libchdr: derive the LZMA decoder dictionary size without an encoder

lzma_codec_init constructed a full LZMA encoder (LzmaEnc_Create /
SetProps / WriteProperties) purely to learn the dictionary size the
compressor would have written into the properties bytes - dragging
the encoder half of the LZMA SDK into every build for a handful of
arithmetic, and making a decode-only LzmaDec.c insufficient for
consumers that bundle their own SDK subset.

Compute it directly: the level-9 default (1 << 28 on 64-bit hosts,
1 << 26 on 32-bit), reduced - when it exceeds the hunk size - to
the smallest {2,3} << i value covering it, with the SDK's 4 KiB
floor. An equivalence sweep against the real encoder (bundled
deps/7zip LzmaEnc) across every chdman CD hunk size, the
power-of-two boundaries from 4 KiB to 32 MiB, and degenerate
inputs matches the encoder-written properties byte-for-byte in all
90 cases, props byte 0 (the constant 93 for lc=3 lp=0 pb=2)
included.

Deliberately NOT ported from the libchdr fork's decode-only
variant: that version applies the reduce clamp as
MAX(min(4K, reduce), dictSize), which never reduces anything, so
every cdlz codec allocates the full level-9 default - a measured
256 MiB LzmaDec dictionary per open CHD where the encoder used
24 KiB. Decode still succeeds with an oversized dictionary, which
is how it shipped unnoticed.

lzma.h drops its LzmaEnc.h include accordingly; the LZMA SDK
requirement is now LzmaDec alone.

libretro-common/formats/libchdr/libchdr_lzma.c

@@ -194,47 +194,79 @@ void lzma_fast_free(void *p, void *address)
  *-------------------------------------------------
  */
 
+/*-------------------------------------------------
+ *  lzma_compute_dictionary_size
+ *
+ *  The decoder only needs the dictionary size the encoder would have
+ *  written into the properties byte stream. Deriving it by actually
+ *  constructing an LZMA encoder (LzmaEncProps_Normalize +
+ *  LzmaEnc_SetProps + LzmaEnc_WriteProperties, as this file
+ *  historically did) drags the whole encoder half of the LZMA SDK
+ *  into every build for a handful of arithmetic.
+ *
+ *  For the fixed inputs used here (level 9, reduceSize = hunkbytes)
+ *  the encoder's answer is: the level default (1 << 28 on 64-bit
+ *  hosts, 1 << 26 on 32-bit), reduced - when it exceeds the hunk
+ *  size - to the smallest value of the form 2 << i or 3 << i that
+ *  covers it, with a 4 KiB floor.  Verified byte-identical to the
+ *  encoder-derived properties across every chdman CD hunk size and
+ *  the power-of-two boundaries around them.
+ *
+ *  Note this is NOT the computation the libchdr fork ships: its
+ *  decode-only derivation applies the reduce clamp as
+ *  MAX(min(4K, reduce), dictSize), which never reduces anything, so
+ *  every cdlz codec allocates the full level-9 default - a measured
+ *  256 MiB LzmaDec dictionary per open CHD where the encoder used
+ *  24 KiB.  Decode still succeeds (any dictionary at least as large
+ *  as the encoder's works), which is how it went unnoticed.
+ *-------------------------------------------------
+ */
+
+static uint32_t lzma_compute_dictionary_size(uint32_t hunkbytes)
+{
+	/* level-9 default dictionary for the host word size, as
+	 * LzmaEncProps_Normalize computes it */
+	uint32_t dictSize = (sizeof(size_t) <= 4)
+		? ((uint32_t)1 << 26) : ((uint32_t)1 << 28);
+
+	if (dictSize > hunkbytes)
+	{
+		/* reduce to the smallest {2,3} << i covering hunkbytes; the
+		 * i = 11 start gives the SDK's 4 KiB floor.  Every ladder
+		 * value below 2 MiB is already in LzmaEnc_WriteProperties'
+		 * aligned form, and every one at or above it is a 1 MiB
+		 * multiple, so no further alignment step is needed. */
+		unsigned i;
+		for (i = 11; i <= 30; i++)
+		{
+			uint32_t v = (uint32_t)2 << i;
+			if (hunkbytes <= v) { dictSize = v; break; }
+			v = (uint32_t)3 << i;
+			if (hunkbytes <= v) { dictSize = v; break; }
+		}
+	}
+	return dictSize;
+}
+
 chd_error lzma_codec_init(void* codec, uint32_t hunkbytes)
 {
-	CLzmaEncHandle enc;
-	CLzmaEncProps encoder_props;
+	unsigned int i;
 	Byte decoder_props[LZMA_PROPS_SIZE];
-	SizeT props_size;
 	lzma_allocator* alloc;
 	lzma_codec_data* lzma_codec = (lzma_codec_data*) codec;
+	const uint32_t dictSize = lzma_compute_dictionary_size(hunkbytes);
 
 	/* construct the decoder */
 	LzmaDec_Construct(&lzma_codec->decoder);
 
-	/* FIXME: this code is written in a way that makes it impossible to safely upgrade the LZMA SDK
-	 * This code assumes that the current version of the encoder imposes the same requirements on the
-	 * decoder as the encoder used to produce the file.  This is not necessarily true.  The format
-	 * needs to be changed so the encoder properties are written to the file.
+	/* properties byte 0 encodes (pb * 5 + lp) * 9 + lc for the encoder
+	 * defaults lc=3 lp=0 pb=2 -> 93; bytes 1..4 the dictionary size LE */
+	decoder_props[0] = 93;
+	for (i = 0; i < LZMA_PROPS_SIZE - 1; ++i)
+		decoder_props[1 + i] = (dictSize >> (8 * i)) & 0xFF;
 
-	 * configure the properties like the compressor did */
-	LzmaEncProps_Init(&encoder_props);
-	encoder_props.level = 9;
-	encoder_props.reduceSize = hunkbytes;
-	LzmaEncProps_Normalize(&encoder_props);
-
-	/* convert to decoder properties */
 	alloc = &lzma_codec->allocator;
 	lzma_allocator_init(alloc);
-	enc = LzmaEnc_Create((ISzAlloc*)alloc);
-	if (!enc)
-		return CHDERR_DECOMPRESSION_ERROR;
-	if (LzmaEnc_SetProps(enc, &encoder_props) != SZ_OK)
-	{
-		LzmaEnc_Destroy(enc, (ISzAlloc*)&alloc, (ISzAlloc*)&alloc);
-		return CHDERR_DECOMPRESSION_ERROR;
-	}
-	props_size = sizeof(decoder_props);
-	if (LzmaEnc_WriteProperties(enc, decoder_props, &props_size) != SZ_OK)
-	{
-		LzmaEnc_Destroy(enc, (ISzAlloc*)alloc, (ISzAlloc*)alloc);
-		return CHDERR_DECOMPRESSION_ERROR;
-	}
-	LzmaEnc_Destroy(enc, (ISzAlloc*)alloc, (ISzAlloc*)alloc);
 
 	/* do memory allocations */
 	if (LzmaDec_Allocate(&lzma_codec->decoder, decoder_props, LZMA_PROPS_SIZE, (ISzAlloc*)alloc) != SZ_OK)

libretro-common/include/libchdr/lzma.h

@@ -15,8 +15,7 @@
 
 #include <stdint.h>
 
-#include <LzmaEnc.h>
-#include <LzmaDec.h>
+#include <LzmaDec.h> /* decode-only: lzma_codec_init derives props without the encoder */
 
 #include <libchdr/libchdr_zlib.h>