Bump version and soname for 5.2.1.

The bug was added in the commit
f48fce093b and thus
affected 5.1.4beta and 5.2.0. Luckily the bug cannot
cause data corruption or other nasty things.

INSTALL

@@ -122,6 +122,10 @@ XZ Utils Installation
     This can be worked around by passing gl_cv_cc_visibility=no
     as an argument to the configure script.
 
+    test_scripts.sh in "make check" may fail if good enough tools are
+    missing from PATH (/usr/xpg4/bin or /usr/xpg6/bin). See sections
+    4.5 and 3.2 for more information.
+
 
 1.2.6. Tru64
 
@@ -474,7 +478,8 @@ XZ Utils Installation
     to POSIX. The configure script tries to find such a shell. If
     it fails, you can force the shell to be used by passing
     gl_cv_posix_shell=/path/to/posix-sh as an argument to the configure
-    script.
+    script. Alternatively you can omit the installation of scripts and
+    this error by passing --disable-scripts to configure.
 
 
 4.3. configure works but build fails at crc32_x86.S

NEWS

@@ -2,6 +2,20 @@
 XZ Utils Release Notes
 ======================
 
+5.2.1 (2015-02-26)
+
+    * Fixed a compression-ratio regression in fast mode of LZMA1 and
+      LZMA2. The bug is present in 5.1.4beta and 5.2.0 releases.
+
+    * Fixed a portability problem in xz that affected at least OpenBSD.
+
+    * Fixed xzdiff to be compatible with FreeBSD's mktemp which differs
+      from most other mktemp implementations.
+
+    * Changed CPU core count detection to use cpuset_getaffinity() on
+      FreeBSD.
+
+
 5.2.0 (2014-12-21)
 
     Since 5.1.4beta:

THANKS

@@ -67,6 +67,7 @@ has been important. :-) In alphabetical order:
   - Andre Noll
   - Peter O'Gorman
   - Peter Pallinger
+  - Rui Paulo
   - Igor Pavlov
   - Diego Elio Pettenò
   - Elbert Pol
@@ -78,12 +79,14 @@ has been important. :-) In alphabetical order:
   - Eric S. Raymond
   - Cristian Rodríguez
   - Christian von Roques
+  - Torsten Rupp
   - Jukka Salmi
   - Alexandre Sauvé
   - Benno Schulenberg
   - Andreas Schwab
   - Dan Shechter
   - Stuart Shelton
+  - Brad Smith
   - Jonathan Stott
   - Dan Stromberg
   - Vincent Torri

configure.ac

@@ -474,7 +474,7 @@ AM_CONDITIONAL([COND_SYMVERS], [test "x$enable_symbol_versions" = xyes])
 
 echo
 gl_POSIX_SHELL
-if test -z "$POSIX_SHELL" ; then
+if test -z "$POSIX_SHELL" && test "x$enable_scripts" = xyes ; then
 	AC_MSG_ERROR([No POSIX conforming shell (sh) was found.])
 fi
 
@@ -626,8 +626,8 @@ gl_GETOPT
 # Find the best function to set timestamps.
 AC_CHECK_FUNCS([futimens futimes futimesat utimes utime], [break])
 
-# This is nice to have but not mandatory.
-AC_CHECK_FUNCS([posix_fadvise])
+# These are nice to have but not mandatory.
+AC_CHECK_FUNCS([posix_fadvise pipe2])
 
 TUKLIB_PROGNAME
 TUKLIB_INTEGER

m4/tuklib_cpucores.m4

@@ -45,6 +45,22 @@ compile error
 #endif
 ]])], [tuklib_cv_cpucores_method=special], [
 
+# FreeBSD has both cpuset and sysctl. Look for cpuset first because
+# it's a better approach.
+AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
+#include <sys/param.h>
+#include <sys/cpuset.h>
+
+int
+main(void)
+{
+	cpuset_t set;
+	cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1,
+			sizeof(set), &set);
+	return 0;
+}
+]])], [tuklib_cv_cpucores_method=cpuset], [
+
 # Look for sysctl() solution first, because on OS/2, both sysconf()
 # and sysctl() pass the tests in this file, but only sysctl()
 # actually works.
@@ -97,9 +113,14 @@ main(void)
 ]])], [tuklib_cv_cpucores_method=pstat_getdynamic], [
 
 	tuklib_cv_cpucores_method=unknown
-])])])])])
+])])])])])])
 
 case $tuklib_cv_cpucores_method in
+	cpuset)
+		AC_DEFINE([TUKLIB_CPUCORES_CPUSET], [1],
+			[Define to 1 if the number of available CPU cores
+			can be detected with cpuset(2).])
+		;;
 	sysctl)
 		AC_DEFINE([TUKLIB_CPUCORES_SYSCTL], [1],
 			[Define to 1 if the number of available CPU cores

src/common/tuklib_cpucores.c

@@ -18,6 +18,11 @@
 #	endif
 #	include <windows.h>
 
+// FreeBSD
+#elif defined(TUKLIB_CPUCORES_CPUSET)
+#	include <sys/param.h>
+#	include <sys/cpuset.h>
+
 #elif defined(TUKLIB_CPUCORES_SYSCTL)
 #	ifdef HAVE_SYS_PARAM_H
 #		include <sys/param.h>
@@ -44,6 +49,19 @@ tuklib_cpucores(void)
 	GetSystemInfo(&sysinfo);
 	ret = sysinfo.dwNumberOfProcessors;
 
+#elif defined(TUKLIB_CPUCORES_CPUSET)
+	cpuset_t set;
+	if (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1,
+			sizeof(set), &set) == 0) {
+#	ifdef CPU_COUNT
+		ret = CPU_COUNT(&set);
+#	else
+		for (unsigned i = 0; i < CPU_SETSIZE; ++i)
+			if (CPU_ISSET(i, &set))
+				++ret;
+#	endif
+	}
+
 #elif defined(TUKLIB_CPUCORES_SYSCTL)
 	int name[2] = { CTL_HW, HW_NCPU };
 	int cpus;

src/common/tuklib_integer.h

@@ -321,8 +321,8 @@ unaligned_read32le(const uint8_t *buf)
 static inline void
 unaligned_write16be(uint8_t *buf, uint16_t num)
 {
-	buf[0] = num >> 8;
-	buf[1] = num;
+	buf[0] = (uint8_t)(num >> 8);
+	buf[1] = (uint8_t)num;
 	return;
 }
 
@@ -330,8 +330,8 @@ unaligned_write16be(uint8_t *buf, uint16_t num)
 static inline void
 unaligned_write16le(uint8_t *buf, uint16_t num)
 {
-	buf[0] = num;
-	buf[1] = num >> 8;
+	buf[0] = (uint8_t)num;
+	buf[1] = (uint8_t)(num >> 8);
 	return;
 }
 
@@ -339,10 +339,10 @@ unaligned_write16le(uint8_t *buf, uint16_t num)
 static inline void
 unaligned_write32be(uint8_t *buf, uint32_t num)
 {
-	buf[0] = num >> 24;
-	buf[1] = num >> 16;
-	buf[2] = num >> 8;
-	buf[3] = num;
+	buf[0] = (uint8_t)(num >> 24);
+	buf[1] = (uint8_t)(num >> 16);
+	buf[2] = (uint8_t)(num >> 8);
+	buf[3] = (uint8_t)num;
 	return;
 }
 
@@ -350,10 +350,10 @@ unaligned_write32be(uint8_t *buf, uint32_t num)
 static inline void
 unaligned_write32le(uint8_t *buf, uint32_t num)
 {
-	buf[0] = num;
-	buf[1] = num >> 8;
-	buf[2] = num >> 16;
-	buf[3] = num >> 24;
+	buf[0] = (uint8_t)num;
+	buf[1] = (uint8_t)(num >> 8);
+	buf[2] = (uint8_t)(num >> 16);
+	buf[3] = (uint8_t)(num >> 24);
 	return;
 }
 

src/liblzma/Makefile.am

@@ -24,7 +24,7 @@ liblzma_la_CPPFLAGS = \
 	-I$(top_srcdir)/src/liblzma/simple \
 	-I$(top_srcdir)/src/common \
 	-DTUKLIB_SYMBOL_PREFIX=lzma_
-liblzma_la_LDFLAGS = -no-undefined -version-info 7:0:2
+liblzma_la_LDFLAGS = -no-undefined -version-info 7:1:2
 
 EXTRA_DIST += liblzma.map validate_map.sh
 if COND_SYMVERS

src/liblzma/api/lzma/version.h

@@ -22,7 +22,7 @@
  */
 #define LZMA_VERSION_MAJOR 5
 #define LZMA_VERSION_MINOR 2
-#define LZMA_VERSION_PATCH 0
+#define LZMA_VERSION_PATCH 1
 #define LZMA_VERSION_STABILITY LZMA_VERSION_STABILITY_STABLE
 
 #ifndef LZMA_VERSION_COMMIT

src/liblzma/common/memcmplen.h

@@ -19,11 +19,6 @@
 #	include <immintrin.h>
 #endif
 
-/// How many extra bytes lzma_memcmplen() may read. This depends on
-/// the method but since it is just a few bytes the biggest possible
-/// value is used here.
-#define LZMA_MEMCMPLEN_EXTRA 16
-
 
 /// Find out how many equal bytes the two buffers have.
 ///
@@ -39,6 +34,11 @@
 ///
 /// \return     Number of equal bytes in the buffers is returned.
 ///             This is always at least len and at most limit.
+///
+/// \note       LZMA_MEMCMPLEN_EXTRA defines how many extra bytes may be read.
+///             It's rounded up to 2^n. This extra amount needs to be
+///             allocated in the buffers being used. It needs to be
+///             initialized too to keep Valgrind quiet.
 static inline uint32_t lzma_attribute((__always_inline__))
 lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2,
 		uint32_t len, uint32_t limit)
@@ -59,6 +59,7 @@ lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2,
 	// to be a good method. This may be fine on other 64-bit CPUs too.
 	// On big endian one should use xor instead of subtraction and switch
 	// to __builtin_clzll().
+#define LZMA_MEMCMPLEN_EXTRA 8
 	while (len < limit) {
 		const uint64_t x = *(const uint64_t *)(buf1 + len)
 				- *(const uint64_t *)(buf2 + len);
@@ -91,6 +92,7 @@ lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2,
 	// version is sometimes significantly faster and sometimes
 	// slightly slower than this SSE2 version, so this SSE2
 	// version isn't used on x86-64.
+#	define LZMA_MEMCMPLEN_EXTRA 16
 	while (len < limit) {
 		const uint32_t x = 0xFFFF ^ _mm_movemask_epi8(_mm_cmpeq_epi8(
 			_mm_loadu_si128((const __m128i *)(buf1 + len)),
@@ -116,6 +118,7 @@ lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2,
 
 #elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) && !defined(WORDS_BIGENDIAN)
 	// Generic 32-bit little endian method
+#	define LZMA_MEMCMPLEN_EXTRA 4
 	while (len < limit) {
 		uint32_t x = *(const uint32_t *)(buf1 + len)
 				- *(const uint32_t *)(buf2 + len);
@@ -138,6 +141,7 @@ lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2,
 
 #elif defined(TUKLIB_FAST_UNALIGNED_ACCESS) && defined(WORDS_BIGENDIAN)
 	// Generic 32-bit big endian method
+#	define LZMA_MEMCMPLEN_EXTRA 4
 	while (len < limit) {
 		uint32_t x = *(const uint32_t *)(buf1 + len)
 				^ *(const uint32_t *)(buf2 + len);
@@ -160,6 +164,7 @@ lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2,
 
 #else
 	// Simple portable version that doesn't use unaligned access.
+#	define LZMA_MEMCMPLEN_EXTRA 0
 	while (len < limit && buf1[len] == buf2[len])
 		++len;
 

src/liblzma/lz/lz_encoder.c

@@ -110,6 +110,12 @@ fill_window(lzma_coder *coder, const lzma_allocator *allocator,
 
 	coder->mf.write_pos = write_pos;
 
+	// Silence Valgrind. lzma_memcmplen() can read extra bytes
+	// and Valgrind will give warnings if those bytes are uninitialized
+	// because Valgrind cannot see that the values of the uninitialized
+	// bytes are eventually ignored.
+	memzero(coder->mf.buffer + write_pos, LZMA_MEMCMPLEN_EXTRA);
+
 	// If end of stream has been reached or flushing completed, we allow
 	// the encoder to process all the input (that is, read_pos is allowed
 	// to reach write_pos). Otherwise we keep keep_size_after bytes

src/liblzma/lzma/lzma_encoder_optimum_fast.c

@@ -152,7 +152,7 @@ lzma_lzma_optimum_fast(lzma_coder *restrict coder, lzma_mf *restrict mf,
 	// the old buf pointer instead of recalculating it with mf_ptr().
 	++buf;
 
-	const uint32_t limit = len_main - 1;
+	const uint32_t limit = my_max(2, len_main - 1);
 
 	for (uint32_t i = 0; i < REPS; ++i) {
 		if (memcmp(buf, buf - coder->reps[i] - 1, limit) == 0) {

src/scripts/xzdiff.in

@@ -140,7 +140,12 @@ elif test $# -eq 2; then
               (exit 2); exit 2
             ' HUP INT PIPE TERM 0
             if type mktemp >/dev/null 2>&1; then
-              tmp=`mktemp -t -d -- "$prog.XXXXXXXXXX"` || exit 2
+              # Note that FreeBSD's mktemp isn't fully compatible with
+              # the implementations from mktemp.org and GNU coreutils.
+              # It is important that the -t argument is the last argument
+              # and that no "--" is used between -t and the template argument.
+              # This way this command works on all implementations.
+              tmp=`mktemp -d -t "$prog.XXXXXXXXXX"` || exit 2
             else
               # Fallback code if mktemp is missing. This isn't as
               # robust as using mktemp since this doesn't try with

src/xz/file_io.c

@@ -82,13 +82,26 @@ io_init(void)
 	// we are root.
 	warn_fchown = geteuid() == 0;
 
-	if (pipe(user_abort_pipe)
-			|| fcntl(user_abort_pipe[0], F_SETFL, O_NONBLOCK)
-				== -1
-			|| fcntl(user_abort_pipe[1], F_SETFL, O_NONBLOCK)
-				== -1)
+	// Create a pipe for the self-pipe trick. If pipe2() is available,
+	// we can avoid the fcntl() calls.
+#	ifdef HAVE_PIPE2
+	if (pipe2(user_abort_pipe, O_NONBLOCK))
 		message_fatal(_("Error creating a pipe: %s"),
 				strerror(errno));
+#	else
+	if (pipe(user_abort_pipe))
+		message_fatal(_("Error creating a pipe: %s"),
+				strerror(errno));
+
+	// Make both ends of the pipe non-blocking.
+	for (unsigned i = 0; i < 2; ++i) {
+		int flags = fcntl(user_abort_pipe[i], F_GETFL);
+		if (flags == -1 || fcntl(user_abort_pipe[i], F_SETFL,
+				flags | O_NONBLOCK) == -1)
+			message_fatal(_("Error creating a pipe: %s"),
+					strerror(errno));
+	}
+#	endif
 #endif
 
 #ifdef __DJGPP__
@@ -393,7 +406,11 @@ io_open_src_real(file_pair *pair)
 #ifdef TUKLIB_DOSLIKE
 		setmode(STDIN_FILENO, O_BINARY);
 #else
-		// Enable O_NONBLOCK for stdin.
+		// Try to set stdin to non-blocking mode. It won't work
+		// e.g. on OpenBSD if stdout is e.g. /dev/null. In such
+		// case we proceed as if stdin were non-blocking anyway
+		// (in case of /dev/null it will be in practice). The
+		// same applies to stdout in io_open_dest_real().
 		stdin_flags = fcntl(STDIN_FILENO, F_GETFL);
 		if (stdin_flags == -1) {
 			message_error(_("Error getting the file status flags "
@@ -402,17 +419,10 @@ io_open_src_real(file_pair *pair)
 			return true;
 		}
 
-		if ((stdin_flags & O_NONBLOCK) == 0) {
-			if (fcntl(STDIN_FILENO, F_SETFL,
-					stdin_flags | O_NONBLOCK) == -1) {
-				message_error(_("Error setting O_NONBLOCK "
-						"on standard input: %s"),
-						strerror(errno));
-				return true;
-			}
-
+		if ((stdin_flags & O_NONBLOCK) == 0
+				&& fcntl(STDIN_FILENO, F_SETFL,
+					stdin_flags | O_NONBLOCK) != -1)
 			restore_stdin_flags = true;
-		}
 #endif
 #ifdef HAVE_POSIX_FADVISE
 		// It will fail if stdin is a pipe and that's fine.
@@ -705,7 +715,10 @@ io_open_dest_real(file_pair *pair)
 #ifdef TUKLIB_DOSLIKE
 		setmode(STDOUT_FILENO, O_BINARY);
 #else
-		// Set O_NONBLOCK if it isn't already set.
+		// Try to set O_NONBLOCK if it isn't already set.
+		// If it fails, we assume that stdout is non-blocking
+		// in practice. See the comments in io_open_src_real()
+		// for similar situation with stdin.
 		//
 		// NOTE: O_APPEND may be unset later in this function
 		// and it relies on stdout_flags being set here.
@@ -717,17 +730,10 @@ io_open_dest_real(file_pair *pair)
 			return true;
 		}
 
-		if ((stdout_flags & O_NONBLOCK) == 0) {
-			if (fcntl(STDOUT_FILENO, F_SETFL,
-					stdout_flags | O_NONBLOCK) == -1) {
-				message_error(_("Error setting O_NONBLOCK "
-						"on standard output: %s"),
-						strerror(errno));
-				return true;
-			}
-
-			restore_stdout_flags = true;
-		}
+		if ((stdout_flags & O_NONBLOCK) == 0
+				&& fcntl(STDOUT_FILENO, F_SETFL,
+					stdout_flags | O_NONBLOCK) != -1)
+				restore_stdout_flags = true;
 #endif
 	} else {
 		pair->dest_name = suffix_get_dest_name(pair->src_name);
@@ -742,6 +748,7 @@ io_open_dest_real(file_pair *pair)
 				message_error("%s: Refusing to write to "
 						"a DOS special file",
 						pair->dest_name);
+				free(pair->dest_name);
 				return true;
 			}
 
@@ -751,6 +758,7 @@ io_open_dest_real(file_pair *pair)
 				message_error("%s: Output file is the same "
 						"as the input file",
 						pair->dest_name);
+				free(pair->dest_name);
 				return true;
 			}
 		}
@@ -829,23 +837,24 @@ io_open_dest_real(file_pair *pair)
 				if (lseek(STDOUT_FILENO, 0, SEEK_END) == -1)
 					return false;
 
-				// O_NONBLOCK was set earlier in this function
-				// so it must be kept here too. If this
-				// fcntl() call fails, we continue but won't
+				// Construct the new file status flags.
+				// If O_NONBLOCK was set earlier in this
+				// function, it must be kept here too.
+				int flags = stdout_flags & ~O_APPEND;
+				if (restore_stdout_flags)
+					flags |= O_NONBLOCK;
+
+				// If this fcntl() fails, we continue but won't
 				// try to create sparse output. The original
 				// flags will still be restored if needed (to
 				// unset O_NONBLOCK) when the file is finished.
-				if (fcntl(STDOUT_FILENO, F_SETFL,
-						(stdout_flags | O_NONBLOCK)
-						& ~O_APPEND) == -1)
+				if (fcntl(STDOUT_FILENO, F_SETFL, flags) == -1)
 					return false;
 
 				// Disabling O_APPEND succeeded. Mark
 				// that the flags should be restored
-				// in io_close_dest(). This quite likely was
-				// already set when enabling O_NONBLOCK but
-				// just in case O_NONBLOCK was already set,
-				// set this again here.
+				// in io_close_dest(). (This may have already
+				// been set when enabling O_NONBLOCK.)
 				restore_stdout_flags = true;
 
 			} else if (lseek(STDOUT_FILENO, 0, SEEK_CUR)

tests/test_files.sh

@@ -21,7 +21,7 @@ fi
 
 for I in "$srcdir"/files/good-*.xz
 do
-	if test -z "$XZ" || "$XZ" -dc "$I" > /dev/null 2>&1; then
+	if test -z "$XZ" || "$XZ" -dc "$I" > /dev/null; then
 		:
 	else
 		echo "Good file failed: $I"
@@ -29,7 +29,7 @@ do
 		exit 1
 	fi
 
-	if test -z "$XZDEC" || "$XZDEC" "$I" > /dev/null 2>&1; then
+	if test -z "$XZDEC" || "$XZDEC" "$I" > /dev/null; then
 		:
 	else
 		echo "Good file failed: $I"