simplify: Improve precision of collapse error sorting

We use a subset of bits of collapse error to produce an approximate
ordering of collapses; to keep stack usage and cache utilization
reasonable, we used 11-bit counting sort which uses 8 bits of exponent
and 3 bits of mantissa.

3 bits of mantissa may not be enough and can result in choosing
suboptimal collapse order. Ideally we should probably use 5 bits here,
but that needs much more stack space.

For now, switch to 8+4 bits but to avoid doubling the stack usage,
constrain the exponent range on the high end so that excessively high
errors (>2^32) are bucketed together, as they are not useful.

In the future there's an opportunity to similarly constrain exponent
space on low end by clamping errors that are too low to zero.

In addition to improving the error selection in some cases this also can
sometimes reduce the amount of simplification passes as eligible
collapses in error limited regime get sorted better and can be processed
earlier.

demo/tests.cpp

@@ -1151,7 +1151,7 @@ static void simplifyAttr(bool skip_g)
 		{
 			vb[y * 3 + x][0] = float(x);
 			vb[y * 3 + x][1] = float(y);
-			vb[y * 3 + x][2] = 0.03f * x + 0.03f * (y % 2);
+			vb[y * 3 + x][2] = 0.03f * x + 0.03f * (y % 2) + (x == 2 && y == 7) * 0.03f;
 			vb[y * 3 + x][3] = r;
 			vb[y * 3 + x][4] = g;
 			vb[y * 3 + x][5] = b;

src/simplifier.cpp

@@ -1056,16 +1056,22 @@ static void rankEdgeCollapses(Collapse* collapses, size_t collapse_count, const
 
 static void sortEdgeCollapses(unsigned int* sort_order, const Collapse* collapses, size_t collapse_count)
 {
-	const int sort_bits = 11;
+	// we use counting sort to order collapses by error; since the exact sort order is not as critical,
+	// only top 12 bits of exponent+mantissa (8 bits of exponent and 4 bits of mantissa) are used.
+	// to avoid excessive stack usage, we clamp the exponent range as collapses with errors much higher than 1 are not useful.
+	const unsigned int sort_bits = 12;
+	const unsigned int sort_bins = 2048 + 512; // exponent range [-127, 32)
 
 	// fill histogram for counting sort
-	unsigned int histogram[1 << sort_bits];
+	unsigned int histogram[sort_bins];
 	memset(histogram, 0, sizeof(histogram));
 
 	for (size_t i = 0; i < collapse_count; ++i)
 	{
 		// skip sign bit since error is non-negative
-		unsigned int key = (collapses[i].errorui << 1) >> (32 - sort_bits);
+		unsigned int error = collapses[i].errorui;
+		unsigned int key = (error << 1) >> (32 - sort_bits);
+		key = key < sort_bins ? key : sort_bins - 1;
 
 		histogram[key]++;
 	}
@@ -1073,7 +1079,7 @@ static void sortEdgeCollapses(unsigned int* sort_order, const Collapse* collapse
 	// compute offsets based on histogram data
 	size_t histogram_sum = 0;
 
-	for (size_t i = 0; i < 1 << sort_bits; ++i)
+	for (size_t i = 0; i < sort_bins; ++i)
 	{
 		size_t count = histogram[i];
 		histogram[i] = unsigned(histogram_sum);
@@ -1086,7 +1092,9 @@ static void sortEdgeCollapses(unsigned int* sort_order, const Collapse* collapse
 	for (size_t i = 0; i < collapse_count; ++i)
 	{
 		// skip sign bit since error is non-negative
-		unsigned int key = (collapses[i].errorui << 1) >> (32 - sort_bits);
+		unsigned int error = collapses[i].errorui;
+		unsigned int key = (error << 1) >> (32 - sort_bits);
+		key = key < sort_bins ? key : sort_bins - 1;
 
 		sort_order[histogram[key]++] = unsigned(i);
 	}