Textures step: progress by bytes every second; bc7build built for this CPU

The texture step showed nothing for the first minutes: bc7build printed
a line every 500 files taken, the largest files go first, and its first
lines sat in the pipe's buffer. It now reports finished files and bytes
once a second, unbuffered, and the progress line counts the bytes
(the time left by files was ~2:41 h on a player's PC). Progress lines
that report rarely, and downloads that stall, keep their clock running.

bc7build was always compiled for x86-64-v3: CPUs without AVX2, which the
game supports with an x86-64-v2 build, would stop at this step with an
illegal instruction. It now takes the CPU level the game is built for.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

tools/bc7_build.py

@@ -20,7 +20,7 @@ import time
 ROOT = pathlib.Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(ROOT / 'tools'))
 from toolchain import CLANG64, env as tool_env  # noqa: E402
-from build_game import run_limited  # noqa: E402
+from build_game import cpu_arch, run_limited  # noqa: E402
 import progress  # noqa: E402
 from romfs import RomFS  # noqa: E402
 
@@ -34,11 +34,12 @@ def build_tool(env):
     if exe.exists() and exe.stat().st_mtime > newest:
         return exe
     objs = []
+    arch = cpu_arch()   # as the game: x86-64-v2 on CPUs without AVX2 (it was always v3: illegal instruction there)
     (ROOT / 'build/obj').mkdir(parents=True, exist_ok=True)
     for s in SOURCES_C + SOURCES_CXX:
         cxx = s.endswith('.cpp')
         obj = ROOT / 'build/obj' / ('bc7build_' + s.replace('/', '_').rsplit('.', 1)[0] + '.o')
-        cmd = [str(CLANG64 / ('clang++.exe' if cxx else 'clang.exe')), '-O2', '-march=x86-64-v3', '-w', '-c', str(ROOT / s), '-o', str(obj)]
+        cmd = [str(CLANG64 / ('clang++.exe' if cxx else 'clang.exe')), '-O2', f'-march={arch}', '-w', '-c', str(ROOT / s), '-o', str(obj)]
         if cxx:
             cmd += ['-std=c++17', '-fno-exceptions', '-fno-rtti']
         subprocess.run(cmd, check=True, env=env)
@@ -72,10 +73,11 @@ def main():
     exe = build_tool(env)
     print(f'{len(files)} texture files -> {cache}', flush=True)
     t0 = time.time()
-    bar = progress.Bar('re-encoding the textures', len(files), 'files')
+    bar = progress.Bar(f're-encoding the textures ({len(files)} files)', sum(f['size'] for f in files), 'MiB', 2**20,
+                       rate=True, live=True)
 
-    def line(text):   # bc7build's "<i> / <n> files, ..." every 500 files: the progress line; the rest as it is
-        m = re.match(r'(\d+) / (\d+) files', text)
+    def line(text):   # bc7build's "<done> / <n> files, <bytes> / <total> bytes, ..." once a second: the progress line
+        m = re.match(r'\d+ / \d+ files, (\d+) / (\d+) bytes', text)
         if m:
             bar.update(int(m.group(1)), int(m.group(2)))
         elif text.startswith('done:'):

tools/bc7build/bc7build.c

@@ -27,6 +27,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/stat.h>
+#include <time.h>
 #include <zstd.h>
 
 #include "../../src/gfx/tex/astc.h"
@@ -40,7 +41,10 @@ enum { BC1_AREA = 100, BC1_LEVEL = 10 };   // larger ASTC blocks: BC1 when opaqu
 static _Atomic uint64_t g_bc1;
 static const char *g_romfs;
 static int g_folder;   // g_romfs is a folder of plain files, not romfs.bin
-static _Atomic int g_next;
+static _Atomic int g_next, g_done;
+static _Atomic unsigned long long g_done_bytes;
+static _Atomic long long g_said_at;   // the second of the last progress line
+static unsigned long long g_total_bytes;
 static _Atomic uint64_t g_surfaces, g_astc_files, g_encoded, g_had, g_bad, g_texels;
 
 // ---- the cache files (the same records as vkb_bc7.c)
@@ -280,11 +284,14 @@ static void *worker(void *unused) {
     size_t cap = 0;
     for (int i; (i = atomic_fetch_add(&g_next, 1)) < g_nfiles;) {
         do_file(rf, &g_files[i], &buf, &cap);
-        if (i % 500 == 0) {
-            printf("%d / %d files, %llu ASTC surfaces, %llu encoded, %.0f Mtexels\n", i, g_nfiles, (unsigned long long)g_surfaces,
-                   (unsigned long long)g_encoded, g_texels / 1e6);
-            fflush(stdout);
-        }
+        // progress once a second, counted in finished files and their bytes (tools/bc7_build.py draws it): the
+        // largest files go first, so "every 500 files" said nothing for minutes at the start
+        int done = atomic_fetch_add(&g_done, 1) + 1;
+        unsigned long long bytes = atomic_fetch_add(&g_done_bytes, g_files[i].size) + g_files[i].size;
+        long long now = (long long)time(NULL), last = atomic_load(&g_said_at);
+        if ((now != last && atomic_compare_exchange_strong(&g_said_at, &last, now)) || done == g_nfiles)
+            printf("%d / %d files, %llu / %llu bytes, %llu surfaces encoded\n", done, g_nfiles, bytes, g_total_bytes,
+                   (unsigned long long)g_encoded);
     }
     free(buf);
     if (rf) fclose(rf);
@@ -294,6 +301,7 @@ static void *worker(void *unused) {
 int main(int argc, char **argv) {
     if (argc < 4) { fprintf(stderr, "usage: bc7build <romfs.bin> <list> <cache dir> [threads] [quality]\n"); return 2; }
     g_romfs = argv[1];
+    setvbuf(stdout, NULL, _IONBF, 0);   // read through a pipe (tools/bc7_build.py): every line as it is printed
     int threads = argc > 4 ? atoi(argv[4]) : 8;
     if (argc > 5) g_quality = atoi(argv[5]);
     if (argc > 6) g_max_area = atoi(argv[6]);
@@ -306,6 +314,7 @@ int main(int argc, char **argv) {
     while (fscanf(lf, "%llu %llu %191[^\n]", (unsigned long long *)&f.off, (unsigned long long *)&f.size, f.path) == 3) {
         if (g_nfiles == cap) { cap = cap ? cap * 2 : 4096; g_files = realloc(g_files, sizeof *g_files * (size_t)cap); }
         g_files[g_nfiles++] = f;
+        g_total_bytes += f.size;
     }
     fclose(lf);
     open_cache(argv[3]);

tools/deps_install.py

@@ -130,7 +130,7 @@ def fetch_all(items, cache, label):
             todo.append((item, dst))
     if not todo:
         return files
-    bar = progress.Bar(f'downloading {label}', sum(i.get('size', 0) for i, _ in todo), 'MiB', 2**20, rate=True)
+    bar = progress.Bar(f'downloading {label}', sum(i.get('size', 0) for i, _ in todo), 'MiB', 2**20, rate=True, live=True)
     for item, dst in todo:
         part = dst.with_name(dst.name + '.part')
         base = bar.done - (part.stat().st_size if part.is_file() else 0)

tools/progress.py

@@ -21,13 +21,24 @@ def _tty():
 
 
 class Bar:
-    def __init__(self, label, total, unit='', scale=1, rate=False):
-        """unit / scale: how amounts are shown (bytes as MiB: unit 'MiB', scale 2**20); rate: show the speed."""
+    def __init__(self, label, total, unit='', scale=1, rate=False, live=False):
+        """unit / scale: how amounts are shown (bytes as MiB: unit 'MiB', scale 2**20); rate: show the speed;
+        live: drawn again every second while no update comes (work that reports rarely, or a stalled download)."""
         self.label, self.total, self.unit, self.scale, self.rate = label, total, unit, scale, rate
         self.t0 = time.monotonic()
         self.last, self.done, self.width, self.tens, self.printed = 0.0, 0, 0, -1, None
         self.samples = [(self.t0, 0)]   # (time, done) of the last seconds: the speed and time left are recent ones
         self.tty = _tty()
+        self.lock = threading.RLock()
+        self.stop = threading.Event()
+        if live and self.tty:
+            threading.Thread(target=self.redraw, daemon=True).start()
+        self.update(0, force=True)
+
+    def redraw(self):
+        while not self.stop.wait(1):
+            if time.monotonic() - self.last >= 1:
+                self.update(self.done, force=True)
 
     def line(self):
         now = time.monotonic()
@@ -51,42 +62,47 @@ class Bar:
 
     def note(self, text):
         """A line of its own (a source that does not answer...), the progress line drawn again below it."""
-        if self.tty:
-            sys.stdout.write('\r' + ' ' * self.width + '\r')
-            self.width = 0
-        print(f'  {text}', flush=True)
-        self.last = 0
+        with self.lock:
+            if self.tty:
+                sys.stdout.write('\r' + ' ' * self.width + '\r')
+                self.width = 0
+            print(f'  {text}', flush=True)
+            self.last = 0
 
     def update(self, done, total=None, force=False):
-        if total is not None:
-            self.total = total
-        self.done = done
-        now = time.monotonic()
-        if now - self.samples[-1][0] >= 0.5:
-            self.samples.append((now, done))
-        if not force and now - self.last < 0.3:
-            return
-        self.last = now
-        text = self.line()
-        if self.tty:
-            sys.stdout.write('\r' + text.ljust(self.width))
-            sys.stdout.flush()
-            self.width = len(text)
-        else:
-            tens = int(10 * self.done / self.total) if self.total else 0
-            if (force or tens > self.tens) and (self.tens, self.done) != self.printed:
-                self.tens = tens
-                self.printed = (tens, self.done)
-                print(text, flush=True)
+        with self.lock:
+            if total is not None:
+                self.total = total
+            self.done = done
+            now = time.monotonic()
+            if now - self.samples[-1][0] >= 0.5:
+                self.samples.append((now, done))
+            if not force and now - self.last < 0.3:
+                return
+            self.last = now
+            text = self.line()
+            if self.tty:
+                sys.stdout.write('\r' + text.ljust(self.width))
+                sys.stdout.flush()
+                self.width = len(text)
+            else:
+                tens = int(10 * self.done / self.total) if self.total else 0
+                if (force or tens > self.tens) and (self.tens, self.done) != self.printed:
+                    self.tens = tens
+                    self.printed = (tens, self.done)
+                    print(text, flush=True)
 
     def add(self, n):
-        self.update(self.done + n)
+        with self.lock:
+            self.update(self.done + n)
 
     def finish(self):
-        self.update(self.done, force=True)
-        if self.tty:
-            sys.stdout.write('\n')
-            sys.stdout.flush()
+        self.stop.set()
+        with self.lock:
+            self.update(self.done, force=True)
+            if self.tty:
+                sys.stdout.write('\n')
+                sys.stdout.flush()
 
 
 class Ticker: