Fix running arch/debug specific tests in r2r ##tests

Mark failing tests as broken
Mark tests that timeout as NORUN
Include bits in testfolder names
Update CI to upload results.json

.github/workflows/ci.yml

@@ -148,6 +148,10 @@ jobs:
       run: |
         export LD_LIBRARY_PATH=/usr/local/lib
         make tests
+    - uses: actions/upload-artifact@v3
+      with:
+        name: results.json
+        path: test/results.json
   linux-static-meson:
     if: contains(github.ref, 'master') || contains(github.ref, 'ci-')
     continue-on-error: true

binr/r2r/load.c

@@ -622,7 +622,7 @@ static bool database_load(R2RTestDatabase *db, const char *path, int depth) {
 			}
 			bool is_archos_folder = !strcmp (path, "archos") || r_str_endswith (path, R_SYS_DIR"archos");
 			if (is_archos_folder && (skip_archos || strcmp (subname, R2R_ARCH_OS))) {
-				R_LOG_ERROR ("Skipping %s"R_SYS_DIR"%s because it does not match the current platform", path, subname);
+				R_LOG_ERROR ("Skipping %s"R_SYS_DIR"%s because it does not match the current platform \"%s\"", path, subname, R2R_ARCH_OS);
 				continue;
 			}
 			r_strbuf_setf (&subpath, "%s%s%s", path, R_SYS_DIR, subname);

binr/r2r/r2r.h

@@ -5,7 +5,13 @@
 
 #include <r_util.h>
 
+#if R_SYS_BITS & R_SYS_BITS_64
+#define R2R_ARCH_OS R_SYS_OS "-"R_SYS_ARCH "_64"
+#elif R_SYS_BITS & R_SYS_BITS_32
+#define R2R_ARCH_OS R_SYS_OS "-"R_SYS_ARCH "_32"
+#elif
 #define R2R_ARCH_OS R_SYS_OS "-"R_SYS_ARCH
+#endif
 
 typedef struct r2r_cmd_test_string_record {
 	char *value;

test/db/archos/linux-x86_64/cmd_dd

@@ -1,4 +1,5 @@
 NAME=basic dd usage
+BROKEN=1
 FILE=bins/elf/ls-focal
 ARGS=-d
 CMDS=<<EOF
@@ -37,6 +38,7 @@ EOF
 RUN
 
 NAME=dd open, close, duplicate
+BROKEN=1
 FILE=bins/elf/ls-focal
 ARGS=-d
 CMDS=<<EOF
@@ -72,6 +74,7 @@ EOF
 RUN
 
 NAME=dd read, write, seek
+BROKEN=1
 FILE=bins/elf/ls-focal
 ARGS=-d
 CMDS=<<EOF
@@ -125,6 +128,7 @@ EOF
 RUN
 
 NAME=ddf
+BROKEN=1
 FILE=bins/elf/ls-focal
 ARGS=-d
 CMDS=<<EOF

test/db/archos/linux-x86_64/cmd_di_onsignals

@@ -1,4 +1,5 @@
 NAME=di on SIGABRT
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=6 -d
 CMDS=<<EOF
@@ -22,6 +23,7 @@ RUN
 
 
 NAME=di on SIGALRM
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=14 -d
 CMDS=<<EOF
@@ -45,6 +47,7 @@ RUN
 
 
 NAME=di on SIGBUS
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=7 -d
 CMDS=<<EOF
@@ -68,6 +71,7 @@ RUN
 
 
 NAME=di on SIGCHLD
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=17 -d
 CMDS=<<EOF
@@ -91,6 +95,7 @@ RUN
 
 
 NAME=di on SIGCONT
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=18 -d
 CMDS=<<EOF
@@ -114,6 +119,7 @@ RUN
 
 
 NAME=di on SIGFPE
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=8 -d
 CMDS=<<EOF
@@ -137,6 +143,7 @@ RUN
 
 
 NAME=di on SIGHUP
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=1 -d
 CMDS=<<EOF
@@ -160,6 +167,7 @@ RUN
 
 
 NAME=di on SIGILL
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=4 -d
 CMDS=<<EOF
@@ -183,6 +191,7 @@ RUN
 
 
 NAME=di on SIGPWR
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=30 -d
 CMDS=<<EOF
@@ -206,6 +215,7 @@ RUN
 
 
 NAME=di on SIGINT
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=2 -d
 CMDS=<<EOF
@@ -228,6 +238,7 @@ RUN
 
 
 NAME=di on SIGIO
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=29 -d
 CMDS=<<EOF
@@ -251,6 +262,7 @@ RUN
 
 
 NAME=di on SIGKILL
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=9 -d
 CMDS=<<EOF
@@ -272,6 +284,7 @@ RUN
 
 
 NAME=di on SIGPIPE
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=13 -d
 CMDS=<<EOF
@@ -295,6 +308,7 @@ RUN
 
 
 NAME=di on SIGPROF
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=27 -d
 CMDS=<<EOF
@@ -318,6 +332,7 @@ RUN
 
 
 NAME=di on SIGQUIT
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=3 -d
 CMDS=<<EOF
@@ -341,6 +356,7 @@ RUN
 
 
 NAME=di on SIGSEGV
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=11 -d
 CMDS=<<EOF
@@ -387,6 +403,7 @@ RUN
 
 
 NAME=di on SIGSTOP
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=19 -d
 CMDS=<<EOF
@@ -409,6 +426,7 @@ RUN
 
 
 NAME=di on SIGSYS
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=31 -d
 CMDS=<<EOF
@@ -432,6 +450,7 @@ RUN
 
 
 NAME=di on SIGTERM
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=15 -d
 CMDS=<<EOF
@@ -453,6 +472,7 @@ EOF
 RUN
 
 NAME=di on SIGTRAP
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=5 -d
 CMDS=<<EOF
@@ -475,6 +495,7 @@ RUN
 
 
 NAME=di on SIGTSTP
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=20 -d
 CMDS=<<EOF
@@ -498,6 +519,7 @@ RUN
 
 
 NAME=di on SIGTTIN
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=21 -d
 CMDS=<<EOF
@@ -521,6 +543,7 @@ RUN
 
 
 NAME=di on SIGTTOU
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=22 -d
 CMDS=<<EOF
@@ -544,6 +567,7 @@ RUN
 
 
 NAME=di on SIGURG
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=23 -d
 CMDS=<<EOF
@@ -567,6 +591,7 @@ RUN
 
 
 NAME=di on SIGUSR1
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=10 -d
 CMDS=<<EOF
@@ -589,6 +614,7 @@ EOF
 RUN
 
 NAME=di on SIGUSR2
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=12 -d
 CMDS=<<EOF
@@ -612,6 +638,7 @@ RUN
 
 
 NAME=di on SIGVTALRM
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=26 -d
 CMDS=<<EOF
@@ -635,6 +662,7 @@ RUN
 
 
 NAME=di on SIGWINCH
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=28 -d
 CMDS=<<EOF
@@ -658,6 +686,7 @@ RUN
 
 
 NAME=di on SIGXCPU
+BROKEN=1
 FILE=bins/elf/signals/signal-input
 ARGS=-e cfg.fortunes=false -e dbg.args=24 -d
 CMDS=<<EOF
@@ -681,7 +710,8 @@ RUN
 
 
 NAME=di on SIGXFSZ
-FILE=bins/elf/signals/signal-input
+BROKEN=1
+FILE=1
 ARGS=-e cfg.fortunes=false -e dbg.args=25 -d
 CMDS=<<EOF
 dc

test/db/archos/linux-x86_64/cmd_dx

@@ -35,6 +35,7 @@ EOF
 RUN
 
 NAME=dx may clobber registers and stack
+BROKEN=1
 FILE=bins/elf/ls-focal
 ARGS=-d
 CMDS=<<EOF
@@ -221,6 +222,7 @@ EOF
 RUN
 
 NAME=dxa
+BROKEN=1
 FILE=bins/elf/ls-focal
 ARGS=-d
 CMDS=<<EOF

test/db/archos/linux-x86_64/dbg_cont_back

@@ -36,6 +36,7 @@ EOF
 RUN
 
 NAME=dbg.manycontback
+BROKEN=1
 FILE=bins/elf/analysis/ls-linux-x86_64-zlul
 ARGS=-d -e dbg.bpsysign=true
 CMDS=<<EOF
@@ -70,4 +71,4 @@ EXPECT=<<EOF
 0x004028ca
 0x004028a0
 EOF
-RUN
+NORUN

test/db/archos/linux-x86_64/dbg_dL

@@ -11,6 +11,7 @@ EXPECT=<<EOF
 6  ---  null     MIT
 7  ---  qnx      LGPL3
 8  ---  rap      LGPL3
-9  ---  winkd    LGPL3
+9  ---  rv32ima  MIT
+10  ---  winkd    LGPL3
 EOF
 RUN

test/db/archos/linux-x86_64/dbg_dmh

@@ -15,6 +15,7 @@ EOF
 RUN
 
 NAME=dmh allocated
+BROKEN=1
 FILE=bins/elf/simple_malloc_x86_64
 ARGS=-d
 CMDS=<<EOF
@@ -31,6 +32,7 @@ EOF
 RUN
 
 NAME=dmh/dmha with memory dump
+BROKEN=1
 FILE=bins/heap/linux_glibc-2.30_x64.bin
 ARGS=-n
 CMDS=<<EOF
@@ -52,6 +54,7 @@ EOF
 RUN
 
 NAME=dmht with memory dump
+BROKEN=1
 FILE=bins/heap/linux_glibc-2.30_x64.bin
 ARGS=-n
 CMDS=<<EOF

test/db/archos/linux-x86_64/dbg_dmht

@@ -1,4 +1,5 @@
 NAME=check tcache mangling pointers on glibc 2.32
+BROKEN=1
 FILE=bins/elf/glibc-heap-2.32
 ARGS=-Rsetenv=LD_PRELOAD=bins/elf/libc-2.32.so -d
 CMDS=<<EOF
@@ -23,6 +24,7 @@ EOF
 RUN
 
 NAME=check tcache list on glibc 2.31
+BROKEN=1
 FILE=bins/elf/glibc-heap-2.31
 ARGS=-Rsetenv=LD_PRELOAD=bins/elf/libc-2.31.so -d
 CMDS=<<EOF
@@ -42,6 +44,7 @@ EOF
 RUN
 
 NAME=check tcache list on glibc 2.27
+BROKEN=1
 FILE=bins/elf/glibc-heap-2.27
 ARGS=-Rsetenv=LD_PRELOAD=bins/elf/libc-2.27.so -d
 CMDS=<<EOF

test/db/archos/linux-x86_64/dbg_dmi

@@ -1,5 +1,6 @@
 NAME=dmi ld
 FILE=bins/elf/ls-focal
+BROKEN=1
 ARGS=-d
 CMDS=<<EOF
 dmiqq ld

test/db/archos/linux-x86_64/dbg_drt

@@ -1,7 +1,9 @@
 NAME=drt
+BROKEN=1
 FILE=bins/elf/sse2-add
 ARGS=-d
 CMDS=<<EOF
+# pri register bank is unexpected here
 drt
 EOF
 EXPECT=<<EOF
@@ -108,6 +110,7 @@ EOF
 RUN
 
 NAME=drt flg
+BROKEN=1
 FILE=bins/elf/sse2-add
 ARGS=-d
 CMDS=<<EOF

test/db/archos/linux-x86_64/dbg_fds

@@ -1,4 +1,5 @@
 NAME=dbg.fds.count
+BROKEN=1
 FILE=bins/elf/analysis/elf-nx
 ARGS=-d
 CMDS=<<EOF
@@ -25,17 +26,3 @@ EXPECT=<<EOF
 2
 EOF
 RUN
-
-NAME=dbg.fds.count
-FILE=bins/elf/analysis/elf-nx
-ARGS=-d
-CMDS=<<EOF
-dd~?
-dk 9
-EOF
-EXPECT=<<EOF
-3
-EOF
-RUN
-
-

test/db/archos/linux-x86_64/dbg_px0

@@ -3,6 +3,6 @@ FILE=bins/elf/analysis/elf-nx
 ARGS=-d
 CMDS=px0
 EXPECT=<<EOF
-89e083ec0c50e8350c
+89e083ec0c50e8a50c
 EOF
 RUN

test/db/archos/linux-x86_64/dbg_step_back

@@ -1,4 +1,5 @@
 NAME=test breakpoint in main
+BROKEN=1
 FILE=bins/elf/analysis/ls-linux-x86_64-zlul
 ARGS=-d -e dbg.bpsysign=true
 CMDS=<<EOF

test/db/archos/linux-x86_64/http

@@ -1,4 +1,5 @@
 NAME==h
+BROKEN=1
 FILE=--
 CMDS=!scripts/test-webserver.sh
 EXPECT=<<EOF
@@ -58,4 +59,4 @@ Starting http server...
 open http://localhost:9292/
 r2 -C http://localhost:9292/cmd/
 EOF
-RUN
+NORUN