Some slight adjustment to DynablocAge and steam/heroic settings

docs/USAGE.md

@@ -212,6 +212,11 @@ Dynarec print the missing opcodes
 * 0 : not print the missing opcode (Default, unless DYNAREC_LOG>=1 or DYNAREC_DUMP>=1 is used)
 * 1 : Will print the missing opcodes
 
+#### BOX64_DYNAREC_AGE *
+Dynarec will check if black can be freed when that atteins a certain age (in dynablock creation count)
+* 0 : do not age dynablocks (don't try to free old one)
+* XXXX: try to free block that are at least XXXX old (default 10240)
+
 #### BOX64_SSE_FLUSHTO0 *
 Handling of SSE Flush to 0 flags
 * 0 : Just track the flag (Default)

src/custommem.c

@@ -73,6 +73,7 @@ typedef struct memprot_s
     uint8_t* prot;
     uint8_t* hot;
 } memprot_t;
+//#define TRACE_MEMSTAT
 #ifdef TRACE_MEMSTAT
 static uint64_t  memprot_allocated = 0, memprot_max_allocated = 0;
 #endif
@@ -481,6 +482,7 @@ void customFree(void* p)
 #define NCHUNK          64
 typedef struct mmaplist_s {
     blocklist_t         chunks[NCHUNK];
+    size_t              minage[NCHUNK];
     mmaplist_t*         next;
 } mmaplist_t;
 
@@ -522,11 +524,13 @@ dynablock_t* FindDynablockFromNativeAddress(void* p)
 #ifdef TRACE_MEMSTAT
 static uint64_t dynarec_allocated = 0;
 #endif
-uintptr_t AllocDynarecMap(size_t size)
+uintptr_t AllocDynarecMap(size_t size, size_t age)
 {
     if(!size)
         return 0;
 
+    static int current = 0;
+
     size = roundSize(size);
 
     mmaplist_t* list = mmaplist;
@@ -534,8 +538,32 @@ uintptr_t AllocDynarecMap(size_t size)
         list = mmaplist = (mmaplist_t*)box_calloc(1, sizeof(mmaplist_t));
     // check if there is space in current open ones
     int i = 0;
+    int current_i = 0;
     uintptr_t sz = size + 2*sizeof(blockmark_t);
     while(1) {
+        size_t newage = 0;
+        if(box64_dynarec_age && current==current_i && (!list->minage[i] || list->minage[i]<=age)) {
+            //check age of blocks for cleaning
+            blockmark_t* sub = (blockmark_t*)list->chunks[i].block;
+            uintptr_t end = (uintptr_t)list->chunks[i].block+list->chunks[i].size-sizeof(blockmark_t);
+            while(sub && (uintptr_t)sub<end) {
+                blockmark_t* n = NEXT_BLOCK(sub);
+                dynablock_t* db = NULL;
+                if(n->next.fill)
+                    db = *(dynablock_t**)((uintptr_t)n+sizeof(blockmark_t));
+                if(db) {
+                    size_t new_age = AgeDynablock(db, age);
+                    if(!new_age)    // db has been removed
+                        n = sub;    // go back 1 step
+                    else if(new_age<newage || !newage)
+                        newage = new_age;
+                }
+                sub = n;
+            }
+            list->minage[i] = newage;
+        }
+        if(current==current_i)
+            current++;
         if(list->chunks[i].maxfree>=size) {
             // looks free, try to alloc!
             size_t rsize = 0;
@@ -551,6 +579,8 @@ uintptr_t AllocDynarecMap(size_t size)
         }
         // check if new
         if(!list->chunks[i].size) {
+            if(current_i+1==current)
+                current = 0;
             // alloc a new block, aversized or not, we are at the end of the list
             size_t allocsize = (sz>MMAPSIZE)?sz:MMAPSIZE;
             // allign sz with pagesize
@@ -576,6 +606,7 @@ uintptr_t AllocDynarecMap(size_t size)
             list->chunks[i].block = p;
             list->chunks[i].first = p;
             list->chunks[i].size = allocsize;
+            list->minage[i] = age + box64_dynarec_age;
             // setup marks
             blockmark_t* m = (blockmark_t*)p;
             m->prev.x32 = 0;
@@ -594,6 +625,7 @@ uintptr_t AllocDynarecMap(size_t size)
         }
         // next chunk...
         ++i;
+        ++current_i;
         if(i==NCHUNK) {
             i = 0;
             if(!list->next)
@@ -656,6 +688,51 @@ static uintptr_t getDBSize(uintptr_t addr, size_t maxsize, dynablock_t** db)
     return (addr&~JMPTABLE_MASK0)+idx0+1;
 }
 
+void freeJmpTable(uintptr_t addr)
+{
+    uintptr_t idx3, idx2, idx1, idx0;
+    idx3 = (((uintptr_t)addr)>>JMPTABL_START3)&JMPTABLE_MASK3;
+    idx2 = (((uintptr_t)addr)>>JMPTABL_START2)&JMPTABLE_MASK2;
+    idx1 = (((uintptr_t)addr)>>JMPTABL_START1)&JMPTABLE_MASK1;
+    idx0 = (((uintptr_t)addr)                )&JMPTABLE_MASK0;
+    if(box64_jmptbl3[idx3] == box64_jmptbldefault2) {
+        return;
+    }
+    if(box64_jmptbl3[idx3][idx2] == box64_jmptbldefault1) {
+        return;
+    }
+    if(box64_jmptbl3[idx3][idx2][idx1] == box64_jmptbldefault0) {
+        return;
+    }
+    uintptr_t* tbl0 = box64_jmptbl3[idx3][idx2][idx1];
+    for(int i=0; i<1<<JMPTABL_SHIFT0; ++i)
+        if(tbl0[i]!=(uintptr_t)native_next)
+            return;
+    box64_jmptbl3[idx3][idx2][idx1] = box64_jmptbldefault0;
+    box_free(tbl0);
+#ifdef TRACE_MEMSTAT
+    jmptbl_allocated -= (1<<JMPTABL_SHIFT0)*sizeof(uintptr_t);
+#endif
+    uintptr_t** tbl1 = box64_jmptbl3[idx3][idx2];
+    for(int i=0; i<1<<JMPTABL_SHIFT1; ++i)
+        if(tbl1[i]!=box64_jmptbldefault0)
+            return;
+    box64_jmptbl3[idx3][idx2] = box64_jmptbldefault1;
+    box_free(tbl1);
+#ifdef TRACE_MEMSTAT
+    jmptbl_allocated -= (1<<JMPTABL_SHIFT1)*sizeof(uintptr_t*);
+#endif
+    uintptr_t*** tbl2 = box64_jmptbl3[idx3];
+    for(int i=0; i<1<<JMPTABL_SHIFT2; ++i)
+        if(tbl2[i]!=box64_jmptbldefault1)
+            return;
+    box64_jmptbl3[idx3] = box64_jmptbldefault2;
+    box_free(tbl2);
+#ifdef TRACE_MEMSTAT
+    jmptbl_allocated -= (1<<JMPTABL_SHIFT2)*sizeof(uintptr_t**);
+#endif
+}
+
 // each dynmap is 64k of size
 
 void addDBFromAddressRange(uintptr_t addr, size_t size)
@@ -670,14 +747,18 @@ void cleanDBFromAddressRange(uintptr_t addr, size_t size, int destroy)
     dynarec_log(LOG_DEBUG, "cleanDBFromAddressRange %p/%p -> %p %s\n", (void*)addr, (void*)start_addr, (void*)(addr+size-1), destroy?"destroy":"mark");
     dynablock_t* db = NULL;
     uintptr_t end = addr+size;
+    uintptr_t old_addr = 0;
     while (start_addr<end) {
-        start_addr = getDBSize(start_addr, end-start_addr, &db);
+        old_addr = getDBSize(start_addr, end-start_addr, &db);
         if(db) {
             if(destroy)
                 FreeRangeDynablock(db, addr, size);
             else
                 MarkRangeDynablock(db, addr, size);
         }
+        /*if(destroy && start_addr && ((start_addr&~JMPTABLE_MASK0)!=(old_addr&~JMPTABLE_MASK0)))
+            freeJmpTable(start_addr);*/ // cannot free jmpTable as some dynablock might rely on direct address (for fixed IP jmp/call)
+        start_addr = old_addr;
     }
 }
 

src/dynarec/arm64/dynarec_arm64_helper.c

@@ -544,6 +544,16 @@ uintptr_t geted16(dynarec_arm_t* dyn, uintptr_t addr, int ninst, uint8_t nextop,
     return addr;
 }
 
+static void exit_block(dynarec_arm_t* dyn, int ninst)
+{
+    ADR_S20(x5, (intptr_t)dyn->enter - (uintptr_t)dyn->block);
+    LDAXRw(x4, x5);
+    SUBw_U12(x4, x4, 1);
+    STLXRw(x3, x4, x5);
+    CBNZw(x3, 4-4*4);
+}
+#define EXIT_SIZE (4*5)
+
 void jump_to_epilog(dynarec_arm_t* dyn, uintptr_t ip, int reg, int ninst)
 {
     MAYUSE(dyn); MAYUSE(ip); MAYUSE(ninst);
@@ -559,6 +569,8 @@ void jump_to_epilog(dynarec_arm_t* dyn, uintptr_t ip, int reg, int ninst)
     NOTEST(x2);
     TABLE64(x2, (uintptr_t)arm64_epilog);
     SMEND();
+    if(box64_dynarec_age)
+        exit_block(dyn, ninst);
     BR(x2);
 }
 
@@ -599,6 +611,8 @@ void jump_to_next(dynarec_arm_t* dyn, uintptr_t ip, int reg, int ninst)
     //MOVx(x3, 15);    no access to PC reg
     #endif
     SMEND();
+    if(box64_dynarec_age)
+        exit_block(dyn, ninst);
     BLR(x2); // save LR...
 }
 
@@ -612,10 +626,12 @@ void ret_to_epilog(dynarec_arm_t* dyn, int ninst, rex_t rex)
     if(box64_dynarec_callret) {
         // pop the actual return address for ARM stack
         LDPx_S7_offset(x2, x6, xSP, 0);
-        CBZx(x6, 5*4);
+        CBZx(x6, 5*4+(box64_dynarec_age?EXIT_SIZE:0));
         ADDx_U12(xSP, xSP, 16);
         SUBx_REG(x6, x6, xRIP); // is it the right address?
-        CBNZx(x6, 2*4);
+        CBNZx(x6, 2*4+(box64_dynarec_age?EXIT_SIZE:0));
+        if(box64_dynarec_age)
+            exit_block(dyn, ninst);
         BLR(x2);
         // not the correct return address, regular jump
     }
@@ -630,6 +646,8 @@ void ret_to_epilog(dynarec_arm_t* dyn, int ninst, rex_t rex)
     LDRx_REG_LSL3(x2, x2, x3);
     UBFXx(x3, xRIP, JMPTABL_START0, JMPTABL_SHIFT0);
     LDRx_REG_LSL3(x2, x2, x3);
+    if(box64_dynarec_age)
+        exit_block(dyn, ninst);
     BLR(x2); // save LR
     CLEARIP();
 }
@@ -650,10 +668,12 @@ void retn_to_epilog(dynarec_arm_t* dyn, int ninst, rex_t rex, int n)
     if(box64_dynarec_callret) {
         // pop the actual return address for ARM stack
         LDPx_S7_offset(x2, x6, xSP, 0);
-        CBZx(x6, 5*4);
+        CBZx(x6, 5*4+(box64_dynarec_age?EXIT_SIZE:0));
         ADDx_U12(xSP, xSP, 16);
         SUBx_REG(x6, x6, xRIP); // is it the right address?
-        CBNZx(x6, 2*4);
+        CBNZx(x6, 2*4+(box64_dynarec_age?EXIT_SIZE:0));
+        if(box64_dynarec_age)
+            exit_block(dyn, ninst);
         BLR(x2);
         // not the correct return address, regular jump
     }
@@ -668,6 +688,8 @@ void retn_to_epilog(dynarec_arm_t* dyn, int ninst, rex_t rex, int n)
     LDRx_REG_LSL3(x2, x2, x3);
     UBFXx(x3, xRIP, JMPTABL_START0, JMPTABL_SHIFT0);
     LDRx_REG_LSL3(x2, x2, x3);
+    if(box64_dynarec_age)
+        exit_block(dyn, ninst);
     BLR(x2); // save LR
     CLEARIP();
 }
@@ -712,6 +734,8 @@ void iret_to_epilog(dynarec_arm_t* dyn, int ninst, int is64bits)
     // Ret....
     MOV64x(x2, (uintptr_t)arm64_epilog);  // epilog on purpose, CS might have changed!
     SMEND();
+    if(box64_dynarec_age)
+        exit_block(dyn, ninst);
     BR(x2);
     CLEARIP();
 }

src/dynarec/arm64/dynarec_arm64_jmpnext.c

@@ -8,4 +8,17 @@ void CreateJmpNext(void* addr, void* next)
     uint32_t* block = (uint32_t*)addr;
     LDRx_literal(x2, (intptr_t)next - (intptr_t)addr);
     BR(x2);
+}
+
+int CreateEnter(void* addr, void* enter)
+{
+    if(enter) {
+        uint32_t* block = (uint32_t*)addr;
+        ADR_S20(x2, (intptr_t)enter - (intptr_t)addr);
+        LDAXRw(x1, x2);
+        ADDw_U12(x1, x1, 1);
+        STLXRw(x3, x1, x2);
+        CBNZw(x3, 4-4*4);
+    }
+    return 4*5;
 }

src/dynarec/arm64/dynarec_arm64_pass0.h

@@ -5,6 +5,8 @@
     dyn->insts[ninst].x64.addr = addr;  \
     if(ninst) dyn->insts[ninst-1].x64.size = dyn->insts[ninst].x64.addr - dyn->insts[ninst-1].x64.addr
 
+#define ENTER() 
+
 #define MESSAGE(A, ...)  
 #define MAYSETFLAGS()   dyn->insts[ninst].x64.may_set = 1
 #define READFLAGS(A)    \

src/dynarec/arm64/dynarec_arm64_pass1.h

@@ -1,5 +1,6 @@
 #define INIT    
 #define FINI
+#define ENTER() 
 #define MESSAGE(A, ...)  
 #define EMIT(A)     
 #define NEW_INST                                \

src/dynarec/arm64/dynarec_arm64_pass2.h

@@ -4,7 +4,7 @@
                 dyn->insts[ninst].address = (dyn->insts[ninst-1].address+dyn->insts[ninst-1].size);     \
                 dyn->insts_size += 1+((dyn->insts[ninst].x64.size>(dyn->insts[ninst].size/4))?dyn->insts[ninst].x64.size:(dyn->insts[ninst].size/4))/15; \
         }
-
+#define ENTER() dyn->block_prolog = CreateEnter(NULL, NULL)
 #define MESSAGE(A, ...)  
 #define EMIT(A)     do{dyn->insts[ninst].size+=4; dyn->native_size+=4;}while(0)
 #define NEW_INST                                                                                        \

src/dynarec/arm64/dynarec_arm64_pass3.h

@@ -3,6 +3,7 @@
     if(ninst)       \
         addInst(dyn->instsize, &dyn->insts_size, dyn->insts[ninst].x64.size, dyn->insts[ninst].size/4); \
     addInst(dyn->instsize, &dyn->insts_size, 0, 0);
+#define ENTER()  dyn->block += CreateEnter(dyn->block, dyn->enter)
 #define EMIT(A)                                         \
     do{                                                 \
         if(box64_dynarec_dump) print_opcode(dyn, ninst, (uint32_t)(A)); \

src/dynarec/arm64/dynarec_arm64_private.h

@@ -88,6 +88,7 @@ typedef struct dynarec_arm_s {
     int32_t             cap;
     uintptr_t           start;      // start of the block
     uint32_t            isize;      // size in byte of x64 instructions included
+    uint32_t            block_prolog; // size in byte of block prolog (not part of 1st instruction)
     void*               block;      // memory pointer where next instruction is emitted
     uintptr_t           native_start;  // start of the arm code
     size_t              native_size;   // size of emitted arm code
@@ -115,6 +116,7 @@ typedef struct dynarec_arm_s {
     uint8_t             doublepush;
     uint8_t             doublepop;
     uint8_t             always_test;
+    int*                enter;
 } dynarec_arm_t;
 
 void add_next(dynarec_arm_t *dyn, uintptr_t addr);
@@ -125,6 +127,7 @@ int is_instructions(dynarec_arm_t *dyn, uintptr_t addr, int n);
 int Table64(dynarec_arm_t *dyn, uint64_t val, int pass);  // add a value to etable64 (if needed) and gives back the imm19 to use in LDR_literal
 
 void CreateJmpNext(void* addr, void* next);
+int CreateEnter(void* addr, void* enter);
 
 #define GO_TRACE(A, B)      \
     GETIP(addr);            \

src/dynarec/dynablock.c

@@ -90,6 +90,43 @@ void FreeDynablock(dynablock_t* db, int need_lock)
     }
 }
 
+size_t AgeDynablock(dynablock_t* db, size_t age)
+{
+    if(db) {
+        dynarec_log(LOG_DEBUG, "AgeDynablock %p %p-%p, age=%zu, db:age/romove=%zu/%d\n", db, db->x64_addr, db->x64_addr+db->x64_size-1, age, db->age, db->try_remove);
+        if(db->age>age || !db->done || db->gone)
+            return db->age;
+        dynablock_t* actual = getDB((uintptr_t)db->x64_addr);
+        if(actual != db) {
+            printf_log(LOG_INFO, "Warning, AgeDynablock with a rogue db %p instead of %p for %p\n", db, actual, db->x64_addr);
+            return age + box64_dynarec_age;
+        }
+        int need_test = getNeedTest((uintptr_t)db->x64_addr);
+        if(db->try_remove) {
+            if(need_test && !*db->active_count) {
+                db = InvalidDynablock(db, 0);
+                FreeInvalidDynablock(db, 0);
+                return 0;
+            } else {
+                db->try_remove = 0;
+                db->age = age + box64_dynarec_age; // re-arm the wait
+            }
+        } else {
+            if(need_test || (getProtection((uintptr_t)db->x64_addr)&PROT_NOPROT) || db->always_test || !db->active_count || *db->active_count) {
+                // don't mess with this kind of dynablock...
+                db->age = age+box64_dynarec_age*2;
+            } else if(setJumpTableIfRef64(db->x64_addr, db->jmpnext, db->block)) {
+                dynarec_log(LOG_DEBUG, "Marking block %p at %p for later removal\n", db, db->x64_addr);
+                db->age = age+box64_dynarec_age;
+                db->try_remove = 1;
+            } else {
+                db->age = age+box64_dynarec_age*2; // failed to swap, try again later
+            }
+        }
+        return db->age;
+    }
+    return 0;
+}
 
 
 void MarkDynablock(dynablock_t* db)
@@ -109,6 +146,8 @@ void MarkDynablock(dynablock_t* db)
                 else
                     db->previous = old;
             }
+        } else {
+            db->try_remove = 0;
         }
     }
 }
@@ -177,7 +216,7 @@ void cancelFillBlock()
 {
     longjmp(&dynarec_jmpbuf, 1);
 }
-
+extern size_t              box64_counter;
 /* 
     return NULL if block is not found / cannot be created. 
     Don't create if create==0
@@ -187,8 +226,10 @@ static dynablock_t* internalDBGetBlock(x64emu_t* emu, uintptr_t addr, uintptr_t
     if(hasAlternate((void*)addr))
         return NULL;
     dynablock_t* block = getDB(addr);
-    if(block || !create)
+    if(block || !create) {
+        if(block) block->age = box64_counter+box64_dynarec_age;
         return block;
+    }
 
     if(need_lock) {
         if(box64_dynarec_wait) {
@@ -250,7 +291,7 @@ dynablock_t* DBGetBlock(x64emu_t* emu, uintptr_t addr, int create, int is32bits)
     if(db && db->done && db->block && getNeedTest(addr)) {
         if(db->always_test)
             sched_yield();  // just calm down...
-        if(AreaInHotPage((uintptr_t)db->x64_addr, (uintptr_t)db->x64_addr + db->x64_size - 1)) {
+        if(!db->try_remove && AreaInHotPage((uintptr_t)db->x64_addr, (uintptr_t)db->x64_addr + db->x64_size - 1)) {
             emu->test.test = 0;
             if(box64_dynarec_fastpage) {
                 uint32_t hash = X31_hash_code(db->x64_addr, db->x64_size);
@@ -289,7 +330,8 @@ dynablock_t* DBGetBlock(x64emu_t* emu, uintptr_t addr, int create, int is32bits)
             // fill back jumptable
             if(isprotectedDB((uintptr_t)db->x64_addr, db->x64_size) && !db->always_test) {
                 setJumpTableIfRef64(db->x64_addr, db->block, db->jmpnext);
-            }
+            } else 
+                db->try_remove = 0;
         }
         if(!need_lock)
             mutex_unlock(&my_context->mutex_dyndump);
@@ -327,7 +369,8 @@ dynablock_t* DBAlternateBlock(x64emu_t* emu, uintptr_t addr, uintptr_t filladdr,
             // fill back jumptable
             if(isprotectedDB((uintptr_t)db->x64_addr, db->x64_size) && !db->always_test) {
                 setJumpTableIfRef64(db->x64_addr, db->block, db->jmpnext);
-            }
+            } else
+                db->try_remove = 0;
         }
         if(!need_lock)
             mutex_unlock(&my_context->mutex_dyndump);

src/dynarec/dynablock_private.h

@@ -10,17 +10,20 @@ typedef struct dynablock_s {
     void*           block;  // block-sizeof(void*) == self
     void*           actual_block;   // the actual start of the block (so block-sizeof(void*))
     struct dynablock_s*    previous;   // a previous block that might need to be freed
-    int             size;
     void*           x64_addr;
     uintptr_t       x64_size;
+    int             size;
     uint32_t        hash;
     uint8_t         done;
     uint8_t         gone;
     uint8_t         always_test;
     uint8_t         dirty;      // if need to be tested as soon as it's created
-    int             isize;
     instsize_t*     instsize;
     void*           jmpnext;    // a branch jmpnext code when block is marked
+    size_t          age;        //age of the block
+    int             try_remove; // block marked as try removed
+    int             isize;
+    uint32_t*       active_count;   // is this block currently being executed, or NULL if not tracked
 } dynablock_t;
 
 #endif //__DYNABLOCK_PRIVATE_H_

src/dynarec/dynarec_native.c

@@ -23,6 +23,8 @@
 #include "dynarec_arch.h"
 #include "dynarec_next.h"
 
+size_t              box64_counter = 0;
+
 void printf_x64_instruction(zydis_dec_t* dec, instruction_x64_t* inst, const char* name) {
     uint8_t *ip = (uint8_t*)inst->addr;
     if(ip[0]==0xcc && ip[1]=='S' && ip[2]=='C') {
@@ -407,7 +409,7 @@ void* CreateEmptyBlock(dynablock_t* block, uintptr_t addr) {
     block->isize = 0;
     block->done = 0;
     size_t sz = 4*sizeof(void*);
-    void* actual_p = (void*)AllocDynarecMap(sz);
+    void* actual_p = (void*)AllocDynarecMap(sz, box64_counter);
     void* p = actual_p + sizeof(void*);
     if(actual_p==NULL) {
         dynarec_log(LOG_INFO, "AllocDynarecMap(%p, %zu) failed, cancelling block\n", block, sz);
@@ -418,6 +420,7 @@ void* CreateEmptyBlock(dynablock_t* block, uintptr_t addr) {
     block->actual_block = actual_p;
     block->block = p;
     block->jmpnext = p;
+    block->age = box64_counter + box64_dynarec_age;
     *(dynablock_t**)actual_p = block;
     *(void**)(p+2*sizeof(void*)) = native_epilog;
     CreateJmpNext(block->jmpnext, p+2*sizeof(void*));
@@ -461,6 +464,7 @@ void* FillBlock64(dynablock_t* block, uintptr_t addr, int alternate, int is32bit
     helper.start = addr;
     uintptr_t start = addr;
     helper.cap = 128;
+    ++box64_counter;
     helper.insts = (instruction_native_t*)dynaCalloc(helper.cap, sizeof(instruction_native_t));
     // pass 0, addresses, x64 jump addresses, overall size of the block
     uintptr_t end = native_pass0(&helper, addr, alternate, is32bits);
@@ -521,15 +525,20 @@ void* FillBlock64(dynablock_t* block, uintptr_t addr, int alternate, int is32bit
     // keep size of instructions for signal handling
     size_t insts_rsize = (helper.insts_size+2)*sizeof(instsize_t);
     insts_rsize = (insts_rsize+7)&~7;   // round the size...
-    size_t native_size = (helper.native_size+7)&~7;   // round the size...
+    size_t native_size = (helper.native_size+helper.block_prolog+7)&~7;   // round the size...
     // ok, now allocate mapped memory, with executable flag on
-    size_t sz = sizeof(void*) + native_size + helper.table64size*sizeof(uint64_t) + 4*sizeof(void*) + insts_rsize;
-    //           dynablock_t*     block (arm insts)            table64               jmpnext code       instsize
-    void* actual_p = (void*)AllocDynarecMap(sz);
+    size_t sz = sizeof(void*)                           // dynablock_t* 
+               + native_size                            // block (arm insts)
+               + helper.table64size*sizeof(uint64_t)    // table64
+               + 4*sizeof(void*)                        // jmpnext code
+               + (box64_dynarec_age?sizeof(void*):0)    // enter_count
+               + insts_rsize;                           // instsize
+
+    void* actual_p = (void*)AllocDynarecMap(sz, box64_counter);
     void* p = (void*)(((uintptr_t)actual_p) + sizeof(void*));
     void* tablestart = p + native_size;
     void* next = tablestart + helper.table64size*sizeof(uint64_t);
-    void* instsize = next + 4*sizeof(void*);
+    void* instsize = next + 4*sizeof(void*) + (box64_dynarec_age?sizeof(void*):0);
     if(actual_p==NULL) {
         dynarec_log(LOG_INFO, "AllocDynarecMap(%p, %zu) failed, cancelling block\n", block, sz);
         CancelBlock64(0);
@@ -537,6 +546,8 @@ void* FillBlock64(dynablock_t* block, uintptr_t addr, int alternate, int is32bit
     }
     helper.block = p;
     block->actual_block = actual_p;
+    block->active_count = box64_dynarec_age?(next + 4*sizeof(void*)):NULL;
+    block->age = box64_counter + box64_dynarec_age;
     helper.native_start = (uintptr_t)p;
     helper.tablestart = (uintptr_t)tablestart;
     helper.jmp_next = (uintptr_t)next+sizeof(void*);
@@ -545,6 +556,7 @@ void* FillBlock64(dynablock_t* block, uintptr_t addr, int alternate, int is32bit
     helper.table64cap = helper.table64size;
     dynaFree(helper.table64);
     helper.table64 = (uint64_t*)helper.tablestart;
+    helper.enter = block->active_count;
     // pass 3, emit (log emit native opcode)
     if(box64_dynarec_dump) {
         dynarec_log(LOG_NONE, "%s%04d|Emitting %zu bytes for %u %s bytes", (box64_dynarec_dump>1)?"\e[01;36m":"", GetTID(), helper.native_size, helper.isize, is32bits?"x86":"x64"); 
@@ -561,7 +573,7 @@ void* FillBlock64(dynablock_t* block, uintptr_t addr, int alternate, int is32bit
     // keep size of instructions for signal handling
     block->instsize = instsize;
     // ok, free the helper now
-    dynaFree(helper.insts);
+    instruction_native_t* pass2_insts = helper.insts;
     helper.insts = NULL;
     helper.table64 = NULL;
     helper.instsize = NULL;
@@ -592,17 +604,19 @@ void* FillBlock64(dynablock_t* block, uintptr_t addr, int alternate, int is32bit
         printf_log(LOG_NONE, "BOX64: Warning, size difference in block between pass2 (%zu) & pass3 (%zu)!\n", sz, helper.native_size+helper.table64size*8);
         uint8_t *dump = (uint8_t*)helper.start;
         printf_log(LOG_NONE, "Dump of %d x64 opcodes:\n", helper.size);
-        for(int i=0; i<helper.size; ++i) {
+        for(int i=0; i<helper.size-1; ++i) {
             printf_log(LOG_NONE, "%p:", dump);
-            for(; dump<(uint8_t*)helper.insts[i+1].x64.addr; ++dump)
+            for(; dump<(uint8_t*)pass2_insts[i+1].x64.addr; ++dump)
                 printf_log(LOG_NONE, " %02X", *dump);
-            printf_log(LOG_NONE, "\t%d -> %d\n", helper.insts[i].size2, helper.insts[i].size);
+            printf_log(LOG_NONE, "\t%d -> %d\n", pass2_insts[i].size2, pass2_insts[i].size);
         }
         printf_log(LOG_NONE, "Table64 \t%d -> %d\n", oldtable64size*8, helper.table64size*8);
         printf_log(LOG_NONE, " ------------\n");
+        dynaFree(pass2_insts);
         CancelBlock64(0);
         return NULL;
     }
+    dynaFree(pass2_insts);
     if(insts_rsize/sizeof(instsize_t)<helper.insts_size) {
         printf_log(LOG_NONE, "BOX64: Warning, ists_size difference in block between pass2 (%zu) and pass3 (%zu), allocated: %zu\n", oldinstsize, helper.insts_size, insts_rsize/sizeof(instsize_t));
     }

src/dynarec/dynarec_native_pass.c

@@ -50,6 +50,9 @@ uintptr_t native_pass(dynarec_native_t* dyn, uintptr_t addr, int alternate, int
     int stopblock = 2+(FindElfAddress(my_context, addr)?0:1); // if block is in elf_memory, it can be extended with bligblocks==2, else it needs 3
     // ok, go now
     INIT;
+    if(box64_dynarec_age) {
+        ENTER();
+    }
     while(ok) {
         ip = addr;
         if (reset_n!=-1) {

src/dynarec/rv64/dynarec_rv64_helper.c

@@ -474,6 +474,15 @@ uintptr_t geted32(dynarec_rv64_t* dyn, uintptr_t addr, int ninst, uint8_t nextop
     return addr;
 }
 
+static void exit_block(dynarec_rv64_t* dyn, int ninst)
+{
+    uintptr_t diff = (intptr_t)dyn->enter - (uintptr_t)dyn->block;
+    ADDI(x4, xZR, -1);
+    AUIPC(x5, SPLIT20(diff));
+    ADDI(x5, x5, SPLIT12(diff));
+    AMOADD_W(xZR, x4, x5, 1, 1);
+}
+
 void jump_to_epilog(dynarec_rv64_t* dyn, uintptr_t ip, int reg, int ninst)
 {
     MAYUSE(dyn); MAYUSE(ip); MAYUSE(ninst);
@@ -488,6 +497,8 @@ void jump_to_epilog(dynarec_rv64_t* dyn, uintptr_t ip, int reg, int ninst)
     }
     TABLE64(x2, (uintptr_t)rv64_epilog);
     SMEND();
+    if(box64_dynarec_age)
+        exit_block(dyn, ninst);
     BR(x2);
 }
 
@@ -505,6 +516,8 @@ void jump_to_epilog_fast(dynarec_rv64_t* dyn, uintptr_t ip, int reg, int ninst)
     }
     TABLE64(x2, (uintptr_t)rv64_epilog_fast);
     SMEND();
+    if(box64_dynarec_age)
+        exit_block(dyn, ninst);
     BR(x2);
 }
 
@@ -560,6 +573,8 @@ void jump_to_next(dynarec_rv64_t* dyn, uintptr_t ip, int reg, int ninst)
     //MOVx(x3, 15);    no access to PC reg
     #endif
     SMEND();
+    if(box64_dynarec_age)
+        exit_block(dyn, ninst);
     JALR(x2); // save LR...
 }
 
@@ -607,6 +622,8 @@ void ret_to_epilog(dynarec_rv64_t* dyn, int ninst, rex_t rex)
     }
     if(rv64_zba) SH3ADD(x3, x2, x3); else {SLLI(x2, x2, 3); ADD(x3, x3, x2);}
     LD(x2, x3, 0);
+    if(box64_dynarec_age)
+        exit_block(dyn, ninst);
     JALR(x2); // save LR
     CLEARIP();
 }
@@ -661,6 +678,8 @@ void retn_to_epilog(dynarec_rv64_t* dyn, int ninst, rex_t rex, int n)
     }
     if(rv64_zba) SH3ADD(x3, x2, x3); else {SLLI(x2, x2, 3); ADD(x3, x3, x2);}
     LD(x2, x3, 0);
+    if(box64_dynarec_age)
+        exit_block(dyn, ninst);
     JALR(x2); // save LR
     CLEARIP();
 }
@@ -704,6 +723,8 @@ void iret_to_epilog(dynarec_rv64_t* dyn, int ninst, int is64bits)
     // Ret....
     MOV64x(x2, (uintptr_t)rv64_epilog);  // epilog on purpose, CS might have changed!
     SMEND();
+    if(box64_dynarec_age)
+        exit_block(dyn, ninst);
     BR(x2);
     CLEARIP();
 }

src/dynarec/rv64/dynarec_rv64_jmpnext.c

@@ -16,4 +16,17 @@ void CreateJmpNext(void* addr, void* next)
     LR_D(x2, x2, 1, 1);
     #endif
     BR(x2);
+}
+
+void CreateEnter(void* addr, void* enter)
+{
+    if(enter) {
+        uint32_t* block = (uint32_t*)addr;
+        uintptr_t diff = (intptr_t)enter - (intptr_t)addr;
+        ADDI(x1, xZR, 1);
+        AUIPC(x2, SPLIT20(diff));
+        ADDI(x2, x2, SPLIT12(diff));
+        AMOADD_W(x3, x1, x2, 1, 1);
+    }
+    return 4*4;
 }

src/dynarec/rv64/dynarec_rv64_pass0.h

@@ -5,6 +5,8 @@
     dyn->insts[ninst].x64.addr = addr;  \
     if(ninst) dyn->insts[ninst-1].x64.size = dyn->insts[ninst].x64.addr - dyn->insts[ninst-1].x64.addr
 
+#define ENTER()
+
 #define MESSAGE(A, ...)  
 #define MAYSETFLAGS()   dyn->insts[ninst].x64.may_set = 1
 #define READFLAGS(A)    \

src/dynarec/rv64/dynarec_rv64_pass1.h

@@ -1,5 +1,6 @@
 #define INIT    
 #define FINI
+#define ENTER() 
 #define MESSAGE(A, ...)  
 #define EMIT(A)     
 #define NEW_INST                                \

src/dynarec/rv64/dynarec_rv64_pass2.h

@@ -4,7 +4,7 @@
                 dyn->insts[ninst].address = (dyn->insts[ninst-1].address+dyn->insts[ninst-1].size);     \
                 dyn->insts_size += 1+((dyn->insts[ninst].x64.size>(dyn->insts[ninst].size/4))?dyn->insts[ninst].x64.size:(dyn->insts[ninst].size/4))/15; \
         }
-
+#define ENTER() dyn->block_prolog = CreateEnter(NULL, NULL)
 #define MESSAGE(A, ...)  
 #define EMIT(A)     do {dyn->insts[ninst].size+=4; dyn->native_size+=4;}while(0)
 #define NEW_INST                                                                                        \

src/dynarec/rv64/dynarec_rv64_pass3.h

@@ -3,6 +3,7 @@
     if(ninst)       \
         addInst(dyn->instsize, &dyn->insts_size, dyn->insts[ninst].x64.size, dyn->insts[ninst].size/4); \
     addInst(dyn->instsize, &dyn->insts_size, 0, 0);
+#define ENTER()  dyn->block += CreateEnter(dyn->block, dyn->enter)
 #define EMIT(A)     \
     do {                                                \
         if(box64_dynarec_dump) print_opcode(dyn, ninst, (uint32_t)A); \
@@ -10,7 +11,6 @@
         dyn->block += 4; dyn->native_size += 4;         \
         dyn->insts[ninst].size2 += 4;                   \
     }while(0)
-
 #define MESSAGE(A, ...)  if(box64_dynarec_dump) dynarec_log(LOG_NONE, __VA_ARGS__)
 #define NEW_INST        \
     if(box64_dynarec_dump) print_newinst(dyn, ninst);   \

src/dynarec/rv64/dynarec_rv64_private.h

@@ -99,6 +99,7 @@ typedef struct dynarec_rv64_s {
     uintptr_t           start;      // start of the block
     uint32_t            isize;      // size in byte of x64 instructions included
     void*               block;      // memory pointer where next instruction is emitted
+    uint32_t            block_prolog; // size in byte of block prolog (not part of 1st instruction)
     uintptr_t           native_start;  // start of the arm code
     size_t              native_size;   // size of emitted arm code
     uintptr_t           last_ip;    // last set IP in RIP (or NULL if unclean state) TODO: move to a cache something
@@ -123,6 +124,7 @@ typedef struct dynarec_rv64_s {
     int32_t             forward_size;   // size at the forward point
     int                 forward_ninst;  // ninst at the forward point
     uint8_t             always_test;
+    int*                enter;
 } dynarec_rv64_t;
 
 // convert idx (0..24) to reg index (10..31 0..1)
@@ -138,6 +140,7 @@ int is_instructions(dynarec_rv64_t *dyn, uintptr_t addr, int n);
 int Table64(dynarec_rv64_t *dyn, uint64_t val, int pass);  // add a value to etable64 (if needed) and gives back the imm19 to use in LDR_literal
 
 void CreateJmpNext(void* addr, void* next);
+int CreateEnter(void* addr, void* enter);
 
 #define GO_TRACE(A, B)      \
     GETIP(addr);            \

src/dynarec/rv64/rv64_emitter.h

@@ -381,6 +381,7 @@ f28–31  ft8–11  FP temporaries                  Caller
 #define SC_W(rd, rs2, rs1, aq, rl)  EMIT(R_type(AQ_RL(0b00011, aq, rl), rs2, rs1, 0b010, rd, 0b0101111))
 
 #define AMOSWAP_W(rd, rs2, rs1, aq, rl)  EMIT(R_type(AQ_RL(0b00001, aq, rl), rs2, rs1, 0b010, rd, 0b0101111))
+#define AMOADD_W(rd, rs2, rs1, aq, rl)   EMIT(R_type(AQ_RL(0b00000, aq, rl), rs2, rs1, 0b010, rd, 0b0101111))
 
 // RV64A
 #define LR_D(rd, rs1, aq, rl)       EMIT(R_type(AQ_RL(0b00010, aq, rl), 0, rs1, 0b011, rd, 0b0101111))

src/include/custommem.h

@@ -19,7 +19,7 @@ void customFree(void* p);
 #ifdef DYNAREC
 typedef struct dynablock_s dynablock_t;
 // custom protection flag to mark Page that are Write protected for Dynarec purpose
-uintptr_t AllocDynarecMap(size_t size);
+uintptr_t AllocDynarecMap(size_t size, size_t age);
 void FreeDynarecMap(uintptr_t addr);
 
 void addDBFromAddressRange(uintptr_t addr, size_t size);

src/include/debug.h

@@ -29,6 +29,7 @@ extern int box64_dynarec_fastpage;
 extern int box64_dynarec_wait;
 extern int box64_dynarec_test;
 extern int box64_dynarec_missing;
+extern size_t box64_dynarec_age;
 #ifdef ARM64
 extern int arm64_asimd;
 extern int arm64_aes;

src/include/dynablock.h

@@ -6,6 +6,7 @@ typedef struct dynablock_s dynablock_t;
 
 uint32_t X31_hash_code(void* addr, int len);
 void FreeDynablock(dynablock_t* db, int need_lock);
+size_t AgeDynablock(dynablock_t* db, size_t age);
 void MarkDynablock(dynablock_t* db);
 void MarkRangeDynablock(dynablock_t* db, uintptr_t addr, uintptr_t size);
 int FreeRangeDynablock(dynablock_t* db, uintptr_t addr, uintptr_t size);

src/main.c

@@ -66,6 +66,7 @@ int box64_dynarec_jvm = 1;
 int box64_dynarec_wait = 1;
 int box64_dynarec_test = 0;
 int box64_dynarec_missing = 0;
+size_t box64_dynarec_age = 0;
 uintptr_t box64_nodynarec_start = 0;
 uintptr_t box64_nodynarec_end = 0;
 #ifdef ARM64
@@ -625,6 +626,18 @@ void LoadLogEnv()
         if(box64_dynarec_missing)
             printf_log(LOG_INFO, "Dynarec will print missing opcodes\n");
     }
+    p = getenv("BOX64_DYNAREC_AGE");
+    if(p) {
+        int val = -1;
+        if(sscanf(p, "%d", &val)==1) {
+            if(val>=0)
+                box64_dynarec_age = val;
+        }
+        if(box64_dynarec_age)
+            printf_log(LOG_INFO, "Dynarec will try to free dynablock older than %d\n", box64_dynarec_age);
+        else
+            printf_log(LOG_INFO, "Dynarec will not age dynablock\n");
+    }
     p = getenv("BOX64_NODYNAREC");
     if(p) {
         if (strchr(p,'-')) {

src/tools/rcfile.c

@@ -38,6 +38,8 @@ static const char default_rcfile[] =
 "[heroic]\n"
 "BOX64_NOSANDBOX=1\n"
 "BOX64_MALLOC_HACK=2\n"
+"BOX64_DYNAREC_BIGBLOCK=0\n"
+"BOX64_DYNAREC8AGE=4096\n"
 "\n"
 "[LotCG.x86_64]\n"
 "BOX64_DYNAREC_FASTROUND=0\n"
@@ -55,7 +57,8 @@ static const char default_rcfile[] =
 "BOX64_NOSANDBOX=1\n"
 "BOX64_MALLOC_HACK=2\n"
 "BOX64_LOG=0\n"
-"BOX64_DYNAREC_BIGBLOCK=0\n"
+"BOX64_DYNAREC_BIGBLOCK=1\n"
+"BOX64_DYNAREC8AGE=4096\n"
 "\n"
 "[steam-runtime-check-requirements]\n"
 "BOX64_EXIT=1\n"
@@ -133,6 +136,7 @@ ENTRYBOOL(BOX64_DYNAREC_JVM, box64_dynarec_jvm)                     \
 ENTRYINT(BOX64_DYNAREC_HOTPAGE, box64_dynarec_hotpage, 0, 255, 8)   \
 ENTRYBOOL(BOX64_DYNAREC_FASTPAGE, box64_dynarec_fastpage)           \
 ENTRYBOOL(BOX64_DYNAREC_WAIT, box64_dynarec_wait)                   \
+ENTRYULONG(BOX64_DYNAREC_AGE, box64_dynarec_age)                    \
 ENTRYSTRING_(BOX64_NODYNAREC, box64_nodynarec)                      \
 ENTRYBOOL(BOX64_DYNAREC_TEST, box64_dynarec_test)                   \
 ENTRYBOOL(BOX64_DYNAREC_MISSING, box64_dynarec_missing)             \
@@ -155,6 +159,7 @@ IGNORE(BOX64_DYNAREC_JVM)                                           \
 IGNORE(BOX64_DYNAREC_HOTPAGE)                                       \
 IGNORE(BOX64_DYNAREC_FASTPAGE)                                      \
 IGNORE(BOX64_DYNAREC_WAIT)                                          \
+IGNORE(BOX64_DYNAREC_AGE)                                           \
 IGNORE(BOX64_NODYNAREC)                                             \
 IGNORE(BOX64_DYNAREC_TEST)                                          \
 IGNORE(BOX64_DYNAREC_MISSING)                                       \

system/box64.box64rc

@@ -49,8 +49,8 @@ BOX64_DYNAREC_FASTROUND=0
 [heroic]
 BOX64_NOSANDBOX=1
 BOX64_MALLOC_HACK=2
-# this is to save some memory
 BOX64_DYNAREC_BIGBLOCK=0
+BOX64_DYNAREC_AGE=4096
 
 [LotCG.x86_64]
 BOX64_DYNAREC_FASTROUND=0
@@ -89,8 +89,8 @@ BOX64_EMULATED_LIBS=libSDL2-2.0.so.0:libSDL2_ttf-2.0.so.0
 BOX64_NOSANDBOX=1
 BOX64_LOG=0
 BOX64_MALLOC_HACK=2
-# this is to save some memory
-BOX64_DYNAREC_BIGBLOCK=0
+BOX64_DYNAREC_BIGBLOCK=1
+BOX64_DYNAREC_AGE=4096
 # if steamwebhelper takes too much memory, enable next line to disable it
 #BOX64_EXIT=1