Improve line:column tracking (#660)

Emit source locations manually for more precise tracking. Don't infer
them from emitted bytecode opcodes because that leads to inaccurate
and sometimes surprising results.

Speeds up code generation (although infinitesimally) as a bonus.

Fixes: https://github.com/quickjs-ng/quickjs/issues/236

gen/function_source.c

@@ -2,9 +2,9 @@
 
 #include "quickjs-libc.h"
 
-const uint32_t qjsc_function_source_size = 338;
+const uint32_t qjsc_function_source_size = 314;
 
-const uint8_t qjsc_function_source[338] = {
+const uint8_t qjsc_function_source[314] = {
  0x11, 0x05, 0x01, 0x30, 0x74, 0x65, 0x73, 0x74,
  0x73, 0x2f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69,
  0x6f, 0x6e, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63,
@@ -23,31 +23,28 @@ const uint8_t qjsc_function_source[338] = {
  0x00, 0xc2, 0x03, 0x00, 0x0d, 0xc0, 0x03, 0x01,
  0x01, 0x0c, 0x43, 0xfa, 0x01, 0xc0, 0x03, 0x00,
  0x00, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, 0xbc,
- 0x2a, 0x28, 0xbc, 0x03, 0x03, 0x01, 0x04, 0x02,
- 0x1e, 0x0c, 0x0e, 0x1a, 0x66, 0x75, 0x6e, 0x63,
- 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x66, 0x28, 0x29,
- 0x20, 0x7b, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72,
- 0x6e, 0x20, 0x34, 0x32, 0x20, 0x7d, 0x0c, 0x03,
- 0xc2, 0x04, 0x08, 0xcc, 0x08, 0xea, 0x05, 0xbf,
- 0x00, 0xe2, 0x29, 0x04, 0xe2, 0x00, 0x00, 0x00,
- 0xe1, 0x61, 0x00, 0x00, 0xde, 0x42, 0x38, 0x00,
- 0x00, 0x00, 0x24, 0x00, 0x00, 0xc9, 0x62, 0x00,
- 0x00, 0x65, 0x00, 0x00, 0xaf, 0xea, 0x0b, 0x38,
- 0x95, 0x00, 0x00, 0x00, 0x62, 0x00, 0x00, 0xef,
- 0x2f, 0x61, 0x02, 0x00, 0x61, 0x01, 0x00, 0x38,
- 0x3b, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x04,
- 0xe0, 0x00, 0x00, 0x00, 0x9d, 0x31, 0x01, 0x00,
- 0x03, 0x00, 0xca, 0x62, 0x01, 0x00, 0x42, 0x38,
- 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0xcb, 0x62,
- 0x02, 0x00, 0x65, 0x00, 0x00, 0xaf, 0xea, 0x0b,
- 0x38, 0x95, 0x00, 0x00, 0x00, 0x62, 0x02, 0x00,
- 0xef, 0x2f, 0x68, 0x02, 0x00, 0x68, 0x01, 0x00,
- 0x06, 0x2e, 0xbc, 0x03, 0x01, 0x01, 0x22, 0x1f,
- 0x01, 0x27, 0x1c, 0x1b, 0x04, 0x0a, 0x1d, 0x12,
- 0x26, 0x35, 0x17, 0x11, 0x18, 0x20, 0x1c, 0x37,
- 0x41, 0x21, 0x1c, 0x2a, 0x18, 0x1b, 0x04, 0x26,
- 0x11, 0x3f, 0x17, 0x11, 0x18, 0x20, 0x1c, 0x53,
- 0x41, 0x00,
+ 0x2a, 0x28, 0xbc, 0x03, 0x03, 0x01, 0x00, 0x1a,
+ 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e,
+ 0x20, 0x66, 0x28, 0x29, 0x20, 0x7b, 0x20, 0x72,
+ 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x34, 0x32,
+ 0x20, 0x7d, 0x0c, 0x03, 0xc2, 0x04, 0x08, 0xcc,
+ 0x08, 0xea, 0x05, 0xbf, 0x00, 0xe2, 0x29, 0x04,
+ 0xe2, 0x00, 0x00, 0x00, 0xe1, 0x61, 0x00, 0x00,
+ 0xde, 0x42, 0x38, 0x00, 0x00, 0x00, 0x24, 0x00,
+ 0x00, 0xc9, 0x62, 0x00, 0x00, 0x65, 0x00, 0x00,
+ 0xaf, 0xea, 0x0b, 0x38, 0x95, 0x00, 0x00, 0x00,
+ 0x62, 0x00, 0x00, 0xef, 0x2f, 0x61, 0x02, 0x00,
+ 0x61, 0x01, 0x00, 0x38, 0x3b, 0x00, 0x00, 0x00,
+ 0x65, 0x00, 0x00, 0x04, 0xe0, 0x00, 0x00, 0x00,
+ 0x9d, 0x31, 0x01, 0x00, 0x03, 0x00, 0xca, 0x62,
+ 0x01, 0x00, 0x42, 0x38, 0x00, 0x00, 0x00, 0x24,
+ 0x00, 0x00, 0xcb, 0x62, 0x02, 0x00, 0x65, 0x00,
+ 0x00, 0xaf, 0xea, 0x0b, 0x38, 0x95, 0x00, 0x00,
+ 0x00, 0x62, 0x02, 0x00, 0xef, 0x2f, 0x68, 0x02,
+ 0x00, 0x68, 0x01, 0x00, 0x06, 0x2e, 0xbc, 0x03,
+ 0x01, 0x01, 0x0e, 0x00, 0x1c, 0x0a, 0x2a, 0x5d,
+ 0x18, 0x00, 0x10, 0x08, 0x25, 0x76, 0x0e, 0x5d,
+ 0x18, 0x00,
 };
 
 static JSContext *JS_NewCustomContext(JSRuntime *rt)

gen/hello.c

@@ -2,9 +2,9 @@
 
 #include "quickjs-libc.h"
 
-const uint32_t qjsc_hello_size = 105;
+const uint32_t qjsc_hello_size = 103;
 
-const uint8_t qjsc_hello[105] = {
+const uint8_t qjsc_hello[103] = {
  0x11, 0x04, 0x01, 0x22, 0x65, 0x78, 0x61, 0x6d,
  0x70, 0x6c, 0x65, 0x73, 0x2f, 0x68, 0x65, 0x6c,
  0x6c, 0x6f, 0x2e, 0x6a, 0x73, 0x01, 0x0e, 0x63,
@@ -17,8 +17,7 @@ const uint8_t qjsc_hello[105] = {
  0x02, 0x29, 0x38, 0xdf, 0x00, 0x00, 0x00, 0x42,
  0xe0, 0x00, 0x00, 0x00, 0x04, 0xe1, 0x00, 0x00,
  0x00, 0x24, 0x01, 0x00, 0x0e, 0x06, 0x2e, 0xbc,
- 0x03, 0x01, 0x01, 0x04, 0x01, 0x01, 0x17, 0x02,
- 0x00,
+ 0x03, 0x01, 0x01, 0x02, 0x48, 0x0e, 0x00,
 };
 
 static JSContext *JS_NewCustomContext(JSRuntime *rt)

gen/hello_module.c

@@ -2,9 +2,9 @@
 
 #include "quickjs-libc.h"
 
-const uint32_t qjsc_fib_module_size = 314;
+const uint32_t qjsc_fib_module_size = 282;
 
-const uint8_t qjsc_fib_module[314] = {
+const uint8_t qjsc_fib_module[282] = {
  0x11, 0x03, 0x01, 0x2c, 0x65, 0x78, 0x61, 0x6d,
  0x70, 0x6c, 0x65, 0x73, 0x2f, 0x66, 0x69, 0x62,
  0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x2e,
@@ -19,37 +19,33 @@ const uint8_t qjsc_fib_module[314] = {
  0xb4, 0xa7, 0xea, 0x03, 0xb4, 0x28, 0xd1, 0xb5,
  0xac, 0xea, 0x03, 0xb5, 0x28, 0xdd, 0xd1, 0xb5,
  0x9e, 0xef, 0xdd, 0xd1, 0xb6, 0x9e, 0xef, 0x9d,
- 0x28, 0xbc, 0x03, 0x02, 0x08, 0x20, 0x04, 0x00,
- 0x07, 0x06, 0x07, 0x06, 0x12, 0x09, 0x08, 0x07,
- 0x07, 0x10, 0x07, 0x06, 0x07, 0x06, 0x12, 0x13,
- 0x08, 0x07, 0x08, 0x16, 0x0c, 0x0c, 0x07, 0x04,
- 0x0c, 0x0a, 0x0c, 0x0c, 0x07, 0x04, 0x8d, 0x01,
- 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e,
- 0x20, 0x66, 0x69, 0x62, 0x28, 0x6e, 0x29, 0x0a,
- 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66,
- 0x20, 0x28, 0x6e, 0x20, 0x3c, 0x3d, 0x20, 0x30,
- 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+ 0x28, 0xbc, 0x03, 0x02, 0x08, 0x06, 0x00, 0x0f,
+ 0x0e, 0x10, 0x1b, 0x1a, 0x8d, 0x01, 0x66, 0x75,
+ 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x66,
+ 0x69, 0x62, 0x28, 0x6e, 0x29, 0x0a, 0x7b, 0x0a,
+ 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28,
+ 0x6e, 0x20, 0x3c, 0x3d, 0x20, 0x30, 0x29, 0x0a,
+ 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+ 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x30,
+ 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6c,
+ 0x73, 0x65, 0x20, 0x69, 0x66, 0x20, 0x28, 0x6e,
+ 0x20, 0x3d, 0x3d, 0x20, 0x31, 0x29, 0x0a, 0x20,
+ 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x72,
+ 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20, 0x31, 0x3b,
+ 0x0a, 0x20, 0x20, 0x20, 0x20, 0x65, 0x6c, 0x73,
+ 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
  0x20, 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e,
- 0x20, 0x30, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20,
- 0x65, 0x6c, 0x73, 0x65, 0x20, 0x69, 0x66, 0x20,
- 0x28, 0x6e, 0x20, 0x3d, 0x3d, 0x20, 0x31, 0x29,
- 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
- 0x20, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x20,
- 0x31, 0x3b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x65,
- 0x6c, 0x73, 0x65, 0x0a, 0x20, 0x20, 0x20, 0x20,
- 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x74, 0x75,
- 0x72, 0x6e, 0x20, 0x66, 0x69, 0x62, 0x28, 0x6e,
- 0x20, 0x2d, 0x20, 0x31, 0x29, 0x20, 0x2b, 0x20,
- 0x66, 0x69, 0x62, 0x28, 0x6e, 0x20, 0x2d, 0x20,
- 0x32, 0x29, 0x3b, 0x0a, 0x7d, 0x08, 0xea, 0x05,
- 0xbf, 0x00, 0xe1, 0x29, 0x06, 0x2e, 0xbc, 0x03,
- 0x01, 0x01, 0x06, 0x01, 0x01, 0x00, 0x07, 0x14,
- 0x02, 0x00,
+ 0x20, 0x66, 0x69, 0x62, 0x28, 0x6e, 0x20, 0x2d,
+ 0x20, 0x31, 0x29, 0x20, 0x2b, 0x20, 0x66, 0x69,
+ 0x62, 0x28, 0x6e, 0x20, 0x2d, 0x20, 0x32, 0x29,
+ 0x3b, 0x0a, 0x7d, 0x08, 0xea, 0x05, 0xbf, 0x00,
+ 0xe1, 0x29, 0x06, 0x2e, 0xbc, 0x03, 0x01, 0x01,
+ 0x00, 0x00,
 };
 
-const uint32_t qjsc_hello_module_size = 185;
+const uint32_t qjsc_hello_module_size = 183;
 
-const uint8_t qjsc_hello_module[185] = {
+const uint8_t qjsc_hello_module[183] = {
  0x11, 0x07, 0x01, 0x30, 0x65, 0x78, 0x61, 0x6d,
  0x70, 0x6c, 0x65, 0x73, 0x2f, 0x68, 0x65, 0x6c,
  0x6c, 0x6f, 0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c,
@@ -71,9 +67,8 @@ const uint8_t qjsc_hello_module[185] = {
  0x00, 0x00, 0x00, 0x42, 0xe2, 0x00, 0x00, 0x00,
  0x04, 0xe4, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00,
  0xbc, 0x0a, 0xef, 0x24, 0x02, 0x00, 0x0e, 0x06,
- 0x2e, 0xbc, 0x03, 0x01, 0x01, 0x0a, 0x01, 0x01,
- 0x00, 0x04, 0x0a, 0x02, 0x62, 0x00, 0x4d, 0x30,
- 0x00,
+ 0x2e, 0xbc, 0x03, 0x01, 0x01, 0x08, 0x18, 0x28,
+ 0x36, 0x17, 0x62, 0x00, 0x2a, 0x20, 0x00,
 };
 
 static JSContext *JS_NewCustomContext(JSRuntime *rt)

gen/test_fib.c

@@ -2,9 +2,9 @@
 
 #include "quickjs-libc.h"
 
-const uint32_t qjsc_test_fib_size = 314;
+const uint32_t qjsc_test_fib_size = 290;
 
-const uint8_t qjsc_test_fib[314] = {
+const uint8_t qjsc_test_fib[290] = {
  0x11, 0x0e, 0x01, 0x28, 0x65, 0x78, 0x61, 0x6d,
  0x70, 0x6c, 0x65, 0x73, 0x2f, 0x74, 0x65, 0x73,
  0x74, 0x5f, 0x66, 0x69, 0x62, 0x2e, 0x6a, 0x73,
@@ -39,12 +39,9 @@ const uint8_t qjsc_test_fib[314] = {
  0xe8, 0x00, 0x00, 0x00, 0x42, 0xe9, 0x00, 0x00,
  0x00, 0x04, 0xeb, 0x00, 0x00, 0x00, 0x65, 0x02,
  0x00, 0xbc, 0x0a, 0xef, 0x24, 0x02, 0x00, 0x0e,
- 0x06, 0x2e, 0xbc, 0x03, 0x01, 0x01, 0x22, 0x01,
- 0x01, 0x00, 0x04, 0x08, 0x1e, 0x2a, 0x18, 0x1b,
- 0x08, 0x0d, 0x3b, 0x1b, 0x0c, 0x07, 0x04, 0x25,
- 0x08, 0x0c, 0x04, 0x07, 0x10, 0x43, 0x2c, 0x25,
- 0x10, 0x25, 0x04, 0x27, 0x6b, 0x62, 0x00, 0x4d,
- 0x30, 0x00,
+ 0x06, 0x2e, 0xbc, 0x03, 0x01, 0x01, 0x0a, 0x00,
+ 0x45, 0x08, 0x6c, 0x36, 0x5b, 0x62, 0x00, 0x2a,
+ 0x20, 0x00,
 };
 
 static JSContext *JS_NewCustomContext(JSRuntime *rt)

quickjs.c

@@ -18711,8 +18711,6 @@ typedef struct JSFunctionDef {
 
     DynBuf byte_code;
     int last_opcode_pos; /* -1 if no last opcode */
-    int last_opcode_line_num;
-    int last_opcode_col_num;
     BOOL use_short_opcodes; /* true if short opcodes are used in byte_code */
 
     LabelSlot *label_slots;
@@ -19558,6 +19556,7 @@ static __exception int next_token(JSParseState *s)
     case '_':
     case '$':
         /* identifier */
+        s->mark = p;
         p++;
         ident_has_escape = FALSE;
     has_ident:
@@ -19873,7 +19872,7 @@ static __exception int next_token(JSParseState *s)
         p++;
         break;
     }
-    s->token.col_num = s->mark - s->eol;
+    s->token.col_num = max_int(1, s->mark - s->eol);
     s->buf_ptr = p;
 
     //    dump_token(s, &s->token);
@@ -20338,22 +20337,21 @@ static void emit_u32(JSParseState *s, uint32_t val)
     dbuf_put_u32(&s->cur_func->byte_code, val);
 }
 
+static void emit_source_loc(JSParseState *s)
+{
+    JSFunctionDef *fd = s->cur_func;
+    DynBuf *bc = &fd->byte_code;
+
+    dbuf_putc(bc, OP_source_loc);
+    dbuf_put_u32(bc, s->last_line_num);
+    dbuf_put_u32(bc, s->last_col_num);
+}
+
 static void emit_op(JSParseState *s, uint8_t val)
 {
     JSFunctionDef *fd = s->cur_func;
     DynBuf *bc = &fd->byte_code;
 
-    /* Use the line and column number of the last token used,
-       not the next token, nor the current offset in the source file.
-     */
-    if (unlikely(fd->last_opcode_line_num != s->last_line_num ||
-                 fd->last_opcode_col_num != s->last_col_num)) {
-        dbuf_putc(bc, OP_source_loc);
-        dbuf_put_u32(bc, s->last_line_num);
-        dbuf_put_u32(bc, s->last_col_num);
-        fd->last_opcode_line_num = s->last_line_num;
-        fd->last_opcode_col_num = s->last_col_num;
-    }
     fd->last_opcode_pos = bc->size;
     dbuf_putc(bc, val);
 }
@@ -23481,6 +23479,7 @@ static __exception int js_parse_postfix_expr(JSParseState *s, int parse_flags)
             emit_atom(s, JS_ATOM_new_target);
             emit_u16(s, 0);
         } else {
+            emit_source_loc(s);
             if (js_parse_postfix_expr(s, 0))
                 return -1;
             accept_lparen = TRUE;
@@ -23580,6 +23579,7 @@ static __exception int js_parse_postfix_expr(JSParseState *s, int parse_flags)
 
             if (call_type == FUNC_CALL_NORMAL) {
             parse_func_call2:
+                emit_source_loc(s);
                 switch(opcode = get_prev_opcode(fd)) {
                 case OP_get_field:
                     /* keep the object on the stack */
@@ -26035,6 +26035,7 @@ static __exception int js_parse_statement_or_decl(JSParseState *s,
 
     default:
     hasexpr:
+        emit_source_loc(s);
         if (js_parse_expr(s))
             goto fail;
         if (s->cur_func->eval_ret_idx >= 0) {
@@ -28329,7 +28330,6 @@ static JSFunctionDef *js_new_function_def(JSContext *ctx,
     js_dbuf_init(ctx, &fd->pc2line);
     //fd->pc2line_last_line_num = line_num;
     //fd->pc2line_last_pc = 0;
-    fd->last_opcode_line_num = line_num;
 
     fd->ic = init_ic(ctx);
     return fd;

tests/test_builtin.js

@@ -7,12 +7,12 @@ function test_exception_source_pos()
     var e;
 
     try {
-        throw new Error(""); // line 10, column 19
+        throw new Error(""); // line 10, column 15
     } catch(_e) {
         e = _e;
     }
 
-    assert(e.stack.includes("test_builtin.js:10:19"));
+    assert(e.stack.includes("test_builtin.js:10:15"));
 }
 
 // Keep this at the top; it tests source positions.
@@ -36,7 +36,7 @@ function test_exception_prepare_stack()
     };
 
     try {
-        throw new Error(""); // line 39, column 19
+        throw new Error(""); // line 39, column 15
     } catch(_e) {
         e = _e;
     }
@@ -48,7 +48,7 @@ function test_exception_prepare_stack()
     assert(f.getFunctionName(), 'test_exception_prepare_stack');
     assert(f.getFileName().endsWith('test_builtin.js'));
     assert(f.getLineNumber(), 39);
-    assert(f.getColumnNumber(), 19);
+    assert(f.getColumnNumber(), 15);
     assert(!f.isNative());
 }
 
@@ -64,7 +64,7 @@ function test_exception_stack_size_limit()
     };
 
     try {
-        throw new Error(""); // line 67, column 19
+        throw new Error(""); // line 67, column 15
     } catch(_e) {
         e = _e;
     }
@@ -77,7 +77,7 @@ function test_exception_stack_size_limit()
     assert(f.getFunctionName(), 'test_exception_stack_size_limit');
     assert(f.getFileName().endsWith('test_builtin.js'));
     assert(f.getLineNumber(), 67);
-    assert(f.getColumnNumber(), 19);
+    assert(f.getColumnNumber(), 15);
     assert(!f.isNative());
 }