in-progress: fixes the inconsistency in static pointers to raw

structures: the code containing directly a pointer would use the address
of the real structure in the current process; but if loading this
address from memory, we would instead see the old recorded address.

9 files changed, 122 insertions, 43 deletions

diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py
index ffe93b6f03..201d4cefe9 100644
--- a/rpython/memory/gctransform/boehm.py
+++ b/rpython/memory/gctransform/boehm.py
@@ -31,7 +31,8 @@ class BoehmGCTransformer(GCTransformer):
         fields = [("hash", lltype.Signed)]
         if translator and translator.config.translation.reverse_debugger:
             fields.append(("uid", lltype.SignedLongLong))
-        self.HDR = lltype.Struct("header", *fields)
+        hints = {'hints': {'gcheader': True}}
+        self.HDR = lltype.Struct("header", *fields, **hints)
         HDRPTR = lltype.Ptr(self.HDR)
 
         if self.translator:
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
index fa2533d529..e025dbc52d 100644
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -695,6 +695,8 @@ class SourceGenerator:
 
         print >> f, '#define PYPY_FILE_NAME "%s"' % os.path.basename(f.name)
         print >> f, '#include "src/g_include.h"'
+        if self.database.reverse_debugger:
+            print >> f, '#include "revdb_def.h"'
         print >> f
 
         nextralines = 11 + 1
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
index c067cd8d6e..2f9985f8e5 100644
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -459,19 +459,23 @@ class ContainerNode(Node):
         self.implementationtypename = db.gettype(
             T, varlength=self.getvarlength())
         parent, parentindex = parentlink(obj)
+        mangled = False
         if obj in exports.EXPORTS_obj2name:
             self.name = exports.EXPORTS_obj2name[obj]
             self.globalcontainer = 2    # meh
         elif parent is None:
             self.name = db.namespace.uniquename('g_' + self.basename())
             self.globalcontainer = True
+            if db.reverse_debugger and T._gckind != 'gc':
+                from rpython.translator.revdb import gencsupp
+                mangled = gencsupp.mangle_name_prebuilt_raw(db, self, T)
         else:
             self.globalcontainer = False
             parentnode = db.getcontainernode(parent)
             defnode = db.gettypedefnode(parentnode.getTYPE())
             self.name = defnode.access_expr(parentnode.name, parentindex)
         if self.typename != self.implementationtypename:
-            if db.gettypedefnode(T).extra_union_for_varlength:
+            if db.gettypedefnode(T).extra_union_for_varlength and not mangled:
                 self.name += '.b'
         self._funccodegen_owner = None
 
@@ -492,19 +496,23 @@ class ContainerNode(Node):
         return getattr(self.obj, self.eci_name, None)
 
     def get_declaration(self):
-        if self.name[-2:] == '.b':
+        name = self.name
+        if name.startswith('RPY_RDB_A('):
+            assert name.endswith(')')
+            name = name[len('RPY_RDB_A('):-1]
+        if name[-2:] == '.b':
             # xxx fish fish
             assert self.implementationtypename.startswith('struct ')
             assert self.implementationtypename.endswith(' @')
             uniontypename = 'union %su @' % self.implementationtypename[7:-2]
-            return uniontypename, self.name[:-2]
+            return uniontypename, name[:-2], True
         else:
-            return self.implementationtypename, self.name
+            return self.implementationtypename, name, False
 
     def forward_declaration(self):
         if llgroup.member_of_group(self.obj):
             return
-        type, name = self.get_declaration()
+        type, name, is_union = self.get_declaration()
         yield '%s;' % (
             forward_cdecl(type, name, self.db.standalone,
                           is_thread_local=self.is_thread_local(),
@@ -514,12 +522,12 @@ class ContainerNode(Node):
         if llgroup.member_of_group(self.obj):
             return []
         lines = list(self.initializationexpr())
-        type, name = self.get_declaration()
-        if name != self.name and len(lines) < 2:
+        type, name, is_union = self.get_declaration()
+        if is_union and len(lines) < 2:
             # a union with length 0
             lines[0] = cdecl(type, name, self.is_thread_local())
         else:
-            if name != self.name:
+            if is_union:
                 lines[0] = '{ ' + lines[0]    # extra braces around the 'a' part
                 lines[-1] += ' }'             # of the union
             lines[0] = '%s = %s' % (
@@ -597,8 +605,8 @@ class StructNode(ContainerNode):
             padding_drop = T._hints['get_padding_drop'](d)
         else:
             padding_drop = []
-        type, name = self.get_declaration()
-        if name != self.name and self.getvarlength() < 1 and len(data) < 2:
+        type, name, is_union = self.get_declaration()
+        if is_union and self.getvarlength() < 1 and len(data) < 2:
             # an empty union
             yield ''
             return
@@ -795,6 +803,11 @@ def generic_initializationexpr(db, value, access_expr, decoration):
             expr = db.get(value)
             if typeOf(value) is Void:
                 comma = ''
+            elif expr.startswith('(&RPY_RDB_A('):
+                # can't use this in static initialization code
+                assert db.reverse_debugger
+                db.late_initializations.append(('%s' % access_expr, expr))
+                expr = 'NULL /* patched later with %s */' % (expr,)
         expr += comma
         i = expr.find('\n')
         if i < 0:
diff --git a/rpython/translator/revdb/gencsupp.py b/rpython/translator/revdb/gencsupp.py
index 165b813481..3115f7acb8 100644
--- a/rpython/translator/revdb/gencsupp.py
+++ b/rpython/translator/revdb/gencsupp.py
@@ -1,4 +1,4 @@
-import py
+import py, random, sys
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import LL_OPERATIONS
 from rpython.translator.c.support import cdecl
@@ -11,6 +11,17 @@ def extra_files():
         srcdir / 'revdb.c',
     ]
 
+def mangle_name_prebuilt_raw(database, node, S):
+    if (S._gckind != 'gc' and not S._hints.get('is_excdata')
+                          and not S._hints.get('static_immutable')
+                          and not S._hints.get('ignore_revdb')
+                          and not S._hints.get('gcheader')):
+        database.all_raw_structures.append(node)
+        node.name = 'RPY_RDB_A(%s)' % (node.name,)
+        return True
+    else:
+        return False
+
 def prepare_function(funcgen):
     stack_bottom = False
     for block in funcgen.graph.iterblocks():
@@ -102,10 +113,13 @@ def prepare_database(db):
     db.get(s)
 
     db.stack_bottom_funcnames = []
+    db.all_raw_structures = []
 
 def write_revdb_def_file(db, target_path):
-    funcnames = sorted(db.stack_bottom_funcnames)
     f = target_path.open('w')
+    funcnames = sorted(db.stack_bottom_funcnames)
+    print >> f, "#define RDB_VERSION  0x%x" % random.randrange(0, sys.maxint)
+    print >> f
     for i, fn in enumerate(funcnames):
         print >> f, '#define RPY_CALLBACKLOC_%s %d' % (fn, i)
     print >> f
@@ -117,4 +131,36 @@ def write_revdb_def_file(db, target_path):
         else:
             tail = ', \\'
         print >> f, '\t(void *)%s%s' % (fn, tail)
+    print >> f
+
+    def _base(name):
+        assert name.startswith('RPY_RDB_A(')
+        if name.endswith('.b'):
+            name = name[:-2]
+        name = name[len('RPY_RDB_A('):-1]
+        return name
+
+    rawstructs = sorted(db.all_raw_structures, key=lambda node: node.name)
+    print >> f, '#define RPY_RDB_A(name)  (*rpy_rdb_struct.name)'
+    print >> f, 'struct rpy_rdb_a_s {'
+    for i, node in enumerate(rawstructs):
+        print >> f, '\t%s;' % (cdecl(node.typename, '*'+_base(node.name)),)
+    if not rawstructs:
+        print >> f, '\tchar dummy;'
+    print >> f, '};'
+    print >> f, 'RPY_EXTERN struct rpy_rdb_a_s rpy_rdb_struct;'
+    print >> f
+    print >> f, '#define RPY_RDB_STRUCT_CONTENT \\'
+    if not rawstructs:
+        print >> f, '\t0'
+    else:
+        for i, node in enumerate(rawstructs):
+            if i == len(rawstructs) - 1:
+                tail = ''
+            else:
+                tail = ', \\'
+            name = '&' + _base(node.name)
+            if node.typename != node.implementationtypename:
+                name = '(%s)%s' % (cdecl(node.typename, '*'), name)
+            print >> f, '\t%s%s' % (name, tail)
     f.close()
diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c
index 2a17ea439c..f3fc7bed97 100644
--- a/rpython/translator/revdb/src-revdb/revdb.c
+++ b/rpython/translator/revdb/src-revdb/revdb.c
@@ -22,7 +22,6 @@
 #include "src-revdb/revdb_include.h"
 
 #define RDB_SIGNATURE   "RevDB:"
-#define RDB_VERSION     0x00FF0002
 
 #define WEAKREF_AFTERWARDS_DEAD    ((char)0xf2)
 #define WEAKREF_AFTERWARDS_ALIVE   ((char)0xeb)
@@ -37,6 +36,7 @@
 typedef struct {
     Signed version;
     uint64_t reserved1, reserved2;
+    unsigned int size_rdb_struct;
     int argc;
     char **argv;
 } rdb_header_t;
@@ -164,9 +164,14 @@ static void setup_record_mode(int argc, char *argv[])
 
         memset(&h, 0, sizeof(h));
         h.version = RDB_VERSION;
+        h.size_rdb_struct = sizeof(rpy_rdb_struct);
         h.argc = argc;
         h.argv = argv;
         write_all((const char *)&h, sizeof(h));
+
+        /* write the whole content of rpy_rdb_struct */
+        write_all((const char *)&rpy_rdb_struct, sizeof(rpy_rdb_struct));
+
         fprintf(stderr, "PID %d: recording revdb log to '%s'\n",
                         (int)getpid(), filename);
     }
@@ -464,14 +469,11 @@ void *rpy_reverse_db_weakref_deref(void *weakref)
 RPY_EXTERN
 void rpy_reverse_db_callback_loc(int locnum)
 {
-    union {
-        unsigned char n[2];
-        uint16_t u;
-    } r;
+    locnum += 300;
     assert(locnum < 0xFC00);
     if (!RPY_RDB_REPLAY) {
-        RPY_REVDB_EMIT(r.n[0] = locnum >> 8; r.n[1] = locnum & 0xFF;,
-                       uint16_t _e, r.u);
+        _RPY_REVDB_EMIT_RECORD(unsigned char _e, (locnum >> 8));
+        _RPY_REVDB_EMIT_RECORD(unsigned char _e, (locnum & 0xFF));
     }
 }
 
@@ -663,6 +665,10 @@ static void setup_replay_mode(int *argc_p, char **argv_p[])
                 (long)h.version, (long)RDB_VERSION);
         exit(1);
     }
+    if (h.size_rdb_struct != sizeof(rpy_rdb_struct)) {
+        fprintf(stderr, "bad size_rdb_struct\n");
+        exit(1);
+    }
     *argc_p = h.argc;
     *argv_p = h.argv;
 
@@ -676,6 +682,9 @@ static void setup_replay_mode(int *argc_p, char **argv_p[])
         exit(1);
     }
 
+    /* read the whole content of rpy_rdb_struct */
+    read_all((char *)&rpy_rdb_struct, sizeof(rpy_rdb_struct));
+
     rpy_revdb.buf_p = rpy_rev_buffer;
     rpy_revdb.buf_limit = rpy_rev_buffer;
     rpy_revdb.buf_readend = rpy_rev_buffer;
@@ -1335,6 +1344,10 @@ static void replay_call_destructors(void)
     fq_trigger();
 }
 
+struct rpy_rdb_a_s rpy_rdb_struct = {
+    RPY_RDB_STRUCT_CONTENT   /* macro from revdb_def.h */
+};
+
 static void *callbacklocs[] = {
     RPY_CALLBACKLOCS     /* macro from revdb_def.h */
 };
@@ -1352,6 +1365,7 @@ void rpy_reverse_db_invoke_callback(unsigned char e)
         void (*pfn)(void);
         _RPY_REVDB_EMIT_REPLAY(unsigned char _e, e2)
         index = (e << 8) | e2;
+        index -= 300;
         if (index >= (sizeof(callbacklocs) / sizeof(callbacklocs[0]))) {
             fprintf(stderr, "bad callback index\n");
             exit(1);
diff --git a/rpython/translator/revdb/src-revdb/revdb_include.h b/rpython/translator/revdb/src-revdb/revdb_include.h
index 2c1300a79e..b930f238e9 100644
--- a/rpython/translator/revdb/src-revdb/revdb_include.h
+++ b/rpython/translator/revdb/src-revdb/revdb_include.h
@@ -29,15 +29,15 @@ RPY_EXTERN rpy_revdb_t rpy_revdb;
 RPY_EXTERN void rpy_reverse_db_setup(int *argc_p, char **argv_p[]);
 RPY_EXTERN void rpy_reverse_db_teardown(void);
 
-#if 0    /* enable to print locations to stderr of all the EMITs */
-#  define _RPY_REVDB_PRINT(mode)                                        \
+#if 1    /* enable to print locations to stderr of all the EMITs */
+#  define _RPY_REVDB_PRINT(mode, _e)                                    \
     fprintf(stderr,                                                     \
             "%s:%d: %0*llx\n",                                          \
             __FILE__, __LINE__, 2 * sizeof(_e),                         \
             ((unsigned long long)_e) & ((2ULL << (8*sizeof(_e)-1)) - 1))
 #endif
 
-#if 0    /* enable to print all mallocs to stderr */
+#if 1    /* enable to print all mallocs to stderr */
 RPY_EXTERN void seeing_uid(uint64_t uid);
 #  define _RPY_REVDB_PRUID()                                    \
     seeing_uid(uid);                                            \
@@ -47,7 +47,7 @@ RPY_EXTERN void seeing_uid(uint64_t uid);
 #endif
 
 #ifndef _RPY_REVDB_PRINT
-#  define _RPY_REVDB_PRINT(mode)  /* nothing */
+#  define _RPY_REVDB_PRINT(mode, _e)  /* nothing */
 #endif
 #ifndef _RPY_REVDB_PRUID
 #  define _RPY_REVDB_PRUID()      /* nothing */
@@ -57,22 +57,12 @@ RPY_EXTERN void seeing_uid(uint64_t uid);
 #define _RPY_REVDB_EMIT_RECORD(decl_e, variable)                        \
         {                                                               \
             decl_e = variable;                                          \
-            _RPY_REVDB_PRINT("write");                                  \
+            _RPY_REVDB_PRINT("write", _e);                              \
             memcpy(rpy_revdb.buf_p, &_e, sizeof(_e));                   \
             if ((rpy_revdb.buf_p += sizeof(_e)) > rpy_revdb.buf_limit)  \
                 rpy_reverse_db_flush();                                 \
         }
 
-#define _RPY_REVDB_EMIT_RECORD_EXTRA(extra, decl_e, variable)           \
-        {                                                               \
-            decl_e = variable;                                          \
-            _RPY_REVDB_PRINT("write");                                  \
-            rpy_revdb.buf_p[0] = extra;                                 \
-            memcpy(rpy_revdb.buf_p + 1, &_e, sizeof(_e));               \
-            if ((rpy_revdb.buf_p += 1 + sizeof(_e)) > rpy_revdb.buf_limit) \
-                rpy_reverse_db_flush();                                 \
-        }
-
 #define _RPY_REVDB_EMIT_REPLAY(decl_e, variable)                        \
         {                                                               \
             decl_e;                                                     \
@@ -80,7 +70,7 @@ RPY_EXTERN void seeing_uid(uint64_t uid);
             char *_end1 = _src + sizeof(_e);                            \
             memcpy(&_e, _src, sizeof(_e));                              \
             rpy_revdb.buf_p = _end1;                                    \
-            _RPY_REVDB_PRINT("read");                                   \
+            _RPY_REVDB_PRINT("read", _e);                               \
             if (_end1 >= rpy_revdb.buf_limit)                           \
                 rpy_reverse_db_fetch(__FILE__, __LINE__);               \
             variable = _e;                                              \
@@ -99,7 +89,8 @@ RPY_EXTERN void seeing_uid(uint64_t uid);
 #define RPY_REVDB_CALL(call_code, decl_e, variable)                     \
     if (!RPY_RDB_REPLAY) {                                              \
         call_code                                                       \
-        _RPY_REVDB_EMIT_RECORD_EXTRA(0xFC, decl_e, variable)            \
+        _RPY_REVDB_EMIT_RECORD(unsigned char _e, 0xFC)                  \
+        _RPY_REVDB_EMIT_RECORD(decl_e, variable)                        \
     } else {                                                            \
         unsigned char _re;                                              \
         _RPY_REVDB_EMIT_REPLAY(unsigned char _e, _re)                   \
diff --git a/rpython/translator/revdb/test/test_basic.py b/rpython/translator/revdb/test/test_basic.py
index b789839cb0..16063e2174 100644
--- a/rpython/translator/revdb/test/test_basic.py
+++ b/rpython/translator/revdb/test/test_basic.py
@@ -22,11 +22,14 @@ class RDB(object):
         #
         self.cur = 0
         x = self.read1('c'); assert x == '\x00'
-        x = self.read1('P'); assert x == 0x00FF0002
+        x = self.read1('P'); #assert x == ...random version number...
         x = self.read1('P'); assert x == 0
         x = self.read1('P'); assert x == 0
-        self.argc = self.read1('P')
+        size_rdb_struct = self.read1('I')
+        self.argc = self.read1('i')
         self.argv = self.read1('P')
+        self.rdb_struct = self.buffer[self.cur : self.cur + size_rdb_struct]
+        self.cur += size_rdb_struct
         self.current_packet_end = self.cur
         self.read_check_argv(expected_argv)
 
diff --git a/rpython/translator/revdb/test/test_callback.py b/rpython/translator/revdb/test/test_callback.py
index bbdae42589..ff3efb17e2 100644
--- a/rpython/translator/revdb/test/test_callback.py
+++ b/rpython/translator/revdb/test/test_callback.py
@@ -66,7 +66,7 @@ class TestRecording(BaseRecordingTests):
         rdb.same_thread()                       # callmesimple()
         x = rdb.next('i'); assert x == 55555
         rdb.write_call('55555\n')
-        b = rdb.next('!h'); assert 0 <= b < 10  # -> callback
+        b = rdb.next('!h'); assert 300 <= b < 310  # -> callback
         x = rdb.next('i'); assert x == 40       # arg n
         x = rdb.next('!h'); assert x == b       # -> callback
         x = rdb.next('i'); assert x == 3        # arg n
@@ -81,7 +81,7 @@ class TestRecording(BaseRecordingTests):
         self.compile(main, backendopt=False)
         out = self.run('Xx')
         rdb = self.fetch_rdb([self.exename, 'Xx'])
-        b = rdb.next('!h'); assert 0 <= b < 10  # -> callback
+        b = rdb.next('!h'); assert 300 <= b < 310  # -> callback
         x = rdb.next('i'); assert x == 40       # arg n
         rdb.write_call('40\n')
         x = rdb.next('!h'); assert x == b       # -> callback again
diff --git a/rpython/translator/revdb/test/test_bug.py b/rpython/translator/revdb/test/test_raw.py
index 1cf08dd8d0..aa19badeaf 100644
--- a/rpython/translator/revdb/test/test_bug.py
+++ b/rpython/translator/revdb/test/test_raw.py
@@ -6,11 +6,12 @@ from rpython.translator.revdb.test.test_basic import InteractiveTests
 from rpython.translator.revdb.message import *
 
 
-class TestReplayingBug(InteractiveTests):
+class TestReplayingRaw(InteractiveTests):
     expected_stop_points = 1
 
     def setup_class(cls):
         from rpython.translator.revdb.test.test_basic import compile, run
+        from rpython.translator.revdb.test.test_basic import fetch_rdb
 
         FOO = lltype.Struct('FOO')
         foo = lltype.malloc(FOO, flavor='raw', immortal=True)
@@ -19,15 +20,23 @@ class TestReplayingBug(InteractiveTests):
         bar = lltype.malloc(BAR, flavor='raw', immortal=True)
         bar.p = foo
 
+        BAZ = lltype.Struct('BAZ', ('p', lltype.Ptr(FOO)), ('q', lltype.Signed),
+                            hints={'union': True})
+        baz = lltype.malloc(BAZ, flavor='raw', immortal=True)
+        baz.p = foo
+
         def main(argv):
             assert bar.p == foo
+            assert baz.p == foo
             revdb.stop_point()
             return 9
 
         compile(cls, main, backendopt=False)
         run(cls, '')
+        rdb = fetch_rdb(cls, [cls.exename])
+        assert len(rdb.rdb_struct) >= 4
 
-    def test_replaying_bug(self):
+    def test_replaying_raw(self):
         # This tiny test seems to always have foo at the same address
         # in multiple runs.  Here we recompile with different options
         # just to change that address.