Re: Help with pdumper

[Daniel Colascione]

On June 29, 2024 3:28:47 PM EDT, Stefan Monnier <monnier@iro.umontreal.ca> 
wrote:
>I'm playing with replacing the linked-list of markers with an ordered
>array of markers.  I'm now stuck (as is sadly often the case in such
>endeavors) with a pdumper problem.


Have you tried running under rr and using watchpoints (which fire even when 
running reversed) to figure out where the bad value is coming from?


>
>You can find below the presumably relevant part of the patch I'm using.
>As you can see, I removed the `next` field in `struct Lisp_Marker`
>and I have a new type `struct Lisp_Markers` which is this "array with
>a gap" of markers.  This type is managed by xmalloc/xfree rather than
>by the GC (not sure if it's relevant at this point).
>
>Now my dumped Emacs crashes with:
>
>    pdumper.c:5282: Emacs fatal error: assertion failed: 
> pdumper_object_p_precise (obj)
>
>with a stack trace that ends with:
>
>    (gdb) bt
>    #0  terminate_due_to_signal
>        (sig=sig@entry=6, backtrace_limit=backtrace_limit@entry=2147483647)
>        at emacs.c:446
>    #1  0x00005555557254df in die
>        (msg=msg@entry=0x555555820ca8 "pdumper_object_p_precise (obj)", 
> file=file@entry=0x55555581f5e0 "pdumper.c", line=line@entry=5282) at 
> alloc.c:7718
>    #2  0x00005555557356cb in pdumper_cold_object_p_impl
>        (obj=obj@entry=0x7ffff1d5c738) at pdumper.c:5282
>    #3  0x0000555555729b29 in pdumper_cold_object_p (obj=0x7ffff1d5c738)
>        at /home/monnier/src/emacs/work/src/pdumper.h:183
>    #4  vector_marked_p (v=0x7ffff1d5c738) at alloc.c:4233
>    #5  0x0000555555729b8e in vectorlike_marked_p (header=<optimized out>)
>        at alloc.c:4258
>    #6  0x0000555555729bd4 in unchain_dead_markers
>        (buffer=buffer@entry=0x7ffff1d5c328) at alloc.c:7479
>    #7  0x000055555572a053 in sweep_buffers () at alloc.c:7498
>    #8  0x000055555572b06b in gc_sweep () at alloc.c:7513
>    #9  0x000055555572c8bd in garbage_collect () at alloc.c:6307
>
>More specifically this happens when `unchain_dead_markers` (i.e. during
>a GC phase) looks at one of the markers of the *scratch* buffer
>
>    (gdb) p *it->t
>    $50 = {
>      size = 9,
>      gap_beg = 0,
>      gap_end = 3,
>      markers = 0x5555559fd5e0
>    }
>    (gdb) print *it.t->markers@9
>    $51 = {0x0, 0x0, 0x0, 0x5555559bae10, 0x5555559bae60, 0x5555559bae38, 
>      0x7ffff1d5c738, 0x7ffff1d5c760, 0x7ffff1d5c788}
>    (gdb) 
>
>everything works fine for the first 3 markers (which according to their
>address of the form 0x555555?????? live in the "normal" heap) but when
>we get to the 4th one, whose address 0x7ffff1d5c738 indicates that it
>lives in the pdump area, we have:
>
>- `pdumper_object_p` returns true.
>- `pdumper_object_p_precise` returns false (hence the assertion failure).
>- `pdumper_find_object_type_impl` returns -1.
>- This seems to be due to `dump_find_relocation` returning:
>
>    (gdb) p *reloc
>    $43 = {
>      raw_offset = 1024492,
>      type = RELOC_NATIVE_SUBR
>    }
>
>The marker object itself looks perfectly healthy:
>
>    (gdb) p it.m
>    $52 = (struct Lisp_Marker *) 0x7ffff1d5c738
>    (gdb) p *it.m
>    $53 = {
>      header = {
>        size = 4611686018477735936
>      },
>      buffer = 0x7ffff1d5c328,
>      need_adjustment = false,
>      insertion_type = false,
>      cache = false,
>      charpos = 12345678,
>      bytepos = 12345678
>    }
>    (gdb)
>
>This `buffer` field points to the right buffer, and it `charpos` and
>`bytepos` fields have the expected value (I have BEG set to 12345678 in
>this build, which happens to come in handy at times, such as here).

The BEG thing is probably worth adding as a configure option, FWIW


>
>Any idea what I'm doing wrong in my code and/or why
>`dump_find_relocation` might return a "relocation type" of
>RELOC_NATIVE_SUBR for this marker?
>Or any other thing I could try to track down the origin of the problem?

Supposing we have the wrong relocation type in the dump itself, you can find 
the offset relative to dump start of the has value and just debug the dumping 
process (recording both dump and load with rr for determinism) and see what 
writes the bad value at that offset.


>
>
>        Stefan
>
>
>diff --git a/src/pdumper.c b/src/pdumper.c
>index c1b0dbd2828..00011d9149f 100644
>--- a/src/pdumper.c
>+++ b/src/pdumper.c
>@@ -2117,8 +2117,6 @@ dump_marker (struct dump_context *ctx, const struct 
>Lisp_Marker *marker)
>     {
>       dump_field_lv_rawptr (ctx, out, marker, &marker->buffer,
>                           Lisp_Vectorlike, WEIGHT_NORMAL);
>-      dump_field_lv_rawptr (ctx, out, marker, &marker->next,
>-                          Lisp_Vectorlike, WEIGHT_STRONG);
>       DUMP_FIELD_COPY (out, marker, charpos);
>       DUMP_FIELD_COPY (out, marker, bytepos);
>     }
>@@ -2126,8 +2124,7 @@ dump_marker (struct dump_context *ctx, const struct 
>Lisp_Marker *marker)
> }
> 
> static dump_off
>-dump_interval_node (struct dump_context *ctx, struct itree_node *node,
>-                    dump_off parent_offset)
>+dump_interval_node (struct dump_context *ctx, struct itree_node *node)
> {
> #if CHECK_STRUCTS && !defined (HASH_itree_node_50DE304F13)
> # error "itree_node changed. See CHECK_STRUCTS comment in config.h."
>@@ -2154,17 +2151,57 @@ dump_interval_node (struct dump_context *ctx, struct 
>itree_node *node,
>       dump_remember_fixup_ptr_raw
>       (ctx,
>        offset + dump_offsetof (struct itree_node, parent),
>-       dump_interval_node (ctx, node->parent, offset));
>+       dump_interval_node (ctx, node->parent));
>   if (node->left)
>       dump_remember_fixup_ptr_raw
>       (ctx,
>        offset + dump_offsetof (struct itree_node, left),
>-       dump_interval_node (ctx, node->left, offset));
>+       dump_interval_node (ctx, node->left));
>   if (node->right)
>       dump_remember_fixup_ptr_raw
>       (ctx,
>        offset + dump_offsetof (struct itree_node, right),
>-       dump_interval_node (ctx, node->right, offset));
>+       dump_interval_node (ctx, node->right));
>+  return offset;
>+}
>+
>+/*****************/
>+/* FIXME: YUCK!! */
>+/*****************/
>+typedef unsigned int m_index_t;
>+struct Lisp_Markers
>+{
>+  m_index_t size;
>+  m_index_t gap_beg;
>+  m_index_t gap_end;
>+  struct Lisp_Marker *markers[FLEXIBLE_ARRAY_MEMBER];
>+};
>+
>+
>+static dump_off
>+dump_markers (struct dump_context *ctx, const struct Lisp_Markers *t)
>+{
>+  ptrdiff_t bytesize = (sizeof (struct Lisp_Markers)
>+                      + t->size * sizeof (struct Lisp_Marker *));
>+  struct Lisp_Markers *out = malloc (bytesize);
>+  dump_object_start (ctx, out, bytesize);
>+  DUMP_FIELD_COPY (out, t, size);
>+  DUMP_FIELD_COPY (out, t, gap_beg);
>+  DUMP_FIELD_COPY (out, t, gap_end);
>+  for (m_index_t i = 0; i < t->size; i++)
>+    if (t->markers[i])
>+      dump_field_fixup_later (ctx, out, t, &t->markers[i]);
>+  dump_off offset = dump_object_finish (ctx, out, bytesize);
>+  free (out);
>+  for (m_index_t i = 0; i < t->size; i++)
>+    {
>+      struct Lisp_Marker *m = t->markers[i];
>+      if (m)
>+      dump_remember_fixup_ptr_raw
>+        (ctx,
>+         offset + dump_offsetof (struct Lisp_Markers, markers[i]),
>+         dump_marker (ctx, m));
>+    }
>   return offset;
> }
> 
>@@ -2181,7 +2218,7 @@ dump_overlay (struct dump_context *ctx, const struct 
>Lisp_Overlay *overlay)
>   dump_remember_fixup_ptr_raw
>     (ctx,
>      offset + dump_offsetof (struct Lisp_Overlay, interval),
>-     dump_interval_node (ctx, overlay->interval, offset));
>+     dump_interval_node (ctx, overlay->interval));
>   return offset;
> }
> 
>@@ -2865,8 +2902,7 @@ dump_buffer (struct dump_context *ctx, const struct 
>buffer *in_buffer)
>       DUMP_FIELD_COPY (out, buffer, own_text.overlay_unchanged_modified);
>       if (buffer->own_text.intervals)
>         dump_field_fixup_later (ctx, out, buffer, 
> &buffer->own_text.intervals);
>-      dump_field_lv_rawptr (ctx, out, buffer, &buffer->own_text.old_markers,
>-                            Lisp_Vectorlike, WEIGHT_NORMAL);
>+      dump_field_fixup_later (ctx, out, buffer, 
>&buffer->own_text.all_markers);
>       DUMP_FIELD_COPY (out, buffer, own_text.inhibit_shrinking);
>       DUMP_FIELD_COPY (out, buffer, own_text.redisplay);
>     }
>@@ -2925,11 +2961,18 @@ dump_buffer (struct dump_context *ctx, const struct 
>buffer *in_buffer)
>   dump_field_lv (ctx, out, buffer, &buffer->undo_list_,
>                  WEIGHT_STRONG);
>   dump_off offset = finish_dump_pvec (ctx, &out->header);
>-  if (!buffer->base_buffer && buffer->own_text.intervals)
>-    dump_remember_fixup_ptr_raw
>-      (ctx,
>-       offset + dump_offsetof (struct buffer, own_text.intervals),
>-       dump_interval_tree (ctx, buffer->own_text.intervals, 0));
>+  if (!buffer->base_buffer)
>+    {
>+      if (buffer->own_text.intervals)
>+      dump_remember_fixup_ptr_raw
>+        (ctx,
>+         offset + dump_offsetof (struct buffer, own_text.intervals),
>+         dump_interval_tree (ctx, buffer->own_text.intervals, 0));
>+      dump_remember_fixup_ptr_raw
>+      (ctx,
>+       offset + dump_offsetof (struct buffer, own_text.all_markers),
>+       dump_markers (ctx, buffer->own_text.all_markers));
>+    }
> 
>   return offset;
> }
>