[Help-smalltalk] Re: command-line-dependent compiler "error"

[Paolo Bonzini]

At last this error has bitten me too.  I suspect it is also the cause of 
the failures in the ANSI test suite that were happening here and there 
and disappearing mysteriously in the next version.

One almost invisible but major difference between 2.2 and 2.3 is that 
the implementation of FileStream>>#fileIn: has changed.  While in 2.2 
the VM would take care of I/O, in 2.3 the VM calls back to the 
FileStream in order to get the data; this has the advantage that we can 
implement Stream>>#fileIn and possibly, in the future, implement nice 
things like gzipped file-in in a completely transparent way.

However, there was an important bug in this.  When the VM calls 
Stream>>#nextHunk, a GC could happen.  The GC can use the buffer in 
str.c to manage finalization and, by doing so, it will ruin the contents 
of the buffer (that lex.c uses to lex strings, large integers, and so on).

The resulting strings were already placed in an obstack, so the patch I 
attach switches to using obstacks also to build the strings.  For me it 
fixes the failures (which didn't always happen and showed different 
error messages, but always involved "truncated" symbols or keywords).

Jochen, if this patch solves the problems you had on x86-64, just tell 
me as I can release 2.3.2 pretty soon now.

Paolo

2006-12-21  Paolo Bonzini  <address@hidden>

        * libgst/lex.c: Build strings on obstack instead of the
        str.c buffer.  Otherwise the buffer could be cleared out
        by oop.c when it uses it to store the live ephemeron OOPs.

--- orig/libgst/lex.c
+++ mod/libgst/lex.c
@@ -506,15 +504,15 @@ scan_symbol (int c,
       return '#';
     }
 
-  _gst_reset_buffer ();
-  _gst_add_str_buf_char (ic);
+  obstack_1grow (_gst_compilation_obstack, ic);
 
   while (((ic = _gst_next_char ()) != EOF)
          && (CHAR_TAB (ic)->char_class & SYMBOL_CHAR))
-    _gst_add_str_buf_char (ic);
+    obstack_1grow (_gst_compilation_obstack, ic);
 
   _gst_unread_char (ic);
-  lvalp->sval = _gst_obstack_cur_str_buf (_gst_compilation_obstack);
+  obstack_1grow (_gst_compilation_obstack, '\0');
+  lvalp->sval = obstack_finish (_gst_compilation_obstack);
   return SYMBOL_LITERAL;
 }
 
@@ -574,8 +572,6 @@ string_literal (int c,
 {
   int ic;
 
-  _gst_reset_buffer ();
-
   for (;;)
     {
       ic = _gst_next_char ();
@@ -595,9 +591,10 @@ string_literal (int c,
              break;
            }
        }
-      _gst_add_str_buf_char (ic);
+      obstack_1grow (_gst_compilation_obstack, ic);
     }
-  lvalp->sval = _gst_obstack_cur_str_buf (_gst_compilation_obstack);
+  obstack_1grow (_gst_compilation_obstack, '\0');
+  lvalp->sval = obstack_finish (_gst_compilation_obstack);
   return (STRING_LITERAL);
 }
 
@@ -607,14 +604,13 @@ scan_ident (int c,
 {
   int ic, identType;
 
-  _gst_reset_buffer ();
-  _gst_add_str_buf_char (c);
+  obstack_1grow (_gst_compilation_obstack, c);
 
   identType = IDENTIFIER;
 
   while (((ic = _gst_next_char ()) != EOF)
         && (CHAR_TAB (ic)->char_class & ID_CHAR))
-    _gst_add_str_buf_char (ic);
+    obstack_1grow (_gst_compilation_obstack, ic);
 
   /* Read a dot as '::' if followed by a letter.  */
   if (ic == '.')
@@ -638,7 +634,7 @@ scan_ident (int c,
        _gst_unread_char (':');
       else
        {
-          _gst_add_str_buf_char (':');
+          obstack_1grow (_gst_compilation_obstack, ':');
           identType = KEYWORD;
        }
     }
@@ -646,7 +642,8 @@ scan_ident (int c,
   else
     _gst_unread_char (ic);
 
-  lvalp->sval = _gst_obstack_cur_str_buf (_gst_compilation_obstack);
+  obstack_1grow (_gst_compilation_obstack, '\0');
+  lvalp->sval = obstack_finish (_gst_compilation_obstack);
   return (identType);
 }
 
@@ -850,6 +847,8 @@ scan_number (int c,
 
   if (float_type)
     {
+      obstack_blank_fast (_gst_compilation_obstack,
+                         -obstack_object_size (_gst_compilation_obstack));
       lvalp->fval = num;
       return (float_type);
     }
@@ -858,6 +857,7 @@ scan_number (int c,
       lvalp->boval = scan_large_integer (isNegative, base);
       return (LARGE_INTEGER_LITERAL);
     }
+  else
     {
       lvalp->ival = (intptr_t) num;
       return (INTEGER_LITERAL);
@@ -873,9 +873,6 @@ scan_digits (int c,
   long double result;
   mst_Boolean oneDigit = false;
 
-  if (largeInteger)
-    _gst_reset_buffer ();
-
   while (c == '_')
     c = _gst_next_char ();
 
@@ -886,7 +883,7 @@ scan_digits (int c,
       result += digit_to_int (c, base);
       if (largeInteger)
        {
-         _gst_add_str_buf_char (digit_to_int (c, base));
+         obstack_1grow (_gst_compilation_obstack, digit_to_int (c, base));
          if (result > -MIN_ST_INT
              || (!negative && result > MAX_ST_INT))
            *largeInteger = true;
@@ -930,7 +927,7 @@ scan_fraction (int c,
 
       if (largeInteger)
         {
-          _gst_add_str_buf_char (digit_to_int (c, base));
+          obstack_1grow (_gst_compilation_obstack, digit_to_int (c, base));
           if (num > MAX_ST_INT)
             *largeInteger = true;
         }
@@ -1006,13 +1003,14 @@ scan_large_integer (mst_Boolean negative
   gst_uchar *digits, *result;
   byte_object bo;
 
-  size = _gst_buffer_size ();
+  /* Copy the contents of the currently grown obstack on the stack.  */
+  size = obstack_object_size (_gst_compilation_obstack);
   digits = (gst_uchar *) alloca (size);
-  _gst_copy_buffer (digits);
+  memcpy (digits, obstack_base (_gst_compilation_obstack), size);
 
-  bo =
-    (byte_object) obstack_alloc (_gst_compilation_obstack,
-                                sizeof (struct byte_object) + size);
+  /* And reuse the area on the obstack for a struct byte_object.  */
+  obstack_blank (_gst_compilation_obstack, sizeof (struct byte_object));
+  bo = (byte_object) obstack_finish (_gst_compilation_obstack);
 
   bo->class =
     negative ? _gst_large_negative_integer_class :