[6162] parsetexi update

[Gavin D. Smith]

Revision: 6162
          http://svn.sv.gnu.org/viewvc/?view=rev&root=texinfo&revision=6162
Author:   gavin
Date:     2015-02-27 12:34:52 +0000 (Fri, 27 Feb 2015)
Log Message:
-----------
parsetexi update

Modified Paths:
--------------
    trunk/parsetexi/ChangeLog
    trunk/parsetexi/README
    trunk/parsetexi/TODO
    trunk/parsetexi/api.c
    trunk/parsetexi/close.c
    trunk/parsetexi/context_stack.c
    trunk/parsetexi/debug.c
    trunk/parsetexi/dump_perl.c
    trunk/parsetexi/end_line.c
    trunk/parsetexi/errors.c
    trunk/parsetexi/errors.h
    trunk/parsetexi/handle_commands.c
    trunk/parsetexi/handle_commands.h
    trunk/parsetexi/macro.c
    trunk/parsetexi/makeinfo
    trunk/parsetexi/parser.c
    trunk/parsetexi/tree_types.h

Modified: trunk/parsetexi/ChangeLog
===================================================================
--- trunk/parsetexi/ChangeLog   2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/ChangeLog   2015-02-27 12:34:52 UTC (rev 6162)
@@ -1,3 +1,11 @@
+2015-02-27  Gavin Smith  <address@hidden>
+
+       * tree_types.h (enum extra_type): Add extra_deleted.
+       * parser.c (abort_empty_line): Remove "spaces_before_argument" 
+       extra value.
+       (process_remaining_on_line): Add check for empty input ("EMPTY 
+       TEXT").
+
 2015-02-22  Gavin Smith  <address@hidden>
 
        * macro.c (store_value, fetch_value): New functions.

Modified: trunk/parsetexi/README
===================================================================
--- trunk/parsetexi/README      2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/README      2015-02-27 12:34:52 UTC (rev 6162)
@@ -5,3 +5,17 @@
 makeinfo in this directory wraps texi2any-C.pl, which is tp/texi2any.pl 
 changed to use the module in the Parsetexi subdirectory instead of 
 Texinfo::Parser.
+
+Notes -
+How to debug with valgrind -
+export PERL5LIB to the value in the makeinfo script
+
+with alias VAL='valgrind --log-file=val.log', can do e.g.
+
+VAL perl ./texi2any-C.pl texinfo.texi
+
+also
+
+valgrind --vgdb-error=0 perl texi2any-C.pl texinfo.texi
+
+

Modified: trunk/parsetexi/TODO
===================================================================
--- trunk/parsetexi/TODO        2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/TODO        2015-02-27 12:34:52 UTC (rev 6162)
@@ -1,4 +1,4 @@
-* @set/@value
+* conditionals on @set/@value, output format
 * Counters for e.g. numbers of arguments read so far for a brace 
 command, or the row in a table.
 * check valid nestings
@@ -11,6 +11,7 @@
 test suites.
 * find where "strlen" and "text_append" are used and try to remove them, for 
 efficiency.  Also newSVpv (..., 0);
+* Add "TODO" anywhere in the code with explanations of what is not done yet.
 
 Integration with rest of Perl code:
 

Modified: trunk/parsetexi/api.c
===================================================================
--- trunk/parsetexi/api.c       2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/api.c       2015-02-27 12:34:52 UTC (rev 6162)
@@ -225,6 +225,9 @@
           char *key = e->extra[i].key;
           ELEMENT *f = e->extra[i].value;
 
+          if (e->extra[i].type == extra_deleted)
+            continue;
+
           switch (e->extra[i].type)
             {
             case extra_element:

Modified: trunk/parsetexi/close.c
===================================================================
--- trunk/parsetexi/close.c     2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/close.c     2015-02-27 12:34:52 UTC (rev 6162)
@@ -16,6 +16,7 @@
 #include <stdlib.h>
 
 #include "parser.h"
+#include "errors.h"
 
 /* In parser.c. */
 ELEMENT *end_paragraph (ELEMENT *current);
@@ -57,8 +58,6 @@
     } // 1635
 }
 
-/* 1638 - 1794 */
-
 /* 1642 */
 static ELEMENT *
 close_current (ELEMENT *current)
@@ -71,22 +70,29 @@
     }
   else if (current->type != ET_NONE)
     {
+      debug ("CLOSING type %s", element_type_names[current->type]);
       enum context c;
       switch (current->type)
         {
         case ET_bracketed:
-          /* error */
+          command_error ("misplaced {");
+          current = current->parent;
           break;
         case ET_menu_comment:
         case ET_menu_entry_description:
           c = pop_context ();
           if (c != ct_preformatted)
+            abort ();
+
+          /* 1700 Remove empty menu_comment */
+          if (current->contents.number == 0)
             {
-              /* error */
+              current = current->parent;
+              destroy_element (pop_element_from_contents (current));
             }
+          else
+            current = current->parent;
 
-          /* close empty menu_comment */
-
           break;
         case ET_misc_line_arg:
         case ET_block_line_arg:
@@ -96,9 +102,12 @@
               /* error */
               abort ();
             }
+          current = current->parent;
           break;
+        default:
+          current = current->parent;
+          break;
         }
-      current = current->parent;
     }
   else
     {
@@ -158,6 +167,10 @@
       *closed_element = current;
       current = current->parent; /* 1788 */
     }
+  else if (closed_command)
+    {
+      line_errorf ("unmatched @end %s", command_data(closed_command).cmdname);
+    }
 
   return current;
 }

Modified: trunk/parsetexi/context_stack.c
===================================================================
--- trunk/parsetexi/context_stack.c     2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/context_stack.c     2015-02-27 12:34:52 UTC (rev 6162)
@@ -30,6 +30,14 @@
       stack = realloc (stack, (space += 5) * sizeof (enum context));
     }
 
+  /*
+  debug (">>>>>>>>>>>>>>>>>.PUSHING STACK AT %d  -- %s", top,
+         c == ct_preformatted ? "preformatted"
+         : c == ct_line ? "line"
+         : c == ct_def ? "def"
+         : c == ct_menu ? "menu"
+         : "");
+  */
   stack[top++] = c;
 }
 
@@ -39,6 +47,7 @@
   if (top == 0)
     abort ();
 
+  /*  debug (">>>>>>>>>>>>>POPPING STACK AT %d", top - 1);  */
   return stack[--top];
 }
 

Modified: trunk/parsetexi/debug.c
===================================================================
--- trunk/parsetexi/debug.c     2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/debug.c     2015-02-27 12:34:52 UTC (rev 6162)
@@ -16,7 +16,7 @@
 #include <stdio.h>
 
 /* Whether to dump debugging output on stderr. */
-int debug_output = 0;
+int debug_output = 1;
 
 void
 debug (char *s, ...)

Modified: trunk/parsetexi/dump_perl.c
===================================================================
--- trunk/parsetexi/dump_perl.c 2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/dump_perl.c 2015-02-27 12:34:52 UTC (rev 6162)
@@ -233,6 +233,8 @@
     {
       for (i = 0; i < e->extra_number; i++)
         {
+          if (e->extra[i].type == extra_deleted)
+            continue;
           if (e->extra[i].type == extra_index_entry)
             {
               /* A "index_entry" extra key on a command defining an index

Modified: trunk/parsetexi/end_line.c
===================================================================
--- trunk/parsetexi/end_line.c  2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/end_line.c  2015-02-27 12:34:52 UTC (rev 6162)
@@ -1,3 +1,4 @@
+/* end_line.c -- what to do at the end of a whole line of input
 /* Copyright 2010, 2011, 2012, 2013, 2014, 2015
    Free Software Foundation, Inc.
 
@@ -795,6 +796,7 @@
   return current;
 }
 
+// 3100
 /* Actions to be taken at the end of an argument to a line command
    not starting a block. */
 static ELEMENT *
@@ -841,16 +843,18 @@
          Common/Text.pm on the first element of current->args. */
       /* however, this makes it impossible to decouple the parser and 
          output stages...  Any use of Texinfo::Convert is problematic. */
+      /* Fortunately Text.pm is not too complicated (unlike Plaintext.pm). */
 
+      // TODO: Convert properly.
       if (current->args.number > 0)
         text = text_convert (current->args.list[0]);
       else
         text = "foo";
 
-      if (!strcmp (text, ""))
+      if (!text || !strcmp (text, ""))
         {
-          /* 3123 warning - missing argument */
-          abort ();
+          // 3123
+          line_warnf ("@%s missing argument", command_data(cmd_id).cmdname);
         }
       else
         {
@@ -861,10 +865,6 @@
               /* Set end_command - used below. */
               end_command = read_command_name (&line);
 
-              /* Check argument meets format of a Texinfo command
-                 (alphanumberic character followed by alphanumeric 
-                 characters or hyphens. */
-
               /* Check if argument is a block Texinfo command. */
               end_id = lookup_command (end_command);
               if (end_id == 0 || !(command_data(end_id).flags & CF_block))

Modified: trunk/parsetexi/errors.c
===================================================================
--- trunk/parsetexi/errors.c    2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/errors.c    2015-02-27 12:34:52 UTC (rev 6162)
@@ -61,7 +61,30 @@
   line_error (message);
 }
 
+void
+line_warn (char *message)
+{
+  if (error_number == error_space)
+    {
+      error_list = realloc (error_list,
+                            (error_space += 10) * sizeof (ERROR_MESSAGE));
+    }
+  error_list[error_number].message = message;
+  error_list[error_number].type = warning;
+  error_list[error_number++].line_nr = line_nr; /* Field-by-field copy. */
+}
 
+void
+line_warnf (char *format, ...)
+{
+  va_list v;
+  char *message;
+
+  va_start (v, format);
+  vasprintf (&message, format, v);
+  line_warn (message);
+}
+
 char *
 dump_errors (void)
 {

Modified: trunk/parsetexi/errors.h
===================================================================
--- trunk/parsetexi/errors.h    2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/errors.h    2015-02-27 12:34:52 UTC (rev 6162)
@@ -1,3 +1,12 @@
 void line_error (char *message);
 void line_errorf (char *format, ...);
+void line_warn (char *message);
+void line_warnf (char *format, ...);
+
+/* TODO: Proper implementations */
+#define command_error line_error
+#define command_errorf line_errorf
+#define command_warn line_warn
+#define command_warnf line_warnf
+
 char *dump_errors (void);

Modified: trunk/parsetexi/handle_commands.c
===================================================================
--- trunk/parsetexi/handle_commands.c   2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/handle_commands.c   2015-02-27 12:34:52 UTC (rev 6162)
@@ -107,7 +107,7 @@
       else /* arg_spec == MISC_special */
         {
           args = parse_special_misc_command (line, cmd_id); //4362
-          add_extra_string (misc, "arg_line", line);
+          add_extra_string (misc, "arg_line", strdup (line));
         }
 
       if ((cmd_id == CM_set || cmd_id == CM_clear)
@@ -317,7 +317,9 @@
               else
                 {
                   // error - deffnx not after deffn
-                  abort ();
+                  line_errorf ("must be after @%s to use @%s",
+                               command_data(base_command).cmdname,
+                               command_data(cmd_id).cmdname);
                 }
             }
         } /* 4571 */
@@ -364,7 +366,7 @@
 /* line 4632 */
 ELEMENT *
 handle_block_command (ELEMENT *current, char **line_inout,
-                      enum command_id cmd_id)
+                      enum command_id cmd_id, int *get_new_line)
 {
   char *line = *line_inout;
   unsigned long flags = command_data(cmd_id).flags;
@@ -379,10 +381,11 @@
       current = macro;
 
       /* 4640 FIXME */
-      /* The line should be advanced to the end, so a new line should be read 
-         immediately after this. */
+      /* A new line should be read immediately after this.  */
       /* Alternative is to use longjmp to go where "last;" does in the Perl 
          version. */
+      line = strchr (line, '\0');
+      get_new_line = 1;
       goto funexit;
     }
   else if (command_data(cmd_id).data == BLOCK_conditional)
@@ -405,8 +408,25 @@
     }
   else /* line 4710 */
     {
-      if (flags & CF_menu)
+      // 4715
+      if (flags & CF_menu
+          && (current->type == ET_menu_comment
+              || current->type == ET_menu_entry_description))
         {
+#if 0
+          /* This is for @detailmenu within @menu */
+          ELEMENT *menu = current->parent;
+          //abort ();
+          if (current->contents.number == 0)
+            {
+              destroy_element (pop_element_from_contents (menu));
+              if (pop_context () != ct_preformatted)
+                abort ();
+            }
+          if (menu->type == ET_menu_entry)
+            menu = menu->parent;
+          current = last_contents_child (menu);
+#endif
         }
 
       // 4740

Modified: trunk/parsetexi/handle_commands.h
===================================================================
--- trunk/parsetexi/handle_commands.h   2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/handle_commands.h   2015-02-27 12:34:52 UTC (rev 6162)
@@ -2,6 +2,6 @@
 ELEMENT *handle_misc_command (ELEMENT *current, char **line_inout,
                      enum command_id cmd_id);
 ELEMENT *handle_block_command (ELEMENT *current, char **line_inout,
-                      enum command_id cmd_id);
+                      enum command_id cmd_id, int *new_line);
 ELEMENT *handle_brace_command (ELEMENT *current, char **line_inout,
                                enum command_id cmd_id);

Modified: trunk/parsetexi/macro.c
===================================================================
--- trunk/parsetexi/macro.c     2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/macro.c     2015-02-27 12:34:52 UTC (rev 6162)
@@ -180,10 +180,11 @@
   return -1;
 }
 
-/* LINE points to after opening brace in a macro invocation. */
+/* LINE points to after the opening brace in a macro invocation.  CMD is the
+   command identifier of the macro command. */
 // 1984
 char **
-expand_macro_arguments (ELEMENT *macro, char **line_inout)
+expand_macro_arguments (ELEMENT *macro, char **line_inout, enum command_id cmd)
 {
   char *line = *line_inout;
   char *pline = line;
@@ -208,11 +209,17 @@
       if (!*sep)
         {
           debug ("MACRO ARG end of line");
-          // FIXME: How to free line?
-          // We could keep a reference to it in a static variable in
-          // new_line.
+          text_append (&arg, pline);
           line = new_line ();
+          if (!line)
+            {
+              line_error ("@%s missing close brace", 
+                          command_data(cmd).cmdname);
+              line = "\n";
+              goto funexit;
+            }
           pline = line;
+          continue;
         }
 
       text_append_n (&arg, pline, sep - pline);
@@ -250,23 +257,20 @@
           // 2021 check for too many args
 
           /* Add the last argument read to the list. */
-          if (arg.end > 0)
+          if (arg_number == arg_space)
             {
-              if (arg_number == arg_space)
-                {
-                  arg_list = realloc (arg_list,
-                                      (1+(arg_space += 5)) * sizeof (char *));
-                  /* Include space for terminating null element. */
-                  if (!arg_list)
-                    abort ();
-                }
-              arg_list[arg_number++] = arg.text;
-              text_init (&arg);
+              arg_list = realloc (arg_list,
+                                  (1+(arg_space += 5)) * sizeof (char *));
+              /* Include space for terminating null element. */
+              if (!arg_list)
+                abort ();
             }
+          if (arg.space > 0)
+            arg_list[arg_number++] = arg.text;
           else
-          // then what? e.g. is "@m {     }" one empty argument or none?
-            if (arg_number != 0)
-              abort ();
+            arg_list[arg_number++] = strdup ("");
+          text_init (&arg);
+          // TODO: is "@m {     }" one empty argument or none?
 
           debug ("MACRO NEW ARG");
           pline = sep + 1;
@@ -280,6 +284,7 @@
   debug ("END MACRO ARGS EXPANSION");
   line = pline;
 
+funexit:
   *line_inout = line;
   arg_list[arg_number] = 0;
   return arg_list;
@@ -371,11 +376,11 @@
 ELEMENT *
 handle_macro (ELEMENT *current, char **line_inout, enum command_id cmd)
 {
-  char *line;
+  char *line, *p;
   MACRO *macro_record;
   ELEMENT *macro;
   TEXT expanded;
-  char **arguments;
+  char **arguments = 0;
 
   line = *line_inout;
   text_init (&expanded);
@@ -387,28 +392,38 @@
 
   // 3907 Get number of args.
 
-  line += strspn (line, whitespace_chars);
-  if (*line == '{')
+  p = line + strspn (line, whitespace_chars);
+  if (*p == '{')
     {
+      line = p;
       line++;
       /* In the Perl version formfeed is excluded for some reason. */
       line += strspn (line, whitespace_chars);
-      arguments = expand_macro_arguments (macro, &line);
+      arguments = expand_macro_arguments (macro, &line, cmd);
     }
+  else
+    {
+      /* TODO: Warning depending on the number of arguments this macro
+         is supposed to take. */
 
+      /* TODO: Otherwise, if it takes a single line of input,
+         and we don't have a full line of input already, call new_line. */
+    }
+
   expand_macro_body (macro, arguments, &expanded);
   debug ("MACROBODY: %s||||||", expanded.text);
 
   /* Free arguments. */
-  {
-    char **s = arguments;
-    while (*s)
-      {
-        free (*s);
-        s++;
-      }
-    free (arguments);
-  }
+  if (arguments)
+    {
+      char **s = arguments;
+      while (*s)
+        {
+          free (*s);
+          s++;
+        }
+      free (arguments);
+    }
 
   // 3958 Pop macro stack
 

Modified: trunk/parsetexi/makeinfo
===================================================================
--- trunk/parsetexi/makeinfo    2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/makeinfo    2015-02-27 12:34:52 UTC (rev 6162)
@@ -1,7 +1,9 @@
 #! /usr/bin/env bash
 
 #DEBUG='perl -d'
+#DEBUG='perl -d:NYTProf'
 #OPTIONS='-c DEBUG=1 --no-validate'
+#VALGRIND='valgrind --vgdb-error=0'
 COMMAND=${COMMAND:-./texi2any-C.pl}
 
-PERL5LIB=../tp:../tp/maintain/lib/libintl-perl/lib:../tp/maintain/lib/Text-Unidecode/lib/:../tp/maintain/lib/Unicode-EastAsianWidth/lib:./Parsetexi/lib:./Parsetexi/blib/lib:./Parsetexi/blib/arch/auto/Parsetexi:../tp/Texinfo/Convert/XSParagraph/lib:../tp/Texinfo/Convert/XSParagraph/blib/arch/auto/XSParagraph
 ${DEBUG} ${COMMAND} ${OPTIONS} "$@"
+PERL5LIB=../tp:../tp/maintain/lib/libintl-perl/lib:../tp/maintain/lib/Text-Unidecode/lib/:../tp/maintain/lib/Unicode-EastAsianWidth/lib:./Parsetexi/lib:./Parsetexi/blib/lib:./Parsetexi/blib/arch/auto/Parsetexi:../tp/Texinfo/Convert/XSParagraph/lib:../tp/Texinfo/Convert/XSParagraph/blib/arch/auto/XSParagraph
 ${VALGRIND} ${DEBUG} ${COMMAND} ${OPTIONS} "$@"

Modified: trunk/parsetexi/parser.c
===================================================================
--- trunk/parsetexi/parser.c    2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/parser.c    2015-02-27 12:34:52 UTC (rev 6162)
@@ -374,9 +374,21 @@
       // FIXME: How and when is this condition exactly met?
       if (last_child->text.end == 0) //2121
         {
-          // 'extra' stuff
+          KEY_PAIR *k = 0;
+          
+          /* FIXME: does extra key get removed from current or 
+             current->parent?  */
+          if (current->parent)
+            k = lookup_extra_key (current->parent, "spaces_before_argument");
+          if (k) // && k->value == last_contents_child (current))
+            {
+              k->key = "";
+              k->value = 0;
+              k->type = extra_deleted;
+            }
 
           destroy_element (pop_element_from_contents (current));
+          /* TODO: Maybe we could avoid adding it in the first place? */
         }
       else if (last_child->type == ET_empty_line) //2132
         {
@@ -595,14 +607,6 @@
   char *command = 0;
   enum command_id cmd_id = CM_NONE;
 
-  /* We could be after a @macro line.  See comment in handle_block_command 
-     4640. */
-  if (!*line)
-    {
-      retval = 0;
-      goto funexit;
-    }
-
   /* If in raw block, or ignored conditional block. */
   // 3727
   if (command_flags(current) & CF_block
@@ -780,6 +784,30 @@
         }
     }
 
+  /* There are cases when we need more input, but we don't want to
+     get it in the top-level loop in parse_texi - this is mostly
+     (always?) when we don't want to start a new, empty line, and
+     need to get more from the current, incomplete line of input. */
+  while (*line == '\0')
+    {
+      static char *allocated_text;
+      debug ("EMPTY TEXT");
+
+      /* Each place we supply Texinfo input we store the supplied
+         input in a static variable like allocated_text, to prevent
+         memory leaks.  */
+      free (allocated_text);
+      line = allocated_text = next_text ();
+
+      if (!line)
+        {
+          /* TODO: Can this only happen at end of file? */
+          current = end_line (current);
+          retval = 0;
+        }
+      goto funexit;
+    }
+
   /* Handle user-defined macros before anything else because their expansion 
      may lead to changes in the line. */
   if (cmd_id && (command_data(cmd_id).flags & CF_MACRO)) // 3894
@@ -945,7 +973,15 @@
       /* line 4632 */
       else if (command_data(cmd_id).flags & CF_block)
         {
-          current = handle_block_command (current, &line, cmd_id);
+          int new_line = 0;
+          current = handle_block_command (current, &line, cmd_id, &new_line);
+          if (new_line)
+            {
+              /* For @macro, to get a new line.  This is done instead of
+                 doing the EMPTY TEXT (3879) code on the next time round. */
+              retval = 0;
+              goto funexit;
+            }
         }
 
       else if (command_data(cmd_id).flags & CF_brace) /* line 4835 */

Modified: trunk/parsetexi/tree_types.h
===================================================================
--- trunk/parsetexi/tree_types.h        2015-02-27 08:49:19 UTC (rev 6161)
+++ trunk/parsetexi/tree_types.h        2015-02-27 12:34:52 UTC (rev 6162)
@@ -34,7 +34,8 @@
     extra_misc_args,
     extra_node_spec,
     extra_node_spec_array,
-    extra_string
+    extra_string,
+    extra_deleted
 };
 
 typedef struct KEY_PAIR {