branch master updated: * tp/Texinfo/XS/parsetexi/end_line.c (end_line_misc_line), tp/Texinfo/ParserNonXS.pm (_end_line_misc_line, _handle_line_command), tp/maintain/regenerate_commands_perl_info.pl: in end_line_misc_line, use item_LINE to get the command data information associated to @item. Keep item_LINE in the perl commands information as in the XS parser, modify _handle_line_command and _end_line_misc_line accordingly, also to be more in line with the XS parser code. For some selected flags that are of inter [...]

[Patrice Dumas]

This is an automated email from the git hooks/post-receive script.

pertusus pushed a commit to branch master
in repository texinfo.

The following commit(s) were added to refs/heads/master by this push:
     new 583dff9753 * tp/Texinfo/XS/parsetexi/end_line.c (end_line_misc_line), 
tp/Texinfo/ParserNonXS.pm (_end_line_misc_line, _handle_line_command), 
tp/maintain/regenerate_commands_perl_info.pl: in end_line_misc_line, use 
item_LINE to get the command data information associated to @item. Keep 
item_LINE in the perl commands information as in the XS parser, modify 
_handle_line_command and _end_line_misc_line accordingly, also to be more in 
line with the XS parser code. For some selected fl [...]
583dff9753 is described below

commit 583dff9753a7484987871648b5b4af98a83fe940
Author: Patrice Dumas <pertusus@free.fr>
AuthorDate: Sat Mar 4 09:34:24 2023 +0100

    * tp/Texinfo/XS/parsetexi/end_line.c (end_line_misc_line),
    tp/Texinfo/ParserNonXS.pm (_end_line_misc_line, _handle_line_command),
    tp/maintain/regenerate_commands_perl_info.pl: in end_line_misc_line,
    use item_LINE to get the command data information associated to
    @item.
    Keep item_LINE in the perl commands information as in the XS parser,
    modify _handle_line_command and _end_line_misc_line accordingly, also
    to be more in line with the XS parser code.
    For some selected flags that are of interest for converters but not
    for the parsers, duplicate the multi category, ie item_LINE
    information for the main command, here item.  In the converters,
    item_LINE is not present and the tree allows to knwo in advance which
    type of item command is processed.
---
 ChangeLog                                    | 16 ++++++++
 tp/TODO                                      |  5 ---
 tp/Texinfo/ParserNonXS.pm                    | 61 ++++++++++++++++++----------
 tp/Texinfo/XS/parsetexi/end_line.c           | 21 ++++++----
 tp/maintain/regenerate_commands_perl_info.pl | 32 +++++++++++++--
 5 files changed, 98 insertions(+), 37 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index f064a1cc63..f22118f29b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,19 @@
+2023-03-03  Patrice Dumas  <pertusus@free.fr>
+
+       * tp/Texinfo/XS/parsetexi/end_line.c (end_line_misc_line),
+       tp/Texinfo/ParserNonXS.pm (_end_line_misc_line, _handle_line_command),
+       tp/maintain/regenerate_commands_perl_info.pl: in end_line_misc_line,
+       use item_LINE to get the command data information associated to
+       @item.
+       Keep item_LINE in the perl commands information as in the XS parser,
+       modify _handle_line_command and _end_line_misc_line accordingly, also
+       to be more in line with the XS parser code.
+       For some selected flags that are of interest for converters but not
+       for the parsers, duplicate the multi category, ie item_LINE
+       information for the main command, here item.  In the converters,
+       item_LINE is not present and the tree allows to knwo in advance which
+       type of item command is processed.
+
 2023-03-03  Patrice Dumas  <pertusus@free.fr>
 
        * tp/Texinfo/ParserNonXS.pm (_process_remaining_on_line), 
diff --git a/tp/TODO b/tp/TODO
index 44e1eabb43..c1889b98f0 100644
--- a/tp/TODO
+++ b/tp/TODO
@@ -13,11 +13,6 @@ Before next release
 Bugs
 ====
 
-duplicate item entries in %close_paragraph_commands and other.
-Also inconsistent entries for item and item as item_LINE, would be
-better to modify the perl parser to use item_LINE and item like
-the XS parser.
-
 source marks in parse_node_manual
 
 Add source marks for alias. Not easy, as command elements are added in many
diff --git a/tp/Texinfo/ParserNonXS.pm b/tp/Texinfo/ParserNonXS.pm
index 2776aa8a35..3f18e5b276 100644
--- a/tp/Texinfo/ParserNonXS.pm
+++ b/tp/Texinfo/ParserNonXS.pm
@@ -3296,9 +3296,14 @@ sub _end_line_misc_line($$$)
   my $source_info = shift;
 
   my $command = $current->{'parent'}->{'cmdname'};
+  my $data_cmdname = $command;
+
+  # we are in a command line context, so the @item command information is
+  # associated to CM_item_LINE
+  $data_cmdname = 'item_LINE' if ($command eq 'item');
 
   if ($self->{'basic_inline_commands'}
-      and $self->{'basic_inline_commands'}->{$command}) {
+      and $self->{'basic_inline_commands'}->{$data_cmdname}) {
     pop @{$self->{'nesting_context'}->{'basic_inline_stack_on_line'}};
   }
   _isolate_last_space($self, $current);
@@ -3316,16 +3321,19 @@ sub _end_line_misc_line($$$)
   my $end_command;
   my $included_file;
   my $include_source_mark;
-  print STDERR "MISC END \@$command: $self->{'line_commands'}->{$command}\n"
+
+  my $arg_spec = $self->{'line_commands'}->{$data_cmdname};
+
+  print STDERR "MISC END \@$command: $arg_spec\n"
      if ($self->{'DEBUG'});
 
-  if ($self->{'line_commands'}->{$command} eq 'specific') {
+  if ($arg_spec eq 'specific') {
     my $args = _parse_line_command_args($self, $current, $source_info);
     if (defined($args)) {
       $current->{'extra'} = {} if (!defined($current->{'extra'}));
       $current->{'extra'}->{'misc_args'} = $args;
     }
-  } elsif ($self->{'line_commands'}->{$command} eq 'text') {
+  } elsif ($arg_spec eq 'text') {
     my ($text, $superfluous_arg)
       = _convert_to_text($current->{'args'}->[0]);
 
@@ -3605,7 +3613,7 @@ sub _end_line_misc_line($$$)
         if (!defined($current->{'parent'}->{'extra'}));
       $current->{'parent'}->{'extra'}->{'columnfractions'} = $misc_cmd;
     }
-  } elsif ($root_commands{$command}) {
+  } elsif ($root_commands{$data_cmdname}) {
     $current = $current->{'contents'}->[-1];
     delete $current->{'remaining_args'};
 
@@ -5107,17 +5115,18 @@ sub _handle_other_command($$$$$)
   return ($current, $line, $retval);
 }
 
-sub _handle_line_command($$$$$)
+sub _handle_line_command($$$$$$)
 {
   my $self = shift;
   my $current = shift;
   my $command = shift;
+  my $data_cmdname = shift;
   my $line = shift;
   my $source_info = shift;
 
   my $retval = $STILL_MORE_TO_PROCESS;
 
-  if ($root_commands{$command} or $command eq 'bye') {
+  if ($root_commands{$data_cmdname} or $command eq 'bye') {
     $current = _close_commands($self, $current, $source_info, undef,
                                $command);
     # if the root command happens in a Texinfo fragment going through
@@ -5148,7 +5157,7 @@ sub _handle_line_command($$$$$)
   }
 
   # text line lineraw special specific
-  my $arg_spec = $self->{'line_commands'}->{$command};
+  my $arg_spec = $self->{'line_commands'}->{$data_cmdname};
   my $misc;
 
   # all the cases using the raw line
@@ -5306,7 +5315,7 @@ sub _handle_line_command($$$$$)
         # command.  This means that spaces are preserved properly
         # when converting back to Texinfo.
         $current = _end_line($self, $current, $source_info);
-      } elsif ($sectioning_heading_commands{$command}) {
+      } elsif ($sectioning_heading_commands{$data_cmdname}) {
         if ($self->{'sections_level'}) {
           $misc->{'extra'} = {'sections_level' => $self->{'sections_level'}};
         }
@@ -5314,7 +5323,7 @@ sub _handle_line_command($$$$$)
       push @{$current->{'contents'}}, $misc;
       $misc->{'parent'} = $current;
       # def*x
-      if ($def_commands{$command}) {
+      if ($def_commands{$data_cmdname}) {
         my $base_command = $command;
         $base_command =~ s/x$//;
         # check that the def*x is first after @def*, no paragraph
@@ -5351,7 +5360,7 @@ sub _handle_line_command($$$$$)
     $current->{'args'} = [{ 'type' => 'line_arg',
                             'parent' => $current }];
     if ($self->{'basic_inline_commands'}
-        and $self->{'basic_inline_commands'}->{$command}) {
+        and $self->{'basic_inline_commands'}->{$data_cmdname}) {
       push @{$self->{'nesting_context'}->{'basic_inline_stack_on_line'}},
            $command;
     }
@@ -5359,7 +5368,7 @@ sub _handle_line_command($$$$$)
     # 'specific' commands arguments are handled in a specific way.
     # The only other line commands that have more than one argument is
     # node, so the following condition only applies to node
-    if ($self->{'line_commands'}->{$command} ne 'specific'
+    if ($arg_spec ne 'specific'
         and $commands_args_number{$command}
         and $commands_args_number{$command} > 1) {
       $current->{'remaining_args'} = $commands_args_number{$command} - 1;
@@ -6130,6 +6139,8 @@ sub _process_remaining_on_line($$$$)
     my $def_line_continuation
       = ($self->_top_context() eq 'ct_def' and $command eq "\n");
 
+    # warn on not appearing at line beginning.  Need to do before closing
+    # paragraph as it also closes the empty line
     if (not $def_line_continuation
            and not _abort_empty_line($self, $current)
            and $begin_line_commands{$command}) {
@@ -6145,6 +6156,12 @@ sub _process_remaining_on_line($$$$)
       $current = _end_preformatted($self, $current, $source_info);
     }
 
+    # command used for gathering data on the command.  For @item command
+    my $data_cmdname = $command;
+    # cannot check parent before closing paragraph/preformatted
+    $data_cmdname = 'item_LINE'
+      if ($command eq 'item' and _item_line_parent($current));
+
     _check_valid_nesting ($self, $current, $command, $source_info);
     _check_valid_nesting_context ($self, $command, $source_info);
 
@@ -6176,27 +6193,27 @@ sub _process_remaining_on_line($$$$)
       }
     }
 
-    unless ($self->{'no_paragraph_commands'}->{$command}) {
+    unless ($self->{'no_paragraph_commands'}->{$data_cmdname}) {
       my $paragraph = _begin_paragraph($self, $current, $source_info);
       $current = $paragraph if ($paragraph);
     }
 
-    if (defined($nobrace_commands{$command})
-        and ($command ne 'item' or !_item_line_parent($current))) {
+    if (defined($nobrace_commands{$data_cmdname})) {
       ($current, $line, $retval)
-         = _handle_other_command($self, $current, $command, $line, 
$source_info);
+        = _handle_other_command($self, $current, $command, $line, 
$source_info);
       # in the XS parser return here if GET_A_NEW_LINE or FINISHED_TOTALLY
     # line commands
-    } elsif (defined($self->{'line_commands'}->{$command})) {
+    } elsif (defined($self->{'line_commands'}->{$data_cmdname})) {
       ($current, $line, $retval)
-       = _handle_line_command($self, $current, $command, $line, $source_info);
+       = _handle_line_command($self, $current, $command, $data_cmdname, $line,
+                              $source_info);
       # in the XS parser return here if GET_A_NEW_LINE or FINISHED_TOTALLY
     # @-command with matching @end opening
-    } elsif (exists($block_commands{$command})) {
+    } elsif (exists($block_commands{$data_cmdname})) {
       ($current, $line, $retval)
        = _handle_block_command($self, $current, $command, $line, $source_info);
       # in the XS parser return here if GET_A_NEW_LINE
-    } elsif (defined($self->{'brace_commands'}->{$command})) {
+    } elsif (defined($self->{'brace_commands'}->{$data_cmdname})) {
       $current = _handle_brace_command($self, $current, $command, 
$source_info);
     }
   } elsif ($separator_match) {
@@ -6828,6 +6845,8 @@ sub _parse_texi($$$)
       print STDERR "NEXT_LINE NO MORE\n" if ($self->{'DEBUG'});
       last;
     }
+#print STDERR 
"@{$self->{'nesting_context'}->{'basic_inline_stack_on_line'}}|$line"
+#if ($self->{'nesting_context'} and 
$self->{'nesting_context'}->{'basic_inline_stack_on_line'});
 
     if ($self->{'DEBUG'}) {
       my $source_info_text = '';
@@ -8622,7 +8641,7 @@ I<columnfractions> key is associated with the element for 
the
 
 =item C<@node>
 
-Explicit directions labels information are in the I<line_arg>
+Explicit directions labels information is in the I<line_arg>
 arguments C<extra> node direction C<@node> arguments.  They consist
 in a hash with the I<node_content> key for an array holding the
 corresponding content, a I<manual_content> key if there is an
diff --git a/tp/Texinfo/XS/parsetexi/end_line.c 
b/tp/Texinfo/XS/parsetexi/end_line.c
index 168b283469..1636c63b5f 100644
--- a/tp/Texinfo/XS/parsetexi/end_line.c
+++ b/tp/Texinfo/XS/parsetexi/end_line.c
@@ -1541,19 +1541,24 @@ ELEMENT *
 end_line_misc_line (ELEMENT *current)
 {
   enum command_id cmd;
-  int arg_type;
+  enum command_id data_cmd;
+  int arg_spec;
   ELEMENT *misc_cmd;
   char *end_command = 0;
   enum command_id end_id = CM_NONE;
   int included_file = 0;
   SOURCE_MARK *include_source_mark;
 
-  cmd = current->parent->cmd;
+  data_cmd = cmd = current->parent->cmd;
+  /* we are in a command line context, so the @item command information is
+     associated to CM_item_LINE */
+  if (cmd == CM_item)
+    data_cmd = CM_item_LINE;
+
   if (!cmd)
     fatal ("command name unknown for line command end");
 
-  if ((command_data(cmd).flags & CF_contain_basic_inline)
-      || cmd == CM_item) /* CM_item_LINE on stack */
+  if (command_data(data_cmd).flags & CF_contain_basic_inline)
     (void) pop_command (&nesting_context.basic_inline_stack_on_line);
   isolate_last_space (current);
 
@@ -1563,20 +1568,20 @@ end_line_misc_line (ELEMENT *current)
   current = current->parent;
   misc_cmd = current;
 
-  arg_type = command_data(cmd).data;
+  arg_spec = command_data(data_cmd).data;
    
   debug ("MISC END %s", command_name(cmd));
 
   if (pop_context () != ct_line)
     fatal ("line context expected");
 
-  if (arg_type == LINE_specific)
+  if (arg_spec == LINE_specific)
     {
       ELEMENT *args = parse_line_command_args (current);
       if (args)
         add_extra_misc_args (current, "misc_args", args);
     }
-  else if (arg_type == LINE_text)
+  else if (arg_spec == LINE_text)
     {
       char *text = 0;
       int superfluous_arg = 0;
@@ -2079,7 +2084,7 @@ end_line_misc_line (ELEMENT *current)
           add_extra_element (current->parent, "columnfractions", misc_cmd);
         }
     }
-  else if (command_data(cmd).flags & CF_root)
+  else if (command_data(data_cmd).flags & CF_root)
     {
       current = last_contents_child (current);
       if (cmd == CM_node)
diff --git a/tp/maintain/regenerate_commands_perl_info.pl 
b/tp/maintain/regenerate_commands_perl_info.pl
index 07a1104c27..1cb31e5169 100755
--- a/tp/maintain/regenerate_commands_perl_info.pl
+++ b/tp/maintain/regenerate_commands_perl_info.pl
@@ -40,6 +40,8 @@ my %command_categories;
 my %flags_hashes;
 my %command_args_nr;
 
+my %multi_category_commands;
+
 while (<>) {
   if (not (/^#/ or /^ *$/)) {
     my ($command, $flags, $data, $args_nr) = split;
@@ -61,6 +63,10 @@ while (<>) {
         die "$command: ".join('|',@flags).": $category: not in flags\n";
       }
     } else {
+      # note that this depends on the order in the file, a category can
+      # only be detected in the flags if it was already seen as data for
+      # another command, so commands without data should be last in their
+      # category in the input file.
       my @categories = grep {exists($command_categories{$_})} @flags;
       if (scalar(@categories) == 0) {
         die "$command: ".join('|',@flags).": cannot find a category ("
@@ -71,9 +77,11 @@ while (<>) {
       }
       $category = $categories[0];
     }
-    # remove _LINE in item_LINE
+    # handle commands in multiple categories, for now @item
     my $uc_category = uc($category);
-    $command =~ s/_$uc_category$//;
+    if ($command =~ /^(.*)_$uc_category$/) {
+      $multi_category_commands{$command} = $1;
+    }
 
     if (defined($args_nr) and $args_nr ne '') {
       $command_args_nr{$command} = $args_nr;
@@ -116,12 +124,30 @@ foreach my $category (sort(keys(%command_categories))) {
 print OUT "\n";
 print OUT "# flag hashes\n";
 
+# for those flags, the information of multi category commands is
+# duplicated.  So, for example, item_LINE has the formatted_line flag
+# associated, it will be associated to item.
+#
+# In general, the hash here should be in the excluded flags in
+# Texinfo/XS/parsetexi/command_data.awk
+my %converter_flag = (
+  'formatted_line' => 1,
+  'formattable_line' => 1,
+);
+
 foreach my $hash_flag (sort(keys(%flags_hashes))) {
-  # happens for 'txiinternalvalue' which is also brace
+  # happens for 'txiinternalvalue', which is in internal category but also
+  # has the brace flag set.  This information cannot be kept, this command
+  # will not appear in brace commands.
   next if ($command_categories{$hash_flag});
   print OUT "our %${hash_flag}_commands = (\n";
   foreach my $command (sort(@{$flags_hashes{$hash_flag}})) {
     print OUT '  '.sprintf('%-25s', '"'.$command.'"')." => 1,\n";
+    if ($multi_category_commands{$command}
+        and $converter_flag{$hash_flag}) {
+      print OUT '  '.sprintf('%-25s', '"'.$multi_category_commands{$command}
+                                                             .'"')." => 1,\n";
+    }
   }
   print OUT ");\n\n";
 }