texinfo/tp Texinfo/Parser.pm t/08misc_commands....

[Patrice Dumas]

CVSROOT:        /sources/texinfo
Module name:    texinfo
Changes by:     Patrice Dumas <pertusus>        10/10/24 14:22:58

Modified files:
        tp/Texinfo     : Parser.pm 
        tp/t           : 08misc_commands.t 
        tp/t/results/include: macro_and_commands_in_early_commands.pl 
        tp/t/results/misc_commands: many_lines.pl 
Added files:
        tp/t/results/misc_commands: empty_documentencoding.pl 

Log message:
        Use documentencoding information to set the binmode layer for this
        encoding.

CVSWeb URLs:
http://cvs.savannah.gnu.org/viewcvs/texinfo/tp/Texinfo/Parser.pm?cvsroot=texinfo&r1=1.101&r2=1.102
http://cvs.savannah.gnu.org/viewcvs/texinfo/tp/t/08misc_commands.t?cvsroot=texinfo&r1=1.4&r2=1.5
http://cvs.savannah.gnu.org/viewcvs/texinfo/tp/t/results/include/macro_and_commands_in_early_commands.pl?cvsroot=texinfo&r1=1.4&r2=1.5
http://cvs.savannah.gnu.org/viewcvs/texinfo/tp/t/results/misc_commands/many_lines.pl?cvsroot=texinfo&r1=1.11&r2=1.12
http://cvs.savannah.gnu.org/viewcvs/texinfo/tp/t/results/misc_commands/empty_documentencoding.pl?cvsroot=texinfo&rev=1.1

Patches:
Index: Texinfo/Parser.pm
===================================================================
RCS file: /sources/texinfo/texinfo/tp/Texinfo/Parser.pm,v
retrieving revision 1.101
retrieving revision 1.102
diff -u -b -r1.101 -r1.102
--- Texinfo/Parser.pm   24 Oct 2010 13:12:24 -0000      1.101
+++ Texinfo/Parser.pm   24 Oct 2010 14:22:58 -0000      1.102
@@ -25,6 +25,8 @@
 use Data::Dumper;
 # to expand file names in @include
 use Texinfo::Convert::Text;
+# to detect if an encoding may be used to open the files
+use Encode;
 use strict;
 
 require Exporter;
@@ -577,6 +579,45 @@
 $full_text_commands{'center'} = 1;
 $full_text_commands{'exdent'} = 1;
 
+my %canonical_texinfo_encodings;
+# These are the encodings from the texinfo manual
+foreach my $canonical_encoding('us-ascii', 'utf-8', 'iso-8859-1',
+  'iso-8859-15','iso-8859-2','koi8-r', 'koi8-u') {
+  $canonical_texinfo_encodings{$canonical_encoding} = 1;
+}
+
+
+my %perl_charset_to_html = (
+              'utf8'       => 'utf-8',
+              'utf-8-strict'       => 'utf-8',
+              'ascii'      => 'us-ascii',
+              'shiftjis'      => 'shift_jis',
+);
+
+# encoding name normalization to html-compatible encoding names
+my %encoding_aliases = (
+              'latin1' => 'iso-8859-1',
+);
+
+foreach my $perl_charset (keys(%perl_charset_to_html)) {
+   $encoding_aliases{$perl_charset} = $perl_charset_to_html{$perl_charset};
+   $encoding_aliases{$perl_charset_to_html{$perl_charset}} 
+        = $perl_charset_to_html{$perl_charset};
+}
+my %makeinfo_encoding_to_map = (
+  "iso-8859-1",  'iso8859_1', 
+  "iso-8859-2",  'iso8859_2',
+  "iso-8859-15", 'iso8859_15',
+  "koi8-r",      'koi8',
+  "koi8-u",      'koi8',
+);  
+    
+foreach my $encoding (keys(%makeinfo_encoding_to_map)) {        
+  $encoding_aliases{$encoding} = $encoding;
+  $encoding_aliases{$makeinfo_encoding_to_map{$encoding}} = $encoding;
+}
+
+
 
 
 # deep copy of a structure
@@ -1693,6 +1734,8 @@
             my $filehandle = do { local *FH };
             if (open ($filehandle, $file)) {
               $included_file = 1;
+              binmode($filehandle, ":encoding($self->{'encoding'})")
+                if (defined($self->{'encoding'}));
               print STDERR "Included $file($filehandle)\n" if 
($self->{'debug'});
               $included_file = 1;
               unshift @{$self->{'input'}}, { 
@@ -1708,6 +1751,28 @@
             _line_error ($self, sprintf($self->__("address@hidden: Cannot find 
%s"), 
                $command, $text), $line_nr);
           }
+        } elsif ($command eq 'documentencoding') {
+          if ($text =~ /\S/) {
+            _line_warn($self, sprintf($self->__("Encoding `%s' is not a 
canonical texinfo encoding"), 
+                                     $text), $line_nr)
+              if (!$canonical_texinfo_encodings{lc($text)});
+            my $encoding = Encode::resolve_alias($text);
+            if (!$encoding) {
+              _line_warn($self, sprintf($self->__("unrecognized encoding name 
`%s'"), 
+                         $text), $line_nr);
+            } else {
+              $encoding = $encoding_aliases{$encoding} 
+                if ($encoding_aliases{$encoding});
+              $self->{'encoding'} = $encoding;
+              print STDERR "Using encoding $encoding\n" if ($self->{'debug'});
+              foreach my $input (@{$self->{'input'}}) {
+                binmode($input->{'fh'}, ":encoding($encoding)") if 
($input->{'fh'});
+              }
+            }
+          } else {
+            _line_warn ($self, sprintf($self->__("address@hidden missing 
argument"), 
+               $command), $line_nr);
+          }
         }
       }
     }

Index: t/08misc_commands.t
===================================================================
RCS file: /sources/texinfo/texinfo/tp/t/08misc_commands.t,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -b -r1.4 -r1.5
--- t/08misc_commands.t 20 Oct 2010 19:26:26 -0000      1.4
+++ t/08misc_commands.t 24 Oct 2010 14:22:58 -0000      1.5
@@ -152,6 +152,9 @@
 'documentlanguage on its line
 @documentlanguage  en  
 line following documentlanguage
+'],
+['empty_documentencoding',
+'@documentencoding   
 ']
 );
 

Index: t/results/include/macro_and_commands_in_early_commands.pl
===================================================================
RCS file: 
/sources/texinfo/texinfo/tp/t/results/include/macro_and_commands_in_early_commands.pl,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -b -r1.4 -r1.5
--- t/results/include/macro_and_commands_in_early_commands.pl   21 Oct 2010 
22:16:41 -0000      1.4
+++ t/results/include/macro_and_commands_in_early_commands.pl   24 Oct 2010 
14:22:58 -0000      1.5
@@ -484,6 +484,24 @@
 
 $result_errors{'macro_and_commands_in_early_commands'} = [
   {
+    'error_line' => ':11: warning: Encoding address@hidden' is not a canonical 
texinfo encoding
+',
+    'file_name' => '',
+    'line_nr' => 11,
+    'macro' => '',
+    'text' => 'Encoding address@hidden' is not a canonical texinfo encoding',
+    'type' => 'warning'
+  },
+  {
+    'error_line' => ':11: warning: unrecognized encoding name address@hidden'
+',
+    'file_name' => '',
+    'line_nr' => 11,
+    'macro' => '',
+    'text' => 'unrecognized encoding name address@hidden'',
+    'type' => 'warning'
+  },
+  {
     'error_line' => ':21: warning: @multiinclude defined with zero or more 
than one argument should be invoked with {}
 ',
     'file_name' => '',

Index: t/results/misc_commands/many_lines.pl
===================================================================
RCS file: /sources/texinfo/texinfo/tp/t/results/misc_commands/many_lines.pl,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -b -r1.11 -r1.12
--- t/results/misc_commands/many_lines.pl       21 Oct 2010 22:16:45 -0000      
1.11
+++ t/results/misc_commands/many_lines.pl       24 Oct 2010 14:22:58 -0000      
1.12
@@ -1554,6 +1554,24 @@
     'type' => 'warning'
   },
   {
+    'error_line' => ':29: warning: Encoding `US-ascii encoding name\' is not a 
canonical texinfo encoding
+',
+    'file_name' => '',
+    'line_nr' => 29,
+    'macro' => '',
+    'text' => 'Encoding `US-ascii encoding name\' is not a canonical texinfo 
encoding',
+    'type' => 'warning'
+  },
+  {
+    'error_line' => ':29: warning: unrecognized encoding name `US-ascii 
encoding name\'
+',
+    'file_name' => '',
+    'line_nr' => 29,
+    'macro' => '',
+    'text' => 'unrecognized encoding name `US-ascii encoding name\'',
+    'type' => 'warning'
+  },
+  {
     'error_line' => ':31: warning: @frenchspacing should only appear at a line 
beginning
 ',
     'file_name' => '',

Index: t/results/misc_commands/empty_documentencoding.pl
===================================================================
RCS file: t/results/misc_commands/empty_documentencoding.pl
diff -N t/results/misc_commands/empty_documentencoding.pl
--- /dev/null   1 Jan 1970 00:00:00 -0000
+++ t/results/misc_commands/empty_documentencoding.pl   24 Oct 2010 14:22:58 
-0000      1.1
@@ -0,0 +1,51 @@
+use vars qw(%result_texis %result_texts %result_trees %result_errors);
+
+$result_trees{'empty_documentencoding'} = {
+  'contents' => [
+    {
+      'args' => [
+        {
+          'contents' => [
+            {
+              'parent' => {},
+              'text' => '   
+',
+              'type' => 'empty_line_after_command'
+            }
+          ],
+          'parent' => {},
+          'type' => 'misc_line_arg'
+        }
+      ],
+      'cmdname' => 'documentencoding',
+      'parent' => {},
+      'special' => {
+        'text_arg' => ''
+      }
+    }
+  ],
+  'type' => 'text_root'
+};
+$result_trees{'empty_documentencoding'}{'contents'}[0]{'args'}[0]{'contents'}[0]{'parent'}
 = $result_trees{'empty_documentencoding'}{'contents'}[0]{'args'}[0];
+$result_trees{'empty_documentencoding'}{'contents'}[0]{'args'}[0]{'parent'} = 
$result_trees{'empty_documentencoding'}{'contents'}[0];
+$result_trees{'empty_documentencoding'}{'contents'}[0]{'parent'} = 
$result_trees{'empty_documentencoding'};
+
+$result_texis{'empty_documentencoding'} = '@documentencoding   
+';
+
+
+$result_texts{'empty_documentencoding'} = '';
+
+$result_errors{'empty_documentencoding'} = [
+  {
+    'error_line' => ':1: warning: @documentencoding missing argument
+',
+    'file_name' => '',
+    'line_nr' => 1,
+    'macro' => '',
+    'text' => '@documentencoding missing argument',
+    'type' => 'warning'
+  }
+];
+
+