coreutils fnmatch bugfix + cleanup

[Paul Eggert]

This fixes a bug in fnmatch (that is also in glibc fnmatch): when the
FNM_EXTMATCH option is used, the code mishandles patterns containing
more than INT_MAX nested subpatterns, a problem that can occur on
64-bit hosts.  The plan is to migrate this fix into glibc via gnulib.

2004-08-02  Paul Eggert  <address@hidden>

        * fnmatch.c: Include <stdbool.h>.
        (errno): Remove decl; we now assume C89 or better.
        * fnmatch_loop.c (EXT, FCT): Use bool when appropriate.
        (FCT): Use size_t, not unsigned int, for sizes.
        (EXT): Use size_t, not int, for sizes.

Index: lib/fnmatch.c
===================================================================
RCS file: /home/eggert/coreutils/cu/lib/fnmatch.c,v
retrieving revision 1.27
retrieving revision 1.28
diff -p -u -r1.27 -r1.28
--- lib/fnmatch.c       21 May 2004 07:50:27 -0000      1.27
+++ lib/fnmatch.c       2 Aug 2004 18:10:35 -0000       1.28
@@ -35,6 +35,7 @@
 #include <ctype.h>
 #include <errno.h>
 #include <stddef.h>
+#include <stdbool.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -155,10 +156,6 @@ extern int fnmatch (const char *pattern,
 
 /* Avoid depending on library functions or files
    whose names are inconsistent.  */
-
-# ifndef errno
-extern int errno;
-# endif
 
 /* Global variable.  */
 static int posixly_correct;
Index: lib/fnmatch_loop.c
===================================================================
RCS file: /home/eggert/coreutils/cu/lib/fnmatch_loop.c,v
retrieving revision 1.6
retrieving revision 1.8
diff -p -u -r1.6 -r1.8
--- lib/fnmatch_loop.c  16 Jan 2004 09:55:15 -0000      1.6
+++ lib/fnmatch_loop.c  2 Aug 2004 18:21:05 -0000       1.8
@@ -18,14 +18,14 @@
 /* Match STRING against the filename pattern PATTERN, returning zero if
    it matches, nonzero if not.  */
 static int EXT (INT opt, const CHAR *pattern, const CHAR *string,
-               const CHAR *string_end, int no_leading_period, int flags)
+               const CHAR *string_end, bool no_leading_period, int flags)
      internal_function;
 static const CHAR *END (const CHAR *patternp) internal_function;
 
 static int
 internal_function
 FCT (const CHAR *pattern, const CHAR *string, const CHAR *string_end,
-     int no_leading_period, int flags)
+     bool no_leading_period, int flags)
 {
   register const CHAR *p = pattern, *n = string;
   register UCHAR c;
@@ -41,7 +41,7 @@ FCT (const CHAR *pattern, const CHAR *st
 
   while ((c = *p++) != L('\0'))
     {
-      int new_no_leading_period = 0;
+      bool new_no_leading_period = false;
       c = FOLD (c);
 
       switch (c)
@@ -161,9 +161,9 @@ FCT (const CHAR *pattern, const CHAR *st
                {
                  int flags2 = ((flags & FNM_FILE_NAME)
                                ? flags : (flags & ~FNM_PERIOD));
-                 int no_leading_period2 = no_leading_period;
+                 bool no_leading_period2 = no_leading_period;
 
-                 for (--p; n < endp; ++n, no_leading_period2 = 0)
+                 for (--p; n < endp; ++n, no_leading_period2 = false)
                    if (FCT (p, n, string_end, no_leading_period2, flags2)
                        == 0)
                      return 0;
@@ -186,7 +186,7 @@ FCT (const CHAR *pattern, const CHAR *st
                  if (c == L('\\') && !(flags & FNM_NOESCAPE))
                    c = *p;
                  c = FOLD (c);
-                 for (--p; n < endp; ++n, no_leading_period2 = 0)
+                 for (--p; n < endp; ++n, no_leading_period2 = false)
                    if (FOLD ((UCHAR) *n) == c
                        && (FCT (p, n, string_end, no_leading_period2, flags2)
                            == 0))
@@ -200,7 +200,7 @@ FCT (const CHAR *pattern, const CHAR *st
        case L('['):
          {
            /* Nonzero if the sense of the character class is inverted.  */
-           register int not;
+           register bool not;
            CHAR cold;
            UCHAR fn;
 
@@ -410,10 +410,10 @@ FCT (const CHAR *pattern, const CHAR *st
                  return FNM_NOMATCH;
                else
                  {
-                   int is_range = 0;
+                   bool is_range = false;
 
 #ifdef _LIBC
-                   int is_seqval = 0;
+                   bool is_seqval = false;
 
                    if (c == L('[') && *p == L('.'))
                      {
@@ -460,7 +460,7 @@ FCT (const CHAR *pattern, const CHAR *st
                            const int32_t *symb_table;
 # ifdef WIDE_CHAR_VERSION
                            char str[c1];
-                           unsigned int strcnt;
+                           size_t strcnt;
 # else
 #  define str (startp + 1)
 # endif
@@ -550,7 +550,7 @@ FCT (const CHAR *pattern, const CHAR *st
                                  }
 
                                /* Get the collation sequence value.  */
-                               is_seqval = 1;
+                               is_seqval = true;
 # ifdef WIDE_CHAR_VERSION
                                cold = wextra[1 + wextra[idx]];
 # else
@@ -629,7 +629,7 @@ FCT (const CHAR *pattern, const CHAR *st
                        lcollseq = is_seqval ? cold : collseq[(UCHAR) cold];
 # endif
 
-                       is_seqval = 0;
+                       is_seqval = false;
                        if (cend == L('[') && *p == L('.'))
                          {
                            uint32_t nrules =
@@ -668,7 +668,7 @@ FCT (const CHAR *pattern, const CHAR *st
                                const int32_t *symb_table;
 # ifdef WIDE_CHAR_VERSION
                                char str[c1];
-                               unsigned int strcnt;
+                               size_t strcnt;
 # else
 #  define str (startp + 1)
 # endif
@@ -738,7 +738,7 @@ FCT (const CHAR *pattern, const CHAR *st
                                    wextra = (int32_t *) &extra[idx + 4];
 # endif
                                    /* Get the collation sequence value.  */
-                                   is_seqval = 1;
+                                   is_seqval = true;
 # ifdef WIDE_CHAR_VERSION
                                    cend = wextra[1 + wextra[idx]];
 # else
@@ -929,7 +929,7 @@ FCT (const CHAR *pattern, const CHAR *st
              if (n == string_end || c != (UCHAR) *n)
                return FNM_NOMATCH;
 
-             new_no_leading_period = 1;
+             new_no_leading_period = true;
              break;
            }
          /* FALLTHROUGH */
@@ -996,10 +996,10 @@ END (const CHAR *pattern)
 static int
 internal_function
 EXT (INT opt, const CHAR *pattern, const CHAR *string, const CHAR *string_end,
-     int no_leading_period, int flags)
+     bool no_leading_period, int flags)
 {
   const CHAR *startp;
-  int level;
+  size_t level;
   struct patternlist
   {
     struct patternlist *next;
@@ -1013,7 +1013,7 @@ EXT (INT opt, const CHAR *pattern, const
 
   /* Parse the pattern.  Store the individual parts in the list.  */
   level = 0;
-  for (startp = p = pattern + 1; level >= 0; ++p)
+  for (startp = p = pattern + 1; ; ++p)
     if (*p == L('\0'))
       /* This is an invalid pattern.  */
       return -1;
@@ -1066,6 +1066,7 @@ EXT (INT opt, const CHAR *pattern, const
            *lastp = newp;                                                    \
            lastp = &newp->next
            NEW_PATTERN;
+           break;
          }
       }
     else if (*p == L('|'))
@@ -1100,7 +1101,7 @@ EXT (INT opt, const CHAR *pattern, const
                && (FCT (p, rs, string_end,
                         rs == string
                         ? no_leading_period
-                        : rs[-1] == '/' && NO_LEADING_PERIOD (flags) ? 1 : 0,
+                        : rs[-1] == '/' && NO_LEADING_PERIOD (flags),
                         flags & FNM_FILE_NAME
                         ? flags : flags & ~FNM_PERIOD) == 0
                    /* This didn't work.  Try the whole pattern.  */
@@ -1108,8 +1109,7 @@ EXT (INT opt, const CHAR *pattern, const
                        && FCT (pattern - 1, rs, string_end,
                                rs == string
                                ? no_leading_period
-                               : (rs[-1] == '/' && NO_LEADING_PERIOD (flags)
-                                  ? 1 : 0),
+                               : rs[-1] == '/' && NO_LEADING_PERIOD (flags),
                                flags & FNM_FILE_NAME
                                ? flags : flags & ~FNM_PERIOD) == 0)))
              /* It worked.  Signal success.  */
@@ -1156,7 +1156,7 @@ EXT (INT opt, const CHAR *pattern, const
              && (FCT (p, rs, string_end,
                       rs == string
                       ? no_leading_period
-                      : rs[-1] == '/' && NO_LEADING_PERIOD (flags) ? 1 : 0,
+                      : rs[-1] == '/' && NO_LEADING_PERIOD (flags),
                       flags & FNM_FILE_NAME ? flags : flags & ~FNM_PERIOD)
                  == 0))
            /* This is successful.  */