Re: Two pass scanning

[Henrik Sorensen]

On Sunday 23 November 2003 21:01, Henrik Sorensen wrote:
> First pass scans the source code and stores tokens and strings away, before
> returning to the parser. The second pass reads the stored tokens instead of
> the actual source code.

In reply to myself, here is how I solved it.
Any comments on this implementation would be highly appreciated.

in the lex source redefine the name of the generated scanner:
#define YY_DECL int pl1lex YY_PROTO(( void ))

in the gramma source, add two tokens to signal start of pass one and two:
%token _PASSONE_ _PASSTWO_

and change the gramma to take the passes into consideration:
pl1pgm:         _PASSONE_ pl1stmtlist _PASSTWO_ pl1stmtlist;

and finally create an yylex internal to the gramma souce:
/*
* Copyright 2002, 2003 Henrik Sorensen
*
* This file is part of GCC front-end for the PL/1 programming language, GCC 
PL/1.
 *
 * GCC PL/1 is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; version 2.
*/
char *pl1gcctemplate="gccpl1gcctokXXXXXX";
short passnumber=0; /* Actual passnumber */
FILE *savefile ;
unsigned int maxlen=0;
unsigned long chksum1=0, chksum2=0;
#define LEXEOF 0
/*
 * yylex: returns tokens from pl1lex for the first pass, and the FILE savefile
 *        for the second pass.
 *        The very first token returned is _PASSONE_, this is returned before
 *        pl1lex is invoked.
 *        When pl1lex returns EOF, the passnumber is increased, and FILE
 *        savefile is reset by rewind(),
 *        and token _PASSTWO_ is returned before the tokens are reread from
 *        the FILE savefile.
 *        Note, the EOF from pass one is only returned at the end of the
 *        second pass, to signal to yyparse, that the parsing is done.
 */
int yylex( void )
{ int t,fd;
  size_t i;
  unsigned int m;
  switch(passnumber)
  {case 0: /* init save file */
      fd=mkstemp(pl1gcctemplate);
      if (fd<0) yyerror(strerror(errno));
      savefile=fdopen(fd,"w+");
      if ( ! savefile ) yyerror(strerror(errno));
      // ready to proceed with the first pass
      passnumber++;
      return _PASSONE_ ;

   case 1: /* first pass */
      t=pl1lex();
      if (t!=LEXEOF)
      { /* save token and yytext, 
        remember max length of yytext for reading in pass two. */
        printf("Saving token %i\n",t);
        m=strlen(yytext);
        if ( m>maxlen) maxlen=m;
        i=fwrite(&t,sizeof(t),1,savefile); 
        if ( ferror( savefile )) yyerror(strerror(errno));
        i=fwrite(&m,sizeof(m),1,savefile); 
        if ( ferror( savefile )) yyerror(strerror(errno));
        if (m>0)
        { i=fwrite((void *)yytext,sizeof(*yytext),m,savefile);
          if ( ferror( savefile )) yyerror(strerror(errno));
        }
        chksum1 += t+m;
        return t;
      }
      // pl1lex returned end of file
      // Prepare for the second pass

      printf("Check sum pass 1: %ul\n",chksum1);
      passnumber++;
      yytext=malloc(maxlen);
      *yytext='\0';
      rewind(savefile);
      return _PASSTWO_ ;

   case 2: /* read and return next token and yytext */
      i=fread(&t,sizeof(t),1,savefile);
      /* if eof then close savefile and return eof */
      if (feof(savefile))
      { i=fclose(savefile);
        if ( i ) yyerror(strerror(errno));
        t=LEXEOF;
        *yytext='\0';
        printf("Check sum pass 2: %ul\n",chksum2);
        if (chksum1==chksum2) return t;

        yyerror("Internal compiler error: Checksum failure.");
      }
      if ( ferror( savefile )) yyerror(strerror(errno));

      printf("Reading token %i\n",t);

      i=fread(&m,sizeof(m),1,savefile); 
      if ( ferror( savefile )) yyerror(strerror(errno));
      if (m>0)
      { i=fread((void *)yytext,sizeof(*yytext),m,savefile);
        if ( ferror( savefile )) yyerror(strerror(errno));
      }
      else
      { *yytext='\0'; }

      chksum2 +=t+m;
      return t;

    default:
        yyerror("Internal compiler error: Passnumbering gone wild.");
  } // end switch passnumber
  return LEXEOF; /* just to remove warning */
} // end yylex


keep hacking

Henrik