[elpa] externals/phps-mode e59d30e 26/96: Improvements in lex-analyzer for parser

[Christian Johansson]

branch: externals/phps-mode
commit e59d30e58e84048b7780a208988e6d7b1854883d
Author: Christian Johansson <christian@cvj.se>
Commit: Christian Johansson <christian@cvj.se>

    Improvements in lex-analyzer for parser
---
 Makefile                        |  2 +-
 phps-mode-automation-grammar.el | 56 +++++++++++++------------------
 phps-mode-parser.el             | 74 ++++++++++++++++++++++++++++-------------
 test/phps-mode-test-parser.el   |  2 +-
 4 files changed, 76 insertions(+), 58 deletions(-)

diff --git a/Makefile b/Makefile
index 3c74b92..7207e62 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,7 @@ parser: clean generate-parser
 
 .PHONY: generate-parser
 generate-parser: 
-       $(EMACS_CMD) -L ~/.emacs.d/emacs-parser-generator/ -l 
phps-mode-lexer.el -l admin/phps-mode-automation.el -eval "(progn (require 
'parser-generator-lr-export)(setq debug-on-signal t)(setq debug-on-error 
t)(phps-mode-automation))"
+       rm phps-mode-automation-grammar.elc; $(EMACS_CMD) -L 
~/.emacs.d/emacs-parser-generator/ -l phps-mode-lexer.el -l 
admin/phps-mode-automation.el -eval "(progn (require 
'parser-generator-lr-export)(setq debug-on-signal t)(setq debug-on-error 
t)(phps-mode-automation))"
 
 .PHONY: compile
 compile:
diff --git a/phps-mode-automation-grammar.el b/phps-mode-automation-grammar.el
index b6ad484..52e933b 100644
--- a/phps-mode-automation-grammar.el
+++ b/phps-mode-automation-grammar.el
@@ -31,32 +31,32 @@
   nil
   "Just placeholder to pass byte-compilation.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--context-sensitive-attributes
   '(%prec)
   "List of context-sensitive attributes.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--lr-context-sensitive-precedence-attribute
   '%prec
   "The LR-parser's context-sensitive precedence attribute.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--global-attributes
   '(%left %nonassoc %precedence %right)
   "List of valid global attributes.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--lr-global-precedence-attributes
   '(%left %nonassoc %precedence %right)
   "The LR-parser's list of global precedence attributes.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--lr--allow-default-conflict-resolution
   t
   "Allow shift resolution to shift/reduce conflicts were precedence is 
missing.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--global-declaration
   '(
     (%precedence T_THROW)
@@ -94,7 +94,7 @@
     )
   "Declaration for grammar.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--non-terminals
   '(
     absolute_trait_method_reference
@@ -253,7 +253,7 @@
     )
   "The non-terminals in grammar.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--terminals
   '(
     "!"
@@ -465,12 +465,12 @@
     )
   "The terminals of grammar.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--look-ahead-number
   1
   "The look-ahead number of grammar.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--productions
   '(
 
@@ -1438,27 +1438,27 @@
     )
   "The productions of grammar.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--header
   "(require 'phps-mode-lexer)\n(require 'semantic)\n(require 'semantic/lex)\n"
   "Header contents for parser.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--start
   'start
   "The entry-point of grammar.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--e-identifier
   '%empty
   "The e-identifier of grammar.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--eof-identifier
   '$
   "The EOF-identifier of grammar.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--lex-analyzer-function
   (lambda (index)
     (save-current-buffer
@@ -1466,10 +1466,10 @@
       (if (= (point) index) nil (goto-char index))
       (if (< index (point-max))
           (progn
-            (if (and
+            (unless
+                (and
                  phps-mode-lexer--generated-new-tokens-index
                  (= phps-mode-lexer--generated-new-tokens-index index))
-                nil
               (phps-mode-lexer--re2c))
             (let ((first (car (reverse 
phps-mode-lexer--generated-new-tokens))))
               (cond
@@ -1478,13 +1478,8 @@
                  (not first)
                  (not (equal index semantic-lex-end-point)))
                 (setq-local
-                 phps-mode-parser-lex-analyzer--index
-                 semantic-lex-end-point)
-                (setq
-                 first
-                 (funcall
-                  phps-mode-parser-lex-analyzer--function
-                  phps-mode-parser-lex-analyzer--index)))
+                 phps-mode-parser-lex-analyzer--move-to-index-flag
+                 semantic-lex-end-point))
 
                ((or
                  (equal (car first) 'T_OPEN_TAG)
@@ -1492,13 +1487,8 @@
                  (equal (car first) 'T_DOC_COMMENT)
                  (equal (car first) 'T_COMMENT))
                 (setq-local
-                 phps-mode-parser-lex-analyzer--index
-                 (cdr (cdr first)))
-                (setq
-                 first
-                 (funcall
-                  phps-mode-parser-lex-analyzer--function
-                  phps-mode-parser-lex-analyzer--index)))
+                 phps-mode-parser-lex-analyzer--move-to-index-flag
+                 (cdr (cdr first))))
 
                ((equal (car first) 'T_OPEN_TAG_WITH_ECHO)
                 (let* ((v first)) (setcar v 'T_ECHO))))
@@ -1506,7 +1496,7 @@
               first)))))
   "The custom lex-analyzer.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--precedence-comparison-function
   (lambda(a-type a-value _b-type b-value)
     (cond
@@ -1548,7 +1538,7 @@
      ))
   "The precedence comparison function of the grammar.")
 
-(defconst
+(defvar
   phps-mode-automation-grammar--lex-analyzer-get-function
   (lambda (token)
     (with-current-buffer "*PHPs Lexer*"
diff --git a/phps-mode-parser.el b/phps-mode-parser.el
index fc84c3b..c02fb65 100644
--- a/phps-mode-parser.el
+++ b/phps-mode-parser.el
@@ -13,77 +13,77 @@
 ;;; Constants:
 
 
-(defconst
+(defvar
   phps-mode-parser--action-tables
   #s(hash-table size 42072 test equal rehash-size 1.5 rehash-threshold 0.8125 
data (0 0 1 1 2 2 3 3 4 4 5 2 6 2 7 2 8 2 9 9 10 10 11 2 12 12 13 2 14 14 15 15 
16 16 17 2 18 18 19 19 20 14 21 21 22 10 23 23 24 24 25 25 26 2 27 2 28 10 29 
10 30 30 31 31 32 32 33 33 34 10 35 10 36 36 37 37 38 38 39 18 40 10 41 10 42 
21 43 2 44 2 45 45 46 46 47 2 48 10 49 49 50 10 51 51 52 10 53 53 54 54 55 55 
56 56 57 57 58 58 59 59 60 2 61 2 62 2 63 2 64 14 65 65 66 66 67 67 68 2 69 10 
70 2 71 71 72 72 73 7 [...]
   "The generated action-tables.")
 
-(defconst
+(defvar
   phps-mode-parser--distinct-action-tables
   #s(hash-table size 18699 test equal rehash-size 1.5 rehash-threshold 0.8125 
data (0 ((("!") reduce 80) (("\"") reduce 80) (($) reduce 80) (("$") reduce 80) 
(("(") reduce 80) (("+") reduce 80) (("-") reduce 80) (("@") reduce 80) 
((T_ABSTRACT) reduce 80) ((T_ARRAY) reduce 80) ((T_ARRAY_CAST) reduce 80) 
((T_ATTRIBUTE) reduce 80) ((T_BOOL_CAST) reduce 80) ((T_BREAK) reduce 80) 
((T_CLASS) reduce 80) ((T_CLASS_C) reduce 80) ((T_CLONE) reduce 80) ((T_CONST) 
reduce 80) ((T_CONSTANT_ENCAPSED_ST [...]
   "The generated distinct action-tables.")
 
-(defconst
+(defvar
   phps-mode-parser--goto-tables
   #s(hash-table size 42072 test equal rehash-size 1.5 rehash-threshold 0.8125 
data (0 0 1 1 2 2 3 3 4 4 5 5 6 6 7 7 8 8 9 9 10 10 11 11 12 12 13 13 14 14 15 
9 16 9 17 15 18 16 19 9 20 17 21 18 22 19 23 9 24 9 25 20 26 21 27 22 28 23 29 
24 30 25 31 9 32 9 33 9 34 26 35 27 36 9 37 9 38 28 39 29 40 30 41 31 42 32 43 
33 44 34 45 9 46 9 47 35 48 36 49 9 50 37 51 9 52 38 53 9 54 39 55 9 56 9 57 9 
58 40 59 9 60 41 61 42 62 43 63 44 64 45 65 46 66 47 67 48 68 49 69 50 70 51 71 
9 72 9 73 52 74 53 [...]
   "The generated goto-tables.")
 
-(defconst
+(defvar
   phps-mode-parser--distinct-goto-tables
   #s(hash-table size 18699 test equal rehash-size 1.5 rehash-threshold 0.8125 
data (0 ((top_statement_list 1)) 1 (("!" 2) ("\"" 3) ("$" 4) ("(" 5) ("+" 6) 
("-" 7) ("@" 8) (T_ABSTRACT 9) (T_ARRAY 10) (T_ARRAY_CAST 11) (T_ATTRIBUTE 12) 
(T_BOOL_CAST 13) (T_BREAK 14) (T_CLASS 15) (T_CLASS_C 16) (T_CLONE 17) (T_CONST 
18) (T_CONSTANT_ENCAPSED_STRING 19) (T_CONTINUE 20) (T_DEC 21) (T_DECLARE 22) 
(T_DIR 23) (T_DNUMBER 24) (T_DO 25) (T_DOUBLE_CAST 26) (T_ECHO 27) (T_EMPTY 28) 
(T_EVAL 29) (T_EXIT  [...]
   "The generated distinct goto-tables.")
 
-(defconst
+(defvar
   phps-mode-parser--table-productions-number-reverse
   #s(hash-table size 730 test equal rehash-size 1.5 rehash-threshold 0.8125 
data (0 ((start) (top_statement_list)) 1 ((reserved_non_modifiers) (T_INCLUDE)) 
2 ((reserved_non_modifiers) (T_INCLUDE_ONCE)) 3 ((reserved_non_modifiers) 
(T_EVAL)) 4 ((reserved_non_modifiers) (T_REQUIRE)) 5 ((reserved_non_modifiers) 
(T_REQUIRE_ONCE)) 6 ((reserved_non_modifiers) (T_LOGICAL_OR)) 7 
((reserved_non_modifiers) (T_LOGICAL_XOR)) 8 ((reserved_non_modifiers) 
(T_LOGICAL_AND)) 9 ((reserved_non_modifiers) (T_ [...]
   "The hash-table indexed by production-number and value is production.")
 
-(defconst
+(defvar
   phps-mode-parser--table-look-aheads
   #s(hash-table size 217 test equal rehash-size 1.5 rehash-threshold 0.8125 
data (("!") t ("\"") t ($) t ("$") t ("%") t ("&") t ("'") t ("(") t (")") t 
("*") t ("+") t (",") t ("-") t (".") t ("/") t (":") t (";") t ("<") t ("=") t 
(">") t ("?") t ("@") t (PREC_ARROW_FUNCTION) t (T_ABSTRACT) t (T_AND_EQUAL) t 
(T_ARG) t (T_ARRAY) t (T_ARRAY_CAST) t (T_ARROW) t (T_AS) t (T_ASSIGN) t 
(T_ATTRIBUTE) t (T_BAD) t (T_BOOLEAN_AND) t (T_BOOLEAN_OR) t (T_BOOL_CAST) t 
(T_BREAK) t (T_CALL) t (T_CALL [...]
   "The hash-table of valid look-aheads.")
 
-(defconst
+(defvar
   phps-mode-parser--table-terminal-p
   #s(hash-table size 217 test equal rehash-size 1.5 rehash-threshold 0.8125 
data ("!" t "%" t "&" t "(" t ")" t "*" t "+" t "." t "," t "-" t "/" t ":" t 
";" t "<" t "=" t ">" t "?" t "@" t "[" t "]" t "}" t "{" t "^" t "|" t "~" t 
"`" t "'" t "\"" t "$" t PREC_ARROW_FUNCTION t T_ABSTRACT t T_AND_EQUAL t T_ARG 
t T_ARRAY t T_ARRAY_CAST t T_ARROW t T_AS t T_ASSIGN t T_ATTRIBUTE t T_BAD t 
T_BOOL_CAST t T_BOOLEAN_AND t T_BOOLEAN_OR t T_BREAK t T_CALL t T_CALLABLE t 
T_CASE t T_CATCH t T_CLASS [...]
   "The hash-table of valid terminals.")
 
-(defconst
+(defvar
   phps-mode-parser--table-non-terminal-p
   #s(hash-table size 217 test equal rehash-size 1.5 rehash-threshold 0.8125 
data (absolute_trait_method_reference t alt_if_stmt t alt_if_stmt_without_else 
t anonymous_class t argument t argument_list t array_object_dereferencable t 
array_pair t array_pair_list t attribute t attribute_decl t attribute_group t 
attributed_class_statement t attributed_parameter t attributed_statement t 
attributes t backticks_expr t backup_doc_comment t backup_fn_flags t 
backup_lex_pos t callable_expr t calla [...]
   "The hash-table of valid non-terminals.")
 
-(defconst
+(defvar
   phps-mode-parser--table-translations
-  #s(hash-table size 65 test equal rehash-size 1.5 rehash-threshold 0.8125 
data (333 (lambda(data) (message data) nil) 411 (lambda(data) data) 459 
(lambda(data) data) 466 (lambda(data) data) ))
+  #s(hash-table size 65 test equal rehash-size 1.5 rehash-threshold 0.8125 
data ())
   "The hash-table of translations.")
 
-(defconst
+(defvar
   phps-mode-parser-lex-analyzer--get-function
   (lambda (token) (save-current-buffer (set-buffer "*PHPs Lexer*") (let 
((start (car (cdr token))) (end (cdr (cdr token)))) (if (<= end (point-max)) 
(progn (buffer-substring-no-properties start end))))))
   "The lex-analyzer get function.")
 
-(defconst
+(defvar
   phps-mode-parser-lex-analyzer--function
-  (lambda (index) (save-current-buffer (set-buffer "*PHPs Lexer*") (if (= 
(point) index) nil (goto-char index)) (if (< index (point-max)) (progn (if (and 
phps-mode-lexer--generated-new-tokens-index (= 
phps-mode-lexer--generated-new-tokens-index index)) nil 
(phps-mode-lexer--re2c)) (let ((first (car (reverse 
phps-mode-lexer--generated-new-tokens)))) (cond ((and (not first) (not (equal 
index semantic-lex-end-point))) (set (make-local-variable 
'phps-mode-parser-lex-analyzer--index) semantic [...]
+  (lambda (index) (save-current-buffer (set-buffer "*PHPs Lexer*") (if (= 
(point) index) nil (goto-char index)) (if (< index (point-max)) (progn (if (and 
phps-mode-lexer--generated-new-tokens-index (= 
phps-mode-lexer--generated-new-tokens-index index)) nil 
(phps-mode-lexer--re2c)) (let ((first (car (reverse 
phps-mode-lexer--generated-new-tokens)))) (cond ((and (not first) (not (equal 
index semantic-lex-end-point))) (set (make-local-variable 
'phps-mode-parser-lex-analyzer--move-to-index-f [...]
   "The lex-analyzer function.")
 
-(defconst
+(defvar
   phps-mode-parser-lex-analyzer--reset-function
   nil
   "The lex-analyzer reset function.")
 
-(defconst
+(defvar
   phps-mode-parser--e-identifier
   '%empty
   "The e-identifier.")
 
-(defconst
+(defvar
   phps-mode-parser--eof-identifier
   '$
   "The end-of-file-identifier.")
 
-(defconst
+(defvar
   phps-mode-parser--look-ahead-number
   1
   "The look-ahead number.")
@@ -96,6 +96,11 @@
   0
   "The current index of the lex-analyzer.")
 
+(defvar-local
+  phps-mode-parser-lex-analyzer--move-to-index-flag
+  nil
+  "Non-nil means move index to value.")
+
 
 ;;; Functions:
 
@@ -149,10 +154,21 @@
             k)
       (condition-case error
           (progn
+            (setq-local
+              phps-mode-parser-lex-analyzer--move-to-index-flag
+              nil)
             (let ((next-look-ahead
                    (funcall
                     phps-mode-parser-lex-analyzer--function
                     index)))
+                  (if phps-mode-parser-lex-analyzer--move-to-index-flag
+                    (progn
+                     (setq-local
+                      phps-mode-parser-lex-analyzer--index
+                      phps-mode-parser-lex-analyzer--move-to-index-flag)
+                    (setq
+                     index
+                     phps-mode-parser-lex-analyzer--index))
               (if next-look-ahead
                   (progn
                     (unless (listp (car next-look-ahead))
@@ -166,7 +182,7 @@
                         (setq index (cdr (cdr next-look-ahead-item))))))
                 (push (list phps-mode-parser--eof-identifier) look-ahead)
                 (setq look-ahead-length (1+ look-ahead-length))
-                (setq index (1+ index)))))
+                (setq index (1+ index))))))
         (error
          (error
           "Lex-analyze failed to peek next look-ahead at %s, error: %s"
@@ -177,15 +193,23 @@
 (defun
   phps-mode-parser-lex-analyzer--pop-token ()
   "Pop next token via lex-analyzer."
-  (let ((iteration 0)
+  (let ((continue t)
         (tokens))
-    (while (< iteration 1)
+    (while continue
       (condition-case error
           (progn
+            (setq-local
+              phps-mode-parser-lex-analyzer--move-to-index-flag
+              nil)
             (let ((token
                    (funcall
                     phps-mode-parser-lex-analyzer--function
                     phps-mode-parser-lex-analyzer--index)))
+              (if phps-mode-parser-lex-analyzer--move-to-index-flag
+                  (progn
+                    (setq-local
+                     phps-mode-parser-lex-analyzer--index
+                     phps-mode-parser-lex-analyzer--move-to-index-flag))
               (when token
                 (unless (listp (car token))
                   (setq token (list token)))
@@ -193,12 +217,16 @@
                   (setq
                    phps-mode-parser-lex-analyzer--index
                    (cdr (cdr first-token)))
-                  (push first-token tokens)))))
+                  (push 
+                   first-token 
+                   tokens)))
+                (setq
+                 continue
+                 nil))))
         (error (error
                 "Lex-analyze failed to pop token at %s, error: %s"
                 phps-mode-parser-lex-analyzer--index
-                (car (cdr error)))))
-      (setq iteration (1+ iteration)))
+                (car (cdr error))))))
     (nreverse tokens)))
 
 
diff --git a/test/phps-mode-test-parser.el b/test/phps-mode-test-parser.el
index 95e3537..fd00015 100644
--- a/test/phps-mode-test-parser.el
+++ b/test/phps-mode-test-parser.el
@@ -153,7 +153,7 @@
    (lambda()
      (should
       (equal
-       '(nil ("echo" nil ";"))
+       '(nil ("echo" "'hello'" ";"))
        (phps-mode-parser-translate)))))
 
   (message "\n-- Ran tests for parser boundaries. --"))