[elpa] externals/parser-generator 9073a387ba 16/19: More fixes for state-based lexer

[Christian Johansson]

branch: externals/parser-generator
commit 9073a387ba81a02a0254e35d0e09c20e63a701d2
Author: Christian Johansson <christian@cvj.se>
Commit: Christian Johansson <christian@cvj.se>

    More fixes for state-based lexer
---
 TODO.md                                    |  8 +++-----
 docs/Lexical-Analysis.md                   |  8 ++++----
 parser-generator-lex-analyzer.el           |  2 +-
 test/parser-generator-lex-analyzer-test.el | 10 +++++-----
 test/parser-generator-lr-test.el           |  2 +-
 5 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/TODO.md b/TODO.md
index b72f9c8b94..65054c83de 100644
--- a/TODO.md
+++ b/TODO.md
@@ -11,11 +11,9 @@ Functions (with validations) to set global variables:
 ## Lex-analyzer
 
 * State-based lex-analyzer
-* Added unit test for state-based lex-analyzer
-* Change lex-function return value count
-* Verify that parser-generator-lex-analyzer--index is used in exported 
lex-analyzers
-* Verify that parser-generator-lex-analyzer--state-init is used in exported 
lex-analyzers
-* Use buffer when lexing in exported parsers as well
+** Verify that parser-generator-lex-analyzer--index is used in exported 
lex-analyzers
+** Verify that parser-generator-lex-analyzer--state-init is used in exported 
lex-analyzers
+** Use buffered lexer when lexing in exported parsers as well
 
 ## LR-Parser
 
diff --git a/docs/Lexical-Analysis.md b/docs/Lexical-Analysis.md
index 842c19a02f..aae2dacbd0 100644
--- a/docs/Lexical-Analysis.md
+++ b/docs/Lexical-Analysis.md
@@ -4,7 +4,7 @@ Set lexical analysis function by setting variable 
`parser-generator-lex-analyzer
 
 The lexical analysis is internally indexed on a local variable 
`parser-generator-lex-analyzer--index` and has it optional state in the local 
variable `parser-generation-lex-analyzer--state`. The initial values for the 
index and state can be set in variables 
`parser-generation-lex-analyzer--index-init` and 
`parser-generator-lex-analyzer--state-init`.
 
-All parsers expect a list as response from lexical-analysis, the first item in 
the list should be a list of one or more tokens. The second is "move 
index"-flag, if it is non-nil it is expected to be a integer representing the 
index to temporarily move the index to and perform a new lex. Third item is not 
used. The fourth item is the new state after the lex.
+All parsers expect a list as response from the lexical-analysis, the first 
item in the list should be a list of one or more tokens. The second is "move 
index"-flag, if it is non-nil it is expected to be a integer representing the 
index to temporarily move the index to and perform a new lex at. The third item 
is the new state after the lex. Return values 2 and 3 are optional.
 
 To enable exporting, the functions need to be specified in a way that the 
entire body is within the same block, do that using `(let)` or `(progn)` for 
example.
 
@@ -21,7 +21,7 @@ To enable exporting, the functions need to be specified in a 
way that the entire
                (< (1- index) max-index))
          (push (nth (1- index) string) tokens)
          (setq index (1+ index)))
-       (list tokens nil nil nil))))
+       (list tokens))))
 ```
 
 ## Token
@@ -62,7 +62,7 @@ Returns the look-ahead number of next terminals in stream, if 
end of stream is r
          (setq new-index (cdr (cdr (nth (1- index) string))))
          (push next-token tokens)
          (setq index (1+ index)))
-       (list (nreverse tokens) nil nil nil))))
+       (list (nreverse tokens)))))
 (parser-generator-lex-analyzer--reset)
 
 (setq parser-generator--look-ahead-number 1)
@@ -104,7 +104,7 @@ Returns the next token in stream and moves the lexical 
analyzer index one point
          (setq new-index (cdr (cdr (nth (1- index) string))))
          (push (nth (1- index) string) tokens)
          (setq index (1+ index)))
-       (list (nreverse tokens) nil nil nil))))
+       (list (nreverse tokens)))))
 (parser-generator-lex-analyzer--reset)
 
 (setq parser-generator--look-ahead-number 1)
diff --git a/parser-generator-lex-analyzer.el b/parser-generator-lex-analyzer.el
index 52a2e841a9..474139ccbe 100644
--- a/parser-generator-lex-analyzer.el
+++ b/parser-generator-lex-analyzer.el
@@ -212,7 +212,7 @@
                      (move-to-index-flag
                       (nth 1 result-list))
                      (new-state
-                      (nth 3 result-list)))
+                      (nth 2 result-list)))
                 (if move-to-index-flag
                     (progn
                       (setq
diff --git a/test/parser-generator-lex-analyzer-test.el 
b/test/parser-generator-lex-analyzer-test.el
index eac8a1eb69..df3f674671 100644
--- a/test/parser-generator-lex-analyzer-test.el
+++ b/test/parser-generator-lex-analyzer-test.el
@@ -38,7 +38,7 @@
          (setq new-index (cdr (cdr (nth (1- index) string))))
          (push next-token tokens)
          (setq index (1+ index)))
-       (list (nreverse tokens) nil new-index nil))))
+       (list (nreverse tokens)))))
   (should-error
    (parser-generator-lex-analyzer--peek-next-look-ahead))
   (parser-generator-lex-analyzer--reset)
@@ -84,7 +84,7 @@
          (setq new-index (cdr (cdr (nth (1- index) string))))
          (push next-token tokens)
          (setq index (1+ index)))
-       (list (nreverse tokens) nil new-index nil))))
+       (list (nreverse tokens)))))
 
   (parser-generator-lex-analyzer--reset)
   (should-error
@@ -132,7 +132,7 @@
            (setq index (1+ index)))
           (t
            (error "Invalid state: %s" state))))
-       (list (nreverse tokens) nil nil new-state))))
+       (list (nreverse tokens) nil new-state))))
   (parser-generator-lex-analyzer--reset)
   (should
    (equal
@@ -167,7 +167,7 @@
          (push (nth (1- index) string) tokens)
          (setq new-index (cdr (cdr (nth (1- index) string))))
          (setq index (1+ index)))
-       (list (nreverse tokens) nil new-index nil))))
+       (list (nreverse tokens)))))
   (should-error
    (parser-generator-lex-analyzer--pop-token))
 
@@ -231,7 +231,7 @@
            (setq index (1+ index)))
           (t
            (error "Invalid state: %s" state))))
-       (list (nreverse tokens) nil nil new-state))))
+       (list (nreverse tokens) nil new-state))))
   (parser-generator-lex-analyzer--reset)
   (should
    (equal
diff --git a/test/parser-generator-lr-test.el b/test/parser-generator-lr-test.el
index 8941bf9d32..687a493a19 100644
--- a/test/parser-generator-lr-test.el
+++ b/test/parser-generator-lr-test.el
@@ -353,7 +353,7 @@
                 token
                 `(,symbol ,(match-beginning 0) . ,(match-end 0)))))
             (t (error "Unexpected input at %d!" index)))
-           (list token nil nil nil) )))))
+           (list token))))))
 
   (setq
    parser-generator-lex-analyzer--get-function