java.io.StreamTokenizer behaves differently from Sun's implementation

[Ito Kazumitsu]

HI,

It has been discussed in the kaffe mailing list [1] that Sun's
implementation of java.io.StreamTokenizer does not respect
the obsoltete document JLS 1st ed. and its working specificatin
is unknown.  Kaffe's java.io.StreamTokenizer has been modified
so that it simulates Sun's current implementation [2].

I checked GNU Classpath's java.io.StreamTokenizer to find
that it behaves differently from Sun's implementation.

Attached is my test program that generates various patterns
of test cases and prints the results of them.

If GNU Classpath is to simulate Sun's implementation,  I hope
these pieces of information can be of some help.

[1] http://www.kaffe.org/pipermail/kaffe/2003-June/042843.html
[2] 
http://www.kaffe.org/cgi-bin/viewcvs.cgi/kaffe/libraries/javalib/java/io/StreamTokenizer.java

Attached program:
bash$ cat StreamTokenizerTest2.java
import java.io.*;
public class StreamTokenizerTest2 {

  private static String testString;
  private static String testChar;

  public static void main(String[] args) throws Exception {
      testString = args[0];
      testChar = args[1];
      String[] a = new String[] {"S", "C", "Q", "W", "N"};
      for (int i=1; i<=5; i++) {
          Permutation.generate(a, i, new MainHandler());
      }
  }

  private static class MainHandler extends Permutation.Handler {
    public void doit(Object[] array) {
        try {
            System.out.print(testString + " " + testChar + " ");
            for (int i=0; i<array.length; i++) {
                System.out.print(array[i] + " ");
            }
            System.out.println();
            test(array);
        }
        catch (Exception e) {
            System.err.println(e);
        }
    }
  }

  private static void test(Object[] args) throws Exception {
    StreamTokenizer tok = new StreamTokenizer(new StringReader(testString));
    tok.resetSyntax();
    int c = testChar.charAt(0);
    for (int i=0; i<args.length; i++) {
       if (args[i].equals("S")) tok.whitespaceChars(c, c);
       else if (args[i].equals("C")) tok.commentChar(c);
       else if (args[i].equals("Q")) tok.quoteChar(c);
       else if (args[i].equals("N")) tok.parseNumbers();
       else if (args[i].equals("W")) tok.wordChars(c, c);
    }
    while (true) {
      int t = tok.nextToken();
      if (t == StreamTokenizer.TT_NUMBER) {
          System.out.println(tok.nval + ": " + t);
      }
      else {
          System.out.println(tok.sval + ": " + t);
      }
      if (t == StreamTokenizer.TT_EOF) break;
    }
  }
}
bash$ cat Permutation.java
public class Permutation {

    public static void generate(Object[] array, int n, Handler h) {
        int l = array.length;
        if (n == 1) {
            for (int i = 0; i < l; i++) {
                h.doit(new Object[] {array[i]});
            }
            return;
        }
        final int N = n;
        final Handler H = h;
        for (int i = 0; i < l; i++) {
            final Object OBJ = array[i];
            Object[] a1 = new Object[l - 1];
            System.arraycopy(array, 0, a1, 0, i);
            System.arraycopy(array, i+1, a1, i, l-i-1);
            generate(a1, n-1, new Handler() {
                public void doit(Object[] a2) {
                    Object[] a3 = new Object[N];
                    System.arraycopy(a2, 0, a3, 1, N-1);
                    a3[0] = OBJ;
                    H.doit(a3);
                }
            });
        }
        return;
    }

    public static class Handler {
        public void doit(Object[] array) {}
    }

}
bash$ java StreamTokenizerTest2 121 1
121 1 S 
null: 50
null: -1
121 1 C 
null: -1
(snip)
121 1 N W Q S C 
null: -1
121 1 N W Q C S 
21.0: -2
null: -1
bash$