Please, can anyone comment on this? Is this possibly a bug? Can you please check the grammar, if valid (from the java language perspective) identifiers with non-usascii characters are tokenized correctly?

Thanks,

Holger

the following patches fix the problem:

--- javassist-3.0/src/main/javassist/preproc/Compiler.java 2005-01-18 15:53:48.000000000 +0100
+++ javassist-3.0_patched/src/main/javassist/preproc/Compiler.java 2005-05-19 12:07:20.038916592 +0200
@@ -199,8 +199,7 @@
 throws IOException
 {
 int c = skipSpaces(reader, ' ');
- while ('A' <= c && c <= 'Z' || 'a' <= c && c <= 'z'
- || '0' <= c && c <= '9' || c == '.' || c == '_') {
+ while (Character.isJavaIdentifierPart((char)c)) {
 buf.append((char)c);
 c = reader.read();
 }

--- javassist-3.0/src/main/javassist/compiler/Lex.java 2005-01-18 15:53:48.000000000 +0100
+++ javassist-3.0_patched/src/main/javassist/compiler/Lex.java 2005-05-19 13:09:20.419332800 +0200
@@ -133,8 +133,7 @@
 return readSeparator('.');
 }
 }
- else if ('A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || c == '_'
- || c == '$')
+ else if (Character.isJavaIdentifierStart((char)c))
 return readIdentifier(c, token);
 else
 return readSeparator(c);
@@ -434,8 +433,7 @@
 do {
 tbuf.append((char)c);
 c = getc();
- } while ('A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || c == '_'
- || c == '$' || '0' <= c && c <= '9');
+ } while (Character.isJavaIdentifierPart((char)c));

 ungetc(c);