replace regex parsing with linear parsing

langserver/java/TokenList.js

@@ -15,6 +15,7 @@ class TokenList {
         this.inc();
         /** @type {ParseProblem[]} */
         this.problems = [];
+        this.marks = [];
     }
 
     inc() {
@@ -25,6 +26,19 @@ class TokenList {
             }
         }
     }
+    
+    mark() {
+        this.marks.unshift(this.idx);
+    }
+
+    /**
+     * Returns the array of tokens from the last mark() point, trimming any trailing whitespace tokens
+     */
+    markEnd() {
+        let i = this.idx;
+        while (this.tokens[--i].kind === 'wsc') { }
+        return this.tokens.slice(this.marks.shift(), i + 1);
+    }
 
     /**
      * Token lookahead. The current token is unaffected by this method.
@@ -43,16 +57,25 @@ class TokenList {
         return token;
     }
 
+    /**
+     * Check if the current token matches the specified value, returns and consumes it
+     * @param {string} value
+     */
+    getIfValue(value) {
+        const token = this.current;
+        if (token && token.value === value) {
+            this.inc();
+            return token;
+        }
+        return null;
+    }
+
     /**
      * Check if the current token matches the specified value and consumes it
      * @param {string} value
      */
     isValue(value) {
-        if (this.current && this.current.value === value) {
+        return this.getIfValue(value) !== null;
-            this.inc();
-            return true;
-        }
-        return false;
     }
 
     /**

langserver/java/body-parser3.js

@@ -6,15 +6,17 @@
  */
 const { JavaType, CEIType, PrimitiveType, ArrayType, UnresolvedType, NullType, WildcardType, TypeVariableType,
     TypeVariable, InferredTypeArgument, Field, Method, ReifiedMethod, Parameter, Constructor, signatureToType } = require('java-mti');
-const { SourceType, SourceTypeIdent, SourceField, SourceMethod, SourceConstructor, SourceInitialiser, SourceParameter, SourceAnnotation } = require('./source-types2');
+const { SourceType, SourceTypeIdent, SourceField, SourceMethod, SourceConstructor, SourceInitialiser, SourceParameter, SourceAnnotation,
+    SourceUnit, SourcePackage, SourceImport } = require('./source-types2');
 const ResolvedImport = require('./parsetypes/resolved-import');
 const ParseProblem = require('./parsetypes/parse-problem');
-const { getOperatorType, Token } = require('./tokenizer');
+const { getOperatorType, tokenize, Token } = require('./tokenizer');
 const { resolveTypeOrPackage, resolveNextTypeOrPackage } = require('./type-resolver');
 const { genericTypeArgs, typeIdent, typeIdentList } = require('./typeident');
 const { TokenList } = require("./TokenList");
 const { AnyMethod, AnyType, AnyValue, ArrayElement, ArrayLiteral, ConstructorCall, Label, LiteralNumber, LiteralValue, Local,
     MethodCall, MethodDeclarations, ResolvedIdent, TernaryValue, Value } = require("./body-types");
+const { resolveImports, resolveSingleImport } = require('../java/import-resolver');
 
 /**
  * @typedef {SourceMethod|SourceConstructor|SourceInitialiser} SourceMC
@@ -76,6 +78,158 @@ function addproblem(tokens, problem) {
     tokens.problems.push(problem);
 }
 
+/**
+ * @param {string} source
+ * @param {Map<string,JavaType>} typemap 
+ */
+function parse(source, typemap) {
+    const unit = new SourceUnit();
+    let tokens;
+    try {
+        console.time('tokenize');
+        tokens = new TokenList(tokenize(source));
+        console.timeEnd('tokenize');
+        console.time('parse');
+        parseUnit(tokens, unit, typemap);
+        console.timeEnd('parse');
+    } catch(err) {
+        if (tokens) {
+            addproblem(tokens, ParseProblem.Error(tokens.current, `Parse failed: ${err.message}`));
+        } else {
+            console.log(`Parse failed: ${err.message}`);
+        }
+    }
+
+    return unit;
+}
+
+/**
+ * @param {TokenList} tokens
+ * @param {SourceUnit} unit
+ * @param {Map<string,JavaType>} typemap 
+ */
+function parseUnit(tokens, unit, typemap) {
+    let package_name = '';
+    // init resolved imports with java.lang.*
+    let resolved_imports = resolveImports(typemap, [], [], null).resolved.slice();
+    // retrieve the implicit imports
+    while (tokens.current) {
+        let modifiers = [], annotations = [];
+        for (;tokens.current;) {
+            if (tokens.current.kind === 'modifier') {
+                modifiers.push(tokens.current);
+                tokens.inc();
+                continue;
+            }
+            if (tokens.current.value === '@') {
+                tokens.inc().value === 'interface'
+                    ? sourceType(modifiers, tokens, package_name, '@interface', unit, resolved_imports, typemap)
+                    : annotations.push(annotation(tokens, null, unit.imports, typemap));
+                continue;
+            }
+            break;
+        }
+        if (!tokens.current) {
+            break;
+        }
+        switch (tokens.current.value) {
+            case 'package':
+                if (unit.package_ !== null) {
+                    addproblem(tokens, ParseProblem.Error(tokens.current, `Multiple package declarations`));
+                }
+                if (modifiers[0]) {
+                    addproblem(tokens, ParseProblem.Error(tokens.current, `Unexpected modifier: ${modifiers[0].source}`));
+                }
+                const pkg = packageDeclaration(tokens);
+                if (!package_name) {
+                    unit.package_ = pkg;
+                    package_name = pkg.name;
+                }
+                continue;
+            case 'import':
+                if (modifiers[0]) {
+                    addproblem(tokens, ParseProblem.Error(tokens.current, `Unexpected modifier: ${modifiers[0].source}`));
+                }
+                const imprt = importDeclaration(tokens, typemap);
+                unit.imports.push(imprt);
+                if (imprt.resolved) {
+                    resolved_imports.push(imprt.resolved);
+                }
+                continue;
+        }
+        if (tokens.current.kind === 'type-kw') {
+            sourceType(modifiers, tokens, package_name, tokens.current.value, unit, resolved_imports, typemap);
+            continue;
+        }
+        addproblem(tokens, ParseProblem.Error(tokens.current, 'Type declaration expected'));
+        // skip until something we recognise
+        while (tokens.current) {
+            if (/@|package|import/.test(tokens.current.value) || /modifier|type-kw/.test(tokens.current.kind)) {
+                break;
+            }
+            tokens.inc();
+        }
+    }
+    return unit;
+}
+
+/**
+ * @param {TokenList} tokens 
+ */
+function packageDeclaration(tokens) {
+    tokens.mark();
+    tokens.expectValue('package');
+    const pkg_name_parts = [];
+    for (;;) {
+        let name = tokens.current;
+        if (!tokens.isKind('ident')) {
+            name = null;
+            addproblem(tokens, ParseProblem.Error(tokens.current, `Package identifier expected`));
+        }
+        if (name) pkg_name_parts.push(name.value);
+        if (tokens.isValue('.')) {
+            continue;
+        }
+        const decl_tokens = tokens.markEnd();
+        semicolon(tokens);
+        return new SourcePackage(decl_tokens, pkg_name_parts.join('.'));
+    }
+}
+
+/**
+ * @param {TokenList} tokens 
+ * @param {Map<string,JavaType>} typemap 
+ */
+function importDeclaration(tokens, typemap) {
+    tokens.mark();
+    tokens.expectValue('import');
+    const static_token = tokens.getIfValue('static');
+    let asterisk_token = null;
+    const pkg_name_parts = [];
+    for (;;) {
+        let name = tokens.current;
+        if (!tokens.isKind('ident')) {
+            name = null;
+            addproblem(tokens, ParseProblem.Error(tokens.current, `Package identifier expected`));
+        }
+        if (name) {
+            pkg_name_parts.push(name);
+        }
+        if (tokens.isValue('.')) {
+            if (!(asterisk_token = tokens.getIfValue('*'))) {
+                continue;
+            }
+        }
+        const decl_tokens = tokens.markEnd();
+        semicolon(tokens);
+
+        const pkg_name = pkg_name_parts.map(x => x.source).join('.');
+        const resolved = resolveSingleImport(typemap, pkg_name, !!static_token, !!asterisk_token, 'import');
+
+        return new SourceImport(decl_tokens, pkg_name_parts, pkg_name, static_token, asterisk_token, resolved);
+    }
+}
+
 /**
  * @param {MethodDeclarations} mdecls 
  * @param {Local[]} new_locals 
@@ -106,7 +260,7 @@ function statement(tokens, mdecls, method, imports, typemap) {
                 tokens.inc();
                 continue;
             case 'type-kw':
-                sourceType(modifiers.splice(0,1e9), tokens, mdecls, method, imports, typemap);
+                sourceType(modifiers.splice(0,1e9), tokens, method, tokens.current.value, mdecls, imports, typemap);
                 continue;
         }
         break;
@@ -239,36 +393,49 @@ class AssertStatement extends Statement {
 /**
 * @param {Token[]} modifiers
 * @param {TokenList} tokens 
-* @param {MethodDeclarations} mdecls
+* @param {Scope|string} scope_or_pkgname
-* @param {Scope} scope 
+* @param {string} typeKind
+* @param {{types:SourceType[]}} owner
 * @param {ResolvedImport[]} imports
 * @param {Map<string,JavaType>} typemap 
 */
-function sourceType(modifiers, tokens, mdecls, scope, imports, typemap) {
+function sourceType(modifiers, tokens, scope_or_pkgname, typeKind, owner, imports, typemap) {
-    const scoped_type = scope instanceof SourceType ? scope : scope.owner;
+    let package_name, scope;
-    const type = typeDeclaration(scoped_type.packageName, scope, modifiers, tokens.current, tokens, imports, typemap);
+    if (typeof scope_or_pkgname === 'string') {
-    mdecls.types.push(type);
+        package_name = scope_or_pkgname;
+        scope = null;
+    } else {
+        const scoped_type = scope_or_pkgname instanceof SourceType ? scope_or_pkgname : scope_or_pkgname.owner;
+        package_name = scoped_type.packageName;
+        scope = scope_or_pkgname;
+    }
+    const type = typeDeclaration(package_name, scope, modifiers, typeKind, tokens.current, tokens, imports, typemap);
+    owner.types.push(type);
+    if (!(owner instanceof MethodDeclarations)) {
+        typemap.set(type.shortSignature, type);
+    }
     if (tokens.isValue('extends')) {
-        const extends_types = typeIdentList(tokens, type, imports, typemap);
+        type.extends_types = typeIdentList(tokens, type, imports, typemap);
     }
     if (tokens.isValue('implements')) {
-        const implement_types = typeIdentList(tokens, type, imports, typemap);
+        type.implements_types = typeIdentList(tokens, type, imports, typemap);
     }
     tokens.expectValue('{');
     if (!tokens.isValue('}')) {
-        typeBody(type, tokens, scope, imports, typemap);
+        typeBody(type, tokens, owner, imports, typemap);
+        tokens.expectValue('}');
     }
 }
 
 /**
 * @param {SourceType} type 
 * @param {TokenList} tokens 
-* @param {Scope} scope 
+* @param {{types:SourceType[]}} owner
 * @param {ResolvedImport[]} imports
 * @param {Map<string,JavaType>} typemap 
 */
-function typeBody(type, tokens, scope, imports, typemap) {
+function typeBody(type, tokens, owner, imports, typemap) {
-    while (!tokens.isValue('}')) {
+    for (;;) {
         let modifiers = [], annotations = [];
         while (tokens.current.kind === 'modifier') {
             modifiers.push(tokens.current);
@@ -280,7 +447,7 @@ function typeBody(type, tokens, scope, imports, typemap) {
                 fmc(modifiers, annotations, [], type, tokens, imports, typemap);
                 continue;
             case 'type-kw':
-                sourceType(modifiers, tokens, new MethodDeclarations(), scope, imports, typemap);
+                sourceType(modifiers, tokens, type, tokens.current.value, owner, imports, typemap);
                 continue;
         }
         switch(tokens.current.value) {
@@ -290,12 +457,17 @@ function typeBody(type, tokens, scope, imports, typemap) {
                 continue;
             case '@':
                 tokens.inc().value === 'interface' 
-                    ? annotationTypeDeclaration(type.packageName, type, modifiers.splice(0,1e9), tokens, imports, typemap)
+                    ? sourceType(modifiers, tokens, type, '@interface', owner, imports, typemap)
                     : annotation(tokens, type, imports, typemap);
                 continue;
             case ';':
                 tokens.inc();
                 continue;
+            case '{':
+                initer(tokens, type, modifiers.splice(0,1e9));
+                continue;
+            case '}':
+                return;
         }
         if (!tokens.inc()) {
             break;
@@ -313,8 +485,8 @@ function typeBody(type, tokens, scope, imports, typemap) {
  * @param {Map<string,JavaType>} typemap 
  */
 function fmc(modifiers, annotations, type_variables, type, tokens, imports, typemap) {
-    const decl_type = typeIdent(tokens, type, imports, typemap);
+    let decl_type_ident = typeIdent(tokens, type, imports, typemap);
-    if (decl_type.rawTypeSignature === type.rawTypeSignature) {
+    if (decl_type_ident.resolved.rawTypeSignature === type.rawTypeSignature) {
         if (tokens.current.value === '(') {
             // constructor
             const { parameters, throws, body } = methodDeclaration(type, tokens, imports, typemap);
@@ -329,22 +501,61 @@ function fmc(modifiers, annotations, type_variables, type, tokens, imports, type
         addproblem(tokens, ParseProblem.Error(tokens.current, `Identifier expected`))
     }
     if (tokens.current.value === '(') {
-        const { parameters, throws, body } = methodDeclaration(type, tokens, imports, typemap);
+        const { postnamearrdims, parameters, throws, body } = methodDeclaration(type, tokens, imports, typemap);
-        const method = new SourceMethod(type, modifiers, annotations, new SourceTypeIdent([], decl_type), name, parameters, throws, body);
+        if (postnamearrdims > 0) {
+            decl_type_ident.resolved = new ArrayType(decl_type_ident.resolved, postnamearrdims);
+        }
+        const method = new SourceMethod(type, modifiers, annotations, decl_type_ident, name, parameters, throws, body);
         type.methods.push(method);
     } else {
         if (name) {
             if (type_variables.length) {
                 addproblem(tokens, ParseProblem.Error(tokens.current, `Fields cannot declare type variables`));
             }
-            const locals = var_ident_list(modifiers, decl_type, name, tokens, new MethodDeclarations(), type, imports, typemap);
+            const locals = var_ident_list(modifiers, decl_type_ident, name, tokens, new MethodDeclarations(), type, imports, typemap);
-            const fields = locals.map(l => new SourceField(type, modifiers, new SourceTypeIdent([], l.type), l.decltoken));
+            const fields = locals.map(l => new SourceField(type, modifiers, l.typeIdent, l.decltoken));
             type.fields.push(...fields);
         }
         semicolon(tokens);
     }
 }
 
+/**
+ * 
+ * @param {TokenList} tokens 
+ * @param {SourceType} type 
+ * @param {Token[]} modifiers 
+ */
+function initer(tokens, type, modifiers) {
+    const i = new SourceInitialiser(type, modifiers, skipBody(tokens));
+    type.initers.push(i);
+}
+
+/**
+ * 
+ * @param {TokenList} tokens 
+ */
+function skipBody(tokens) {
+    let body = null;
+    const start_idx = tokens.idx;
+    if (tokens.expectValue('{')) {
+        // skip the method body
+        for (let balance=1; balance;) {
+            switch (tokens.current.value) {
+                case '{': balance++; break;
+                case '}': {
+                    if (--balance === 0) {
+                        body = tokens.tokens.slice(start_idx, tokens.idx + 1);
+                    }
+                    break;
+                }
+            }
+            tokens.inc();
+        }
+    }
+    return body;
+}
+
 /**
  * 
  * @param {TokenList} tokens 
@@ -361,37 +572,27 @@ function annotation(tokens, scope, imports, typemap) {
             tokens.expectValue(')');
         }
     }
-}
+    return new SourceAnnotation(annotation_type);
-
-/**
- * @param {string} package_name
- * @param {Scope} scope
- * @param {Token[]} modifiers
- * @param {TokenList} tokens 
- * @param {ResolvedImport[]} imports
- * @param {Map<string,JavaType>} typemap
- */
-function annotationTypeDeclaration(package_name, scope, modifiers, tokens, imports, typemap) {
-    const type = typeDeclaration(package_name, scope, modifiers, tokens.current, tokens, imports, typemap);
 }
     
 /**
  * @param {string} package_name
  * @param {Scope} scope
  * @param {Token[]} modifiers
+ * @param {string} typeKind
  * @param {Token} kind_token
  * @param {TokenList} tokens 
  * @param {ResolvedImport[]} imports
  * @param {Map<string,JavaType>} typemap
  */
-function typeDeclaration(package_name, scope, modifiers, kind_token, tokens, imports, typemap) {
+function typeDeclaration(package_name, scope, modifiers, typeKind, kind_token, tokens, imports, typemap) {
     let name = tokens.inc();
     if (!tokens.isKind('ident')) {
         name = null;
         addproblem(tokens, ParseProblem.Error(tokens.current, `Type identifier expected`));
         return;
     }
-    const type = new SourceType(package_name, scope, '', modifiers.map(m => m.source), kind_token, name, typemap);
+    const type = new SourceType(package_name, scope, '', modifiers, typeKind, kind_token, name, typemap);
     type.typeVariables = tokens.current.value === '<'
         ? typeVariableList(type, tokens, scope, imports, typemap)
         : [];
@@ -419,7 +620,8 @@ function typeVariableList(owner, tokens, scope, imports, typemap) {
         switch (tokens.current.value) {
             case 'extends':
             case 'super':
-                const type_bounds = typeIdent(tokens, scope, imports, typemap);
+                tokens.inc();
+                const {resolved: type_bounds} = typeIdent(tokens, scope, imports, typemap);
                 bounds.push(new TypeVariable.Bound(owner, type_bounds.typeSignature, type_bounds.typeKind === 'interface'));
                 break;
         }
@@ -448,7 +650,7 @@ function typeVariableList(owner, tokens, scope, imports, typemap) {
  */
 function methodDeclaration(owner, tokens, imports, typemap) {
     tokens.expectValue('(');
-    let parameters = [], throws = [], body = null;
+    let parameters = [], throws = [], postnamearrdims = 0, body = null;
     if (!tokens.isValue(')')) {
         for(;;) {
             const p = parameterDeclaration(owner, tokens, imports, typemap);
@@ -460,28 +662,18 @@ function methodDeclaration(owner, tokens, imports, typemap) {
             break;
         }
     }
+    while (tokens.isValue('[')) {
+        postnamearrdims += 1;
+        tokens.expectValue(']');
+    }
     if (tokens.isValue('throws')) {
         throws = typeIdentList(tokens, owner, imports, typemap);
     }
     if (!tokens.isValue(';')) {
-        const start_idx = tokens.idx;
+        body = skipBody(tokens);
-        if (tokens.expectValue('{')) {
-            // skip the method body
-            for (let balance=1; balance;) {
-                switch (tokens.current.value) {
-                    case '{': balance++; break;
-                    case '}': {
-                        if (--balance === 0) {
-                            body = tokens.tokens.slice(start_idx, tokens.idx + 1);
-                        }
-                        break;
-                    }
-                }
-                tokens.inc();
-            }
-        }
     }
     return {
+        postnamearrdims,
         parameters,
         throws,
         body,
@@ -501,7 +693,7 @@ function parameterDeclaration(owner, tokens, imports, typemap) {
         tokens.inc();
     }
     checkLocalModifiers(tokens, modifiers);
-    let type = typeIdent(tokens, owner, imports, typemap);
+    let type_ident = typeIdent(tokens, owner, imports, typemap);
     const varargs = tokens.isValue('...');
     let name_token = tokens.current;
     if (!tokens.isKind('ident')) {
@@ -514,12 +706,12 @@ function parameterDeclaration(owner, tokens, imports, typemap) {
         tokens.expectValue(']');
     }
     if (postnamearrdims > 0) {
-        type = new ArrayType(type, postnamearrdims);
+        type_ident.resolved = new ArrayType(type_ident.resolved, postnamearrdims);
     }
     if (varargs) {
-        type = new ArrayType(type, 1);
+        type_ident.resolved = new ArrayType(type_ident.resolved, 1);
     }
-    return new SourceParameter(modifiers, new SourceTypeIdent([], type), varargs, name_token);
+    return new SourceParameter(modifiers, type_ident, varargs, name_token);
 }
 
 /**
@@ -1087,7 +1279,7 @@ function var_decl(mods, tokens, mdecls, scope, imports, typemap) {
 /**
  * 
  * @param {Token[]} mods 
- * @param {JavaType} type 
+ * @param {SourceTypeIdent} type 
  * @param {Token} first_ident 
  * @param {TokenList} tokens 
  * @param {MethodDeclarations} mdecls 
@@ -1150,7 +1342,7 @@ function expression_or_var_decl(tokens, mdecls, scope, imports, typemap) {
 
     // if theres at least one type followed by an ident, we assume a variable declaration
     if (matches.types[0] && tokens.current.kind === 'ident') {
-        return var_ident_list([], matches.types[0], null, tokens, mdecls, scope, imports, typemap);
+        return var_ident_list([], new SourceTypeIdent(matches.tokens, matches.types[0]), null, tokens, mdecls, scope, imports, typemap);
     }
 
     return matches;
@@ -1220,6 +1412,7 @@ const operator_precedences = {
  * @param {Map<string,JavaType>} typemap 
  */
 function expression(tokens, mdecls, scope, imports, typemap, precedence_stack = [13]) {
+    tokens.mark();
     /** @type {ResolvedIdent} */
     let matches = qualifiedTerm(tokens, mdecls, scope, imports, typemap);
 
@@ -1251,6 +1444,7 @@ function expression(tokens, mdecls, scope, imports, typemap, precedence_stack =
         }
     }
 
+    matches.tokens = tokens.markEnd();
     return matches;
 }
 
@@ -2073,7 +2267,7 @@ function rootTerm(tokens, mdecls, scope, imports, typemap) {
 function newTerm(tokens, mdecls, scope, imports, typemap) {
     tokens.expectValue('new');
     const type_start_token = tokens.idx;
-    const ctr_type = typeIdent(tokens, scope, imports, typemap, false);
+    const { resolved: ctr_type } = typeIdent(tokens, scope, imports, typemap, false);
     if (ctr_type instanceof AnyType) {
         const toks = tokens.tokens.slice(type_start_token, tokens.idx);
         addproblem(tokens, ParseProblem.Error(toks, `Unresolved type: '${toks.map(t => t.source).join('')}'`));
@@ -2574,4 +2768,5 @@ function findIdentifier(ident, mdecls, scope, imports, typemap) {
 
 exports.addproblem = addproblem;
 exports.parseBody = parseBody;
+exports.parse = parse;
 exports.flattenBlocks = flattenBlocks;

langserver/java/body-types.js

@@ -15,6 +15,8 @@ class ResolvedIdent {
         this.methods = methods;
         this.types = types;
         this.package_name = package_name;
+        /** @type {Token[]} */
+        this.tokens = [];
     }
 }
 
@@ -62,17 +64,23 @@ class Local {
      * @param {Token[]} modifiers 
      * @param {string} name 
      * @param {Token} decltoken 
-     * @param {JavaType} type 
+     * @param {import('./source-type').SourceTypeIdent} typeIdent 
-     * @param {JavaType} type 
      * @param {number} postnamearrdims 
      */
-    constructor(modifiers, name, decltoken, type, postnamearrdims) {
+    constructor(modifiers, name, decltoken, typeIdent, postnamearrdims) {
         this.finalToken = modifiers.find(m => m.source === 'final') || null;
         this.name = name;
         this.decltoken = decltoken;
-        this.type = postnamearrdims > 0 ? new ArrayType(type, postnamearrdims): type;
+        if (postnamearrdims > 0) {
+            typeIdent.resolved = new ArrayType(typeIdent.resolved, postnamearrdims);
+        }
+        this.typeIdent = typeIdent;
         this.init = null;
     }
+
+    get type() {
+        return this.typeIdent.resolved;
+    }
 }
 
 class Label {

langserver/java/import-resolver.js

@@ -1,3 +1,4 @@
+
 const { ImportBlock } = require('./parser9');
 const ResolvedImport = require('./parsetypes/resolved-import');
 
@@ -29,6 +30,25 @@ function resolveImportTypes(typenames, import_decl) {
     return fetchImportedTypes(typenames, import_decl.name, import_decl.isDemandLoad);
 }
 
+/**
+ * Resolve a single parsed import
+ * 
+ * @param {Map<string, import('java-mti').JavaType>} typemap
+ * @param {string} dotted_name
+ * @param {boolean} is_static
+ * @param {boolean} on_demand
+ * @param {'owner-package'|'import'|'implicit-import'} import_kind
+ */
+function resolveSingleImport(typemap, dotted_name, is_static, on_demand, import_kind) {
+    // construct the list of typenames
+    const typenames = [...typemap.keys()].join('\n');
+    const matches = fetchImportedTypes(typenames, dotted_name, on_demand);
+    if (matches) {
+        return new ResolvedImport(null, matches, typemap, import_kind);
+    }
+    return null;
+}
+
 /**
  * Resolve a set of imports for a module.
  * 
@@ -106,5 +126,6 @@ function resolveImports(androidLibrary, sourceTypes, imports, package_name, impl
 
 module.exports = {
     resolveImports,
+    resolveSingleImport,
     ResolvedImport,
 }

langserver/java/source-type.js

@@ -308,4 +308,6 @@ exports.SourceConstructor = source_types.SourceConstructor;
 exports.DefaultConstructor = DefaultConstructor;
 exports.SourceInitialiser = source_types.SourceInitialiser;
 exports.SourceAnnotation = source_types.SourceAnnotation;
-exports.ResolvableType = ResolvableType;
+exports.SourceUnit = source_types.SourceUnit;
+exports.SourcePackage = source_types.SourcePackage;
+exports.SourceImport = source_types.SourceImport;

langserver/java/source-types2.js

@@ -25,16 +25,18 @@ class SourceType extends CEIType {
      * @param {string} packageName
      * @param {SourceType|SourceMethod|SourceConstructor|SourceInitialiser} outer_scope
      * @param {string} docs 
-     * @param {string[]} modifiers 
+     * @param {Token[]} modifiers 
+     * @param {string} typeKind 
      * @param {Token} kind_token 
      * @param {Token} name_token 
      */
-    constructor(packageName, outer_scope, docs, modifiers, kind_token, name_token, typemap) {
+    constructor(packageName, outer_scope, docs, modifiers, typeKind, kind_token, name_token, typemap) {
         // @ts-ignore
-        super(generateShortSignature(outer_scope || packageName, name_token), kind_token.source, modifiers, docs);
+        super(generateShortSignature(outer_scope || packageName, name_token), typeKind, modifiers.map(m => m.source), docs);
         super.packageName = packageName;
+        this.modifierTokens = modifiers;
         this.kind_token = kind_token;
-        this.name_token = name_token;
+        this.nameToken = name_token;
         this.scope = outer_scope;
         this.typemap = typemap;
         /**
@@ -57,7 +59,7 @@ class SourceType extends CEIType {
     }
 
     get supers() {
-        const supertypes = [...this.extends_types, ...this.implements_types].map(x => x.type);
+        const supertypes = [...this.extends_types, ...this.implements_types].map(x => x.resolved);
         if (this.typeKind === 'enum') {
             /** @type {CEIType} */
             const enumtype = this.typemap.get('java/lang/Enum');
@@ -76,8 +78,8 @@ class SourceTypeIdent {
      * @param {JavaType} type 
      */
     constructor(tokens, type) {
-        this.typeTokens = tokens;
+        this.tokens = tokens;
-        this.type = type;
+        this.resolved = type;
     }
 }
 
@@ -85,13 +87,14 @@ class SourceField extends Field {
     /**
      * @param {SourceType} owner 
      * @param {Token[]} modifiers 
-     * @param {SourceTypeIdent} field_type 
+     * @param {SourceTypeIdent} field_type_ident 
      * @param {Token} name_token 
      */
-    constructor(owner, modifiers, field_type, name_token) {
+    constructor(owner, modifiers, field_type_ident, name_token) {
         super(modifiers.map(m => m.value), '');
         this.owner = owner;
-        this.fieldType = field_type;
+        this.modifierTokens = modifiers;
+        this.fieldTypeIdent = field_type_ident;
         this.nameToken = name_token;
     }
 
@@ -100,7 +103,7 @@ class SourceField extends Field {
     }
 
     get type() {
-        return this.fieldType.type;
+        return this.fieldTypeIdent.resolved;
     }
 }
 
@@ -115,6 +118,7 @@ class SourceConstructor extends Constructor {
     constructor(owner, modifiers, parameters, throws, body) {
         super(owner, modifiers.map(m => m.value), '');
         this.owner = owner;
+        this.modifierTokens = modifiers;
         this.sourceParameters = parameters;
         this.throws = throws;
         this.body = body;
@@ -158,7 +162,9 @@ class SourceMethod extends Method {
         super(owner, name_token ? name_token.value : '', modifiers.map(m => m.value), '');
         this.annotations = annotations;
         this.owner = owner;
-        this.methodTypeIdent = method_type_ident;
+        this.modifierTokens = modifiers;
+        this.returnTypeIdent = method_type_ident;
+        this.nameToken = name_token;
         this.sourceParameters = parameters;
         this.throws = throws;
         this.body = body;
@@ -183,7 +189,7 @@ class SourceMethod extends Method {
      * @returns {JavaType}
      */
     get returnType() {
-        return this.methodTypeIdent.type;
+        return this.returnTypeIdent.resolved;
     }
 }
 
@@ -197,6 +203,7 @@ class SourceInitialiser extends MethodBase {
         super(owner, modifiers.map(m => m.value), '');
         /** @type {SourceType} */
         this.owner = owner;
+        this.modifierTokens = modifiers;
         this.body = body;
     }
 
@@ -220,11 +227,15 @@ class SourceParameter extends Parameter {
      * @param {Token} name_token 
      */
     constructor(modifiers, typeident, varargs, name_token) {
-        super(name_token ? name_token.value : '', typeident.type, varargs);
+        super(name_token ? name_token.value : '', typeident.resolved, varargs);
-        this.name_token = name_token;
+        this.nameToken = name_token;
-        this.modifiers = modifiers;
+        this.modifierTokens = modifiers;
         this.paramTypeIdent = typeident;
     }
+
+    get type() {
+        return this.paramTypeIdent.resolved;
+    }
 }
 
 class SourceAnnotation {
@@ -234,6 +245,58 @@ class SourceAnnotation {
     constructor(typeident) {
         this.annotationTypeIdent = typeident;
     }
+
+    get type() {
+        return this.annotationTypeIdent.resolved;
+    }
+}
+
+class SourcePackage {
+    /**
+     * @param {Token[]} tokens 
+     * @param {string} name 
+     */
+    constructor(tokens, name) {
+        this.tokens = tokens;
+        this.name = name;
+    }
+}
+
+class SourceImport {
+
+    /**
+     * @param {Token[]} tokens 
+     * @param {Token[]} name_tokens 
+     * @param {string} pkg_name 
+     * @param {Token} static_token 
+     * @param {Token} asterisk_token 
+     * @param {import('./parsetypes/resolved-import')} resolved 
+     */
+    constructor(tokens, name_tokens, pkg_name, static_token, asterisk_token, resolved) {
+        this.tokens = tokens;
+        this.nameTokens = name_tokens;
+        this.package_name = pkg_name;
+        this.staticToken = static_token;
+        this.asteriskToken = asterisk_token;
+        this.resolved = resolved;
+    }
+
+    get isDemandLoad() {
+        return !!this.asteriskToken;
+    }
+
+    get isStatic() {
+        return !!this.staticToken;
+    }
+}
+
+class SourceUnit {
+    /** @type {SourcePackage} */
+    package_ = null;
+    /** @type {SourceImport[]} */
+    imports = [];
+    /** @type {SourceType[]} */
+    types = [];
 }
 
 exports.SourceType = SourceType;
@@ -244,3 +307,6 @@ exports.SourceParameter = SourceParameter;
 exports.SourceConstructor = SourceConstructor;
 exports.SourceInitialiser = SourceInitialiser;
 exports.SourceAnnotation = SourceAnnotation;
+exports.SourceUnit = SourceUnit;
+exports.SourcePackage = SourcePackage;
+exports.SourceImport = SourceImport;

langserver/java/typeident.js

@@ -1,5 +1,5 @@
 const { ArrayType, CEIType, JavaType, PrimitiveType, MethodBase, WildcardType } = require('java-mti');
-const { SourceMethod, SourceConstructor, SourceInitialiser } = require('./source-type');
+const { SourceTypeIdent, SourceMethod, SourceConstructor, SourceInitialiser } = require('./source-type');
 const ResolvedImport = require('./parsetypes/resolved-import');
 const { resolveTypeOrPackage, resolveNextTypeOrPackage } = require('./type-resolver');
 const { Token } = require('./tokenizer');
@@ -35,9 +35,22 @@ function typeIdentList(tokens, scope, imports, typemap) {
  * @param {boolean} allow_array_qualifiers
  */
 function typeIdent(tokens, scope, imports, typemap, allow_array_qualifiers = true) {
+    tokens.mark();
+    const type = singleTypeIdent(tokens, scope, imports, typemap, allow_array_qualifiers);
+    return new SourceTypeIdent(tokens.markEnd(), type);
+}
+
+/**
+ * @param {TokenList} tokens 
+ * @param {CEIType|MethodBase} scope 
+ * @param {ResolvedImport[]} imports
+ * @param {Map<string,JavaType>} typemap 
+ * @param {boolean} allow_array_qualifiers
+ */
+function singleTypeIdent(tokens, scope, imports, typemap, allow_array_qualifiers = true) {
     /** @type {JavaType[]} */
     let types = [], package_name = '';
-    const start_idx = tokens.idx;
+    tokens.mark();
     switch(tokens.current.kind) {
         case 'ident':
             ({ types, package_name } = resolveTypeOrPackage(tokens.current.value, scope, imports, typemap));
@@ -65,8 +78,9 @@ function typeIdent(tokens, scope, imports, typemap, allow_array_qualifiers = tru
         }
     }
 
+    const type_tokens = tokens.markEnd();
     if (!types[0]) {
-        const anytype = new AnyType(tokens.tokens.slice(start_idx, tokens.idx).map(t => t.source).join('').trim());
+        const anytype = new AnyType(type_tokens.map(t => t.source).join(''));
         types.push(anytype);
     }
 
@@ -104,7 +118,7 @@ function genericTypeArgs(tokens, types, scope, imports, typemap) {
         });
         return;
     }
-    const type_arguments = typeIdentList(tokens, scope, imports, typemap);
+    const type_arguments = typeIdentList(tokens, scope, imports, typemap).map(s => s.resolved);
     types.forEach((t,i,arr) => {
         if (t instanceof CEIType) {
             let specialised = t.specialise(type_arguments);
@@ -141,7 +155,7 @@ function wildcardTypeArgument(tokens, scope, imports, typemap) {
             tokens.inc();
             bound = {
                 kind,
-                type: typeIdent(tokens, scope, imports, typemap),
+                type: singleTypeIdent(tokens, scope, imports, typemap),
             }
             break;
     }

langserver/java/validater.js

@@ -1,150 +1,44 @@
-const { ArrayType, JavaType, TypeVariable } = require('java-mti');
+const { JavaType } = require('java-mti');
-const { ModuleBlock, TypeDeclBlock } = require('./parser9');
 const { resolveImports } = require('../java/import-resolver');
-const ResolvedImport = require('../java/parsetypes/resolved-import');
+const { SourceUnit } = require('./source-type');
-const { SourceType, SourceTypeIdent, SourceField, SourceMethod, SourceConstructor, SourceInitialiser, SourceParameter, SourceAnnotation } = require('./source-type');
+const { parseBody } = require('./body-parser3');
-const { parseBody, flattenBlocks } = require('./body-parser3');
-const { TokenList } = require('./TokenList');
-const { typeIdent } = require('./typeident');
 
 
 /**
- * @param {ModuleBlock} mod
+ * @param {SourceUnit} unit 
- * @param {SourceType} outer_type
- * @param {ModuleBlock|TypeDeclBlock} parent 
- * @param {SourceType[]} source_types 
- * @param {Map<string,JavaType>} typemap
- */
-function getSourceTypes(mod, outer_type, parent, source_types, typemap) {
-    parent.types.forEach(type => {
-        const t = new SourceType(mod.packageName, outer_type, '', type.modifiers.map(m => m.value), type.kindToken, type.name_token, typemap);
-        t.typeVariables = type.typevars.map(tv => new TypeVariable(t, tv.name, [new TypeVariable.Bound(t, 'Ljava/lang/Object;', false)]));
-        source_types.push(t);
-        getSourceTypes(mod, t, type, source_types, typemap);
-    });
-}
-
-/**
- * @param {TokenList} tokens
- * @param {ModuleBlock|TypeDeclBlock} parent 
- * @param {ResolvedImport[]} imports 
- * @param {Map<string,JavaType>} typemap
- */
-function populateTypes(tokens, parent, imports, typemap) {
-
-    parent.types.forEach(type => {
-        const source_type = typemap.get(type.shortSignature);
-        if (source_type instanceof SourceType) {
-            if (type.extends_decl)
-                source_type.extends_types = resolveTypeList(source_type, type.extends_decl);
-            if (type.implements_decl)
-                source_type.implements_types = resolveTypeList(source_type, type.implements_decl);
-            
-            // fields
-            source_type.fields = type.fields.map(f => {
-                const field_type = resolveTypeFromTokens(source_type, f);
-                return new SourceField(source_type, f.modifiers, field_type, f.name_token);
-            });
-            // methods
-            source_type.methods = type.methods.map(m => {
-                const method_type = resolveTypeFromTokens(source_type, m);
-                const params = m.parameters.map(p => {
-                    let param_type = resolveTypeFromTokens(source_type, p);
-                    return new SourceParameter(p.modifiers, param_type, p.isVarArgs, p.name_token);
-                })
-                const annotations = m.annotations.map(a => new SourceAnnotation(resolveTypeFromTokens(source_type, {typeTokens: [a.blockArray().blocks.slice().pop()]})))
-                return new SourceMethod(source_type, m.modifiers, annotations, method_type, m.name_token, params, [], flattenBlocks([m.body()], true));
-            })
-            // constructors
-            source_type.constructors = type.constructors.map(c => {
-                const params = c.parameters.map(p => {
-                    const param_type = resolveTypeFromTokens(source_type, p);
-                    return new SourceParameter(p.modifiers, param_type, p.isVarArgs, p.name_token);
-                })
-                return new SourceConstructor(source_type, c.modifiers, params, [], flattenBlocks([c.body()], true));
-            })
-            // initialisers
-            source_type.initers = type.initialisers.map(i => {
-                return new SourceInitialiser(source_type, i.modifiers, flattenBlocks([i.body()], true));
-            })
-        }
-        populateTypes(tokens, type, imports, typemap);
-    });
-
-    function resolveTypeFromTokens(scope, decl) {
-        const typetokens = flattenBlocks([decl.typeTokens[0]], false);
-        tokens.current = tokens.tokens[tokens.idx = tokens.tokens.indexOf(typetokens[0])];
-        let type = typeIdent(tokens, scope, imports, typemap);
-        if (decl.varBlock && decl.varBlock.post_name_arr_token) {
-            type = new ArrayType(type, decl.varBlock.post_name_arr_token.source.replace(/[^\[]/g,'').length);
-        }
-        if (decl.isVarArgs) {
-            type = new ArrayType(type, 1);
-        }
-        return new SourceTypeIdent(typetokens, type);
-    }
-
-    function resolveTypeList(scope, eit_decl) {
-        const types = [];
-        const eit_tokens = flattenBlocks([eit_decl], false);
-        tokens.current = tokens.tokens[tokens.idx = tokens.tokens.indexOf(eit_tokens[0])];
-        tokens.inc();   // bypass extends/implements/throws keyword
-        for (;;) {
-            const start = tokens.idx;
-            const type = typeIdent(tokens, scope, imports, typemap);
-            let end = tokens.idx - 1;
-            while (tokens.tokens[end].kind === 'wsc') {
-                end -= 1;
-            }
-            types.push(new SourceTypeIdent(tokens.tokens.slice(start, end + 1), type));
-            if (!tokens.isValue(',')) {
-                break;
-            }
-        }
-        return types;
-    }
-}
-
-
-/**
- * @param {ModuleBlock} mod 
  * @param {Map<string, JavaType>} androidLibrary
  */
-function validate(mod, androidLibrary) {
+function validate(unit, androidLibrary) {
     console.time('validation');
 
-    /** @type {SourceType[]} */
-    const source_types = [];
-    getSourceTypes(mod, null, mod, source_types, androidLibrary);
-
-    const imports = resolveImports(androidLibrary, source_types, mod.imports, mod.packageName);
-
-    populateTypes(new TokenList(flattenBlocks(mod.blocks, false)), mod, imports.resolved, imports.typemap);
-
     let probs = [];
-    source_types.forEach(t => {
+    const resolved_types = [
+        ...resolveImports(androidLibrary, [], [], null).resolved,
+        ...unit.imports.filter(i => i.resolved).map(i => i.resolved),
+    ]
+    unit.types.forEach(t => {
         t.initers.forEach(i => {
-            const parsed = parseBody(i, imports.resolved, imports.typemap);
+            const parsed = parseBody(i, resolved_types, androidLibrary);
             if (parsed)
                 probs = probs.concat(parsed.problems)
         })
         t.constructors.forEach(c => {
-            const parsed = parseBody(c, imports.resolved, imports.typemap);
+            const parsed = parseBody(c, resolved_types, androidLibrary);
             if (parsed)
                 probs = probs.concat(parsed.problems)
         })
         t.methods.forEach(m => {
-            const parsed = parseBody(m, imports.resolved, imports.typemap);
+            const parsed = parseBody(m, resolved_types, androidLibrary);
             if (parsed)
                 probs = probs.concat(parsed.problems)
         })
     })
 
     const module_validaters = [
-        require('./validation/multiple-package-decls'),
+        // require('./validation/multiple-package-decls'),
-        require('./validation/unit-decl-order'),
+        // require('./validation/unit-decl-order'),
-        require('./validation/duplicate-members'),
+        // require('./validation/duplicate-members'),
-        require('./validation/parse-errors'),
+        // require('./validation/parse-errors'),
         require('./validation/modifier-errors'),
         require('./validation/unresolved-imports'),
         require('./validation/invalid-types'),
@@ -155,7 +49,7 @@ function validate(mod, androidLibrary) {
         require('./validation/missing-constructor'),
     ];
     let problems = [
-        module_validaters.map(v => v(mod, imports, source_types)),
+        module_validaters.map(v => v(unit.types, unit)),
         ...probs,
     ];
     console.timeEnd('validation');

langserver/java/validation/bad-extends.js

@@ -1,6 +1,5 @@
-const { ModuleBlock, TypeDeclBlock } = require('../parser9');
+const { SourceType, SourceTypeIdent } = require('../source-type');
 const ParseProblem = require('../parsetypes/parse-problem');
-const {SourceType} = require('../source-type');
 
 /**
  * @param {SourceType} source_type 
@@ -10,41 +9,39 @@ function checkExtends(source_type, probs) {
     if (source_type.extends_types.length === 0) {
         return;
     }
-    const supertypes = source_type.extends_types.map(st => st.type);
+    const supertypes = source_type.extends_types.map(st => st.resolved);
     const supertype = supertypes[0];
     if (source_type.typeKind === 'enum') {
-        probs.push(ParseProblem.Error(source_type.extends_types[0].typeTokens, `Enum types cannot declare a superclass`));
+        probs.push(ParseProblem.Error(source_type.extends_types[0].tokens, `Enum types cannot declare a superclass`));
     }
     if (source_type.typeKind === 'class' && supertypes.length > 1) {
-        probs.push(ParseProblem.Error(source_type.extends_types[1].typeTokens, `Class types cannot inherit from more than one type`));
+        probs.push(ParseProblem.Error(source_type.extends_types[1].tokens, `Class types cannot inherit from more than one type`));
     }
     if (source_type.typeKind === 'class' && supertype.typeKind !== 'class') {
-        probs.push(ParseProblem.Error(source_type.extends_types[0].typeTokens, `Class '${source_type.fullyDottedRawName}' cannot inherit from ${supertype.typeKind} type: '${supertype.fullyDottedRawName}'`));
+        probs.push(ParseProblem.Error(source_type.extends_types[0].tokens, `Class '${source_type.fullyDottedRawName}' cannot inherit from ${supertype.typeKind} type: '${supertype.fullyDottedRawName}'`));
     }
     if (source_type.typeKind === 'class' && supertype.typeKind === 'class' && supertype.modifiers.includes('final')) {
-        probs.push(ParseProblem.Error(source_type.extends_types[0].typeTokens, `Class '${source_type.fullyDottedRawName}' cannot inherit from final class: '${supertype.fullyDottedRawName}'`));
+        probs.push(ParseProblem.Error(source_type.extends_types[0].tokens, `Class '${source_type.fullyDottedRawName}' cannot inherit from final class: '${supertype.fullyDottedRawName}'`));
     }
     if (source_type.typeKind === 'class' && supertype === source_type) {
-        probs.push(ParseProblem.Error(source_type.extends_types[0].typeTokens, `Class '${source_type.fullyDottedRawName}' cannot inherit from itself`));
+        probs.push(ParseProblem.Error(source_type.extends_types[0].tokens, `Class '${source_type.fullyDottedRawName}' cannot inherit from itself`));
     }
     if (source_type.typeKind === 'interface') {
         supertypes.forEach((supertype, i) => {
             if (supertype.typeKind !== 'interface') {
-                probs.push(ParseProblem.Error(source_type.extends_types[i].typeTokens, `Interface '${source_type.fullyDottedRawName}' cannot inherit from ${supertype.typeKind} type: '${supertype.fullyDottedRawName}'`));
+                probs.push(ParseProblem.Error(source_type.extends_types[i].tokens, `Interface '${source_type.fullyDottedRawName}' cannot inherit from ${supertype.typeKind} type: '${supertype.fullyDottedRawName}'`));
             }
             if (supertype === source_type) {
-                probs.push(ParseProblem.Error(source_type.extends_types[i].typeTokens, `Interface '${source_type.fullyDottedRawName}' cannot inherit from itself`));
+                probs.push(ParseProblem.Error(source_type.extends_types[i].tokens, `Interface '${source_type.fullyDottedRawName}' cannot inherit from itself`));
             }
         })
     }
 }
 
 /**
- * @param {ModuleBlock} mod 
- * @param {*} imports
  * @param {SourceType[]} source_types
  */
-module.exports = function(mod, imports, source_types) {
+module.exports = function(source_types) {
     /** @type {ParseProblem[]} */
     const probs = [];
       

langserver/java/validation/bad-implements.js

@@ -1,4 +1,3 @@
-const { ModuleBlock } = require('../parser9');
 const ParseProblem = require('../parsetypes/parse-problem');
 const {SourceType} = require('../source-type');
 const { UnresolvedType } = require('java-mti');
@@ -11,9 +10,9 @@ function checkImplements(source_type, probs) {
     if (source_type.implements_types.length === 0) {
         return;
     }
-    const interfaces = source_type.implements_types.map(it => it.type);
+    const interfaces = source_type.implements_types.map(it => it.resolved);
     if (source_type.typeKind === 'interface') {
-        probs.push(ParseProblem.Error(source_type.implements_types[0].typeTokens, `Interface types cannot declare an implements section`));
+        probs.push(ParseProblem.Error(source_type.implements_types[0].tokens, `Interface types cannot declare an implements section`));
     }
     if (source_type.typeKind === 'class') {
         interfaces.forEach((intf, i) => {
@@ -21,18 +20,16 @@ function checkImplements(source_type, probs) {
                 return;
             }
             if (intf.typeKind !== 'interface') {
-                probs.push(ParseProblem.Error(source_type.implements_types[i].typeTokens, `Class '${source_type.fullyDottedRawName}' cannot implement ${intf.typeKind} type: '${intf.fullyDottedRawName}'`));
+                probs.push(ParseProblem.Error(source_type.implements_types[i].tokens, `Class '${source_type.fullyDottedRawName}' cannot implement ${intf.typeKind} type: '${intf.fullyDottedRawName}'`));
             }
         })
     }
 }
 
 /**
- * @param {ModuleBlock} mod 
- * @param {*} imports
  * @param {SourceType[]} source_types
  */
-module.exports = function(mod, imports, source_types) {
+module.exports = function(source_types) {
     /** @type {ParseProblem[]} */
     const probs = [];
       

langserver/java/validation/bad-overrides.js

@@ -1,12 +1,7 @@
-const { ModuleBlock } = require('../parser9');
 const ParseProblem = require('../parsetypes/parse-problem');
 const {SourceType, SourceAnnotation} = require('../source-type');
 const {CEIType, Method} = require('java-mti');
 
-function nonAbstractLabel(label) {
-    return label.replace(/\babstract /g, '');
-}
-
 /**
  * @param {SourceType} source_type 
  * @param {*} probs 
@@ -22,7 +17,7 @@ function checkOverrides(source_type, probs) {
     /** @type {{ann:SourceAnnotation, method:Method, method_id:string}[]} */
     const overriden_methods = [];
     source_type.methods.reduce((arr, method) => {
-        const ann = method.annotations.find(a => /^Override$/.test(a.annotationTypeIdent.type.simpleTypeName));
+        const ann = method.annotations.find(a => a.type.simpleTypeName === 'Override');
         if (ann) {
             arr.push({
                 ann,
@@ -52,17 +47,15 @@ function checkOverrides(source_type, probs) {
 
     overriden_methods.forEach(x => {
         if (!methods.has(x.method_id)) {
-            probs.push(ParseProblem.Error(x.ann.annotationTypeIdent.typeTokens, `${x.method.label} does not override a matching method in any inherited type or interface`));
+            probs.push(ParseProblem.Error(x.ann.annotationTypeIdent.tokens, `${x.method.label} does not override a matching method in any inherited type or interface`));
         }
     })
 }
 
 /**
- * @param {ModuleBlock} mod 
- * @param {*} imports
  * @param {SourceType[]} source_types
  */
-module.exports = function(mod, imports, source_types) {
+module.exports = function(source_types) {
     /** @type {ParseProblem[]} */
     const probs = [];
       

langserver/java/validation/invalid-types.js

@@ -1,26 +1,22 @@
-const { ModuleBlock } = require('../parser9');
+const { SourceType, SourceTypeIdent } = require('../source-type');
 const ParseProblem = require('../parsetypes/parse-problem');
-const {SourceType} = require('../source-type');
-const {Token} = require('../tokenizer');
-const {JavaType} = require('java-mti');
 
 /**
- * @param {JavaType} type 
+ * @param {SourceTypeIdent} type 
  * @param {boolean} is_return_type
- * @param {Token[]} typeTokens
  * @param {ParseProblem[]} probs
  */
-function checkType(type, is_return_type, typeTokens, probs) {
+function checkType(type, is_return_type, probs) {
-    const typesig = type.typeSignature;
+    const typesig = type.resolved.typeSignature;
     if (/^\[*U/.test(typesig)) {
-        probs.push(ParseProblem.Error(typeTokens, `Unresolved type '${type.label}'`))
+        probs.push(ParseProblem.Error(type.tokens, `Unresolved type '${type.resolved.label}'`))
         return;
     }
     if (typesig === 'V' && !is_return_type) {
-        probs.push(ParseProblem.Error(typeTokens, `'void' is not a valid type for variables`))
+        probs.push(ParseProblem.Error(type.tokens, `'void' is not a valid type for variables`))
     }
     if (/^\[+V/.test(typesig)) {
-        probs.push(ParseProblem.Error(typeTokens, `Illegal type: '${type.label}'`))
+        probs.push(ParseProblem.Error(type.tokens, `Illegal type: '${type.resolved.label}'`))
     }
 }
 
@@ -29,22 +25,25 @@ function checkType(type, is_return_type, typeTokens, probs) {
  * @param {*} probs 
  */
 function checkInvalidTypes(type, probs) {
-    type.fields.forEach(f => checkType(f.type, false, f.fieldType.typeTokens, probs));
+    type.fields.forEach(f => checkType(f.fieldTypeIdent, false, probs));
     type.methods.forEach(m => {
-        checkType(m.returnType, true, m.methodTypeIdent.typeTokens, probs);
+        checkType(m.returnTypeIdent, true, probs);
         m.parameters.forEach(p => {
-            checkType(p.type, false, p.paramTypeIdent.typeTokens, probs);
+            checkType(p.paramTypeIdent, false, probs);
+        })
+    })
+    type.constructors.forEach(c => {
+        c.parameters.forEach(p => {
+            checkType(p.paramTypeIdent, false, probs);
         })
     })
 }
 
 
 /**
- * @param {ModuleBlock} mod 
- * @param {*} imports
  * @param {SourceType[]} source_types
  */
-module.exports = function(mod, imports, source_types) {
+module.exports = function(source_types) {
     /** @type {ParseProblem[]} */
     const probs = [];
       

langserver/java/validation/missing-constructor.js

@@ -1,6 +1,5 @@
-const { ModuleBlock, TypeDeclBlock } = require('../parser9');
-const ParseProblem = require('../parsetypes/parse-problem');
 const {SourceType, SourceConstructor} = require('../source-type');
+const ParseProblem = require('../parsetypes/parse-problem');
 
 /**
  * @param {SourceType} source_type 
@@ -22,16 +21,14 @@ function checkConstructor(source_type, probs) {
     if (!superclass.constructors.find(c => c.parameterCount === 0)) {
         // the source type has no declared constructors, but the superclass
         // does not include a default (parameterless) constructor
-        probs.push(ParseProblem.Error(source_type.name_token, `Class '${source_type.fullyDottedRawName}' requires a constructor to be declared because the inherited class '${superclass.fullyDottedRawName}' does not define a default constructor.`));
+        probs.push(ParseProblem.Error(source_type.nameToken, `Class '${source_type.fullyDottedRawName}' requires a constructor to be declared because the inherited class '${superclass.fullyDottedRawName}' does not define a default constructor.`));
     }
 }
 
 /**
- * @param {ModuleBlock} mod 
- * @param {*} imports
  * @param {SourceType[]} source_types
  */
-module.exports = function(mod, imports, source_types) {
+module.exports = function(source_types) {
     /** @type {ParseProblem[]} */
     const probs = [];
       

langserver/java/validation/modifier-errors.js

@@ -1,8 +1,9 @@
-const { TextBlock, ModuleBlock, FieldBlock, MethodBlock, ConstructorBlock, InitialiserBlock, TypeDeclBlock } = require('../parser9');
+const { SourceType, SourceMethod, SourceField, SourceConstructor, SourceInitialiser } = require('../source-type');
+const { Token } = require('../tokenizer');
 const ParseProblem = require('../parsetypes/parse-problem');
 
 /**
- * @param {TextBlock[]} mods 
+ * @param {Token[]} mods 
  * @param {ParseProblem[]} probs 
  */
 function checkDuplicate(mods, probs) {
@@ -21,7 +22,7 @@ function checkDuplicate(mods, probs) {
 }
 
 /**
- * @param {TextBlock[]} mods 
+ * @param {Token[]} mods 
  * @param {ParseProblem[]} probs 
  */
 function checkConflictingAccess(mods, probs) {
@@ -49,13 +50,13 @@ function checkConflictingAccess(mods, probs) {
 }
 
 /**
- * @param {FieldBlock} field 
+ * @param {SourceField} field 
  * @param {ParseProblem[]} probs 
  */
 function checkFieldModifiers(field, probs) {
-    checkDuplicate(field.modifiers, probs);
+    checkDuplicate(field.modifierTokens, probs);
-    checkConflictingAccess(field.modifiers, probs);
+    checkConflictingAccess(field.modifierTokens, probs);
-    for (let mod of field.modifiers) {
+    for (let mod of field.modifierTokens) {
         switch (mod.source) {
             case 'abstract':
                 probs.push(ParseProblem.Error(mod, 'Field declarations cannot be abstract'));
@@ -68,18 +69,18 @@ function checkFieldModifiers(field, probs) {
 }
 
 /**
- * @param {TypeDeclBlock} type
+ * @param {SourceType} type
  * @param {Map<string,*>} ownertypemods
- * @param {MethodBlock} method 
+ * @param {SourceMethod} method 
  * @param {ParseProblem[]} probs 
  */
 function checkMethodModifiers(type, ownertypemods, method, probs) {
-    checkDuplicate(method.modifiers, probs);
+    checkDuplicate(method.modifierTokens, probs);
-    checkConflictingAccess(method.modifiers, probs);
+    checkConflictingAccess(method.modifierTokens, probs);
 
-    const allmods = new Map(method.modifiers.map(m => [m.source, m]));
+    const allmods = new Map(method.modifierTokens.map(m => [m.source, m]));
-    const is_interface_kind = /@?interface/.test(type.kind());
+    const is_interface_kind = /@?interface/.test(type.typeKind);
-    const has_body = method.body().simplified.startsWith('B');
+    const has_body = method.hasImplementation;
 
     if (allmods.has('abstract') && allmods.has('final')) {
         probs.push(ParseProblem.Error(allmods.get('abstract'), 'Method declarations cannot be abstract and final'));
@@ -91,77 +92,76 @@ function checkMethodModifiers(type, ownertypemods, method, probs) {
         probs.push(ParseProblem.Error(allmods.get('abstract'), 'Method declarations marked as abstract cannot have a method body'));
     }
     if (!is_interface_kind && !allmods.has('abstract') && !allmods.has('native') && !has_body) {
-        probs.push(ParseProblem.Error(method, `Method '${method.name}' must have an implementation or be defined as abstract or native`));
+        probs.push(ParseProblem.Error(method.nameToken, `Method '${method.name}' must have an implementation or be defined as abstract or native`));
     }
     if (!is_interface_kind && allmods.has('abstract') && !ownertypemods.has('abstract')) {
-        probs.push(ParseProblem.Error(method, `Method '${method.name}' cannot be declared abstract inside a non-abstract type`));
+        probs.push(ParseProblem.Error(allmods.get('abstract'), `Method '${method.name}' cannot be declared abstract inside a non-abstract type`));
     }
     if (is_interface_kind && has_body && !allmods.has('default')) {
-        probs.push(ParseProblem.Error(method, `Non-default interface methods cannot have a method body`));
+        probs.push(ParseProblem.Error(method.body[0], `Non-default interface methods cannot have a method body`));
     }
     if (allmods.has('native') && has_body) {
         probs.push(ParseProblem.Error(allmods.get('native'), 'Method declarations marked as native cannot have a method body'));
     }
     // JLS8
-    if (type.kind() !== 'interface' && allmods.has('default')) {
+    if (type.typeKind !== 'interface' && allmods.has('default')) {
-        probs.push(ParseProblem.Error(method, `Default method declarations are only allowed inside interfaces`));
+        probs.push(ParseProblem.Error(allmods.get('default'), `Default method declarations are only allowed inside interfaces`));
     }
     if (allmods.has('default') && !has_body) {
-        probs.push(ParseProblem.Error(method, `Default method declarations must have an implementation`));
+        probs.push(ParseProblem.Error(allmods.get('default'), `Default method declarations must have an implementation`));
     }
 }
 
 /**
- * @param {ConstructorBlock} field 
+ * @param {SourceConstructor} field 
  * @param {ParseProblem[]} probs 
  */
 function checkConstructorModifiers(field, probs) {
 }
 
 /**
- * @param {InitialiserBlock} initialiser 
+ * @param {SourceInitialiser} initialiser 
  * @param {ParseProblem[]} probs 
  */
 function checkInitialiserModifiers(initialiser, probs) {
 }
 
 /**
- * @param {TypeDeclBlock} type 
+ * @param {SourceType} type 
  * @param {ParseProblem[]} probs 
  */
 function checkTypeModifiers(type, probs) {
-    const typemods = new Map(type.modifiers.map(m => [m.source, m]));
+    const typemods = new Map(type.modifierTokens.map(m => [m.source, m]));
-    checkDuplicate(type.modifiers, probs);
+    checkDuplicate(type.modifierTokens, probs);
 
-    if (type.kind() === 'interface' && typemods.has('final')) {
+    if (type.typeKind === 'interface' && typemods.has('final')) {
         probs.push(ParseProblem.Error(typemods.get('final'), 'Interface declarations cannot be marked as final'));
     }
-    if (type.kind() === 'enum' && typemods.has('abstract')) {
+    if (type.typeKind === 'enum' && typemods.has('abstract')) {
         probs.push(ParseProblem.Error(typemods.get('abstract'), 'Enum declarations cannot be marked as abstract'));
     }
-    // top-level types cannot be private, protected or static
+    if (/[$]/.test(type._rawShortSignature)) {
-    for (let mod of ['private','protected', 'static']) {
+        checkConflictingAccess(type.modifierTokens, probs);
-        if (!type.outer_type && typemods.has(mod)) {
+    } else {
-            probs.push(ParseProblem.Error(typemods.get(mod), `Top-level declarations cannot be marked as ${mod}`));
+        // top-level types cannot be private, protected or static
+        for (let mod of ['private','protected', 'static']) {
+            if (typemods.has(mod)) {
+                probs.push(ParseProblem.Error(typemods.get(mod), `Top-level declarations cannot be marked as ${mod}`));
+            }
         }
     }
-    if (type.outer_type) {
-        checkConflictingAccess(type.modifiers, probs);
-    }
 
     type.fields.forEach(field => checkFieldModifiers(field, probs));
     type.methods.forEach(method => checkMethodModifiers(type, typemods, method, probs));
     type.constructors.forEach(ctr => checkConstructorModifiers(ctr, probs));
-    //type.initialisers.forEach(initer => checkInitModifiers(initer, probs));
+    type.initers.forEach(initer => checkInitialiserModifiers(initer, probs));
-    // check enclosed types
-    type.types.forEach(type => checkTypeModifiers(type, probs));
 }
 
 /**
- * @param {ModuleBlock} mod 
+ * @param {SourceType[]} types 
  */
-module.exports = function(mod) {
+module.exports = function(types) {
     const probs = [];
-    mod.types.forEach(type => checkTypeModifiers(type, probs));
+    types.forEach(type => checkTypeModifiers(type, probs));
     return probs;
 }

langserver/java/validation/non-implemented-interfaces.js

@@ -1,4 +1,3 @@
-const { ModuleBlock } = require('../parser9');
 const ParseProblem = require('../parsetypes/parse-problem');
 const { SourceType } = require('../source-type');
 const {CEIType} = require('java-mti');
@@ -49,17 +48,15 @@ function checkImplementedInterfaces(source_type, probs) {
             }
         })
         if (missing_methods.length) {
-            probs.push(ParseProblem.Error(source_type.kind_token, `Non-abstract ${source_type.typeKind} '${source_type.fullyDottedRawName}' does not implement the following methods from interface '${intf.fullyDottedRawName}':\n${missing_methods.join('\n')}`));
+            probs.push(ParseProblem.Error(source_type.kind_token, `Non-abstract ${source_type.typeKind} '${source_type.fullyDottedRawName}' does not implement the following methods from interface '${intf.fullyDottedTypeName}':\n${missing_methods.join('\n')}`));
         }
     });
 }
 
 /**
- * @param {ModuleBlock} mod 
- * @param {*} imports
  * @param {SourceType[]} source_types
  */
-module.exports = function(mod, imports, source_types) {
+module.exports = function(source_types) {
     /** @type {ParseProblem[]} */
     const probs = [];
       

langserver/java/validation/unresolved-imports.js

@@ -1,16 +1,16 @@
-const { ModuleBlock } = require('../parser9');
+const { SourceUnit } = require('../source-type');
 const ParseProblem = require('../parsetypes/parse-problem');
 
 /**
- * @param {ModuleBlock} mod 
+ * @param {SourceUnit} unit
- * @param {{unresolved:*[]}} imports
  */
-module.exports = function(mod, imports) {
+module.exports = function(mod, unit) {
     /** @type {ParseProblem[]} */
     const probs = [];
       
-    imports.unresolved.forEach(import_tokens => {
+    unit.imports.forEach(i => {
-        probs.push(ParseProblem.Warning(import_tokens, `Unresolved import: ${import_tokens.name}`));
+        if (!i.resolved)
+            probs.push(ParseProblem.Warning(i.nameTokens, `Unresolved import: ${i.package_name}`));
     })
       
     return probs;

langserver/server.js

@@ -20,7 +20,8 @@ const { TextDocument } = require('vscode-languageserver-textdocument');
 const { loadAndroidLibrary, JavaType } = require('java-mti');
 
 const { ParseProblem } = require('./java/parser');
-const { parse, ModuleBlock } = require('./java/parser9');
+const { parse } = require('./java/body-parser3');
+const { SourceUnit } = require('./java/source-type');
 const { validate } = require('./java/validater');
 
 /**
@@ -42,11 +43,14 @@ let connection = createConnection(ProposedFeatures.all);
 
 ///** @type {LiveParseInfo[]} */
 //const liveParsers = [];
-/** @type {{content: string, uri: string, result: ModuleBlock, positionAt:(n) => Position, indexAt:(p:Position) => number}} */
+/** @type {{content: string, uri: string, result: SourceUnit, positionAt:(n) => Position, indexAt:(p:Position) => number}} */
 let parsed = null;
 
 function reparse(uri, content) {
-    const result = parse(content);
+    if (androidLibrary instanceof Promise) {
+        return;
+    }
+    const result = parse(content, new Map(androidLibrary));
     parsed = {
         content,
         uri,