generate source types before parsing

langserver/java/body-parser3.js

@@ -78,6 +78,76 @@ function addproblem(tokens, problem) {
     tokens.problems.push(problem);
 }
 
+/**
+ * @param {TokenList} tokens 
+ * @param {*} typemap 
+ */
+function extractSourceTypes(tokens, typemap) {
+    // first strip out any comments, chars, strings etc which might confuse the parsing
+    const normalised_source = tokens.tokens.map(t => {
+        return /wsc|string-literal|char-literal/.test(t.kind) 
+          ? ' '.repeat(t.length)
+          : t.source
+    }).join('')
+    
+    // look for scope boundaries, package and type declarations
+    const re = /(\{)|(\})|\bpackage +(\w+(?: *\. *\w+)*)|\b(class|enum|interface|@ *interface) +(\w+)/g;
+    let package_name = null;
+    let type_stack = [];
+    let code_balance = 0;
+    const source_types = [];
+    function findTokenAt(idx) {
+        return tokens.tokens.find(t => t.range.start === idx);        
+    }
+    for (let m; m = re.exec(normalised_source);) {
+        if (code_balance) {
+            if (m[1]) code_balance += 1;
+            else if (m[2]) code_balance -= 1;
+            continue;
+        }
+        if (m[1]) {
+            // open brace
+            if (!type_stack[0]) {
+                continue;   // ignore - we haven't started a type yet
+            }
+            if (!type_stack[0].type_open) {
+                type_stack[0].type_open = true; // start of type body
+                continue;
+            }
+            // start of method body or array expression
+            code_balance = 1;
+        } else if (m[2]) {
+            // close brace
+            if (!type_stack[0]) {
+                continue;   // we're outside any type
+            }
+            type_stack.shift();
+        } else if (m[3]) {
+            // package name
+            if (package_name !== null) {
+                continue;   // ignore - we already have a package name or started parsing types
+            }
+            package_name = m[3].replace(/ +/g, '');
+        } else if (m[4]) {
+            // named type decl
+            package_name = package_name || '';
+            const typeKind = m[4].replace(/ +/g, ''),
+              kind_token = findTokenAt(m.index),
+              name_token = findTokenAt(m.index + m[0].match(/\w+$/).index),
+              outer_type = type_stack[0] && type_stack[0].source_type,
+              source_type = new SourceType(package_name, outer_type, '', [], typeKind, kind_token, name_token, typemap);
+              
+            type_stack.unshift({
+                source_type,
+                type_open: false,
+            });
+            source_types.unshift(source_type);
+        }
+    }
+    console.log(source_types.map(t => t.shortSignature))
+    return source_types;
+}
+
 /**
  * @param {string} source
  * @param {Map<string,JavaType>} typemap 
@@ -89,6 +159,12 @@ function parse(source, typemap) {
         console.time('tokenize');
         tokens = new TokenList(tokenize(source));
         console.timeEnd('tokenize');
+
+        // in order to resolve types as we parse, we must extract the set of source types first
+        const source_types = extractSourceTypes(tokens, typemap);
+        // add them to the type map
+        source_types.forEach(t => typemap.set(t.shortSignature, t));
+
         console.time('parse');
         parseUnit(tokens, unit, typemap);
         console.timeEnd('parse');
@@ -592,7 +668,17 @@ function typeDeclaration(package_name, scope, modifiers, typeKind, kind_token, t
         addproblem(tokens, ParseProblem.Error(tokens.current, `Type identifier expected`));
         return;
     }
-    const type = new SourceType(package_name, scope, '', modifiers, typeKind, kind_token, name, typemap);
+    const type_short_sig = SourceType.getShortSignature(package_name, scope, name.value);
+    // the source type object should already exist in the type map
+    /** @type {SourceType} */
+    // @ts-ignore
+    let type = typemap.get(type_short_sig);
+    if (type instanceof SourceType) {
+        // update the missing parts
+        type.setModifierTokens(modifiers);
+    } else {
+        type = new SourceType(package_name, scope, '', modifiers, typeKind, kind_token, name, typemap);
+    }
     type.typeVariables = tokens.current.value === '<'
         ? typeVariableList(type, tokens, scope, imports, typemap)
         : [];

langserver/java/source-types2.js

@@ -3,21 +3,21 @@ const { Token } = require('./tokenizer');
 
 /**
  * @param {SourceType|SourceMethod|SourceConstructor|SourceInitialiser|string} scope_or_package_name 
- * @param {Token} name 
+ * @param {string} name 
  */
 function generateShortSignature(scope_or_package_name, name) {
     if (scope_or_package_name instanceof SourceType) {
         const type = scope_or_package_name;
-        return `${type._rawShortSignature}$${name.value}`;
+        return `${type._rawShortSignature}$${name}`;
     }
     if (scope_or_package_name instanceof SourceMethod
         || scope_or_package_name instanceof SourceConstructor
         || scope_or_package_name instanceof SourceInitialiser) {
         const method = scope_or_package_name;
-        return `${method.owner._rawShortSignature}$${method.owner.localTypeCount += 1}${name.value}`;
+        return `${method.owner._rawShortSignature}$${method.owner.localTypeCount += 1}${name}`;
     }
     const pkgname = scope_or_package_name;
-    return pkgname ?`${pkgname.replace(/\./g, '/')}/${name.value}` : name.value;
+    return pkgname ?`${pkgname.replace(/\./g, '/')}/${name}` : name;
 }
 
 class SourceType extends CEIType {
@@ -32,7 +32,7 @@ class SourceType extends CEIType {
      */
     constructor(packageName, outer_scope, docs, modifiers, typeKind, kind_token, name_token, typemap) {
         // @ts-ignore
-        super(generateShortSignature(outer_scope || packageName, name_token), typeKind, modifiers.map(m => m.source), docs);
+        super(generateShortSignature(outer_scope || packageName, name_token.value), typeKind, modifiers.map(m => m.source), docs);
         super.packageName = packageName;
         this.modifierTokens = modifiers;
         this.kind_token = kind_token;
@@ -58,6 +58,23 @@ class SourceType extends CEIType {
         this.initers = [];
     }
 
+    /**
+     * @param {string} package_name
+     * @param {SourceType|SourceMethod|SourceConstructor|SourceInitialiser} outer_scope
+     * @param {string} name
+     */
+    static getShortSignature(package_name, outer_scope, name) {
+        return generateShortSignature(outer_scope || package_name || '', name);
+    }
+
+    /**
+     * @param {Token[]} mods 
+     */
+    setModifierTokens(mods) {
+        this.modifierTokens = mods;
+        this.modifiers = mods.map(m => m.source);
+    }
+
     get supers() {
         const supertypes = [...this.extends_types, ...this.implements_types].map(x => x.resolved);
         if (this.typeKind === 'enum') {

langserver/server.js

@@ -43,18 +43,20 @@ let connection = createConnection(ProposedFeatures.all);
 
 ///** @type {LiveParseInfo[]} */
 //const liveParsers = [];
-/** @type {{content: string, uri: string, result: SourceUnit, positionAt:(n) => Position, indexAt:(p:Position) => number}} */
+/** @type {{content: string, uri: string, result: SourceUnit, typemap:Map<string,JavaType>, positionAt:(n) => Position, indexAt:(p:Position) => number}} */
 let parsed = null;
 
 function reparse(uri, content) {
     if (androidLibrary instanceof Promise) {
         return;
     }
-    const result = parse(content, new Map(androidLibrary));
+    const typemap = new Map(androidLibrary);
+    const result = parse(content, typemap);
     parsed = {
         content,
         uri,
         result,
+        typemap,
         positionAt(n) {
             let line = 0,
                 last_nl_idx = 0,
@@ -259,7 +261,7 @@ async function validateTextDocument(textDocument) {
 
     if (parsed && parsed.result) {
         try {
-            problems = validate(parsed.result, androidLibrary);
+            problems = validate(parsed.result, parsed.typemap);
         } catch(err) {
             console.error(err);
         }
@@ -351,7 +353,7 @@ connection.onCompletion(
         if (androidLibrary instanceof Promise) {
             androidLibrary = await androidLibrary;
         }
-        const lib = androidLibrary;
+        const lib = (parsed && parsed.typemap) || androidLibrary;
         if (!lib) return [];
         const typeKindMap = {
             class: CompletionItemKind.Class,
@@ -367,12 +369,12 @@ connection.onCompletion(
                     kind: CompletionItemKind.Keyword,
                     data: -1,
                 })),
-                ...'public private protected static final abstract volatile native'.split(' ').map((t) => ({
+                ...'public private protected static final abstract volatile native transient strictfp'.split(' ').map((t) => ({
                     label: t,
                     kind: CompletionItemKind.Keyword,
                     data: -1,
                 })),
-                ...'false true null'.split(' ').map((t) => ({
+                ...'false true null this super'.split(' ').map((t) => ({
                     label: t,
                     kind: CompletionItemKind.Value,
                     data: -1,