different attempt to parse using collapsable text

ranges
This commit is contained in:
Dave Holoway
2020-05-23 13:20:51 +01:00
parent bdc5b1d4cd
commit 23dc6d3871
18 changed files with 1904 additions and 345 deletions

View File

@@ -2,6 +2,7 @@ const Token = require('./parsetypes/token');
const Declaration = require('./parsetypes/declaration');
const TypeIdent = require('./parsetypes/typeident');
const { parse_expression, ExpressionText, ParsedExpression } = require('../../src/expression/parse');
const { TextBlock, TextBlockArray, BlockRange } = require('./parsetypes/textblock');
class LocalVariableDeclaration extends Declaration {
/**
@@ -40,10 +41,11 @@ class LocalVariable {
* @param {number} index
*/
function extractExpression(text, index = 0) {
const e = new ExpressionText(text.slice(index));
const src = text.slice(index);
const e = new ExpressionText(src);
const parsed = parse_expression(e);
console.log(parsed);
let consumed = text.indexOf(e.expr);
//console.log(parsed);
let consumed = index + src.lastIndexOf(e.expr);
return {
parsed,
index: consumed,
@@ -69,18 +71,19 @@ function parseBody(text, text_index = 0) {
const tokens = new TextBlockArray('body');
// preprocess - strip any comments and normalise strings
text = text.replace(/(\/\/.*|\/\*[\D\d]*?\*\/)|(".+?")/g, (_,comment,str) =>
text = text.replace(/(\/\/.*|\/\*[\D\d]*?\*\/|\s+)|(".+?")/g, (_,comment,str) =>
str ?
`"${' '.repeat(str.length-2)}"`
: _.replace(/[^\r\n]/g,' ')
);
: ' '
).replace(/;/g,';\n');
const re = /(\s+)|(["'\d]|\b(?:true|false|null)\b)|\b(if|switch|while|else|for|case|default|do|try|finally|catch|return|break|continue)\b|(\bnew\b)|(\w+)|([;{}():])|(.)/g;
const re = /(\s+)|(["'\d]|\b(?:true|false|null)\b)|\b(if|switch|while|else|for|case|default|do|try|finally|catch|return|break|continue)\b|(\bnew\b)|(\w+|\d+(?:\.\d*)?[eE][+-]?\w*|[!~+-])|([;{}():])|(.)/g;
for (let m; m = re.exec(text);) {
if (m[1]) {
// ignore ws + comments
continue;
}
console.log(re.lastIndex)
if (m[2]) {
// string, character, number, boolean or null literal - parse as an expression
const { parsed, index } = extractExpression(text, m.index);
@@ -101,19 +104,19 @@ function parseBody(text, text_index = 0) {
}
if (m[5]) {
// word - first check if this looks like a variable declaration
const local_var_re = /(\w+(?: *\. *\w+)*(?: *<.*?>)?(?: *\[ *\])*)( +)(\w+)( *\[ *\])*/g;
const local_var_re = /(final +)?(\w+(?: *\. *\w+)*(?: *<.*?>)?(?: *\[ *\])*)( +)(\w+)( *\[ *\])*/g;
local_var_re.lastIndex = m.index;
const local_var_match = local_var_re.exec(text);
if (local_var_match && local_var_match.index === m.index) {
m = local_var_match;
// it looks like a local variable declaration
const typeident = new TypeIdent([new Token(text_index + m.index, m[1], '', null)]);
const typeident = new TypeIdent([new Token(text_index + m.index, m[2], '', null)]);
const local_var_decl = new LocalVariableDeclaration([], typeident);
let name_token = new Token(text_index + m.index + m[1].length + m[2].length, m[3], '', null);
let postarray_token = m[4] ? new Token(name_token.source_idx + m[3].length, m[4], '', null) : null;
let name_token = new Token(text_index + m.index + (m[1]||'').length + m[2].length + m[3].length, m[4], '', null);
let postarray_token = m[4] ? new Token(name_token.source_idx + m[4].length, m[5], '', null) : null;
const vars = [new LocalVariable(local_var_decl, name_token, postarray_token)];
const next = /( *=)|( *, *)(\w+)( *\[ *\])*/g;
const next = /( *= *)|( *, *)(\w+)( *\[ *\])*/g;
let lastIndex = local_var_re.lastIndex;
for (;;) {
next.lastIndex = lastIndex;
@@ -184,7 +187,7 @@ function parseBody(text, text_index = 0) {
for (let m; m = re.exec(sourcemap.simplified);) {
let start = sourcemap.map[m.index];
let end = sourcemap.map[m.index + m[0].length];
tokens.shrink(ids[idx], start, end - start, replacements[idx]);
tokens.shrink(ids[idx], start, end - start, m, replacements[idx]);
sourcemap = tokens.sourcemap();
re.lastIndex = 0;
}
@@ -192,7 +195,7 @@ function parseBody(text, text_index = 0) {
chunks = [
/\{([SBVE;]*)(\})/g, // statement block -> B
/I([SBVE;])L?[SBVE;]/g, // if (Expression) Statement/Block Else -> S
/I([SBVE;])(L[SBVE;])?/g, // if (Expression) Statement/Block Else -> S
/F[SBVE;]/g, // for loop -> S
/P(\{)(Q+[SBVE]*)*(\}?)/g, // switch(Expression){ Q(caseblock),... } -> S
/try(B)(C?B?)(N?B?)/g, // try, Block, catch/finally -> S
@@ -208,7 +211,7 @@ function parseBody(text, text_index = 0) {
for (let m; m = re.exec(sourcemap.simplified);) {
let start = sourcemap.map[m.index];
let end = sourcemap.map[m.index + m[0].length];
tokens.shrink(ids[idx], start, end - start, replacements[idx]);
tokens.shrink(ids[idx], start, end - start, m, replacements[idx]);
sourcemap = tokens.sourcemap();
re.lastIndex = 0;
}
@@ -220,124 +223,6 @@ function parseBody(text, text_index = 0) {
}
const expressions = [
'1 for(){}',
'a',
'true',
'null',
`""`,
`'c'`,
// operators
`1 + 2`,
`1 - 2`,
`1 * 2`,
`1 / 2`,
`1 % 2`,
`1 & 2`,
`1 | 2`,
`1 ^ 2`,
`1 < 2`,
`1 <= 2`,
`1 << 2`,
`1 > 2`,
`1 >= 2`,
`1 >> 2`,
`1 == 2`,
`1 instanceof 2`,
// assignment operators
`a += 2`,
`a -= 2`,
`a *= 2`,
`a /= 2`,
`a %= 2`,
`a &= 2`,
`a |= 2`,
`a ^= 2`,
`a <<= 2`,
`a >>= 2`,
// member, array, methodcall
`a.b`,
`a.b.c`,
`a[1]`,
`a[1,2]`,
`a[1][2]`,
`a()`,
`a(b)`,
`a(b, "")`,
`a.b()`,
`a.b()[1]`,
];
expressions.map(e => {
extractExpression(e);
})
const src =
`for (int i=0; i < 10; i++) {
do {
if (i) {
System.out.println("1234");
}
#
switch(x) {
case 4:
case 5:
return x;
case 6:
default:
return;
}
while (x > 0) true;
} while (i > 0);
while (x > 0)
System.out.println("1234");
}
`
class BlockRange {
get end() { return this.start + this.length }
get text() { return this.source.slice(this.start, this.end) }
/**
*
* @param {string} source
* @param {number} start
* @param {number} length
*/
constructor(source, start, length) {
this.source = source;
this.start = start;
this.length = length;
}
}
class TextBlock {
/**
* @param {BlockRange|TextBlockArray} range
* @param {string} simplified
*/
constructor(range, simplified) {
this.range = range;
this.simplified = simplified;
}
/**
* @param {string} source
* @param {number} start
* @param {number} length
* @param {string} [simplified]
*/
static from(source, start, length, simplified) {
const range = new BlockRange(source, start, length);
return new TextBlock(range, simplified || range.text);
}
toSource() {
return this.range instanceof BlockRange
? this.range.text
: this.range.toSource()
}
}
class ParsedExpressionBlock extends TextBlock {
/**
* @param {string} source
@@ -375,55 +260,7 @@ class InvalidTextBlock extends TextBlock {
}
}
class TextBlockArray {
/**
* @param {string} id
* @param {TextBlock[]} [blocks]
*/
constructor(id, blocks = []) {
this.id = id;
this.blocks = blocks;
}
get simplified() {
return this.blocks.map(tb => tb.simplified).join('');
}
sourcemap() {
let idx = 0;
const parts = [];
/** @type {number[]} */
const map = this.blocks.reduce((arr,tb,i) => {
arr[idx] = i;
parts.push(tb.simplified);
idx += tb.simplified.length;
return arr;
}, []);
map[idx] = this.blocks.length;
return {
simplified: parts.join(''),
map,
}
}
/**
* @param {string} id
* @param {number} start
* @param {number} count
* @param {string} simplified
*/
shrink(id, start, count, simplified) {
if (count <= 0) return;
const collapsed = new TextBlockArray(id, this.blocks.splice(start, count, null));
this.blocks[start] = new TextBlock(collapsed, simplified);
}
get source() { return this.toSource() }
toSource() {
return this.blocks.map(tb => tb.toSource()).join('');
}
module.exports = {
parseBody,
}
parseBody(src);

View File

@@ -1,6 +1,4 @@
/**
* @typedef {import('./parsetypes/import')} ImportDeclaration
*/
const { ImportBlock } = require('./parser9');
const ResolvedImport = require('./parsetypes/resolved-import');
/**
@@ -25,11 +23,10 @@ function fetchImportedTypes(typenames, dotted_import, demandload) {
/**
* @param {string} typenames newline-separated list of fully qualified type names
* @param {import('./parsetypes/import')} import_decl import declaration
* @param {ImportBlock} import_decl import declaration
*/
function resolveImportTypes(typenames, import_decl) {
const dotted = import_decl.getDottedName();
return fetchImportedTypes(typenames, dotted, !!import_decl.asterisk);
return fetchImportedTypes(typenames, import_decl.name, import_decl.isDemandLoad);
}
/**
@@ -41,7 +38,7 @@ function resolveImportTypes(typenames, import_decl) {
* - followed by implicit packages
*
* @param {*} androidLibrary imported types from the Android platform library
* @param {import('./parsetypes/import')[]} imports list of declared imports in the module
* @param {ImportBlock[]} imports list of declared imports in the module
* @param {string} package_name package name of the module
* @param {import('./mti').Type[]} source_mtis MTIs representing types declared in the source
* @param {string[]} [implicitPackages] list of implicit demand-load packages
@@ -63,7 +60,7 @@ function resolveImports(androidLibrary, imports, package_name, source_mtis, impl
/**
* The list of explicit import declarations we are unable to resolve
* @type {ImportDeclaration[]}
* @type {ImportBlock[]}
*/
const unresolved = [];

View File

@@ -86,14 +86,15 @@ class MTI extends MinifiableInfo {
* @param {string[]} modifiers
* @param {'class'|'enum'|'interface'|'@interface'} typeKind
* @param {string} name
* @param {string[]} typeVarNames
*/
addType(package_name, docs, modifiers, typeKind, name) {
addType(package_name, docs, modifiers, typeKind, name, typeVarNames) {
const t = {
d: docs,
p: this.addPackage(package_name),
m: getTypeMods(modifiers, typeKind),
n: name.replace(/\./g,'$'),
v: [],
v: typeVarNames.map(name => this.addRefType('', name)),
e: /interface/.test(typeKind) ? []
: typeKind === 'enum' ? this.addRefType('java.lang', 'Enum')
: this.addRefType('java.lang', 'Object'),
@@ -353,6 +354,11 @@ class MTITypeBase extends MinifiableInfo {
*/
get methods() { return [] }
/**
* @type {ReferencedType[]}
*/
get typevars() { return [] }
/**
* @param {string} name
*/

1028
langserver/java/parser9.js Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
const ProblemSeverity = require('./problem-severity');
const Token = require('./token');
const { TextBlock } = require('./textblock');
/**
* @typedef {import('./import')} ImportDeclaration
@@ -11,123 +11,61 @@ const Token = require('./token');
class ParseProblem {
/**
* @param {Token|Token[]} token
* @param {TextBlock|TextBlock[]} token
* @param {string} message
* @param {Severity} severity
*/
constructor(token, message, severity) {
this.startIdx = (Array.isArray(token) ? token[0] : token).source_idx;
const lastToken = (Array.isArray(token) ? token[token.length - 1] : token);
this.endIdx = lastToken.source_idx + lastToken.text.length;
if (Array.isArray(token)) {
this.startIdx = token[0].range.start;
const lastToken = token[token.length - 1];
this.endIdx = lastToken.range.start + lastToken.range.length;
} else {
this.startIdx = token.range.start;
this.endIdx = this.startIdx + token.range.length;
}
this.message = message;
this.severity = severity;
}
/**
* @param {Modifier[]} mods
* @param {TextBlock|TextBlock[]} token
* @param {string} message
*/
static checkDuplicateModifiers(mods) {
const done = new Set();
const res = [];
for (let mod of mods) {
if (mod instanceof Token) {
if (done.has(mod.text)) {
res.push(new ParseProblem(mod, `Duplicate modifier: ${mod.text}`, ProblemSeverity.Error));
}
done.add(mod.text);
}
}
return res;
}
static checkConflictingModifiers(mods) {
const modmap = new Map();
let res = [];
mods.filter(m => m instanceof Token).forEach(m => modmap.set(m.text, m));
const names = [...modmap.keys()];
const visibilities = names.filter(m => /^(public|private|protected)$/.test(m));
if (visibilities.length > 1) {
const visnames = visibilities.map(m => `'${m}'`).join(', ').replace(/, (?='\w+'$)/, ' and ');
res = visibilities.map(m => new ParseProblem(modmap.get(m), `Conflicting modifiers: ${visnames}`, ProblemSeverity.Error));
}
if (names.includes('abstract')) {
if (names.includes('final')) {
res.push(new ParseProblem(modmap.get('final'), `Declarations cannot be both 'abstract' and 'final`, ProblemSeverity.Error));
}
if (names.includes('native')) {
res.push(new ParseProblem(modmap.get('native'), `Declarations cannot be both 'abstract' and 'native`, ProblemSeverity.Error));
}
}
return res;
static Error(token, message) {
return new ParseProblem(token, message, ProblemSeverity.Error);
}
/**
* @param {Modifier[]} mods
* @param {'class'|'interface'|'enum'|'@interface'|'field'|'method'|'constructor'|'initializer'} decl_kind
* @param {TextBlock|TextBlock[]} token
* @param {string} message
*/
static checkAccessModifiers(mods, decl_kind) {
let valid_mods = /^$/;
switch (decl_kind) {
case 'class': valid_mods = /^(public|final|abstract|strictfp)$/; break;
case 'interface': valid_mods = /^(public|abstract|strictfp)$/; break;
case '@interface': valid_mods = /^(public)$/; break;
case 'enum': valid_mods = /^(public|final)$/; break;
case 'field': valid_mods = /^(public|private|protected|static|final|volatile|transient)$/; break;
case 'method': valid_mods = /^(public|private|protected|static|final|abstract|native|strictfp|synchronized)$/; break;
case 'constructor': valid_mods = /^(public|protected|native)$/; break;
case 'initializer': valid_mods = /^(static)$/; break;
}
const problems = [];
for (let mod of mods) {
if (mod instanceof Token) {
if (!valid_mods.test(mod.text)) {
problems.push(new ParseProblem(mod, `'${mod.text}' is not a valid modifier for ${decl_kind} type declarations`, ProblemSeverity.Warning));
}
const redundant = (mod.text === 'abstract' && decl_kind === 'interface')
|| (mod.text === 'final' && decl_kind === 'enum');
if (redundant) {
problems.push(new ParseProblem(mod, `'${mod.text}' is redundant for a ${decl_kind} declaration`, ProblemSeverity.Hint));
}
}
}
return problems;
static Warning(token, message) {
return new ParseProblem(token, message, ProblemSeverity.Warning);
}
/**
* @param {PackageDeclaration|ImportDeclaration} o
* @param {TextBlock|TextBlock[]} token
* @param {string} message
*/
static checkSemicolon(o) {
if (!o.semicolon) {
const lastToken = o.lastToken();
return new ParseProblem(lastToken, 'Missing operator or semicolon', ProblemSeverity.Error);
}
static Information(token, message) {
return new ParseProblem(token, message, ProblemSeverity.Information);
}
/**
* @param {Token[]} tokens
* @param {TextBlock|TextBlock[]} token
* @param {string} message
*/
static checkNonKeywordIdents(tokens) {
const res = [];
const KEYWORDS = /^(abstract|assert|break|case|catch|class|const|continue|default|do|else|enum|extends|final|finally|for|goto|if|implements|import|interface|native|new|package|private|protected|public|return|static|strictfp|super|switch|synchronized|throw|throws|transient|try|volatile|while)$/;
const PRIMITIVE_TYPE_KEYWORDS = /^(int|boolean|byte|char|double|float|long|short|void)$/
const LITERAL_VALUE_KEYWORDS = /^(this|true|false|null)$/;
const OPERATOR_KEYWORDS = /^(instanceof)$/;
for (let token of tokens) {
let iskw = KEYWORDS.test(token.text) || PRIMITIVE_TYPE_KEYWORDS.test(token.text) || LITERAL_VALUE_KEYWORDS.test(token.text) || OPERATOR_KEYWORDS.test(token.text);
if (iskw) {
const problem = new ParseProblem(token, `'${token.text}' is a keyword and cannot be used as an identifier`, ProblemSeverity.Error);
res.push(problem);
}
}
return res;
static Hint(token, message) {
return new ParseProblem(token, message, ProblemSeverity.Hint);
}
/**
* @param {Token} token
* @param {TextBlock|TextBlock[]} token
*/
static syntaxError(token) {
if (!token) return null;
return new ParseProblem(token, 'Unsupported, invalid or incomplete declaration', ProblemSeverity.Error);
return ParseProblem.Error(token, 'Unsupported, invalid or incomplete declaration');
}
}

View File

@@ -1,6 +1,4 @@
/**
* @typedef {import('./import')} ImportDeclaration
*/
const { ImportBlock } = require('../parser9');
/**
* Class representing a resolved import.
@@ -11,9 +9,10 @@
*/
class ResolvedImport {
/**
* @param {ImportDeclaration} import_decl
* @param {ImportBlock} import_decl
* @param {RegExpMatchArray} matches
* @param {'owner-package'|'import'|'implicit-import'} import_kind;
* @param {Map<string,import('../mti').Type>} typemap
* @param {'owner-package'|'import'|'implicit-import'} import_kind
*/
constructor(import_decl, matches, typemap, import_kind) {
/**

View File

@@ -58,6 +58,10 @@ class ResolvedType {
get label() {
return this.name + (this.typeargs ? `<${this.typeargs.map(arg => arg.label).join(',')}>` : '');
}
get rawlabel() {
return this.name;
}
}
/** @type {ResolvedType.TypePart[]} */
@@ -79,6 +83,13 @@ class ResolvedType {
*/
mtis = [];
/**
* @param {boolean} [isTypeArg]
*/
constructor(isTypeArg = false) {
this.isTypeArg = isTypeArg;
}
/**
* During parsing, add a new type part
* @param {string} [name]
@@ -97,9 +108,20 @@ class ResolvedType {
return this.parts.map(p => p.name).join('.');
}
get isPrimitive() {
if (this.arrdims > 0 || this.parts.length !== 1) {
return false;
}
return /^(int|boolean|char|void|byte|long|double|float|short)$/.test(this.parts[0].name);
}
get label() {
return this.parts.map(p => p.label).join('.') + '[]'.repeat(this.arrdims);
}
get rawlabel() {
return this.parts.map(p => p.rawlabel).join('.') + '[]'.repeat(this.arrdims);
}
};
module.exports = ResolvedType;

View File

@@ -0,0 +1,138 @@
class BlockRange {
get end() { return this.start + this.length }
get text() { return this.source.slice(this.start, this.end) }
/**
*
* @param {string} source
* @param {number} start
* @param {number} length
*/
constructor(source, start, length) {
this.source = source;
this.start = start;
this.length = length;
}
}
class TextBlock {
/**
* @param {BlockRange|TextBlockArray} range
* @param {string} simplified
*/
constructor(range, simplified) {
this.range = range;
this.simplified = simplified;
}
blockArray() {
return this.range instanceof TextBlockArray ? this.range : null;
}
/**
* Returns the length of the original source
* @returns {number}
*/
get length() {
return this.range.length;
}
/**
* @param {string} source
* @param {number} start
* @param {number} length
* @param {string} [simplified]
*/
static from(source, start, length, simplified) {
const range = new BlockRange(source, start, length);
return new TextBlock(range, simplified || range.text);
}
get source() { return this.toSource() }
/**
* @returns {string}
*/
toSource() {
return this.range instanceof BlockRange
? this.range.text
: this.range.toSource()
}
}
class TextBlockArray {
/**
* @param {string} id
* @param {TextBlock[]} [blocks]
*/
constructor(id, blocks = []) {
this.id = id;
this.blocks = blocks;
}
/**
* Returns the length of the original source
* @returns {number}
*/
get length() {
return this.blocks.reduce(((len,b) => len + b.length), 0);
}
get simplified() {
return this.blocks.map(tb => tb.simplified).join('');
}
/** @returns {number} */
get start() {
return this.blocks[0].range.start;
}
sourcemap() {
let idx = 0;
const parts = [];
/** @type {number[]} */
const map = this.blocks.reduce((arr,tb,i) => {
arr[idx] = i;
if (!tb) {
throw this.blocks;
}
parts.push(tb.simplified);
idx += tb.simplified.length;
return arr;
}, []);
map[idx] = this.blocks.length;
return {
simplified: parts.join(''),
map,
}
}
/**
* @param {string} id
* @param {number} start_block_idx
* @param {number} block_count
* @param {RegExpMatchArray} match
* @param {string} marker
* @param {*} [parseClass]
*/
shrink(id, start_block_idx, block_count, match, marker, parseClass) {
if (block_count <= 0) return;
const collapsed = new TextBlockArray(id, this.blocks.splice(start_block_idx, block_count, null));
const simplified = collapsed.source.replace(/./g, ' ').replace(/^./, marker);
return this.blocks[start_block_idx] = parseClass
? new parseClass(collapsed, simplified, match)
: new TextBlock(collapsed, simplified);
}
get source() { return this.toSource() }
toSource() {
return this.blocks.map(tb => tb.toSource()).join('');
}
}
module.exports = {
BlockRange,
TextBlock,
TextBlockArray,
}

View File

@@ -26,7 +26,7 @@ function parse_type(label) {
if (m[0] === '<') {
if (!parts[0].typeargs && !parts[0].owner.arrdims) {
// start of type arguments - start a new type
const t = new ResolvedType();
const t = new ResolvedType(true);
parts[0].typeargs = [t];
parts.unshift(t.addTypePart());
continue;
@@ -36,7 +36,7 @@ function parse_type(label) {
if (m[0] === ',') {
if (parts[1] && parts[1].typeargs) {
// type argument separator - replace the type on the stack
const t = new ResolvedType();
const t = new ResolvedType(true);
parts[1].typeargs.push(t);
parts[0] = t.addTypePart();
continue;
@@ -202,7 +202,7 @@ function findRawTypeMTIs(dotted_raw_typename, fully_qualified_scope, resolved_im
}
// if the type matches multiple import entries, exact imports take prioirity over demand-load imports
let exact_import_matches = matched_types.filter(x => x.ri.import && !x.ri.import.asterisk);
let exact_import_matches = matched_types.filter(x => x.ri.import && !x.ri.import.isDemandLoad);
if (exact_import_matches.length) {
if (exact_import_matches.length < matched_types.length) {
matched_types = exact_import_matches;

View File

@@ -0,0 +1,66 @@
const { ModuleBlock, TypeDeclBlock } = require('./parser9');
const { resolveImports } = require('../java/import-resolver');
const MTI = require('./mti');
/**
* @param {string} package_name
* @param {string} owner_typename
* @param {ModuleBlock|TypeDeclBlock} parent
* @param {MTI.Type[]} mtis
*/
function getSourceMTIs(package_name, owner_typename, parent, mtis) {
parent.types.forEach(type => {
const mods = type.modifiers.map(m => m.source);
const qualifiedTypeName = `${owner_typename}${type.simpleName}`;
// we add the names of type variables here, but we resolve any bounds later
const typevar_names = type.typevars.map(tv => tv.name);
const mti = new MTI().addType(package_name, '', mods, type.kind(), qualifiedTypeName, typevar_names);
mtis.push(mti);
getSourceMTIs(package_name, `${qualifiedTypeName}$`, type, mtis);
});
}
/**
* @param {ModuleBlock} mod
*/
function validate(mod, androidLibrary) {
console.time('validation');
const source_mtis = [];
getSourceMTIs(mod.packageName, '', mod, source_mtis);
const imports = resolveImports(androidLibrary, mod.imports, mod.packageName, source_mtis);
const module_validaters = [
require('./validation/multiple-package-decls'),
require('./validation/unit-decl-order'),
require('./validation/duplicate-members'),
require('./validation/parse-errors'),
require('./validation/modifier-errors'),
require('./validation/unresolved-imports'),
require('./validation/resolved-types'),
];
let problems = [
module_validaters.map(v => v(mod, imports)),
];
console.timeEnd('validation');
function flatten(arr) {
let res = arr;
for (;;) {
const idx = res.findIndex(x => Array.isArray(x));
if (idx < 0) {
return res;
}
res = [...res.slice(0, idx), ...res[idx], ...res.slice(idx+1)]
}
}
let flattened = flatten(problems).filter(x => x);
return flattened;
}
module.exports = {
validate,
}

View File

@@ -0,0 +1,93 @@
const { ModuleBlock, FieldBlock, TypeDeclBlock } = require('../parser9');
const ParseProblem = require('../parsetypes/parse-problem');
/**
*
* @param {TypeDeclBlock} type
* @param {ParseProblem[]} probs
*/
function checkDuplicateFieldName(type, probs) {
/** @type {Map<string,FieldBlock>} */
let names = new Map();
type.fields.forEach(field => {
if (!field.name) {
return;
}
const value = names.get(field.name);
if (value === undefined) {
names.set(field.name, field);
} else {
if (value !== null) {
probs.push(ParseProblem.Error(value, `Duplicate field: ${field.name}`));
names.set(field.name, null);
}
probs.push(ParseProblem.Error(field, `Duplicate field: ${field.name}`));
}
})
// check enclosed types
type.types.forEach(type => checkDuplicateFieldName(type, probs));
}
/**
* @param {string} outername
* @param {TypeDeclBlock[]} types
* @param {ParseProblem[]} probs
*/
function checkDuplicateTypeNames(outername, types, probs) {
/** @type {Map<string,TypeDeclBlock>} */
let names = new Map();
types.forEach(type => {
const name = type.simpleName;
if (!name) {
return;
}
const value = names.get(name);
if (value === undefined) {
names.set(name, type);
} else {
if (value !== null) {
probs.push(ParseProblem.Error(value.name_token, `Duplicate type: ${outername}${name}`));
names.set(name, null);
}
probs.push(ParseProblem.Error(type.name_token, `Duplicate type: ${outername}${name}`));
}
})
// check enclosed types
types.forEach(type => {
checkDuplicateTypeNames(`${outername}${type.simpleName}.`, type.types, probs);
});
}
/**
* @param {TypeDeclBlock} type
* @param {ParseProblem[]} probs
*/
function checkDuplicateTypeVariableName(type, probs) {
type.typevars.forEach((tv, i) => {
const name = tv.name;
if (tv.name === '?') {
return;
}
if (type.typevars.findIndex(tv => tv.name === name) < i) {
probs.push(ParseProblem.Error(tv.decl, `Duplicate type variable: ${name}`));
}
})
// check enclosed types
type.types.forEach(type => {
checkDuplicateTypeVariableName(type, probs);
});
}
/**
* @param {ModuleBlock} mod
*/
module.exports = function(mod) {
const probs = [];
mod.types.forEach(type => {
checkDuplicateFieldName(type, probs);
checkDuplicateTypeVariableName(type, probs);
});
checkDuplicateTypeNames('', mod.types, probs);
return probs;
}

View File

@@ -0,0 +1,138 @@
const { TextBlock, ModuleBlock, FieldBlock, MethodBlock, ConstructorBlock, InitialiserBlock, TypeDeclBlock } = require('../parser9');
const ParseProblem = require('../parsetypes/parse-problem');
/**
* @param {TextBlock[]} mods
* @param {ParseProblem[]} probs
*/
function checkDuplicate(mods, probs) {
if (mods.length <= 1) {
return;
}
const m = new Map();
for (let mod of mods) {
const firstmod = m.get(mod.source);
if (firstmod === undefined) {
m.set(mod.source, mod);
} else {
probs.push(ParseProblem.Error(mod, 'Duplicate modifier'));
if (firstmod !== null) {
probs.push(ParseProblem.Error(firstmod, 'Duplicate modifier'));
m.set(mod.source, null);
}
}
}
}
/**
* @param {TextBlock[]} mods
* @param {ParseProblem[]} probs
*/
function checkConflictingAccess(mods, probs) {
if (mods.length <= 1) {
return;
}
const allmods = mods.map(m => m.source).join(' ');
for (let mod of mods) {
let match;
switch (mod.source) {
case 'private':
match = allmods.match(/protected|public/);
break;
case 'protected':
match = allmods.match(/private|public/);
break;
case 'public':
match = allmods.match(/private|protected/);
break;
}
if (match) {
probs.push(ParseProblem.Error(mod, `Access modifier '${mod.source}' conflicts with '${match[0]}'`));
}
}
}
/**
* @param {FieldBlock} field
* @param {ParseProblem[]} probs
*/
function checkFieldModifiers(field, probs) {
checkDuplicate(field.modifiers, probs);
checkConflictingAccess(field.modifiers, probs);
for (let mod of field.modifiers) {
switch (mod.source) {
case 'abstract':
probs.push(ParseProblem.Error(mod, 'Field declarations cannot be abstract'));
break;
case 'native':
probs.push(ParseProblem.Error(mod, 'Field declarations cannot be native'));
break;
}
}
}
/**
* @param {Set<string>} ownertypemods
* @param {MethodBlock} method
* @param {ParseProblem[]} probs
*/
function checkMethodModifiers(ownertypemods, method, probs) {
checkDuplicate(method.modifiers, probs);
checkConflictingAccess(method.modifiers, probs);
const allmods = new Map(method.modifiers.map(m => [m.source, m]));
if (allmods.has('abstract') && allmods.has('final')) {
probs.push(ParseProblem.Error(allmods.get('abstract'), 'Method declarations cannot be abstract and final'));
}
if (allmods.has('abstract') && allmods.has('native')) {
probs.push(ParseProblem.Error(allmods.get('abstract'), 'Method declarations cannot be abstract and native'));
}
if (allmods.has('abstract') && method.body().simplified.startsWith('B')) {
probs.push(ParseProblem.Error(allmods.get('abstract'), 'Method declarations marked as abstract cannot have a method body'));
}
if (!allmods.has('abstract') && !allmods.has('native') && !method.body().simplified.startsWith('B')) {
probs.push(ParseProblem.Error(method, `Method '${method.name}' must have an implementation or be defined as abstract or native`));
}
if (allmods.has('abstract') && !ownertypemods.has('abstract')) {
probs.push(ParseProblem.Error(method, `Method '${method.name}' cannot be declared abstract inside a non-abstract type`));
}
if (allmods.has('native') && method.body().simplified.startsWith('B')) {
probs.push(ParseProblem.Error(allmods.get('native'), 'Method declarations marked as native cannot have a method body'));
}
}
/**
* @param {ConstructorBlock} field
* @param {ParseProblem[]} probs
*/
function checkConstructorModifiers(field, probs) {
}
/**
* @param {InitialiserBlock} initialiser
* @param {ParseProblem[]} probs
*/
function checkInitialiserModifiers(initialiser, probs) {
}
/**
* @param {TypeDeclBlock} type
* @param {ParseProblem[]} probs
*/
function checkTypeModifiers(type, probs) {
const typemods = new Set(type.modifiers.map(m => m.source));
type.fields.forEach(field => checkFieldModifiers(field, probs));
type.methods.forEach(method => checkMethodModifiers(typemods, method, probs));
type.constructors.forEach(ctr => checkConstructorModifiers(ctr, probs));
//type.initialisers.forEach(initer => checkInitModifiers(initer, probs));
// check enclosed types
type.types.forEach(type => checkTypeModifiers(type, probs));
}
/**
* @param {ModuleBlock} mod
*/
module.exports = function(mod) {
const probs = [];
mod.types.forEach(type => checkTypeModifiers(type, probs));
return probs;
}

View File

@@ -0,0 +1,13 @@
const { ModuleBlock } = require('./../parser9');
const ParseProblem = require('./../parsetypes/parse-problem');
/**
* @param {ModuleBlock} mod
*/
module.exports = function(mod) {
return mod.packages.slice(1).map(
pkg => {
return ParseProblem.Error(pkg, 'Additional package declaration');
}
)
}

View File

@@ -0,0 +1,29 @@
const { ModuleBlock, TypeDeclBlock, MethodBlock } = require('../parser9');
const ParseProblem = require('../parsetypes/parse-problem');
/**
* @param {TypeDeclBlock} type
* @param {ParseProblem[]} probs
*/
function checkTypeParseErrors(type, probs) {
type.parseErrors.forEach(err => probs.push(ParseProblem.Error(err, `Invalid, incomplete or unsupported declaration`)));
type.methods.filter(m => m.parseErrors).forEach(m => checkMethodParseErrors(m, probs));
type.types.forEach(type => checkTypeParseErrors(type, probs));
}
/**
* @param {MethodBlock} method
* @param {ParseProblem[]} probs
*/
function checkMethodParseErrors(method, probs) {
method.parseErrors.forEach(err => probs.push(ParseProblem.Error(err, `Invalid, incomplete or unsupported declaration`)));
}
/**
* @param {ModuleBlock} mod
*/
module.exports = function(mod) {
const probs = [];
mod.types.forEach(type => checkTypeParseErrors(type, probs));
return probs;
}

View File

@@ -0,0 +1,255 @@
/**
* @typedef {import('../parsetypes/resolved-import')} ResolvedImport
*/
const { ModuleBlock, TypeDeclBlock, DeclaredVariableBlock, MethodBlock, ParameterBlock, TextBlock } = require('../parser9');
const ParseProblem = require('../parsetypes/parse-problem');
const { resolveTypes } = require('../type-resolver');
const ResolvedType = require('../parsetypes/resolved-type');
/**
* @param {DeclaredVariableBlock|MethodBlock|TextBlock[] & {typeTokens: *[]}} decl
* @param {ResolvedType|ResolvedType[]} resolved
* @param {ParseProblem[]} probs
*/
function checkResolvedTypes(decl, resolved, probs) {
if (Array.isArray(resolved)) {
resolved.forEach(resolved => checkResolvedTypes(decl, resolved, probs));
return;
}
if (resolved.error) {
probs.push(ParseProblem.Error(decl, resolved.error));
return;
}
// the parser will detect varargs (...) on all variable declarations
if (decl instanceof DeclaredVariableBlock && decl.isVarArgs && !(decl instanceof ParameterBlock)) {
probs.push(ParseProblem.Error(decl.varBlock.varargs_token, `Variable-arity can only be applied to parameter declarations.`));
}
// void arrays are illegal
if (/^void\[/.test(resolved.rawlabel)) {
probs.push(ParseProblem.Error(decl.typeTokens, `Invalid type: ${resolved.rawlabel}`));
return;
}
// void can only be used for method declarations
if (resolved.rawlabel === 'void' && decl instanceof DeclaredVariableBlock) {
probs.push(ParseProblem.Error(decl.typeTokens, `'void' is not a valid type for fields, parameters or variables`));
return;
}
// no primitive type arguments
if (resolved.isTypeArg && resolved.isPrimitive) {
probs.push(ParseProblem.Error(decl.typeTokens, `Primitive types cannot be used as type arguments.`));
return;
}
switch (resolved.mtis.length) {
case 0:
probs.push(ParseProblem.Error(decl.typeTokens, `Unresolved type: '${resolved.rawlabel}'`));
break;
case 1:
break;
default:
const matchlist = resolved.mtis.map(m => `'${m.fullyDottedRawName}'`).join(', ');
probs.push(ParseProblem.Error(decl.typeTokens, `Ambiguous type: '${resolved.rawlabel}'. Possible matches: ${matchlist}.`));
break;
}
// check type arguments
resolved.parts
.filter(typepart => typepart.typeargs)
.forEach(typepart => {
checkResolvedTypes(decl, typepart.typeargs, probs);
// check number of type arguments match
if (resolved.mtis.length === 1 && typepart.typeargs.length !== resolved.mtis[0].typevars.length) {
const msg = resolved.mtis[0].typevars.length === 0
? `Type '${resolved.mtis[0].fullyDottedRawName}' is not declared as a parameterized type and cannot be used with type arguments.`
: `Wrong number of type arguments for: '${resolved.mtis[0].fullyDottedRawName}'. Expected ${resolved.mtis[0].typevars.length} but found ${typepart.typeargs.length}.`;
probs.push(ParseProblem.Error(decl.typeTokens, msg));
}
});
}
/**
* @param {string} outername
* @param {TypeDeclBlock} owner_type
* @param {''|'.'|'$'} qualifier
* @param {ResolvedImport[]} resolved_imports
* @param {Map<string, import('../mti').Type>} typemap
* @param {ParseProblem[]} probs
*/
function resolveFieldTypes(outername, owner_type, qualifier, resolved_imports, typemap, probs) {
const fieldtypes = owner_type.fields.map(f => f.type);
const fully_qualified_scope_name = `${outername}${qualifier}${owner_type.simpleName}`;
const resolved = resolveTypes(fieldtypes, fully_qualified_scope_name, resolved_imports, typemap);
owner_type.fields.forEach((field,i) => {
checkResolvedTypes(field, resolved[i], probs);
})
// check enclosed types
owner_type.types.forEach(type => {
resolveFieldTypes(fully_qualified_scope_name, type, '$', resolved_imports, typemap, probs);
});
}
function extractTypeList(decl) {
if (!decl) {
return [];
}
const types = [];
const re = /[WD]( *[WDT.])*/g;
decl = decl.blockArray();
const sm = decl.sourcemap();
for (let m; m = re.exec(sm.simplified);) {
const start = sm.map[m.index], end = sm.map[m.index + m[0].length-1];
const block_range = decl.blocks.slice(start, end+1);
const typename = block_range.map(b => b.source).join('');
block_range.typename = typename;
block_range.typeTokens = block_range;
types.push(block_range);
}
return types;
}
/**
* @param {string} outername
* @param {TypeDeclBlock} owner_type
* @param {''|'.'|'$'} qualifier
* @param {ResolvedImport[]} resolved_imports
* @param {Map<string, import('../mti').Type>} typemap
* @param {ParseProblem[]} probs
*/
function resolveExtends(outername, owner_type, qualifier, resolved_imports, typemap, probs) {
if (!owner_type.extends_token) {
return;
}
// the scope for extends and implements needs to include any type variables, but not enclosed types
const fully_qualified_scope_name = `${outername}${qualifier}${owner_type.simpleName}`;
if (!/^(class|interface)/.test(owner_type.kind())) {
probs.push(ParseProblem.Error(owner_type.extends_token, `extends declaration is not valid for ${owner_type.kind()} type: ${fully_qualified_scope_name}`));
return;
}
const eit_types = extractTypeList(owner_type.extends_token);
const resolved = resolveTypes(eit_types.map(x => x.typename), fully_qualified_scope_name, resolved_imports, typemap);
eit_types.forEach((eit_type,i) => {
checkResolvedTypes(eit_type, resolved[i], probs);
})
switch(owner_type.kind()) {
case 'class':
if (eit_types[0] && resolved[0].mtis.length === 1 && resolved[0].mtis[0].typeKind !== 'class') {
probs.push(ParseProblem.Error(eit_types[0], `Class '${fully_qualified_scope_name}' cannot extend from ${resolved[0].mtis[0].typeKind} type '${resolved[0].mtis[0].fullyDottedRawName}'`));
}
if (eit_types.length > 1) {
probs.push(ParseProblem.Error(eit_types[1], `Class types cannot extend from more than one type`));
}
break;
case "interface":
eit_types.forEach((eit_type, i) => {
const mti = resolved[i].mtis[0];
if (resolved[i].mtis.length === 1 && mti.typeKind !== 'interface') {
probs.push(ParseProblem.Error(eit_type, `Interface '${fully_qualified_scope_name}' cannot extend from ${mti.typeKind} type '${mti.fullyDottedRawName}'`));
}
// check for repeated types
if (resolved[i].mtis.length === 1) {
const name = resolved[i].mtis[0].fullyDottedRawName;
if (resolved.findIndex(r => r.mtis.length === 1 && r.mtis[0].fullyDottedRawName === name) < i) {
probs.push(ParseProblem.Error(eit_types[1], `Repeated type: ${name}`));
}
}
})
break;
}
// check enclosed types
owner_type.types.forEach(type => {
resolveExtends(fully_qualified_scope_name, type, '$', resolved_imports, typemap, probs);
});
}
/**
* @param {string} outername
* @param {TypeDeclBlock} owner_type
* @param {''|'.'|'$'} qualifier
* @param {ResolvedImport[]} resolved_imports
* @param {Map<string, import('../mti').Type>} typemap
* @param {ParseProblem[]} probs
*/
function resolveImplements(outername, owner_type, qualifier, resolved_imports, typemap, probs) {
if (!owner_type.implements_token) {
return;
}
const fully_qualified_scope_name = `${outername}${qualifier}${owner_type.simpleName}`;
if (!/class/.test(owner_type.kind())) {
probs.push(ParseProblem.Error(owner_type.implements_token, `implements declaration is not valid for ${owner_type.kind()} type: ${fully_qualified_scope_name}`));
return;
}
const eit_types = extractTypeList(owner_type.implements_token);
// the scope for extends and implements needs to include any type variables, but not enclosed types
const resolved = resolveTypes(eit_types.map(x => x.typename), fully_qualified_scope_name, resolved_imports, typemap);
eit_types.forEach((eit_type,i) => {
checkResolvedTypes(eit_type, resolved[i], probs);
})
eit_types.forEach((eit_type, i) => {
const mti = resolved[i].mtis[0];
if (resolved[i].mtis.length === 1 && mti.typeKind !== 'interface') {
probs.push(ParseProblem.Error(eit_type, `Interface '${fully_qualified_scope_name}' cannot extend from ${mti.typeKind} type '${mti.fullyDottedRawName}'`));
}
// check for repeated types
if (resolved[i].mtis.length === 1) {
const name = resolved[i].mtis[0].fullyDottedRawName;
if (resolved.findIndex(r => r.mtis.length === 1 && r.mtis[0].fullyDottedRawName === name) < i) {
probs.push(ParseProblem.Error(eit_types[1], `Repeated type: ${name}`));
}
}
})
// check enclosed types
owner_type.types.forEach(type => {
resolveImplements(fully_qualified_scope_name, type, '$', resolved_imports, typemap, probs);
});
}
/**
* @param {string} outername
* @param {TypeDeclBlock} owner_type
* @param {''|'.'|'$'} qualifier
* @param {ResolvedImport[]} resolved_imports
* @param {Map<string, import('../mti').Type>} typemap
* @param {ParseProblem[]} probs
*/
function resolveMethodTypes(outername, owner_type, qualifier, resolved_imports, typemap, probs) {
const method_type_names = [];
owner_type.methods.forEach(m => {
method_type_names.push(m.type);
m.parameters.forEach(p => {
method_type_names.push(p.type);
});
});
const fully_qualified_scope_name = `${outername}${qualifier}${owner_type.simpleName}`;
const resolved = resolveTypes(method_type_names, fully_qualified_scope_name, resolved_imports, typemap);
let i = 0;
owner_type.methods.forEach(method => {
checkResolvedTypes(method, resolved[i++], probs);
method.parameters.forEach((parameter, idx, arr) => {
checkResolvedTypes(parameter, resolved[i++], probs);
if (parameter.isVarArgs && idx !== arr.length-1) {
probs.push(ParseProblem.Error(parameter, `Variable-arity parameters must be declared last.`));
}
});
})
// check enclosed types
owner_type.types.forEach(type => {
resolveMethodTypes(fully_qualified_scope_name, type, '$', resolved_imports, typemap, probs);
});
}
/**
* @param {ModuleBlock} mod
*/
module.exports = function(mod, imports) {
/** @type {ParseProblem[]} */
const probs = [];
mod.types.forEach(type => {
const qualifier = mod.packageName ? '.' : '';
resolveExtends(mod.packageName, type, qualifier, imports.resolved, imports.typemap, probs);
resolveImplements(mod.packageName, type, qualifier, imports.resolved, imports.typemap, probs);
resolveFieldTypes(mod.packageName, type, qualifier, imports.resolved, imports.typemap, probs);
resolveMethodTypes(mod.packageName, type, qualifier, imports.resolved, imports.typemap, probs);
});
return probs;
}

View File

@@ -0,0 +1,33 @@
const { ModuleBlock, PackageBlock, ImportBlock, TypeDeclBlock } = require('../parser9');
const ParseProblem = require('../parsetypes/parse-problem');
/**
* @param {ModuleBlock} mod
*/
module.exports = function(mod) {
let have_imports, have_type;
const problems = [];
for (let decl of mod.decls()) {
let p;
switch (true) {
case decl instanceof PackageBlock:
if (have_imports || have_type) {
p = ParseProblem.Error(decl, 'package must be declared before import and type declarations');
}
break;
case decl instanceof ImportBlock:
if (have_type) {
p = ParseProblem.Error(decl, 'imports must be declared before type declarations');
}
have_imports = true;
break;
case decl instanceof TypeDeclBlock:
have_type = true;
break;
}
if (p) {
problems.push(p)
}
}
return problems;
}

View File

@@ -0,0 +1,16 @@
const { ModuleBlock } = require('../parser9');
const ParseProblem = require('../parsetypes/parse-problem');
/**
* @param {ModuleBlock} mod
*/
module.exports = function(mod, imports) {
/** @type {ParseProblem[]} */
const probs = [];
imports.unresolved.forEach(i => {
probs.push(ParseProblem.Warning(i, `Unresolved import: ${i.name}`));
})
return probs;
}

View File

@@ -18,8 +18,9 @@ const {
const { TextDocument } = require('vscode-languageserver-textdocument');
const MTI = require('./java/mti');
const { parse, ParseProblem, ProblemSeverity, ParseResult } = require('./java/parser');
const { resolveImports } = require('./java/import-resolver');
const { ParseProblem } = require('./java/parser');
const { parse, ModuleBlock } = require('./java/parser9');
const { validate } = require('./java/validater');
let androidLibrary = null;
function loadAndroidLibrary(retry) {
@@ -70,7 +71,7 @@ let connection = createConnection(ProposedFeatures.all);
///** @type {LiveParseInfo[]} */
//const liveParsers = [];
/** @type {{content: string, uri: string, result: ParseResult, positionAt:(n) => Position, indexAt:(p:Position) => number}} */
/** @type {{content: string, uri: string, result: ModuleBlock, positionAt:(n) => Position, indexAt:(p:Position) => number}} */
let parsed = null;
function reparse(uri, content) {
@@ -270,57 +271,7 @@ async function validateTextDocument(textDocument) {
connection.console.log('validateTextDocument');
if (parsed && parsed.result) {
// package problem
if (parsed.result.package) {
problems = [...problems, ...parsed.result.package.validate()];
}
// import problems
problems = parsed.result.imports.reduce((problems, import_decl) => {
return [...problems, ...import_decl.validate()];
}, problems);
// type problems
problems = parsed.result.types.reduce((problems, type_decl) => {
return [...problems, ...type_decl.validate()];
}, problems);
// syntax problems
problems = parsed.result.invalids.reduce((problems, invalid) => {
return [...problems, ...invalid.validate()];
}, problems);
const package_name = parsed.result.package ? parsed.result.package.dottedName() : '';
const source_mtis = parsed.result.types.map(type_decl => {
return new MTI().addType(package_name, type_decl.getDocString(), type_decl.getAccessModifierValues(), type_decl.kind, type_decl.qualifiedName());
})
const imports = resolveImports(androidLibrary, parsed.result.imports, package_name, source_mtis);
// missing/invalid imports
problems = imports.unresolved.reduce((problems, unresolved) => {
const fqn = unresolved.nameparts.join('.');
return [...problems, new ParseProblem(unresolved.nameparts, `Unresolved import: ${fqn}`, ProblemSeverity.Warning)];
}, problems);
// resolved types
problems = parsed.result.types.reduce((problems, type_decl) => {
return [...problems, ...type_decl.validateTypes(package_name, imports.resolved, imports.typemap)];
}, problems);
// duplicate type names
/** @type {Map<string,import('./java/parsetypes/type')[]>} */
const typenames = new Map();
parsed.result.types.forEach(type_decl => {
const qname = type_decl.qualifiedName();
let list = typenames.get(qname);
if (!list) typenames.set(qname, list = []);
list.push(type_decl);
});
[...typenames.values()]
.filter(list => list.length > 1)
.forEach(list => {
problems = [...problems, ...list.map(type_decl => new ParseProblem(type_decl.name, `Duplicate type: ${type_decl.qualifiedDottedName()}`, ProblemSeverity.Error))];
});
problems = validate(parsed.result, androidLibrary);
}
const diagnostics = problems