mirror of https://github.com/buggins/dlangide.git
libdparser
This commit is contained in:
Vadim Lopatin
parent
e88b72f629
commit
b217c895c2
|
|
@ -80,15 +80,212 @@ class DParsedModule {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static class ImportInfo {
|
|
||||||
|
private IdentPositionIterator _identPositionIterator;
|
||||||
|
IdentDefinitionLookupResult findTokenNode(const(Token)* tokenToFindPositionFor, const(Token)* tokenToFindReferencesFor) {
|
||||||
|
if (!_identPositionIterator)
|
||||||
|
_identPositionIterator = new IdentPositionIterator();
|
||||||
|
auto foundNode = _identPositionIterator.run(this, _ast, tokenToFindPositionFor, tokenToFindReferencesFor);
|
||||||
|
return foundNode;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void findDeclaration(int bytePosition, DParsedModule[string] scanned) {
|
||||||
|
const(Token) * token = findIdentTokenByBytePosition(bytePosition);
|
||||||
|
if (!token)
|
||||||
|
return;
|
||||||
|
|
||||||
|
Log.d("Identifier token found by position: ", token.text);
|
||||||
|
IdentDefinitionLookupResult res = findTokenNode(token, token);
|
||||||
|
if (!res.found)
|
||||||
|
return;
|
||||||
|
Log.d("Found in node:");
|
||||||
|
res.found.dump();
|
||||||
|
}
|
||||||
|
|
||||||
|
const(Token) * findIdentTokenByBytePosition(int bytePosition) {
|
||||||
|
const(Token) * res = null;
|
||||||
|
for(int i = 0; i < _tokens.length; i++) {
|
||||||
|
auto t = &_tokens[i];
|
||||||
|
if (t.index >= bytePosition) {
|
||||||
|
if (res && *res == tok!"identifier")
|
||||||
|
return res; // return this or previous identifier token
|
||||||
|
if (t.index == bytePosition && (*t) == tok!"identifier")
|
||||||
|
return t; // return next identifier token
|
||||||
|
}
|
||||||
|
res = t;
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
void parse(ubyte[] sourceCode) {
|
||||||
|
_sourceCode = sourceCode;
|
||||||
|
_tokens = getTokensForParser(sourceCode, _lexerConfig, _cache);
|
||||||
|
uint errorCount;
|
||||||
|
uint warningCount;
|
||||||
|
_ast = parseModule(_tokens, _moduleFile, null, &msgFunction, &errorCount, &warningCount);
|
||||||
|
_moduleName = _ast.moduleDeclaration ? importDeclToModuleName(_ast.moduleDeclaration.moduleName) : null;
|
||||||
|
scanImports();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addImport(string m) {
|
||||||
|
foreach(imp; _imports)
|
||||||
|
if (imp.equal(m))
|
||||||
|
return;
|
||||||
|
_imports ~= m;
|
||||||
|
}
|
||||||
|
|
||||||
|
@property string[] imports() {
|
||||||
|
return _imports;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// D source code parsing service
|
||||||
|
class DParsingService {
|
||||||
|
|
||||||
|
protected static __gshared DParsingService _instance;
|
||||||
|
/// singleton
|
||||||
|
static @property DParsingService instance() {
|
||||||
|
if (!_instance) {
|
||||||
|
_instance = new DParsingService();
|
||||||
|
}
|
||||||
|
return _instance;
|
||||||
|
}
|
||||||
|
/// destroy singleton
|
||||||
|
static void shutdown() {
|
||||||
|
destroy(_instance);
|
||||||
|
_instance = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected StringCache _cache;
|
||||||
|
protected string[] _importPaths;
|
||||||
|
protected DParsedModule[] _modules;
|
||||||
|
protected DParsedModule[string] _moduleByName;
|
||||||
|
protected DParsedModule[string] _moduleByFile;
|
||||||
|
protected bool[string] _notFoundModules;
|
||||||
|
protected DParsedModule _currentModule; // current module
|
||||||
|
|
||||||
|
this() {
|
||||||
|
_cache = StringCache(16);
|
||||||
|
}
|
||||||
|
|
||||||
|
void scanDeps(DParsedModule m, ref DParsedModule[string]scanned) {
|
||||||
|
foreach(imp; m.imports) {
|
||||||
|
if (imp !in scanned) {
|
||||||
|
DParsedModule impModule = getOrParseModule(imp);
|
||||||
|
scanned[imp] = impModule;
|
||||||
|
if (impModule)
|
||||||
|
scanDeps(impModule, scanned);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
DParsedModule scan(ubyte[] sourceCode, string filename, ref DParsedModule[string] scanned) {
|
||||||
|
Log.d("scanning ", filename);
|
||||||
|
destroy(_notFoundModules);
|
||||||
|
DParsedModule res = new DParsedModule(&_cache, filename);
|
||||||
|
res.parse(sourceCode);
|
||||||
|
_currentModule = res;
|
||||||
|
Log.d("moduleName: ", res.moduleName, " imports: ", res.imports);
|
||||||
|
Log.d("deps:");
|
||||||
|
scanned[res.moduleName] = res;
|
||||||
|
scanDeps(res, scanned);
|
||||||
|
foreach(key, value; scanned) {
|
||||||
|
Log.d(" module ", key, " : ", value ? value.filename : "NOT FOUND");
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
DParsedModule findDeclaration(ubyte[] sourceCode, string filename, int bytePosition) {
|
||||||
|
DParsedModule[string] scanned;
|
||||||
|
DParsedModule m = scan(sourceCode, filename, scanned);
|
||||||
|
m.findDeclaration(bytePosition, scanned);
|
||||||
|
return m;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// converts some.module.name to some/module/name.d
|
||||||
|
string moduleNameToPackagePath(string moduleName) {
|
||||||
|
string[] pathSegments = moduleName.split(".");
|
||||||
|
string normalized = buildNormalizedPath(pathSegments);
|
||||||
|
return normalized ~ ".d";
|
||||||
|
}
|
||||||
|
|
||||||
|
string findModuleFile(string moduleName) {
|
||||||
|
string packagePath = moduleNameToPackagePath(moduleName);
|
||||||
|
foreach(ip; _importPaths) {
|
||||||
|
//Log.d("packagePath: ", packagePath, " importPath: ", ip);
|
||||||
|
string path = buildNormalizedPath(ip, packagePath);
|
||||||
|
if (path.exists && path.isFile) {
|
||||||
|
//Log.d("found ", path);
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
string pathImports = path ~ "i";
|
||||||
|
if (pathImports.exists && pathImports.isFile) {
|
||||||
|
//Log.d("found ", pathImports);
|
||||||
|
return pathImports;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
DParsedModule getOrParseModule(string moduleName) {
|
||||||
|
if (_currentModule) {
|
||||||
|
if (moduleName.equal(_currentModule.moduleName))
|
||||||
|
return _currentModule; // module being scanned
|
||||||
|
}
|
||||||
|
if (auto m = moduleName in _moduleByName) {
|
||||||
|
return *m;
|
||||||
|
}
|
||||||
|
if (moduleName in _notFoundModules) {
|
||||||
|
Log.d("module is in not found: ", moduleName);
|
||||||
|
return null; // already listed as not found
|
||||||
|
}
|
||||||
|
string filename = findModuleFile(moduleName);
|
||||||
|
if (!filename) {
|
||||||
|
Log.d("module not found: ", moduleName);
|
||||||
|
_notFoundModules[moduleName] = true;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
DParsedModule res = new DParsedModule(&_cache, filename);
|
||||||
|
ubyte[] sourceCode = cast(ubyte[])read(filename);
|
||||||
|
res.parse(sourceCode);
|
||||||
|
_moduleByName[moduleName] = res;
|
||||||
|
_moduleByFile[filename] = res;
|
||||||
|
return res;
|
||||||
|
} catch (Exception e) {
|
||||||
|
Log.d("exception while parsing: ", moduleName, " : ", e);
|
||||||
|
_notFoundModules[moduleName] = true;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void addImportPaths(in string[] paths) {
|
||||||
|
Log.d("addImportPaths: ", paths);
|
||||||
|
foreach(p; paths) {
|
||||||
|
string ap = absolutePath(buildNormalizedPath(p));
|
||||||
|
bool found = false;
|
||||||
|
foreach(ip; _importPaths)
|
||||||
|
if (ip.equal(ap)) {
|
||||||
|
found = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (!found)
|
||||||
|
_importPaths ~= ap;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static class ImportInfo {
|
||||||
string moduleName;
|
string moduleName;
|
||||||
const ImportDeclaration decl;
|
const ImportDeclaration decl;
|
||||||
this(const ImportDeclaration decl, string moduleName) {
|
this(const ImportDeclaration decl, string moduleName) {
|
||||||
this.decl = decl;
|
this.decl = decl;
|
||||||
this.moduleName = moduleName;
|
this.moduleName = moduleName;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
enum DeclarationType {
|
enum DeclarationType {
|
||||||
none,
|
none,
|
||||||
classVariableDeclaration,
|
classVariableDeclaration,
|
||||||
structVariableDeclaration,
|
structVariableDeclaration,
|
||||||
|
|
@ -103,8 +300,8 @@ class DParsedModule {
|
||||||
classTemplateTypeParameter,
|
classTemplateTypeParameter,
|
||||||
structTemplateTypeParameter,
|
structTemplateTypeParameter,
|
||||||
templateTypeParameter,
|
templateTypeParameter,
|
||||||
}
|
}
|
||||||
static class IdentContext {
|
static class IdentContext {
|
||||||
DParsedModule mod;
|
DParsedModule mod;
|
||||||
Token token;
|
Token token;
|
||||||
ImportInfo[] imports;
|
ImportInfo[] imports;
|
||||||
|
|
@ -214,9 +411,24 @@ class DParsedModule {
|
||||||
" declNode: ", declarationNode, " baseDeclNode: ", baseDeclarationNode,
|
" declNode: ", declarationNode, " baseDeclNode: ", baseDeclarationNode,
|
||||||
"\n\timports: ", imports, "\n\tcontext: ", stack);
|
"\n\timports: ", imports, "\n\tcontext: ", stack);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static class IdentPositionIterator : ASTVisitor {
|
static class IdentDefinitionLookupResult {
|
||||||
|
DParsedModule mod;
|
||||||
|
const(Token) tokenToFind;
|
||||||
|
const(Token) tokenToFindReferences;
|
||||||
|
IdentContext found;
|
||||||
|
IdentContext[] references;
|
||||||
|
this(DParsedModule mod, const(Token) * tokenToFind, const(Token) * tokenToFindReferences, IdentContext found, IdentContext[] references) {
|
||||||
|
this.mod = mod;
|
||||||
|
this.tokenToFind = *tokenToFind;
|
||||||
|
this.tokenToFindReferences = *tokenToFindReferences;
|
||||||
|
this.found = found;
|
||||||
|
this.references = references;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static class IdentPositionIterator : ASTVisitor {
|
||||||
|
|
||||||
private const(Token) * _tokenToFind;
|
private const(Token) * _tokenToFind;
|
||||||
private const(Token) * _tokenToFindReferences;
|
private const(Token) * _tokenToFindReferences;
|
||||||
|
|
@ -245,7 +457,7 @@ class DParsedModule {
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
IdentContext run(DParsedModule mod, Module ast, const(Token) * tokenToFind, const(Token) * tokenToFindReferences) {
|
IdentDefinitionLookupResult run(DParsedModule mod, Module ast, const(Token) * tokenToFind, const(Token) * tokenToFindReferences) {
|
||||||
_mod = mod;
|
_mod = mod;
|
||||||
_stack.length = 0;
|
_stack.length = 0;
|
||||||
_references.length = 0;
|
_references.length = 0;
|
||||||
|
|
@ -258,7 +470,7 @@ class DParsedModule {
|
||||||
foreach(r; _references)
|
foreach(r; _references)
|
||||||
r.dump();
|
r.dump();
|
||||||
}
|
}
|
||||||
return _found;
|
return new IdentDefinitionLookupResult(_mod, _tokenToFind, _tokenToFindReferences, _found, _references);
|
||||||
}
|
}
|
||||||
|
|
||||||
//alias visit = ASTVisitor.visit;
|
//alias visit = ASTVisitor.visit;
|
||||||
|
|
@ -542,199 +754,4 @@ class DParsedModule {
|
||||||
override void visit(const WhileStatement whileStatement) { mixin(def("whileStatement")); }
|
override void visit(const WhileStatement whileStatement) { mixin(def("whileStatement")); }
|
||||||
override void visit(const WithStatement withStatement) { mixin(def("withStatement")); }
|
override void visit(const WithStatement withStatement) { mixin(def("withStatement")); }
|
||||||
override void visit(const XorExpression xorExpression) { mixin(def("xorExpression")); }
|
override void visit(const XorExpression xorExpression) { mixin(def("xorExpression")); }
|
||||||
}
|
|
||||||
|
|
||||||
private IdentPositionIterator _identPositionIterator;
|
|
||||||
IdentContext findTokenNode(const(Token)* tokenToFindPositionFor, const(Token)* tokenToFindReferencesFor) {
|
|
||||||
if (!_identPositionIterator)
|
|
||||||
_identPositionIterator = new IdentPositionIterator();
|
|
||||||
auto foundNode = _identPositionIterator.run(this, _ast, tokenToFindPositionFor, tokenToFindReferencesFor);
|
|
||||||
return foundNode;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void findDeclaration(int bytePosition, DParsedModule[string] scanned) {
|
|
||||||
const(Token) * token = findIdentTokenByBytePosition(bytePosition);
|
|
||||||
if (!token)
|
|
||||||
return;
|
|
||||||
|
|
||||||
Log.d("Identifier token found by position: ", token.text);
|
|
||||||
auto found = findTokenNode(token, token);
|
|
||||||
if (!found)
|
|
||||||
return;
|
|
||||||
Log.d("Found in node:");
|
|
||||||
found.dump();
|
|
||||||
}
|
|
||||||
|
|
||||||
const(Token) * findIdentTokenByBytePosition(int bytePosition) {
|
|
||||||
const(Token) * res = null;
|
|
||||||
for(int i = 0; i < _tokens.length; i++) {
|
|
||||||
auto t = &_tokens[i];
|
|
||||||
if (t.index >= bytePosition) {
|
|
||||||
if (res && *res == tok!"identifier")
|
|
||||||
return res; // return this or previous identifier token
|
|
||||||
if (t.index == bytePosition && (*t) == tok!"identifier")
|
|
||||||
return t; // return next identifier token
|
|
||||||
}
|
|
||||||
res = t;
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
void parse(ubyte[] sourceCode) {
|
|
||||||
_sourceCode = sourceCode;
|
|
||||||
_tokens = getTokensForParser(sourceCode, _lexerConfig, _cache);
|
|
||||||
uint errorCount;
|
|
||||||
uint warningCount;
|
|
||||||
_ast = parseModule(_tokens, _moduleFile, null, &msgFunction, &errorCount, &warningCount);
|
|
||||||
_moduleName = _ast.moduleDeclaration ? importDeclToModuleName(_ast.moduleDeclaration.moduleName) : null;
|
|
||||||
scanImports();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void addImport(string m) {
|
|
||||||
foreach(imp; _imports)
|
|
||||||
if (imp.equal(m))
|
|
||||||
return;
|
|
||||||
_imports ~= m;
|
|
||||||
}
|
|
||||||
|
|
||||||
@property string[] imports() {
|
|
||||||
return _imports;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// D source code parsing service
|
|
||||||
class DParsingService {
|
|
||||||
|
|
||||||
protected static __gshared DParsingService _instance;
|
|
||||||
/// singleton
|
|
||||||
static @property DParsingService instance() {
|
|
||||||
if (!_instance) {
|
|
||||||
_instance = new DParsingService();
|
|
||||||
}
|
|
||||||
return _instance;
|
|
||||||
}
|
|
||||||
/// destroy singleton
|
|
||||||
static void shutdown() {
|
|
||||||
destroy(_instance);
|
|
||||||
_instance = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected StringCache _cache;
|
|
||||||
protected string[] _importPaths;
|
|
||||||
protected DParsedModule[] _modules;
|
|
||||||
protected DParsedModule[string] _moduleByName;
|
|
||||||
protected DParsedModule[string] _moduleByFile;
|
|
||||||
protected bool[string] _notFoundModules;
|
|
||||||
protected DParsedModule _currentModule; // current module
|
|
||||||
|
|
||||||
this() {
|
|
||||||
_cache = StringCache(16);
|
|
||||||
}
|
|
||||||
|
|
||||||
void scanDeps(DParsedModule m, ref DParsedModule[string]scanned) {
|
|
||||||
foreach(imp; m.imports) {
|
|
||||||
if (imp !in scanned) {
|
|
||||||
DParsedModule impModule = getOrParseModule(imp);
|
|
||||||
scanned[imp] = impModule;
|
|
||||||
if (impModule)
|
|
||||||
scanDeps(impModule, scanned);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
DParsedModule scan(ubyte[] sourceCode, string filename, ref DParsedModule[string] scanned) {
|
|
||||||
Log.d("scanning ", filename);
|
|
||||||
destroy(_notFoundModules);
|
|
||||||
DParsedModule res = new DParsedModule(&_cache, filename);
|
|
||||||
res.parse(sourceCode);
|
|
||||||
_currentModule = res;
|
|
||||||
Log.d("moduleName: ", res.moduleName, " imports: ", res.imports);
|
|
||||||
Log.d("deps:");
|
|
||||||
scanned[res.moduleName] = res;
|
|
||||||
scanDeps(res, scanned);
|
|
||||||
foreach(key, value; scanned) {
|
|
||||||
Log.d(" module ", key, " : ", value ? value.filename : "NOT FOUND");
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
DParsedModule findDeclaration(ubyte[] sourceCode, string filename, int bytePosition) {
|
|
||||||
DParsedModule[string] scanned;
|
|
||||||
DParsedModule m = scan(sourceCode, filename, scanned);
|
|
||||||
m.findDeclaration(bytePosition, scanned);
|
|
||||||
return m;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// converts some.module.name to some/module/name.d
|
|
||||||
string moduleNameToPackagePath(string moduleName) {
|
|
||||||
string[] pathSegments = moduleName.split(".");
|
|
||||||
string normalized = buildNormalizedPath(pathSegments);
|
|
||||||
return normalized ~ ".d";
|
|
||||||
}
|
|
||||||
|
|
||||||
string findModuleFile(string moduleName) {
|
|
||||||
string packagePath = moduleNameToPackagePath(moduleName);
|
|
||||||
foreach(ip; _importPaths) {
|
|
||||||
//Log.d("packagePath: ", packagePath, " importPath: ", ip);
|
|
||||||
string path = buildNormalizedPath(ip, packagePath);
|
|
||||||
if (path.exists && path.isFile) {
|
|
||||||
//Log.d("found ", path);
|
|
||||||
return path;
|
|
||||||
}
|
|
||||||
string pathImports = path ~ "i";
|
|
||||||
if (pathImports.exists && pathImports.isFile) {
|
|
||||||
//Log.d("found ", pathImports);
|
|
||||||
return pathImports;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
DParsedModule getOrParseModule(string moduleName) {
|
|
||||||
if (_currentModule) {
|
|
||||||
if (moduleName.equal(_currentModule.moduleName))
|
|
||||||
return _currentModule; // module being scanned
|
|
||||||
}
|
|
||||||
if (auto m = moduleName in _moduleByName) {
|
|
||||||
return *m;
|
|
||||||
}
|
|
||||||
if (moduleName in _notFoundModules) {
|
|
||||||
Log.d("module is in not found: ", moduleName);
|
|
||||||
return null; // already listed as not found
|
|
||||||
}
|
|
||||||
string filename = findModuleFile(moduleName);
|
|
||||||
if (!filename) {
|
|
||||||
Log.d("module not found: ", moduleName);
|
|
||||||
_notFoundModules[moduleName] = true;
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
DParsedModule res = new DParsedModule(&_cache, filename);
|
|
||||||
ubyte[] sourceCode = cast(ubyte[])read(filename);
|
|
||||||
res.parse(sourceCode);
|
|
||||||
_moduleByName[moduleName] = res;
|
|
||||||
_moduleByFile[filename] = res;
|
|
||||||
return res;
|
|
||||||
} catch (Exception e) {
|
|
||||||
Log.d("exception while parsing: ", moduleName, " : ", e);
|
|
||||||
_notFoundModules[moduleName] = true;
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void addImportPaths(in string[] paths) {
|
|
||||||
Log.d("addImportPaths: ", paths);
|
|
||||||
foreach(p; paths) {
|
|
||||||
string ap = absolutePath(buildNormalizedPath(p));
|
|
||||||
bool found = false;
|
|
||||||
foreach(ip; _importPaths)
|
|
||||||
if (ip.equal(ap)) {
|
|
||||||
found = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (!found)
|
|
||||||
_importPaths ~= ap;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue