diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbfb20c..b901a90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,8 +4,6 @@ on: push: branches: - master - release: - types: [published] jobs: Build: @@ -17,13 +15,20 @@ jobs: dc: - ldc-latest - dmd-latest + build: [debug, release] arch: - x86_64 + libdparse-version: [min, max] include: # windows x86 - os: windows-latest arch: x86 dc: ldc-latest + build: debug + libdparse-version: min + # old compiler tests + - { os: ubuntu-latest, dc: dmd-2.095.1, libdparse-version: min, build: debug, arch: x86_64 } + - { os: ubuntu-latest, dc: ldc-1.25.0, libdparse-version: min, build: debug, arch: x86_64 } runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 @@ -39,11 +44,27 @@ jobs: - name: Build run: | - dub build --build=release --config=client --arch=${{ matrix.arch }} - dub build --build=release --config=server --arch=${{ matrix.arch }} + rdmd ./d-test-utils/test_with_package.d $LIBDPARSE_VERSION libdparse -- dub build --build=${{ matrix.build }} --config=client --arch=${{ matrix.arch }} + rdmd ./d-test-utils/test_with_package.d $LIBDPARSE_VERSION libdparse -- dub build --build=${{ matrix.build }} --config=server --arch=${{ matrix.arch }} # Tests + - name: Build DSymbol + env: + DC: ${{matrix.dc}} + LIBDPARSE_VERSION: ${{ matrix.libdparse-version }} + run: | + cd dsymbol + rdmd ../d-test-utils/test_with_package.d $LIBDPARSE_VERSION libdparse -- dub build --build=${{ matrix.build }} + + - name: Test DSymbol + env: + DC: ${{matrix.dc}} + LIBDPARSE_VERSION: ${{ matrix.libdparse-version }} + run: | + cd dsymbol + rdmd ../d-test-utils/test_with_package.d $LIBDPARSE_VERSION libdparse -- dub test + - name: Linux Tests if: contains(matrix.os, 'ubuntu') run: | @@ -58,54 +79,3 @@ jobs: working-directory: tests shell: bash continue-on-error: true - - - # Package Release - - - name: Package the artificats - if: github.event_name == 'release' && contains(matrix.dc, 'ldc') - shell: pwsh - working-directory: bin - run: | - if ("${{ matrix.os }}" -like 'windows*') { - 7z a -tzip ..\dcd.zip dcd-client.exe dcd-server.exe - } elseif ("${{ matrix.os }}" -like 'macos*') { - gtar -cvzf ../dcd.tar.gz dcd-client dcd-server - } else { - tar -cvzf ../dcd.tar.gz dcd-client dcd-server - } - - # Release - - - name: Release Linux - if: github.event_name == 'release' && contains(matrix.os, 'ubuntu') && contains(matrix.dc, 'ldc') - uses: WebFreak001/upload-asset@v1.0.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OS: linux - with: - file: dcd.tar.gz - name: dcd-${TAG_RAW}-${OS}-${{ matrix.arch }}.tar.gz - mime: application/tar+gzip - - - name: Release Macos - if: github.event_name == 'release' && contains(matrix.os, 'macos') && contains(matrix.dc, 'ldc') - uses: WebFreak001/upload-asset@v1.0.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OS: osx - with: - file: dcd.tar.gz - name: dcd-${TAG_RAW}-${OS}-${{ matrix.arch }}.tar.gz - mime: application/tar+gzip - - - name: Release Windows - if: github.event_name == 'release' && contains(matrix.os, 'windows') && contains(matrix.dc, 'ldc') - uses: WebFreak001/upload-asset@v1.0.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OS: windows - with: - file: dcd.zip - name: dcd-${TAG_RAW}-${OS}-${{ matrix.arch }}.zip - mime: application/zip diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..eb83b06 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,81 @@ +name: Publish Releases +on: + release: + types: [published] + +jobs: + Build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + dc: + - ldc-latest + arch: + - x86_64 + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Setup D + uses: dlang-community/setup-dlang@v1 + with: + compiler: ${{ matrix.dc }} + + # Build + + - name: Build + run: | + dub build --build=release --config=client --arch=${{ matrix.arch }} + dub build --build=release --config=server --arch=${{ matrix.arch }} + + # Package Release + + - name: Package the artificats + shell: pwsh + working-directory: bin + run: | + if ("${{ matrix.os }}" -like 'windows*') { + 7z a -tzip ..\dcd.zip dcd-client.exe dcd-server.exe + } elseif ("${{ matrix.os }}" -like 'macos*') { + gtar -cvzf ../dcd.tar.gz dcd-client dcd-server + } else { + tar -cvzf ../dcd.tar.gz dcd-client dcd-server + } + + # Release + + - name: Release Linux + if: contains(matrix.os, 'ubuntu') + uses: WebFreak001/upload-asset@v1.0.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OS: linux + with: + file: dcd.tar.gz + name: dcd-${TAG_RAW}-${OS}-${{ matrix.arch }}.tar.gz + mime: application/tar+gzip + + - name: Release Macos + if: contains(matrix.os, 'macos') + uses: WebFreak001/upload-asset@v1.0.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OS: osx + with: + file: dcd.tar.gz + name: dcd-${TAG_RAW}-${OS}-${{ matrix.arch }}.tar.gz + mime: application/tar+gzip + + - name: Release Windows + if: contains(matrix.os, 'windows') + uses: WebFreak001/upload-asset@v1.0.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OS: windows + with: + file: dcd.zip + name: dcd-${TAG_RAW}-${OS}-${{ matrix.arch }}.zip + mime: application/zip diff --git a/.gitmodules b/.gitmodules index 02de1fd..b4bb0c1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,9 +8,6 @@ path = libdparse url = https://github.com/dlang-community/libdparse.git branch = master -[submodule "dsymbol"] - path = dsymbol - url = https://github.com/dlang-community/dsymbol.git [submodule "stdx-allocator"] path = stdx-allocator url = https://github.com/dlang-community/stdx-allocator diff --git a/dsymbol b/dsymbol deleted file mode 160000 index 46873f0..0000000 --- a/dsymbol +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 46873f01f74844ab0bea0af14643cdd791573505 diff --git a/dsymbol/.editorconfig b/dsymbol/.editorconfig new file mode 100644 index 0000000..9ab011e --- /dev/null +++ b/dsymbol/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*.d] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = tab +indent_size = 4 +trim_trailing_whitespace = true diff --git a/dsymbol/.gitignore b/dsymbol/.gitignore new file mode 100644 index 0000000..8d26d9b --- /dev/null +++ b/dsymbol/.gitignore @@ -0,0 +1,32 @@ +# Compiled Object files +*.o +*.obj + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Compiled Static libraries +*.a +*.lib + +# Executables +*.exe + +# DUB +.dub/ +docs/ +docs.json +__dummy.html + +# Code coverage +*.lst + +# Others +build/ +*.~* +*.*~ +dub.selections.json +trace.def +.vscode/ diff --git a/dsymbol/LICENSE_1_0.txt b/dsymbol/LICENSE_1_0.txt new file mode 100644 index 0000000..36b7cd9 --- /dev/null +++ b/dsymbol/LICENSE_1_0.txt @@ -0,0 +1,23 @@ +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/dsymbol/README.md b/dsymbol/README.md new file mode 100644 index 0000000..f1900d6 --- /dev/null +++ b/dsymbol/README.md @@ -0,0 +1,3 @@ +# dsymbol + +Symbol lookup support for [libdparse](https://github.com/dlang-community/libdparse) diff --git a/dsymbol/dub.json b/dsymbol/dub.json new file mode 100644 index 0000000..e3a6934 --- /dev/null +++ b/dsymbol/dub.json @@ -0,0 +1,12 @@ +{ + "name": "dsymbol", + "description": "Symbol lookup support for libdparse", + "license": "BSL-1.0", + "authors": ["Brian Schott"], + "targetPath": "build", + "targetType": "library", + "dependencies": { + "libdparse": ">=0.20.0 <0.21.0", + "emsi_containers": "~>0.9.0" + } +} diff --git a/dsymbol/meson.build b/dsymbol/meson.build new file mode 100644 index 0000000..a0d6561 --- /dev/null +++ b/dsymbol/meson.build @@ -0,0 +1,84 @@ + project('dsymbol', 'd', + meson_version: '>=0.44', + license: 'BSL-1.0', + version: '0.4.8' +) + +project_soversion = '0' + +pkgc = import('pkgconfig') +dparse_dep = dependency('dparse', version: '>= 0.9.0', fallback: ['dparse', 'dparse_dep']) +dcontainers_dep = dependency('dcontainers', version: '>= 0.8.0', fallback: ['dcontainers', 'dcontainers_dep']) + +# +# Sources +# +dsymbol_src = [ + 'src/dsymbol/builtin/names.d', + 'src/dsymbol/builtin/symbols.d', + 'src/dsymbol/cache_entry.d', + 'src/dsymbol/conversion/first.d', + 'src/dsymbol/conversion/package.d', + 'src/dsymbol/conversion/second.d', + 'src/dsymbol/deferred.d', + 'src/dsymbol/import_.d', + 'src/dsymbol/modulecache.d', + 'src/dsymbol/scope_.d', + 'src/dsymbol/semantic.d', + 'src/dsymbol/string_interning.d', + 'src/dsymbol/symbol.d', + 'src/dsymbol/tests.d', + 'src/dsymbol/type_lookup.d', +] + +src_dir = include_directories('src/') + +# +# Targets +# +dsymbol_lib = library('dsymbol', + [dsymbol_src], + include_directories: [src_dir], + install: true, + version: meson.project_version(), + soversion: project_soversion, + dependencies: [dparse_dep, dcontainers_dep] +) + +pkgc.generate(name: 'dsymbol', + libraries: [dsymbol_lib], + subdirs: 'd/dsymbol', + requires: ['dparse', 'dcontainers'], + version: meson.project_version(), + description: 'Library for lexing and parsing D source code.' +) + +# for use by others which embed this as subproject +dsymbol_dep = declare_dependency( + link_with: [dsymbol_lib], + include_directories: [src_dir], + dependencies: [dparse_dep, dcontainers_dep] +) + +# +# Tests +# +if meson.get_compiler('d').get_id() == 'llvm' + extra_args = ['-main', '-link-defaultlib-shared'] +else + extra_args = ['-main'] +endif + +dsymbol_test_exe = executable('test_dsymbol', + [dsymbol_src], + include_directories: [src_dir], + dependencies: [dparse_dep, dcontainers_dep], + d_unittest: true, + link_args: extra_args +) +test('test_dsymbol', dsymbol_test_exe) + +# +# Install +# +install_subdir('src/dsymbol/', install_dir: 'include/d/dsymbol/') diff --git a/dsymbol/src/dsymbol/builtin/names.d b/dsymbol/src/dsymbol/builtin/names.d new file mode 100644 index 0000000..7cd6795 --- /dev/null +++ b/dsymbol/src/dsymbol/builtin/names.d @@ -0,0 +1,208 @@ +module dsymbol.builtin.names; + +import dparse.lexer; +import dsymbol.string_interning; + +package istring[24] builtinTypeNames; + +/// Constants for buit-in or dummy symbol names +istring FUNCTION_SYMBOL_NAME; +/// ditto +istring FUNCTION_LITERAL_SYMBOL_NAME; +/// ditto +istring IMPORT_SYMBOL_NAME; +/// ditto +istring ARRAY_SYMBOL_NAME; +/// ditto +istring ARRAY_LITERAL_SYMBOL_NAME; +/// ditto +istring ASSOC_ARRAY_SYMBOL_NAME; +/// ditto +istring POINTER_SYMBOL_NAME; +/// ditto +istring PARAMETERS_SYMBOL_NAME; +/// ditto +istring WITH_SYMBOL_NAME; +/// ditto +istring CONSTRUCTOR_SYMBOL_NAME; +/// ditto +istring DESTRUCTOR_SYMBOL_NAME; +/// ditto +istring ARGPTR_SYMBOL_NAME; +/// ditto +istring ARGUMENTS_SYMBOL_NAME; +/// ditto +istring THIS_SYMBOL_NAME; +/// ditto +istring SUPER_SYMBOL_NAME; +/// ditto +istring UNITTEST_SYMBOL_NAME; +/// ditto +istring DOUBLE_LITERAL_SYMBOL_NAME; +/// ditto +istring FLOAT_LITERAL_SYMBOL_NAME; +/// ditto +istring IDOUBLE_LITERAL_SYMBOL_NAME; +/// ditto +istring IFLOAT_LITERAL_SYMBOL_NAME; +/// ditto +istring INT_LITERAL_SYMBOL_NAME; +/// ditto +istring LONG_LITERAL_SYMBOL_NAME; +/// ditto +istring REAL_LITERAL_SYMBOL_NAME; +/// ditto +istring IREAL_LITERAL_SYMBOL_NAME; +/// ditto +istring UINT_LITERAL_SYMBOL_NAME; +/// ditto +istring ULONG_LITERAL_SYMBOL_NAME; +/// ditto +istring CHAR_LITERAL_SYMBOL_NAME; +/// ditto +istring DSTRING_LITERAL_SYMBOL_NAME; +/// ditto +istring STRING_LITERAL_SYMBOL_NAME; +/// ditto +istring WSTRING_LITERAL_SYMBOL_NAME; +/// ditto +istring BOOL_VALUE_SYMBOL_NAME; +/// ditto +istring VOID_SYMBOL_NAME; + +/** + * Translates the IDs for built-in types into an interned string. + */ +istring getBuiltinTypeName(IdType id) nothrow @nogc @safe +{ + switch (id) + { + case tok!"int": return builtinTypeNames[0]; + case tok!"uint": return builtinTypeNames[1]; + case tok!"double": return builtinTypeNames[2]; + case tok!"idouble": return builtinTypeNames[3]; + case tok!"float": return builtinTypeNames[4]; + case tok!"ifloat": return builtinTypeNames[5]; + case tok!"short": return builtinTypeNames[6]; + case tok!"ushort": return builtinTypeNames[7]; + case tok!"long": return builtinTypeNames[8]; + case tok!"ulong": return builtinTypeNames[9]; + case tok!"char": return builtinTypeNames[10]; + case tok!"wchar": return builtinTypeNames[11]; + case tok!"dchar": return builtinTypeNames[12]; + case tok!"bool": return builtinTypeNames[13]; + case tok!"void": return builtinTypeNames[14]; + case tok!"cent": return builtinTypeNames[15]; + case tok!"ucent": return builtinTypeNames[16]; + case tok!"real": return builtinTypeNames[17]; + case tok!"ireal": return builtinTypeNames[18]; + case tok!"byte": return builtinTypeNames[19]; + case tok!"ubyte": return builtinTypeNames[20]; + case tok!"cdouble": return builtinTypeNames[21]; + case tok!"cfloat": return builtinTypeNames[22]; + case tok!"creal": return builtinTypeNames[23]; + default: assert (false); + } +} + + +/** + * Initializes builtin types and the various properties of builtin types + */ +static this() +{ + builtinTypeNames[0] = internString("int"); + builtinTypeNames[1] = internString("uint"); + builtinTypeNames[2] = internString("double"); + builtinTypeNames[3] = internString("idouble"); + builtinTypeNames[4] = internString("float"); + builtinTypeNames[5] = internString("ifloat"); + builtinTypeNames[6] = internString("short"); + builtinTypeNames[7] = internString("ushort"); + builtinTypeNames[8] = internString("long"); + builtinTypeNames[9] = internString("ulong"); + builtinTypeNames[10] = internString("char"); + builtinTypeNames[11] = internString("wchar"); + builtinTypeNames[12] = internString("dchar"); + builtinTypeNames[13] = internString("bool"); + builtinTypeNames[14] = internString("void"); + builtinTypeNames[15] = internString("cent"); + builtinTypeNames[16] = internString("ucent"); + builtinTypeNames[17] = internString("real"); + builtinTypeNames[18] = internString("ireal"); + builtinTypeNames[19] = internString("byte"); + builtinTypeNames[20] = internString("ubyte"); + builtinTypeNames[21] = internString("cdouble"); + builtinTypeNames[22] = internString("cfloat"); + builtinTypeNames[23] = internString("creal"); + + FUNCTION_SYMBOL_NAME = internString("function"); + FUNCTION_LITERAL_SYMBOL_NAME = internString("*function-literal*"); + IMPORT_SYMBOL_NAME = internString("import"); + ARRAY_SYMBOL_NAME = internString("*arr*"); + ARRAY_LITERAL_SYMBOL_NAME = internString("*arr-literal*"); + ASSOC_ARRAY_SYMBOL_NAME = internString("*aa*"); + POINTER_SYMBOL_NAME = internString("*"); + PARAMETERS_SYMBOL_NAME = internString("*parameters*"); + WITH_SYMBOL_NAME = internString("with"); + CONSTRUCTOR_SYMBOL_NAME = internString("*constructor*"); + DESTRUCTOR_SYMBOL_NAME = internString("~this"); + ARGPTR_SYMBOL_NAME = internString("_argptr"); + ARGUMENTS_SYMBOL_NAME = internString("_arguments"); + THIS_SYMBOL_NAME = internString("this"); + SUPER_SYMBOL_NAME = internString("super"); + UNITTEST_SYMBOL_NAME = internString("*unittest*"); + DOUBLE_LITERAL_SYMBOL_NAME = internString("*double"); + FLOAT_LITERAL_SYMBOL_NAME = internString("*float"); + IDOUBLE_LITERAL_SYMBOL_NAME = internString("*idouble"); + IFLOAT_LITERAL_SYMBOL_NAME = internString("*ifloat"); + INT_LITERAL_SYMBOL_NAME = internString("*int"); + LONG_LITERAL_SYMBOL_NAME = internString("*long"); + REAL_LITERAL_SYMBOL_NAME = internString("*real"); + IREAL_LITERAL_SYMBOL_NAME = internString("*ireal"); + UINT_LITERAL_SYMBOL_NAME = internString("*uint"); + ULONG_LITERAL_SYMBOL_NAME = internString("*ulong"); + CHAR_LITERAL_SYMBOL_NAME = internString("*char"); + DSTRING_LITERAL_SYMBOL_NAME = internString("*dstring"); + STRING_LITERAL_SYMBOL_NAME = internString("*string"); + WSTRING_LITERAL_SYMBOL_NAME = internString("*wstring"); + BOOL_VALUE_SYMBOL_NAME = internString("*bool"); + VOID_SYMBOL_NAME = internString("*void"); +} + +istring symbolNameToTypeName(istring name) +{ + if (name == DOUBLE_LITERAL_SYMBOL_NAME) + return builtinTypeNames[2]; + if (name == FLOAT_LITERAL_SYMBOL_NAME) + return builtinTypeNames[4]; + if (name == IDOUBLE_LITERAL_SYMBOL_NAME) + return builtinTypeNames[3]; + if (name == IFLOAT_LITERAL_SYMBOL_NAME) + return builtinTypeNames[5]; + if (name == INT_LITERAL_SYMBOL_NAME) + return builtinTypeNames[0]; + if (name == LONG_LITERAL_SYMBOL_NAME) + return builtinTypeNames[8]; + if (name == REAL_LITERAL_SYMBOL_NAME) + return builtinTypeNames[17]; + if (name == IREAL_LITERAL_SYMBOL_NAME) + return builtinTypeNames[18]; + if (name == UINT_LITERAL_SYMBOL_NAME) + return builtinTypeNames[1]; + if (name == ULONG_LITERAL_SYMBOL_NAME) + return builtinTypeNames[9]; + if (name == CHAR_LITERAL_SYMBOL_NAME) + return builtinTypeNames[10]; + if (name == DSTRING_LITERAL_SYMBOL_NAME) + return istring("dstring"); + if (name == STRING_LITERAL_SYMBOL_NAME) + return istring("string"); + if (name == WSTRING_LITERAL_SYMBOL_NAME) + return istring("wstring"); + if (name == BOOL_VALUE_SYMBOL_NAME) + return istring("bool"); + if (name == VOID_SYMBOL_NAME) + return istring("void"); + return name; +} diff --git a/dsymbol/src/dsymbol/builtin/symbols.d b/dsymbol/src/dsymbol/builtin/symbols.d new file mode 100644 index 0000000..94422c5 --- /dev/null +++ b/dsymbol/src/dsymbol/builtin/symbols.d @@ -0,0 +1,310 @@ +module dsymbol.builtin.symbols; + +import containers.hashset; +import containers.ttree; +import dparse.rollback_allocator; +import dsymbol.builtin.names; +import dsymbol.string_interning; +import dsymbol.symbol; +import std.experimental.allocator.mallocator : Mallocator; + +alias SymbolsAllocator = Mallocator; + +/** + * Symbols for the built in types + */ +TTree!(DSymbol*, SymbolsAllocator, true, "a < b") builtinSymbols; + +/** + * Array properties + */ +TTree!(DSymbol*, SymbolsAllocator, true, "a < b") arraySymbols; + +/** + * Associative array properties + */ +TTree!(DSymbol*, SymbolsAllocator, true, "a < b") assocArraySymbols; + +/** + * Struct, enum, union, class, and interface properties + */ +TTree!(DSymbol*, SymbolsAllocator, true, "a < b") aggregateSymbols; + +/** + * Class properties + */ +TTree!(DSymbol*, SymbolsAllocator, true, "a < b") classSymbols; + +/** + * Enum properties + */ +TTree!(DSymbol*, SymbolsAllocator, true, "a < b") enumSymbols; + +/** + * Variadic template parameters properties + */ +DSymbol* variadicTmpParamSymbol; + +/** + * Type template parameters properties (when no colon constraint) + */ +DSymbol* typeTmpParamSymbol; + +static this() +{ + auto bool_ = makeSymbol(builtinTypeNames[13], CompletionKind.keyword); + auto int_ = makeSymbol(builtinTypeNames[0], CompletionKind.keyword); + auto long_ = makeSymbol(builtinTypeNames[8], CompletionKind.keyword); + auto byte_ = makeSymbol(builtinTypeNames[19], CompletionKind.keyword); + auto char_ = makeSymbol(builtinTypeNames[10], CompletionKind.keyword); + auto dchar_ = makeSymbol(builtinTypeNames[12], CompletionKind.keyword); + auto short_ = makeSymbol(builtinTypeNames[6], CompletionKind.keyword); + auto ubyte_ = makeSymbol(builtinTypeNames[20], CompletionKind.keyword); + auto uint_ = makeSymbol(builtinTypeNames[1], CompletionKind.keyword); + auto ulong_ = makeSymbol(builtinTypeNames[9], CompletionKind.keyword); + auto ushort_ = makeSymbol(builtinTypeNames[7], CompletionKind.keyword); + auto wchar_ = makeSymbol(builtinTypeNames[11], CompletionKind.keyword); + + auto alignof_ = makeSymbol("alignof", CompletionKind.keyword); + auto mangleof_ = makeSymbol("mangleof", CompletionKind.keyword); + auto sizeof_ = makeSymbol("sizeof", CompletionKind.keyword); + auto stringof_ = makeSymbol("stringof", CompletionKind.keyword); + auto init = makeSymbol("init", CompletionKind.keyword); + auto min = makeSymbol("min", CompletionKind.keyword); + auto max = makeSymbol("max", CompletionKind.keyword); + auto dup = makeSymbol("dup", CompletionKind.keyword); + auto length = makeSymbol("length", CompletionKind.keyword, ulong_); + auto tupleof = makeSymbol("tupleof", CompletionKind.keyword); + + variadicTmpParamSymbol = makeSymbol("variadicTmpParam", CompletionKind.keyword); + variadicTmpParamSymbol.addChild(init, false); + variadicTmpParamSymbol.addChild(length, false); + variadicTmpParamSymbol.addChild(stringof_, false); + + typeTmpParamSymbol = makeSymbol("typeTmpParam", CompletionKind.keyword); + typeTmpParamSymbol.addChild(alignof_, false); + typeTmpParamSymbol.addChild(init, false); + typeTmpParamSymbol.addChild(mangleof_, false); + typeTmpParamSymbol.addChild(sizeof_, false); + typeTmpParamSymbol.addChild(stringof_, false); + + arraySymbols.insert(alignof_); + arraySymbols.insert(dup); + arraySymbols.insert(makeSymbol("idup", CompletionKind.keyword)); + arraySymbols.insert(init); + arraySymbols.insert(length); + arraySymbols.insert(mangleof_); + arraySymbols.insert(makeSymbol("ptr", CompletionKind.keyword)); + arraySymbols.insert(sizeof_); + arraySymbols.insert(stringof_); + + assocArraySymbols.insert(alignof_); + assocArraySymbols.insert(makeSymbol("byKey", CompletionKind.keyword)); + assocArraySymbols.insert(makeSymbol("byValue", CompletionKind.keyword)); + assocArraySymbols.insert(makeSymbol("clear", CompletionKind.keyword)); + assocArraySymbols.insert(dup); + assocArraySymbols.insert(makeSymbol("get", CompletionKind.keyword)); + assocArraySymbols.insert(init); + assocArraySymbols.insert(makeSymbol("keys", CompletionKind.keyword)); + assocArraySymbols.insert(length); + assocArraySymbols.insert(mangleof_); + assocArraySymbols.insert(makeSymbol("rehash", CompletionKind.keyword)); + assocArraySymbols.insert(sizeof_); + assocArraySymbols.insert(stringof_); + assocArraySymbols.insert(init); + assocArraySymbols.insert(makeSymbol("values", CompletionKind.keyword)); + + DSymbol*[12] integralTypeArray; + integralTypeArray[0] = bool_; + integralTypeArray[1] = int_; + integralTypeArray[2] = long_; + integralTypeArray[3] = byte_; + integralTypeArray[4] = char_; + integralTypeArray[5] = dchar_; + integralTypeArray[6] = short_; + integralTypeArray[7] = ubyte_; + integralTypeArray[8] = uint_; + integralTypeArray[9] = ulong_; + integralTypeArray[10] = ushort_; + integralTypeArray[11] = wchar_; + + foreach (s; integralTypeArray) + { + s.addChild(makeSymbol("init", CompletionKind.keyword, s), false); + s.addChild(makeSymbol("min", CompletionKind.keyword, s), false); + s.addChild(makeSymbol("max", CompletionKind.keyword, s), false); + s.addChild(alignof_, false); + s.addChild(sizeof_, false); + s.addChild(stringof_, false); + s.addChild(mangleof_, false); + } + + auto cdouble_ = makeSymbol(builtinTypeNames[21], CompletionKind.keyword); + auto cent_ = makeSymbol(builtinTypeNames[15], CompletionKind.keyword); + auto cfloat_ = makeSymbol(builtinTypeNames[22], CompletionKind.keyword); + auto creal_ = makeSymbol(builtinTypeNames[23], CompletionKind.keyword); + auto double_ = makeSymbol(builtinTypeNames[2], CompletionKind.keyword); + auto float_ = makeSymbol(builtinTypeNames[4], CompletionKind.keyword); + auto idouble_ = makeSymbol(builtinTypeNames[3], CompletionKind.keyword); + auto ifloat_ = makeSymbol(builtinTypeNames[5], CompletionKind.keyword); + auto ireal_ = makeSymbol(builtinTypeNames[18], CompletionKind.keyword); + auto real_ = makeSymbol(builtinTypeNames[17], CompletionKind.keyword); + auto ucent_ = makeSymbol(builtinTypeNames[16], CompletionKind.keyword); + + DSymbol*[11] floatTypeArray; + floatTypeArray[0] = cdouble_; + floatTypeArray[1] = cent_; + floatTypeArray[2] = cfloat_; + floatTypeArray[3] = creal_; + floatTypeArray[4] = double_; + floatTypeArray[5] = float_; + floatTypeArray[6] = idouble_; + floatTypeArray[7] = ifloat_; + floatTypeArray[8] = ireal_; + floatTypeArray[9] = real_; + floatTypeArray[10] = ucent_; + + foreach (s; floatTypeArray) + { + s.addChild(alignof_, false); + s.addChild(makeSymbol("dig", CompletionKind.keyword, s), false); + s.addChild(makeSymbol("epsilon", CompletionKind.keyword, s), false); + s.addChild(makeSymbol("infinity", CompletionKind.keyword, s), false); + s.addChild(makeSymbol("init", CompletionKind.keyword, s), false); + s.addChild(mangleof_, false); + s.addChild(makeSymbol("mant_dig", CompletionKind.keyword, int_), false); + s.addChild(makeSymbol("max", CompletionKind.keyword, s), false); + s.addChild(makeSymbol("max_10_exp", CompletionKind.keyword, int_), false); + s.addChild(makeSymbol("max_exp", CompletionKind.keyword, int_), false); + s.addChild(makeSymbol("min_exp", CompletionKind.keyword, int_), false); + s.addChild(makeSymbol("min_10_exp", CompletionKind.keyword, int_), false); + s.addChild(makeSymbol("min_normal", CompletionKind.keyword, s), false); + s.addChild(makeSymbol("nan", CompletionKind.keyword, s), false); + s.addChild(sizeof_, false); + s.addChild(stringof_, false); + } + + aggregateSymbols.insert(tupleof); + aggregateSymbols.insert(mangleof_); + aggregateSymbols.insert(alignof_); + aggregateSymbols.insert(sizeof_); + aggregateSymbols.insert(stringof_); + aggregateSymbols.insert(init); + + classSymbols.insert(makeSymbol("classinfo", CompletionKind.variableName)); + classSymbols.insert(tupleof); + classSymbols.insert(makeSymbol("__vptr", CompletionKind.variableName)); + classSymbols.insert(makeSymbol("__monitor", CompletionKind.variableName)); + classSymbols.insert(mangleof_); + classSymbols.insert(alignof_); + classSymbols.insert(sizeof_); + classSymbols.insert(stringof_); + classSymbols.insert(init); + + enumSymbols.insert(init); + enumSymbols.insert(sizeof_); + enumSymbols.insert(alignof_); + enumSymbols.insert(mangleof_); + enumSymbols.insert(stringof_); + enumSymbols.insert(min); + enumSymbols.insert(max); + + + ireal_.addChild(makeSymbol("im", CompletionKind.keyword, real_), false); + ifloat_.addChild(makeSymbol("im", CompletionKind.keyword, float_), false); + idouble_.addChild(makeSymbol("im", CompletionKind.keyword, double_), false); + ireal_.addChild(makeSymbol("re", CompletionKind.keyword, real_), false); + ifloat_.addChild(makeSymbol("re", CompletionKind.keyword, float_), false); + idouble_.addChild(makeSymbol("re", CompletionKind.keyword, double_), false); + + auto void_ = makeSymbol(builtinTypeNames[14], CompletionKind.keyword); + + builtinSymbols.insert(bool_); + bool_.type = bool_; + builtinSymbols.insert(int_); + int_.type = int_; + builtinSymbols.insert(long_); + long_.type = long_; + builtinSymbols.insert(byte_); + byte_.type = byte_; + builtinSymbols.insert(char_); + char_.type = char_; + builtinSymbols.insert(dchar_); + dchar_.type = dchar_; + builtinSymbols.insert(short_); + short_.type = short_; + builtinSymbols.insert(ubyte_); + ubyte_.type = ubyte_; + builtinSymbols.insert(uint_); + uint_.type = uint_; + builtinSymbols.insert(ulong_); + ulong_.type = ulong_; + builtinSymbols.insert(ushort_); + ushort_.type = ushort_; + builtinSymbols.insert(wchar_); + wchar_.type = wchar_; + builtinSymbols.insert(cdouble_); + cdouble_.type = cdouble_; + builtinSymbols.insert(cent_); + cent_.type = cent_; + builtinSymbols.insert(cfloat_); + cfloat_.type = cfloat_; + builtinSymbols.insert(creal_); + creal_.type = creal_; + builtinSymbols.insert(double_); + double_.type = double_; + builtinSymbols.insert(float_); + float_.type = float_; + builtinSymbols.insert(idouble_); + idouble_.type = idouble_; + builtinSymbols.insert(ifloat_); + ifloat_.type = ifloat_; + builtinSymbols.insert(ireal_); + ireal_.type = ireal_; + builtinSymbols.insert(real_); + real_.type = real_; + builtinSymbols.insert(ucent_); + ucent_.type = ucent_; + builtinSymbols.insert(void_); + void_.type = void_; + + + foreach (s; ["__DATE__", "__EOF__", "__TIME__", "__TIMESTAMP__", "__VENDOR__", + "__VERSION__", "__FUNCTION__", "__PRETTY_FUNCTION__", "__MODULE__", + "__FILE__", "__LINE__", "__FILE_FULL_PATH__"]) + builtinSymbols.insert(makeSymbol(s, CompletionKind.keyword)); +} + +static ~this() +{ + destroy(builtinSymbols); + destroy(arraySymbols); + destroy(assocArraySymbols); + destroy(aggregateSymbols); + destroy(classSymbols); + destroy(enumSymbols); + + foreach (sym; symbolsMadeHere[]) + destroy(*sym); + + destroy(symbolsMadeHere); + destroy(rba); +} + +private RollbackAllocator rba; +private HashSet!(DSymbol*) symbolsMadeHere; + +private DSymbol* makeSymbol(string s, CompletionKind kind, DSymbol* type = null) +{ + auto sym = rba.make!DSymbol(istring(s), kind, type); + sym.ownType = false; + symbolsMadeHere.insert(sym); + return sym; +} +private DSymbol* makeSymbol(istring s, CompletionKind kind, DSymbol* type = null) +{ + auto sym = rba.make!DSymbol(s, kind, type); + sym.ownType = false; + symbolsMadeHere.insert(sym); + return sym; +} diff --git a/dsymbol/src/dsymbol/cache_entry.d b/dsymbol/src/dsymbol/cache_entry.d new file mode 100644 index 0000000..fb6260a --- /dev/null +++ b/dsymbol/src/dsymbol/cache_entry.d @@ -0,0 +1,49 @@ +module dsymbol.cache_entry; + +import dsymbol.symbol; +import std.datetime; +import containers.openhashset; +import containers.unrolledlist; + +/** + * Module cache entry + */ +struct CacheEntry +{ + /// Module root symbol + DSymbol* symbol; + + /// Modification time when this file was last cached + SysTime modificationTime; + + /// The path to the module + istring path; + + /// The modules that this module depends on + OpenHashSet!istring dependencies; + + ~this() + { + if (symbol !is null) + typeid(DSymbol).destroy(symbol); + } + +pure nothrow @nogc @safe: + + ptrdiff_t opCmp(ref const CacheEntry other) const + { + return path.opCmpFast(other.path); + } + + bool opEquals(ref const CacheEntry other) const + { + return path == other.path; + } + + size_t toHash() const + { + return path.toHash(); + } + + @disable void opAssign(ref const CacheEntry other); +} diff --git a/dsymbol/src/dsymbol/conversion/first.d b/dsymbol/src/dsymbol/conversion/first.d new file mode 100644 index 0000000..69b51a0 --- /dev/null +++ b/dsymbol/src/dsymbol/conversion/first.d @@ -0,0 +1,1597 @@ +/** + * This file is part of DCD, a development tool for the D programming language. + * Copyright (C) 2014 Brian Schott + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +module dsymbol.conversion.first; + +import containers.unrolledlist; +import dparse.ast; +import dparse.formatter; +import dparse.lexer; +import dsymbol.builtin.names; +import dsymbol.builtin.symbols; +import dsymbol.cache_entry; +import dsymbol.import_; +import dsymbol.modulecache; +import dsymbol.scope_; +import dsymbol.semantic; +import dsymbol.string_interning; +import dsymbol.symbol; +import dsymbol.type_lookup; +import std.algorithm.iteration : map; +import std.experimental.allocator; +import std.experimental.allocator.gc_allocator : GCAllocator; +import std.experimental.logger; +import std.typecons : Rebindable; + +/** + * First Pass handles the following: + * $(UL + * $(LI symbol name) + * $(LI symbol location) + * $(LI alias this locations) + * $(LI base class names) + * $(LI protection level) + * $(LI symbol kind) + * $(LI function call tip) + * $(LI symbol file path) + * ) + */ +final class FirstPass : ASTVisitor +{ + alias SymbolAllocator = GCAllocator; // NOTE using First`Pass.symbolAllocator` instead fails when analyzing Phobos master + alias ScopeAllocator = GCAllocator; // NOTE using `Mallocator` instead fails when analyzing Phobos master + + /** + * Params: + * mod = the module to visit + * symbolFile = path to the file being converted + * symbolAllocator = allocator used for the auto-complete symbols + * semanticAllocator = allocator used for semantic symbols + */ + this(const Module mod, istring symbolFile, RCIAllocator symbolAllocator, + RCIAllocator semanticAllocator, + ModuleCache* cache, CacheEntry* entry = null) + in + { + assert(mod); + assert(!symbolAllocator.isNull); + assert(!semanticAllocator.isNull); + assert(cache); + } + do + { + this.mod = mod; + this.symbolFile = symbolFile; + this.symbolAllocator = symbolAllocator; + this.semanticAllocator = semanticAllocator; + this.entry = entry; + this.cache = cache; + } + + /** + * Runs the against the AST and produces symbols. + */ + void run() + { + visit(mod); + } + + override void visit(const Unittest u) + { + // Create a dummy symbol because we don't want unit test symbols leaking + // into the symbol they're declared in. + pushSymbol(UNITTEST_SYMBOL_NAME, + CompletionKind.dummy, istring(null)); + scope(exit) popSymbol(); + u.accept(this); + } + + override void visit(const Constructor con) + { + visitConstructor(con.location, con.parameters, con.templateParameters, con.functionBody, con.comment); + } + + override void visit(const SharedStaticConstructor con) + { + visitConstructor(con.location, null, null, con.functionBody, con.comment); + } + + override void visit(const StaticConstructor con) + { + visitConstructor(con.location, null, null, con.functionBody, con.comment); + } + + override void visit(const Destructor des) + { + visitDestructor(des.index, des.functionBody, des.comment); + } + + override void visit(const SharedStaticDestructor des) + { + visitDestructor(des.location, des.functionBody, des.comment); + } + + override void visit(const StaticDestructor des) + { + visitDestructor(des.location, des.functionBody, des.comment); + } + + override void visit(const FunctionDeclaration dec) + { + assert(dec); + pushSymbol(dec.name.text, CompletionKind.functionName, symbolFile, + dec.name.index, dec.returnType); + scope (exit) popSymbol(); + currentSymbol.acSymbol.protection = protection.current; + currentSymbol.acSymbol.doc = makeDocumentation(dec.comment); + + istring lastComment = this.lastComment; + this.lastComment = istring.init; + scope(exit) this.lastComment = lastComment; + + if (dec.functionBody !is null) + { + pushFunctionScope(dec.functionBody, semanticAllocator, + dec.name.index + dec.name.text.length); + scope (exit) popScope(); + processParameters(currentSymbol, dec.returnType, + currentSymbol.acSymbol.name, dec.parameters, dec.templateParameters); + dec.functionBody.accept(this); + } + else + { + processParameters(currentSymbol, dec.returnType, + currentSymbol.acSymbol.name, dec.parameters, dec.templateParameters); + } + } + + override void visit(const FunctionLiteralExpression exp) + { + assert(exp); + + auto fbody = exp.specifiedFunctionBody; + if (fbody is null) + return; + auto block = fbody.blockStatement; + if (block is null) + return; + + pushSymbol(FUNCTION_LITERAL_SYMBOL_NAME, CompletionKind.dummy, symbolFile, + block.startLocation, null); + scope(exit) popSymbol(); + + pushScope(block.startLocation, block.endLocation); + scope (exit) popScope(); + processParameters(currentSymbol, exp.returnType, + FUNCTION_LITERAL_SYMBOL_NAME, exp.parameters, null); + block.accept(this); + } + + override void visit(const ClassDeclaration dec) + { + visitAggregateDeclaration(dec, CompletionKind.className); + } + + override void visit(const TemplateDeclaration dec) + { + visitAggregateDeclaration(dec, CompletionKind.templateName); + } + + override void visit(const InterfaceDeclaration dec) + { + visitAggregateDeclaration(dec, CompletionKind.interfaceName); + } + + override void visit(const UnionDeclaration dec) + { + visitAggregateDeclaration(dec, CompletionKind.unionName); + } + + override void visit(const StructDeclaration dec) + { + visitAggregateDeclaration(dec, CompletionKind.structName); + } + + override void visit(const NewAnonClassExpression nace) + { + // its base classes would be added as "inherit" breadcrumbs in the current symbol + skipBaseClassesOfNewAnon = true; + nace.accept(this); + skipBaseClassesOfNewAnon = false; + } + + override void visit(const BaseClass bc) + { + if (skipBaseClassesOfNewAnon) + return; + if (bc.type2.typeIdentifierPart is null || + bc.type2.typeIdentifierPart.identifierOrTemplateInstance is null) + return; + auto lookup = TypeLookupsAllocator.instance.make!TypeLookup(TypeLookupKind.inherit); + writeIotcTo(bc.type2.typeIdentifierPart, lookup.breadcrumbs); + currentSymbol.typeLookups.insert(lookup); + + // create an alias to the BaseClass to allow completions + // of the form : `instance.BaseClass.`, which is + // mostly used to bypass the most derived overrides. + const idt = lookup.breadcrumbs.back; + if (!idt.length) + return; + SemanticSymbol* symbol = allocateSemanticSymbol(idt, + CompletionKind.aliasName, symbolFile, currentScope.endLocation); + Type t = TypeLookupsAllocator.instance.make!Type; + t.type2 = cast() bc.type2; + addTypeToLookups(symbol.typeLookups, t); + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol, true); + symbol.acSymbol.protection = protection.current; + } + + override void visit(const VariableDeclaration dec) + { + assert (currentSymbol); + foreach (declarator; dec.declarators) + { + SemanticSymbol* symbol = allocateSemanticSymbol( + declarator.name.text, CompletionKind.variableName, + symbolFile, declarator.name.index); + if (dec.type !is null) + addTypeToLookups(symbol.typeLookups, dec.type); + symbol.parent = currentSymbol; + symbol.acSymbol.protection = protection.current; + symbol.acSymbol.doc = makeDocumentation(declarator.comment); + currentSymbol.addChild(symbol, true); + currentScope.addSymbol(symbol.acSymbol, false); + + if (currentSymbol.acSymbol.kind == CompletionKind.structName + || currentSymbol.acSymbol.kind == CompletionKind.unionName) + { + structFieldNames.insert(symbol.acSymbol.name); + // TODO: remove this cast. See the note on structFieldTypes + structFieldTypes.insert(cast() dec.type); + } + } + if (dec.autoDeclaration !is null) + { + foreach (part; dec.autoDeclaration.parts) + { + SemanticSymbol* symbol = allocateSemanticSymbol( + part.identifier.text, CompletionKind.variableName, + symbolFile, part.identifier.index); + symbol.parent = currentSymbol; + populateInitializer(symbol, part.initializer); + symbol.acSymbol.protection = protection.current; + symbol.acSymbol.doc = makeDocumentation(dec.comment); + currentSymbol.addChild(symbol, true); + currentScope.addSymbol(symbol.acSymbol, false); + + if (currentSymbol.acSymbol.kind == CompletionKind.structName + || currentSymbol.acSymbol.kind == CompletionKind.unionName) + { + structFieldNames.insert(symbol.acSymbol.name); + // TODO: remove this cast. See the note on structFieldTypes + structFieldTypes.insert(null); + } + } + } + } + + override void visit(const AliasDeclaration aliasDeclaration) + { + if (aliasDeclaration.initializers.length == 0) + { + foreach (name; aliasDeclaration.declaratorIdentifierList.identifiers) + { + SemanticSymbol* symbol = allocateSemanticSymbol( + name.text, CompletionKind.aliasName, symbolFile, name.index); + if (aliasDeclaration.type !is null) + addTypeToLookups(symbol.typeLookups, aliasDeclaration.type); + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol, true); + currentScope.addSymbol(symbol.acSymbol, false); + symbol.acSymbol.protection = protection.current; + symbol.acSymbol.doc = makeDocumentation(aliasDeclaration.comment); + } + } + else + { + foreach (initializer; aliasDeclaration.initializers) + { + SemanticSymbol* symbol = allocateSemanticSymbol( + initializer.name.text, CompletionKind.aliasName, + symbolFile, initializer.name.index); + if (initializer.type !is null) + addTypeToLookups(symbol.typeLookups, initializer.type); + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol, true); + currentScope.addSymbol(symbol.acSymbol, false); + symbol.acSymbol.protection = protection.current; + symbol.acSymbol.doc = makeDocumentation(aliasDeclaration.comment); + } + } + } + + override void visit(const AliasThisDeclaration dec) + { + const k = currentSymbol.acSymbol.kind; + if (k != CompletionKind.structName && k != CompletionKind.className && + k != CompletionKind.unionName && k != CompletionKind.mixinTemplateName) + { + return; + } + currentSymbol.typeLookups.insert(TypeLookupsAllocator.instance.make!TypeLookup( + internString(dec.identifier.text), TypeLookupKind.aliasThis)); + } + + override void visit(const Declaration dec) + { + if (dec.attributeDeclaration !is null + && isProtection(dec.attributeDeclaration.attribute.attribute.type)) + { + protection.addScope(dec.attributeDeclaration.attribute.attribute.type); + return; + } + IdType p; + foreach (const Attribute attr; dec.attributes) + { + if (isProtection(attr.attribute.type)) + p = attr.attribute.type; + } + if (p != tok!"") + { + protection.beginLocal(p); + if (dec.declarations.length > 0) + { + protection.beginScope(); + dec.accept(this); + protection.endScope(); + } + else + dec.accept(this); + protection.endLocal(); + } + else + dec.accept(this); + } + + override void visit(const Module mod) + { + rootSymbol = allocateSemanticSymbol(null, CompletionKind.moduleName, + symbolFile); + currentSymbol = rootSymbol; + moduleScope = GCAllocator.instance.make!Scope(0, uint.max); // NOTE using `semanticAllocator` here fails as `Segmentation fault (core dumped)` + currentScope = moduleScope; + auto objectLocation = cache.resolveImportLocation("object"); + if (objectLocation is null) + warning("Could not locate object.d or object.di"); + else + { + auto objectImport = allocateSemanticSymbol(IMPORT_SYMBOL_NAME, + CompletionKind.importSymbol, objectLocation); + objectImport.acSymbol.skipOver = true; + currentSymbol.addChild(objectImport, true); + currentScope.addSymbol(objectImport.acSymbol, false); + } + foreach (s; builtinSymbols[]) + currentScope.addSymbol(s, false); + mod.accept(this); + } + + override void visit(const EnumDeclaration dec) + { + assert (currentSymbol); + SemanticSymbol* symbol = allocateSemanticSymbol(dec.name.text, + CompletionKind.enumName, symbolFile, dec.name.index); + if (dec.type !is null) + addTypeToLookups(symbol.typeLookups, dec.type); + symbol.acSymbol.addChildren(enumSymbols[], false); + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol, true); + currentScope.addSymbol(symbol.acSymbol, false); + symbol.acSymbol.doc = makeDocumentation(dec.comment); + + istring lastComment = this.lastComment; + this.lastComment = istring.init; + scope(exit) this.lastComment = lastComment; + + currentSymbol = symbol; + + if (dec.enumBody !is null) + { + pushScope(dec.enumBody.startLocation, dec.enumBody.endLocation); + dec.enumBody.accept(this); + popScope(); + } + + currentSymbol = currentSymbol.parent; + } + + mixin visitEnumMember!EnumMember; + mixin visitEnumMember!AnonymousEnumMember; + + override void visit(const ModuleDeclaration moduleDeclaration) + { + const parts = moduleDeclaration.moduleName.identifiers; + rootSymbol.acSymbol.name = internString(parts.length ? parts[$ - 1].text : null); + } + + override void visit(const StructBody structBody) + { + import std.algorithm : move; + + pushScope(structBody.startLocation, structBody.endLocation); + scope (exit) popScope(); + protection.beginScope(); + scope (exit) protection.endScope(); + + auto savedStructFieldNames = move(structFieldNames); + auto savedStructFieldTypes = move(structFieldTypes); + scope(exit) structFieldNames = move(savedStructFieldNames); + scope(exit) structFieldTypes = move(savedStructFieldTypes); + + DSymbol* thisSymbol = SymbolAllocator.instance.make!DSymbol(THIS_SYMBOL_NAME, + CompletionKind.variableName, currentSymbol.acSymbol); + thisSymbol.location = currentScope.startLocation; + thisSymbol.symbolFile = symbolFile; + thisSymbol.type = currentSymbol.acSymbol; + thisSymbol.ownType = false; + currentScope.addSymbol(thisSymbol, false); + + foreach (dec; structBody.declarations) + visit(dec); + + // If no constructor is found, generate one + if ((currentSymbol.acSymbol.kind == CompletionKind.structName + || currentSymbol.acSymbol.kind == CompletionKind.unionName) + && currentSymbol.acSymbol.getFirstPartNamed(CONSTRUCTOR_SYMBOL_NAME) is null) + createConstructor(); + } + + override void visit(const ImportDeclaration importDeclaration) + { + import std.algorithm : filter, map; + import std.path : buildPath; + import std.typecons : Tuple; + + foreach (single; importDeclaration.singleImports.filter!( + a => a !is null && a.identifierChain !is null)) + { + immutable importPath = convertChainToImportPath(single.identifierChain); + istring modulePath = cache.resolveImportLocation(importPath); + if (modulePath is null) + { + warning("Could not resolve location of module '", importPath.data, "'"); + continue; + } + SemanticSymbol* importSymbol = allocateSemanticSymbol(IMPORT_SYMBOL_NAME, + CompletionKind.importSymbol, modulePath); + importSymbol.acSymbol.skipOver = protection.currentForImport != tok!"public"; + if (single.rename == tok!"") + { + size_t i = 0; + DSymbol* currentImportSymbol; + foreach (p; single.identifierChain.identifiers.map!(a => a.text)) + { + immutable bool first = i == 0; + immutable bool last = i + 1 >= single.identifierChain.identifiers.length; + immutable CompletionKind kind = last ? CompletionKind.moduleName + : CompletionKind.packageName; + istring ip = internString(p); + if (first) + { + auto s = currentScope.getSymbolsByName(ip); + if (s.length == 0) + { + currentImportSymbol = SymbolAllocator.instance.make!DSymbol(ip, kind); + currentScope.addSymbol(currentImportSymbol, true); + if (last) + { + currentImportSymbol.symbolFile = modulePath; + currentImportSymbol.type = importSymbol.acSymbol; + currentImportSymbol.ownType = false; + } + } + else + currentImportSymbol = s[0]; + } + else + { + auto s = currentImportSymbol.getPartsByName(ip); + if (s.length == 0) + { + auto sym = SymbolAllocator.instance.make!DSymbol(ip, kind); + currentImportSymbol.addChild(sym, true); + currentImportSymbol = sym; + if (last) + { + currentImportSymbol.symbolFile = modulePath; + currentImportSymbol.type = importSymbol.acSymbol; + currentImportSymbol.ownType = false; + } + } + else + currentImportSymbol = s[0]; + } + i++; + } + currentSymbol.addChild(importSymbol, true); + currentScope.addSymbol(importSymbol.acSymbol, false); + } + else + { + SemanticSymbol* renameSymbol = allocateSemanticSymbol( + internString(single.rename.text), CompletionKind.aliasName, + modulePath); + renameSymbol.acSymbol.skipOver = protection.currentForImport != tok!"public"; + renameSymbol.acSymbol.type = importSymbol.acSymbol; + renameSymbol.acSymbol.ownType = true; + renameSymbol.addChild(importSymbol, true); + currentSymbol.addChild(renameSymbol, true); + currentScope.addSymbol(renameSymbol.acSymbol, false); + } + if (entry !is null) + entry.dependencies.insert(modulePath); + } + if (importDeclaration.importBindings is null) return; + if (importDeclaration.importBindings.singleImport.identifierChain is null) return; + + immutable chain = convertChainToImportPath(importDeclaration.importBindings.singleImport.identifierChain); + istring modulePath = cache.resolveImportLocation(chain); + if (modulePath is null) + { + warning("Could not resolve location of module '", chain, "'"); + return; + } + + foreach (bind; importDeclaration.importBindings.importBinds) + { + TypeLookup* lookup = TypeLookupsAllocator.instance.make!TypeLookup( + TypeLookupKind.selectiveImport); + + immutable bool isRenamed = bind.right != tok!""; + + // The second phase must change this `importSymbol` kind to + // `aliasName` for symbol lookup to work. + SemanticSymbol* importSymbol = allocateSemanticSymbol( + isRenamed ? bind.left.text : IMPORT_SYMBOL_NAME, + CompletionKind.importSymbol, modulePath); + + if (isRenamed) + { + lookup.breadcrumbs.insert(internString(bind.right.text)); + importSymbol.acSymbol.location = bind.left.index; + importSymbol.acSymbol.altFile = symbolFile; + } + lookup.breadcrumbs.insert(internString(bind.left.text)); + + importSymbol.acSymbol.qualifier = SymbolQualifier.selectiveImport; + importSymbol.typeLookups.insert(lookup); + importSymbol.acSymbol.skipOver = protection.currentForImport != tok!"public"; + currentSymbol.addChild(importSymbol, true); + currentScope.addSymbol(importSymbol.acSymbol, false); + } + + if (entry !is null) + entry.dependencies.insert(modulePath); + } + + // Create scope for block statements + override void visit(const BlockStatement blockStatement) + { + if (blockStatement.declarationsAndStatements !is null) + { + pushScope(blockStatement.startLocation, blockStatement.endLocation); + scope(exit) popScope(); + visit (blockStatement.declarationsAndStatements); + } + } + + // Create attribute/protection scope for conditional compilation declaration + // blocks. + override void visit(const ConditionalDeclaration conditionalDecl) + { + if (conditionalDecl.compileCondition !is null) + visit(conditionalDecl.compileCondition); + + if (conditionalDecl.trueDeclarations.length) + { + protection.beginScope(); + scope (exit) protection.endScope(); + + foreach (decl; conditionalDecl.trueDeclarations) + if (decl !is null) + visit (decl); + } + + if (conditionalDecl.falseDeclarations.length) + { + protection.beginScope(); + scope (exit) protection.endScope(); + + foreach (decl; conditionalDecl.falseDeclarations) + if (decl !is null) + visit (decl); + } + } + + override void visit(const TemplateMixinExpression tme) + { + // TODO: support typeof here + if (tme.mixinTemplateName.symbol is null) + return; + const Symbol sym = tme.mixinTemplateName.symbol; + auto lookup = TypeLookupsAllocator.instance.make!TypeLookup(TypeLookupKind.mixinTemplate); + + writeIotcTo(tme.mixinTemplateName.symbol.identifierOrTemplateChain, + lookup.breadcrumbs); + + if (currentSymbol.acSymbol.kind != CompletionKind.functionName) + currentSymbol.typeLookups.insert(lookup); + + /* If the mixin is named then do like if `mixin F f;` would be `mixin F; alias f = F;` + which's been empirically verified to produce the right completions for `f.`, + */ + if (tme.identifier != tok!"" && sym.identifierOrTemplateChain && + sym.identifierOrTemplateChain.identifiersOrTemplateInstances.length) + { + SemanticSymbol* symbol = allocateSemanticSymbol(tme.identifier.text, + CompletionKind.aliasName, symbolFile, tme.identifier.index); + Type tp = TypeLookupsAllocator.instance.make!Type; + tp.type2 = TypeLookupsAllocator.instance.make!Type2; + TypeIdentifierPart root; + TypeIdentifierPart current; + foreach(ioti; sym.identifierOrTemplateChain.identifiersOrTemplateInstances) + { + TypeIdentifierPart old = current; + current = TypeLookupsAllocator.instance.make!TypeIdentifierPart; + if (old) + { + old.typeIdentifierPart = current; + } + else + { + root = current; + } + current.identifierOrTemplateInstance = cast() ioti; + } + tp.type2.typeIdentifierPart = root; + addTypeToLookups(symbol.typeLookups, tp); + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol, true); + currentScope.addSymbol(symbol.acSymbol, false); + symbol.acSymbol.protection = protection.current; + } + } + + override void visit(const ForeachStatement feStatement) + { + if (feStatement.declarationOrStatement !is null + && feStatement.declarationOrStatement.statement !is null + && feStatement.declarationOrStatement.statement.statementNoCaseNoDefault !is null + && feStatement.declarationOrStatement.statement.statementNoCaseNoDefault.blockStatement !is null) + { + const BlockStatement bs = + feStatement.declarationOrStatement.statement.statementNoCaseNoDefault.blockStatement; + pushScope(feStatement.startIndex, bs.endLocation); + scope(exit) popScope(); + feExpression = feStatement.low.items[$ - 1]; + feStatement.accept(this); + feExpression = null; + } + else + { + const ubyte o1 = foreachTypeIndexOfInterest; + const ubyte o2 = foreachTypeIndex; + feStatement.accept(this); + foreachTypeIndexOfInterest = o1; + foreachTypeIndex = o2; + } + } + + override void visit(const ForeachTypeList feTypeList) + { + foreachTypeIndex = 0; + foreachTypeIndexOfInterest = cast(ubyte)(feTypeList.items.length - 1); + feTypeList.accept(this); + } + + override void visit(const ForeachType feType) + { + if (foreachTypeIndex++ == foreachTypeIndexOfInterest) + { + SemanticSymbol* symbol = allocateSemanticSymbol(feType.identifier.text, + CompletionKind.variableName, symbolFile, feType.identifier.index); + if (feType.type !is null) + addTypeToLookups(symbol.typeLookups, feType.type); + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol, true); + currentScope.addSymbol(symbol.acSymbol, true); + if (symbol.typeLookups.empty && feExpression !is null) + populateInitializer(symbol, feExpression, true); + } + } + + override void visit(const IfStatement ifs) + { + if (ifs.identifier != tok!"" && ifs.thenStatement) + { + pushScope(ifs.thenStatement.startLocation, ifs.thenStatement.endLocation); + scope(exit) popScope(); + + SemanticSymbol* symbol = allocateSemanticSymbol(ifs.identifier.text, + CompletionKind.variableName, symbolFile, ifs.identifier.index); + if (ifs.type !is null) + addTypeToLookups(symbol.typeLookups, ifs.type); + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol, true); + currentScope.addSymbol(symbol.acSymbol, true); + if (symbol.typeLookups.empty && ifs.expression !is null) + populateInitializer(symbol, ifs.expression, false); + } + ifs.accept(this); + } + + override void visit(const WithStatement withStatement) + { + if (withStatement.expression !is null + && withStatement.declarationOrStatement !is null) + { + pushScope(withStatement.declarationOrStatement.startLocation, + withStatement.declarationOrStatement.endLocation); + scope(exit) popScope(); + + pushSymbol(WITH_SYMBOL_NAME, CompletionKind.withSymbol, symbolFile, + currentScope.startLocation, null); + scope(exit) popSymbol(); + + populateInitializer(currentSymbol, withStatement.expression, false); + withStatement.accept(this); + + } + else + withStatement.accept(this); + } + + override void visit(const ArgumentList list) + { + scope visitor = new ArgumentListVisitor(this); + visitor.visit(list); + } + + alias visit = ASTVisitor.visit; + + /// Module scope + Scope* moduleScope; + + /// The module + SemanticSymbol* rootSymbol; + + /// Allocator used for symbol allocation + RCIAllocator symbolAllocator; + + /// Number of symbols allocated + uint symbolsAllocated; + +private: + + void createConstructor() + { + import std.array : appender; + import std.range : zip; + + auto app = appender!string(); + app.put("this("); + bool first = true; + foreach (field; zip(structFieldTypes[], structFieldNames[])) + { + if (first) + first = false; + else + app.put(", "); + if (field[0] is null) + app.put("auto "); + else + { + app.formatNode(field[0]); + app.put(" "); + } + app.put(field[1].data); + } + app.put(")"); + SemanticSymbol* symbol = allocateSemanticSymbol(CONSTRUCTOR_SYMBOL_NAME, + CompletionKind.functionName, symbolFile, currentSymbol.acSymbol.location); + symbol.acSymbol.callTip = istring(app.data); + currentSymbol.addChild(symbol, true); + } + + void pushScope(size_t startLocation, size_t endLocation) + { + assert (startLocation < uint.max); + assert (endLocation < uint.max || endLocation == size_t.max); + Scope* s = ScopeAllocator.instance.make!Scope(cast(uint) startLocation, cast(uint) endLocation); + s.parent = currentScope; + currentScope.children.insert(s); + currentScope = s; + } + + void popScope() + { + currentScope = currentScope.parent; + } + + void pushFunctionScope(const FunctionBody functionBody, + RCIAllocator semanticAllocator, size_t scopeBegin) + { + Scope* s = ScopeAllocator.instance.make!Scope(cast(uint) scopeBegin, + cast(uint) functionBody.endLocation); + s.parent = currentScope; + currentScope.children.insert(s); + currentScope = s; + } + + void pushSymbol(string name, CompletionKind kind, istring symbolFile, + size_t location = 0, const Type type = null) + { + SemanticSymbol* symbol = allocateSemanticSymbol(name, kind, symbolFile, + location); + if (type !is null) + addTypeToLookups(symbol.typeLookups, type); + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol, true); + currentScope.addSymbol(symbol.acSymbol, false); + currentSymbol = symbol; + } + + void popSymbol() + { + currentSymbol = currentSymbol.parent; + } + + template visitEnumMember(T) + { + override void visit(const T member) + { + pushSymbol(member.name.text, CompletionKind.enumMember, symbolFile, + member.name.index, member.type); + scope(exit) popSymbol(); + currentSymbol.acSymbol.doc = makeDocumentation(member.comment); + } + } + + void visitAggregateDeclaration(AggType)(AggType dec, CompletionKind kind) + { + if ((kind == CompletionKind.unionName || kind == CompletionKind.structName) && + dec.name == tok!"") + { + dec.accept(this); + return; + } + pushSymbol(dec.name.text, kind, symbolFile, dec.name.index); + scope(exit) popSymbol(); + + if (kind == CompletionKind.className) + currentSymbol.acSymbol.addChildren(classSymbols[], false); + else + currentSymbol.acSymbol.addChildren(aggregateSymbols[], false); + currentSymbol.acSymbol.protection = protection.current; + currentSymbol.acSymbol.doc = makeDocumentation(dec.comment); + + istring lastComment = this.lastComment; + this.lastComment = istring.init; + scope(exit) this.lastComment = lastComment; + + immutable size_t scopeBegin = dec.name.index + dec.name.text.length; + static if (is (AggType == const(TemplateDeclaration))) + immutable size_t scopeEnd = dec.endLocation; + else + immutable size_t scopeEnd = dec.structBody is null ? scopeBegin : dec.structBody.endLocation; + pushScope(scopeBegin, scopeEnd); + scope(exit) popScope(); + protection.beginScope(); + scope (exit) protection.endScope(); + processTemplateParameters(currentSymbol, dec.templateParameters); + dec.accept(this); + } + + void visitConstructor(size_t location, const Parameters parameters, + const TemplateParameters templateParameters, + const FunctionBody functionBody, string doc) + { + SemanticSymbol* symbol = allocateSemanticSymbol(CONSTRUCTOR_SYMBOL_NAME, + CompletionKind.functionName, symbolFile, location); + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol, true); + processParameters(symbol, null, THIS_SYMBOL_NAME, parameters, templateParameters); + symbol.acSymbol.protection = protection.current; + symbol.acSymbol.doc = makeDocumentation(doc); + + istring lastComment = this.lastComment; + this.lastComment = istring.init; + scope(exit) this.lastComment = lastComment; + + if (functionBody !is null) + { + pushFunctionScope(functionBody, semanticAllocator, + location + 4); // 4 == "this".length + scope(exit) popScope(); + currentSymbol = symbol; + functionBody.accept(this); + currentSymbol = currentSymbol.parent; + } + } + + void visitDestructor(size_t location, const FunctionBody functionBody, string doc) + { + SemanticSymbol* symbol = allocateSemanticSymbol(DESTRUCTOR_SYMBOL_NAME, + CompletionKind.functionName, symbolFile, location); + symbol.parent = currentSymbol; + currentSymbol.addChild(symbol, true); + symbol.acSymbol.callTip = internString("~this()"); + symbol.acSymbol.protection = protection.current; + symbol.acSymbol.doc = makeDocumentation(doc); + + istring lastComment = this.lastComment; + this.lastComment = istring.init; + scope(exit) this.lastComment = lastComment; + + if (functionBody !is null) + { + pushFunctionScope(functionBody, semanticAllocator, location + 4); // 4 == "this".length + scope(exit) popScope(); + currentSymbol = symbol; + functionBody.accept(this); + currentSymbol = currentSymbol.parent; + } + } + + void processParameters(SemanticSymbol* symbol, const Type returnType, + string functionName, const Parameters parameters, + const TemplateParameters templateParameters) + { + processTemplateParameters(symbol, templateParameters); + if (parameters !is null) + { + currentSymbol.acSymbol.functionParameters.reserve(parameters.parameters.length); + foreach (const Parameter p; parameters.parameters) + { + SemanticSymbol* parameter = allocateSemanticSymbol( + p.name.text, CompletionKind.variableName, symbolFile, + p.name.index); + if (p.type !is null) + addTypeToLookups(parameter.typeLookups, p.type); + parameter.parent = currentSymbol; + currentSymbol.acSymbol.argNames.insert(parameter.acSymbol.name); + + currentSymbol.acSymbol.functionParameters ~= parameter.acSymbol; + + currentSymbol.addChild(parameter, true); + currentScope.addSymbol(parameter.acSymbol, false); + } + if (parameters.hasVarargs) + { + SemanticSymbol* argptr = allocateSemanticSymbol(ARGPTR_SYMBOL_NAME, + CompletionKind.variableName, istring(null), size_t.max); + addTypeToLookups(argptr.typeLookups, argptrType); + argptr.parent = currentSymbol; + currentSymbol.addChild(argptr, true); + currentScope.addSymbol(argptr.acSymbol, false); + + SemanticSymbol* arguments = allocateSemanticSymbol( + ARGUMENTS_SYMBOL_NAME, CompletionKind.variableName, + istring(null), size_t.max); + addTypeToLookups(arguments.typeLookups, argumentsType); + arguments.parent = currentSymbol; + currentSymbol.addChild(arguments, true); + currentScope.addSymbol(arguments.acSymbol, false); + } + } + symbol.acSymbol.callTip = formatCallTip(returnType, functionName, + parameters, templateParameters); + } + + void processTemplateParameters(SemanticSymbol* symbol, const TemplateParameters templateParameters) + { + if (templateParameters !is null + && templateParameters.templateParameterList !is null) + { + foreach (const TemplateParameter p; templateParameters.templateParameterList.items) + { + string name; + CompletionKind kind; + size_t index; + Rebindable!(const(Type)) type; + if (p.templateAliasParameter !is null) + { + name = p.templateAliasParameter.identifier.text; + kind = CompletionKind.aliasName; + index = p.templateAliasParameter.identifier.index; + } + else if (p.templateTypeParameter !is null) + { + name = p.templateTypeParameter.identifier.text; + kind = CompletionKind.aliasName; + index = p.templateTypeParameter.identifier.index; + // even if templates are not solved we can get the completions + // for the type the template parameter implicitly converts to, + // which is often useful for aggregate types. + if (p.templateTypeParameter.colonType) + type = p.templateTypeParameter.colonType; + // otherwise just provide standard type properties + else + kind = CompletionKind.typeTmpParam; + } + else if (p.templateValueParameter !is null) + { + name = p.templateValueParameter.identifier.text; + kind = CompletionKind.variableName; + index = p.templateValueParameter.identifier.index; + type = p.templateValueParameter.type; + } + else if (p.templateTupleParameter !is null) + { + name = p.templateTupleParameter.identifier.text; + kind = CompletionKind.variadicTmpParam; + index = p.templateTupleParameter.identifier.index; + } + else + continue; + SemanticSymbol* templateParameter = allocateSemanticSymbol(name, + kind, symbolFile, index); + if (type !is null) + addTypeToLookups(templateParameter.typeLookups, type); + + if (p.templateTupleParameter !is null) + { + TypeLookup* tl = TypeLookupsAllocator.instance.make!TypeLookup( + istring(name), TypeLookupKind.varOrFunType); + templateParameter.typeLookups.insert(tl); + } + else if (p.templateTypeParameter && kind == CompletionKind.typeTmpParam) + { + TypeLookup* tl = TypeLookupsAllocator.instance.make!TypeLookup( + istring(name), TypeLookupKind.varOrFunType); + templateParameter.typeLookups.insert(tl); + } + + templateParameter.parent = symbol; + symbol.addChild(templateParameter, true); + if (currentScope) + currentScope.addSymbol(templateParameter.acSymbol, false); + } + } + } + + istring formatCallTip(const Type returnType, string name, + const Parameters parameters, const TemplateParameters templateParameters) + { + import std.array : appender; + + auto app = appender!string(); + if (returnType !is null) + { + app.formatNode(returnType); + app.put(' '); + } + app.put(name); + if (templateParameters !is null) + app.formatNode(templateParameters); + if (parameters is null) + app.put("()"); + else + app.formatNode(parameters); + return istring(app.data); + } + + void populateInitializer(T)(SemanticSymbol* symbol, const T initializer, + bool appendForeach = false) + { + auto lookup = TypeLookupsAllocator.instance.make!TypeLookup(TypeLookupKind.initializer); + scope visitor = new InitializerVisitor(lookup, appendForeach, this); + symbol.typeLookups.insert(lookup); + visitor.visit(initializer); + } + + SemanticSymbol* allocateSemanticSymbol(string name, CompletionKind kind, + istring symbolFile, size_t location = 0) + in + { + assert (!symbolAllocator.isNull); + } + do + { + DSymbol* acSymbol = SymbolAllocator.instance.make!DSymbol(istring(name), kind); + acSymbol.location = location; + acSymbol.symbolFile = symbolFile; + symbolsAllocated++; + return SymbolAllocator.instance.make!SemanticSymbol(acSymbol); // NOTE using semanticAllocator here breaks when analysing phobos as: `Segmentation fault (core dumped)‘’ + } + + void addTypeToLookups(ref TypeLookups lookups, + const Type type, TypeLookup* l = null) + { + auto lookup = l !is null ? l : TypeLookupsAllocator.instance.make!TypeLookup( + TypeLookupKind.varOrFunType); + auto t2 = type.type2; + if (t2.type !is null) + addTypeToLookups(lookups, t2.type, lookup); + else if (t2.superOrThis is tok!"this") + lookup.breadcrumbs.insert(internString("this")); + else if (t2.superOrThis is tok!"super") + lookup.breadcrumbs.insert(internString("super")); + else if (t2.builtinType !is tok!"") + lookup.breadcrumbs.insert(getBuiltinTypeName(t2.builtinType)); + else if (t2.typeIdentifierPart !is null) + writeIotcTo(t2.typeIdentifierPart, lookup.breadcrumbs); + else + { + // TODO: Add support for typeof expressions + // TODO: Add support for __vector +// warning("typeof() and __vector are not yet supported"); + } + + foreach (suffix; type.typeSuffixes) + { + if (suffix.star != tok!"") + continue; + else if (suffix.type) + lookup.breadcrumbs.insert(ASSOC_ARRAY_SYMBOL_NAME); + else if (suffix.array) + lookup.breadcrumbs.insert(ARRAY_SYMBOL_NAME); + else if (suffix.star != tok!"") + lookup.breadcrumbs.insert(POINTER_SYMBOL_NAME); + else if (suffix.delegateOrFunction != tok!"") + { + import std.array : appender; + auto app = appender!string(); + formatNode(app, type); + istring callTip = istring(app.data); + // Insert the call tip and THEN the "function" string because + // the breadcrumbs are processed in reverse order + lookup.breadcrumbs.insert(callTip); + lookup.breadcrumbs.insert(FUNCTION_SYMBOL_NAME); + } + } + if (l is null) + lookups.insert(lookup); + } + + DocString makeDocumentation(string documentation) + { + if (documentation.isDitto) + return DocString(lastComment, true); + else + { + lastComment = internString(documentation); + return DocString(lastComment, false); + } + } + + /// Current protection type + ProtectionStack protection; + + /// Current scope + Scope* currentScope; + + /// Current symbol + SemanticSymbol* currentSymbol; + + /// Path to the file being converted + istring symbolFile; + + /// Field types used for generating struct constructors if no constructor + /// was defined + // TODO: This should be `const Type`, but Rebindable and opEquals don't play + // well together + UnrolledList!(Type) structFieldTypes; + + /// Field names for struct constructor generation + UnrolledList!(istring) structFieldNames; + + /// Last comment for ditto-ing + istring lastComment; + + const Module mod; + + RCIAllocator semanticAllocator; + + Rebindable!(const ExpressionNode) feExpression; + + CacheEntry* entry; + + ModuleCache* cache; + + bool skipBaseClassesOfNewAnon; + + ubyte foreachTypeIndexOfInterest; + ubyte foreachTypeIndex; +} + +struct ProtectionStack +{ + invariant + { + import std.algorithm.iteration : filter, joiner, map; + import std.conv:to; + import std.range : walkLength; + + assert(stack.length == stack[].filter!(a => isProtection(a) + || a == tok!":" || a == tok!"{").walkLength(), to!string(stack[].map!(a => str(a)).joiner(", "))); + } + + IdType currentForImport() const + { + return stack.empty ? tok!"default" : current(); + } + + IdType current() const + { + import std.algorithm.iteration : filter; + import std.range : choose, only; + + IdType retVal; + foreach (t; choose(stack.empty, only(tok!"public"), stack[]).filter!( + a => a != tok!"{" && a != tok!":")) + retVal = cast(IdType) t; + return retVal; + } + + void beginScope() + { + stack.insertBack(tok!"{"); + } + + void endScope() + { + import std.algorithm.iteration : joiner; + import std.conv : to; + import std.range : walkLength; + + while (!stack.empty && stack.back == tok!":") + { + assert(stack.length >= 2); + stack.popBack(); + stack.popBack(); + } + assert(stack.length == stack[].walkLength()); + assert(!stack.empty && stack.back == tok!"{", to!string(stack[].map!(a => str(a)).joiner(", "))); + stack.popBack(); + } + + void beginLocal(const IdType t) + { + assert (t != tok!"", "DERP!"); + stack.insertBack(t); + } + + void endLocal() + { + import std.algorithm.iteration : joiner; + import std.conv : to; + + assert(!stack.empty && stack.back != tok!":" && stack.back != tok!"{", + to!string(stack[].map!(a => str(a)).joiner(", "))); + stack.popBack(); + } + + void addScope(const IdType t) + { + assert(t != tok!"", "DERP!"); + assert(isProtection(t)); + if (!stack.empty && stack.back == tok!":") + { + assert(stack.length >= 2); + stack.popBack(); + assert(isProtection(stack.back)); + stack.popBack(); + } + stack.insertBack(t); + stack.insertBack(tok!":"); + } + +private: + + UnrolledList!IdType stack; +} + +void formatNode(A, T)(ref A appender, const T node) +{ + if (node is null) + return; + scope f = new Formatter!(A*)(&appender); + f.format(node); +} + +private: + +bool isDitto(scope const(char)[] comment) +{ + import std.uni : icmp; + + return comment.length == 5 && icmp(comment, "ditto") == 0; +} + +void writeIotcTo(T)(const TypeIdentifierPart tip, ref T output) nothrow +{ + if (!tip.identifierOrTemplateInstance) + return; + if (tip.identifierOrTemplateInstance.identifier != tok!"") + output.insert(internString(tip.identifierOrTemplateInstance.identifier.text)); + else + output.insert(internString(tip.identifierOrTemplateInstance.templateInstance.identifier.text)); + + // the indexer of a TypeIdentifierPart means either that there's + // a static array dimension or that a type is selected in a type list. + // we can only handle the first case since dsymbol does not process templates yet. + if (tip.indexer) + output.insert(ARRAY_SYMBOL_NAME); + + if (tip.typeIdentifierPart) + writeIotcTo(tip.typeIdentifierPart, output); +} + +auto byIdentifier(const IdentifierOrTemplateChain iotc) nothrow +{ + import std.algorithm : map; + + return iotc.identifiersOrTemplateInstances.map!(a => a.identifier == tok!"" + ? a.templateInstance.identifier.text + : a.identifier.text); +} + +void writeIotcTo(T)(const IdentifierOrTemplateChain iotc, ref T output) nothrow +{ + import std.algorithm : each; + + byIdentifier(iotc).each!(a => output.insert(internString(a))); +} + +static istring convertChainToImportPath(const IdentifierChain ic) +{ + import std.path : dirSeparator; + import std.array : appender; + auto app = appender!string(); + foreach (i, ident; ic.identifiers) + { + app.put(ident.text); + if (i + 1 < ic.identifiers.length) + app.put(dirSeparator); + } + return istring(app.data); +} + +class InitializerVisitor : ASTVisitor +{ + this (TypeLookup* lookup, bool appendForeach, FirstPass fp) + { + this.lookup = lookup; + this.appendForeach = appendForeach; + this.fp = fp; + } + + alias visit = ASTVisitor.visit; + + override void visit(const FunctionLiteralExpression exp) + { + fp.visit(exp); + } + + override void visit(const IdentifierOrTemplateInstance ioti) + { + if (on && ioti.identifier != tok!"") + lookup.breadcrumbs.insert(internString(ioti.identifier.text)); + else if (on && ioti.templateInstance.identifier != tok!"") + lookup.breadcrumbs.insert(internString(ioti.templateInstance.identifier.text)); + ioti.accept(this); + } + + override void visit(const PrimaryExpression primary) + { + // Add identifiers without processing. Convert literals to strings with + // the prefix '*' so that that the second pass can tell the difference + // between "int.abc" and "10.abc". + if (on && primary.basicType != tok!"") + lookup.breadcrumbs.insert(internString(str(primary.basicType.type))); + if (on) switch (primary.primary.type) + { + case tok!"identifier": + lookup.breadcrumbs.insert(internString(primary.primary.text)); + break; + case tok!"doubleLiteral": + lookup.breadcrumbs.insert(DOUBLE_LITERAL_SYMBOL_NAME); + break; + case tok!"floatLiteral": + lookup.breadcrumbs.insert(FLOAT_LITERAL_SYMBOL_NAME); + break; + case tok!"idoubleLiteral": + lookup.breadcrumbs.insert(IDOUBLE_LITERAL_SYMBOL_NAME); + break; + case tok!"ifloatLiteral": + lookup.breadcrumbs.insert(IFLOAT_LITERAL_SYMBOL_NAME); + break; + case tok!"intLiteral": + lookup.breadcrumbs.insert(INT_LITERAL_SYMBOL_NAME); + break; + case tok!"longLiteral": + lookup.breadcrumbs.insert(LONG_LITERAL_SYMBOL_NAME); + break; + case tok!"realLiteral": + lookup.breadcrumbs.insert(REAL_LITERAL_SYMBOL_NAME); + break; + case tok!"irealLiteral": + lookup.breadcrumbs.insert(IREAL_LITERAL_SYMBOL_NAME); + break; + case tok!"uintLiteral": + lookup.breadcrumbs.insert(UINT_LITERAL_SYMBOL_NAME); + break; + case tok!"ulongLiteral": + lookup.breadcrumbs.insert(ULONG_LITERAL_SYMBOL_NAME); + break; + case tok!"characterLiteral": + lookup.breadcrumbs.insert(CHAR_LITERAL_SYMBOL_NAME); + break; + case tok!"dstringLiteral": + lookup.breadcrumbs.insert(DSTRING_LITERAL_SYMBOL_NAME); + break; + case tok!"stringLiteral": + lookup.breadcrumbs.insert(STRING_LITERAL_SYMBOL_NAME); + break; + case tok!"wstringLiteral": + lookup.breadcrumbs.insert(WSTRING_LITERAL_SYMBOL_NAME); + break; + case tok!"false": + case tok!"true": + lookup.breadcrumbs.insert(BOOL_VALUE_SYMBOL_NAME); + break; + default: + break; + } + primary.accept(this); + } + + override void visit(const IndexExpression expr) + { + expr.unaryExpression.accept(this); + foreach (index; expr.indexes) + if (index.high is null) + lookup.breadcrumbs.insert(ARRAY_SYMBOL_NAME); + } + + override void visit(const Initializer initializer) + { + on = true; + initializer.accept(this); + on = false; + } + + override void visit(const ArrayInitializer ai) + { + // If the array has any elements, assume all elements have the + // same type as the first element. + if (ai.arrayMemberInitializations.length) + ai.arrayMemberInitializations[0].accept(this); + else + lookup.breadcrumbs.insert(VOID_SYMBOL_NAME); + + lookup.breadcrumbs.insert(ARRAY_LITERAL_SYMBOL_NAME); + } + + override void visit(const ArrayLiteral al) + { + // ditto + if (al.argumentList) + { + if (al.argumentList.items.length) + al.argumentList.items[0].accept(this); + else + lookup.breadcrumbs.insert(VOID_SYMBOL_NAME); + } + lookup.breadcrumbs.insert(ARRAY_LITERAL_SYMBOL_NAME); + } + + // Skip it + override void visit(const NewAnonClassExpression) {} + + override void visit(const NewExpression ne) + { + if (ne.newAnonClassExpression) + lowerNewAnonToNew((cast() ne)); + ne.accept(this); + } + + private void lowerNewAnonToNew(NewExpression ne) + { + import std.format : format; + + // here we follow DMDFE naming style + __gshared size_t anonIndex; + const idt = istring("__anonclass%d".format(++anonIndex)); + + // the goal is to replace it so we null the field + NewAnonClassExpression nace = ne.newAnonClassExpression; + ne.newAnonClassExpression = null; + + // Lower the AnonClass body to a standard ClassDeclaration and visit it. + ClassDeclaration cd = theAllocator.make!(ClassDeclaration); + cd.name = Token(tok!"identifier", idt, 1, 1, nace.structBody.startLocation - idt.length); + cd.baseClassList = nace.baseClassList; + cd.structBody = nace.structBody; + fp.visit(cd); + + // Change the NewAnonClassExpression to a standard NewExpression using + // the ClassDeclaration created in previous step + ne.type = theAllocator.make!(Type); + ne.type.type2 = theAllocator.make!(Type2); + ne.type.type2.typeIdentifierPart = theAllocator.make!(TypeIdentifierPart); + ne.type.type2.typeIdentifierPart.identifierOrTemplateInstance = theAllocator.make!(IdentifierOrTemplateInstance); + ne.type.type2.typeIdentifierPart.identifierOrTemplateInstance.identifier = cd.name; + ne.arguments = nace.constructorArguments; + } + + override void visit(const ArgumentList list) + { + scope visitor = new ArgumentListVisitor(fp); + visitor.visit(list); + } + + override void visit(const Expression expression) + { + on = true; + expression.accept(this); + if (appendForeach) + lookup.breadcrumbs.insert(internString("foreach")); + on = false; + } + + override void visit(const ExpressionNode expression) + { + on = true; + expression.accept(this); + if (appendForeach) + lookup.breadcrumbs.insert(internString("foreach")); + on = false; + } + + TypeLookup* lookup; + bool on = false; + const bool appendForeach; + FirstPass fp; +} + +class ArgumentListVisitor : ASTVisitor +{ + this(FirstPass fp) + { + assert(fp); + this.fp = fp; + } + + alias visit = ASTVisitor.visit; + + override void visit(const FunctionLiteralExpression exp) + { + fp.visit(exp); + } + + override void visit(const NewAnonClassExpression exp) + { + fp.visit(exp); + } + +private: + FirstPass fp; +} diff --git a/dsymbol/src/dsymbol/conversion/package.d b/dsymbol/src/dsymbol/conversion/package.d new file mode 100644 index 0000000..9249173 --- /dev/null +++ b/dsymbol/src/dsymbol/conversion/package.d @@ -0,0 +1,253 @@ +/** + * This file is part of DCD, a development tool for the D programming language. + * Copyright (C) 2014 Brian Schott + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +module dsymbol.conversion; + +import dsymbol.cache_entry; +import dsymbol.conversion.first; +import dsymbol.conversion.second; +import dsymbol.modulecache; +import dsymbol.scope_; +import dsymbol.string_interning; +import dsymbol.symbol; +import dsymbol.semantic; +import dparse.ast; +import dparse.lexer; +import dparse.parser; +import dparse.rollback_allocator; +import std.experimental.allocator; + +/** + * Used by autocompletion. + */ +ScopeSymbolPair generateAutocompleteTrees(const(Token)[] tokens, + RCIAllocator symbolAllocator, RollbackAllocator* parseAllocator, + size_t cursorPosition, ref ModuleCache cache) +{ + Module m = parseModuleForAutocomplete(tokens, internString("stdin"), + parseAllocator, cursorPosition); + + scope first = new FirstPass(m, internString("stdin"), symbolAllocator, + symbolAllocator, &cache); + first.run(); + + secondPass(first.rootSymbol, first.moduleScope, cache); + auto r = first.rootSymbol.acSymbol; + typeid(SemanticSymbol).destroy(first.rootSymbol); + return ScopeSymbolPair(r, first.moduleScope); +} + +struct ScopeSymbolPair +{ + void destroy() + { + typeid(DSymbol).destroy(symbol); + typeid(Scope).destroy(scope_); + } + + DSymbol* symbol; + Scope* scope_; +} + +/** + * Used by import symbol caching. + * + * Params: + * tokens = the tokens that compose the file + * fileName = the name of the file being parsed + * parseAllocator = the allocator to use for the AST + * Returns: the parsed module + */ +Module parseModuleSimple(const(Token)[] tokens, string fileName, RollbackAllocator* parseAllocator) +{ + assert (parseAllocator !is null); + scope parser = new SimpleParser(); + parser.fileName = fileName; + parser.tokens = tokens; + parser.messageFunction = &doesNothing; + parser.allocator = parseAllocator; + return parser.parseModule(); +} + +private: + +Module parseModuleForAutocomplete(const(Token)[] tokens, string fileName, + RollbackAllocator* parseAllocator, size_t cursorPosition) +{ + scope parser = new AutocompleteParser(); + parser.fileName = fileName; + parser.tokens = tokens; + parser.messageFunction = &doesNothing; + parser.allocator = parseAllocator; + parser.cursorPosition = cursorPosition; + return parser.parseModule(); +} + +class AutocompleteParser : Parser +{ + override BlockStatement parseBlockStatement() + { + if (!currentIs(tok!"{")) + return null; + if (current.index > cursorPosition) + { + BlockStatement bs = allocator.make!(BlockStatement); + bs.startLocation = current.index; + skipBraces(); + bs.endLocation = tokens[index - 1].index; + return bs; + } + immutable start = current.index; + auto b = setBookmark(); + skipBraces(); + if (tokens[index - 1].index < cursorPosition) + { + abandonBookmark(b); + BlockStatement bs = allocator.make!BlockStatement(); + bs.startLocation = start; + bs.endLocation = tokens[index - 1].index; + return bs; + } + else + { + goToBookmark(b); + return super.parseBlockStatement(); + } + } + +private: + size_t cursorPosition; +} + +class SimpleParser : Parser +{ + override Unittest parseUnittest() + { + expect(tok!"unittest"); + if (currentIs(tok!"{")) + skipBraces(); + return allocator.make!Unittest; + } + + override MissingFunctionBody parseMissingFunctionBody() + { + // Unlike many of the other parsing functions, it is valid and expected + // for this one to return `null` on valid code. Returning `null` in + // this function means that we are looking at a SpecifiedFunctionBody + // or ShortenedFunctionBody. + // + // The super-class will handle re-trying with the correct parsing + // function. + + const bool needDo = skipContracts(); + if (needDo && moreTokens && (currentIs(tok!"do") || current.text == "body")) + return null; + if (currentIs(tok!";")) + advance(); + else + return null; + return allocator.make!MissingFunctionBody; + } + + override SpecifiedFunctionBody parseSpecifiedFunctionBody() + { + if (currentIs(tok!"{")) + skipBraces(); + else + { + skipContracts(); + if (currentIs(tok!"do") || (currentIs(tok!"identifier") && current.text == "body")) + advance(); + if (currentIs(tok!"{")) + skipBraces(); + } + return allocator.make!SpecifiedFunctionBody; + } + + override ShortenedFunctionBody parseShortenedFunctionBody() + { + skipContracts(); + if (currentIs(tok!"=>")) + { + while (!currentIs(tok!";") && moreTokens) + { + if (currentIs(tok!"{")) // potential function literal + skipBraces(); + else + advance(); + } + if (moreTokens) + advance(); + return allocator.make!ShortenedFunctionBody; + } + else + { + return null; + } + } + + /** + * Skip contracts, and return `true` if the type of contract used requires + * that the next token is `do`. + */ + private bool skipContracts() + { + bool needDo; + + while (true) + { + if (currentIs(tok!"in")) + { + advance(); + if (currentIs(tok!"{")) + { + skipBraces(); + needDo = true; + } + if (currentIs(tok!"(")) + skipParens(); + } + else if (currentIs(tok!"out")) + { + advance(); + if (currentIs(tok!"(")) + { + immutable bool asExpr = peekIs(tok!";") + || (peekIs(tok!"identifier") + && index + 2 < tokens.length && tokens[index + 2].type == tok!";"); + skipParens(); + if (asExpr) + { + needDo = false; + continue; + } + } + if (currentIs(tok!"{")) + { + skipBraces(); + needDo = true; + } + } + else + break; + } + return needDo; + } +} + +void doesNothing(string, size_t, size_t, string, bool) {} diff --git a/dsymbol/src/dsymbol/conversion/second.d b/dsymbol/src/dsymbol/conversion/second.d new file mode 100644 index 0000000..f1d0e02 --- /dev/null +++ b/dsymbol/src/dsymbol/conversion/second.d @@ -0,0 +1,527 @@ +/** + * This file is part of DCD, a development tool for the D programming language. + * Copyright (C) 2014 Brian Schott + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +module dsymbol.conversion.second; + +import dsymbol.semantic; +import dsymbol.string_interning; +import dsymbol.symbol; +import dsymbol.scope_; +import dsymbol.builtin.names; +import dsymbol.builtin.symbols; +import dsymbol.type_lookup; +import dsymbol.deferred; +import dsymbol.import_; +import dsymbol.modulecache; +import std.experimental.allocator; +import std.experimental.allocator.gc_allocator : GCAllocator; +import std.experimental.logger; +import dparse.ast; +import dparse.lexer; + +alias SymbolAllocator = GCAllocator; // NOTE using cache.symbolAllocator instead fails when analyzing Phobos master + +void secondPass(SemanticSymbol* currentSymbol, Scope* moduleScope, ref ModuleCache cache) +{ + with (CompletionKind) final switch (currentSymbol.acSymbol.kind) + { + case className: + case interfaceName: + resolveInheritance(currentSymbol.acSymbol, currentSymbol.typeLookups, + moduleScope, cache); + break; + case withSymbol: + case variableName: + case memberVariableName: + case functionName: + case aliasName: + // type may not be null in the case of a renamed import + if (currentSymbol.acSymbol.type is null) + { + resolveType(currentSymbol.acSymbol, currentSymbol.typeLookups, + moduleScope, cache); + } + break; + case importSymbol: + if (currentSymbol.acSymbol.type is null) + resolveImport(currentSymbol.acSymbol, currentSymbol.typeLookups, cache); + break; + case variadicTmpParam: + currentSymbol.acSymbol.type = variadicTmpParamSymbol; + break; + case typeTmpParam: + currentSymbol.acSymbol.type = typeTmpParamSymbol; + break; + case structName: + case unionName: + case enumName: + case keyword: + case enumMember: + case packageName: + case moduleName: + case dummy: + case templateName: + case mixinTemplateName: + break; + } + + foreach (child; currentSymbol.children) + secondPass(child, moduleScope, cache); + + // Alias this and mixin templates are resolved after child nodes are + // resolved so that the correct symbol information will be available. + with (CompletionKind) switch (currentSymbol.acSymbol.kind) + { + case className: + case interfaceName: + case structName: + case unionName: + resolveAliasThis(currentSymbol.acSymbol, currentSymbol.typeLookups, moduleScope, cache); + resolveMixinTemplates(currentSymbol.acSymbol, currentSymbol.typeLookups, + moduleScope, cache); + break; + default: + break; + } +} + +void resolveImport(DSymbol* acSymbol, ref TypeLookups typeLookups, + ref ModuleCache cache) +in +{ + assert(acSymbol.kind == CompletionKind.importSymbol); + assert(acSymbol.symbolFile !is null); +} +do +{ + DSymbol* moduleSymbol = cache.cacheModule(acSymbol.symbolFile); + if (acSymbol.qualifier == SymbolQualifier.selectiveImport) + { + if (moduleSymbol is null) + { + tryAgain: + DeferredSymbol* deferred = TypeLookupsAllocator.instance.make!DeferredSymbol(acSymbol); + deferred.typeLookups.insert(typeLookups[]); + // Get rid of the old references to the lookups, this new deferred + // symbol owns them now + typeLookups.clear(); + cache.deferredSymbols.insert(deferred); + } + else + { + immutable size_t breadcrumbCount = typeLookups.front.breadcrumbs.length; + assert(breadcrumbCount <= 2 && breadcrumbCount > 0, "Malformed selective import"); + + istring symbolName = typeLookups.front.breadcrumbs.front; + DSymbol* selected = moduleSymbol.getFirstPartNamed(symbolName); + if (selected is null) + goto tryAgain; + acSymbol.type = selected; + acSymbol.ownType = false; + + // count of 1 means selective import + // count of 2 means a renamed selective import + if (breadcrumbCount == 2) + { + acSymbol.kind = CompletionKind.aliasName; + acSymbol.symbolFile = acSymbol.altFile; + } + } + } + else + { + if (moduleSymbol is null) + { + DeferredSymbol* deferred = DeferredSymbolsAllocator.instance.make!DeferredSymbol(acSymbol); + cache.deferredSymbols.insert(deferred); + } + else + { + acSymbol.type = moduleSymbol; + acSymbol.ownType = false; + } + } +} + +void resolveTypeFromType(DSymbol* symbol, TypeLookup* lookup, Scope* moduleScope, + ref ModuleCache cache, Imports* imports) +in +{ + if (imports !is null) + foreach (i; imports.opSlice()) + assert(i.kind == CompletionKind.importSymbol); +} +do +{ + // The left-most suffix + DSymbol* suffix; + // The right-most suffix + DSymbol* lastSuffix; + + // Create symbols for the type suffixes such as array and + // associative array + while (!lookup.breadcrumbs.empty) + { + auto back = lookup.breadcrumbs.back; + immutable bool isArr = back == ARRAY_SYMBOL_NAME; + immutable bool isAssoc = back == ASSOC_ARRAY_SYMBOL_NAME; + immutable bool isFunction = back == FUNCTION_SYMBOL_NAME; + if (back == POINTER_SYMBOL_NAME) + { + lastSuffix.isPointer = true; + lookup.breadcrumbs.popBack(); + continue; + } + if (!isArr && !isAssoc && !isFunction) + break; + immutable qualifier = isAssoc ? SymbolQualifier.assocArray : + (isFunction ? SymbolQualifier.func : SymbolQualifier.array); + lastSuffix = SymbolAllocator.instance.make!DSymbol(back, CompletionKind.dummy, lastSuffix); + lastSuffix.qualifier = qualifier; + lastSuffix.ownType = true; + if (isFunction) + { + lookup.breadcrumbs.popBack(); + lastSuffix.callTip = lookup.breadcrumbs.back(); + } + else + lastSuffix.addChildren(isArr ? arraySymbols[] : assocArraySymbols[], false); + + if (suffix is null) + suffix = lastSuffix; + lookup.breadcrumbs.popBack(); + } + + Imports remainingImports; + + DSymbol* currentSymbol; + + void getSymbolFromImports(Imports* importList, istring name) + { + foreach (im; importList.opSlice()) + { + assert(im.symbolFile !is null); + // Try to find a cached version of the module + DSymbol* moduleSymbol = cache.getModuleSymbol(im.symbolFile); + // If the module has not been cached yet, store it in the + // remaining imports list + if (moduleSymbol is null) + { + remainingImports.insert(im); + continue; + } + // Try to get the symbol from the imported module + currentSymbol = moduleSymbol.getFirstPartNamed(name); + if (currentSymbol is null) + continue; + } + } + + // Follow all the names and try to resolve them + size_t i = 0; + foreach (part; lookup.breadcrumbs[]) + { + if (i == 0) + { + if (moduleScope is null) + getSymbolFromImports(imports, part); + else + { + auto symbols = moduleScope.getSymbolsByNameAndCursor(part, symbol.location); + if (symbols.length > 0) + currentSymbol = symbols[0]; + else + return; + } + } + else + { + if (currentSymbol.kind == CompletionKind.aliasName) + currentSymbol = currentSymbol.type; + if (currentSymbol is null) + return; + if (currentSymbol.kind == CompletionKind.moduleName && currentSymbol.type !is null) + currentSymbol = currentSymbol.type; + if (currentSymbol is null) + return; + if (currentSymbol.kind == CompletionKind.importSymbol) + currentSymbol = currentSymbol.type; + if (currentSymbol is null) + return; + currentSymbol = currentSymbol.getFirstPartNamed(part); + } + ++i; + if (currentSymbol is null) + return; + } + + if (lastSuffix !is null) + { + assert(suffix !is null); + suffix.type = currentSymbol; + suffix.ownType = false; + symbol.type = lastSuffix; + symbol.ownType = true; + if (currentSymbol is null && !remainingImports.empty) + { +// info("Deferring type resolution for ", symbol.name); + auto deferred = DeferredSymbolsAllocator.instance.make!DeferredSymbol(suffix); + // TODO: The scope has ownership of the import information + deferred.imports.insert(remainingImports[]); + deferred.typeLookups.insert(lookup); + cache.deferredSymbols.insert(deferred); + } + } + else if (currentSymbol !is null) + { + symbol.type = currentSymbol; + symbol.ownType = false; + } + else if (!remainingImports.empty) + { + auto deferred = DeferredSymbolsAllocator.instance.make!DeferredSymbol(symbol); +// info("Deferring type resolution for ", symbol.name); + // TODO: The scope has ownership of the import information + deferred.imports.insert(remainingImports[]); + deferred.typeLookups.insert(lookup); + cache.deferredSymbols.insert(deferred); + } +} + +private: + +void resolveInheritance(DSymbol* symbol, ref TypeLookups typeLookups, + Scope* moduleScope, ref ModuleCache cache) +{ + import std.algorithm : filter; + + outer: foreach (TypeLookup* lookup; typeLookups[]) + { + if (lookup.kind != TypeLookupKind.inherit) + continue; + DSymbol* baseClass; + assert(lookup.breadcrumbs.length > 0); + + // TODO: Delayed type lookup + auto symbolScope = moduleScope.getScopeByCursor( + symbol.location + symbol.name.length); + auto symbols = moduleScope.getSymbolsByNameAndCursor(lookup.breadcrumbs.front, + symbol.location); + if (symbols.length == 0) + continue; + + baseClass = symbols[0]; + lookup.breadcrumbs.popFront(); + foreach (part; lookup.breadcrumbs[]) + { + symbols = baseClass.getPartsByName(part); + if (symbols.length == 0) + continue outer; + baseClass = symbols[0]; + } + + DSymbol* imp = SymbolAllocator.instance.make!DSymbol(IMPORT_SYMBOL_NAME, + CompletionKind.importSymbol, baseClass); + symbol.addChild(imp, true); + symbolScope.addSymbol(imp, false); + if (baseClass.kind == CompletionKind.className) + { + auto s = SymbolAllocator.instance.make!DSymbol(SUPER_SYMBOL_NAME, + CompletionKind.variableName, baseClass); + symbolScope.addSymbol(s, true); + } + } +} + +void resolveAliasThis(DSymbol* symbol, + ref TypeLookups typeLookups, Scope* moduleScope, ref ModuleCache cache) +{ + import std.algorithm : filter; + + foreach (aliasThis; typeLookups[].filter!(a => a.kind == TypeLookupKind.aliasThis)) + { + assert(aliasThis.breadcrumbs.length > 0); + auto parts = symbol.getPartsByName(aliasThis.breadcrumbs.front); + if (parts.length == 0 || parts[0].type is null) + continue; + DSymbol* s = SymbolAllocator.instance.make!DSymbol(IMPORT_SYMBOL_NAME, + CompletionKind.importSymbol, parts[0].type); + symbol.addChild(s, true); + auto symbolScope = moduleScope.getScopeByCursor(s.location); + if (symbolScope !is null) + symbolScope.addSymbol(s, false); + } +} + +void resolveMixinTemplates(DSymbol* symbol, + ref TypeLookups typeLookups, Scope* moduleScope, ref ModuleCache cache) +{ + import std.algorithm : filter; + + foreach (mix; typeLookups[].filter!(a => a.kind == TypeLookupKind.mixinTemplate)) + { + assert(mix.breadcrumbs.length > 0); + auto symbols = moduleScope.getSymbolsByNameAndCursor(mix.breadcrumbs.front, + symbol.location); + if (symbols.length == 0) + continue; + auto currentSymbol = symbols[0]; + mix.breadcrumbs.popFront(); + foreach (m; mix.breadcrumbs[]) + { + auto s = currentSymbol.getPartsByName(m); + if (s.length == 0) + { + currentSymbol = null; + break; + } + else + currentSymbol = s[0]; + } + if (currentSymbol !is null) + { + auto i = SymbolAllocator.instance.make!DSymbol(IMPORT_SYMBOL_NAME, + CompletionKind.importSymbol, currentSymbol); + i.ownType = false; + symbol.addChild(i, true); + } + } +} + +void resolveType(DSymbol* symbol, ref TypeLookups typeLookups, + Scope* moduleScope, ref ModuleCache cache) +{ + + import std.conv; + + if (typeLookups.length == 0) + return; + assert(typeLookups.length == 1); + auto lookup = typeLookups.front; + if (lookup.kind == TypeLookupKind.varOrFunType) + resolveTypeFromType(symbol, lookup, moduleScope, cache, null); + else if (lookup.kind == TypeLookupKind.initializer) + resolveTypeFromInitializer(symbol, lookup, moduleScope, cache); + // issue 94 + else if (lookup.kind == TypeLookupKind.inherit) + resolveInheritance(symbol, typeLookups, moduleScope, cache); + else + assert(false, "How did this happen?"); +} + + +void resolveTypeFromInitializer(DSymbol* symbol, TypeLookup* lookup, + Scope* moduleScope, ref ModuleCache cache) +{ + if (lookup.breadcrumbs.length == 0) + return; + DSymbol* currentSymbol = null; + size_t i = 0; + + auto crumbs = lookup.breadcrumbs[]; + foreach (crumb; crumbs) + { + if (i == 0) + { + currentSymbol = moduleScope.getFirstSymbolByNameAndCursor( + symbolNameToTypeName(crumb), symbol.location); + + if (currentSymbol is null) + return; + } + else + if (crumb == ARRAY_LITERAL_SYMBOL_NAME) + { + auto arr = SymbolAllocator.instance.make!(DSymbol)(ARRAY_LITERAL_SYMBOL_NAME, CompletionKind.dummy, currentSymbol); + arr.qualifier = SymbolQualifier.array; + currentSymbol = arr; + } + else if (crumb == ARRAY_SYMBOL_NAME) + { + typeSwap(currentSymbol); + if (currentSymbol is null) + return; + + // Index expressions can be an array index or an AA index + if (currentSymbol.qualifier == SymbolQualifier.array + || currentSymbol.qualifier == SymbolQualifier.assocArray + || currentSymbol.kind == CompletionKind.aliasName) + { + if (currentSymbol.type !is null) + currentSymbol = currentSymbol.type; + else + return; + } + else + { + auto opIndex = currentSymbol.getFirstPartNamed(internString("opIndex")); + if (opIndex !is null) + currentSymbol = opIndex.type; + else + return; + } + } + else if (crumb == "foreach") + { + typeSwap(currentSymbol); + if (currentSymbol is null) + return; + if (currentSymbol.qualifier == SymbolQualifier.array + || currentSymbol.qualifier == SymbolQualifier.assocArray) + { + currentSymbol = currentSymbol.type; + break; + } + auto front = currentSymbol.getFirstPartNamed(internString("front")); + if (front !is null) + { + currentSymbol = front.type; + break; + } + auto opApply = currentSymbol.getFirstPartNamed(internString("opApply")); + if (opApply !is null) + { + currentSymbol = opApply.type; + break; + } + } + else + { + typeSwap(currentSymbol); + if (currentSymbol is null ) + return; + currentSymbol = currentSymbol.getFirstPartNamed(crumb); + } + ++i; + if (currentSymbol is null) + return; + } + typeSwap(currentSymbol); + symbol.type = currentSymbol; + symbol.ownType = false; +} + +void typeSwap(ref DSymbol* currentSymbol) +{ + while (currentSymbol !is null && currentSymbol.type !is currentSymbol + && (currentSymbol.kind == CompletionKind.variableName + || currentSymbol.kind == CompletionKind.importSymbol + || currentSymbol.kind == CompletionKind.withSymbol + || currentSymbol.kind == CompletionKind.aliasName)) + currentSymbol = currentSymbol.type; +} diff --git a/dsymbol/src/dsymbol/deferred.d b/dsymbol/src/dsymbol/deferred.d new file mode 100644 index 0000000..10dac5a --- /dev/null +++ b/dsymbol/src/dsymbol/deferred.d @@ -0,0 +1,61 @@ +/** + * This file is part of DCD, a development tool for the D programming language. + * Copyright (C) 2014 Brian Schott + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +module dsymbol.deferred; + +import containers.unrolledlist; +import containers.openhashset; +import dsymbol.string_interning; +import dsymbol.import_; +import dsymbol.symbol; +import dsymbol.type_lookup; +import std.experimental.allocator : dispose; +import std.experimental.allocator.mallocator : Mallocator; +import dsymbol.semantic : TypeLookups, TypeLookupsAllocator; + +alias ImportsAllocator = Mallocator; +alias Imports = UnrolledList!(DSymbol*, ImportsAllocator); + +/** + * Contains information for deferred type resolution + */ +struct DeferredSymbol +{ + ~this() + { + foreach (l; typeLookups[]) + TypeLookupsAllocator.instance.dispose(l); + foreach (i; imports[]) + ImportsAllocator.instance.dispose(i); + } + + bool dependsOn(istring modulePath) + { + foreach (i; imports[]) + if (i.symbolFile == modulePath) + return true; + return false; + } + + /// The symbol that needs its type resolved + DSymbol* symbol; + /// The imports that were in scope for the symbol's declaration' + Imports imports; + /// The type lookup information + TypeLookups typeLookups; +} diff --git a/dsymbol/src/dsymbol/import_.d b/dsymbol/src/dsymbol/import_.d new file mode 100644 index 0000000..29dc16b --- /dev/null +++ b/dsymbol/src/dsymbol/import_.d @@ -0,0 +1,32 @@ +/** + * This file is part of DCD, a development tool for the D programming language. + * Copyright (C) 2014 Brian Schott + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +module dsymbol.import_; + +/** + * Import information + */ +//struct ImportInformation +//{ +// /// module resolved path +// istring modulePath; +// /// symbols to import from this module +// UnrolledList!(Tuple!(istring, istring), false) importedSymbols; +// /// true if the import is public +// bool isPublic; +//} diff --git a/dsymbol/src/dsymbol/modulecache.d b/dsymbol/src/dsymbol/modulecache.d new file mode 100644 index 0000000..0985126 --- /dev/null +++ b/dsymbol/src/dsymbol/modulecache.d @@ -0,0 +1,539 @@ +/** + * This file is part of DCD, a development tool for the D programming language. + * Copyright (C) 2014 Brian Schott + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +module dsymbol.modulecache; + +import containers.dynamicarray; +import containers.hashset; +import containers.ttree; +import containers.unrolledlist; +import dsymbol.conversion; +import dsymbol.conversion.first; +import dsymbol.conversion.second; +import dsymbol.cache_entry; +import dsymbol.scope_; +import dsymbol.semantic; +import dsymbol.symbol; +import dsymbol.string_interning; +import dsymbol.deferred; +import std.algorithm; +import std.experimental.allocator; +import std.experimental.allocator.building_blocks.allocator_list; +import std.experimental.allocator.building_blocks.region; +import std.experimental.allocator.building_blocks.null_allocator; +import std.experimental.allocator.mallocator : Mallocator; +import std.experimental.allocator.gc_allocator : GCAllocator; +import std.conv; +import dparse.ast; +import std.datetime; +import dparse.lexer; +import dparse.parser; +import std.experimental.logger; +import std.file; +import std.experimental.lexer; +import std.path; + +alias ASTAllocator = AllocatorList!(n => Region!Mallocator(1024 * 128), Mallocator); + +/** + * Returns: true if a file exists at the given path. + */ +bool existanceCheck(A)(A path) +{ + if (path.exists()) + return true; + warning("Cannot cache modules in ", path, " because it does not exist"); + return false; +} + +alias DeferredSymbolsAllocator = GCAllocator; // NOTE using `Mallocator` here fails when analysing Phobos as `free(): invalid pointer` + +/** + * Caches pre-parsed module information. + */ +struct ModuleCache +{ + /// No copying. + @disable this(this); + + @disable this(); + + this(RCIAllocator symbolAllocator) + { + this.symbolAllocator = symbolAllocator; + } + + ~this() + { + clear(); + } + + /** + * Adds the given paths to the list of directories checked for imports. + * Performs duplicate checking, so multiple instances of the same path will + * not be present. + */ + void addImportPaths(const string[] paths) + { + import std.path : baseName; + import std.array : array; + + auto newPaths = paths + .map!(a => absolutePath(expandTilde(a))) + .filter!(a => existanceCheck(a) && !importPaths[].canFind!(b => b.path == a)) + .map!(a => ImportPath(istring(a))) + .array; + importPaths.insert(newPaths); + } + + /** + * Removes the given paths from the list of directories checked for + * imports. Corresponding cache entries are removed. + */ + void removeImportPaths(const string[] paths) + { + foreach (path; paths[]) + { + if (!importPaths[].canFind!(a => a.path == path)) + { + warning("Cannot remove ", path, " because it is not imported"); + continue; + } + + foreach (ref importPath; importPaths[].filter!(a => a.path == path)) + importPaths.remove(importPath); + + foreach (cacheEntry; cache[]) + { + if (cacheEntry.path.data.startsWith(path)) + { + foreach (deferredSymbol; deferredSymbols[].find!(d => d.symbol.symbolFile.data.startsWith(cacheEntry.path.data))) + { + deferredSymbols.remove(deferredSymbol); + DeferredSymbolsAllocator.instance.dispose(deferredSymbol); + } + + cache.remove(cacheEntry); + CacheAllocator.instance.dispose(cacheEntry); + } + } + } + } + + /** + * Clears the cache from all import paths + */ + void clear() + { + foreach (entry; cache[]) + CacheAllocator.instance.dispose(entry); + foreach (symbol; deferredSymbols[]) + DeferredSymbolsAllocator.instance.dispose(symbol); + + // TODO: This call to deallocateAll is a workaround for issues of + // CAllocatorImpl and GCAllocator not interacting well. + symbolAllocator.deallocateAll(); + cache.clear(); + deferredSymbols.clear(); + importPaths.clear(); + } + + /** + * Caches the module at the given location + */ + DSymbol* cacheModule(string location) + { + import std.stdio : File; + + assert (location !is null); + + const cachedLocation = istring(location); + + if (recursionGuard.contains(&cachedLocation.data[0])) + return null; + + if (!needsReparsing(cachedLocation)) + return getEntryFor(cachedLocation).symbol; + + recursionGuard.insert(&cachedLocation.data[0]); + + File f = File(cachedLocation); + immutable fileSize = cast(size_t) f.size; + if (fileSize == 0) + return null; + + const(Token)[] tokens; + auto parseStringCache = StringCache(fileSize.optimalBucketCount); + { + ubyte[] source = cast(ubyte[]) Mallocator.instance.allocate(fileSize); + scope (exit) Mallocator.instance.deallocate(source); + f.rawRead(source); + LexerConfig config; + config.fileName = cachedLocation; + + // The first three bytes are sliced off here if the file starts with a + // Unicode byte order mark. The lexer/parser don't handle them. + tokens = getTokensForParser( + (source.length >= 3 && source[0 .. 3] == "\xef\xbb\xbf"c) + ? source[3 .. $] : source, + config, &parseStringCache); + } + + CacheEntry* newEntry = CacheAllocator.instance.make!CacheEntry(); + + scope semanticAllocator = new ASTAllocator(); + import dparse.rollback_allocator:RollbackAllocator; + RollbackAllocator parseAllocator; + Module m = parseModuleSimple(tokens[], cachedLocation, &parseAllocator); + + assert (!symbolAllocator.isNull); + scope first = new FirstPass(m, cachedLocation, symbolAllocator, + semanticAllocator.allocatorObject, &this, newEntry); + first.run(); + + secondPass(first.rootSymbol, first.moduleScope, this); + + typeid(Scope).destroy(first.moduleScope); + symbolsAllocated += first.symbolsAllocated; + + SysTime access; + SysTime modification; + getTimes(cachedLocation.data, access, modification); + + newEntry.symbol = first.rootSymbol.acSymbol; + newEntry.modificationTime = modification; + newEntry.path = cachedLocation; + + CacheEntry* oldEntry = getEntryFor(cachedLocation); + if (oldEntry !is null) + { + // Generate update mapping from the old symbol to the new one + UpdatePairCollection updatePairs; + generateUpdatePairs(oldEntry.symbol, newEntry.symbol, updatePairs); + + // Apply updates to all symbols in modules that depend on this one + cache[].filter!(a => a.dependencies.contains(cachedLocation)).each!( + upstream => upstream.symbol.updateTypes(updatePairs)); + + // Remove the old symbol. + cache.remove(oldEntry, entry => CacheAllocator.instance.dispose(entry)); + } + + cache.insert(newEntry); + recursionGuard.remove(&cachedLocation.data[0]); + + resolveDeferredTypes(cachedLocation); + + typeid(SemanticSymbol).destroy(first.rootSymbol); + + return newEntry.symbol; + } + + /** + * Resolves types for deferred symbols + */ + void resolveDeferredTypes(istring location) + { + DeferredSymbols temp; + temp.insert(deferredSymbols[]); + deferredSymbols.clear(); + foreach (deferred; temp[]) + { + if (!deferred.imports.empty && !deferred.dependsOn(location)) + { + deferredSymbols.insert(deferred); + continue; + } + assert(deferred.symbol.type is null); + if (deferred.symbol.kind == CompletionKind.importSymbol) + { + resolveImport(deferred.symbol, deferred.typeLookups, this); + } + else if (!deferred.typeLookups.empty) + { + // TODO: Is .front the right thing to do here? + resolveTypeFromType(deferred.symbol, deferred.typeLookups.front, null, + this, &deferred.imports); + } + DeferredSymbolsAllocator.instance.dispose(deferred); + } + } + + /** + * Params: + * moduleName = the name of the module in "a/b/c" form + * Returns: + * The symbols defined in the given module, or null if the module is + * not cached yet. + */ + DSymbol* getModuleSymbol(istring location) + { + auto existing = getEntryFor(location); + return existing ? existing.symbol : cacheModule(location); + } + + /** + * Params: + * moduleName = the name of the module being imported, in "a/b/c" style + * Returns: + * The absolute path to the file that contains the module, or null if + * not found. + */ + istring resolveImportLocation(string moduleName) + { + assert(moduleName !is null, "module name is null"); + if (isRooted(moduleName)) + return istring(moduleName); + string alternative; + foreach (importPath; importPaths[]) + { + auto path = importPath.path; + // import path is a filename + // first check string if this is a feasable path (no filesystem usage) + if (path.stripExtension.endsWith(moduleName) + && path.existsAnd!isFile) + { + // prefer exact import names above .di/package.d files + return istring(path); + } + // no exact matches and no .di/package.d matches either + else if (!alternative.length) + { + string dotDi = buildPath(path, moduleName) ~ ".di"; + string dotD = dotDi[0 .. $ - 1]; + string withoutSuffix = dotDi[0 .. $ - 3]; + if (existsAnd!isFile(dotD)) + return istring(dotD); // return early for exactly matching .d files + else if (existsAnd!isFile(dotDi)) + alternative = dotDi; + else if (existsAnd!isDir(withoutSuffix)) + { + string packagePath = buildPath(withoutSuffix, "package.di"); + if (existsAnd!isFile(packagePath[0 .. $ - 1])) + alternative = packagePath[0 .. $ - 1]; + else if (existsAnd!isFile(packagePath)) + alternative = packagePath; + } + } + // we have a potential .di/package.d file but continue searching for + // exact .d file matches to use instead + else + { + string dotD = buildPath(path, moduleName) ~ ".d"; + if (existsAnd!isFile(dotD)) + return istring(dotD); // return early for exactly matching .d files + } + } + return alternative.length > 0 ? istring(alternative) : istring(null); + } + + auto getImportPaths() const + { + return importPaths[].map!(a => a.path); + } + + auto getAllSymbols() + { + scanAll(); + return cache[]; + } + + RCIAllocator symbolAllocator; + + alias DeferredSymbols = UnrolledList!(DeferredSymbol*, DeferredSymbolsAllocator); + DeferredSymbols deferredSymbols; + + /// Count of autocomplete symbols that have been allocated + uint symbolsAllocated; + +private: + + CacheEntry* getEntryFor(istring cachedLocation) + { + CacheEntry dummy; + dummy.path = cachedLocation; + auto r = cache.equalRange(&dummy); + return r.empty ? null : r.front; + } + + /** + * Params: + * mod = the path to the module + * Returns: + * true if the module needs to be reparsed, false otherwise + */ + bool needsReparsing(istring mod) + { + if (!exists(mod.data)) + return true; + CacheEntry e; + e.path = mod; + auto r = cache.equalRange(&e); + if (r.empty) + return true; + SysTime access; + SysTime modification; + getTimes(mod.data, access, modification); + return r.front.modificationTime != modification; + } + + void scanAll() + { + foreach (ref importPath; importPaths) + { + if (importPath.scanned) + continue; + scope(success) importPath.scanned = true; + + if (importPath.path.existsAnd!isFile) + { + if (importPath.path.baseName.startsWith(".#")) + continue; + cacheModule(importPath.path); + } + else + { + void scanFrom(const string root) + { + if (exists(buildPath(root, ".no-dcd"))) + return; + + try foreach (f; dirEntries(root, SpanMode.shallow)) + { + if (f.name.existsAnd!isFile) + { + if (!f.name.extension.among(".d", ".di") || f.name.baseName.startsWith(".#")) + continue; + cacheModule(f.name); + } + else scanFrom(f.name); + } + catch(FileException) {} + } + scanFrom(importPath.path); + } + } + } + + // Mapping of file paths to their cached symbols. + alias CacheAllocator = GCAllocator; // NOTE using `Mallocator` here fails when analysing Phobos as `Segmentation fault (core dumped)` + alias Cache = TTree!(CacheEntry*, CacheAllocator); + Cache cache; + + HashSet!(immutable(char)*) recursionGuard; + + struct ImportPath + { + string path; + bool scanned; + } + + // Listing of paths to check for imports + UnrolledList!ImportPath importPaths; +} + +/// Wrapper to check some attribute of a path, ignoring errors +/// (such as on a broken symlink). +private static bool existsAnd(alias fun)(string file) +{ + try + return fun(file); + catch (FileException e) + return false; +} + +/// same as getAttributes without throwing +/// Returns: true if exists, false otherwise +private static bool getFileAttributesFast(R)(R name, uint* attributes) +{ + version (Windows) + { + import std.internal.cstring : tempCStringW; + import core.sys.windows.winnt : INVALID_FILE_ATTRIBUTES; + import core.sys.windows.winbase : GetFileAttributesW; + + auto namez = tempCStringW(name); + static auto trustedGetFileAttributesW(const(wchar)* namez) @trusted + { + return GetFileAttributesW(namez); + } + *attributes = trustedGetFileAttributesW(namez); + return *attributes != INVALID_FILE_ATTRIBUTES; + } + else version (Posix) + { + import core.sys.posix.sys.stat : stat, stat_t; + import std.internal.cstring : tempCString; + + auto namez = tempCString(name); + static auto trustedStat(const(char)* namez, out stat_t statbuf) @trusted + { + return stat(namez, &statbuf); + } + + stat_t statbuf; + const ret = trustedStat(namez, statbuf) == 0; + *attributes = statbuf.st_mode; + return ret; + } + else + { + static assert(false, "Unimplemented getAttributes check"); + } +} + +private static bool existsAnd(alias fun : isFile)(string file) +{ + uint attributes; + if (!getFileAttributesFast(file, &attributes)) + return false; + return attrIsFile(attributes); +} + +private static bool existsAnd(alias fun : isDir)(string file) +{ + uint attributes; + if (!getFileAttributesFast(file, &attributes)) + return false; + return attrIsDir(attributes); +} + +version (Windows) +{ + unittest + { + assert(existsAnd!isFile(`C:\Windows\regedit.exe`)); + assert(existsAnd!isDir(`C:\Windows`)); + assert(!existsAnd!isDir(`C:\Windows\regedit.exe`)); + assert(!existsAnd!isDir(`C:\SomewhereNonExistant\nonexistant.exe`)); + assert(!existsAnd!isFile(`C:\SomewhereNonExistant\nonexistant.exe`)); + assert(!existsAnd!isFile(`C:\Windows`)); + } +} +else version (Posix) +{ + unittest + { + assert(existsAnd!isFile(`/bin/sh`)); + assert(existsAnd!isDir(`/bin`)); + assert(!existsAnd!isDir(`/bin/sh`)); + assert(!existsAnd!isDir(`/nonexistant_dir/__nonexistant`)); + assert(!existsAnd!isFile(`/nonexistant_dir/__nonexistant`)); + assert(!existsAnd!isFile(`/bin`)); + } +} diff --git a/dsymbol/src/dsymbol/scope_.d b/dsymbol/src/dsymbol/scope_.d new file mode 100644 index 0000000..798c3a9 --- /dev/null +++ b/dsymbol/src/dsymbol/scope_.d @@ -0,0 +1,257 @@ +/** + * This file is part of DCD, a development tool for the D programming language. + * Copyright (C) 2014 Brian Schott + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +module dsymbol.scope_; + +import dsymbol.symbol; +import dsymbol.import_; +import dsymbol.builtin.names; +import containers.ttree; +import containers.unrolledlist; +import std.algorithm : canFind, any; +import std.experimental.logger; +import std.experimental.allocator.gc_allocator : GCAllocator; + +/** + * Contains symbols and supports lookup of symbols by cursor position. + */ +struct Scope +{ + @disable this(this); + @disable this(); + + /** + * Params: + * begin = the beginning byte index + * end = the ending byte index + */ + this (uint begin, uint end) + { + this.startLocation = begin; + this.endLocation = end; + } + + ~this() + { + foreach (child; children[]) + typeid(Scope).destroy(child); + foreach (symbol; _symbols) + { + if (symbol.owned) + typeid(DSymbol).destroy(symbol.ptr); + } + } + + /** + * Params: + * cursorPosition = the cursor position in bytes + * Returns: + * the innermost scope that contains the given cursor position + */ + Scope* getScopeByCursor(size_t cursorPosition) return pure @nogc + { + if (cursorPosition < startLocation) return null; + if (cursorPosition > endLocation) return null; + foreach (child; children[]) + { + auto childScope = child.getScopeByCursor(cursorPosition); + if (childScope !is null) + return childScope; + } + return cast(typeof(return)) &this; + } + + /** + * Params: + * cursorPosition = the cursor position in bytes + * Returns: + * all symbols in the scope containing the cursor position, as well as + * the symbols in parent scopes of that scope. + */ + DSymbol*[] getSymbolsInCursorScope(size_t cursorPosition) + { + import std.array : array; + import std.algorithm.iteration : map; + + auto s = getScopeByCursor(cursorPosition); + if (s is null) + return null; + + UnrolledList!(DSymbol*) retVal; + Scope* sc = s; + while (sc !is null) + { + foreach (item; sc._symbols[]) + { + if (item.ptr.kind == CompletionKind.withSymbol) + { + if (item.ptr.type !is null) + foreach (i; item.ptr.type.opSlice()) + retVal.insert(i); + } + else if (item.ptr.type !is null && item.ptr.kind == CompletionKind.importSymbol) + { + if (item.ptr.qualifier != SymbolQualifier.selectiveImport) + { + foreach (i; item.ptr.type.opSlice()) + retVal.insert(i); + } + else + retVal.insert(item.ptr.type); + } + else + retVal.insert(item.ptr); + } + sc = sc.parent; + } + return array(retVal[]); + } + + /** + * Params: + * name = the symbol name to search for + * Returns: + * all symbols in this scope or parent scopes with the given name + */ + inout(DSymbol)*[] getSymbolsByName(istring name) inout + { + import std.array : array, appender; + import std.algorithm.iteration : map; + + DSymbol s = DSymbol(name); + auto er = _symbols.equalRange(SymbolOwnership(&s)); + if (!er.empty) + return cast(typeof(return)) array(er.map!(a => a.ptr)); + + // Check symbols from "with" statement + DSymbol ir2 = DSymbol(WITH_SYMBOL_NAME); + auto r2 = _symbols.equalRange(SymbolOwnership(&ir2)); + if (!r2.empty) + { + auto app = appender!(DSymbol*[])(); + foreach (e; r2) + { + if (e.type is null) + continue; + foreach (withSymbol; e.type.getPartsByName(s.name)) + app.put(cast(DSymbol*) withSymbol); + } + if (app.data.length > 0) + return cast(typeof(return)) app.data; + } + + if (name != CONSTRUCTOR_SYMBOL_NAME && + name != DESTRUCTOR_SYMBOL_NAME && + name != UNITTEST_SYMBOL_NAME && + name != THIS_SYMBOL_NAME) + { + // Check imported symbols + DSymbol ir = DSymbol(IMPORT_SYMBOL_NAME); + + auto app = appender!(DSymbol*[])(); + foreach (e; _symbols.equalRange(SymbolOwnership(&ir))) + { + if (e.type is null) + continue; + if (e.qualifier == SymbolQualifier.selectiveImport && e.type.name == name) + app.put(cast(DSymbol*) e.type); + else + foreach (importedSymbol; e.type.getPartsByName(s.name)) + app.put(cast(DSymbol*) importedSymbol); + } + if (app.data.length > 0) + return cast(typeof(return)) app.data; + } + if (parent is null) + return []; + return parent.getSymbolsByName(name); + } + + /** + * Params: + * name = the symbol name to search for + * cursorPosition = the cursor position in bytes + * Returns: + * all symbols with the given name in the scope containing the cursor + * and its parent scopes + */ + DSymbol*[] getSymbolsByNameAndCursor(istring name, size_t cursorPosition) + { + auto s = getScopeByCursor(cursorPosition); + if (s is null) + return []; + return s.getSymbolsByName(name); + } + + DSymbol* getFirstSymbolByNameAndCursor(istring name, size_t cursorPosition) + { + auto s = getSymbolsByNameAndCursor(name, cursorPosition); + return s.length > 0 ? s[0] : null; + } + + /** + * Returns an array of symbols that are present at global scope + */ + inout(DSymbol)*[] getSymbolsAtGlobalScope(istring name) inout + { + if (parent !is null) + return parent.getSymbolsAtGlobalScope(name); + return getSymbolsByName(name); + } + + bool hasSymbolRecursive(const(DSymbol)* symbol) const + { + return _symbols[].canFind!(a => a == symbol) || children[].any!(a => a.hasSymbolRecursive(symbol)); + } + + /// The scope that contains this one + Scope* parent; + + /// Child scopes + alias ChildrenAllocator = GCAllocator; // NOTE using `Mallocator` here fails when analysing Phobos + alias Children = UnrolledList!(Scope*, ChildrenAllocator); + Children children; + + /// Start location of this scope in bytes + uint startLocation; + + /// End location of this scope in bytes + uint endLocation; + + auto symbols() @property + { + return _symbols[]; + } + + /** + * Adds the given symbol to this scope. + * Params: + * symbol = the symbol to add + * owns = if true, the symbol's destructor will be called when this + * scope's destructor is called. + */ + void addSymbol(DSymbol* symbol, bool owns) + { + assert(symbol !is null); + _symbols.insert(SymbolOwnership(symbol, owns)); + } + +private: + /// Symbols contained in this scope + TTree!(SymbolOwnership, GCAllocator, true, "a.opCmp(b) < 0") _symbols; // NOTE using `Mallocator` here fails when analysing Phobos +} diff --git a/dsymbol/src/dsymbol/semantic.d b/dsymbol/src/dsymbol/semantic.d new file mode 100644 index 0000000..159fce7 --- /dev/null +++ b/dsymbol/src/dsymbol/semantic.d @@ -0,0 +1,156 @@ +/** + * This file is part of DCD, a development tool for the D programming language. + * Copyright (C) 2014 Brian Schott + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +module dsymbol.semantic; + +import dsymbol.symbol; +import dparse.ast; +import dparse.lexer; +import containers.unrolledlist; +import dsymbol.type_lookup; +import std.experimental.allocator.mallocator : Mallocator; +import std.experimental.allocator.gc_allocator : GCAllocator; + +enum ResolutionFlags : ubyte +{ + inheritance = 0b0000_0001, + type = 0b0000_0010, + mixinTemplates = 0b0000_0100, +} + +alias TypeLookupsAllocator = GCAllocator; // NOTE using `Mallocator` here fails when analysing Phobos as: `munmap_chunk(): invalid pointer` +alias TypeLookups = UnrolledList!(TypeLookup*, TypeLookupsAllocator); + +/** + * Intermediate form between DSymbol and the AST classes. Stores enough + * information to resolve things like base classes and alias this. + */ +struct SemanticSymbol +{ +public: + + /// Disable default construction. + @disable this(); + /// Disable copy construction + @disable this(this); + + /** + * Params: + * name = the name + */ + this(DSymbol* acSymbol) + { + this.acSymbol = acSymbol; + } + + ~this() + { + import std.experimental.allocator : dispose; + + foreach (child; children[]) + typeid(SemanticSymbol).destroy(child); + foreach (lookup; typeLookups[]) + TypeLookupsAllocator.instance.dispose(lookup); + } + + /** + * Adds a child to the children field and updates the acSymbol's parts field + */ + void addChild(SemanticSymbol* child, bool owns) + { + children.insert(child); + acSymbol.addChild(child.acSymbol, owns); + } + + /// Information used to do type resolution, inheritance, mixins, and alias this + TypeLookups typeLookups; + + /// Child symbols + UnrolledList!(SemanticSymbol*, GCAllocator) children; // NOTE using `Mallocator` here fails when analysing Phobos + + /// Autocompletion symbol + DSymbol* acSymbol; + + /// Parent symbol + SemanticSymbol* parent; + + /// Protection level for this symobol + deprecated("Use acSymbol.protection instead") ref inout(IdType) protection() @property inout + { + return acSymbol.protection; + } +} + +/** + * Type of the _argptr variable + */ +Type argptrType; + +/** + * Type of _arguments + */ +Type argumentsType; + +alias GlobalsAllocator = Mallocator; + +static this() +{ + import dsymbol.string_interning : internString; + import std.experimental.allocator : make; + + // TODO: Replace these with DSymbols + + // _argptr has type void* + argptrType = GlobalsAllocator.instance.make!Type(); + argptrType.type2 = GlobalsAllocator.instance.make!Type2(); + argptrType.type2.builtinType = tok!"void"; + TypeSuffix argptrTypeSuffix = GlobalsAllocator.instance.make!TypeSuffix(); + argptrTypeSuffix.star = Token(tok!"*"); + argptrType.typeSuffixes = cast(TypeSuffix[]) GlobalsAllocator.instance.allocate(TypeSuffix.sizeof); + argptrType.typeSuffixes[0] = argptrTypeSuffix; + + // _arguments has type TypeInfo[] + argumentsType = GlobalsAllocator.instance.make!Type(); + argumentsType.type2 = GlobalsAllocator.instance.make!Type2(); + argumentsType.type2.typeIdentifierPart = GlobalsAllocator.instance.make!TypeIdentifierPart(); + IdentifierOrTemplateInstance i = GlobalsAllocator.instance.make!IdentifierOrTemplateInstance(); + i.identifier.text = internString("TypeInfo"); + i.identifier.type = tok!"identifier"; + argumentsType.type2.typeIdentifierPart.identifierOrTemplateInstance = i; + TypeSuffix argumentsTypeSuffix = GlobalsAllocator.instance.make!TypeSuffix(); + argumentsTypeSuffix.array = true; + argumentsType.typeSuffixes = cast(TypeSuffix[]) GlobalsAllocator.instance.allocate(TypeSuffix.sizeof); + argumentsType.typeSuffixes[0] = argumentsTypeSuffix; +} + +static ~this() +{ + import std.experimental.allocator : dispose; + GlobalsAllocator.instance.dispose(argumentsType.typeSuffixes[0]); + GlobalsAllocator.instance.dispose(argumentsType.type2.typeIdentifierPart.identifierOrTemplateInstance); + GlobalsAllocator.instance.dispose(argumentsType.type2.typeIdentifierPart); + GlobalsAllocator.instance.dispose(argumentsType.type2); + GlobalsAllocator.instance.dispose(argptrType.typeSuffixes[0]); + GlobalsAllocator.instance.dispose(argptrType.type2); + + GlobalsAllocator.instance.deallocate(argumentsType.typeSuffixes); + GlobalsAllocator.instance.deallocate(argptrType.typeSuffixes); + + GlobalsAllocator.instance.dispose(argumentsType); + GlobalsAllocator.instance.dispose(argptrType); +} diff --git a/dsymbol/src/dsymbol/string_interning.d b/dsymbol/src/dsymbol/string_interning.d new file mode 100644 index 0000000..0f8039e --- /dev/null +++ b/dsymbol/src/dsymbol/string_interning.d @@ -0,0 +1,97 @@ +/** + * This file is part of DCD, a development tool for the D programming language. + * Copyright (C) 2014 Brian Schott + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +module dsymbol.string_interning; + +import std.traits : Unqual; +import dparse.lexer; + +/// Obsolete, use `istring` constructor instead +istring internString(string s) nothrow @nogc @safe +{ + return istring(s); +} + +static this() +{ + stringCache = StringCache(StringCache.defaultBucketCount); +} + +static ~this() +{ + destroy(stringCache); +} + +private StringCache stringCache = void; + +struct istring +{ +nothrow @nogc @safe: + /// Interns the given string and returns the interned version. Handles empty strings too. + this(string s) + { + if (s.length > 0) + _data = stringCache.intern(s); + } + +pure: + void opAssign(T)(T other) if (is(Unqual!T == istring)) + { + _data = other._data; + } + + bool opCast(To : bool)() const + { + return _data.length > 0; + } + + ptrdiff_t opCmpFast(const istring another) const @trusted + { + // Interned strings can be compared by the pointers. + // Identical strings MUST have the same address + return (cast(ptrdiff_t) _data.ptr) - (cast(ptrdiff_t) another._data.ptr); + } + ptrdiff_t opCmp(const string another) const + { + import std.algorithm.comparison : cmp; + // Compare as usual, because another string may come from somewhere else + return cmp(_data, another); + } + + bool opEquals(const istring another) const @trusted + { + return _data.ptr is another._data.ptr; + } + bool opEquals(const string another) const + { + return _data == another; + } + + size_t toHash() const @trusted + { + return (cast(size_t) _data.ptr) * 27_644_437; + } + + string data() const + { + return _data; + } + + alias data this; + private string _data; +} diff --git a/dsymbol/src/dsymbol/symbol.d b/dsymbol/src/dsymbol/symbol.d new file mode 100644 index 0000000..44263e1 --- /dev/null +++ b/dsymbol/src/dsymbol/symbol.d @@ -0,0 +1,492 @@ +/** + * This file is part of DCD, a development tool for the D programming language. + * Copyright (C) 2014 Brian Schott + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +module dsymbol.symbol; + +import std.array; + +import std.experimental.allocator.mallocator : Mallocator; +import std.experimental.allocator.gc_allocator : GCAllocator; +import containers.ttree; +import containers.unrolledlist; +import containers.slist; +import containers.hashset; +import dparse.lexer; +import std.bitmanip; + +import dsymbol.builtin.names; +public import dsymbol.string_interning; + +import std.range : isOutputRange; + +/** + * Identifies the kind of the item in an identifier completion list + */ +enum CompletionKind : char +{ + /// Invalid completion kind. This is used internally and will never + /// be returned in a completion response. + dummy = '?', + + /// Import symbol. This is used internally and will never + /// be returned in a completion response. + importSymbol = '*', + + /// With symbol. This is used internally and will never + /// be returned in a completion response. + withSymbol = 'w', + + /// class names + className = 'c', + + /// interface names + interfaceName = 'i', + + /// structure names + structName = 's', + + /// union name + unionName = 'u', + + /// variable name + variableName = 'v', + + /// member variable + memberVariableName = 'm', + + /// keyword, built-in version, scope statement + keyword = 'k', + + /// function or method + functionName = 'f', + + /// enum name + enumName = 'g', + + /// enum member + enumMember = 'e', + + /// package name + packageName = 'P', + + /// module name + moduleName = 'M', + + /// alias name + aliasName = 'l', + + /// template name + templateName = 't', + + /// mixin template name + mixinTemplateName = 'T', + + /// variadic template parameter + variadicTmpParam = 'p', + + /// type template parameter when no constraint + typeTmpParam = 'h', +} + +/** + * Returns: true if `kind` is something that can be returned to the client + */ +bool isPublicCompletionKind(CompletionKind kind) pure nothrow @safe @nogc +{ + return kind != CompletionKind.dummy && kind != CompletionKind.importSymbol + && kind != CompletionKind.withSymbol; +} + + +/** + * Any special information about a variable declaration symbol. + */ +enum SymbolQualifier : ubyte +{ + /// None + none, + /// The symbol is an array + array, + /// The symbol is a associative array + assocArray, + /// The symbol is a function or delegate pointer + func, + /// Selective import + selectiveImport, +} + +/** + * Autocompletion symbol + */ +struct DSymbol +{ + // Copying is disabled + @disable this(); + @disable this(this); + + /** + * Params: + * name = the symbol's name + * kind = the symbol's completion kind + * type = the resolved type of the symbol + */ + this(string name, CompletionKind kind = CompletionKind.dummy, DSymbol* type = null) nothrow @nogc @safe + { + this.name = istring(name); + this.kind = kind; + this.type = type; + } + /// ditto + this(istring name, CompletionKind kind = CompletionKind.dummy, DSymbol* type = null) nothrow @nogc @safe + { + this.name = name; + this.kind = kind; + this.type = type; + } + + ~this() + { + foreach (ref part; parts[]) + { + if (part.owned) + { + assert(part.ptr !is null); + typeid(DSymbol).destroy(part.ptr); + } + else + part.ptr = null; + } + if (ownType) + typeid(DSymbol).destroy(type); + } + + ptrdiff_t opCmp(ref const DSymbol other) const pure nothrow @nogc @safe + { + return name.opCmpFast(other.name); + } + + bool opEquals(ref const DSymbol other) const pure nothrow @nogc @safe + { + return name == other.name; + } + + size_t toHash() const pure nothrow @nogc @safe + { + return name.toHash(); + } + + /** + * Gets all parts whose name matches the given string. + */ + inout(DSymbol)*[] getPartsByName(istring name) inout + { + auto app = appender!(DSymbol*[])(); + HashSet!size_t visited; + getParts(name, app, visited); + return cast(typeof(return)) app.data; + } + + inout(DSymbol)* getFirstPartNamed(this This)(istring name) inout + { + auto app = appender!(DSymbol*[])(); + HashSet!size_t visited; + getParts(name, app, visited); + return app.data.length > 0 ? cast(typeof(return)) app.data[0] : null; + } + + /** + * Gets all parts and imported parts. Filters based on the part's name if + * the `name` argument is not null. Stores results in `app`. + */ + void getParts(OR)(istring name, ref OR app, ref HashSet!size_t visited, + bool onlyOne = false) inout + if (isOutputRange!(OR, DSymbol*)) + { + import std.algorithm.iteration : filter; + + if (&this is null) + return; + if (visited.contains(cast(size_t) &this)) + return; + visited.insert(cast(size_t) &this); + + if (name is null) + { + foreach (part; parts[].filter!(a => a.name != IMPORT_SYMBOL_NAME)) + { + app.put(cast(DSymbol*) part); + if (onlyOne) + return; + } + DSymbol p = DSymbol(IMPORT_SYMBOL_NAME); + foreach (im; parts.equalRange(SymbolOwnership(&p))) + { + if (im.type !is null && !im.skipOver) + { + if (im.qualifier == SymbolQualifier.selectiveImport) + { + app.put(cast(DSymbol*) im.type); + if (onlyOne) + return; + } + else + im.type.getParts(name, app, visited, onlyOne); + } + } + } + else + { + DSymbol s = DSymbol(name); + foreach (part; parts.equalRange(SymbolOwnership(&s))) + { + app.put(cast(DSymbol*) part); + if (onlyOne) + return; + } + if (name == CONSTRUCTOR_SYMBOL_NAME || + name == DESTRUCTOR_SYMBOL_NAME || + name == UNITTEST_SYMBOL_NAME || + name == THIS_SYMBOL_NAME) + return; // these symbols should not be imported + + DSymbol p = DSymbol(IMPORT_SYMBOL_NAME); + foreach (im; parts.equalRange(SymbolOwnership(&p))) + { + if (im.type !is null && !im.skipOver) + { + if (im.qualifier == SymbolQualifier.selectiveImport) + { + if (im.type.name == name) + { + app.put(cast(DSymbol*) im.type); + if (onlyOne) + return; + } + } + else + im.type.getParts(name, app, visited, onlyOne); + } + } + } + } + + /** + * Returns: a range over this symbol's parts and publicly visible imports + */ + inout(DSymbol)*[] opSlice(this This)() inout + { + auto app = appender!(DSymbol*[])(); + HashSet!size_t visited; + getParts!(typeof(app))(istring(null), app, visited); + return cast(typeof(return)) app.data; + } + + void addChild(DSymbol* symbol, bool owns) + { + assert(symbol !is null); + parts.insert(SymbolOwnership(symbol, owns)); + } + + void addChildren(R)(R symbols, bool owns) + { + foreach (symbol; symbols) + { + assert(symbol !is null); + parts.insert(SymbolOwnership(symbol, owns)); + } + } + + void addChildren(DSymbol*[] symbols, bool owns) + { + foreach (symbol; symbols) + { + assert(symbol !is null); + parts.insert(SymbolOwnership(symbol, owns)); + } + } + + /** + * Updates the type field based on the mappings contained in the given + * collection. + */ + void updateTypes(ref UpdatePairCollection collection) + { + auto r = collection.equalRange(UpdatePair(type, null)); + if (!r.empty) + type = r.front.newSymbol; + foreach (part; parts[]) + part.updateTypes(collection); + } + + /** + * Symbols that compose this symbol, such as enum members, class variables, + * methods, parameters, etc. + */ + alias PartsAllocator = GCAllocator; // NOTE using `Mallocator` here fails when analysing Phobos + alias Parts = TTree!(SymbolOwnership, PartsAllocator, true, "a < b"); + private Parts parts; + + /** + * DSymbol's name + */ + istring name; + + /** + * Calltip to display if this is a function + */ + istring callTip; + + /** + * Used for storing information for selective renamed imports + */ + alias altFile = callTip; + + /** + * Module containing the symbol. + */ + istring symbolFile; + + /** + * Documentation for the symbol. + */ + DocString doc; + + /** + * The symbol that represents the type. + */ + // TODO: assert that the type is not a function + DSymbol* type; + + /** + * Names of function arguments + */ + // TODO: remove since we have function arguments + UnrolledList!(istring) argNames; + + /** + * Function parameter symbols + */ + DSymbol*[] functionParameters; + + private uint _location; + + /** + * DSymbol location + */ + size_t location() const pure nothrow @nogc @property @safe + { + return _location; + } + + void location(size_t location) pure nothrow @nogc @property @safe + { + // If the symbol was declared in a file, assert that it has a location + // in that file. Built-in symbols don't need a location. + assert(symbolFile is null || location < uint.max); + _location = cast(uint) location; + } + + /** + * The kind of symbol + */ + CompletionKind kind; + + /** + * DSymbol qualifier + */ + SymbolQualifier qualifier; + + /** + * If true, this symbol owns its type and will free it on destruction + */ + // dfmt off + mixin(bitfields!(bool, "ownType", 1, + bool, "skipOver", 1, + bool, "isPointer", 1, + ubyte, "", 5)); + // dfmt on + + /// Protection level for this symbol + IdType protection; + +} + +/** + * istring with actual content and information if it was ditto + */ +struct DocString +{ + /// Creates a non-ditto comment. + this(istring content) + { + this.content = content; + } + + /// Creates a comment which may have been ditto, but has been resolved. + this(istring content, bool ditto) + { + this.content = content; + this.ditto = ditto; + } + + alias content this; + + /// Contains the documentation string associated with this symbol, resolves ditto to the previous comment with correct scope. + istring content; + /// `true` if the documentation was just a "ditto" comment copying from the previous comment. + bool ditto; +} + +struct UpdatePair +{ + ptrdiff_t opCmp(ref const UpdatePair other) const pure nothrow @nogc @safe + { + return (cast(ptrdiff_t) other.oldSymbol) - (cast(ptrdiff_t) this.oldSymbol); + } + + DSymbol* oldSymbol; + DSymbol* newSymbol; +} + +alias UpdatePairCollectionAllocator = Mallocator; +alias UpdatePairCollection = TTree!(UpdatePair, UpdatePairCollectionAllocator, false, "a < b"); + +void generateUpdatePairs(DSymbol* oldSymbol, DSymbol* newSymbol, ref UpdatePairCollection results) +{ + results.insert(UpdatePair(oldSymbol, newSymbol)); + foreach (part; oldSymbol.parts[]) + { + auto temp = DSymbol(oldSymbol.name); + auto r = newSymbol.parts.equalRange(SymbolOwnership(&temp)); + if (r.empty) + continue; + generateUpdatePairs(part, r.front, results); + } +} + +struct SymbolOwnership +{ + ptrdiff_t opCmp(ref const SymbolOwnership other) const @nogc + { + return this.ptr.opCmp(*other.ptr); + } + + DSymbol* ptr; + bool owned; + alias ptr this; +} diff --git a/dsymbol/src/dsymbol/tests.d b/dsymbol/src/dsymbol/tests.d new file mode 100644 index 0000000..4a56b69 --- /dev/null +++ b/dsymbol/src/dsymbol/tests.d @@ -0,0 +1,561 @@ +module dsymbol.tests; + +import std.experimental.allocator; +import dparse.ast, dparse.parser, dparse.lexer, dparse.rollback_allocator; +import dsymbol.cache_entry, dsymbol.modulecache, dsymbol.symbol; +import dsymbol.conversion, dsymbol.conversion.first, dsymbol.conversion.second; +import dsymbol.semantic, dsymbol.string_interning, dsymbol.builtin.names; +import std.file, std.path, std.format; +import std.stdio : writeln, stdout; + +/** + * Parses `source`, caches its symbols and compares the the cache content + * with the `results`. + * + * Params: + * source = The source code to test. + * results = An array of string array. Each slot represents the variable name + * followed by the type strings. + */ +version (unittest): +void expectSymbolsAndTypes(const string source, const string[][] results, + string file = __FILE_FULL_PATH__, size_t line = __LINE__) +{ + import core.exception : AssertError; + import std.exception : enforce; + + ModuleCache mcache = ModuleCache(theAllocator); + auto pair = generateAutocompleteTrees(source, mcache); + scope(exit) pair.destroy(); + + size_t i; + foreach(ss; (*pair.symbol)[]) + { + if (ss.type) + { + enforce!AssertError(i <= results.length, "not enough results", file, line); + enforce!AssertError(results[i].length > 1, + "at least one type must be present in a result row", file, line); + enforce!AssertError(ss.name == results[i][0], + "expected variableName: `%s` but got `%s`".format(results[i][0], ss.name), + file, line); + + auto t = cast() ss.type; + foreach (immutable j; 1..results[i].length) + { + enforce!AssertError(t != null, "null symbol", file, line); + enforce!AssertError(t.name == results[i][j], + "expected typeName: `%s` but got `%s`".format(results[i][j], t.name), + file, line); + if (t.type is t && t.name.length && t.name[0] != '*') + break; + t = t.type; + } + i++; + } + } + enforce!AssertError(i == results.length, "too many expected results, %s is left".format(results[i .. $]), file, line); +} + +@system unittest +{ + writeln("Running type deduction tests..."); + q{bool b; int i;}.expectSymbolsAndTypes([["b", "bool"],["i", "int"]]); + q{auto b = false;}.expectSymbolsAndTypes([["b", "bool"]]); + q{auto b = true;}.expectSymbolsAndTypes([["b", "bool"]]); + q{auto b = [0];}.expectSymbolsAndTypes([["b", "*arr-literal*", "int"]]); + q{auto b = [[0]];}.expectSymbolsAndTypes([["b", "*arr-literal*", "*arr-literal*", "int"]]); + q{auto b = [[[0]]];}.expectSymbolsAndTypes([["b", "*arr-literal*", "*arr-literal*", "*arr-literal*", "int"]]); + q{auto b = [];}.expectSymbolsAndTypes([["b", "*arr-literal*", "void"]]); + q{auto b = [[]];}.expectSymbolsAndTypes([["b", "*arr-literal*", "*arr-literal*", "void"]]); + //q{int* b;}.expectSymbolsAndTypes([["b", "*", "int"]]); + //q{int*[] b;}.expectSymbolsAndTypes([["b", "*arr*", "*", "int"]]); + + q{auto b = new class {int i;};}.expectSymbolsAndTypes([["b", "__anonclass1"]]); + + // got a crash before but solving is not yet working ("foo" instead of "__anonclass1"); + q{class Bar{} auto foo(){return new class Bar{};} auto b = foo();}.expectSymbolsAndTypes([["b", "foo"]]); +} + +// this one used to crash, see #125 +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + auto source = q{ auto a = true ? [42] : []; }; + auto pair = generateAutocompleteTrees(source, cache); +} + +// https://github.com/dlang-community/D-Scanner/issues/749 +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + auto source = q{ void test() { foo(new class A {});} }; + auto pair = generateAutocompleteTrees(source, cache); +} + +// https://github.com/dlang-community/D-Scanner/issues/738 +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + auto source = q{ void b() { c = } alias b this; }; + auto pair = generateAutocompleteTrees(source, cache); +} + +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + + writeln("Running function literal tests..."); + const sources = [ + q{ int a; auto dg = { }; }, + q{ void f() { int a; auto dg = { }; } }, + q{ auto f = (int a) { }; }, + q{ auto f() { return (int a) { }; } }, + q{ auto f() { return g((int a) { }); } }, + q{ void f() { g((int a) { }); } }, + q{ void f() { auto x = (int a) { }; } }, + q{ void f() { auto x = g((int a) { }); } }, + ]; + foreach (src; sources) + { + auto pair = generateAutocompleteTrees(src, cache); + auto a = pair.scope_.getFirstSymbolByNameAndCursor(istring("a"), 35); + assert(a, src); + assert(a.type, src); + assert(a.type.name == "int", src); + } +} + +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + + writeln("Running struct constructor tests..."); + auto source = q{ struct A {int a; struct B {bool b;} int c;} }; + auto pair = generateAutocompleteTrees(source, cache); + auto A = pair.symbol.getFirstPartNamed(internString("A")); + auto B = A.getFirstPartNamed(internString("B")); + auto ACtor = A.getFirstPartNamed(CONSTRUCTOR_SYMBOL_NAME); + auto BCtor = B.getFirstPartNamed(CONSTRUCTOR_SYMBOL_NAME); + assert(ACtor.callTip == "this(int a, int c)"); + assert(BCtor.callTip == "this(bool b)"); +} + +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + + writeln("Running union constructor tests..."); + auto source = q{ union A {int a; bool b;} }; + auto pair = generateAutocompleteTrees(source, cache); + auto A = pair.symbol.getFirstPartNamed(internString("A")); + auto ACtor = A.getFirstPartNamed(CONSTRUCTOR_SYMBOL_NAME); + assert(ACtor.callTip == "this(int a, bool b)"); +} + +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + writeln("Running non-importable symbols tests..."); + auto source = q{ + class A { this(int a){} } + class B : A {} + class C { A f; alias f this; } + }; + auto pair = generateAutocompleteTrees(source, cache); + auto A = pair.symbol.getFirstPartNamed(internString("A")); + auto B = pair.symbol.getFirstPartNamed(internString("B")); + auto C = pair.symbol.getFirstPartNamed(internString("C")); + assert(A.getFirstPartNamed(CONSTRUCTOR_SYMBOL_NAME) !is null); + assert(B.getFirstPartNamed(CONSTRUCTOR_SYMBOL_NAME) is null); + assert(C.getFirstPartNamed(CONSTRUCTOR_SYMBOL_NAME) is null); +} + +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + + writeln("Running alias this tests..."); + auto source = q{ struct A {int f;} struct B { A a; alias a this; void fun() { auto var = f; };} }; + auto pair = generateAutocompleteTrees(source, cache); + auto A = pair.symbol.getFirstPartNamed(internString("A")); + auto B = pair.symbol.getFirstPartNamed(internString("B")); + auto Af = A.getFirstPartNamed(internString("f")); + auto fun = B.getFirstPartNamed(internString("fun")); + auto var = fun.getFirstPartNamed(internString("var")); + assert(Af is pair.scope_.getFirstSymbolByNameAndCursor(internString("f"), var.location)); +} + +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + + writeln("Running anon struct tests..."); + auto source = q{ struct A { struct {int a;}} }; + auto pair = generateAutocompleteTrees(source, cache); + auto A = pair.symbol.getFirstPartNamed(internString("A")); + assert(A); + auto Aa = A.getFirstPartNamed(internString("a")); + assert(Aa); +} + +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + + writeln("Running anon class tests..."); + const sources = [ + q{ auto a = new class Object { int i; }; }, + q{ auto a = new class Object { int i; void m() { } }; }, + q{ auto a = g(new class Object { int i; }); }, + q{ auto a = g(new class Object { int i; void m() { } }); }, + q{ void f() { new class Object { int i; }; } }, + q{ void f() { new class Object { int i; void m() { } }; } }, + q{ void f() { g(new class Object { int i; }); } }, + q{ void f() { g(new class Object { int i; void m() { } }); } }, + q{ void f() { auto a = new class Object { int i; }; } }, + q{ void f() { auto a = new class Object { int i; void m() { } }; } }, + q{ void f() { auto a = g(new class Object { int i; }); } }, + q{ void f() { auto a = g(new class Object { int i; void m() { } }); } }, + ]; + foreach (src; sources) + { + auto pair = generateAutocompleteTrees(src, cache); + auto a = pair.scope_.getFirstSymbolByNameAndCursor(istring("i"), 60); + assert(a, src); + assert(a.type, src); + assert(a.type.name == "int", src); + } +} + +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + + writeln("Running the deduction from index expr tests..."); + { + auto source = q{struct S{} S[] s; auto b = s[i];}; + auto pair = generateAutocompleteTrees(source, cache); + DSymbol* S = pair.symbol.getFirstPartNamed(internString("S")); + DSymbol* b = pair.symbol.getFirstPartNamed(internString("b")); + assert(S); + assert(b.type is S); + } + { + auto source = q{struct S{} S[1] s; auto b = s[i];}; + auto pair = generateAutocompleteTrees(source, cache); + DSymbol* S = pair.symbol.getFirstPartNamed(internString("S")); + DSymbol* b = pair.symbol.getFirstPartNamed(internString("b")); + assert(S); + assert(b.type is S); + } + { + auto source = q{struct S{} S[][] s; auto b = s[0];}; + auto pair = generateAutocompleteTrees(source, cache); + DSymbol* S = pair.symbol.getFirstPartNamed(internString("S")); + DSymbol* b = pair.symbol.getFirstPartNamed(internString("b")); + assert(S); + assert(b.type.type is S); + } + { + auto source = q{struct S{} S[][][] s; auto b = s[0][0];}; + auto pair = generateAutocompleteTrees(source, cache); + DSymbol* S = pair.symbol.getFirstPartNamed(internString("S")); + DSymbol* b = pair.symbol.getFirstPartNamed(internString("b")); + assert(S); + assert(b.type.name == ARRAY_SYMBOL_NAME); + assert(b.type.type is S); + } + { + auto source = q{struct S{} S s; auto b = [s][0];}; + auto pair = generateAutocompleteTrees(source, cache); + DSymbol* S = pair.symbol.getFirstPartNamed(internString("S")); + DSymbol* b = pair.symbol.getFirstPartNamed(internString("b")); + assert(S); + assert(b.type is S); + } +} + +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + + writeln("Running `super` tests..."); + auto source = q{ class A {} class B : A {} }; + auto pair = generateAutocompleteTrees(source, cache); + assert(pair.symbol); + auto A = pair.symbol.getFirstPartNamed(internString("A")); + auto B = pair.symbol.getFirstPartNamed(internString("B")); + auto scopeA = (pair.scope_.getScopeByCursor(A.location + A.name.length)); + auto scopeB = (pair.scope_.getScopeByCursor(B.location + B.name.length)); + assert(scopeA !is scopeB); + + assert(!scopeA.getSymbolsByName(SUPER_SYMBOL_NAME).length); + assert(scopeB.getSymbolsByName(SUPER_SYMBOL_NAME)[0].type is A); +} + +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + + writeln("Running the \"access chain with inherited type\" tests..."); + auto source = q{ class A {} class B : A {} }; + auto pair = generateAutocompleteTrees(source, cache); + assert(pair.symbol); + auto A = pair.symbol.getFirstPartNamed(internString("A")); + assert(A); + auto B = pair.symbol.getFirstPartNamed(internString("B")); + assert(B); + auto AfromB = B.getFirstPartNamed(internString("A")); + assert(AfromB.kind == CompletionKind.aliasName); + assert(AfromB.type is A); +} + +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + + writeln("Running template type parameters tests..."); + { + auto source = q{ struct Foo(T : int){} struct Bar(T : Foo){} }; + auto pair = generateAutocompleteTrees(source, "", 0, cache); + DSymbol* T1 = pair.symbol.getFirstPartNamed(internString("Foo")); + DSymbol* T2 = T1.getFirstPartNamed(internString("T")); + assert(T2.type.name == "int"); + DSymbol* T3 = pair.symbol.getFirstPartNamed(internString("Bar")); + DSymbol* T4 = T3.getFirstPartNamed(internString("T")); + assert(T4.type); + assert(T4.type == T1); + } + { + auto source = q{ struct Foo(T){ }}; + auto pair = generateAutocompleteTrees(source, "", 0, cache); + DSymbol* T1 = pair.symbol.getFirstPartNamed(internString("Foo")); + assert(T1); + DSymbol* T2 = T1.getFirstPartNamed(internString("T")); + assert(T2); + assert(T2.kind == CompletionKind.typeTmpParam); + } +} + +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + + writeln("Running template variadic parameters tests..."); + auto source = q{ struct Foo(T...){ }}; + auto pair = generateAutocompleteTrees(source, "", 0, cache); + DSymbol* T1 = pair.symbol.getFirstPartNamed(internString("Foo")); + assert(T1); + DSymbol* T2 = T1.getFirstPartNamed(internString("T")); + assert(T2); + assert(T2.kind == CompletionKind.variadicTmpParam); +} + +unittest +{ + writeln("Running public import tests..."); + + const dir = buildPath(tempDir(), "dsymbol"); + const fnameA = buildPath(dir, "a.d"); + const fnameB = buildPath(dir, "b.d"); + const fnameC = buildPath(dir, "c.d"); + const fnameD = buildPath(dir, "d.d"); + const srcA = q{ int x; int w; }; + const srcB = q{ float y; private float z; }; + const srcC = q{ public { import a : x; import b; } import a : w; long t; }; + const srcD = q{ public import c; }; + // A simpler diagram: + // a = x w + // b = y [z] + // c = t + (x y) [w] + // d = (t x y) + + mkdir(dir); + write(fnameA, srcA); + write(fnameB, srcB); + write(fnameC, srcC); + write(fnameD, srcD); + scope (exit) + { + remove(fnameA); + remove(fnameB); + remove(fnameC); + remove(fnameD); + rmdir(dir); + } + + ModuleCache cache = ModuleCache(theAllocator); + cache.addImportPaths([dir]); + + const a = cache.getModuleSymbol(istring(fnameA)); + const b = cache.getModuleSymbol(istring(fnameB)); + const c = cache.getModuleSymbol(istring(fnameC)); + const d = cache.getModuleSymbol(istring(fnameD)); + const ax = a.getFirstPartNamed(istring("x")); + const aw = a.getFirstPartNamed(istring("w")); + assert(ax); + assert(aw); + assert(ax.type && ax.type.name == "int"); + assert(aw.type && aw.type.name == "int"); + const by = b.getFirstPartNamed(istring("y")); + const bz = b.getFirstPartNamed(istring("z")); + assert(by); + assert(bz); + assert(by.type && by.type.name == "float"); + assert(bz.type && bz.type.name == "float"); + const ct = c.getFirstPartNamed(istring("t")); + const cw = c.getFirstPartNamed(istring("w")); + const cx = c.getFirstPartNamed(istring("x")); + const cy = c.getFirstPartNamed(istring("y")); + const cz = c.getFirstPartNamed(istring("z")); + assert(ct); + assert(ct.type && ct.type.name == "long"); + assert(cw is null); // skipOver is true + assert(cx is ax); + assert(cy is by); + assert(cz is bz); // should not be there, but it is handled by DCD + const dt = d.getFirstPartNamed(istring("t")); + const dw = d.getFirstPartNamed(istring("w")); + const dx = d.getFirstPartNamed(istring("x")); + const dy = d.getFirstPartNamed(istring("y")); + const dz = d.getFirstPartNamed(istring("z")); + assert(dt is ct); + assert(dw is null); + assert(dx is cx); + assert(dy is cy); + assert(dz is cz); +} + +unittest +{ + ModuleCache cache = ModuleCache(theAllocator); + + writeln("Testing protection scopes"); + auto source = q{version(all) { private: } struct Foo{ }}; + auto pair = generateAutocompleteTrees(source, "", 0, cache); + DSymbol* T1 = pair.symbol.getFirstPartNamed(internString("Foo")); + assert(T1); + assert(T1.protection != tok!"private"); +} + +// check for memory leaks on thread termination (in static constructors) +version (linux) +unittest +{ + import core.memory : GC; + import core.thread : Thread; + import fs = std.file; + import std.array : split; + import std.conv : to; + + // get the resident set size + static long getRSS() + { + GC.collect(); + GC.minimize(); + // read Linux process statistics + const txt = fs.readText("/proc/self/stat"); + const parts = split(txt); + return to!long(parts[23]); + } + + const rssBefore = getRSS(); + // create and destroy a lot of dummy threads + foreach (j; 0 .. 50) + { + Thread[100] arr; + foreach (i; 0 .. 100) + arr[i] = new Thread({}).start(); + foreach (i; 0 .. 100) + arr[i].join(); + } + const rssAfter = getRSS(); + // check the process memory increase with some eyeballed threshold + assert(rssAfter - rssBefore < 5000); +} + +// this is for testing that internString data is always on the same address +// since we use this special property for modulecache recursion guard +unittest +{ + istring a = internString("foo_bar_baz".idup); + istring b = internString("foo_bar_baz".idup); + assert(a.data.ptr == b.data.ptr); +} + +private StringCache stringCache = void; +static this() +{ + stringCache = StringCache(StringCache.defaultBucketCount); +} +static ~this() +{ + destroy(stringCache); +} + +const(Token)[] lex(string source) +{ + return lex(source, null); +} + +const(Token)[] lex(string source, string filename) +{ + import dparse.lexer : getTokensForParser; + import std.string : representation; + LexerConfig config; + config.fileName = filename; + return getTokensForParser(source.dup.representation, config, &stringCache); +} + +unittest +{ + auto tokens = lex(q{int a = 9;}); + foreach(i, t; + cast(IdType[]) [tok!"int", tok!"identifier", tok!"=", tok!"intLiteral", tok!";"]) + { + assert(tokens[i] == t); + } + assert(tokens[1].text == "a", tokens[1].text); + assert(tokens[3].text == "9", tokens[3].text); +} + +string randomDFilename() +{ + import std.uuid : randomUUID; + return "dsymbol_" ~ randomUUID().toString() ~ ".d"; +} + +ScopeSymbolPair generateAutocompleteTrees(string source, ref ModuleCache cache) +{ + return generateAutocompleteTrees(source, randomDFilename, cache); +} + +ScopeSymbolPair generateAutocompleteTrees(string source, string filename, ref ModuleCache cache) +{ + auto tokens = lex(source); + RollbackAllocator rba; + Module m = parseModule(tokens, filename, &rba); + + scope first = new FirstPass(m, internString(filename), + theAllocator, theAllocator, &cache); + first.run(); + + secondPass(first.rootSymbol, first.moduleScope, cache); + auto r = first.rootSymbol.acSymbol; + typeid(SemanticSymbol).destroy(first.rootSymbol); + return ScopeSymbolPair(r, first.moduleScope); +} + +ScopeSymbolPair generateAutocompleteTrees(string source, size_t cursorPosition, ref ModuleCache cache) +{ + return generateAutocompleteTrees(source, null, cache); +} + +ScopeSymbolPair generateAutocompleteTrees(string source, string filename, size_t cursorPosition, ref ModuleCache cache) +{ + auto tokens = lex(source); + RollbackAllocator rba; + return dsymbol.conversion.generateAutocompleteTrees( + tokens, theAllocator, &rba, cursorPosition, cache); +} diff --git a/dsymbol/src/dsymbol/type_lookup.d b/dsymbol/src/dsymbol/type_lookup.d new file mode 100644 index 0000000..2260e57 --- /dev/null +++ b/dsymbol/src/dsymbol/type_lookup.d @@ -0,0 +1,40 @@ +module dsymbol.type_lookup; + +import dsymbol.string_interning; +import containers.unrolledlist; + +/** + * The type lookup kind. + */ +enum TypeLookupKind : ubyte +{ + inherit, + aliasThis, + initializer, + mixinTemplate, + varOrFunType, + selectiveImport, +} + +/** + * information used by the symbol resolver to determine types, inheritance, + * mixins, and alias this. + */ +struct TypeLookup +{ + this(TypeLookupKind kind) + { + this.kind = kind; + } + + this(istring name, TypeLookupKind kind) + { + breadcrumbs.insert(name); + this.kind = kind; + } + + /// Strings used to resolve the type + UnrolledList!istring breadcrumbs; + /// The kind of type lookup + TypeLookupKind kind; +} diff --git a/dsymbol/subprojects/dcontainers.wrap b/dsymbol/subprojects/dcontainers.wrap new file mode 100644 index 0000000..08d9249 --- /dev/null +++ b/dsymbol/subprojects/dcontainers.wrap @@ -0,0 +1,4 @@ +[wrap-git] +directory = dcontainers +url = https://github.com/dlang-community/containers.git +revision = head diff --git a/dsymbol/subprojects/dparse.wrap b/dsymbol/subprojects/dparse.wrap new file mode 100644 index 0000000..d629018 --- /dev/null +++ b/dsymbol/subprojects/dparse.wrap @@ -0,0 +1,4 @@ +[wrap-git] +directory = dparse +url = https://github.com/dlang-community/libdparse.git +revision = head diff --git a/dub.json b/dub.json index 65ba4eb..48a5517 100644 --- a/dub.json +++ b/dub.json @@ -7,12 +7,12 @@ ], "license": "GPL-3.0", "dependencies": { - "dsymbol": ">=0.14.0 <0.15.0", + ":dsymbol": "*", "libdparse": ">=0.20.0 <0.21.0", ":common": "*", - "emsi_containers": "~>0.8.0" + "emsi_containers": "~>0.9.0" }, - "subPackages": ["common"], + "subPackages": ["dsymbol", "common"], "versions": ["built_with_dub"], "configurations": [ { diff --git a/tests/run_tests.sh b/tests/run_tests.sh index ea1c0ca..f26f78c 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -6,6 +6,13 @@ NORMAL="\033[0m" IMPORTS=$(pwd)/imports export IMPORTS +if [ -z "${1:-}" ]; +then + TESTCASES="tc*" +else + TESTCASES="$1" +fi + fail_count=0 pass_count=0 client="../bin/dcd-client" @@ -51,7 +58,7 @@ for socket in unix tcp; do done # Run tests - for testCase in tc*; do + for testCase in $TESTCASES; do cd $testCase ./run.sh "$tcp"