code cleanup

createTable.sql

@@ -1,4 +1,4 @@
-create table files (path, mtime, id);
+create table modules (path, mtime, id);
 create table publicImports (importerId, importedId);
-create table containers (name, protection, fileId, id);
+create table containers (name, protection, moduleId, id);
 create table symbols (name, type, kind, containerId, id);

main.d

@@ -110,9 +110,9 @@ int main(string[] args)
 
 	try
 	{
-		getopt(args, "I", &importDirs,/+ "dotComplete", &dotComplete,+/ "sloc", &sloc,
+		getopt(args, "I", &importDirs,/+ "dotComplete|d", &dotComplete,+/ "sloc|l", &sloc,
-			/+"json", &json,+/ /+"parenComplete", &parenComplete,+/ "highlight", &highlight,
+			/+"json|j", &json,+/ /+"parenComplete|p", &parenComplete,+/ "highlight", &highlight,
-			"ctags", &ctags, "recursive|r|R", &recursiveCtags, "help|h", &help,
+			"ctags|c", &ctags, "recursive|r|R", &recursiveCtags, "help|h", &help,
 			"tokenCount", &tokenCount, "frequencyCount", &frequencyCount);
 	}
 	catch (Exception e)
@@ -120,27 +120,30 @@ int main(string[] args)
 		stderr.writeln(e.msg);
 	}
 
-	if (help || (!sloc && /+!dotComplete &&+/ /+!json &&+/ /+!parenComplete &&+/ !highlight
+    if (help)
-		&& !ctags && !format && !tokenCount && !frequencyCount))
     {
-		printHelp();
+        printHelp(args[0]);
         return 0;
     }
 
-	importDirs ~= loadConfig();
+    auto optionCount = count!"a"([sloc, highlight, ctags, json, tokenCount]);
-
+    if (optionCount > 1)
-	if (tokenCount)
     {
-		import core.memory;
+        stderr.writeln("Too many options specified");
-		/+if (args.length == 1)
+        return 1;
-		{
-			writeln((cast (ubyte[]) stdin.byLine(KeepTerminator.yes).join()).byToken().walkLength());
     }
-		else
+    else if (optionCount < 1)
-		{+/
+    {
+        printHelp(args[0]);
+        return 1;
+    }
+
+	if (tokenCount || sloc)
+	{
 		LexerConfig config;
         config.tokenStyle = TokenStyle.doNotReplaceSpecial;
-			foreach (arg; args[1..$])
+		ulong[] counts = new ulong[args.length - 1];
+		foreach (i, arg; parallel(args[1..$]))
 		{
 			config.fileName = arg;
 			uint count;
@@ -148,160 +151,65 @@ int main(string[] args)
             ubyte[] buffer = uninitializedArray!(ubyte[])(f.size);
 			foreach (t; byToken(f.rawRead(buffer), config))
             {
-					++count;
+                if (tokenCount)
+                    ++counts[i];
+                else if (isLineOfCode(t.type))
+                    ++counts[i];
             }
-				writefln("%s: %d", arg, count);
-			}
-		/+}+/
-	}
-    /+else if (frequencyCount)
-    {
-        uint[TokenType] frequency;
-        foreach (t; byToken(cast(ubyte[]) File(args[1]).byLine(KeepTerminator.yes).join()))
-        {
-            frequency[t.type]++;
-        }
-        foreach (k, v; frequency)
-        {
-            writeln(v, ":", cast(TokenType) k);
-        }
-    }+/
 
-	/+if (sloc)
-	{
-		if (args.length == 1)
-		{
-			writeln(stdin.byLine(KeepTerminator.yes).join().byToken().count!(a => isLineOfCode(a.type))());
 		}
-		else
+		foreach(i; 0 .. counts.length)
-		{
+			writefln("%s: %d", args[i + 1], counts[i]);
-			writeln(args[1..$].map!(a => File(a).byLine(KeepTerminator.yes).join().byToken(a))()
-                .joiner().count!(a => isLineOfCode(a.type))());
 	}
-		return 0;
+    else if (highlight)
-	}+/
-
-	if (highlight)
 	{
 		LexerConfig config;
 		config.iterStyle = IterationStyle.everything;
 		config.tokenStyle = TokenStyle.source;
         File f = args.length == 1 ? stdin : File(args[1]);
-        highlighter.highlight((cast(ubyte[]) f.byLine(KeepTerminator.yes).join()).byToken(config),
+        ubyte[] buffer = uninitializedArray!(ubyte[])(f.size);
+        highlighter.highlight(byToken(f.rawRead(buffer), config),
 			args.length == 1 ? "stdin" : args[1]);
 		return 0;
 	}
 
-	/+if (dotComplete || parenComplete)
-	{
-		if (isAbsolute(args[1]))
-			importDirs ~= dirName(args[1]);
-		else
-			importDirs ~= getcwd();
-		Token[] tokens;
-		try
-		{
-			to!size_t(args[1]);
-			auto f = appender!string();
-			char[] buf;
-			while (stdin.readln(buf))
-				f.put(buf);
-			tokens = f.data.byToken().array();
-		}
-		catch(ConvException e)
-		{
-			tokens = args[1].readText().byToken().array();
-			args.popFront();
-		}
-		auto mod = parseModule(tokens);
-		CompletionContext context = new CompletionContext(mod);
-		context.importDirectories = importDirs;
-		foreach (im; parallel(mod.imports))
-		{
-			auto p = findAbsPath(importDirs, im);
-			if (p is null || !p.exists())
-				continue;
-			context.addModule(p.readText().byToken().array().parseModule());
-		}
-		auto complete = AutoComplete(tokens, context);
-		if (parenComplete)
-			writeln(complete.parenComplete(to!size_t(args[1])));
-		else if (dotComplete)
-			writeln(complete.dotComplete(to!size_t(args[1])));
-		return 0;
-	}+/
-
-	/+if (json)
-	{
-		CircularBuffer!(Token) tokens;
-        File f = args.length == 1 ? stdin : File(args[1]);
-        tokens = new CircularBuffer!(Token)(CIRC_BUFF_SIZE,
-            f.byLine(KeepTerminator.yes).join().byToken!(char[])());
-		auto mod = parseModule(tokens);
-		mod.writeJSONTo(stdout);
-		return 0;
-	}+/
-
-//	if (ctags)
-//	{
-//		if (!recursiveCtags)
-//		{
-//			auto tokens = byToken(readText(args[1]));
-//			auto mod = parseModule(tokens.array());
-//			mod.writeCtagsTo(stdout, args[1]);
-//		}
-//		else
-//		{
-//			Module m;
-//			foreach (dirEntry; dirEntries(args[1], SpanMode.breadth))
-//			{
-//				if (!dirEntry.name.endsWith(".d", ".di"))
-//					continue;
-//				stderr.writeln("Generating tags for ", dirEntry.name);
-//				auto tokens = byToken(readText(dirEntry.name));
-//				if (m is null)
-//					m = parseModule(tokens.array());
-//				else
-//				{
-//					auto mod = parseModule(tokens.array());
-//					m.merge(mod);
-//				}
-//			}
-//			m.writeCtagsTo(stdout, "");
-//		}
-//	}
 	return 0;
 }
 
-void printHelp()
+void printHelp(string programName)
 {
-	writeln(
+	writefln(
-q{
+`
-    Usage: dscanner options
+    Usage: %s options
 
 options:
     --help | -h
         Prints this help message
 
-    --sloc [sourceFiles]
+    --sloc | -l [sourceFiles]
         count the number of logical lines of code in the given
         source files. If no files are specified, a file is read from stdin.
 
-    --json [sourceFile]
+    --json | -j [sourceFile]
         Generate a JSON summary of the given source file. If no file is
         specifed, the file is read from stdin.
 
-    --dotComplete [sourceFile] cursorPosition
+    --dotComplete | -d [sourceFile] cursorPosition
         Provide autocompletion for the insertion of the dot operator. The cursor
         position is the character position in the *file*, not the position in
         the line. If no file is specified, the file is read from stdin.
 
-    --parenComplete [sourceFile] cursorPosition
+    --parenComplete | -p [sourceFile] cursorPosition
-        Provides a listing of function parameters or pre-defined version
+        Provide a listing of function parameters or pre-defined version
         identifiers at the cursor position. The cursor position is the character
         position in the *file*, not the line. If no file is specified, the
         contents are read from stdin.
 
+    --symbolComplete | -s [sourceFile] cursorPosition
+        Provide a listing of classes, structs, interfaces, variables, functions,
+        and methods available in the current scope that begin with the text
+        before the cursor position.
+
     --highlight [sourceFile] - Syntax-highlight the given source file. The
         resulting HTML will be written to standard output.
 
@@ -311,12 +219,13 @@ options:
         well as any paths specified in /etc/dmd.conf. This is only used for the
         --parenComplete and --dotComplete options.
 
-    --ctags sourceFile
+    --ctags | -c sourceFile
         Generates ctags information from the given source code file. Note that
         ctags information requires a filename, so stdin cannot be used in place
         of a filename.
 
     --recursive | -R | -r directory
         When used with --ctags, dscanner will produce ctags output for all .d
-        and .di files contained within directory and its sub-directories.});
+        and .di files contained within directory and its sub-directories.`,
+        programName);
 }

std/d/lexer.d

@@ -1100,10 +1100,8 @@ private:
 			foreach (b; std.range.chunks(src.slice[2 .. $ - 1], 2))
 			{
 				auto s = cast(char[])b;
-				dchar ch = cast(dchar)parse!uint(s, 16);
+				ubyte ch = cast(ubyte)parse!uint(s, 16);
-				char[4] utf8Buf;
+				a.put(ch);
-				size_t utf8Size = encode(utf8Buf, ch);
-				a.put(utf8Buf[0..utf8Size]);
 			}
 			// can safely assume ownership of data
 			current.value = cast(string)a.data;
@@ -1915,18 +1913,19 @@ private:
 	}
 	body
 	{
-		static size_t reencodeNumeric(ubyte[] src, int radix, OutputRange dest)
+		size_t reencodeNumeric(ubyte[] src, int radix, OutputRange dest)
 		{
-			/*scope(failure) //TODO: get rid of std.stdio in lexer
-			{
-				import std.stdio;
-				("Failed on line ", lineNumber, " of file ",
-					config.fileName);
-			}*/
 			char[] chunk = cast(char[])src;
 			char[4] utfBuf;
 			uint codepoint = parse!uint(chunk, radix);
-			size_t len = encode(utfBuf, codepoint);
+			size_t len;
+			try
+				 len = encode(utfBuf, codepoint);
+			catch (UTFException ex)
+			{
+				errorMessage(ex.msg);
+				return 0;
+			}
 			dest.put(cast(ubyte[]) utfBuf[0..len]);
 			return len;
 		}