Merge branch 'master' of https://github.com/Hackerpilot/Dscanner

README.md

@@ -19,7 +19,7 @@ The following examples assume that we are analyzing a simple file called hellowo
 ### Token Count
 The "--tokenCount" or "-t" option prints the number of tokens in the given file
 
-	$ dscanner --tokencount helloworld.d
+	$ dscanner --tokenCount helloworld.d
 	20
 
 ### Import Listing
@@ -198,6 +198,6 @@ but not yet implemented.
 
 # Useful code
 The source code for DScanner has a complete lexer, parser, and abstact syntax
-tree library for D code under the stdx/d/ directory. It is intended that these
+tree library for D code under the std/d/ directory. It is intended that these
 modules eventually end up in Phobos, so feel free to use them for your own D
 tools.

astprinter.d

@@ -1314,8 +1314,9 @@ class XMLPrinter : ASTVisitor
 		case tok!"stringLiteral": tagName = "stringLiteral"; break;
 		case tok!"dstringLiteral": tagName = "dstringLiteral"; break;
 		case tok!"wstringLiteral": tagName = "wstringLiteral"; break;
+		case tok!"scriptLine": tagName = "scriptLine"; break;
 		case tok!"$": output.writeln("<dollar/>"); return;
-		default: output.writeln("<", str(token.type), "/>"); return;
+		default: tagName = "token"; break;
 		}
 		output.writeln("<", tagName, ">", xmlEscape(token.text), "</", tagName, ">");
 	}

build.sh

@@ -24,8 +24,8 @@ dmd\
 #	astprinter.d\
 #	formatter.d\
 #	outliner.d\
-#	stdx/*.d\
+#	std/*.d\
-#	stdx/d/*.d\
+#	std/d/*.d\
 #	analysis/*.d\
 #	-O3 -frelease -fno-bounds-check\
 #	-odscanner\
@@ -39,8 +39,8 @@ dmd\
 #	astprinter.d\
 #	formatter.d\
 #	outliner.d\
-#	stdx/*.d\
+#	std/*.d\
-#	stdx/d/*.d\
+#	std/d/*.d\
 #	analysis/*.d\
 #	-O3 -release\
 #	-oq -of=dscanner\

ctags.d

@@ -90,6 +90,26 @@ class CTagsPrinter : ASTVisitor
 		context = c;
 	}
 
+	override void visit(const Constructor dec)
+	{
+		tagLines ~= "this\t%s\t%d;\"\tf\tarity:%d%s\n".format(fileName,
+			dec.line, dec.parameters.parameters.length, context);
+		auto c = context;
+		context = "\tfunction: this";
+		dec.accept(this);
+		context = c;
+	}
+
+	override void visit(const Destructor dec)
+	{
+		tagLines ~= "~this\t%s\t%d;\"\tf%s\n".format(fileName, dec.line,
+			context);
+		auto c = context;
+		context = "\tfunction: this";
+		dec.accept(this);
+		context = c;
+	}
+
 	override void visit(const EnumDeclaration dec)
 	{
 		if (dec.name == tok!"")
@@ -136,6 +156,11 @@ class CTagsPrinter : ASTVisitor
 		dec.accept(this);
 	}
 
+	override void visit(const Invariant dec)
+	{
+		tagLines ~= "invariant\t%s\t%d;\"\tv%s\n".format(fileName, dec.line, context);
+	}
+
 	alias visit = ASTVisitor.visit;
 
 	string fileName;

dub.json

@@ -9,7 +9,7 @@
         {
             "name": "library",
             "targetType": "library",
-            "sourcePaths": ["stdx"],
+            "sourcePaths": ["std"],
         },
         {
             "name": "dscanner",

highlighter.d

@@ -49,6 +49,8 @@ html  { background-color: #fdf6e3; color: #002b36; }
 			writeSpan("num", t.text);
 		else if (isOperator(t.type))
 			writeSpan("op", str(t.type));
+		else if (t.type == tok!"specialTokenSequence" || t.type == tok!"scriptLine")
+			writeSpan("cons", t.text.replace("<", "&lt;"));
 		else
 		{
 			version(Windows)

main.d

@@ -101,6 +101,7 @@ int main(string[] args)
 		config.whitespaceBehavior = WhitespaceBehavior.include;
 		config.stringBehavior = StringBehavior.source;
 		config.commentBehavior = CommentBehavior.include;
+		config.specialTokenBehavior = SpecialTokenBehavior.include;
 		auto tokens = byToken(bytes, config, cache);
 		if (highlight)
 		{

std/allocator.d

@@ -2860,7 +2860,7 @@ private struct BasicRegion(uint minAlign = platformAlignment)
         static if (minAlign > 1)
         {
             auto newStore = cast(void*) roundUpToMultipleOf(
-                cast(ulong) store.ptr,
+                cast(size_t) store.ptr,
                 alignment);
             assert(newStore <= store.ptr + store.length);
             _current = newStore;
@@ -2904,7 +2904,7 @@ private struct BasicRegion(uint minAlign = platformAlignment)
         // Just bump the pointer to the next good allocation
         auto save = _current;
         _current = cast(void*) roundUpToMultipleOf(
-            cast(ulong) _current, a);
+            cast(size_t) _current, a);
         if (auto b = allocate(bytes)) return b;
         // Failed, rollback
         _current = save;
@@ -3064,7 +3064,7 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment)
     {
         assert(!_crt);
         _crt = cast(void*) roundUpToMultipleOf(
-            cast(ulong) _store.ptr, alignment);
+            cast(size_t) _store.ptr, alignment);
         _end = _store.ptr + _store.length;
     }
 
@@ -3109,7 +3109,7 @@ struct InSituRegion(size_t size, size_t minAlign = platformAlignment)
         // Just bump the pointer to the next good allocation
         auto save = _crt;
         _crt = cast(void*) roundUpToMultipleOf(
-            cast(ulong) _crt, a);
+            cast(size_t) _crt, a);
         if (auto b = allocate(bytes)) return b;
         // Failed, rollback
         _crt = save;

std/d/ast.d

@@ -1239,7 +1239,9 @@ public:
     }
     /** */ MemberFunctionAttribute[] memberFunctionAttributes;
     /** */ FunctionBody functionBody;
-    /** */ size_t location;
+    /** */ size_t line;
+    /** */ size_t column;
+    /** */ size_t index;
     /** */ string comment;
     mixin OpEquals;
 }
@@ -1780,6 +1782,8 @@ public:
     }
     /** */ BlockStatement blockStatement;
     /** */ string comment;
+    size_t line;
+    size_t index;
     mixin OpEquals;
 }
 
@@ -1952,8 +1956,9 @@ final class Module : ASTNode
 public:
     override void accept(ASTVisitor visitor) const
     {
-        mixin (visitIfNotNull!(moduleDeclaration, declarations));
+        mixin (visitIfNotNull!(scriptLine, moduleDeclaration, declarations));
     }
+	/** */ Token scriptLine;
     /** */ ModuleDeclaration moduleDeclaration;
     /** */ Declaration[] declarations;
     mixin OpEquals;

std/d/lexer.d

@@ -41,7 +41,7 @@ private enum dynamicTokens = [
 	"whitespace", "doubleLiteral", "floatLiteral", "idoubleLiteral",
 	"ifloatLiteral", "intLiteral", "longLiteral", "realLiteral",
 	"irealLiteral", "uintLiteral", "ulongLiteral", "characterLiteral",
-	"dstringLiteral", "stringLiteral", "wstringLiteral", "scriptLine"
+	"dstringLiteral", "stringLiteral", "wstringLiteral"
 ];
 
 private enum pseudoTokenHandlers = [
@@ -119,6 +119,18 @@ public enum WhitespaceBehavior : ubyte
 	/// Whitespace is treated as a token
 	include
 }
+
+/**
+ * Configure special token handling behavior
+ */
+public enum SpecialTokenBehavior : ubyte
+{
+	/// Special tokens are skipped
+	skip,
+	/// Special tokens are treated as a token
+	include
+}
+
 /**
  * Configure comment handling behavior
  */
@@ -136,6 +148,7 @@ public struct LexerConfig
 	StringBehavior stringBehavior;
 	WhitespaceBehavior whitespaceBehavior;
 	CommentBehavior commentBehavior;
+	SpecialTokenBehavior specialTokenBehavior;
 }
 
 public bool isBasicType(IdType type) nothrow pure @safe
@@ -434,6 +447,7 @@ public struct DLexer
 					}
 					do _popFront(); while (front == tok!"comment");
 					if (front == tok!"whitespace") goto case tok!"whitespace";
+					if (front == tok!"specialTokenSequence") goto case tok!"specialTokenSequence";
 				}
 				break;
 			case tok!"whitespace":
@@ -441,6 +455,15 @@ public struct DLexer
 				{
 					do _popFront(); while (front == tok!"whitespace");
 					if (front == tok!"comment") goto case tok!"comment";
+					if (front == tok!"specialTokenSequence") goto case tok!"specialTokenSequence";
+				}
+				break;
+			case tok!"specialTokenSequence":
+				if (config.specialTokenBehavior == SpecialTokenBehavior.skip)
+				{
+					do _popFront(); while (front == tok!"specialTokenSequence");
+					if (front == tok!"comment") goto case tok!"comment";
+					if (front == tok!"whitespace") goto case tok!"whitespace";
 				}
 				break;
 			default:

std/d/parser.d

@@ -1976,6 +1976,9 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         node.comment = comment;
         comment = null;
         if (expect(tok!"~") is null) return null;
+        node.index = current.index;
+        node.line = current.line;
+        node.column = current.column;
         if (expect(tok!"this") is null) return null;
         if (expect(tok!"(") is null) return null;
         if (expect(tok!")") is null) return null;
@@ -2281,7 +2284,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
         {
             if (!canBeRange)
             {
-                error(`Cannot have more than one foreach varible for a foreach range statement`);
+                error(`Cannot have more than one foreach variable for a foreach range statement`);
                 return null;
             }
             advance();
@@ -3184,6 +3187,8 @@ interface "Four"
     Invariant parseInvariant()
     {
         auto node = allocate!Invariant;
+        node.index = current.index;
+        node.line = current.line;
         if (expect(tok!"invariant") is null) return null;
         if (currentIs(tok!"("))
         {
@@ -3556,7 +3561,7 @@ invariant() foo();
         mixin(traceEnterAndExit!(__FUNCTION__));
         Module m = allocate!Module;
         if (currentIs(tok!"scriptLine"))
-            advance();
+            m.scriptLine = advance();
         if (currentIs(tok!"module"))
             m.moduleDeclaration = parseModuleDeclaration();
         while (moreTokens())

std/lexer.d

@@ -33,7 +33,7 @@
  * )
  * Examples:
  * $(UL
- * $(LI A _lexer for D is available $(LINK2 https://github.com/Hackerpilot/Dscanner/blob/master/stdx/d/lexer.d, here).)
+ * $(LI A _lexer for D is available $(LINK2 https://github.com/Hackerpilot/Dscanner/blob/master/std/d/lexer.d, here).)
  * $(LI A _lexer for Lua is available $(LINK2 https://github.com/Hackerpilot/lexer-demo/blob/master/lualexer.d, here).)
  * $(LI A _lexer for JSON is available $(LINK2 https://github.com/Hackerpilot/lexer-demo/blob/master/jsonlexer.d, here).)
  * )