Merge remote-tracking branch 'upstream/stable' into merge_stable

.github/workflows/pre_commit.yml

@@ -31,3 +31,19 @@ jobs:
         git config --global diff.wsErrorHighlight "all"
     - uses: actions/setup-python@v3.0.0
     - uses: pre-commit/action@v3.0.0
+    - name: Check changelog entries
+      run: |
+        check_prefix="$(find changelog -type f -name '*\.dd' -a ! -name 'dmd\.*' -a ! -name 'druntime\.*')"
+        if [ ! -z "${check_prefix}" ]; then
+          echo 'All changelog entries must begin with either `dmd.` or `druntime.`'
+          echo 'Found:'
+          echo "${check_prefix}"
+          exit 1
+        fi
+        check_ext="$(find changelog -type f ! -name 'README\.md' -a ! -name '*\.dd')"
+        if [ ! -z "${check_ext}" ]; then
+          echo 'All changelog entries must end with `.dd`'
+          echo 'Found:'
+          echo "${check_ext}"
+          exit 1
+        fi

VERSION

@@ -1 +1 @@
-v2.101.0-beta.1
+v2.101.0-rc.1

changelog/README.md

@@ -4,9 +4,10 @@ merged into stable prior to a new release.
 How to add a new changelog entry to the pending changelog?
 ==========================================================
 
-Create a new file in the `changelog` folder. It should end with `.dd` and look
-similar to a git commit message. The first line represents the title of the change.
-After an empty line follows the long description:
+Create a new file in the `changelog` folder. It should begin with either `dmd.`
+or `druntime.` and end with `.dd`. The contents of the entry should look
+similar to a git commit message. The first line represents the title of the
+change.  After an empty line follows the long description:
 
 ```
 My fancy title of the new feature

changelog/dmd.unicode-directionality.dd

@@ -0,0 +1,10 @@
+Source files may no longer contain Unicode directionality overrides
+
+[Trojan Source: Invisible Vulnerabilities](https://github.com/nickboucher/trojan-source) shows how they can be used to maliciously hide code in various programming languages.
+[D is also affected](https://github.com/nickboucher/trojan-source/pull/16).
+
+To prevent this, the compiler now raises an error when they appear in source files.
+If you need a string literal containing directionality overrides, use escape sequences instead:
+---
+string s = "\u202E\u2066";
+---

changelog/dmd.windows_stack_limit.dd

@@ -0,0 +1,6 @@
+Windows: Double DMD stack limit to 16 MB
+
+The compilation of some projects require more stack space than previously available.
+Now DMD has the same limit as LDC, which
+$(LINK2 https://github.com/ldc-developers/ldc/pull/3921, increased its stack limit)
+in its 1.29.0 release.

compiler/src/build.d

@@ -472,7 +472,7 @@ alias dmdExe = makeRuleWithArgs!((MethodInitializer!BuildRule builder, BuildRule
 
     string[] platformArgs;
     version (Windows)
-        platformArgs = ["-L/STACK:8388608"];
+        platformArgs = ["-L/STACK:16777216"];
 
     auto lexer = lexer(targetSuffix, depFlags);
     auto backend = backend(targetSuffix, depFlags);

compiler/src/dmd/blockexit.d

@@ -162,14 +162,7 @@ int blockExit(Statement s, FuncDeclaration func, bool mustNotThrow)
                     {
                         if (blockExit(s, func, mustNotThrow) != BE.halt && s.hasCode() &&
                             s.loc != Loc.initial) // don't emit warning for generated code
-                        {
-                            auto parent1 = func.toParent();
-                            if (parent1 && parent1.isTemplateInstance())
-                                s.warning("statement is not reachable in template instance %s", func.toPrettyChars());
-                            else
-                                s.warning("statement is not reachable");
-                        }
-
+                            s.warning("statement is not reachable");
                     }
                     else
                     {
@@ -554,7 +547,7 @@ BE checkThrow(ref const Loc loc, Expression exp, const bool mustNotThrow)
     ClassDeclaration cd = t.isClassHandle();
     assert(cd);
 
-    if (cd == ClassDeclaration.errorException || ClassDeclaration.errorException.isBaseOf(cd, null))
+    if (cd.isErrorException())
     {
         return BE.errthrow;
     }

compiler/src/dmd/dclass.d

@@ -991,6 +991,11 @@ extern (C++) class ClassDeclaration : AggregateDeclaration
         return vtblsym;
     }
 
+    extern (D) final bool isErrorException()
+    {
+        return errorException && (this == errorException || errorException.isBaseOf(this, null));
+    }
+
     override final inout(ClassDeclaration) isClassDeclaration() inout @nogc nothrow pure @safe
     {
         return this;

compiler/src/dmd/dinterpret.d

@@ -1521,11 +1521,6 @@ public:
         result = e;
     }
 
-    static bool isAnErrorException(ClassDeclaration cd)
-    {
-        return cd == ClassDeclaration.errorException || ClassDeclaration.errorException.isBaseOf(cd, null);
-    }
-
     static ThrownExceptionExp chainExceptions(ThrownExceptionExp oldest, ThrownExceptionExp newest)
     {
         debug (LOG)
@@ -1537,7 +1532,7 @@ public:
         const next = 4;                         // index of Throwable.next
         assert((*boss.value.elements)[next].type.ty == Tclass); // Throwable.next
         ClassReferenceExp collateral = newest.thrown;
-        if (isAnErrorException(collateral.originalClass()) && !isAnErrorException(boss.originalClass()))
+        if (collateral.originalClass().isErrorException() && !boss.originalClass().isErrorException())
         {
             /* Find the index of the Error.bypassException field
              */
@@ -2872,6 +2867,12 @@ public:
                         else
                             m = v.getConstInitializer(true);
                     }
+                    else if (v.type.isTypeNoreturn())
+                    {
+                        // Noreturn field with default initializer
+                        (*elems)[fieldsSoFar + i] = null;
+                        continue;
+                    }
                     else
                         m = v.type.defaultInitLiteral(e.loc);
                     if (exceptionOrCant(m))

compiler/src/dmd/lexer.d

@@ -126,18 +126,26 @@ class Lexer
         if (p && p[0] == '#' && p[1] == '!')
         {
             p += 2;
-            while (1)
+            for (;;p++)
             {
-                char c = *p++;
+                char c = *p;
                 switch (c)
                 {
+                case '\n':
+                    p++;
+                    goto case;
                 case 0:
                 case 0x1A:
-                    p--;
-                    goto case;
-                case '\n':
                     break;
+
                 default:
+                    // Note: We do allow malformed UTF-8 on shebang line.
+                    // It could have a meaning if the native system
+                    // encoding is not Unicode. See test compilable/test13512.d
+                    // for example encoded in KOI-8.
+                    // We also allow bidirectional control characters.
+                    // We do not execute the shebang line, so it can't be used
+                    // to conceal code. It is up to the shell to sanitize it.
                     continue;
                 }
                 break;
@@ -2829,6 +2837,20 @@ class Lexer
      * Return decoded character, advance p to last character in UTF sequence.
      */
     private uint decodeUTF()
+    {
+        string msg;
+        auto result = decodeUTFpure(msg);
+
+        if (msg)
+            error("%.*s", cast(int)msg.length, msg.ptr);
+        return result;
+    }
+
+    /********************************************
+     * Same as above, but the potential error message is stored to the
+     * msg parameter instead of being issued.
+     */
+    private pure uint decodeUTFpure(out string msg)
     {
         const s = p;
         assert(*s & 0x80);
@@ -2839,12 +2861,10 @@ class Lexer
         }
         size_t idx = 0;
         dchar u;
-        const msg = utf_decodeChar(s[0 .. len], idx, u);
+        msg = utf_decodeChar(s[0 .. len], idx, u);
         p += idx - 1;
-        if (msg)
-        {
-            error("%.*s", cast(int)msg.length, msg.ptr);
-        }
+        if (!msg && isBidiControl(u))
+            msg = "Bidirectional control characters are disallowed for security reasons.";
         return u;
     }
 

compiler/src/dmd/root/utf.d

@@ -395,6 +395,26 @@ void utf_encode(int sz, void* s, dchar c)
     }
 }
 
+/********************************************
+ * Checks whether an Unicode code point is a bidirectional
+ * control character.
+ */
+@safe bool isBidiControl(dchar c)
+{
+    // Source: https://www.unicode.org/versions/Unicode15.0.0, table 23-3.
+    switch(c)
+    {
+        case '\u061C':
+        case '\u200E':
+        case '\u200F':
+        case '\u202A': .. case '\u202E':
+        case '\u2066': .. case '\u2069':
+            return true;
+        default:
+            return false;
+    }
+}
+
 /********************************************
  * Decode a UTF-8 sequence as a single UTF-32 code point.
  * Params:

compiler/test/compilable/test23431.d

@@ -0,0 +1,14 @@
+// https://issues.dlang.org/show_bug.cgi?id=23431
+// REQUIRED_ARGS: -lowmem
+void test23431()
+{
+    int a;
+    try
+    {
+        throw new Exception("test1");
+        a++;
+    }
+    finally
+    {
+    }
+}

compiler/test/compilable/test23431_minimal.d

@@ -0,0 +1,28 @@
+// https://issues.dlang.org/show_bug.cgi?id=23431
+// REQUIRED_ARGS: -lowmem
+module object;
+
+alias string  = immutable(char)[];
+class Throwable { }
+class Exception : Throwable
+{
+    this(string )
+    {
+    }
+}
+
+class Error { }
+
+void test23431()
+{
+    int a;
+
+    try
+    {
+        throw new Exception("test1");
+        a++;
+    }
+    finally
+    {
+    }
+}

compiler/test/compilable/test23433.d

@@ -0,0 +1,16 @@
+// https://issues.dlang.org/show_bug.cgi?id=23433
+module object;
+
+class Throwable { }
+class Exception : Throwable { this(immutable(char)[]) { } }
+
+void test23433()
+{
+    try
+    {
+        throw new Exception("ice");
+    }
+    finally
+    {
+    }
+}

compiler/test/compilable/test23439.d

@@ -0,0 +1,8 @@
+// https://issues.dlang.org/show_bug.cgi?id=23439
+// PERMUTE_ARGS: -lowmem
+class C23439
+{
+    noreturn f23439;
+}
+
+__gshared ice23439 = new C23439();

compiler/test/fail_compilation/fail23439.d

@@ -0,0 +1,13 @@
+// https://issues.dlang.org/show_bug.cgi?id=23439
+// PERMUTE_ARGS: -lowmem
+/* TEST_OUTPUT:
+---
+fail_compilation/fail23439.d(13): Error: variable `fail23439.ice23439` is a thread-local class and cannot have a static initializer. Use `static this()` to initialize instead.
+---
+*/
+class C23439
+{
+    noreturn f23439;
+}
+
+static ice23439 = new C23439();

compiler/test/fail_compilation/warn14905.d

@@ -1,23 +0,0 @@
-// REQUIRED_ARGS: -o- -w
-
-/*
-TEST_OUTPUT:
----
-fail_compilation/warn14905.d(16): Warning: statement is not reachable in template instance warn14905.fun!"a".fun
-fail_compilation/warn14905.d(16): Warning: statement is not reachable in template instance warn14905.fun!"b".fun
-Error: warnings are treated as errors
-       Use -wi if you wish to treat warnings only as informational.
----
-*/
-
-bool fun(string s)()
-{
-    return true;
-    return false;
-}
-
-void main()
-{
-    cast(void)fun!"a";
-    cast(void)fun!"b";
-}

compiler/test/unit/lexer/lexer_dmdlib.d

@@ -252,3 +252,59 @@ unittest
     assert(result == expected);
     assert(lexer.empty);
 }
+
+// Issue 22495
+unittest
+{
+    import std.conv : text, to;
+    import std.string : fromStringz;
+
+    import core.stdc.stdarg : va_list;
+
+    import dmd.frontend;
+    import dmd.globals : Loc;
+    import dmd.common.outbuffer;
+    import dmd.console : Color;
+
+    const(char)[][2][] diagnosticMessages;
+    nothrow bool diagnosticHandler(const ref Loc loc, Color headerColor, const(char)* header,
+                                   const(char)* format, va_list ap, const(char)* p1, const(char)* p2)
+    {
+        OutBuffer tmp;
+        tmp.vprintf(format, ap);
+        diagnosticMessages ~= [loc.filename.fromStringz, to!string(tmp.peekChars())];
+        return true;
+    }
+
+    initDMD(&diagnosticHandler);
+    scope(exit) deinitializeDMD();
+
+    immutable codes = [
+        "enum myString = \"\u061C\";",
+        "enum myString = `\u202E\u2066 \u2069\u2066`;",
+        "void test(){} // \u200E comment \u200F"
+    ];
+
+    foreach (codeNum, code; codes)
+    {
+        auto fileName = text("file", codeNum, '\0');
+        Lexer lexer = new Lexer(fileName.ptr, code.ptr, 0, code.length, false, false);
+        // Generate the errors
+        foreach(unused; lexer){}
+    }
+
+    string bidiErrorMessage =
+        "Bidirectional control characters are disallowed for security reasons.";
+
+    string[2][] excepted = [
+        ["file0", bidiErrorMessage],
+        ["file1", bidiErrorMessage],
+        ["file1", bidiErrorMessage],
+        ["file1", bidiErrorMessage],
+        ["file1", bidiErrorMessage],
+        ["file2", bidiErrorMessage],
+        ["file2", bidiErrorMessage],
+    ];
+
+    assert(diagnosticMessages == excepted);
+}