Azure CI: Get rid of additional host DMD requirement for LDC jobs

This requires to build run.d (incl. its tools) and druntime (which also
involves building a little tool - with the host compiler - to copy the
imports...) before tampering with the LIB and DFLAGS environment
variables (for MinGW).

.azure-pipelines/windows-msbuild.bat

@@ -16,21 +16,17 @@ set N=3
 set DM_MAKE=%DMD_DIR%\dm\path\make.exe
 set LDC_DIR=%DMD_DIR%\ldc2-%LDC_VERSION%-windows-multilib
 
+if "%D_COMPILER%" == "ldc" set HOST_DMD=%LDC_DIR%\bin\ldmd2.exe
+if "%D_COMPILER%" == "dmd" set HOST_DMD=%DMD_DIR%\dmd2\windows\bin\dmd.exe
+
 REM take the first found cl.exe, in case there was already one in the path when vcvarsall.bat was called
-FOR /F "tokens=* USEBACKQ" %%F IN (`where cl.exe`) DO (SET MSVC_CC=%%~fsF && goto CC_DONE)
+FOR /F "tokens=* USEBACKQ" %%F IN (`where cl.exe`) DO (SET MSVC_CC=%%~fsF
+  goto CC_DONE)
 :CC_DONE
 FOR /F "tokens=* USEBACKQ" %%F IN (`where lib.exe`) DO (SET MSVC_AR=%%~fsF)
-REM this returns two lines (GNU's link.exe is on the second line)
-REM Just take the first one
-FOR /F "tokens=* USEBACKQ" %%F IN (`where link.exe`) DO (SET MSVC_LD=%%~fsF
-  GOTO :Next)
-:Next
-FOR /F "tokens=* USEBACKQ" %%F IN (`where make.exe`) DO (SET GNU_MAKE=%%~fsF)
 
-REM expose dm_make as default make
+REM add grep to PATH
-set PATH=%DMD_DIR%\dm\path;%DMD_DIR%\tools;%PATH%
+set PATH=%DMD_DIR%\tools;%PATH%
-dir "%DMD_DIR%\tools"
-"%DMD_DIR%\tools\grep.exe" --version
 echo %PATH%
 grep --version
 
@@ -40,13 +36,25 @@ if "%D_COMPILER%" == "dmd" reg add "HKLM\SOFTWARE\DMD" /v InstallationFolder /t
 REM configure LDC path
 if "%D_COMPILER%" == "ldc" reg add "HKLM\SOFTWARE\LDC" /v InstallationFolder /t REG_SZ /d "%LDC_DIR%" /reg:32 /f
 
-REM build via VS projects with LDC
+echo [STEP]: Building DMD via VS projects
 cd src
 if "%D_COMPILER%" == "ldc" set LDC_ARGS=%LDC_ARGS% /p:DCompiler=LDC
 msbuild /target:dmd /p:Configuration=%CONFIGURATION% /p:Platform=%PLATFORM% %LDC_ARGS% vcbuild\dmd.sln || exit /B 1
-
 %DMD% --version
-grep --version
+
+echo [STEP]: Building druntime
+cd "%DMD_DIR%\..\druntime"
+"%DM_MAKE%" -f win64.mak MODEL=%MODEL% "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" "HOST_DMD=%HOST_DMD%" target || exit /B 2
+
+echo [STEP]: Building phobos
+cd "%DMD_DIR%\..\phobos"
+"%DM_MAKE%" -f win64.mak MODEL=%MODEL% "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "AR=%MSVC_AR%" "MAKE=%DM_MAKE%" || exit /B 3
+
+echo [STEP]: Building run.d testrunner and its tools
+REM needs to be done before tampering with LIB and DFLAGS env variables (affecting the ldmd2 host compiler too)
+cd "%DMD_DIR%\test"
+"%HOST_DMD%" -m%MODEL% -g -i run.d || exit /B 4
+run.exe tools "BUILD=%CONFIGURATION%" "DMD_MODEL=%PLATFORM%" || exit /B 4
 
 set DMD_TESTS=all
 set DRUNTIME_TESTS=test_all
@@ -66,50 +74,29 @@ if not "%C_RUNTIME%" == "mingw" goto not_mingw
     set DFLAGS=-mscrtlib=msvcrt120
     set LIB=%DMD_DIR%\mingw\dmd2\windows\lib%MODEL%\mingw
     set REQUIRED_ARGS=-mscrtlib=msvcrt120 "-L/LIBPATH:%DMD_DIR%\mingw\dmd2\windows\lib%MODEL%\mingw"
-    rem delete C++ ABI tests
+    rem skip runnable_cxx tests (incompatible MSVC runtime versions - 2017 (cl.exe) vs. 2013)
-    set DMD_TESTS=runnable compilable fail_compilation dshell
+    set DMD_TESTS=runnable compilable fail_compilation dshell unit_tests
     rem FIXME: debug info incomplete when linking through lld-link
     del test\runnable\testpdb.d
 
     set DRUNTIME_TESTS=test_mingw
 :not_mingw
 
-REM Check: run druntime unittests
+echo [STEP]: Building and running druntime tests
 cd "%DMD_DIR%\..\druntime"
-"%DM_MAKE%" -f win64.mak MODEL=%MODEL% "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" target || exit /B 2
+"%DM_MAKE%" -f win64.mak MODEL=%MODEL% "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" unittest %DRUNTIME_TESTS% || exit /B 5
-echo "[DRUNTIME] running tests..."
-"%DM_MAKE%" -f win64.mak MODEL=%MODEL% "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" unittest || exit /B 3
-"%DM_MAKE%" -f win64.mak MODEL=%MODEL% "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" %DRUNTIME_TESTS% || exit /B 4
 
-REM Check: build phobos
+echo [STEP]: Running DMD testsuite
-cd "%DMD_DIR%\..\phobos"
-"%DM_MAKE%" -f win64.mak MODEL=%MODEL% "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "AR=%MSVC_AR%" "MAKE=%DM_MAKE%" || exit /B 5
-
-REM Run DMD testsuite
 cd "%DMD_DIR%\test"
-set ARGS=-O -inline -g
+set CC=%MSVC_CC%
-set OS=windows
+run.exe --environment --jobs=%N% %DMD_TESTS% "ARGS=-O -inline -g" "BUILD=%CONFIGURATION%" "DMD_MODEL=%PLATFORM%" || exit /B 6
-set CC=cl.exe
-set DMD_MODEL=%PLATFORM%
-set BUILD=%CONFIGURATION%
-if "%BUILD%" == "RelWithAsserts" (
-    set BUILD=Release
-    mkdir %DMD_DIR%\generated\Windows\Release\%PLATFORM%
-    xcopy %DMD_DIR%\generated\Windows\RelWithAsserts\%PLATFORM% %DMD_DIR%\generated\Windows\Release\%PLATFORM% /s
-)
-set HOST_DMD=%DMD_DIR%\dmd2\windows\bin\dmd.exe
-del phobos%MODEL%.lib
-
-"%HOST_DMD%" -m%MODEL% -i run.d || exit /B 6
-run.exe %DMD_TESTS% || exit /B 6
 
+echo [STEP]: Building and running Phobos unittests
 rem FIXME: lld-link fails to link phobos unittests ("error: relocation against symbol in discarded section: __TMP2427")
 if "%C_RUNTIME%" == "mingw" exit /B 0
-
 cd "%DMD_DIR%\..\phobos"
-REM Check: build phobos unittests
 if "%D_COMPILER%_%MODEL%" == "ldc_64" copy %LDC_DIR%\lib64\libcurl.dll .
 if "%D_COMPILER%_%MODEL%" == "ldc_32mscoff" copy %LDC_DIR%\lib32\libcurl.dll .
 if "%D_COMPILER%_%MODEL%" == "dmd_64" copy %DMD_DIR%\dmd2\windows\bin64\libcurl.dll .
 if "%D_COMPILER%_%MODEL%" == "dmd_32mscoff" copy %DMD_DIR%\dmd2\windows\bin\libcurl.dll .
-"%DM_MAKE%" -f win64.mak unittest MODEL=%MODEL% "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%"
+"%DM_MAKE%" -f win64.mak MODEL=%MODEL% "DMD=%DMD%" "VCDIR=%VCINSTALLDIR%." "CC=%MSVC_CC%" "MAKE=%DM_MAKE%" unittest || exit /B 7

.azure-pipelines/windows-visual-studio.sh

@@ -53,11 +53,12 @@ dm_make() {
     cp "$DMC" "$DM_MAKE" "dm/path"
 }
 
-echo "[STEP]: Downloading DMD"
-install_host_dmd
 if [ "$D_COMPILER" == "ldc" ]; then
     echo "[STEP]: Downloading LDC"
     ldc
+else
+    echo "[STEP]: Downloading DMD"
+    install_host_dmd
 fi
 
 echo "[STEP]: Downloading VisualD"

azure-pipelines.yml

@@ -49,7 +49,6 @@ jobs:
     pool:
       vmImage: 'vs2017-win2016'
     variables:
-      HOST_DMD_VERSION: 2.090.0
       D_COMPILER: ldc
       VISUALD_VER: v0.49.0
       LDC_VERSION: 1.20.0

test/run.d

@@ -143,6 +143,13 @@ Options:
         return spawnProcess(unitTestRunnerCommand ~ args).wait();
     }
 
+    if (args == ["tools"])
+    {
+        verifyCompilerExists(env);
+        ensureToolsExists(env, EnumMembers!TestTools);
+        return 0;
+    }
+
     // default target
     if (!args.length)
         args = ["all"];

test/tools/d_do_test.d

@@ -803,20 +803,13 @@ bool collectExtraSources (in string input_dir, in string output_dir, in string[]
         auto curSrc = input_dir ~ envData.sep ~"extra-files" ~ envData.sep ~ cur;
         auto curObj = output_dir ~ envData.sep ~ cur ~ envData.obj;
         string command = quoteSpaces(compiler);
-        if (envData.compiler == "dmd")
+        if (envData.usingMicrosoftCompiler)
         {
-            if (envData.usingMicrosoftCompiler)
+            command ~= ` /c /nologo `~curSrc~` /Fo`~curObj;
-            {
+        }
-                command ~= ` /c /nologo `~curSrc~` /Fo`~curObj;
+        else if (envData.compiler == "dmd" && envData.os == "windows" && envData.model == "32")
-            }
+        {
-            else if (envData.os == "windows" && envData.model == "32")
+            command ~= " -c "~curSrc~" -o"~curObj;
-            {
-                command ~= " -c "~curSrc~" -o"~curObj;
-            }
-            else
-            {
-                command ~= " -m"~envData.model~" -c "~curSrc~" -o "~curObj;
-            }
         }
         else
         {