diff --git a/ci.sh b/ci.sh
index 8146ef019f..9ca1c6687b 100755
--- a/ci.sh
+++ b/ci.sh
@@ -86,9 +86,9 @@ test() {
 test_dmd() {
     # test fewer compiler argument permutations for PRs to reduce CI load
     if [ "$FULL_BUILD" == "true" ] && [ "$OS_NAME" == "linux"  ]; then
-        make -j1 -C test start_all_tests MODEL=$MODEL N=$N # all ARGS by default
+        make -j1 -C test auto-tester-test MODEL=$MODEL N=$N # all ARGS by default
     else
-        make -j1 -C test start_all_tests MODEL=$MODEL N=$N ARGS="-O -inline -release"
+        make -j1 -C test auto-tester-test MODEL=$MODEL N=$N ARGS="-O -inline -release"
     fi
 }
 
diff --git a/posix.mak b/posix.mak
index 2c0f74fb05..77c273dec6 100644
--- a/posix.mak
+++ b/posix.mak
@@ -19,7 +19,14 @@ ifneq (,$(findstring Darwin_64_32, $(PWD)))
 auto-tester-test:
 	echo "Darwin_64_32_disabled"
 else
+ifneq (,$(findstring windows, $(PWD)))
 auto-tester-test: test
+else # POSIX
+# Like test, but without runnable_cxx
+auto-tester-test:
+	$(QUIET)$(MAKE) -C src -f posix.mak unittest
+	$(QUIET)$(MAKE) -C test -f Makefile
+endif
 endif
 
 buildkite-test: test
diff --git a/test/Makefile b/test/Makefile
index cea615d104..c7ea261e3b 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -208,6 +208,10 @@ start_all_tests: $(RUNNER)
 	@echo "Running all tests"
 	$(EXECUTE_RUNNER) all
 
+# The auto-tester cannot run runnable_cxx as its compiler is too old
+auto-tester-test: $(RUNNER)
+	$(EXECUTE_RUNNER) runnable compilable fail_compilation dshell
+
 $(RESULTS_DIR)/d_do_test$(EXE): tools/d_do_test.d tools/sanitize_json.d $(RESULTS_DIR)/.created
 	@echo "Building d_do_test tool"
 	@echo "OS: '$(OS)'"