diff --git a/test/issue103-single-file-package-json.d b/test/issue103-single-file-package-json.d
index 978593b..f2479a9 100644
--- a/test/issue103-single-file-package-json.d
+++ b/test/issue103-single-file-package-json.d
@@ -1,5 +1,5 @@
 /+ dub.json: {
-   "name": "hello_world"
+   "name": "single-file-test"
 } +/
 module hello;
 
diff --git a/test/issue103-single-file-package-w-dep.d b/test/issue103-single-file-package-w-dep.d
index 54f4d28..5f66241 100644
--- a/test/issue103-single-file-package-w-dep.d
+++ b/test/issue103-single-file-package-w-dep.d
@@ -1,5 +1,5 @@
 /+ dub.sdl:
-name "hello_world"
+name "single-file-test"
 dependency "sourcelib-simple" path="1-sourceLib-simple"
 +/
 module hello;
diff --git a/test/issue103-single-file-package.d b/test/issue103-single-file-package.d
index b8c8494..8c76638 100755
--- a/test/issue103-single-file-package.d
+++ b/test/issue103-single-file-package.d
@@ -1,6 +1,6 @@
 #!../bin/dub
 /+ dub.sdl:
-   name "hello_world"
+   name "single-file-test"
 +/
 module hello;
 
diff --git a/test/issue103-single-file-package.sh b/test/issue103-single-file-package.sh
index 3b223d3..64c5848 100755
--- a/test/issue103-single-file-package.sh
+++ b/test/issue103-single-file-package.sh
@@ -1,6 +1,20 @@
 #!/bin/sh
 set -e
 cd ${CURR_DIR}
-./issue103-single-file-package.d foo -- bar
+rm -f single-file-test
+
 ${DUB} run --single issue103-single-file-package-json.d --compiler=${DC}
-${DUB} issue103-single-file-package-w-dep.d --compiler=${DC}
+if [ ! -f single-file-test ]; then
+	echo "Normal invocation did not produce a binary in the current directory"
+	exit 1
+fi
+rm single-file-test
+
+./issue103-single-file-package.d foo -- bar
+
+${DUB} issue103-single-file-package-w-dep.d
+
+if [ -f single-file-test ]; then
+	echo "Shebang invocation produced binary in current directory"
+	exit 1
+fi