diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index d08f65d..badad33 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,6 +32,11 @@ jobs:
     - uses: actions/checkout@v3
     - name: Test
       run: go test -timeout=15m ./...
+    - uses: actions/upload-artifact@v3
+      if: failure()
+      with:
+        name: bincmp_output
+        path: bincmp_output/
     - name: Test with -race
       # macos and windows tend to be a bit slower,
       # and it's rare that a race in garble would be OS-specific.
diff --git a/.gitignore b/.gitignore
index 3a3e340..dc5a973 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 /garble
-/test
\ No newline at end of file
+/test
+/bincmp_output/
diff --git a/main_test.go b/main_test.go
index e04f261..0a001ed 100644
--- a/main_test.go
+++ b/main_test.go
@@ -196,15 +196,34 @@ func bincmp(ts *testscript.TestScript, neg bool, args []string) {
 		return
 	}
 	if data1 != data2 {
-		if _, err := exec.LookPath("diffoscope"); err != nil {
-			ts.Logf("diffoscope is not installing; skipping binary diff")
-		} else {
+		if _, err := exec.LookPath("diffoscope"); err == nil {
 			// We'll error below; ignore the exec error here.
 			ts.Exec("diffoscope",
 				"--diff-context", "2", // down from 7 by default
 				"--max-text-report-size", "4096", // no limit (in bytes) by default; avoid huge output
 				ts.MkAbs(args[0]), ts.MkAbs(args[1]))
+		} else {
+			ts.Logf("diffoscope not found; skipping")
 		}
+		outDir := "bincmp_output"
+		err := os.MkdirAll(outDir, 0o777)
+		ts.Check(err)
+
+		file1, err := os.CreateTemp(outDir, "file1-*")
+		ts.Check(err)
+		_, err = file1.Write([]byte(data1))
+		ts.Check(err)
+		err = file1.Close()
+		ts.Check(err)
+
+		file2, err := os.CreateTemp(outDir, "file2-*")
+		ts.Check(err)
+		_, err = file2.Write([]byte(data2))
+		ts.Check(err)
+		err = file2.Close()
+		ts.Check(err)
+
+		ts.Logf("wrote files to %s and %s", file1.Name(), file2.Name())
 		sizeDiff := len(data2) - len(data1)
 		ts.Fatalf("%s and %s differ; diffoscope above, size diff: %+d",
 			args[0], args[1], sizeDiff)