diff --git a/cfbs/masterfiles/analyze.py b/cfbs/masterfiles/analyze.py
index 86867d98..48d329fa 100644
--- a/cfbs/masterfiles/analyze.py
+++ b/cfbs/masterfiles/analyze.py
@@ -87,8 +87,8 @@ def finalize_vcf(versions_dict, checksums_dict, files_dict):
     # sort version numbers, in decreasing order
     versions_dict["versions"] = OrderedDict(
         sorted(
-            versions_dict["versions"].items(),
-            key=lambda p: (version_as_comparable_list(p[0]), p[1]),
+            working_dict.items(),
+            key=lambda p: version_as_comparable_list(p[0]),
             reverse=True,
         )
     )
diff --git a/cfbs/masterfiles/check_download_matches_git.py b/cfbs/masterfiles/check_download_matches_git.py
index a90cc87c..bd6f2c5e 100644
--- a/cfbs/masterfiles/check_download_matches_git.py
+++ b/cfbs/masterfiles/check_download_matches_git.py
@@ -1,6 +1,7 @@
-import os
+from collections import OrderedDict
 
-from cfbs.utils import dict_diff, read_json, user_error
+from cfbs.masterfiles.analyze import version_as_comparable_list
+from cfbs.utils import dict_diff, read_json, user_error, write_json
 
 
 def check_download_matches_git(versions):
@@ -14,7 +15,11 @@ def check_download_matches_git(versions):
     download_versions_dict = read_json("versions.json")
     git_versions_dict = read_json("versions-git.json")
 
-    os.makedirs("differences", exist_ok=True)
+    diffs_dict = {"differences": {}}
+
+    nonmatching_versions = []
+    extraneous_count = 0
+    differing_count = 0
 
     for version in versions:
         download_version_dict = download_versions_dict["versions"][version]["files"]
@@ -29,18 +34,64 @@ def check_download_matches_git(versions):
             new_download_dict[key] = value
         download_version_dict = new_download_dict
 
-        with open("differences/difference-" + version + ".txt", "w") as f:
-            only_dl, only_git, value_diff = dict_diff(
-                download_version_dict, git_version_dict
-            )
-
-            print("Files only in the downloaded version:", only_dl, file=f)
-            print("Files only in the git version:", only_git, file=f)
-            print("Files with different contents:", value_diff, file=f)
-
-            if len(only_dl) > 0 or len(value_diff) > 0:
-                user_error(
-                    "Downloadable files of version "
-                    + version
-                    + " do not match git files"
-                )
+        version_diffs_dict = {}
+        version_diffs_dict["files_only_in_downloads"] = []
+        version_diffs_dict["files_only_in_git"] = []
+        version_diffs_dict["files_with_different_content"] = []
+
+        only_dl, only_git, value_diff = dict_diff(
+            download_version_dict, git_version_dict
+        )
+
+        for filepath in only_dl:
+            version_diffs_dict["files_only_in_downloads"].append(filepath)
+        for filepath in only_git:
+            version_diffs_dict["files_only_in_git"].append(filepath)
+        for filepath, _, _ in value_diff:
+            version_diffs_dict["files_with_different_content"].append(filepath)
+
+        diffs_dict["differences"][version] = version_diffs_dict
+
+        if len(only_dl) > 0 or len(value_diff) > 0:
+            nonmatching_versions.append(version)
+            extraneous_count += len(only_dl)
+            differing_count += len(value_diff)
+
+    nonmatching_versions.sort(key=lambda v: version_as_comparable_list(v), reverse=True)
+
+    # fully sort differences.json:
+    working_dict = diffs_dict["differences"]
+    # sort filepaths of each version, alphabetically
+    for k in working_dict.keys():
+        working_dict[k]["files_only_in_downloads"].sort()
+        working_dict[k]["files_only_in_git"].sort()
+        working_dict[k]["files_with_different_content"].sort()
+    # sort version numbers, in decreasing order
+    diffs_dict["differences"] = OrderedDict(
+        sorted(
+            working_dict.items(),
+            key=lambda p: version_as_comparable_list(p[0]),
+            reverse=True,
+        )
+    )
+
+    write_json("differences.json", diffs_dict)
+
+    if len(nonmatching_versions) > 0:
+        user_error(
+            "The masterfiles downloaded from github.com and cfengine.com do not match - found "
+            + str(extraneous_count)
+            + " extraneous file"
+            + ("" if extraneous_count == 1 else "s")
+            + " and "
+            + str(differing_count)
+            + " differing file"
+            + ("" if differing_count == 1 else "s")
+            + " across "
+            + str(len(nonmatching_versions))
+            + " version"
+            + ("" if len(nonmatching_versions) == 1 else "s")
+            + " ("
+            + ", ".join(nonmatching_versions)
+            + "). See ./differences.json"
+        )