diff --git a/.github/workflows/add_validate_build_all.sh b/.github/workflows/add_validate_build_all.sh
new file mode 100644
index 00000000..3e996773
--- /dev/null
+++ b/.github/workflows/add_validate_build_all.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env
+
+# Requires / expects cfbs to already be installed
+# Run from project root (will look for ./cfbs.json)
+# in current working directory
+
+set -e
+set -x
+
+rm -rf ./tmp/
+mkdir -p ./tmp/
+
+# This script is written to also work in for example cfengine/cfbs
+# where we'd need to downooad cfbs.json.
+if [ ! -f ./cfbs.json ] ; then
+    echo "No cfbs.json found in current working directory, downloading from GitHub"
+    curl -L https://raw.githubusercontent.com/cfengine/build-index/refs/heads/master/cfbs.json -o ./tmp/cfbs.json
+else
+    cp ./cfbs.json ./tmp/cfbs.json
+fi
+
+cd ./tmp/
+
+# Move the index to another filename
+mv cfbs.json index.json
+
+# Initialize cfbs project
+cfbs --non-interactive --index ./index.json init --masterfiles=no
+
+# Add masterfiles
+cfbs --non-interactive add masterfiles
+
+# Validate the minimal project
+cfbs validate
+
+# Add all modules
+cfbs search --index ./index.json | \
+awk '{print $1}' | \
+xargs -n1 cfbs --index ./index.json add
+
+# Validate all modules
+cfbs validate
+
+# Build
+cfbs build --ignore-versions-json
+
+# Status
+cfbs status
+
+# Here we use root for everything to avoid permissions problems in policy that accesses files
+# TODO: Enable later
+#- name: Validate built policy
+#  run: |
+#    sudo pip3 install cfbs
+#    wget https://s3.amazonaws.com/cfengine.packages/quick-install-cfengine-enterprise.sh  && sudo bash ./quick-install-cfengine-enterprise.sh agent
+#    sudo cfbs install
+#    # fake a bootstrap, lib-fim module expects stdlib to be in inputs with $(sys.libdir)
+#    sudo cp -r /var/cfengine/masterfiles/lib /var/cfengine/inputs
+#    sudo cf-promises -f /var/cfengine/masterfiles/promises.cf
+#    sudo cf-promises -vf /var/cfengine/masterfiles/update.cf
diff --git a/.github/workflows/add_validate_build_all.yaml b/.github/workflows/add_validate_build_all.yaml
new file mode 100644
index 00000000..d0773d2e
--- /dev/null
+++ b/.github/workflows/add_validate_build_all.yaml
@@ -0,0 +1,32 @@
+name: Add, validate, and build all modules
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on master every night at 3:15 am
+  schedule:
+    - cron: "15 3 * * *"
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      - name: Install cfbs with pip
+        run: pip3 install cfbs
+
+      # Downloading cfbs.json is needed to run this action in another repo, like cfengine/cfbs:
+      - name: Run test script
+        run: bash .github/workflows/add_validate_build_all.sh
diff --git a/cfbs/cfbs_json.py b/cfbs/cfbs_json.py
index 4ea011fc..26f32e14 100644
--- a/cfbs/cfbs_json.py
+++ b/cfbs/cfbs_json.py
@@ -4,7 +4,7 @@
 
 from cfbs.index import Index
 from cfbs.pretty import pretty, TOP_LEVEL_KEYS, MODULE_KEYS
-from cfbs.utils import read_json, CFBSExitError
+from cfbs.utils import CFBSValidationError, read_json, CFBSExitError
 
 
 def _construct_provided_module(name, data, url, commit, added_by="cfbs add"):
@@ -81,12 +81,12 @@ def _find_all_module_objects(self):
             modules += data["build"]
         return modules
 
-    def warn_about_unknown_keys(self):
+    def warn_about_unknown_keys(self, raise_exceptions=False):
         """Basic validation to warn the user when a cfbs.json has unknown keys.
 
         Unknown keys are typically due to
         typos, or an outdated version of cfbs. This basic type of
-        validation only produces warnings (we want cfbs to still work),
+        validation only produces warnings (we want cfbs build to still work),
         and is run for various cfbs commands, not just cfbs build / validate.
         For the more complete validation, see validate.py.
         """
@@ -97,20 +97,30 @@ def warn_about_unknown_keys(self):
 
         for key in data:
             if key not in TOP_LEVEL_KEYS:
-                log.warning(
+                msg = (
                     'The top level key "%s" is not known to this version of cfbs.\n'
                     + "Is it a typo? If not, try upgrading cfbs:\n"
                     + "pip3 install --upgrade cfbs"
                 )
+                if raise_exceptions:
+                    raise CFBSValidationError(msg)
+                log.warning(msg)
+        already_printed = []
         for module in self._find_all_module_objects():
             for key in module:
+                if key in already_printed:
+                    continue
                 if key not in MODULE_KEYS:
-                    log.warning(
+                    msg = (
                         'The module level key "%s" is not known to this version of cfbs.\n'
                         % key
                         + "Is it a typo? If not, try upgrading cfbs:\n"
                         + "pip3 install --upgrade cfbs"
                     )
+                    if raise_exceptions:
+                        raise CFBSValidationError(msg)
+                    log.warning(msg)
+                    already_printed.append(key)
 
     def _get_all_module_names(self, search_in=("build", "provides", "index")):
         modules = []
diff --git a/cfbs/validate.py b/cfbs/validate.py
index c9e6678a..eb11be86 100644
--- a/cfbs/validate.py
+++ b/cfbs/validate.py
@@ -193,9 +193,11 @@ def validate_module_name_content(name):
     log.debug("Validated name of module %s" % name)
 
 
-def _validate_config(config, empty_build_list_ok=False):
+def validate_config_raise_exceptions(config, empty_build_list_ok=False):
     # First validate the config i.e. the user's cfbs.json
-    config.warn_about_unknown_keys()
+    # Here we can raise exceptions, that's what the rest of
+    # the function does, and they are caught by validate_config()
+    config.warn_about_unknown_keys(raise_exceptions=True)
     _validate_top_level_keys(config)
     raw_data = config.raw_data
 
@@ -217,7 +219,7 @@ def _validate_config(config, empty_build_list_ok=False):
 def validate_config(config, empty_build_list_ok=False):
     """Returns `0` if there are no validation errors, and `1` otherwise."""
     try:
-        _validate_config(config, empty_build_list_ok)
+        validate_config_raise_exceptions(config, empty_build_list_ok)
         return 0
     except CFBSValidationError as e:
         print(e)
diff --git a/tests/shell/011_update.sh b/tests/shell/011_update.sh
index 5243a576..6bab18cd 100644
--- a/tests/shell/011_update.sh
+++ b/tests/shell/011_update.sh
@@ -43,7 +43,7 @@ cat cfbs.json | grep -F "version" | grep -F "."
 cat cfbs.json | grep -F "commit"
 cat cfbs.json | grep -F "subdirectory" | grep -F "libraries/python"
 cat cfbs.json | grep -F "added_by" | grep -F "promise-type-ansible"
-cat cfbs.json | grep -F "steps" | grep -F "copy cfengine.py modules/promises/"
+cat cfbs.json | grep -F "copy cfengine_module_library.py modules/promises/cfengine_module_library.py"
 
 cfbs status
 cfbs build
diff --git a/tests/test_validate_mock_index.py b/tests/test_validate_mock_index.py
index ae8aeedf..85ce92ad 100644
--- a/tests/test_validate_mock_index.py
+++ b/tests/test_validate_mock_index.py
@@ -12,7 +12,7 @@ class MockConfig(OrderedDict):
     def raw_data(self):
         return deepcopy(self)
 
-    def warn_about_unknown_keys(self):
+    def warn_about_unknown_keys(self, raise_exceptions=False):
         pass
 
 
diff --git a/tests/test_validate_mock_input.py b/tests/test_validate_mock_input.py
index 60cb0980..58c7d0cf 100644
--- a/tests/test_validate_mock_input.py
+++ b/tests/test_validate_mock_input.py
@@ -12,7 +12,7 @@ class MockConfig(OrderedDict):
     def raw_data(self):
         return deepcopy(self)
 
-    def warn_about_unknown_keys(self):
+    def warn_about_unknown_keys(self, raise_exceptions=False):
         pass