diff --git a/JSON.md b/JSON.md index 0b61ff1d..0c5ac4ca 100644 --- a/JSON.md +++ b/JSON.md @@ -264,6 +264,15 @@ In `cfbs.json`'s `"steps"`, the build step name must be separated from the rest - Converts the input data for a module into the augments format and merges it with the target augments file. - Source is relative to module directory and target is relative to `out/masterfiles`. - In most cases, the build step should be: `input ./input.json def.json` +- `replace ` + + - Replace string `` with string ``, exactly `` times, in file `filename`. + - string `` must not contain string ``, as that could lead to confusing / recursive replacement situations. + - The number of occurences is strict: It will error if the string cannot be found, cannot be replaced exactly `` times, or can still be found after replacements are done. + (This is to try to catch mistakes). + - `n` must be an integer, from 1 to 1000, and may optionally have a trailing `+` to signify "or more". + At most 1000 replacements will be performed, regardless of whether you specify `+` or not. + - `replace_version ` - Replace the string inside the file with the version number of that module. - The module must have a version and the string must occur exactly once in the file. diff --git a/cfbs/build.py b/cfbs/build.py index 7c8edca7..3e6994a0 100644 --- a/cfbs/build.py +++ b/cfbs/build.py @@ -1,7 +1,18 @@ +""" +Functions for performing the core part of 'cfbs build' + +This module contains the code for performing the actual build, +converting a project into a ready to deploy policy set. +To achieve this, we iterate over all the build steps in all +the modules running the appropriate file and shell operations. + +There are some preliminary parts of 'cfbs build' implemented +elsewhere, like validation and downloading modules. +""" + import os import logging as log import shutil -from typing import List, Tuple from cfbs.utils import ( canonify, cp, @@ -19,19 +30,12 @@ write_json, ) from cfbs.pretty import pretty, pretty_file - -AVAILABLE_BUILD_STEPS = { - "copy": 2, - "run": "1+", - "delete": "1+", - "json": 2, - "append": 2, - "directory": 2, - "input": 2, - "policy_files": "1+", - "bundles": "1+", - "replace_version": 2, # string to replace and filename -} +from cfbs.validate import ( + AVAILABLE_BUILD_STEPS, + MAX_REPLACEMENTS, + step_has_valid_arg_count, + split_build_step, +) def init_out_folder(): @@ -74,27 +78,53 @@ def _generate_augment(module_name, input_data): return augment -def split_build_step(command) -> Tuple[str, List[str]]: - terms = command.split(" ") - operation, args = terms[0], terms[1:] - return operation, args - - -def step_has_valid_arg_count(args, expected): - actual = len(args) - - if type(expected) is int: - if actual != expected: - return False - - else: - # Only other option is a string of 1+, 2+ or similar: - assert type(expected) is str and expected.endswith("+") - expected = int(expected[0:-1]) - if actual < expected: - return False +def _perform_replace_step(n, a, b, filename): + or_more = False + if n.endswith("+"): + n = n[0:-1] + or_more = True + n = int(n) + if n <= 0: + user_error("replace build step cannot replace something %s times" % (n)) + if n > MAX_REPLACEMENTS or n == MAX_REPLACEMENTS and or_more: + user_error( + "replace build step cannot replace something more than %s times" + % (MAX_REPLACEMENTS) + ) + if a in b and (n >= 2 or or_more): + user_error( + "'%s' must not contain '%s' (could lead to recursive replacing)" % (a, b) + ) + if not os.path.isfile(filename): + user_error("No such file '%s' in replace build step" % (filename,)) + try: + with open(filename, "r") as f: + content = f.read() + except: + user_error("Could not open/read '%s' in replace build step" % (filename,)) + new_content = previous_content = content + for i in range(0, n): + previous_content = new_content + new_content = previous_content.replace(a, b, 1) + if new_content == previous_content: + user_error( + "replace build step could only replace '%s' in '%s' %s times, not %s times (required)" + % (a, filename, i, n) + ) - return True + if or_more: + for i in range(n, MAX_REPLACEMENTS): + previous_content = new_content + new_content = previous_content.replace(a, b, 1) + if new_content == previous_content: + break + if a in new_content: + user_error("too many occurences of '%s' in '%s'" % (a, filename)) + try: + with open(filename, "w") as f: + f.write(new_content) + except: + user_error("Failed to write to '%s'" % (filename,)) def _perform_build_step(module, step, max_length): @@ -260,6 +290,12 @@ def _perform_build_step(module, step, max_length): merged = augment log.debug("Merged def.json: %s", pretty(merged)) write_json(path, merged) + elif operation == "replace": + assert len(args) == 4 + print("%s replace '%s'" % (prefix, "' '".join(args))) + n, a, b, file = args + file = os.path.join(destination, file) + _perform_replace_step(n, a, b, file) elif operation == "replace_version": assert len(args) == 2 print("%s replace_version '%s'" % (prefix, "' '".join(args))) diff --git a/cfbs/validate.py b/cfbs/validate.py index 3f9fa553..d6bf1864 100644 --- a/cfbs/validate.py +++ b/cfbs/validate.py @@ -1,7 +1,29 @@ +""" +Functions for performing the core part of 'cfbs validate' + +Iterate over the JSON structure from cfbs.json, and check +the contents against validation rules. + +Currently, we are not very strict with validation in other +commands, when you run something like 'cfbs build', +many things only produce warnings. This is for backwards +compatibility and we might choose to turn those warnings +into errors in the future. + +Be careful about introducing dependencies to other parts +of the codebase, such as build.py - We want validate.py +to be relatively easy to reuse in various places without +accidentally introducing circular dependencies. +Thus, for example, the common parts needed by both build.py +and validate.py, should be in utils.py or validate.py, +not in build.py. +""" + import argparse import sys import re from collections import OrderedDict +from typing import List, Tuple from cfbs.utils import ( is_a_commit_hash, @@ -9,7 +31,45 @@ ) from cfbs.pretty import TOP_LEVEL_KEYS, MODULE_KEYS from cfbs.cfbs_config import CFBSConfig -from cfbs.build import AVAILABLE_BUILD_STEPS, step_has_valid_arg_count, split_build_step + +AVAILABLE_BUILD_STEPS = { + "copy": 2, + "run": "1+", + "delete": "1+", + "json": 2, + "append": 2, + "directory": 2, + "input": 2, + "policy_files": "1+", + "bundles": "1+", + "replace": 4, # n, a, b, filename + "replace_version": 2, # string to replace and filename +} + +MAX_REPLACEMENTS = 1000 + + +def split_build_step(command) -> Tuple[str, List[str]]: + terms = command.split(" ") + operation, args = terms[0], terms[1:] + return operation, args + + +def step_has_valid_arg_count(args, expected): + actual = len(args) + + if type(expected) is int: + if actual != expected: + return False + + else: + # Only other option is a string of 1+, 2+ or similar: + assert type(expected) is str and expected.endswith("+") + expected = int(expected[0:-1]) + if actual < expected: + return False + + return True class CFBSValidationError(Exception): @@ -130,6 +190,31 @@ def validate_config(config, empty_build_list_ok=False): return 1 +def validate_build_step(name, i, operation, args): + if not operation in AVAILABLE_BUILD_STEPS: + raise CFBSValidationError( + name, + 'Unknown operation "%s" in "steps", must be one of: %s (build step %s in module "%s")' + % (operation, ", ".join(AVAILABLE_BUILD_STEPS), i, name), + ) + expected = AVAILABLE_BUILD_STEPS[operation] + actual = len(args) + if not step_has_valid_arg_count(args, expected): + if type(expected) is int: + raise CFBSValidationError( + name, + "The %s build step expects %d arguments, %d were given (build step " + % (operation, expected, actual), + ) + else: + expected = int(expected[0:-1]) + raise CFBSValidationError( + name, + "The %s build step expects %d or more arguments, %d were given" + % (operation, expected, actual), + ) + + def _validate_module_object(context, name, module, config): def validate_alias(name, module, context): if context == "index": @@ -261,7 +346,7 @@ def validate_steps(name, module): raise CFBSValidationError(name, '"steps" must be of type list') if not module["steps"]: raise CFBSValidationError(name, '"steps" must be non-empty') - for step in module["steps"]: + for i, step in enumerate(module["steps"]): if type(step) != str: raise CFBSValidationError(name, '"steps" must be a list of strings') if not step or step.strip() == "": @@ -269,29 +354,7 @@ def validate_steps(name, module): name, '"steps" must be a list of non-empty / non-whitespace strings' ) operation, args = split_build_step(step) - if not operation in AVAILABLE_BUILD_STEPS: - x = ", ".join(AVAILABLE_BUILD_STEPS) - raise CFBSValidationError( - name, - 'Unknown operation "%s" in "steps", must be one of: (%s)' - % (operation, x), - ) - expected = AVAILABLE_BUILD_STEPS[operation] - actual = len(args) - if not step_has_valid_arg_count(args, expected): - if type(expected) is int: - raise CFBSValidationError( - name, - "The %s build step expects %d arguments, %d were given" - % (operation, expected, actual), - ) - else: - expected = int(expected[0:-1]) - raise CFBSValidationError( - name, - "The %s build step expects %d or more arguments, %d were given" - % (operation, expected, actual), - ) + validate_build_step(name, i, operation, args) def validate_url_field(name, module, field): assert field in module diff --git a/tests/shell/044_replace.sh b/tests/shell/044_replace.sh new file mode 100644 index 00000000..0bbd18bc --- /dev/null +++ b/tests/shell/044_replace.sh @@ -0,0 +1,21 @@ +set -e +set -x +cd tests/ +mkdir -p ./tmp/ +cd ./tmp/ + +# Set up the project we will build: +cp ../shell/044_replace/example-cfbs.json ./cfbs.json +mkdir -p subdir +cp ../shell/044_replace/subdir/example.py ./subdir/example.py +cp ../shell/044_replace/subdir/example.expected.py ./subdir/example.expected.py + +cfbs build + +ls out/masterfiles/services/cfbs/subdir/example.py + +# Replace should have changed it: +! diff ./subdir/example.py out/masterfiles/services/cfbs/subdir/example.py > /dev/null + +# This is the expected content: +diff ./subdir/example.expected.py out/masterfiles/services/cfbs/subdir/example.py diff --git a/tests/shell/044_replace/example-cfbs.json b/tests/shell/044_replace/example-cfbs.json new file mode 100644 index 00000000..98a66c1e --- /dev/null +++ b/tests/shell/044_replace/example-cfbs.json @@ -0,0 +1,19 @@ +{ + "name": "Example project", + "description": "Example description", + "type": "policy-set", + "git": true, + "build": [ + { + "name": "./subdir/", + "description": "Local subdirectory added using cfbs command line", + "added_by": "cfbs add", + "steps": [ + "copy example.py services/cfbs/subdir/example.py", + "replace 1 foo bar services/cfbs/subdir/example.py", + "replace 2 alice bob services/cfbs/subdir/example.py", + "replace 1+ lorem ipsum services/cfbs/subdir/example.py" + ] + } + ] +} diff --git a/tests/shell/044_replace/subdir/example.expected.py b/tests/shell/044_replace/subdir/example.expected.py new file mode 100644 index 00000000..acaa90af --- /dev/null +++ b/tests/shell/044_replace/subdir/example.expected.py @@ -0,0 +1,4 @@ +if __name__ == "__main__": + print("bar") + print("bob,bob") + print("ipsum,ipsum,ipsum") diff --git a/tests/shell/044_replace/subdir/example.py b/tests/shell/044_replace/subdir/example.py new file mode 100644 index 00000000..bba3a1cd --- /dev/null +++ b/tests/shell/044_replace/subdir/example.py @@ -0,0 +1,4 @@ +if __name__ == "__main__": + print("foo") + print("alice,alice") + print("lorem,lorem,lorem") diff --git a/tests/shell/all.sh b/tests/shell/all.sh index ed6c5272..ab083a0e 100644 --- a/tests/shell/all.sh +++ b/tests/shell/all.sh @@ -47,5 +47,6 @@ bash tests/shell/040_add_added_by_field_update_2.sh bash tests/shell/041_add_multidep.sh bash tests/shell/042_update_from_url.sh bash tests/shell/043_replace_version.sh +bash tests/shell/044_replace.sh echo "All cfbs shell tests completed successfully!"