diff --git a/cfbs/commands.py b/cfbs/commands.py index d2b26eeb..8fd09e31 100644 --- a/cfbs/commands.py +++ b/cfbs/commands.py @@ -60,12 +60,15 @@ def search_command(terms: List[str]): from cfbs.cfbs_json import CFBSJson from cfbs.cfbs_types import CFBSCommandExitCode, CFBSCommandGitResult from cfbs.masterfiles.analyze import most_relevant_version +from cfbs.masterfiles.download import download_single_version from cfbs.updates import ModuleUpdates, update_module from cfbs.utils import ( CFBSNetworkError, CFBSUserError, CFBSValidationError, + cfbs_dir, cfbs_filename, + display_diff, is_cfbs_repo, read_json, CFBSExitError, @@ -1093,7 +1096,7 @@ def analyze_command( @cfbs_command("convert") def convert_command(non_interactive=False, offline=False): def cfbs_convert_cleanup(): - os.unlink(cfbs_filename()) + rm(cfbs_filename(), missing_ok=True) rm(".git", missing_ok=True) def cfbs_convert_git_commit( @@ -1134,14 +1137,18 @@ def cfbs_convert_git_commit( ) print("Analyzing '" + path_string + "'...") - analyzed_files, _ = analyze_policyset( - path=dir_name, - is_parentpath=False, - reference_version=None, - masterfiles_dir=dir_name, - ignored_path_components=None, - offline=offline, - ) + try: + analyzed_files, _ = analyze_policyset( + path=dir_name, + is_parentpath=False, + reference_version=None, + masterfiles_dir=dir_name, + ignored_path_components=None, + offline=offline, + ) + except: + print("Analyzing the policy set failed, aborting conversion.") + raise current_index = CFBSConfig.get_instance().index default_version = current_index.get_module_object("masterfiles")["version"] @@ -1167,7 +1174,20 @@ def cfbs_convert_git_commit( print("Initializing a new CFBS project...") # since there should be no other files than the masterfiles-name directory, there shouldn't be a .git directory assert not is_git_repo() - r = init_command(masterfiles="no", non_interactive=non_interactive, use_git=True) + try: + r = init_command( + masterfiles="no", non_interactive=non_interactive, use_git=True + ) + except CFBSGitError: + cfbs_convert_cleanup() + print( + "A Git operation failed during initialization of a new CFBS project, aborting conversion." + ) + raise + except: + print("Initializing a new CFBS project failed, aborting conversion.") + cfbs_convert_cleanup() + raise # the cfbs-init should've created a Git repository assert is_git_repo() if r != 0: @@ -1177,19 +1197,33 @@ def cfbs_convert_git_commit( print("Adding masterfiles %s to the project..." % masterfiles_version) masterfiles_to_add = ["masterfiles@%s" % masterfiles_version] - r = add_command(masterfiles_to_add, added_by="cfbs convert") + try: + r = add_command(masterfiles_to_add, added_by="cfbs convert") + except: + print( + "Adding the masterfiles module to the project failed, aborting conversion." + ) + cfbs_convert_cleanup() + raise if r != 0: - print("Adding the masterfiles module failed, aborting conversion.") + print( + "Adding the masterfiles module to the project failed, aborting conversion." + ) cfbs_convert_cleanup() return r print("Adding the policy files...") local_module_to_add = [path_string] - r = add_command( - local_module_to_add, - added_by="cfbs convert", - explicit_build_steps=["copy ./ ./"], - ) + try: + r = add_command( + local_module_to_add, + added_by="cfbs convert", + explicit_build_steps=["copy ./ ./"], + ) + except: + print("Adding the policy files module failed, aborting conversion.") + cfbs_convert_cleanup() + raise if r != 0: print("Adding the policy files module failed, aborting conversion.") cfbs_convert_cleanup() @@ -1247,7 +1281,7 @@ def cfbs_convert_git_commit( if prompt_user_yesno( non_interactive, "Delete files from other versions? (Recommended)" ): - print("Deleting %s files." % len(files_to_delete)) + print("Deleting %s files..." % len(files_to_delete)) for file_d in files_to_delete: rm(os.path.join(dir_name, file_d)) @@ -1261,7 +1295,71 @@ def cfbs_convert_git_commit( print( "The next conversion step is to handle files which have custom modifications." ) - print("This is not implemented yet.") + if not prompt_user_yesno(non_interactive, "Do you want to continue?"): + raise CFBSExitError("User did not proceed, exiting.") + print("The following files have custom modifications:") + modified_files = analyzed_files.modified + for modified_file in modified_files: + print("-", modified_file) + for i, modified_file in enumerate(modified_files, start=1): + # program failures in the middle of this loop would be very user-unfriendly, + # so we will catch exceptions and continue the conversion when handling errors + print("\nFile", i, "diff -", modified_file + ":") + mpf_dir_path = os.path.join(cfbs_dir(), "masterfiles") + mpf_version_dir_path = os.path.join( + mpf_dir_path, masterfiles_version, "tarball", "masterfiles" + ) + mpf_filepath = os.path.join(mpf_version_dir_path, modified_file) + display_diffs = True + if not os.path.exists(mpf_version_dir_path): + try: + download_single_version(mpf_dir_path, masterfiles_version) + except Exception as e: + print( + "Downloading original masterfiles failed (%s), continuing conversion without displaying file diffs." + % str(e) + ) + display_diffs = False + if display_diffs: + try: + display_diff(mpf_filepath, os.path.join(dir_name, modified_file)) + except: + log.warning( + "Displaying a diff between your file and the default file failed, continuing without displaying a diff..." + ) + if i == 1: + if display_diffs: + print( + "Above you can see the differences between your file and the default file." + ) + print( + "As much as possible, we recommend getting rid of these custom modifications." + ) + print( + "Usually, the same thing can be achieved by adding a variable to def.json, or through adding your own policy file (inside 'services/')." + ) + prompt_str = "\nChoose an option:\n" + prompt_str += "1) Drop modifications - They are not important, or can be achieved in another way.\n" + prompt_str += "2) Keep modified file - File is kept as is, and can be handled later. Can make future upgrades more complicated.\n" + prompt_str += "3) (Not implemented yet) Keep patch file - " + prompt_str += "File is converted into a patch file (diff) that hopefully will apply to future versions as well.\n" + + response = prompt_user(non_interactive, prompt_str, ["1", "2"], "1") + + if response == "1": + print("Deleting './%s'..." % modified_file) + rm(os.path.join(dir_name, modified_file)) + commit_message = "Deleted './%s'" % modified_file + print("Creating Git commit - %s..." % commit_message) + try: + cfbs_convert_git_commit(commit_message) + except: + log.warning("Git commit failed, continuing without committing...") + if response == "2": + print("Keeping file as is, nothing to do.") + + print("Conversion finished successfully.") + return 0 diff --git a/cfbs/git.py b/cfbs/git.py index 87a9686a..335515bb 100644 --- a/cfbs/git.py +++ b/cfbs/git.py @@ -49,7 +49,7 @@ def ls_remote(remote, branch): def is_git_repo(path=None): - """Is the given path a Git repository?) + """Is the given path a Git repository? :param:`path` defaults to CWD (if `None`) diff --git a/cfbs/main.py b/cfbs/main.py index 92c225b7..b5b6b82c 100644 --- a/cfbs/main.py +++ b/cfbs/main.py @@ -145,6 +145,7 @@ def _main() -> int: "clean", "update", "input", + "convert", ): raise CFBSUserError( "The option --non-interactive is not for cfbs %s" % (args.command) diff --git a/cfbs/masterfiles/download.py b/cfbs/masterfiles/download.py new file mode 100644 index 00000000..43685887 --- /dev/null +++ b/cfbs/masterfiles/download.py @@ -0,0 +1,190 @@ +import os +import shutil + +from cfbs.masterfiles.analyze import version_is_at_least +from cfbs.utils import CFBSNetworkError, fetch_url, get_json, mkdir, CFBSExitError + +ENTERPRISE_RELEASES_URL = "https://cfengine.com/release-data/enterprise/releases.json" + + +COMMUNITY_ONLY_VERSIONS = ["3.12.0b1", "3.10.0b1"] +"""Masterfiles versions which do not appear in Enterprise releases but appear in Community releases.""" + +MISSING_DATA_VERSIONS = ["3.10.0", "3.9.2"] +"""Rationale for each version: +* 3.10.0:\\ + For some reason, the `"Masterfiles ready-to-install tarball"` is a .tar.gz tarball, rather than a .pkg.tar.gz tarball. + However, an unlisted analoguous URL for the .pkg.tar.gz tarball does exist. +* 3.9.2:\\ + No masterfiles are listed in the release data, but an unlisted analoguous URL does exist.""" + +HARDCODED_VERSIONS = COMMUNITY_ONLY_VERSIONS + MISSING_DATA_VERSIONS + +HARDCODED_URLS = { + "3.12.0b1": "https://cfengine-package-repos.s3.amazonaws.com/community_binaries/Community-3.12.0b1/misc/cfengine-masterfiles-3.12.0b1.pkg.tar.gz", + "3.10.0b1": "https://cfengine-package-repos.s3.amazonaws.com/tarballs/cfengine-masterfiles-3.10.0b1.pkg.tar.gz", + "3.10.0": "https://cfengine-package-repos.s3.amazonaws.com/tarballs/cfengine-masterfiles-3.10.0.pkg.tar.gz", + "3.9.2": "https://cfengine-package-repos.s3.amazonaws.com/tarballs/cfengine-masterfiles-3.9.2.pkg.tar.gz", +} +HARDCODED_CHECKSUMS = { + "3.12.0b1": "ede305dae7be3edfac04fc5b7f63b46adb3a5b1612f4755e855ee8e6b8d344d7", + "3.10.0b1": "09291617254705d79dea2531b23dbd0754f09029e90ce0b43b275aa02c1223a3", + "3.10.0": "7b5e237529e11ce4ae295922dad1a681f13b95f3a7d247d39d3f5088f1a1d7d3", + "3.9.2": "ae1a758530d4a4aad5b6812b61fc37ad1b5900b755f88a1ab98da7fd05a9f5cc", +} + + +def get_download_urls_enterprise(min_version=None): + download_urls = {} + reported_checksums = {} + + print("* gathering download URLs...") + + try: + data = get_json(ENTERPRISE_RELEASES_URL) + except CFBSNetworkError: + raise CFBSExitError( + "Downloading CFEngine release data failed - check your Wi-Fi / network settings." + ) + + for release_data in data["releases"]: + version = release_data["version"] + + if not version_is_at_least(version, min_version): + continue + + if version in MISSING_DATA_VERSIONS: + download_urls[version] = HARDCODED_URLS[version] + reported_checksums[version] = HARDCODED_CHECKSUMS[version] + continue + + release_url = release_data["URL"] + try: + subdata = get_json(release_url) + except CFBSNetworkError: + raise CFBSExitError( + "Downloading CFEngine release data for version %s failed - check your Wi-Fi / network settings." + % version + ) + artifacts_data = subdata["artifacts"] + + if "Additional Assets" not in artifacts_data: + # happens for 3.9.0b1, 3.8.0b1, 3.6.1, 3.6.0 + continue + + masterfiles_data = None + for asset in artifacts_data["Additional Assets"]: + if asset["Title"] == "Masterfiles ready-to-install tarball": + masterfiles_data = asset + + if masterfiles_data is None: + # happens for 3.9.2, 3.9.0, 3.8.2, 3.8.1, 3.8.0, 3.7.4--3.6.2 + # 3.9.2: see above + # 3.9.0 and below: no masterfiles listed, and unlisted analogous URLs seemingly do not exist + continue + + download_urls[version] = masterfiles_data["URL"] + reported_checksums[version] = masterfiles_data["SHA256"] + + return download_urls, reported_checksums + + +def get_all_download_urls(min_version=None): + download_urls, reported_checksums = get_download_urls_enterprise(min_version) + + for version in COMMUNITY_ONLY_VERSIONS: + if version_is_at_least(version, min_version): + download_urls[version] = HARDCODED_URLS[version] + reported_checksums[version] = HARDCODED_CHECKSUMS[version] + + return download_urls, reported_checksums + + +def get_single_download_url(version): + if version in HARDCODED_VERSIONS: + download_url = HARDCODED_URLS[version] + reported_checksum = HARDCODED_CHECKSUMS[version] + return (download_url, reported_checksum) + + try: + data = get_json(ENTERPRISE_RELEASES_URL) + except CFBSNetworkError: + raise CFBSExitError( + "Downloading CFEngine release data failed - check your Wi-Fi / network settings." + ) + + for release_data in data["releases"]: + release_version = release_data["version"] + + if release_version == version: + release_url = release_data["URL"] + try: + subdata = get_json(release_url) + except CFBSNetworkError: + raise CFBSExitError( + "Downloading CFEngine release data for version %s failed - check your Wi-Fi / network settings." + % version + ) + artifacts_data = subdata["artifacts"] + + if "Additional Assets" not in artifacts_data: + break + + for asset in artifacts_data["Additional Assets"]: + if asset["Title"] == "Masterfiles ready-to-install tarball": + download_url = asset["URL"] + reported_checksum = asset["SHA256"] + + return (download_url, reported_checksum) + + raise CFBSExitError("Download URL of given MPF version was not found") + + +def download_versions_from_urls(download_path, download_urls, reported_checksums): + downloaded_versions = [] + + mkdir(download_path) + + for version, url in download_urls.items(): + # ignore master and .x versions + if url.startswith("http://buildcache"): + continue + + print("* downloading from", url) + downloaded_versions.append(version) + + version_path = os.path.join(download_path, version) + mkdir(version_path) + + # download a version, and verify the reported checksum matches + filename = url.split("/")[-1] + tarball_path = os.path.join(version_path, filename) + checksum = reported_checksums[version] + try: + fetch_url(url, tarball_path, checksum) + except CFBSNetworkError as e: + raise CFBSExitError("For version " + version + ": " + str(e)) + + tarball_dir_path = os.path.join(version_path, "tarball") + shutil.unpack_archive(tarball_path, tarball_dir_path) + + return downloaded_versions + + +def download_all_versions(download_path, min_version=None): + download_urls, reported_checksums = get_all_download_urls(min_version) + + downloaded_versions = download_versions_from_urls( + download_path, download_urls, reported_checksums + ) + + return downloaded_versions + + +def download_single_version(download_path, version): + download_url, reported_checksum = get_single_download_url(version) + + download_urls = {version: download_url} + reported_checksums = {version: reported_checksum} + + download_versions_from_urls(download_path, download_urls, reported_checksums) diff --git a/cfbs/masterfiles/download_all_versions.py b/cfbs/masterfiles/download_all_versions.py deleted file mode 100644 index 968a8cb2..00000000 --- a/cfbs/masterfiles/download_all_versions.py +++ /dev/null @@ -1,132 +0,0 @@ -import os -import shutil - -from cfbs.masterfiles.analyze import version_is_at_least -from cfbs.utils import CFBSNetworkError, fetch_url, get_json, mkdir, CFBSExitError - -ENTERPRISE_RELEASES_URL = "https://cfengine.com/release-data/enterprise/releases.json" - - -def get_download_urls_enterprise(min_version=None): - download_urls = {} - reported_checksums = {} - - print("* gathering download URLs...") - - try: - data = get_json(ENTERPRISE_RELEASES_URL) - except CFBSNetworkError: - raise CFBSExitError( - "Downloading CFEngine release data failed - check your Wi-Fi / network settings." - ) - - for release_data in data["releases"]: - version = release_data["version"] - - if not version_is_at_least(version, min_version): - continue - - if version == "3.10.0": - # for 3.10.0, for some reason, the "Masterfiles ready-to-install tarball" is a .tar.gz tarball, rather than a .pkg.tar.gz tarball - # download the .pkg.tar.gz tarball from an unlisted analoguous URL instead - download_url = "https://cfengine-package-repos.s3.amazonaws.com/tarballs/cfengine-masterfiles-3.10.0.pkg.tar.gz" - digest = "7b5e237529e11ce4ae295922dad1a681f13b95f3a7d247d39d3f5088f1a1d7d3" - download_urls[version] = download_url - reported_checksums[version] = digest - continue - if version == "3.9.2": - # for 3.9.2, no masterfiles are listed, but an unlisted analoguous URL exists - download_url = "https://cfengine-package-repos.s3.amazonaws.com/tarballs/cfengine-masterfiles-3.9.2.pkg.tar.gz" - digest = "ae1a758530d4a4aad5b6812b61fc37ad1b5900b755f88a1ab98da7fd05a9f5cc" - download_urls[version] = download_url - reported_checksums[version] = digest - continue - - release_url = release_data["URL"] - try: - subdata = get_json(release_url) - except CFBSNetworkError: - raise CFBSExitError( - "Downloading CFEngine release data for version %s failed - check your Wi-Fi / network settings." - % version - ) - artifacts_data = subdata["artifacts"] - - if "Additional Assets" not in artifacts_data: - # happens for 3.9.0b1, 3.8.0b1, 3.6.1, 3.6.0 - continue - - assets_data = artifacts_data["Additional Assets"] - masterfiles_data = None - - for asset in assets_data: - if asset["Title"] == "Masterfiles ready-to-install tarball": - masterfiles_data = asset - - if masterfiles_data is None: - # happens for 3.9.2, 3.9.0, 3.8.2, 3.8.1, 3.8.0, 3.7.4--3.6.2 - # 3.9.2: see above - # 3.9.0 and below: no masterfiles listed, and unlisted analogous URLs seemingly do not exist - continue - - download_urls[version] = masterfiles_data["URL"] - reported_checksums[version] = masterfiles_data["SHA256"] - - return download_urls, reported_checksums - - -def download_versions_from_urls(download_path, download_urls, reported_checksums): - downloaded_versions = [] - - mkdir(download_path) - - for version, url in download_urls.items(): - # ignore master and .x versions - if url.startswith("http://buildcache"): - continue - - print("* downloading from", url) - downloaded_versions.append(version) - - version_path = os.path.join(download_path, version) - mkdir(version_path) - - # download a version, and verify the reported checksum matches - filename = url.split("/")[-1] - tarball_path = os.path.join(version_path, filename) - checksum = reported_checksums[version] - try: - fetch_url(url, tarball_path, checksum) - except CFBSNetworkError as e: - raise CFBSExitError("For version " + version + ": " + str(e)) - - tarball_dir_path = os.path.join(version_path, "tarball") - shutil.unpack_archive(tarball_path, tarball_dir_path) - - return downloaded_versions - - -def download_all_versions(download_path, min_version=None): - download_urls, reported_checksums = get_download_urls_enterprise(min_version) - - # add masterfiles versions which do not appear in Enterprise releases but appear in Community releases - # 3.12.0b1 - version = "3.12.0b1" - if version_is_at_least(version, min_version): - download_url = "https://cfengine-package-repos.s3.amazonaws.com/community_binaries/Community-3.12.0b1/misc/cfengine-masterfiles-3.12.0b1.pkg.tar.gz" - digest = "ede305dae7be3edfac04fc5b7f63b46adb3a5b1612f4755e855ee8e6b8d344d7" - download_urls[version] = download_url - reported_checksums[version] = digest - # 3.10.0b1 - version = "3.10.0b1" - if version_is_at_least(version, min_version): - download_url = "https://cfengine-package-repos.s3.amazonaws.com/tarballs/cfengine-masterfiles-3.10.0b1.pkg.tar.gz" - digest = "09291617254705d79dea2531b23dbd0754f09029e90ce0b43b275aa02c1223a3" - download_urls[version] = download_url - reported_checksums[version] = digest - - downloaded_versions = download_versions_from_urls( - download_path, download_urls, reported_checksums - ) - - return downloaded_versions diff --git a/cfbs/masterfiles/generate_release_information.py b/cfbs/masterfiles/generate_release_information.py index 02b3403e..2e4ea7e2 100644 --- a/cfbs/masterfiles/generate_release_information.py +++ b/cfbs/masterfiles/generate_release_information.py @@ -1,5 +1,5 @@ from cfbs.masterfiles.analyze import version_is_at_least -from cfbs.masterfiles.download_all_versions import download_all_versions +from cfbs.masterfiles.download import download_all_versions from cfbs.masterfiles.generate_vcf_download import generate_vcf_download from cfbs.masterfiles.generate_vcf_git_checkout import generate_vcf_git_checkout from cfbs.masterfiles.check_download_matches_git import check_download_matches_git diff --git a/cfbs/utils.py b/cfbs/utils.py index b79280f3..6ae97b6e 100644 --- a/cfbs/utils.py +++ b/cfbs/utils.py @@ -102,6 +102,15 @@ def sh(cmd: str, directory=None): _sh("%s" % cmd) +def display_diff(path_A, path_B): + """Also displays `stderr`.""" + cmd = "diff -u " + path_A + " " + path_B + # `diff`'s exit code is 1 for success when there's a difference, so don't use `check=True` + cp = subprocess.run(cmd, shell=True) + if cp.returncode not in (0, 1): + raise + + def mkdir(path: str, exist_ok=True): os.makedirs(path, exist_ok=exist_ok)