From 7d7eb62bfa007fb50a065f95d6358aa2b12d4e10 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Mon, 26 May 2025 20:18:52 -0400 Subject: [PATCH 1/4] feat: import existing landoscript payload builder code --- src/mozilla_taskgraph/worker_types.py | 202 ++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) diff --git a/src/mozilla_taskgraph/worker_types.py b/src/mozilla_taskgraph/worker_types.py index 6ac2c2d..1aba709 100644 --- a/src/mozilla_taskgraph/worker_types.py +++ b/src/mozilla_taskgraph/worker_types.py @@ -183,3 +183,205 @@ def build_signing_payload(config, task, task_def): ) ) task["attributes"]["release_artifacts"] = sorted(list(artifacts)) + + +@payload_builder( + "scriptworker-landoscript", + schema={ + Required("lando-repo"): str, + Optional("hg-repo-url"): str, + Optional("ignore-closed-tree"): bool, + Optional("dontbuild"): bool, + Optional("tags"): [Any("buildN", "release", None)], + Optional("force-dry-run"): bool, + Optional("android-l10n-import-info"): { + Required("from-repo-url"): str, + Required("toml-info"): [ + { + Required("toml-path"): str, + Required("dest-path"): str, + } + ], + }, + Optional("android-l10n-sync-info"): { + Required("from-branch"): str, + Required("toml-info"): [ + { + Required("toml-path"): str, + } + ], + }, + Optional("l10n-bump-info"): [ + { + Required("name"): str, + Required("path"): str, + Optional("l10n-repo-url"): str, + Optional("l10n-repo-target-branch"): str, + Optional("ignore-config"): object, + Required("platform-configs"): [ + { + Required("platforms"): [str], + Required("path"): str, + Optional("format"): str, + } + ], + } + ], + Optional("bump-files"): [str], + Optional("merge-info"): object, + }, +) +def build_landoscript_payload(config, task, task_def): + worker = task["worker"] + release_config = get_release_config(config) + task_def["payload"] = {"actions": [], "lando_repo": worker["lando-repo"]} + actions = task_def["payload"]["actions"] + + if worker.get("ignore-closed-tree") is not None: + task_def["payload"]["ignore_closed_tree"] = worker["ignore-closed-tree"] + + if worker.get("dontbuild"): + task_def["payload"]["dontbuild"] = True + + if worker.get("force-dry-run"): + task_def["payload"]["dry_run"] = True + + if worker.get("android-l10n-import-info"): + android_l10n_import_info = {} + for k, v in worker["android-l10n-import-info"].items(): + android_l10n_import_info[k.replace("-", "_")] = worker[ + "android-l10n-import-info" + ][k] + android_l10n_import_info["toml_info"] = [ + { + param_name.replace("-", "_"): param_value + for param_name, param_value in entry.items() + } + for entry in worker["android-l10n-import-info"]["toml-info"] + ] + task_def["payload"]["android_l10n_import_info"] = android_l10n_import_info + actions.append("android_l10n_import") + + if worker.get("android-l10n-sync-info"): + android_l10n_sync_info = {} + for k, v in worker["android-l10n-sync-info"].items(): + android_l10n_sync_info[k.replace("-", "_")] = worker[ + "android-l10n-sync-info" + ][k] + android_l10n_sync_info["toml_info"] = [ + { + param_name.replace("-", "_"): param_value + for param_name, param_value in entry.items() + } + for entry in worker["android-l10n-sync-info"]["toml-info"] + ] + task_def["payload"]["android_l10n_sync_info"] = android_l10n_sync_info + actions.append("android_l10n_sync") + + if worker.get("l10n-bump-info"): + l10n_bump_info = [] + l10n_repo_urls = set() + for lbi in worker["l10n-bump-info"]: + new_lbi = {} + if "l10n-repo-url" in lbi: + l10n_repo_urls.add(lbi["l10n-repo-url"]) + for k, v in lbi.items(): + new_lbi[k.replace("-", "_")] = lbi[k] + l10n_bump_info.append(new_lbi) + + task_def["payload"]["l10n_bump_info"] = l10n_bump_info + if len(l10n_repo_urls) > 1: + raise Exception( + "Must use the same l10n-repo-url for all files in the same task!" + ) + elif len(l10n_repo_urls) == 1: + actions.append("l10n_bump") + + if worker.get("tags"): + tag_names = [] + product = task["shipping-product"].upper() + version = release_config["version"].replace(".", "_") + buildnum = release_config["build_number"] + if "buildN" in worker["tags"]: + tag_names.extend( + [ + f"{product}_{version}_BUILD{buildnum}", + ] + ) + if "release" in worker["tags"]: + tag_names.extend([f"{product}_{version}_RELEASE"]) + tag_info = { + "tags": tag_names, + "hg_repo_url": worker["hg-repo-url"], + "revision": config.params[ + "{}head_rev".format(worker.get("repo-param-prefix", "")) + ], + } + task_def["payload"]["tag_info"] = tag_info + actions.append("tag") + + if worker.get("bump-files"): + bump_info = {} + bump_info["next_version"] = release_config["next_version"] + bump_info["files"] = worker["bump-files"] + task_def["payload"]["version_bump_info"] = bump_info + actions.append("version_bump") + + if worker.get("merge-info"): + merge_info = { + merge_param_name.replace("-", "_"): merge_param_value + for merge_param_name, merge_param_value in worker["merge-info"].items() + if merge_param_name != "version-files" + } + merge_info["version_files"] = [ + { + file_param_name.replace("-", "_"): file_param_value + for file_param_name, file_param_value in file_entry.items() + } + for file_entry in worker["merge-info"]["version-files"] + ] + # hack alert: co-opt the l10n_bump_info into the merge_info section + # this should be cleaned up to avoid l10n_bump_info ever existing + # in the payload + if task_def["payload"].get("l10n_bump_info"): + actions.remove("l10n_bump") + merge_info["l10n_bump_info"] = task_def["payload"].pop("l10n_bump_info") + + task_def["payload"]["merge_info"] = merge_info + actions.append("merge_day") + + scopes = set(task_def.get("scopes", [])) + scopes.add(f"project:releng:lando:repo:{worker['lando-repo']}") + scopes.update([f"project:releng:lando:action:{action}" for action in actions]) + task_def["scopes"] = sorted(scopes) + + +def get_release_config(config): + """Get the build number and version for a release task. + + Currently only applies to beetmover tasks. + + Args: + config (TransformConfig): The configuration for the kind being transformed. + + Returns: + dict: containing both `build_number` and `version`. This can be used to + update `task.payload`. + """ + release_config = { + "version": config.params["version"], + "appVersion": config.params["app_version"], + "next_version": config.params["next_version"], + "build_number": config.params["build_number"], + } + if pv := config.params.get("partial_versions") and config.kind in ( + "release-bouncer-sub", + "release-bouncer-check", + "release-update-verify-config", + "release-secondary-update-verify-config", + "release-balrog-submit-toplevel", + "release-secondary-balrog-submit-toplevel", + ): + release_config["partial_versions"] = pv + + return release_config From 5390719fa47a013a152392a93d4af73b9efb832b Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Mon, 26 May 2025 21:36:41 -0400 Subject: [PATCH 2/4] feat: add tests for landoscript payload builder and standardize the name/tags This brings mostly inline with existing code, and gets us to a better place to start making improvements. --- src/mozilla_taskgraph/worker_types.py | 19 +- test/conftest.py | 5 +- test/test_worker_types.py | 297 +++++++++++++++++++++++++- 3 files changed, 308 insertions(+), 13 deletions(-) diff --git a/src/mozilla_taskgraph/worker_types.py b/src/mozilla_taskgraph/worker_types.py index 1aba709..9f89c07 100644 --- a/src/mozilla_taskgraph/worker_types.py +++ b/src/mozilla_taskgraph/worker_types.py @@ -186,7 +186,7 @@ def build_signing_payload(config, task, task_def): @payload_builder( - "scriptworker-landoscript", + "scriptworker-lando", schema={ Required("lando-repo"): str, Optional("hg-repo-url"): str, @@ -231,10 +231,11 @@ def build_signing_payload(config, task, task_def): Optional("merge-info"): object, }, ) -def build_landoscript_payload(config, task, task_def): +def build_lando_payload(config, task, task_def): worker = task["worker"] release_config = get_release_config(config) task_def["payload"] = {"actions": [], "lando_repo": worker["lando-repo"]} + task_def["tags"]["worker-implementation"] = "scriptworker" actions = task_def["payload"]["actions"] if worker.get("ignore-closed-tree") is not None: @@ -375,13 +376,13 @@ def get_release_config(config): "build_number": config.params["build_number"], } if pv := config.params.get("partial_versions") and config.kind in ( - "release-bouncer-sub", - "release-bouncer-check", - "release-update-verify-config", - "release-secondary-update-verify-config", - "release-balrog-submit-toplevel", - "release-secondary-balrog-submit-toplevel", - ): + "release-bouncer-sub", + "release-bouncer-check", + "release-update-verify-config", + "release-secondary-update-verify-config", + "release-balrog-submit-toplevel", + "release-secondary-balrog-submit-toplevel", + ): release_config["partial_versions"] = pv return release_config diff --git a/test/conftest.py b/test/conftest.py index a8b670a..f521212 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -74,6 +74,7 @@ def file_url(self, path, pretty=False): def parameters(): return FakeParameters( { + "app_version": "99.0", "base_ref": "123456", "base_repository": "http://example.com/base/repo", "build_date": 0, @@ -85,7 +86,7 @@ def parameters(): "head_tag": "", "level": "1", "moz_build_date": 0, - "next_version": "1.0.1", + "next_version": "100.0", "owner": "some-owner", "project": "some-project", "pushlog_id": 1, @@ -93,7 +94,7 @@ def parameters(): "target_tasks_method": "test_method", "tasks_for": "hg-push", "try_mode": None, - "version": "", + "version": "99.0", } ) diff --git a/test/test_worker_types.py b/test/test_worker_types.py index 7d73cdc..a731f50 100644 --- a/test/test_worker_types.py +++ b/test/test_worker_types.py @@ -12,7 +12,7 @@ @pytest.fixture def build_payload(make_graph_config, make_transform_config, parameters): graph_config = make_graph_config( - extra_config={"scriptworker": {"scope-prefix": "foo"}} + extra_config={"scriptworker": {"scope-prefix": "foo"}}, ) def inner( @@ -25,7 +25,7 @@ def inner( worker.setdefault("implementation", name) - task = {"worker": worker} + task = {"worker": worker, "shipping-product": "product"} task_def = {"tags": {}} parameters.update(extra_params) @@ -463,3 +463,296 @@ def test_build_signing_payload_gpg_asc(build_payload): "foo/bar.asc", ] } + + +def test_lando_android_l10n_import(build_payload): + worker = { + "lando-repo": "testrepo", + "android-l10n-import-info": { + "from-repo-url": "https://from", + "toml-info": [ + { + "toml-path": "foo/bar/l10n.toml", + "dest-path": "foo-bar", + } + ], + }, + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["android_l10n_import"], + "lando_repo": "testrepo", + "android_l10n_import_info": { + "from_repo_url": "https://from", + "toml_info": [ + {"toml_path": "foo/bar/l10n.toml", "dest_path": "foo-bar"}, + ], + }, + }, + "scopes": [ + "project:releng:lando:action:android_l10n_import", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +def test_lando_android_l10n_sync(build_payload): + worker = { + "lando-repo": "testrepo", + "android-l10n-sync-info": { + "from-branch": "branchy", + "toml-info": [ + { + "toml-path": "foo/bar/l10n.toml", + } + ], + }, + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["android_l10n_sync"], + "lando_repo": "testrepo", + "android_l10n_sync_info": { + "from_branch": "branchy", + "toml_info": [ + { + "toml_path": "foo/bar/l10n.toml", + }, + ], + }, + }, + "scopes": [ + "project:releng:lando:action:android_l10n_sync", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +def test_lando_l10n_bump(build_payload): + worker = { + "lando-repo": "testrepo", + "dontbuild": True, + "ignore-closed-tree": True, + "l10n-bump-info": [ + { + "name": "l10n", + "path": "foo/bar/changesets.json", + "l10n-repo-url": "https://l10n", + "l10n-repo-target-branch": "branchy", + "ignore-config": {"ab": ["foo"]}, + "platform-configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["l10n_bump"], + "lando_repo": "testrepo", + "dontbuild": True, + "ignore_closed_tree": True, + "l10n_bump_info": [ + { + "name": "l10n", + "path": "foo/bar/changesets.json", + "l10n_repo_url": "https://l10n", + "l10n_repo_target_branch": "branchy", + "ignore_config": { + "ab": ["foo"], + }, + "platform_configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + } + ], + }, + "scopes": [ + "project:releng:lando:action:l10n_bump", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +def test_lando_tag(build_payload): + worker = { + "lando-repo": "testrepo", + "tags": ["buildN", "release"], + "hg-repo-url": "https://hg/repo", + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["tag"], + "lando_repo": "testrepo", + "tag_info": { + "hg_repo_url": "https://hg/repo", + "revision": "abcdef", + "tags": [ + "PRODUCT_99_0_BUILD1", + "PRODUCT_99_0_RELEASE", + ], + }, + }, + "scopes": [ + "project:releng:lando:action:tag", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +def test_lando_version_bump(build_payload): + worker = { + "lando-repo": "testrepo", + "bump-files": [ + "foo/bar/a.txt", + "another/file.txt", + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["version_bump"], + "lando_repo": "testrepo", + "version_bump_info": { + "files": [ + "foo/bar/a.txt", + "another/file.txt", + ], + "next_version": "100.0", + }, + }, + "scopes": [ + "project:releng:lando:action:version_bump", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +def test_lando_merge(build_payload): + worker = { + "lando-repo": "testrepo", + "dontbuild": False, + "l10n-bump-info": [ + { + "name": "l10n", + "path": "foo/bar/changesets.json", + "l10n-repo-url": "https://l10n", + "l10n-repo-target-branch": "branchy", + "ignore-config": {"ab": ["foo"]}, + "platform-configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + } + ], + "merge-info": { + "fetch-version-from": "version.txt", + "version-files": [ + { + "filename": "version.txt", + "new-suffix": "", + }, + { + "filename": "other.txt", + "new-suffix": "b1", + }, + ], + "replacements": [ + [ + "something.txt", + "foo", + "bar", + ] + ], + "regex-replacements": [ + [ + "somethingelse.txt", + "thing[0-9]+.0", + "bar{next_major_version}.0", + ] + ], + "merge-old-head": True, + "base-tag": "{major_version}_BASE", + "end-tag": "{major_version}_END", + "from-branch": "from-b", + "to-branch": "to-b", + }, + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["merge_day"], + "lando_repo": "testrepo", + "merge_info": { + "fetch_version_from": "version.txt", + "version_files": [ + { + "filename": "version.txt", + "new_suffix": "", + }, + { + "filename": "other.txt", + "new_suffix": "b1", + }, + ], + "replacements": [ + [ + "something.txt", + "foo", + "bar", + ] + ], + "regex_replacements": [ + [ + "somethingelse.txt", + "thing[0-9]+.0", + "bar{next_major_version}.0", + ] + ], + "merge_old_head": True, + "base_tag": "{major_version}_BASE", + "end_tag": "{major_version}_END", + "from_branch": "from-b", + "to_branch": "to-b", + "l10n_bump_info": [ + { + "name": "l10n", + "path": "foo/bar/changesets.json", + "l10n_repo_url": "https://l10n", + "l10n_repo_target_branch": "branchy", + "ignore_config": { + "ab": ["foo"], + }, + "platform_configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + } + ], + }, + }, + "scopes": [ + "project:releng:lando:action:merge_day", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } From ff0772e301dcff7b7bff3feaf2fc83a2e41fa07f Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Tue, 27 May 2025 09:38:22 -0400 Subject: [PATCH 3/4] refactor: get rid of `partial_versions` from `get_release_config` This is not really release-specific (it's used for nightlies as well), and allows us to drop the hardcoded list of kinds from this function. When Gecko picks up this change, some small adjustments will be needed to pull `partial_versions` directly from parameters instead. --- src/mozilla_taskgraph/worker_types.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/mozilla_taskgraph/worker_types.py b/src/mozilla_taskgraph/worker_types.py index 9f89c07..2b6cc0a 100644 --- a/src/mozilla_taskgraph/worker_types.py +++ b/src/mozilla_taskgraph/worker_types.py @@ -369,20 +369,9 @@ def get_release_config(config): dict: containing both `build_number` and `version`. This can be used to update `task.payload`. """ - release_config = { + return { "version": config.params["version"], "appVersion": config.params["app_version"], "next_version": config.params["next_version"], "build_number": config.params["build_number"], } - if pv := config.params.get("partial_versions") and config.kind in ( - "release-bouncer-sub", - "release-bouncer-check", - "release-update-verify-config", - "release-secondary-update-verify-config", - "release-balrog-submit-toplevel", - "release-secondary-balrog-submit-toplevel", - ): - release_config["partial_versions"] = pv - - return release_config From ffc5f1e1e9a42a93a9cdc72ce36aa08d9d2013da Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Tue, 27 May 2025 10:17:14 -0400 Subject: [PATCH 4/4] refactor: rework landoscript payload schema A few notable changes: * Move action-specific things into an `actions` list (per the suggestion in https://phabricator.services.mozilla.com/D246671) * Add separate schemas for different types of merge, to enhance validation * Reduce what's needed in the `worker` section to enable Matrix notifications --- src/mozilla_taskgraph/worker_types.py | 360 ++++++++++------- test/test_worker_types.py | 541 +++++++++++++++++++++----- 2 files changed, 674 insertions(+), 227 deletions(-) diff --git a/src/mozilla_taskgraph/worker_types.py b/src/mozilla_taskgraph/worker_types.py index 2b6cc0a..76f9b12 100644 --- a/src/mozilla_taskgraph/worker_types.py +++ b/src/mozilla_taskgraph/worker_types.py @@ -185,50 +185,142 @@ def build_signing_payload(config, task, task_def): task["attributes"]["release_artifacts"] = sorted(list(artifacts)) +l10n_bump_info_schema = [ + { + Required("name"): str, + Required("path"): str, + Required("l10n-repo-url"): str, + Required("l10n-repo-target-branch"): str, + Optional("ignore-config"): object, + Required("platform-configs"): [ + { + Required("platforms"): [str], + Required("path"): str, + Optional("format"): str, + } + ], + } +] + + +def process_l10n_bump_info(info): + l10n_bump_info = [] + l10n_repo_urls = set() + for lbi in info: + l10n_repo_urls.add(lbi["l10n-repo-url"]) + l10n_bump_info.append(dash_to_underscore(lbi)) + + if len(l10n_repo_urls) > 1: + raise Exception( + "Must use the same l10n-repo-url for all files in the same task!" + ) + + return l10n_bump_info + + @payload_builder( "scriptworker-lando", schema={ Required("lando-repo"): str, - Optional("hg-repo-url"): str, - Optional("ignore-closed-tree"): bool, - Optional("dontbuild"): bool, - Optional("tags"): [Any("buildN", "release", None)], - Optional("force-dry-run"): bool, - Optional("android-l10n-import-info"): { - Required("from-repo-url"): str, - Required("toml-info"): [ + Required("actions"): [ + Any( { - Required("toml-path"): str, - Required("dest-path"): str, - } - ], - }, - Optional("android-l10n-sync-info"): { - Required("from-branch"): str, - Required("toml-info"): [ + Required("android-l10n-sync"): { + Required("from-branch"): str, + Required("toml-info"): [ + { + Required("toml-path"): str, + } + ], + }, + }, { - Required("toml-path"): str, - } - ], - }, - Optional("l10n-bump-info"): [ - { - Required("name"): str, - Required("path"): str, - Optional("l10n-repo-url"): str, - Optional("l10n-repo-target-branch"): str, - Optional("ignore-config"): object, - Required("platform-configs"): [ - { - Required("platforms"): [str], - Required("path"): str, - Optional("format"): str, + Required("android-l10n-import"): { + Required("from-repo-url"): str, + Required("toml-info"): [ + { + Required("toml-path"): str, + Required("dest-path"): str, + } + ], + }, + }, + { + Required("l10n-bump"): l10n_bump_info_schema, + }, + { + Required("tag"): { + Required("types"): [Any("buildN", "release")], + Required("hg-repo-url"): str, } - ], - } + }, + { + Required("version-bump"): { + Required("bump-files"): [str], + }, + }, + # the remaining action types all end up using the "merge_day" + # landoscript action. however, these are quite varied tasks, + # and separating them out allows us to have stronger schemas. + { + Required("esr-bump"): { + Required("to-branch"): str, + Required("fetch-version-from"): str, + Required("version-files"): [ + { + Required("filename"): str, + Required("version-bump"): str, + Optional("new-suffix"): str, + }, + ], + }, + }, + { + Required("main-bump"): { + Required("to-branch"): str, + Required("fetch-version-from"): str, + Required("version-files"): [ + { + Required("filename"): str, + Required("version-bump"): str, + Optional("new-suffix"): str, + }, + ], + Optional("replacements"): [[str]], + Optional("regex-replacements"): [[str]], + Optional("end-tag"): str, + }, + }, + { + Required("early-to-late-beta"): { + Required("to-branch"): str, + Optional("replacements"): [[str]], + }, + }, + { + Required("uplift"): { + Required("fetch-version-from"): str, + Required("version-files"): [ + { + Required("filename"): str, + Optional("version-bump"): str, + Optional("new-suffix"): str, + }, + ], + Required("from-branch"): str, + Required("to-branch"): str, + Optional("replacements"): [[str]], + Optional("base-tag"): str, + Optional("end-tag"): str, + Optional("l10n-bump-info"): l10n_bump_info_schema, + }, + }, + ) ], - Optional("bump-files"): [str], - Optional("merge-info"): object, + Optional("ignore-closed-tree"): bool, + Optional("dontbuild"): bool, + Optional("force-dry-run"): bool, + Optional("matrix-rooms"): [str], }, ) def build_lando_payload(config, task, task_def): @@ -247,113 +339,102 @@ def build_lando_payload(config, task, task_def): if worker.get("force-dry-run"): task_def["payload"]["dry_run"] = True - if worker.get("android-l10n-import-info"): - android_l10n_import_info = {} - for k, v in worker["android-l10n-import-info"].items(): - android_l10n_import_info[k.replace("-", "_")] = worker[ - "android-l10n-import-info" - ][k] - android_l10n_import_info["toml_info"] = [ - { - param_name.replace("-", "_"): param_value - for param_name, param_value in entry.items() - } - for entry in worker["android-l10n-import-info"]["toml-info"] - ] - task_def["payload"]["android_l10n_import_info"] = android_l10n_import_info - actions.append("android_l10n_import") - - if worker.get("android-l10n-sync-info"): - android_l10n_sync_info = {} - for k, v in worker["android-l10n-sync-info"].items(): - android_l10n_sync_info[k.replace("-", "_")] = worker[ - "android-l10n-sync-info" - ][k] - android_l10n_sync_info["toml_info"] = [ - { - param_name.replace("-", "_"): param_value - for param_name, param_value in entry.items() - } - for entry in worker["android-l10n-sync-info"]["toml-info"] - ] - task_def["payload"]["android_l10n_sync_info"] = android_l10n_sync_info - actions.append("android_l10n_sync") - - if worker.get("l10n-bump-info"): - l10n_bump_info = [] - l10n_repo_urls = set() - for lbi in worker["l10n-bump-info"]: - new_lbi = {} - if "l10n-repo-url" in lbi: - l10n_repo_urls.add(lbi["l10n-repo-url"]) - for k, v in lbi.items(): - new_lbi[k.replace("-", "_")] = lbi[k] - l10n_bump_info.append(new_lbi) - - task_def["payload"]["l10n_bump_info"] = l10n_bump_info - if len(l10n_repo_urls) > 1: - raise Exception( - "Must use the same l10n-repo-url for all files in the same task!" - ) - elif len(l10n_repo_urls) == 1: + for action in worker["actions"]: + if info := action.get("android-l10n-import"): + android_l10n_import_info = dash_to_underscore(info) + android_l10n_import_info["toml_info"] = [ + dash_to_underscore(ti) for ti in android_l10n_import_info["toml_info"] + ] + task_def["payload"]["android_l10n_import_info"] = android_l10n_import_info + actions.append("android_l10n_import") + + if info := action.get("android-l10n-sync"): + android_l10n_sync_info = dash_to_underscore(info) + android_l10n_sync_info["toml_info"] = [ + dash_to_underscore(ti) for ti in android_l10n_sync_info["toml_info"] + ] + task_def["payload"]["android_l10n_sync_info"] = android_l10n_sync_info + actions.append("android_l10n_sync") + + if info := action.get("l10n-bump"): + task_def["payload"]["l10n_bump_info"] = process_l10n_bump_info(info) actions.append("l10n_bump") - if worker.get("tags"): - tag_names = [] - product = task["shipping-product"].upper() - version = release_config["version"].replace(".", "_") - buildnum = release_config["build_number"] - if "buildN" in worker["tags"]: - tag_names.extend( - [ - f"{product}_{version}_BUILD{buildnum}", - ] - ) - if "release" in worker["tags"]: - tag_names.extend([f"{product}_{version}_RELEASE"]) - tag_info = { - "tags": tag_names, - "hg_repo_url": worker["hg-repo-url"], - "revision": config.params[ - "{}head_rev".format(worker.get("repo-param-prefix", "")) - ], - } - task_def["payload"]["tag_info"] = tag_info - actions.append("tag") - - if worker.get("bump-files"): - bump_info = {} - bump_info["next_version"] = release_config["next_version"] - bump_info["files"] = worker["bump-files"] - task_def["payload"]["version_bump_info"] = bump_info - actions.append("version_bump") - - if worker.get("merge-info"): - merge_info = { - merge_param_name.replace("-", "_"): merge_param_value - for merge_param_name, merge_param_value in worker["merge-info"].items() - if merge_param_name != "version-files" - } - merge_info["version_files"] = [ - { - file_param_name.replace("-", "_"): file_param_value - for file_param_name, file_param_value in file_entry.items() + if info := action.get("tag"): + tag_types = info["types"] + tag_names = [] + product = task["shipping-product"].upper() + version = release_config["version"].replace(".", "_") + buildnum = release_config["build_number"] + if "buildN" in tag_types: + tag_names.extend( + [ + f"{product}_{version}_BUILD{buildnum}", + ] + ) + if "release" in tag_types: + tag_names.extend([f"{product}_{version}_RELEASE"]) + tag_info = { + "tags": tag_names, + "hg_repo_url": info["hg-repo-url"], + "revision": config.params[ + "{}head_rev".format(worker.get("repo-param-prefix", "")) + ], } - for file_entry in worker["merge-info"]["version-files"] - ] - # hack alert: co-opt the l10n_bump_info into the merge_info section - # this should be cleaned up to avoid l10n_bump_info ever existing - # in the payload - if task_def["payload"].get("l10n_bump_info"): - actions.remove("l10n_bump") - merge_info["l10n_bump_info"] = task_def["payload"].pop("l10n_bump_info") - - task_def["payload"]["merge_info"] = merge_info - actions.append("merge_day") + task_def["payload"]["tag_info"] = tag_info + actions.append("tag") + + if info := action.get("version-bump"): + bump_info = {} + bump_info["next_version"] = release_config["next_version"] + bump_info["files"] = info["bump-files"] + task_def["payload"]["version_bump_info"] = bump_info + actions.append("version_bump") + + if info := action.get("esr-bump"): + merge_info = dash_to_underscore(info) + merge_info["version_files"] = [ + dash_to_underscore(vf) for vf in info["version-files"] + ] + task_def["payload"]["merge_info"] = merge_info + actions.append("merge_day") + + if info := action.get("main-bump"): + merge_info = dash_to_underscore(info) + + merge_info["version_files"] = [ + dash_to_underscore(vf) for vf in info["version-files"] + ] + task_def["payload"]["merge_info"] = merge_info + actions.append("merge_day") + + if info := action.get("early-to-late-beta"): + task_def["payload"]["merge_info"] = dash_to_underscore(info) + actions.append("merge_day") + + if info := action.get("uplift"): + merge_info = dash_to_underscore(info) + merge_info["merge_old_head"] = True + + merge_info["version_files"] = [ + dash_to_underscore(vf) for vf in info["version-files"] + ] + if lbi := info.get("l10n-bump-info"): + merge_info["l10n_bump_info"] = process_l10n_bump_info(lbi) + + task_def["payload"]["merge_info"] = merge_info + actions.append("merge_day") scopes = set(task_def.get("scopes", [])) scopes.add(f"project:releng:lando:repo:{worker['lando-repo']}") scopes.update([f"project:releng:lando:action:{action}" for action in actions]) + + for matrix_room in worker.get("matrix-rooms", []): + task_def.setdefault("routes", []) + task_def["routes"].append(f"notify.matrix-room.{matrix_room}.on-pending") + task_def["routes"].append(f"notify.matrix-room.{matrix_room}.on-resolved") + scopes.add("queue:route:notify.matrix-room.*") + task_def["scopes"] = sorted(scopes) @@ -375,3 +456,10 @@ def get_release_config(config): "next_version": config.params["next_version"], "build_number": config.params["build_number"], } + + +def dash_to_underscore(obj): + new_obj = {} + for k, v in obj.items(): + new_obj[k.replace("-", "_")] = v + return new_obj diff --git a/test/test_worker_types.py b/test/test_worker_types.py index a731f50..ae8eba7 100644 --- a/test/test_worker_types.py +++ b/test/test_worker_types.py @@ -468,15 +468,19 @@ def test_build_signing_payload_gpg_asc(build_payload): def test_lando_android_l10n_import(build_payload): worker = { "lando-repo": "testrepo", - "android-l10n-import-info": { - "from-repo-url": "https://from", - "toml-info": [ - { - "toml-path": "foo/bar/l10n.toml", - "dest-path": "foo-bar", - } - ], - }, + "actions": [ + { + "android-l10n-import": { + "from-repo-url": "https://from", + "toml-info": [ + { + "toml-path": "foo/bar/l10n.toml", + "dest-path": "foo-bar", + } + ], + }, + } + ], } _, task_def = build_payload("scriptworker-lando", worker=worker) assert task_def == { @@ -501,14 +505,18 @@ def test_lando_android_l10n_import(build_payload): def test_lando_android_l10n_sync(build_payload): worker = { "lando-repo": "testrepo", - "android-l10n-sync-info": { - "from-branch": "branchy", - "toml-info": [ - { - "toml-path": "foo/bar/l10n.toml", - } - ], - }, + "actions": [ + { + "android-l10n-sync": { + "from-branch": "branchy", + "toml-info": [ + { + "toml-path": "foo/bar/l10n.toml", + } + ], + }, + } + ], } _, task_def = build_payload("scriptworker-lando", worker=worker) assert task_def == { @@ -532,22 +540,84 @@ def test_lando_android_l10n_sync(build_payload): } +def test_lando_l10n_bump_mismatched_urls(build_payload): + worker = { + "lando-repo": "testrepo", + "dontbuild": True, + "ignore-closed-tree": True, + "actions": [ + { + "l10n-bump": [ + { + "name": "l10n", + "path": "foo/bar/changesets.json", + "l10n-repo-url": "https://l10n", + "l10n-repo-target-branch": "branchy", + "ignore-config": {"ab": ["foo"]}, + "platform-configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + }, + { + "name": "l10n2", + "path": "foo/baz/changesets.json", + "l10n-repo-url": "https://l10n2", + "l10n-repo-target-branch": "branchy", + "ignore-config": {"ab": ["foo"]}, + "platform-configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + }, + ], + } + ], + } + try: + build_payload("scriptworker-lando", worker=worker) + assert False, "should've raised an exception" + except Exception as e: + assert "Must use the same l10n-repo-url for all files" in e.args[0] + + def test_lando_l10n_bump(build_payload): worker = { "lando-repo": "testrepo", "dontbuild": True, "ignore-closed-tree": True, - "l10n-bump-info": [ + "actions": [ { - "name": "l10n", - "path": "foo/bar/changesets.json", - "l10n-repo-url": "https://l10n", - "l10n-repo-target-branch": "branchy", - "ignore-config": {"ab": ["foo"]}, - "platform-configs": [ + "l10n-bump": [ { - "platforms": ["p1", "p2"], - "path": "foo/bar/locales", + "name": "l10n", + "path": "foo/bar/changesets.json", + "l10n-repo-url": "https://l10n", + "l10n-repo-target-branch": "branchy", + "ignore-config": {"ab": ["foo"]}, + "platform-configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + }, + { + "name": "l10n2", + "path": "foo/baz/changesets.json", + "l10n-repo-url": "https://l10n", + "l10n-repo-target-branch": "branchy", + "ignore-config": {"ab": ["foo"]}, + "platform-configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], }, ], } @@ -575,7 +645,22 @@ def test_lando_l10n_bump(build_payload): "path": "foo/bar/locales", }, ], - } + }, + { + "name": "l10n2", + "path": "foo/baz/changesets.json", + "l10n_repo_url": "https://l10n", + "l10n_repo_target_branch": "branchy", + "ignore_config": { + "ab": ["foo"], + }, + "platform_configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + }, ], }, "scopes": [ @@ -586,11 +671,37 @@ def test_lando_l10n_bump(build_payload): } -def test_lando_tag(build_payload): +@pytest.mark.parametrize( + "types,expected", + ( + pytest.param( + ["buildN"], + ["PRODUCT_99_0_BUILD1"], + id="buildN", + ), + pytest.param( + ["release"], + ["PRODUCT_99_0_RELEASE"], + id="release", + ), + pytest.param( + ["buildN", "release"], + ["PRODUCT_99_0_BUILD1", "PRODUCT_99_0_RELEASE"], + id="buildN_and_release", + ), + ), +) +def test_lando_tag(build_payload, types, expected): worker = { "lando-repo": "testrepo", - "tags": ["buildN", "release"], - "hg-repo-url": "https://hg/repo", + "actions": [ + { + "tag": { + "types": types, + "hg-repo-url": "https://hg/repo", + } + } + ], } _, task_def = build_payload("scriptworker-lando", worker=worker) assert task_def == { @@ -600,10 +711,7 @@ def test_lando_tag(build_payload): "tag_info": { "hg_repo_url": "https://hg/repo", "revision": "abcdef", - "tags": [ - "PRODUCT_99_0_BUILD1", - "PRODUCT_99_0_RELEASE", - ], + "tags": expected, }, }, "scopes": [ @@ -617,9 +725,15 @@ def test_lando_tag(build_payload): def test_lando_version_bump(build_payload): worker = { "lando-repo": "testrepo", - "bump-files": [ - "foo/bar/a.txt", - "another/file.txt", + "actions": [ + { + "version-bump": { + "bump-files": [ + "foo/bar/a.txt", + "another/file.txt", + ], + } + } ], } _, task_def = build_payload("scriptworker-lando", worker=worker) @@ -643,57 +757,100 @@ def test_lando_version_bump(build_payload): } -def test_lando_merge(build_payload): +@pytest.mark.parametrize( + "dry_run", (pytest.param(True, id="dry_run"), pytest.param(False, id="not_dry_run")) +) +def test_lando_merge_bump_esr(build_payload, dry_run): worker = { "lando-repo": "testrepo", - "dontbuild": False, - "l10n-bump-info": [ + "force-dry-run": dry_run, + "actions": [ { - "name": "l10n", - "path": "foo/bar/changesets.json", - "l10n-repo-url": "https://l10n", - "l10n-repo-target-branch": "branchy", - "ignore-config": {"ab": ["foo"]}, - "platform-configs": [ + "esr-bump": { + "fetch-version-from": "version.txt", + "version-files": [ + { + "filename": "version.txt", + "version-bump": "minor", + }, + { + "filename": "other.txt", + "new-suffix": "esr", + "version-bump": "minor", + }, + ], + "to-branch": "to-b", + } + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + expected = { + "payload": { + "actions": ["merge_day"], + "lando_repo": "testrepo", + "merge_info": { + "fetch_version_from": "version.txt", + "version_files": [ { - "platforms": ["p1", "p2"], - "path": "foo/bar/locales", + "filename": "version.txt", + "version_bump": "minor", + }, + { + "filename": "other.txt", + "new_suffix": "esr", + "version_bump": "minor", }, ], + "to_branch": "to-b", + }, + }, + "scopes": [ + "project:releng:lando:action:merge_day", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + if dry_run: + expected["payload"]["dry_run"] = dry_run + + assert task_def == expected + + +def test_lando_merge_bump_main(build_payload): + worker = { + "lando-repo": "testrepo", + "actions": [ + { + "main-bump": { + "fetch-version-from": "version.txt", + "version-files": [ + { + "filename": "version.txt", + "new-suffix": "a1", + "version-bump": "minor", + }, + { + "filename": "other.txt", + "new-suffix": "a1", + "version-bump": "minor", + }, + ], + "to-branch": "to-b", + "replacements": [ + [ + "filename", + "before.{current_weave_version}", + "after.{current_weave_version}", + ] + ], + "regex-replacements": [ + ["filename", "foo [0-9]+.0", "foo {next_major_version}.0"] + ], + "end-tag": "PRODUCT_{major_version}_END", + } } ], - "merge-info": { - "fetch-version-from": "version.txt", - "version-files": [ - { - "filename": "version.txt", - "new-suffix": "", - }, - { - "filename": "other.txt", - "new-suffix": "b1", - }, - ], - "replacements": [ - [ - "something.txt", - "foo", - "bar", - ] - ], - "regex-replacements": [ - [ - "somethingelse.txt", - "thing[0-9]+.0", - "bar{next_major_version}.0", - ] - ], - "merge-old-head": True, - "base-tag": "{major_version}_BASE", - "end-tag": "{major_version}_END", - "from-branch": "from-b", - "to-branch": "to-b", - }, } _, task_def = build_payload("scriptworker-lando", worker=worker) assert task_def == { @@ -705,32 +862,154 @@ def test_lando_merge(build_payload): "version_files": [ { "filename": "version.txt", - "new_suffix": "", + "new_suffix": "a1", + "version_bump": "minor", }, { "filename": "other.txt", - "new_suffix": "b1", + "new_suffix": "a1", + "version_bump": "minor", }, ], "replacements": [ [ - "something.txt", - "foo", - "bar", + "filename", + "before.{current_weave_version}", + "after.{current_weave_version}", ] ], "regex_replacements": [ + ["filename", "foo [0-9]+.0", "foo {next_major_version}.0"] + ], + "end_tag": "PRODUCT_{major_version}_END", + "to_branch": "to-b", + }, + }, + "scopes": [ + "project:releng:lando:action:merge_day", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +def test_lando_merge_early_to_late_beta(build_payload): + worker = { + "lando-repo": "testrepo", + "actions": [ + { + "early-to-late-beta": { + "replacements": [ + [ + "defines", + "foo=1", + "foo=", + ] + ], + "to-branch": "to-b", + } + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["merge_day"], + "lando_repo": "testrepo", + "merge_info": { + "replacements": [ [ - "somethingelse.txt", - "thing[0-9]+.0", - "bar{next_major_version}.0", + "defines", + "foo=1", + "foo=", + ] + ], + "to_branch": "to-b", + }, + }, + "scopes": [ + "project:releng:lando:action:merge_day", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +def test_lando_merge_main_to_beta(build_payload): + worker = { + "lando-repo": "testrepo", + "matrix-rooms": ["!foobar:mozilla.org"], + "actions": [ + { + "uplift": { + "fetch-version-from": "version.txt", + "version-files": [ + { + "filename": "version.txt", + }, + { + "filename": "other.txt", + "new-suffix": "b1", + }, + ], + "replacements": [ + [ + "mozconfig", + "nightly", + "official", + ] + ], + "base-tag": "PRODUCT_{major_version}_BASE", + "end-tag": "PRODUCT_{major_version}_END", + "to-branch": "to-b", + "from-branch": "from-b", + "l10n-bump-info": [ + { + "name": "l10n", + "path": "foo/bar/changesets.json", + "l10n-repo-url": "https://l10n", + "l10n-repo-target-branch": "branchy", + "ignore-config": {"ab": ["foo"]}, + "platform-configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + } + ], + } + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["merge_day"], + "lando_repo": "testrepo", + "merge_info": { + "fetch_version_from": "version.txt", + "version_files": [ + { + "filename": "version.txt", + }, + { + "filename": "other.txt", + "new_suffix": "b1", + }, + ], + "replacements": [ + [ + "mozconfig", + "nightly", + "official", ] ], "merge_old_head": True, - "base_tag": "{major_version}_BASE", - "end_tag": "{major_version}_END", - "from_branch": "from-b", + "base_tag": "PRODUCT_{major_version}_BASE", + "end_tag": "PRODUCT_{major_version}_END", "to_branch": "to-b", + "from_branch": "from-b", "l10n_bump_info": [ { "name": "l10n", @@ -750,9 +1029,89 @@ def test_lando_merge(build_payload): ], }, }, + "routes": [ + "notify.matrix-room.!foobar:mozilla.org.on-pending", + "notify.matrix-room.!foobar:mozilla.org.on-resolved", + ], + "scopes": [ + "project:releng:lando:action:merge_day", + "project:releng:lando:repo:testrepo", + "queue:route:notify.matrix-room.*", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +def test_lando_merge_beta_to_release(build_payload): + worker = { + "lando-repo": "testrepo", + "matrix-rooms": ["!foobar:mozilla.org"], + "actions": [ + { + "uplift": { + "fetch-version-from": "version.txt", + "version-files": [ + { + "filename": "version.txt", + }, + { + "filename": "other.txt", + "new-suffix": "b1", + }, + ], + "replacements": [ + [ + ".arcconfig", + "BETA", + "RELEASE", + ] + ], + "base-tag": "PRODUCT_{major_version}_BASE", + "end-tag": "PRODUCT_{major_version}_END", + "to-branch": "to-b", + "from-branch": "from-b", + } + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["merge_day"], + "lando_repo": "testrepo", + "merge_info": { + "fetch_version_from": "version.txt", + "version_files": [ + { + "filename": "version.txt", + }, + { + "filename": "other.txt", + "new_suffix": "b1", + }, + ], + "replacements": [ + [ + ".arcconfig", + "BETA", + "RELEASE", + ] + ], + "merge_old_head": True, + "base_tag": "PRODUCT_{major_version}_BASE", + "end_tag": "PRODUCT_{major_version}_END", + "to_branch": "to-b", + "from_branch": "from-b", + }, + }, + "routes": [ + "notify.matrix-room.!foobar:mozilla.org.on-pending", + "notify.matrix-room.!foobar:mozilla.org.on-resolved", + ], "scopes": [ "project:releng:lando:action:merge_day", "project:releng:lando:repo:testrepo", + "queue:route:notify.matrix-room.*", ], "tags": {"worker-implementation": "scriptworker"}, }