diff --git a/src/mozilla_taskgraph/worker_types.py b/src/mozilla_taskgraph/worker_types.py index 6ac2c2d..76f9b12 100644 --- a/src/mozilla_taskgraph/worker_types.py +++ b/src/mozilla_taskgraph/worker_types.py @@ -183,3 +183,283 @@ def build_signing_payload(config, task, task_def): ) ) task["attributes"]["release_artifacts"] = sorted(list(artifacts)) + + +l10n_bump_info_schema = [ + { + Required("name"): str, + Required("path"): str, + Required("l10n-repo-url"): str, + Required("l10n-repo-target-branch"): str, + Optional("ignore-config"): object, + Required("platform-configs"): [ + { + Required("platforms"): [str], + Required("path"): str, + Optional("format"): str, + } + ], + } +] + + +def process_l10n_bump_info(info): + l10n_bump_info = [] + l10n_repo_urls = set() + for lbi in info: + l10n_repo_urls.add(lbi["l10n-repo-url"]) + l10n_bump_info.append(dash_to_underscore(lbi)) + + if len(l10n_repo_urls) > 1: + raise Exception( + "Must use the same l10n-repo-url for all files in the same task!" + ) + + return l10n_bump_info + + +@payload_builder( + "scriptworker-lando", + schema={ + Required("lando-repo"): str, + Required("actions"): [ + Any( + { + Required("android-l10n-sync"): { + Required("from-branch"): str, + Required("toml-info"): [ + { + Required("toml-path"): str, + } + ], + }, + }, + { + Required("android-l10n-import"): { + Required("from-repo-url"): str, + Required("toml-info"): [ + { + Required("toml-path"): str, + Required("dest-path"): str, + } + ], + }, + }, + { + Required("l10n-bump"): l10n_bump_info_schema, + }, + { + Required("tag"): { + Required("types"): [Any("buildN", "release")], + Required("hg-repo-url"): str, + } + }, + { + Required("version-bump"): { + Required("bump-files"): [str], + }, + }, + # the remaining action types all end up using the "merge_day" + # landoscript action. however, these are quite varied tasks, + # and separating them out allows us to have stronger schemas. + { + Required("esr-bump"): { + Required("to-branch"): str, + Required("fetch-version-from"): str, + Required("version-files"): [ + { + Required("filename"): str, + Required("version-bump"): str, + Optional("new-suffix"): str, + }, + ], + }, + }, + { + Required("main-bump"): { + Required("to-branch"): str, + Required("fetch-version-from"): str, + Required("version-files"): [ + { + Required("filename"): str, + Required("version-bump"): str, + Optional("new-suffix"): str, + }, + ], + Optional("replacements"): [[str]], + Optional("regex-replacements"): [[str]], + Optional("end-tag"): str, + }, + }, + { + Required("early-to-late-beta"): { + Required("to-branch"): str, + Optional("replacements"): [[str]], + }, + }, + { + Required("uplift"): { + Required("fetch-version-from"): str, + Required("version-files"): [ + { + Required("filename"): str, + Optional("version-bump"): str, + Optional("new-suffix"): str, + }, + ], + Required("from-branch"): str, + Required("to-branch"): str, + Optional("replacements"): [[str]], + Optional("base-tag"): str, + Optional("end-tag"): str, + Optional("l10n-bump-info"): l10n_bump_info_schema, + }, + }, + ) + ], + Optional("ignore-closed-tree"): bool, + Optional("dontbuild"): bool, + Optional("force-dry-run"): bool, + Optional("matrix-rooms"): [str], + }, +) +def build_lando_payload(config, task, task_def): + worker = task["worker"] + release_config = get_release_config(config) + task_def["payload"] = {"actions": [], "lando_repo": worker["lando-repo"]} + task_def["tags"]["worker-implementation"] = "scriptworker" + actions = task_def["payload"]["actions"] + + if worker.get("ignore-closed-tree") is not None: + task_def["payload"]["ignore_closed_tree"] = worker["ignore-closed-tree"] + + if worker.get("dontbuild"): + task_def["payload"]["dontbuild"] = True + + if worker.get("force-dry-run"): + task_def["payload"]["dry_run"] = True + + for action in worker["actions"]: + if info := action.get("android-l10n-import"): + android_l10n_import_info = dash_to_underscore(info) + android_l10n_import_info["toml_info"] = [ + dash_to_underscore(ti) for ti in android_l10n_import_info["toml_info"] + ] + task_def["payload"]["android_l10n_import_info"] = android_l10n_import_info + actions.append("android_l10n_import") + + if info := action.get("android-l10n-sync"): + android_l10n_sync_info = dash_to_underscore(info) + android_l10n_sync_info["toml_info"] = [ + dash_to_underscore(ti) for ti in android_l10n_sync_info["toml_info"] + ] + task_def["payload"]["android_l10n_sync_info"] = android_l10n_sync_info + actions.append("android_l10n_sync") + + if info := action.get("l10n-bump"): + task_def["payload"]["l10n_bump_info"] = process_l10n_bump_info(info) + actions.append("l10n_bump") + + if info := action.get("tag"): + tag_types = info["types"] + tag_names = [] + product = task["shipping-product"].upper() + version = release_config["version"].replace(".", "_") + buildnum = release_config["build_number"] + if "buildN" in tag_types: + tag_names.extend( + [ + f"{product}_{version}_BUILD{buildnum}", + ] + ) + if "release" in tag_types: + tag_names.extend([f"{product}_{version}_RELEASE"]) + tag_info = { + "tags": tag_names, + "hg_repo_url": info["hg-repo-url"], + "revision": config.params[ + "{}head_rev".format(worker.get("repo-param-prefix", "")) + ], + } + task_def["payload"]["tag_info"] = tag_info + actions.append("tag") + + if info := action.get("version-bump"): + bump_info = {} + bump_info["next_version"] = release_config["next_version"] + bump_info["files"] = info["bump-files"] + task_def["payload"]["version_bump_info"] = bump_info + actions.append("version_bump") + + if info := action.get("esr-bump"): + merge_info = dash_to_underscore(info) + merge_info["version_files"] = [ + dash_to_underscore(vf) for vf in info["version-files"] + ] + task_def["payload"]["merge_info"] = merge_info + actions.append("merge_day") + + if info := action.get("main-bump"): + merge_info = dash_to_underscore(info) + + merge_info["version_files"] = [ + dash_to_underscore(vf) for vf in info["version-files"] + ] + task_def["payload"]["merge_info"] = merge_info + actions.append("merge_day") + + if info := action.get("early-to-late-beta"): + task_def["payload"]["merge_info"] = dash_to_underscore(info) + actions.append("merge_day") + + if info := action.get("uplift"): + merge_info = dash_to_underscore(info) + merge_info["merge_old_head"] = True + + merge_info["version_files"] = [ + dash_to_underscore(vf) for vf in info["version-files"] + ] + if lbi := info.get("l10n-bump-info"): + merge_info["l10n_bump_info"] = process_l10n_bump_info(lbi) + + task_def["payload"]["merge_info"] = merge_info + actions.append("merge_day") + + scopes = set(task_def.get("scopes", [])) + scopes.add(f"project:releng:lando:repo:{worker['lando-repo']}") + scopes.update([f"project:releng:lando:action:{action}" for action in actions]) + + for matrix_room in worker.get("matrix-rooms", []): + task_def.setdefault("routes", []) + task_def["routes"].append(f"notify.matrix-room.{matrix_room}.on-pending") + task_def["routes"].append(f"notify.matrix-room.{matrix_room}.on-resolved") + scopes.add("queue:route:notify.matrix-room.*") + + task_def["scopes"] = sorted(scopes) + + +def get_release_config(config): + """Get the build number and version for a release task. + + Currently only applies to beetmover tasks. + + Args: + config (TransformConfig): The configuration for the kind being transformed. + + Returns: + dict: containing both `build_number` and `version`. This can be used to + update `task.payload`. + """ + return { + "version": config.params["version"], + "appVersion": config.params["app_version"], + "next_version": config.params["next_version"], + "build_number": config.params["build_number"], + } + + +def dash_to_underscore(obj): + new_obj = {} + for k, v in obj.items(): + new_obj[k.replace("-", "_")] = v + return new_obj diff --git a/test/conftest.py b/test/conftest.py index a8b670a..f521212 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -74,6 +74,7 @@ def file_url(self, path, pretty=False): def parameters(): return FakeParameters( { + "app_version": "99.0", "base_ref": "123456", "base_repository": "http://example.com/base/repo", "build_date": 0, @@ -85,7 +86,7 @@ def parameters(): "head_tag": "", "level": "1", "moz_build_date": 0, - "next_version": "1.0.1", + "next_version": "100.0", "owner": "some-owner", "project": "some-project", "pushlog_id": 1, @@ -93,7 +94,7 @@ def parameters(): "target_tasks_method": "test_method", "tasks_for": "hg-push", "try_mode": None, - "version": "", + "version": "99.0", } ) diff --git a/test/test_worker_types.py b/test/test_worker_types.py index 7d73cdc..ae8eba7 100644 --- a/test/test_worker_types.py +++ b/test/test_worker_types.py @@ -12,7 +12,7 @@ @pytest.fixture def build_payload(make_graph_config, make_transform_config, parameters): graph_config = make_graph_config( - extra_config={"scriptworker": {"scope-prefix": "foo"}} + extra_config={"scriptworker": {"scope-prefix": "foo"}}, ) def inner( @@ -25,7 +25,7 @@ def inner( worker.setdefault("implementation", name) - task = {"worker": worker} + task = {"worker": worker, "shipping-product": "product"} task_def = {"tags": {}} parameters.update(extra_params) @@ -463,3 +463,655 @@ def test_build_signing_payload_gpg_asc(build_payload): "foo/bar.asc", ] } + + +def test_lando_android_l10n_import(build_payload): + worker = { + "lando-repo": "testrepo", + "actions": [ + { + "android-l10n-import": { + "from-repo-url": "https://from", + "toml-info": [ + { + "toml-path": "foo/bar/l10n.toml", + "dest-path": "foo-bar", + } + ], + }, + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["android_l10n_import"], + "lando_repo": "testrepo", + "android_l10n_import_info": { + "from_repo_url": "https://from", + "toml_info": [ + {"toml_path": "foo/bar/l10n.toml", "dest_path": "foo-bar"}, + ], + }, + }, + "scopes": [ + "project:releng:lando:action:android_l10n_import", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +def test_lando_android_l10n_sync(build_payload): + worker = { + "lando-repo": "testrepo", + "actions": [ + { + "android-l10n-sync": { + "from-branch": "branchy", + "toml-info": [ + { + "toml-path": "foo/bar/l10n.toml", + } + ], + }, + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["android_l10n_sync"], + "lando_repo": "testrepo", + "android_l10n_sync_info": { + "from_branch": "branchy", + "toml_info": [ + { + "toml_path": "foo/bar/l10n.toml", + }, + ], + }, + }, + "scopes": [ + "project:releng:lando:action:android_l10n_sync", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +def test_lando_l10n_bump_mismatched_urls(build_payload): + worker = { + "lando-repo": "testrepo", + "dontbuild": True, + "ignore-closed-tree": True, + "actions": [ + { + "l10n-bump": [ + { + "name": "l10n", + "path": "foo/bar/changesets.json", + "l10n-repo-url": "https://l10n", + "l10n-repo-target-branch": "branchy", + "ignore-config": {"ab": ["foo"]}, + "platform-configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + }, + { + "name": "l10n2", + "path": "foo/baz/changesets.json", + "l10n-repo-url": "https://l10n2", + "l10n-repo-target-branch": "branchy", + "ignore-config": {"ab": ["foo"]}, + "platform-configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + }, + ], + } + ], + } + try: + build_payload("scriptworker-lando", worker=worker) + assert False, "should've raised an exception" + except Exception as e: + assert "Must use the same l10n-repo-url for all files" in e.args[0] + + +def test_lando_l10n_bump(build_payload): + worker = { + "lando-repo": "testrepo", + "dontbuild": True, + "ignore-closed-tree": True, + "actions": [ + { + "l10n-bump": [ + { + "name": "l10n", + "path": "foo/bar/changesets.json", + "l10n-repo-url": "https://l10n", + "l10n-repo-target-branch": "branchy", + "ignore-config": {"ab": ["foo"]}, + "platform-configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + }, + { + "name": "l10n2", + "path": "foo/baz/changesets.json", + "l10n-repo-url": "https://l10n", + "l10n-repo-target-branch": "branchy", + "ignore-config": {"ab": ["foo"]}, + "platform-configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + }, + ], + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["l10n_bump"], + "lando_repo": "testrepo", + "dontbuild": True, + "ignore_closed_tree": True, + "l10n_bump_info": [ + { + "name": "l10n", + "path": "foo/bar/changesets.json", + "l10n_repo_url": "https://l10n", + "l10n_repo_target_branch": "branchy", + "ignore_config": { + "ab": ["foo"], + }, + "platform_configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + }, + { + "name": "l10n2", + "path": "foo/baz/changesets.json", + "l10n_repo_url": "https://l10n", + "l10n_repo_target_branch": "branchy", + "ignore_config": { + "ab": ["foo"], + }, + "platform_configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + }, + ], + }, + "scopes": [ + "project:releng:lando:action:l10n_bump", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +@pytest.mark.parametrize( + "types,expected", + ( + pytest.param( + ["buildN"], + ["PRODUCT_99_0_BUILD1"], + id="buildN", + ), + pytest.param( + ["release"], + ["PRODUCT_99_0_RELEASE"], + id="release", + ), + pytest.param( + ["buildN", "release"], + ["PRODUCT_99_0_BUILD1", "PRODUCT_99_0_RELEASE"], + id="buildN_and_release", + ), + ), +) +def test_lando_tag(build_payload, types, expected): + worker = { + "lando-repo": "testrepo", + "actions": [ + { + "tag": { + "types": types, + "hg-repo-url": "https://hg/repo", + } + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["tag"], + "lando_repo": "testrepo", + "tag_info": { + "hg_repo_url": "https://hg/repo", + "revision": "abcdef", + "tags": expected, + }, + }, + "scopes": [ + "project:releng:lando:action:tag", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +def test_lando_version_bump(build_payload): + worker = { + "lando-repo": "testrepo", + "actions": [ + { + "version-bump": { + "bump-files": [ + "foo/bar/a.txt", + "another/file.txt", + ], + } + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["version_bump"], + "lando_repo": "testrepo", + "version_bump_info": { + "files": [ + "foo/bar/a.txt", + "another/file.txt", + ], + "next_version": "100.0", + }, + }, + "scopes": [ + "project:releng:lando:action:version_bump", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +@pytest.mark.parametrize( + "dry_run", (pytest.param(True, id="dry_run"), pytest.param(False, id="not_dry_run")) +) +def test_lando_merge_bump_esr(build_payload, dry_run): + worker = { + "lando-repo": "testrepo", + "force-dry-run": dry_run, + "actions": [ + { + "esr-bump": { + "fetch-version-from": "version.txt", + "version-files": [ + { + "filename": "version.txt", + "version-bump": "minor", + }, + { + "filename": "other.txt", + "new-suffix": "esr", + "version-bump": "minor", + }, + ], + "to-branch": "to-b", + } + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + expected = { + "payload": { + "actions": ["merge_day"], + "lando_repo": "testrepo", + "merge_info": { + "fetch_version_from": "version.txt", + "version_files": [ + { + "filename": "version.txt", + "version_bump": "minor", + }, + { + "filename": "other.txt", + "new_suffix": "esr", + "version_bump": "minor", + }, + ], + "to_branch": "to-b", + }, + }, + "scopes": [ + "project:releng:lando:action:merge_day", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + if dry_run: + expected["payload"]["dry_run"] = dry_run + + assert task_def == expected + + +def test_lando_merge_bump_main(build_payload): + worker = { + "lando-repo": "testrepo", + "actions": [ + { + "main-bump": { + "fetch-version-from": "version.txt", + "version-files": [ + { + "filename": "version.txt", + "new-suffix": "a1", + "version-bump": "minor", + }, + { + "filename": "other.txt", + "new-suffix": "a1", + "version-bump": "minor", + }, + ], + "to-branch": "to-b", + "replacements": [ + [ + "filename", + "before.{current_weave_version}", + "after.{current_weave_version}", + ] + ], + "regex-replacements": [ + ["filename", "foo [0-9]+.0", "foo {next_major_version}.0"] + ], + "end-tag": "PRODUCT_{major_version}_END", + } + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["merge_day"], + "lando_repo": "testrepo", + "merge_info": { + "fetch_version_from": "version.txt", + "version_files": [ + { + "filename": "version.txt", + "new_suffix": "a1", + "version_bump": "minor", + }, + { + "filename": "other.txt", + "new_suffix": "a1", + "version_bump": "minor", + }, + ], + "replacements": [ + [ + "filename", + "before.{current_weave_version}", + "after.{current_weave_version}", + ] + ], + "regex_replacements": [ + ["filename", "foo [0-9]+.0", "foo {next_major_version}.0"] + ], + "end_tag": "PRODUCT_{major_version}_END", + "to_branch": "to-b", + }, + }, + "scopes": [ + "project:releng:lando:action:merge_day", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +def test_lando_merge_early_to_late_beta(build_payload): + worker = { + "lando-repo": "testrepo", + "actions": [ + { + "early-to-late-beta": { + "replacements": [ + [ + "defines", + "foo=1", + "foo=", + ] + ], + "to-branch": "to-b", + } + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["merge_day"], + "lando_repo": "testrepo", + "merge_info": { + "replacements": [ + [ + "defines", + "foo=1", + "foo=", + ] + ], + "to_branch": "to-b", + }, + }, + "scopes": [ + "project:releng:lando:action:merge_day", + "project:releng:lando:repo:testrepo", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +def test_lando_merge_main_to_beta(build_payload): + worker = { + "lando-repo": "testrepo", + "matrix-rooms": ["!foobar:mozilla.org"], + "actions": [ + { + "uplift": { + "fetch-version-from": "version.txt", + "version-files": [ + { + "filename": "version.txt", + }, + { + "filename": "other.txt", + "new-suffix": "b1", + }, + ], + "replacements": [ + [ + "mozconfig", + "nightly", + "official", + ] + ], + "base-tag": "PRODUCT_{major_version}_BASE", + "end-tag": "PRODUCT_{major_version}_END", + "to-branch": "to-b", + "from-branch": "from-b", + "l10n-bump-info": [ + { + "name": "l10n", + "path": "foo/bar/changesets.json", + "l10n-repo-url": "https://l10n", + "l10n-repo-target-branch": "branchy", + "ignore-config": {"ab": ["foo"]}, + "platform-configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + } + ], + } + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["merge_day"], + "lando_repo": "testrepo", + "merge_info": { + "fetch_version_from": "version.txt", + "version_files": [ + { + "filename": "version.txt", + }, + { + "filename": "other.txt", + "new_suffix": "b1", + }, + ], + "replacements": [ + [ + "mozconfig", + "nightly", + "official", + ] + ], + "merge_old_head": True, + "base_tag": "PRODUCT_{major_version}_BASE", + "end_tag": "PRODUCT_{major_version}_END", + "to_branch": "to-b", + "from_branch": "from-b", + "l10n_bump_info": [ + { + "name": "l10n", + "path": "foo/bar/changesets.json", + "l10n_repo_url": "https://l10n", + "l10n_repo_target_branch": "branchy", + "ignore_config": { + "ab": ["foo"], + }, + "platform_configs": [ + { + "platforms": ["p1", "p2"], + "path": "foo/bar/locales", + }, + ], + } + ], + }, + }, + "routes": [ + "notify.matrix-room.!foobar:mozilla.org.on-pending", + "notify.matrix-room.!foobar:mozilla.org.on-resolved", + ], + "scopes": [ + "project:releng:lando:action:merge_day", + "project:releng:lando:repo:testrepo", + "queue:route:notify.matrix-room.*", + ], + "tags": {"worker-implementation": "scriptworker"}, + } + + +def test_lando_merge_beta_to_release(build_payload): + worker = { + "lando-repo": "testrepo", + "matrix-rooms": ["!foobar:mozilla.org"], + "actions": [ + { + "uplift": { + "fetch-version-from": "version.txt", + "version-files": [ + { + "filename": "version.txt", + }, + { + "filename": "other.txt", + "new-suffix": "b1", + }, + ], + "replacements": [ + [ + ".arcconfig", + "BETA", + "RELEASE", + ] + ], + "base-tag": "PRODUCT_{major_version}_BASE", + "end-tag": "PRODUCT_{major_version}_END", + "to-branch": "to-b", + "from-branch": "from-b", + } + } + ], + } + _, task_def = build_payload("scriptworker-lando", worker=worker) + assert task_def == { + "payload": { + "actions": ["merge_day"], + "lando_repo": "testrepo", + "merge_info": { + "fetch_version_from": "version.txt", + "version_files": [ + { + "filename": "version.txt", + }, + { + "filename": "other.txt", + "new_suffix": "b1", + }, + ], + "replacements": [ + [ + ".arcconfig", + "BETA", + "RELEASE", + ] + ], + "merge_old_head": True, + "base_tag": "PRODUCT_{major_version}_BASE", + "end_tag": "PRODUCT_{major_version}_END", + "to_branch": "to-b", + "from_branch": "from-b", + }, + }, + "routes": [ + "notify.matrix-room.!foobar:mozilla.org.on-pending", + "notify.matrix-room.!foobar:mozilla.org.on-resolved", + ], + "scopes": [ + "project:releng:lando:action:merge_day", + "project:releng:lando:repo:testrepo", + "queue:route:notify.matrix-room.*", + ], + "tags": {"worker-implementation": "scriptworker"}, + }