diff --git a/ansible/collections/ansible_collections/nhsd/apigee/plugins/action/add_policy_to_pre_flow.py b/ansible/collections/ansible_collections/nhsd/apigee/plugins/action/add_policy_to_pre_flow.py index 46b82086..fc0de294 100644 --- a/ansible/collections/ansible_collections/nhsd/apigee/plugins/action/add_policy_to_pre_flow.py +++ b/ansible/collections/ansible_collections/nhsd/apigee/plugins/action/add_policy_to_pre_flow.py @@ -32,7 +32,7 @@ def run(self, tmp=None, task_vars=None): proxies_dir = args.dist_dir.joinpath( "proxies", args.proxy_dir, "apiproxy/proxies" ) - proxies_files = [f for f in proxies_dir.glob("*.xml")] + proxies_files = list(proxies_dir.glob("*.xml")) if len(proxies_files) != 1: return { @@ -57,13 +57,12 @@ def run(self, tmp=None, task_vars=None): name = step.find("Name") if name.text == args.policy_name: return {"changed": False} - break result = {"changed": True} if diff_mode: result["diff"] = { "before": etree.tostring(tree, pretty_print=True).decode(), - "before_header": str(proxies_file) + "before_header": str(proxies_file), } step = etree.Element("Step") @@ -76,8 +75,8 @@ def run(self, tmp=None, task_vars=None): result["diff"].update( { "after": etree.tostring(tree, pretty_print=True).decode(), - "after_header": str(proxies_file) - } + "after_header": str(proxies_file), + } ) if check_mode or result["changed"] is False: diff --git a/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/apigee_action.py b/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/apigee_action.py index 428b4372..de007242 100644 --- a/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/apigee_action.py +++ b/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/apigee_action.py @@ -4,10 +4,10 @@ class ApigeeAction(ansible.plugins.action.ActionBase): - def validate_args(self, Validator: pydantic.BaseModel): + def validate_args(self, validator: type[pydantic.BaseModel]): """Returns two-length tuple of validated_args and errors dicts.""" try: - args = Validator(**self._task.args) + args = validator(**self._task.args) return args, {} except pydantic.ValidationError as e: return ( diff --git a/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/ansible/validate_manifest.py b/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/ansible/validate_manifest.py index a8777fca..93cee623 100644 --- a/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/ansible/validate_manifest.py +++ b/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/ansible/validate_manifest.py @@ -53,14 +53,14 @@ def check_service_name(cls, service_name, values): def prepend_dist_dir_to_spec_paths(cls, manifest, values): dist_dir = values.get("dist_dir") print(dist_dir) - if not dist_dir: - return manifest - apigee = manifest["apigee"] - for env_dict in apigee["environments"]: - for spec_dict in env_dict["specs"]: - path = spec_dict.get("path") - if path is not None: - spec_dict["path"] = os.path.join(dist_dir, path) + if dist_dir: + apigee = manifest["apigee"] + for env_dict in apigee["environments"]: + for spec_dict in env_dict["specs"]: + path = spec_dict.get("path") + if path is not None: + spec_dict["path"] = os.path.join(dist_dir, path) + return manifest @pydantic.validator("manifest") diff --git a/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/apigee/product.py b/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/apigee/product.py index da473499..905c6dc7 100644 --- a/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/apigee/product.py +++ b/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/apigee/product.py @@ -1,5 +1,5 @@ import json -from typing import Union, Literal, List, Dict, Any, Type +from typing import Optional, Union, Literal, List, Dict, Any, Type from typing_extensions import Annotated from pydantic import ( Field, @@ -110,19 +110,22 @@ def _literal_name(class_): + ")$)" ) + class ApigeeProductAttributeOther(BaseModel): name: constr(regex=PRODUCT_ATTRIBUTE_REGEX) value: str + ApigeeProductAttributeSpecial = Annotated[ - Union [ + Union[ ApigeeProductAttributeAccess, ApigeeProductAttributeRateLimit, ApigeeProductAttributeRateLimiting, ], - Field(discriminator="name") + Field(discriminator="name"), ] + def _count_cls(items: List[Any], cls: Type): return sum(isinstance(item, cls) for item in items) @@ -130,9 +133,11 @@ def _count_cls(items: List[Any], cls: Type): class ApigeeProduct(BaseModel): name: str approvalType: Literal["auto", "manual"] = "manual" - attributes: List[Union[ApigeeProductAttributeSpecial, ApigeeProductAttributeOther]] = [{"name": "access", "value": "private"}] - description: str = None - displayName: str = None + attributes: List[ + Union[ApigeeProductAttributeSpecial, ApigeeProductAttributeOther] + ] = [{"name": "access", "value": "private"}] + description: Optional[str] = None + displayName: Optional[str] = None # Note: This value is manually inserted by apigee_environment # object that contains this product. So if you do not provide a diff --git a/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/apigee/rate_limiting_config.py b/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/apigee/rate_limiting_config.py index 404af292..967c412d 100644 --- a/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/apigee/rate_limiting_config.py +++ b/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/apigee/rate_limiting_config.py @@ -2,13 +2,13 @@ Pydantic class for the rateliming config JSON, attached to products and apps to control the ApplyRateLimiting shared flow. """ -from typing import Literal + +from typing import Literal, Optional from pydantic import BaseModel, conint, constr, Extra class ExcludeNoneModel(BaseModel): - """ Providing default values for ratelimiting here would mean that changing defaults required a redeploy for all proxies. @@ -21,26 +21,27 @@ class ExcludeNoneModel(BaseModel): update the defaults for everyone by just by updating the shared flow. """ + def dict(self, **kwargs): kwargs["exclude_none"] = True return super().dict(**kwargs) class Config: - extra=Extra.forbid + extra = Extra.forbid class QuotaConfig(ExcludeNoneModel): - enabled: bool = None + enabled: Optional[bool] = None interval: conint(gt=0) = None limit: conint(gt=0) = None timeunit: Literal["minute", "hour", "day", "week", "month"] = None class SpikeArrestConfig(ExcludeNoneModel): - enabled: bool = None - ratelimit: constr(regex=r"^[1-9][0-9]*(ps|pm)$") = None + enabled: Optional[bool] = None + ratelimit: Optional[constr(regex=r"^[1-9][0-9]*(ps|pm)$")] = None class RateLimitingConfig(ExcludeNoneModel): - quota: QuotaConfig = None - spikeArrest: SpikeArrestConfig = None + quota: Optional[QuotaConfig] = None + spikeArrest: Optional[SpikeArrestConfig] = None diff --git a/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/manifest/meta.py b/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/manifest/meta.py index d66e8931..d6210af3 100644 --- a/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/manifest/meta.py +++ b/ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/models/manifest/meta.py @@ -9,7 +9,6 @@ _REGISTRY_DATA = {} - class ManifestMetaApi(pydantic.BaseModel): name: pydantic.constr(regex=r"^[a-z][a-z0-9]*(-[a-z0-9]+)*$") id: typing.Optional[pydantic.UUID4] = pydantic.Field( @@ -29,7 +28,7 @@ def dict(self, **kwargs): { "guid": str(native["guid"]), "spec_guids": [str(guid) for guid in spec_guids], - } + } ) return native @@ -58,7 +57,9 @@ def validate_guid(cls, guid, values): guid = _REGISTRY_DATA[name]["guid"] registered_guid = _REGISTRY_DATA[name]["guid"] if str(guid) != registered_guid: - raise ValueError(f"Supplied guid {guid} does not match registered guid {registered_guid}") + raise ValueError( + f"Supplied guid {guid} does not match registered guid {registered_guid}" + ) return guid @pydantic.validator("spec_guids") @@ -78,11 +79,12 @@ def validate_spec_guids(cls, spec_guids, values): if str(spec_guid) not in registered_spec_guids: invalid.append(str(spec_guid)) if len(invalid) > 0: - raise ValueError(f"Supplied spec_guids {invalid} do not match registered spec_guids {registered_spec_guids}") + raise ValueError( + f"Supplied spec_guids {invalid} do not match registered spec_guids {registered_spec_guids}" + ) return spec_guids - class ManifestMeta(pydantic.BaseModel): schema_version: pydantic.constr(regex=r"[1-9][0-9]*(\.[0-9]+){0,2}") api: ManifestMetaApi @@ -91,7 +93,7 @@ class ManifestMeta(pydantic.BaseModel): def validate_schema_version(cls, schema_version): semantic_parts = schema_version.split(".") - MAJOR, MINOR, PATCH = [int(x) for x in SCHEMA_VERSION.split(".")] + MAJOR, _, _ = [int(x) for x in SCHEMA_VERSION.split(".")] major = int(semantic_parts[0]) # Checking against minor/patch would not allow to us deploy diff --git a/ansible/collections/ansible_collections/nhsd/apigee/scripts/update_schema.py b/ansible/collections/ansible_collections/nhsd/apigee/scripts/update_schema.py index e2881726..9ed1d3ef 100644 --- a/ansible/collections/ansible_collections/nhsd/apigee/scripts/update_schema.py +++ b/ansible/collections/ansible_collections/nhsd/apigee/scripts/update_schema.py @@ -8,10 +8,11 @@ import typing from ansible_collections.nhsd.apigee.plugins.module_utils.models.manifest import meta -from ansible_collections.nhsd.apigee.plugins.module_utils.models.manifest.manifest import Manifest +from ansible_collections.nhsd.apigee.plugins.module_utils.models.manifest.manifest import ( + Manifest, +) - -SCHEMA_VERSION_REGEX = re.compile(r"^([1-9][0-9]*)\.([0-9]+)\.([0-9]+)$") +SCHEMA_VERSION_REGEX = re.compile(r"^([1-9]\d*)\.(\d+)\.(\d+)$") class SchemaString: @@ -23,7 +24,9 @@ def __init__(self, schema_or_major: typing.Union[str, int], minor=None, patch=No match = re.match(SCHEMA_VERSION_REGEX, schema_version) if not match: raise ValueError(f"Invalid SchemaString {schema_version}") - self.major, self.minor, self.patch = [int(match.group(x)) for x in range(1, 4)] + self.major, self.minor, self.patch = [ + int(match.group(x)) for x in range(1, 4) + ] elif isinstance(schema_or_major, int): if not isinstance(minor, int) and isinstance(patch, int): raise ValueError("SchemaString.__init__ requires major, minor, patch") @@ -47,41 +50,51 @@ def __lt__(self, other): return ( self.major < other.major or (self.major == other.major and self.minor < other.minor) - or (self.major == other.major and self.minor == other.minor and self.patch < other.patch) + or ( + self.major == other.major + and self.minor == other.minor + and self.patch < other.patch + ) ) def __le__(self, other): return self < other or self == other def __gt__(self, other): - return not self <= other + return (self.major, self.minor, self.patch) > ( + other.major, + other.minor, + other.patch, + ) def __ge__(self, other): - return not self < other + return (self.major, self.minor, self.patch) >= ( + other.major, + other.minor, + other.patch, + ) def valid_increments(self): return [ SchemaString(self.major + 1, 0, 0), SchemaString(self.major, self.minor + 1, 0), - SchemaString(self.major, self.minor, self.patch + 1) + SchemaString(self.major, self.minor, self.patch + 1), ] def main(): - """ - - """ + """ """ UPDATED_SCHEMA_VERSION = SchemaString(meta.SCHEMA_VERSION) script_dir = pathlib.Path(__file__).parent - relative_schema_dir = script_dir.joinpath(f"../tests/unit/plugins/module_utils/models/manifest/schema_versions/") - - # last_schema_file_name = pathlib.Path(f"v{}.json") + relative_schema_dir = script_dir.joinpath( + "../tests/unit/plugins/module_utils/models/manifest/schema_versions/" + ) schema_dir_glob = relative_schema_dir.glob("v*.json") SCHEMA_VERSION = SchemaString("1.0.0") - SCHEMA_FILE_NAME_PATTERN = re.compile(r"v([1-9][0-9]*\.[0-9]+\.[0-9]+)\.json$") + SCHEMA_FILE_NAME_PATTERN = re.compile(r"v([1-9]\d*\.\d+\.\d+)\.json$") for schema_file in schema_dir_glob: match = re.match(SCHEMA_FILE_NAME_PATTERN, schema_file.name) schema_version = SchemaString(match.group(1)) @@ -100,17 +113,22 @@ def main(): fromfile=str(SCHEMA_VERSION), tofile=str(UPDATED_SCHEMA_VERSION), ) - deltas = [delta for delta in deltas] + deltas = list(deltas) if not deltas: - raise ValueError(f"No difference between proposed {UPDATED_SCHEMA_VERSION} schema and current {meta.SCHEMA_VERSION}") + raise ValueError( + f"No difference between proposed {UPDATED_SCHEMA_VERSION} schema and current {meta.SCHEMA_VERSION}" + ) if UPDATED_SCHEMA_VERSION not in SCHEMA_VERSION.valid_increments(): - raise ValueError(f"""{UPDATED_SCHEMA_VERSION} is invalid increment after current {SCHEMA_VERSION}. + raise ValueError( + f"""{UPDATED_SCHEMA_VERSION} is invalid increment after current {SCHEMA_VERSION}. Please increment major, minor or patch integer, e.g: -""" + f"\n".join(str(x) for x in SCHEMA_VERSION.valid_increments())) +""" + + "\n".join(str(x) for x in SCHEMA_VERSION.valid_increments()) + ) - print("-"*50) + print("-" * 50) for delta in deltas: if delta.startswith("+"): col = colorama.Fore.GREEN @@ -122,7 +140,7 @@ def main(): print(colorama.Fore.RESET) _input = None - print("-"*50) + print("-" * 50) print("Confirm spec changes? ", end="") while _input not in ["y", "n"]: if _input is not None: @@ -139,8 +157,7 @@ def main(): with open(new_schema_file, "w") as f: f.write(new_schema) - print( - f"""Wrote {new_schema_file}" + print(f"""Wrote {new_schema_file}" Validate this file by executing $ make test in {script_dir.parent}""") diff --git a/ansible/collections/ansible_collections/nhsd/apigee/tests/unit/plugins/module_utils/models/manifest/test_product.py b/ansible/collections/ansible_collections/nhsd/apigee/tests/unit/plugins/module_utils/models/manifest/test_product.py index 17ad7338..b5803472 100644 --- a/ansible/collections/ansible_collections/nhsd/apigee/tests/unit/plugins/module_utils/models/manifest/test_product.py +++ b/ansible/collections/ansible_collections/nhsd/apigee/tests/unit/plugins/module_utils/models/manifest/test_product.py @@ -30,7 +30,7 @@ def _make_product_dict(name, environment="internal-dev", approval_type="auto"): @pytest.mark.parametrize( - "env,initial_approvalType,final_approvalType", + "env,initial_approvaltype,final_approvaltype", [ ("prod", "auto", "manual"), ("prod", "manual", "manual"), @@ -38,18 +38,18 @@ def _make_product_dict(name, environment="internal-dev", approval_type="auto"): ("int", "manual", "manual"), ], ) -def test_prod_cannot_have_auto_approvalType_on_products( - env, initial_approvalType, final_approvalType +def test_prod_cannot_have_auto_approvaltype_on_products( + env, initial_approvaltype, final_approvaltype ): product_dict = _make_product_dict( - "test-service", environment=env, approval_type=initial_approvalType + "test-service", environment=env, approval_type=initial_approvaltype ) product = ApigeeProduct(**product_dict) - assert product.approvalType == final_approvalType + assert product.approvalType == final_approvaltype @pytest.mark.parametrize( - "name,env,initial_approvalType,final_approvalType", + "name,env,initial_approvaltype,final_approvaltype", [ ("canary-api", "prod", "auto", "auto"), ("canary-api", "prod", "manual", "manual"), @@ -58,13 +58,13 @@ def test_prod_cannot_have_auto_approvalType_on_products( ], ) def test_manual_approval_exception_list_on_prod( - name, env, initial_approvalType, final_approvalType + name, env, initial_approvaltype, final_approvaltype ): product_dict = _make_product_dict( - name, environment=env, approval_type=initial_approvalType + name, environment=env, approval_type=initial_approvaltype ) product = ApigeeProduct(**product_dict) - assert product.approvalType == final_approvalType + assert product.approvalType == final_approvaltype def test_ratelimiting_product_attribute_initialized_with_dict_or_string(): @@ -75,15 +75,13 @@ def test_ratelimiting_product_attribute_initialized_with_dict_or_string(): "spikeArrest": {"ratelimit": "30000pm"}, # 5000 tps } attr_dict = {product_dict["name"]: ratelimiting_dict} - + # Init with dict-like - product_dict["attributes"].append( - {"name": "ratelimiting", "value": attr_dict} - ) + product_dict["attributes"].append({"name": "ratelimiting", "value": attr_dict}) product1 = ApigeeProduct(**product_dict) # Assert ratelimiting attribute gets serialized to a string - assert type(product1.attributes[-1].value) == str + assert isinstance(product1.attributes[-1].value, str) # Init with attribute values already a string attr_str = json.dumps(attr_dict) diff --git a/ansible/filter_plugins/apigee_helpers.py b/ansible/filter_plugins/apigee_helpers.py index c44117a8..993c1a2f 100644 --- a/ansible/filter_plugins/apigee_helpers.py +++ b/ansible/filter_plugins/apigee_helpers.py @@ -4,20 +4,20 @@ def apigee_apps_to_product_map(apps_list: List[dict], product_filter: str = None): - result = dict() + result = {} not_matched = [] for app in apps_list: - credentials = app.get('credentials', []) + credentials = app.get("credentials", []) for cred in credentials: - api_products = cred.get('apiProducts', []) + api_products = cred.get("apiProducts", []) for product in api_products: - api_product = product['apiproduct'] + api_product = product["apiproduct"] if product_filter and not re.match(product_filter, api_product): not_matched.append(api_product) continue @@ -25,8 +25,8 @@ def apigee_apps_to_product_map(apps_list: List[dict], product_filter: str = None if api_product not in result: result[api_product] = [] - company_exists = "companyName" in app.keys() - developer_exists = "developerId" in app.keys() + company_exists = "companyName" in app + developer_exists = "developerId" in app if developer_exists and not company_exists: owner = app["developerId"] elif company_exists and not developer_exists: @@ -35,36 +35,38 @@ def apigee_apps_to_product_map(apps_list: List[dict], product_filter: str = None raise RuntimeError(f"Invalid owner for app {app['appId']}") result[api_product].append( - dict( - appId=app["appId"], - appName=app["name"], - owner=owner, - ownerEndpoint="companies" if company_exists else "developers", - consumerKey=cred["consumerKey"], - apiproduct=api_product - ) + { + "appId": app["appId"], + "appName": app["name"], + "owner": owner, + "ownerEndpoint": ( + "companies" if company_exists else "developers" + ), + "consumerKey": cred["consumerKey"], + "apiproduct": api_product, + } ) for product in sorted(set(not_matched)): - print(f'did not match: {product}') + print(f"did not match: {product}") return result def product_app_mapping_to_owner_display(dev_id_to_email: dict, product_app: dict): - if product_app['ownerEndpoint'] == 'developers': - return dev_id_to_email[product_app['owner']] + if product_app["ownerEndpoint"] == "developers": + return dev_id_to_email[product_app["owner"]] else: - return product_app['owner'] + return product_app["owner"] def apigee_products_to_api_map(products: List[dict], proxy_filter: str = None): - result = dict() + result = {} not_matched = [] for product in products: - proxies = product.get('proxies', []) + proxies = product.get("proxies", []) for proxy in proxies: @@ -75,36 +77,41 @@ def apigee_products_to_api_map(products: List[dict], proxy_filter: str = None): if proxy not in result: result[proxy] = [] - result[proxy].append(product['name']) + result[proxy].append(product["name"]) for proxy in sorted(set(not_matched)): - print(f'did not match: {proxy}') + print(f"did not match: {proxy}") return result def apigee_remove_proxy_from_product(product: dict, proxy_to_remove): - product["proxies"] = [ - p for p in product.get("proxies", []) - if p != proxy_to_remove - ] + product["proxies"] = [p for p in product.get("proxies", []) if p != proxy_to_remove] return product def apigee_team_to_admin(team): - return next((attr.get('value') for attr in team['attributes'] if attr['name'] == 'ADMIN_EMAIL'), None) + return next( + ( + attr.get("value") + for attr in team["attributes"] + if attr["name"] == "ADMIN_EMAIL" + ), + None, + ) def apigee_teams_map(team_members: List[dict], teams: List[dict]): from collections import defaultdict + joined = defaultdict(dict) for item in team_members + teams: - joined[item['name']].update(item) + joined[item["name"]].update(item) full_teams = list(joined.values()) return { - team['name']: { + team["name"]: { "contact": apigee_team_to_admin(team), - "members": team['members'] + "members": team["members"], } for team in full_teams } @@ -116,19 +123,20 @@ def apigee_teams_to_point_of_contact(teams: List[dict]): def apigee_teams_to_members(teams: List[dict]): - return { - team['owner']: team.get('members', []) for team in teams - } + return {team["owner"]: team.get("members", []) for team in teams} def apigee_product_developers( - product_app_map: dict, dev_id_to_email: dict, teams_map: dict, product_filter: str = None + product_app_map: dict, + dev_id_to_email: dict, + teams_map: dict, + product_filter: str = None, ): if not product_app_map: - raise ValueError('product_app_map not set') + raise ValueError("product_app_map not set") if not dev_id_to_email: - raise ValueError('dev_id_to_email not set') + raise ValueError("dev_id_to_email not set") teams_map = teams_map or {} result = {} @@ -139,22 +147,18 @@ def apigee_product_developers( result[product_name] = [] for app in apps: - entry = { - "app": app["appName"] - } + entry = {"app": app["appName"]} - if app['ownerEndpoint'] == 'companies': + if app["ownerEndpoint"] == "companies": team = teams_map.get(app.get("owner"), {}) - entry['developer'] = team['contact'] - entry['team'] = team['members'] + entry["developer"] = team["contact"] + entry["team"] = team["members"] else: - entry['developer'] = dev_id_to_email[app["owner"]] + entry["developer"] = dev_id_to_email[app["owner"]] result[product_name].append(entry) - result = { - key: sorted(apps, key=lambda x: x['app']) for key, apps in result.items() - } + result = {key: sorted(apps, key=lambda x: x["app"]) for key, apps in result.items()} return result @@ -165,12 +169,12 @@ class FilterModule: def filters(): return { # jinja2 overrides - 'apigee_apps_to_product_map': apigee_apps_to_product_map, - 'apigee_products_to_api_map': apigee_products_to_api_map, - 'apigee_remove_proxy_from_product': apigee_remove_proxy_from_product, - 'apigee_teams_to_point_of_contact': apigee_teams_to_point_of_contact, - 'apigee_teams_to_members': apigee_teams_to_members, - 'apigee_teams_map': apigee_teams_map, - 'apigee_product_developers': apigee_product_developers, - 'product_app_mapping_to_owner_display': product_app_mapping_to_owner_display + "apigee_apps_to_product_map": apigee_apps_to_product_map, + "apigee_products_to_api_map": apigee_products_to_api_map, + "apigee_remove_proxy_from_product": apigee_remove_proxy_from_product, + "apigee_teams_to_point_of_contact": apigee_teams_to_point_of_contact, + "apigee_teams_to_members": apigee_teams_to_members, + "apigee_teams_map": apigee_teams_map, + "apigee_product_developers": apigee_product_developers, + "product_app_mapping_to_owner_display": product_app_mapping_to_owner_display, } diff --git a/ansible/filter_plugins/ecs_service.py b/ansible/filter_plugins/ecs_service.py index 13fb3e5f..b1f93676 100644 --- a/ansible/filter_plugins/ecs_service.py +++ b/ansible/filter_plugins/ecs_service.py @@ -2,64 +2,70 @@ from ansible.plugins.filter.core import combine from collections import Counter -_container_defaults = dict( - cpu=0, - port=9000, - protocol="TCP", - environment=[], - mountPoints=[], - secrets=[], - expose=True, - essential=True, - health_check=dict( - matcher="200", - path="/" - ), - lb_protocol="HTTP" - -) +_container_defaults = { + "cpu": 0, + "port": 9000, + "protocol": "TCP", + "environment": [], + "mountPoints": [], + "secrets": [], + "expose": True, + "essential": True, + "health_check": {"matcher": "200", "path": "/"}, + "lb_protocol": "HTTP", +} def as_ecs_service(docker_service): - ecs_service = [combine(_container_defaults, container) for container in docker_service] + ecs_service = [ + combine(_container_defaults, container) for container in docker_service + ] if sum(1 for _ in ecs_service if _["expose"]) != 1: - raise AnsibleFilterError("you must have exactly one exposed service, set 'expose: false' on others") + raise AnsibleFilterError( + "you must have exactly one exposed service, set 'expose: false' on others" + ) - port_counts = Counter(container['port'] for container in ecs_service) + port_counts = Counter(container["port"] for container in ecs_service) if port_counts.get(9000, 0) != 1: - raise AnsibleFilterError("ecs_service should expose exactly one container with hostPort 9000") + raise AnsibleFilterError( + "ecs_service should expose exactly one container with hostPort 9000" + ) for port, count in port_counts.items(): if count > 1: - raise AnsibleFilterError(f"ecs_service should not have multiple containers on the same port ({port})") + raise AnsibleFilterError( + f"ecs_service should not have multiple containers on the same port ({port})" + ) - if sum(1 for _ in ecs_service if _.get('name') is None): + if sum(1 for _ in ecs_service if _.get("name") is None): raise AnsibleFilterError("all ecs containers should hava name") - name_counts = Counter(container['name'] for container in ecs_service) + name_counts = Counter(container["name"] for container in ecs_service) for name, count in name_counts.items(): if count > 1: - raise AnsibleFilterError(f"ecs_service should not have multiple containers with the same name ({name})") + raise AnsibleFilterError( + f"ecs_service should not have multiple containers with the same name ({name})" + ) return ecs_service -_app_scaling_defaults = dict( - service_metric='', - target_value=0, - scale_in_cooldown=300, - scale_out_cooldown=60, - enabled=False -) +_app_scaling_defaults = { + "service_metric": "", + "target_value": 0, + "scale_in_cooldown": 300, + "scale_out_cooldown": 60, + "enabled": False, +} _ecs_service_metrics = ( - 'ALBRequestCountPerTarget', - 'ECSServiceAverageCPUUtilization', - 'ECSServiceAverageMemoryUtilization' + "ALBRequestCountPerTarget", + "ECSServiceAverageCPUUtilization", + "ECSServiceAverageMemoryUtilization", ) @@ -69,25 +75,31 @@ def as_ecs_autoscaling(docker_service_autoscaling): return _app_scaling_defaults autoscaling_policy = combine(_app_scaling_defaults, docker_service_autoscaling) - service_metric = autoscaling_policy.get('service_metric') + service_metric = autoscaling_policy.get("service_metric") if service_metric not in _ecs_service_metrics: - raise AnsibleFilterError(f"scaling policy must have a service_metric in set {_ecs_service_metrics}") + raise AnsibleFilterError( + f"scaling policy must have a service_metric in set {_ecs_service_metrics}" + ) - target_value = str(autoscaling_policy.get('target_value', '')) + target_value = str(autoscaling_policy.get("target_value", "")) if not target_value.isdigit(): - raise AnsibleFilterError(f"scaling policy must have a target_value as a positive integer") + raise AnsibleFilterError( + "scaling policy must have a target_value as a positive integer" + ) target_value = int(target_value) - autoscaling_policy['target_value'] = target_value - autoscaling_policy['enabled'] = True + autoscaling_policy["target_value"] = target_value + autoscaling_policy["enabled"] = True if target_value < 10: - raise AnsibleFilterError(f"scaling policy target_value should be >= 10") + raise AnsibleFilterError("scaling policy target_value should be >= 10") - if service_metric == 'ALBRequestCountPerTarget': + if service_metric == "ALBRequestCountPerTarget": return autoscaling_policy if target_value > 90: - raise AnsibleFilterError(f"Utilization scaling policy target_value should be between 10 and 90") + raise AnsibleFilterError( + "Utilization scaling policy target_value should be between 10 and 90" + ) return autoscaling_policy @@ -98,7 +110,6 @@ class FilterModule: def filters(): return { # jinja2 overrides - 'as_ecs_service': as_ecs_service, - 'as_ecs_autoscaling': as_ecs_autoscaling + "as_ecs_service": as_ecs_service, + "as_ecs_autoscaling": as_ecs_autoscaling, } - diff --git a/ansible/library/filter_api_products.py b/ansible/library/filter_api_products.py index 9d151e1f..bc24f9bc 100644 --- a/ansible/library/filter_api_products.py +++ b/ansible/library/filter_api_products.py @@ -1,28 +1,32 @@ #!/usr/bin/python -from ansible.module_utils.basic import * +from ansible.module_utils.basic import AnsibleModule import json + def main(): fields = { "products": {"default": True, "type": "str"}, - "environment": {"default": True, "type": "str"} + "environment": {"default": True, "type": "str"}, } module = AnsibleModule(argument_spec=fields) - products_string = module.params['products'] - json_acceptable_string = products_string.replace("'", "\"") + products_string = module.params["products"] + json_acceptable_string = products_string.replace("'", '"') products = json.loads(json_acceptable_string) current_env = module.params["environment"] for product in products: - filtered_product = list(filter(lambda env: env == current_env, product["environments"])) + filtered_product = list( + filter(lambda env: env == current_env, product["environments"]) + ) product["environments"] = filtered_product module.params.update({"products": products}) - + module.exit_json(changed=True, meta=module.params) -if __name__ == '__main__': - main() \ No newline at end of file + +if __name__ == "__main__": + main() diff --git a/azure/build-and-push-ecs-proxies.yml b/azure/build-and-push-ecs-proxies.yml index 51f70518..a6047e43 100644 --- a/azure/build-and-push-ecs-proxies.yml +++ b/azure/build-and-push-ecs-proxies.yml @@ -1,17 +1,21 @@ parameters: - - name: 'container_vars' + - name: "container_vars" type: string - default: 'ecs-proxies-containers.yml' - - name: 'env_vars_dir' + default: "ecs-proxies-containers.yml" + - name: "env_vars_dir" type: string - default: './' - - name: 'utils_dir' + default: "./" + - name: "utils_dir" type: string - default: 'utils' + default: "utils" steps: - bash: | - source "${{ parameters.env_vars_dir }}/.build_env_vars" - CONTAINER_VARS_FILE="$(realpath ${{ parameters.container_vars }})" \ - make --no-print-directory -C ${{ parameters.utils_dir }}/ansible build-ecs-proxies + source "$env_vars_dir/.build_env_vars" + CONTAINER_VARS_FILE="$(realpath "$container_vars")" \ + make --no-print-directory -C "$utils_dir/ansible" build-ecs-proxies + env: + env_vars_dir: ${{ parameters.env_vars_dir }} + container_vars: ${{ parameters.container_vars }} + utils_dir: ${{ parameters.utils_dir }} displayName: "Build and push ECS proxies" diff --git a/azure/build-prereqs.yml b/azure/build-prereqs.yml index 11374528..79dbd93f 100644 --- a/azure/build-prereqs.yml +++ b/azure/build-prereqs.yml @@ -1,7 +1,7 @@ parameters: - - name: 'utils_dir' + - name: "utils_dir" type: string - default: 'utils' + default: "utils" steps: - bash: | @@ -13,21 +13,13 @@ steps: echo "##vso[task.setvariable variable=PY_VER]$PATCH" echo "Resolved latest python version: $PATCH" echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/" - displayName: 'Query and set python tool cache path' - + displayName: "Query and set python tool cache path" - task: UsePythonVersion@0 name: UsePy - displayName: 'Use Python 3.13' + displayName: "Use Python 3.13" inputs: - versionSpec: '$(PY_VER)' - - # - bash: | - # echo "Checking the python version in use to set LD_LIBRARY_PATH" - # echo "Python location: $(UsePy.pythonLocation)" - # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" - # displayName: 'Set LD_LIBRARY_PATH' - + versionSpec: "$(PY_VER)" - bash: | tfenv use 0.14.6 @@ -41,6 +33,8 @@ steps: if ! hash poetry 2>/dev/null; then pip3 install poetry fi - cd ${{ parameters.utils_dir }} + cd "$utils_dir" make install + env: + utils_dir: ${{ parameters.utils_dir }} displayName: "Install Utils dependencies" diff --git a/azure/cleanup-ecs-pr-proxies.yml b/azure/cleanup-ecs-pr-proxies.yml index c6479bcc..bbe10d47 100644 --- a/azure/cleanup-ecs-pr-proxies.yml +++ b/azure/cleanup-ecs-pr-proxies.yml @@ -30,13 +30,12 @@ jobs: displayName: Cleanup ECS PR Proxies timeoutInMinutes: 360 pool: - name: 'AWS-ECS' + name: "AWS-ECS" workspace: clean: all steps: - - checkout: self - bash: | @@ -52,13 +51,13 @@ jobs: - bash: | tfenv use 1.2.3 displayName: setup terraform - - - task: s3-cache-action@1 + + - task: s3-cache-action@1.3.4 inputs: key: poetry | utils | poetry.lock location: ".venv" debug: true - displayName: cache utils pre-requisites + displayName: cache utils pre-requisites - bash: | pyversion="3.13" @@ -69,34 +68,26 @@ jobs: echo "##vso[task.setvariable variable=PY_VER]$PATCH" echo "Resolved latest python version: $PATCH" echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/" - displayName: 'Query and set python tool cache path' - + displayName: "Query and set python tool cache path" - task: UsePythonVersion@0 name: UsePy - displayName: 'Use Python 3.13' + displayName: "Use Python 3.13" inputs: - versionSpec: '$(PY_VER)' - - # - bash: | - # echo "Checking the python version in use to set LD_LIBRARY_PATH" - # echo "Python location: $(UsePy.pythonLocation)" - # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" - # displayName: 'Set LD_LIBRARY_PATH' + versionSpec: "$(PY_VER)" - bash: | make install displayName: install dependencies - ${{ each retry in parameters.retries }}: - - template: ./components/cleanup-ecs-pr-proxies-job.yml - parameters: - retry: '${{ retry }}' - retain_hours: '${{ parameters.retain_hours }}' + - template: ./components/cleanup-ecs-pr-proxies-job.yml + parameters: + retry: "${{ retry }}" + retain_hours: "${{ parameters.retain_hours }}" - bash: | echo "AWS role session has timed out after multiple retries" exit -1 displayName: Trigger failure if role has timed out condition: eq(variables['should_retry'], 'true') - diff --git a/azure/cleanup-pr-portal-apis-and-specs.yml b/azure/cleanup-pr-portal-apis-and-specs.yml index b4ffc940..50c0afab 100644 --- a/azure/cleanup-pr-portal-apis-and-specs.yml +++ b/azure/cleanup-pr-portal-apis-and-specs.yml @@ -16,12 +16,11 @@ jobs: displayName: Cleanup PR specs timeoutInMinutes: 240 pool: - name: 'AWS-ECS' + name: "AWS-ECS" workspace: clean: all steps: - - checkout: self - bash: | @@ -43,12 +42,12 @@ jobs: tfenv use 0.14.6 displayName: setup terraform - - task: s3-cache-action@1 + - task: s3-cache-action@1.3.4 inputs: key: poetry | utils | poetry.lock location: ".venv" debug: true - alias: 'Utils' + alias: "Utils" displayName: cache utils pre-requisites - bash: | diff --git a/azure/common/apigee-build.yml b/azure/common/apigee-build.yml index 873fe838..294326df 100644 --- a/azure/common/apigee-build.yml +++ b/azure/common/apigee-build.yml @@ -45,10 +45,13 @@ jobs: variables: ${{ each var in parameters.variables }}: ${{ var.key }}: ${{ var.value }} - steps: + service_name: ${{ parameters.service_name }} + service_short_name: ${{ parameters.short_service_name }} + python_version: ${{ parameters.python_version }} + steps: - bash: | - if [ ! -z "$(ls -A \"$(Pipeline.Workspace)/s/${{ parameters.service_name }}\" 2>/dev/null)" ]; then + if [ ! -z "$(ls -A \"$(Pipeline.Workspace)/s/$(service_name)\" 2>/dev/null)" ]; then echo "workspace directory is not empty!" exit 1 fi @@ -67,63 +70,63 @@ jobs: - template: ../components/aws-clean-config.yml - ${{ if parameters.notify }}: - - template: ../components/aws-assume-role.yml - parameters: - role: "auto-ops" - profile: "apm_ptl" - - - template: ../components/get-aws-secrets-and-ssm-params.yml - parameters: - secret_file_ids: - - ${{ each secret_file_id in parameters.secret_file_ids }}: - - ${{ secret_file_id }} - secret_ids: - - ptl/access-tokens/github/repo-status-update/GITHUB_ACCESS_TOKEN - - ${{ each secret_id in parameters.secret_ids }}: - - ${{ secret_id }} - config_ids: - - /ptl/azure-devops/GITHUB_USER - - ${{ each config_id in parameters.config_ids }}: - - ${{ config_id }} - - - bash: | - echo "Build.SourceBranch: $(Build.SourceBranch)" - echo "Build.SourceBranchName: $(Build.SourceBranchName)" - echo "Build.SourceVersion: $(Build.SourceVersion)" - echo "Build.SourceVersionMessage: $(Build.SourceVersionMessage)" - - if [[ ! -z $(NOTIFY_COMMIT_SHA) ]]; then - echo "##[debug]Using already provided NOTIFY_COMMIT_SHA=$(NOTIFY_COMMIT_SHA)" - else - NOTIFY_COMMIT_SHA="" - - if [[ "$(Build.SourceBranch)" =~ ^refs/tags/.+$ ]]; then - echo "##[debug]Build appears to be a tag build" - echo "##[debug]Using Build.SourceVersion as NOTIFY_COMMIT_SHA" - NOTIFY_COMMIT_SHA="$(Build.SourceVersion)" - fi - - if [[ "$(Build.SourceBranch)" =~ ^refs/pull/.+$ ]]; then - echo "##[debug]Build appears to be a pull request build" - echo "##[debug]Extracting NOTIFY_COMMIT_SHA from Build.SourceVersionMessage" - NOTIFY_COMMIT_SHA=`echo "$(Build.SourceVersionMessage)" | cut -d' ' -f2` - fi - - if [[ -z $NOTIFY_COMMIT_SHA ]]; then - echo "##[debug]Build does not appear to be pull or tag build" - echo "##[debug]Using Build.SourceVersion as NOTIFY_COMMIT_SHA" - NOTIFY_COMMIT_SHA="$(Build.SourceVersion)" - fi - - echo "##vso[task.setvariable variable=NOTIFY_COMMIT_SHA]$NOTIFY_COMMIT_SHA" - fi - displayName: Set NOTIFY_COMMIT_SHA - condition: always() - - - template: '../components/update-github-status.yml' - parameters: - state: pending - description: "Build started" + - template: ../components/aws-assume-role.yml + parameters: + role: "auto-ops" + profile: "apm_ptl" + + - template: ../components/get-aws-secrets-and-ssm-params.yml + parameters: + secret_file_ids: + - ${{ each secret_file_id in parameters.secret_file_ids }}: + - ${{ secret_file_id }} + secret_ids: + - ptl/access-tokens/github/repo-status-update/GITHUB_ACCESS_TOKEN + - ${{ each secret_id in parameters.secret_ids }}: + - ${{ secret_id }} + config_ids: + - /ptl/azure-devops/GITHUB_USER + - ${{ each config_id in parameters.config_ids }}: + - ${{ config_id }} + + - bash: | + echo "Build.SourceBranch: $(Build.SourceBranch)" + echo "Build.SourceBranchName: $(Build.SourceBranchName)" + echo "Build.SourceVersion: $(Build.SourceVersion)" + echo "Build.SourceVersionMessage: $(Build.SourceVersionMessage)" + + if [[ ! -z $(NOTIFY_COMMIT_SHA) ]]; then + echo "##[debug]Using already provided NOTIFY_COMMIT_SHA=$(NOTIFY_COMMIT_SHA)" + else + NOTIFY_COMMIT_SHA="" + + if [[ "$(Build.SourceBranch)" =~ ^refs/tags/.+$ ]]; then + echo "##[debug]Build appears to be a tag build" + echo "##[debug]Using Build.SourceVersion as NOTIFY_COMMIT_SHA" + NOTIFY_COMMIT_SHA="$(Build.SourceVersion)" + fi + + if [[ "$(Build.SourceBranch)" =~ ^refs/pull/.+$ ]]; then + echo "##[debug]Build appears to be a pull request build" + echo "##[debug]Extracting NOTIFY_COMMIT_SHA from Build.SourceVersionMessage" + NOTIFY_COMMIT_SHA=`echo "$(Build.SourceVersionMessage)" | cut -d' ' -f2` + fi + + if [[ -z $NOTIFY_COMMIT_SHA ]]; then + echo "##[debug]Build does not appear to be pull or tag build" + echo "##[debug]Using Build.SourceVersion as NOTIFY_COMMIT_SHA" + NOTIFY_COMMIT_SHA="$(Build.SourceVersion)" + fi + + echo "##vso[task.setvariable variable=NOTIFY_COMMIT_SHA]$NOTIFY_COMMIT_SHA" + fi + displayName: Set NOTIFY_COMMIT_SHA + condition: always() + + - template: "../components/update-github-status.yml" + parameters: + state: pending + description: "Build started" - bash: | if [[ ! -z $(UTILS_PR_NUMBER) ]]; then @@ -144,7 +147,7 @@ jobs: service_name: "${{ parameters.service_name }}" - bash: | - pyversion=${{ parameters.python_version }} + pyversion="$(python_version)" PATCH=$(curl -fsSL https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json \ | jq -r --arg pyversion "$pyversion" '[ .[] | select(.stable == true) | .version | select(test("^" + ($pyversion|gsub("\\.";"\\.")) + "\\.\\d+$")) | split(".") | map(tonumber)] | max | join(".")') @@ -152,20 +155,13 @@ jobs: echo "##vso[task.setvariable variable=PY_VER]$PATCH" echo "Resolved latest python version: $PATCH" echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/" - displayName: 'Query and set python tool cache path' - + displayName: "Query and set python tool cache path" - task: UsePythonVersion@0 name: UsePy - displayName: 'Use Python ${{ parameters.python_version }}' + displayName: "Use Python ${{ parameters.python_version }}" inputs: - versionSpec: '$(PY_VER)' - - # - bash: | - # echo "Checking the python version in use to set LD_LIBRARY_PATH" - # echo "Python location: $(UsePy.pythonLocation)" - # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" - # displayName: 'Set LD_LIBRARY_PATH' + versionSpec: "$(PY_VER)" - ${{ each cache_step in parameters.cache_steps }}: - ${{ cache_step }} @@ -207,31 +203,31 @@ jobs: - checkout: common path: "s/${{ parameters.service_name }}/utils" - - task: s3-cache-action@1 + - task: s3-cache-action@1.3.4 inputs: key: poetry | $(System.DefaultWorkingDirectory) | ${{ parameters.service_name }}/utils/poetry.lock location: "${{ parameters.service_name }}/utils/.venv" debug: true - alias: 'Utils' + alias: "Utils" displayName: cache utils dependencies - bash: | - make install - sleep 5 + make install + sleep 5 workingDirectory: "${{ parameters.service_name }}/utils" condition: ne(variables['CacheRestored-Utils'], 'true') displayName: "Install utils " - bash: | - export out_dir="$(realpath ${{ parameters.service_name }}/dist)" - export commit_hash="$(git -C ${{ parameters.service_name }} rev-parse --short HEAD)" + export out_dir="$(realpath $(service_name)/dist)" + export commit_hash="$(git -C $(service_name) rev-parse --short HEAD)" # sha prefix is required docker gets upset if names contain -0 iirc export build_label="$(Build.BuildId)-sha${commit_hash}" export pr_number="$(PR_NUMBER)" - export service_name="${{ parameters.service_name }}" - export service_id="${{ parameters.short_service_name }}" + export service_name="$(service_name)" + export service_id="$(service_short_name)" export ANSIBLE_FORCE_COLOR=yes - make --no-print-directory -C ${{ parameters.service_name }}/utils/ansible create-build-env-vars + make --no-print-directory -C "$(service_name)/utils/ansible" create-build-env-vars displayName: output build env vars for artifact condition: and(succeeded(), eq(variables['build_containers'], 'true')) @@ -241,8 +237,8 @@ jobs: condition: and(succeeded(), eq(variables['build_containers'], 'true')) - bash: | - source "${{ parameters.service_name }}/dist/.build_env_vars" - account=ptl make --no-print-directory -C ${{ parameters.service_name }}/utils/ansible create-ecr-build-role + source "$(service_name)/dist/.build_env_vars" + account=ptl make --no-print-directory -C "$(service_name)/utils/ansible" create-ecr-build-role echo "##vso[task.setvariable variable=BUILD_ROLE]build-${service_id}" displayName: "ensure build role" condition: and(succeeded(), eq(variables['build_containers'], 'true')) @@ -253,9 +249,9 @@ jobs: profile: "$(BUILD_ROLE)" - bash: | - source "${{ parameters.service_name }}/dist/.build_env_vars" - CONTAINER_VARS_FILE="$(realpath ${{ parameters.service_name }}/ecs-proxies-containers.yml)" \ - make --no-print-directory -C ${{ parameters.service_name }}/utils/ansible build-ecs-proxies + source "$(service_name)/dist/.build_env_vars" + CONTAINER_VARS_FILE="$(realpath $(service_name)/ecs-proxies-containers.yml)" \ + make --no-print-directory -C "$(service_name)/utils/ansible" build-ecs-proxies displayName: "Build and push ECS proxies" condition: and(succeeded(), eq(variables['build_containers'], 'true')) @@ -263,7 +259,7 @@ jobs: - ${{ post_ecs_push }} - bash: | - cp ${{ parameters.service_name }}/ecs-*.yml ${{ parameters.service_name }}/dist || true + cp "$(service_name)"/ecs-*.yml "$(service_name)"/dist || true displayName: "Copy ecs-proxies-deploy configs into build artifact" condition: and(succeeded(), eq(variables['build_containers'], 'true')) @@ -288,7 +284,7 @@ jobs: displayName: "Validate manifest (template) and copy to artifact" - bash: | - cd ${{ parameters.service_name }}/utils + cd "$(service_name)/utils" git rev-parse HEAD > ../dist/.utils-version displayName: Snapshot utils version @@ -296,14 +292,14 @@ jobs: artifact: "$(Build.BuildNumber)" - ${{ if parameters.notify }}: - - template: '../components/update-github-status.yml' - parameters: - state: success - on_success: true - description: "Build succeeded" - - - template: '../components/update-github-status.yml' - parameters: - state: failure - on_failure: true - description: "Build failed" + - template: "../components/update-github-status.yml" + parameters: + state: success + on_success: true + description: "Build succeeded" + + - template: "../components/update-github-status.yml" + parameters: + state: failure + on_failure: true + description: "Build failed" diff --git a/azure/common/deploy-stage.yml b/azure/common/deploy-stage.yml index 4951b60c..fdd45920 100644 --- a/azure/common/deploy-stage.yml +++ b/azure/common/deploy-stage.yml @@ -22,7 +22,7 @@ parameters: - name: fully_qualified_service_name - name: service_base_path - name: pr_label - default: '' + default: "" - name: secret_file_ids type: object - name: secret_ids @@ -123,7 +123,7 @@ stages: role: "auto-ops" profile: "apm_${{ parameters.aws_account }}" aws_account: "${{ parameters.aws_account }}" - + - bash: | pyversion="3.13" PATCH=$(curl -fsSL https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json \ @@ -133,20 +133,13 @@ stages: echo "##vso[task.setvariable variable=PY_VER]$PATCH" echo "Resolved latest python version: $PATCH" echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/" - displayName: 'Query and set python tool cache path' - + displayName: "Query and set python tool cache path" - task: UsePythonVersion@0 name: UsePy - displayName: 'Use Python 3.13' + displayName: "Use Python 3.13" inputs: - versionSpec: '$(PY_VER)' - - # - bash: | - # echo "Checking the python version in use to set LD_LIBRARY_PATH" - # echo "Python location: $(UsePy.pythonLocation)" - # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" - # displayName: 'Set LD_LIBRARY_PATH' + versionSpec: "$(PY_VER)" - template: "../components/set-facts.yml" parameters: @@ -173,27 +166,27 @@ stages: apigee_organization: ${{ parameters.apigee_organization }} - ${{ if parameters.notify }}: - - bash: | - if [[ -z $(NOTIFY_COMMIT_SHA) ]]; then - export NOTIFY_COMMIT_SHA=`echo "$(Build.SourceVersionMessage)" | cut -d' ' -f2` - echo "##vso[task.setvariable variable=NOTIFY_COMMIT_SHA]$NOTIFY_COMMIT_SHA" - else - echo "##[debug]Using already provided NOTIFY_COMMIT_SHA=$(NOTIFY_COMMIT_SHA)" - fi - displayName: Set NOTIFY_COMMIT_SHA + - bash: | + if [[ -z $(NOTIFY_COMMIT_SHA) ]]; then + export NOTIFY_COMMIT_SHA=`echo "$(Build.SourceVersionMessage)" | cut -d' ' -f2` + echo "##vso[task.setvariable variable=NOTIFY_COMMIT_SHA]$NOTIFY_COMMIT_SHA" + else + echo "##[debug]Using already provided NOTIFY_COMMIT_SHA=$(NOTIFY_COMMIT_SHA)" + fi + displayName: Set NOTIFY_COMMIT_SHA - - template: '../components/update-github-status.yml' - parameters: - state: pending - description: "Deploy to ${{ parameters.environment }} started" - environment: ${{ parameters.environment }} + - template: "../components/update-github-status.yml" + parameters: + state: pending + description: "Deploy to ${{ parameters.environment }} started" + environment: ${{ parameters.environment }} - ${{ each param in parameters }}: - - ${{ if not(containsValue(parameters._expose_blacklist, param.key)) }}: - - bash: | - set -euo pipefail - echo "##vso[task.setvariable variable=${{ upper(param.key) }}]${{ param.value }}" - displayName: Setting ${{ upper(param.key) }}=${{ param.value }} + - ${{ if not(containsValue(parameters._expose_blacklist, param.key)) }}: + - bash: | + set -euo pipefail + echo "##vso[task.setvariable variable=${{ upper(param.key) }}]${{ param.value }}" + displayName: Setting ${{ upper(param.key) }}=${{ param.value }} - bash: | set -euo pipefail @@ -261,7 +254,7 @@ stages: displayName: Check supplied names against API registry continueOnError: true - - template: '../templates/deploy-service.yml' + - template: "../templates/deploy-service.yml" parameters: service_name: ${{ parameters.service_name }} short_service_name: ${{ parameters.short_service_name }} @@ -282,7 +275,7 @@ stages: post_deploy: ${{ parameters.post_deploy }} ping: ${{ parameters.ping }} enable_monitoring: ${{ parameters.enable_monitoring }} - enable_status_monitoring : ${{ parameters.enable_status_monitoring }} + enable_status_monitoring: ${{ parameters.enable_status_monitoring }} portal_api_requires_callback_url: ${{ parameters.portal_api_requires_callback_url }} make_spec_visible: ${{ parameters.make_spec_visible }} friendly_api_name: ${{ parameters.friendly_api_name }} @@ -291,16 +284,16 @@ stages: aws_account: ${{ parameters.aws_account }} - ${{ if parameters.notify }}: - - template: '../components/update-github-status.yml' - parameters: - state: success - on_success: true - description: "Deploy to ${{ parameters.environment }} succeeded" - environment: ${{ parameters.environment }} + - template: "../components/update-github-status.yml" + parameters: + state: success + on_success: true + description: "Deploy to ${{ parameters.environment }} succeeded" + environment: ${{ parameters.environment }} - - template: '../components/update-github-status.yml' - parameters: - state: failure - on_failure: true - description: "Deploy to ${{ parameters.environment }} failed" - environment: ${{ parameters.environment }} + - template: "../components/update-github-status.yml" + parameters: + state: failure + on_failure: true + description: "Deploy to ${{ parameters.environment }} failed" + environment: ${{ parameters.environment }} diff --git a/azure/components/aws-assume-role.yml b/azure/components/aws-assume-role.yml index 2d5bb337..76fc3c1a 100644 --- a/azure/components/aws-assume-role.yml +++ b/azure/components/aws-assume-role.yml @@ -1,84 +1,88 @@ parameters: - - name: 'role' + - name: "role" type: string - - name: 'profile' + - name: "profile" type: string - default: '' - - name: 'aws_account' + default: "" + - name: "aws_account" type: string - default: 'ptl' + default: "ptl" steps: - - template: ./aws-clean-config.yml + - template: ./aws-clean-config.yml - - bash: | - set -e - echo "##vso[task.setvariable variable=ROLE]${{ parameters.role }}" - displayName: get role name + - bash: | + set -e + echo "##vso[task.setvariable variable=ROLE]$role" + env: + role: ${{ parameters.role }} + displayName: get role name - - bash: | - set -e + - bash: | + set -e - aws_role="$(ROLE)" - echo "assume role: '${aws_role}'" + aws_role="$(ROLE)" + echo "assume role: '${aws_role}'" - if [[ "${{ parameters.aws_account }}" =~ ^(prod|dev)$ ]]; then - echo "account is ${{ parameters.aws_account }}" - account_id="$(aws ssm get-parameter --name /account-ids/${{ parameters.aws_account }} --query Parameter.Value --output text)" - aws_role="arn:aws:iam::${account_id}:role/${aws_role}" - fi + if [[ "$AWS_ACCOUNT" =~ ^(prod|dev)$ ]]; then + echo "account is $AWS_ACCOUNT" + account_id="$(aws ssm get-parameter --name /account-ids/$AWS_ACCOUNT --query Parameter.Value --output text)" + aws_role="arn:aws:iam::${account_id}:role/${aws_role}" + fi - if [[ "${aws_role}" != arn:aws:iam:* ]]; then - echo "check if role exists" - # iam synchronisation issues can take a few to make the role appear - for i in {1..15}; do - if aws iam get-role --role-name ${aws_role} > /dev/null; then - echo role exists - sleep 2 - break - fi - echo waiting for role ... + if [[ "${aws_role}" != arn:aws:iam:* ]]; then + echo "check if role exists" + # iam synchronisation issues can take a few to make the role appear + for i in {1..15}; do + if aws iam get-role --role-name ${aws_role} > /dev/null; then + echo role exists sleep 2 - done - - account_id="$(aws sts get-caller-identity --query Account --output text)" - aws_role="arn:aws:iam::${account_id}:role/${aws_role}" - fi + break + fi + echo waiting for role ... + sleep 2 + done - cp ~/.aws/config.default ~/.aws/config - tmp_file="$(Agent.TempDirectory)/.aws.tmp.creds.json" + account_id="$(aws sts get-caller-identity --query Account --output text)" + aws_role="arn:aws:iam::${account_id}:role/${aws_role}" + fi - # add some backoff to allow for eventual consistency of IAM - for i in {2..4}; - do - if aws sts assume-role --role-arn "${aws_role}" --role-session-name build-assume-role > ${tmp_file}; then - echo assumed role - assumed_role="yes" - break - fi - let "sleep_for=$i*10"; - sleep $sleep_for - done + cp ~/.aws/config.default ~/.aws/config + tmp_file="$(Agent.TempDirectory)/.aws.tmp.creds.json" - if [[ "${assumed_role}" != "yes" ]]; then - echo "assume role failed" - exit -1 - fi + # add some backoff to allow for eventual consistency of IAM + for i in {2..4}; + do + if aws sts assume-role --role-arn "${aws_role}" --role-session-name build-assume-role > ${tmp_file}; then + echo assumed role + assumed_role="yes" + break + fi + let "sleep_for=$i*10"; + sleep $sleep_for + done - echo "aws_access_key_id = $(jq -r .Credentials.AccessKeyId ${tmp_file})" >> ~/.aws/config - echo "aws_secret_access_key = $(jq -r .Credentials.SecretAccessKey ${tmp_file})" >> ~/.aws/config - echo "aws_session_token = $(jq -r .Credentials.SessionToken ${tmp_file})" >> ~/.aws/config + if [[ "${assumed_role}" != "yes" ]]; then + echo "assume role failed" + exit -1 + fi - expiry=$(jq -r .Credentials.Expiration ${tmp_file}) - echo "##vso[task.setvariable variable=ASSUME_ROLE_EXPIRY;]$expiry" + echo "aws_access_key_id = $(jq -r .Credentials.AccessKeyId ${tmp_file})" >> ~/.aws/config + echo "aws_secret_access_key = $(jq -r .Credentials.SecretAccessKey ${tmp_file})" >> ~/.aws/config + echo "aws_session_token = $(jq -r .Credentials.SessionToken ${tmp_file})" >> ~/.aws/config - rm ${tmp_file} + expiry=$(jq -r .Credentials.Expiration ${tmp_file}) + echo "##vso[task.setvariable variable=ASSUME_ROLE_EXPIRY;]$expiry" - profile="${{ parameters.profile }}" - if [[ ! -z "${profile}" ]]; then - echo as profile ${profile} - sed -i "s#\[default\]#\[profile ${profile}\]#" ~/.aws/config - fi + rm ${tmp_file} - displayName: assume role - condition: and(succeeded(), ne(variables['ROLE'], '')) + profile="$PROFILE" + if [[ ! -z "${profile}" ]]; then + echo as profile ${profile} + sed -i "s#\[default\]#\[profile ${profile}\]#" ~/.aws/config + fi + env: + AWS_ACCOUNT: ${{ parameters.aws_account }} + PROFILE: ${{ parameters.profile }} + displayName: assume role + condition: and(succeeded(), ne(variables['ROLE'], '')) diff --git a/azure/components/cleanup-ecs-pr-proxies-job.yml b/azure/components/cleanup-ecs-pr-proxies-job.yml index a30f2f8b..f97a4b10 100644 --- a/azure/components/cleanup-ecs-pr-proxies-job.yml +++ b/azure/components/cleanup-ecs-pr-proxies-job.yml @@ -14,7 +14,7 @@ steps: displayName: Remove stale locks - bash: | - export retain_hours="${{ parameters.retain_hours }}" + export retain_hours="$retain_hours" ANSIBLE_FORCE_COLOR=yes make -C ansible remove-old-ecs-pr-deploys ERROR_CODE=$? echo ERROR_CODE - $ERROR_CODE @@ -26,6 +26,7 @@ steps: else echo "##vso[task.setvariable variable=should_retry;]false" fi - + env: + retain_hours: ${{ parameters.retain_hours }} displayName: cleanup older pr deploys condition: or(eq( ${{ parameters.retry }}, '0'), eq(variables['should_retry'], 'true')) diff --git a/azure/components/curl.yml b/azure/components/curl.yml index ccabcd6a..afa47db8 100644 --- a/azure/components/curl.yml +++ b/azure/components/curl.yml @@ -5,14 +5,18 @@ parameters: - GET - name: url - name: headers - default: '' + default: "" - name: body - default: '' + default: "" - name: display_name - default: '' - + default: "" steps: - bash: | - curl --fail -X '${{ parameters.method }}' -H '${{ parameters.headers }}' -d '${{ parameters.body }}' '${{ parameters.url }}' + curl --fail -X "$method" -H "$headers" -d "$body" "$url" + env: + method: ${{ parameters.method }} + url: ${{ parameters.url }} + headers: ${{ parameters.headers }} + body: ${{ parameters.body }} displayName: ${{ coalesce(parameters.display_name, 'curl') }} diff --git a/azure/components/get-access-token.yml b/azure/components/get-access-token.yml index 2be25fb3..c614c81d 100644 --- a/azure/components/get-access-token.yml +++ b/azure/components/get-access-token.yml @@ -11,26 +11,34 @@ parameters: default: nonprod: login.apigee.com prod: nhs-digital-prod.login.apigee.com - + steps: - bash: | set -euo pipefail - echo 'apigee_username: ${{ parameters.apigee_username }}' - echo 'apigee_organization: ${{ parameters.apigee_organization }}' - echo 'auth_url: ${{ parameters._auth_server[parameters.apigee_organization] }}' - displayName: 'Print access token debug info' + echo "apigee_username: $apigee_username" + echo "apigee_organization: $apigee_organization" + echo "auth_url: $auth_url" + env: + apigee_username: ${{ parameters.apigee_username }} + apigee_organization: ${{ parameters.apigee_organization }} + auth_url: ${{ parameters._auth_server[parameters.apigee_organization] }} + displayName: "Print access token debug info" - bash: | set -euo pipefail # fetch the access token - curl -X POST https://${{ parameters._auth_server[parameters.apigee_organization] }}/oauth/token \ + curl -X POST https://$auth_url/oauth/token \ -H "Content-Type: application/x-www-form-urlencoded" \ -H "Accept: application/json;charset=utf-8" \ -H "Authorization: Basic ZWRnZWNsaTplZGdlY2xpc2VjcmV0" \ - -d "username=${{ parameters.apigee_username }}&password=${{ parameters.apigee_password }}&mfa_token=$(secret.MFACode)&grant_type=password" | jq -e .access_token > .token + -d "username=$apigee_username&password=$apigee_password&mfa_token=$(secret.MFACode)&grant_type=password" | jq -e .access_token > .token # Set token into variable echo "##vso[task.setvariable variable=secret.AccessToken;issecret=true]`cat .token`" - displayName: 'Get Apigee Access Token' - retryCountOnTaskFailure: 5 \ No newline at end of file + env: + apigee_username: ${{ parameters.apigee_username }} + apigee_password: ${{ parameters.apigee_password }} + auth_url: ${{ parameters._auth_server[parameters.apigee_organization] }} + displayName: "Get Apigee Access Token" + retryCountOnTaskFailure: 5 diff --git a/azure/components/get-mfa-code.yml b/azure/components/get-mfa-code.yml index d8ea34ea..008ddac6 100644 --- a/azure/components/get-mfa-code.yml +++ b/azure/components/get-mfa-code.yml @@ -3,11 +3,13 @@ parameters: type: string steps: -- bash: | - # Install requirements - set -e - - # Save code to variable - export MFA_CODE=`oathtool --totp -b '${{ parameters.apigee_otp_key }}'` - echo "##vso[task.setvariable variable=secret.MFACode;issecret=true]$MFA_CODE" - displayName: 'Get MFA Code for Apigee Login' + - bash: | + # Install requirements + set -e + + # Save code to variable + export MFA_CODE=`oathtool --totp -b "$apigee_otp_key"` + echo "##vso[task.setvariable variable=secret.MFACode;issecret=true]$MFA_CODE" + env: + apigee_otp_key: ${{ parameters.apigee_otp_key }} + displayName: "Get MFA Code for Apigee Login" diff --git a/azure/components/remove-target-proxy.yml b/azure/components/remove-target-proxy.yml index 36251c9f..0708eaf4 100644 --- a/azure/components/remove-target-proxy.yml +++ b/azure/components/remove-target-proxy.yml @@ -27,7 +27,7 @@ parameters: stages: - stage: approve_then_remove_target_proxy - displayName: Find and remove all components for ${{ parameters.service_name }} proxy in ${{ parameters.apigee_org }} org + displayName: Find and remove all components for ${{ parameters.service_name }} proxy in ${{ parameters.apigee_org }} org dependsOn: [] jobs: - deployment: build @@ -39,66 +39,65 @@ stages: workspace: clean: all strategy: - runOnce: - deploy: - steps: - - checkout: self + runOnce: + deploy: + steps: + - checkout: self - - template: ./print-aws-info.yml + - template: ./print-aws-info.yml - - task: s3-cache-action@1 - inputs: - key: poetry | utils | poetry.lock - location: ".venv" - debug: true - displayName: cache utils pre-requisites + - task: s3-cache-action@1.3.4 + inputs: + key: poetry | utils | poetry.lock + location: ".venv" + debug: true + displayName: cache utils pre-requisites - - bash: make install - displayName: Install project dependencies + - bash: make install + displayName: Install project dependencies - - template: ./aws-assume-role.yml - parameters: - role: auto-ops - aws_account: ${{ parameters.aws_account }} - profile: ${{ parameters.aws_profile }} + - template: ./aws-assume-role.yml + parameters: + role: auto-ops + aws_account: ${{ parameters.aws_account }} + profile: ${{ parameters.aws_profile }} - - bash: | - tfenv use 0.14.6 - displayName: setup terraform + - bash: | + tfenv use 0.14.6 + displayName: setup terraform - - template: ./get-aws-secrets-and-ssm-params.yml - parameters: - secret_ids: - - ${{ parameters.aws_account }}/azure-devops/apigee-${{ parameters.apigee_org }}/APIGEE_OTP_KEY - - ${{ parameters.aws_account }}/azure-devops/apigee-${{ parameters.apigee_org }}/APIGEE_PASSWORD - config_ids: - - /${{ parameters.aws_account }}/azure-devops/apigee-${{ parameters.apigee_org }}/APIGEE_USERNAME - aws_account: ${{ parameters.aws_account }} + - template: ./get-aws-secrets-and-ssm-params.yml + parameters: + secret_ids: + - ${{ parameters.aws_account }}/azure-devops/apigee-${{ parameters.apigee_org }}/APIGEE_OTP_KEY + - ${{ parameters.aws_account }}/azure-devops/apigee-${{ parameters.apigee_org }}/APIGEE_PASSWORD + config_ids: + - /${{ parameters.aws_account }}/azure-devops/apigee-${{ parameters.apigee_org }}/APIGEE_USERNAME + aws_account: ${{ parameters.aws_account }} - - bash: | - export MFA_CODE=`oathtool --totp -b "$(APIGEE_OTP_KEY)"` - # basic auth value here is an apigee public value .. https://docs.apigee.com/api-platform/system-administration/management-api-tokens (not secret) - curl -X POST https://${{ parameters._auth_server[parameters.apigee_org] }}/oauth/token \ - -H "Content-Type: application/x-www-form-urlencoded" \ - -H "Accept: application/json;charset=utf-8" \ - -H "Authorization: Basic ZWRnZWNsaTplZGdlY2xpc2VjcmV0" \ - -d "username=$(APIGEE_USERNAME)&password=$(APIGEE_PASSWORD)&mfa_token=${MFA_CODE}&grant_type=password" | jq .access_token > .token - # Set token into variable - echo "##vso[task.setvariable variable=secret.${{ upper(parameters.apigee_org) }}_API_ACCESS_TOKEN;issecret=true]`cat .token`" - displayName: Get access token + - bash: | + export MFA_CODE=`oathtool --totp -b "$(APIGEE_OTP_KEY)"` + # basic auth value here is an apigee public value .. https://docs.apigee.com/api-platform/system-administration/management-api-tokens (not secret) + curl -X POST https://${{ parameters._auth_server[parameters.apigee_org] }}/oauth/token \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -H "Accept: application/json;charset=utf-8" \ + -H "Authorization: Basic ZWRnZWNsaTplZGdlY2xpc2VjcmV0" \ + -d "username=$(APIGEE_USERNAME)&password=$(APIGEE_PASSWORD)&mfa_token=${MFA_CODE}&grant_type=password" | jq .access_token > .token + # Set token into variable + echo "##vso[task.setvariable variable=secret.${{ upper(parameters.apigee_org) }}_API_ACCESS_TOKEN;issecret=true]`cat .token`" + displayName: Get access token - - bash: | - set -euo pipefail - export SERVICE_NAME="${{ parameters.service_name }}" - export SERVICE_NAME_SHORT="${{ parameters.service_name_short }}" - export APIGEE_ENVIRONMENT="${{ parameters.environment }}" - export APIGEE_ORG="nhsd-${{ parameters.apigee_org }}" - export APIGEE_ACCESS_TOKEN="$(secret.${{ parameters.apigee_org }}_API_ACCESS_TOKEN)" - export AWS_ACCOUNT="${{ parameters.aws_account }}" - export AWS_PROFILE="${{ parameters.aws_profile }}" - export has_ecs_backend="${{ parameters.has_ecs_backend }}" - export remove_ecr_build_role="${{ parameters.remove_ecr_build_role }}" - export force_remove_from_apps="${{ parameters.force_remove_from_apps }}" - ANSIBLE_FORCE_COLOR=yes make -C ansible remove-target-proxy - displayName: "Cleanup target proxy" - + - bash: | + set -euo pipefail + export SERVICE_NAME="${{ parameters.service_name }}" + export SERVICE_NAME_SHORT="${{ parameters.service_name_short }}" + export APIGEE_ENVIRONMENT="${{ parameters.environment }}" + export APIGEE_ORG="nhsd-${{ parameters.apigee_org }}" + export APIGEE_ACCESS_TOKEN="$(secret.${{ parameters.apigee_org }}_API_ACCESS_TOKEN)" + export AWS_ACCOUNT="${{ parameters.aws_account }}" + export AWS_PROFILE="${{ parameters.aws_profile }}" + export has_ecs_backend="${{ parameters.has_ecs_backend }}" + export remove_ecr_build_role="${{ parameters.remove_ecr_build_role }}" + export force_remove_from_apps="${{ parameters.force_remove_from_apps }}" + ANSIBLE_FORCE_COLOR=yes make -C ansible remove-target-proxy + displayName: "Cleanup target proxy" diff --git a/azure/components/set-commit-hash.yml b/azure/components/set-commit-hash.yml index a289acad..6c2b36e2 100644 --- a/azure/components/set-commit-hash.yml +++ b/azure/components/set-commit-hash.yml @@ -1,4 +1,6 @@ steps: - bash: | - echo "##vso[task.setvariable variable=COMMIT_HASH]`echo $(Build.SourceVersionMessage) | cut -d' ' -f2`" + echo "##vso[task.setvariable variable=COMMIT_HASH]$(echo "$BUILD_SOURCEVERSIONMESSAGE" | cut -d' ' -f2)" + env: + BUILD_SOURCEVERSIONMESSAGE: $(Build.SourceVersionMessage) displayName: Set COMMIT_HASH diff --git a/azure/components/set-facts.yml b/azure/components/set-facts.yml index b85aec82..eef93504 100644 --- a/azure/components/set-facts.yml +++ b/azure/components/set-facts.yml @@ -12,36 +12,45 @@ parameters: type: object - name: aws_account type: string - default: 'ptl' + default: "ptl" - name: apigee_organization type: string steps: - bash: | set -euo pipefail - echo "!!! Using ls -t will set the latest file for the artifact. If you get an error here, it is because '${{ parameters.service_name }}' is not the source alias name of the artifact" + echo "!!! Using ls -t will set the latest file for the artifact. If you get an error here, it is because '$service_name' is not the source alias name of the artifact" ls -R $(Pipeline.Workspace) - export SERVICE_ARTIFACT_NAME=`ls -t $(Pipeline.Workspace)/s/${{ parameters.service_name }}` + export SERVICE_ARTIFACT_NAME=`ls -t $(Pipeline.Workspace)/s/$service_name` echo "##vso[task.setvariable variable=SERVICE_ARTIFACT_NAME]$SERVICE_ARTIFACT_NAME" echo "Set Artifact Name of: $SERVICE_ARTIFACT_NAME" - displayName: 'Set SERVICE_ARTIFACT_NAME' + env: + service_name: ${{ parameters.service_name }} + displayName: "Set SERVICE_ARTIFACT_NAME" - bash: | - echo "##vso[task.setvariable variable=SERVICE_DIR]$(Pipeline.Workspace)/s/${{ parameters.service_name }}/$(SERVICE_ARTIFACT_NAME)" - echo "##vso[task.setvariable variable=UTILS_DIR]$(Pipeline.Workspace)/s/${{ parameters.service_name }}/$(SERVICE_ARTIFACT_NAME)/utils" - displayName: Set SERVICE_DIR and UTILS_DIR for deploy-service + echo "##vso[task.setvariable variable=SERVICE_DIR]$(Pipeline.Workspace)/s/$service_name/$(SERVICE_ARTIFACT_NAME)" + echo "##vso[task.setvariable variable=UTILS_DIR]$(Pipeline.Workspace)/s/$service_name/$(SERVICE_ARTIFACT_NAME)/utils" + env: + service_name: ${{ parameters.service_name }} + displayName: "Set SERVICE_DIR and UTILS_DIR for deploy-service" - bash: | - if [ -n "$(System.PullRequest.SourceBranch)" ]; then - export SOURCE_BRANCH="$(System.PullRequest.SourceBranch)" + if [ -n "$System_PullRequest_SourceBranch" ]; then + export SOURCE_BRANCH="$System_PullRequest_SourceBranch" else - export SOURCE_BRANCH="$(Build.SourceBranchName)" + export SOURCE_BRANCH="$Build_SourceBranchName" fi echo "##vso[task.setvariable variable=BRANCH_NAME]`echo $SOURCE_BRANCH | sed -r 's/[/|\\@":<>?*]+/-/g'`" + env: + System_PullRequest_SourceBranch: $(System.PullRequest.SourceBranch) + Build_SourceBranchName: $(Build.SourceBranchName) displayName: Set and replace invalid characters in branch name - - bash: 'echo "##vso[build.updatebuildnumber]${{ parameters.service_name }}-$(BRANCH_NAME)+$(Build.BuildID)"' + - bash: 'echo "##vso[build.updatebuildnumber]$service_name-$(BRANCH_NAME)+$(Build.BuildID)"' condition: eq(variables['Build.SourceBranchName'], 'merge') + env: + service_name: ${{ parameters.service_name }} displayName: Update build ID if this is a pull request - template: ../components/get-aws-secrets-and-ssm-params.yml diff --git a/azure/components/update-github-status.yml b/azure/components/update-github-status.yml index ec93d770..9d77e925 100644 --- a/azure/components/update-github-status.yml +++ b/azure/components/update-github-status.yml @@ -3,10 +3,10 @@ parameters: displayName: State type: string values: - - pending # A build is underway - - success # A build succeeded - - failure # A build failed in an expected way, e.g. the tests failed - - error # A build finishes for an unexpected reason, e.g. error in bash step, bad config + - pending # A build is underway + - success # A build succeeded + - failure # A build failed in an expected way, e.g. the tests failed + - error # A build finishes for an unexpected reason, e.g. error in bash step, bad config - name: description displayName: Description type: string @@ -36,11 +36,14 @@ steps: if [[ "$(NOTIFY_COMMIT_SHA)" =~ ^[0-9a-f]+$ ]]; then echo "Reporting state ${{ parameters.state }} to ${NOTIFY_GITHUB_REPOSITORY}" echo "##[debug] Hitting https://api.github.com/repos/${NOTIFY_GITHUB_REPOSITORY}/statuses/$(NOTIFY_COMMIT_SHA)" - curl --fail -q -X POST "https://api.github.com/repos/${NOTIFY_GITHUB_REPOSITORY}/statuses/$(NOTIFY_COMMIT_SHA)" -d '{"state": "${{ parameters.state }}", "context": "$(Build.DefinitionName) ${{ parameters.environment }}", "description": "${{ parameters.description }}", "target_url": "https://dev.azure.com/NHSD-APIM/API Platform/_build/results?buildId=$(Build.BuildID)"}' --user $(GITHUB_USER):$(GITHUB_ACCESS_TOKEN) + curl --fail -q -X POST "https://api.github.com/repos/${NOTIFY_GITHUB_REPOSITORY}/statuses/$(NOTIFY_COMMIT_SHA)" -d '{"state": "${{ parameters.state }}", "context": "$(Build.DefinitionName) $environment", "description": "$description", "target_url": "https://dev.azure.com/NHSD-APIM/API Platform/_build/results?buildId=$(Build.BuildID)"}' --user $(GITHUB_USER):$(GITHUB_ACCESS_TOKEN) else echo "##[warning]$(NOTIFY_COMMIT_SHA) doesn't look like a commit hash" echo "##[task.logissue type=warning]Task wasn't able to find the right commit sha value to notify github." fi + env: + description: ${{ parameters.description }} + environment: ${{ parameters.environment }} displayName: "Notify GitHub: ${{ parameters.state }}" condition: | or( diff --git a/azure/create-build-env-vars.yml b/azure/create-build-env-vars.yml index 740df5c1..4ccc4cac 100644 --- a/azure/create-build-env-vars.yml +++ b/azure/create-build-env-vars.yml @@ -1,34 +1,34 @@ - parameters: - - name: 'out_dir' + - name: "out_dir" type: string - default: './' - - name: 'source_root' + default: "./" + - name: "source_root" type: string - default: './' - - name: 'service_name' + default: "./" + - name: "service_name" type: string - default: '' - - name: 'service_id' + default: "" + - name: "service_id" type: string - default: '' - - name: 'utils_dir' + default: "" + - name: "utils_dir" type: string - default: 'utils' + default: "utils" steps: - - bash: | - export out_dir="$(realpath ${{ parameters.out_dir }})" - export commit_hash="$(git -C ${{ parameters.source_root }} rev-parse --short HEAD)" + export out_dir="$(realpath $out_dir)" + export commit_hash="$(git -C $source_root rev-parse --short HEAD)" # sha prefix is required docker gets upset if names contain -0 iirc export build_label="$(Build.BuildId)-sha${commit_hash}" export pr_number="$(System.PullRequest.PullRequestNumber)" - export service_name="${{ parameters.service_name }}" - export service_id="${{ parameters.service_id }}" + export service_name="$service_name" + export service_id="$service_id" export ANSIBLE_FORCE_COLOR=yes - make --no-print-directory -C ${{ parameters.utils_dir }}/ansible create-build-env-vars - + make --no-print-directory -C $utils_dir/ansible create-build-env-vars + env: + out_dir: ${{ parameters.out_dir }} + source_root: ${{ parameters.source_root }} + service_name: ${{ parameters.service_name }} + service_id: ${{ parameters.service_id }} + utils_dir: ${{ parameters.utils_dir }} displayName: output build env vars for artifact - - - diff --git a/azure/deploy-ecs-proxies-retag.yml b/azure/deploy-ecs-proxies-retag.yml index 02679c72..06604a92 100644 --- a/azure/deploy-ecs-proxies-retag.yml +++ b/azure/deploy-ecs-proxies-retag.yml @@ -1,22 +1,25 @@ parameters: - - name: 'container_vars' + - name: "container_vars" type: string - default: 'ecs-proxies-containers.yml' + default: "ecs-proxies-containers.yml" - - name: 'env_vars_dir' + - name: "env_vars_dir" type: string - default: './' + default: "./" - - name: 'utils_dir' + - name: "utils_dir" type: string - default: 'utils' + default: "utils" steps: - bash: | set -e - source "${{ parameters.env_vars_dir }}/.build_env_vars" + source "$env_vars_dir/.build_env_vars" export CONTAINER_VARS_FILE="$SERVICE_DIR/ecs-proxies-containers.yml - make --no-print-directory -C ${{ parameters.utils_dir }}/ansible deploy-ecs-proxies-retag + make --no-print-directory -C $utils_dir/ansible deploy-ecs-proxies-retag + env: + env_vars_dir: ${{ parameters.env_vars_dir }} + utils_dir: ${{ parameters.utils_dir }} displayName: "Retag ECS proxies" diff --git a/azure/templates/deploy-service.yml b/azure/templates/deploy-service.yml index 45ae358e..52f0f198 100644 --- a/azure/templates/deploy-service.yml +++ b/azure/templates/deploy-service.yml @@ -34,7 +34,7 @@ parameters: type: stepList - name: spec_file - name: friendly_api_name - default: '' + default: "" - name: portal_api_requires_callback_url - name: make_spec_visible # To mark a pipeline variable as deprecated, go to the @@ -52,29 +52,31 @@ parameters: steps: # Warn user they are passing deprecated variables to the deployment pipeline - ${{ each variable_name in parameters._deprecated_pipeline_variables }}: - - ${{ if ne(parameters[variable_name], 'DEPRECATED') }}: - - bash: | - echo "##[warning]Pipeline parameter ${{ variable_name }} is deprecated" - echo "##[warning]The value that appears in Apigee is derived from your repository's manifest.yml/manifest_template.yml file." - echo "##[warning]To remove this warning stop passing the variable into the pipeline parameters." - echo "##vso[task.complete result=SucceededWithIssues;]DONE" - displayName: Warn ${{ variable_name }} is a deprecated pipeline parameter + - ${{ if ne(parameters[variable_name], 'DEPRECATED') }}: + - bash: | + echo "##[warning]Pipeline parameter ${{ variable_name }} is deprecated" + echo "##[warning]The value that appears in Apigee is derived from your repository's manifest.yml/manifest_template.yml file." + echo "##[warning]To remove this warning stop passing the variable into the pipeline parameters." + echo "##vso[task.complete result=SucceededWithIssues;]DONE" + displayName: Warn ${{ variable_name }} is a deprecated pipeline parameter - bash: | ls -R - if [[ -f ecs-deploy-${{ parameters.apigee_environment }}.yml ]] || [[ -f ecs-proxies-deploy.yml ]] || [[ -f ecs-proxies-deploy-sandbox.yml ]]; then + if [[ -f ecs-deploy-$apigee_environment.yml ]] || [[ -f ecs-proxies-deploy.yml ]] || [[ -f ecs-proxies-deploy-sandbox.yml ]]; then echo "##vso[task.setvariable variable=deploy_containers]true" else echo "##vso[task.setvariable variable=deploy_containers]false" fi workingDirectory: "$(SERVICE_DIR)" + env: + apigee_environment: ${{ parameters.apigee_environment }} displayName: "Check for ECS proxy definitions" - - task: s3-cache-action@1 + - task: s3-cache-action@1.3.4 inputs: key: poetry | utils | $(UTILS_DIR)/poetry.lock location: "$(UTILS_DIR)/.venv" debug: true - alias: 'UtilsPreReq' + alias: "UtilsPreReq" displayName: cache utils pre-requisites - bash: | @@ -94,79 +96,91 @@ steps: condition: and(succeeded(), eq(variables['deploy_containers'], 'true')) - ${{ if parameters.proxy_path }}: - - bash: | - echo "Deploying ${{ parameters.service_name }} artifact $(SERVICE_ARTIFACT_NAME) as ${{ parameters.fully_qualified_service_name }} to ${{ parameters.service_base_path }} on $(ENV_URL)" - displayName: 'Deploy Info' - - - ${{ each pre_template_step in parameters.pre_template }}: - - ${{ pre_template_step }} - - # pre_template steps might have been doing cross account stuff - # make sure we bring everything back to the correct AWS role here - - template: ../components/aws-assume-role.yml - parameters: - role: "auto-ops" - profile: "apm_${{ parameters.aws_account }}" - aws_account: "${{ parameters.aws_account }}" - - - ${{ if parameters.jinja_templates }}: - - bash: mkdir -p group_vars/all && touch jinja_templates.yml - workingDirectory: "$(UTILS_DIR)/ansible/" - displayName: Prepare extra template-proxy vars - - - ${{ each jinja_template in parameters.jinja_templates }}: - - bash: 'echo "${{ jinja_template.key }}: ${{ jinja_template.value }}" >> jinja_templates.yml' - workingDirectory: "$(UTILS_DIR)/ansible/group_vars/all" - displayName: "Set ${{ jinja_template.key }} to ${{ jinja_template.value }}" - - - bash: | - set -euo pipefail - - export SERVICE_NAME="${{ parameters.service_name }}" - export PROXIES_DIR="$(SERVICE_DIR)/proxies" - export SERVICE_BASE_PATH="${{ parameters.service_base_path }}" - export APIGEE_ENVIRONMENT="${{ parameters.apigee_environment }}" - export HOSTED_TARGET_CONNECTION_PATH_SUFFIX="${{ parameters.hosted_target_connection_path_suffix }}" - export HOSTED_TARGET_HEALTHCHECK_SUFFIX="${{ parameters.hosted_target_healthcheck_suffix }}" - export SOURCE_COMMIT_ID=$(Build.SourceVersion) - export RELEASE_RELEASEID=$(Build.BuildId) - - export ASSUMED_VERSION=`echo $SERVICE_ARTIFACT_NAME | ( grep -E -o "v[0-9]+\.[0-9]+\.[0-9]+-?[a-z]*" || true ) | tail -1` - if [[ ! -z $ASSUMED_VERSION ]]; then - export DEPLOYED_VERSION=$ASSUMED_VERSION - else - export DEPLOYED_VERSION="${{ parameters.fully_qualified_service_name }}" - fi - - if [ -f $(SERVICE_DIR)/.build_env_vars ]; then - source $(SERVICE_DIR)/.build_env_vars - fi - - cd $(UTILS_DIR) - ANSIBLE_FORCE_COLOR=yes \ - make --no-print-directory -C ansible template-proxies - - if [ -f $(SERVICE_DIR)/manifest.yml ]; then - DIST_DIR=$(SERVICE_DIR) \ - PROXY_DIR=${{ parameters.proxy_path }} \ - ANSIBLE_FORCE_COLOR=yes \ - make --no-print-directory -C ansible add-apim-guids-policy - fi - displayName: Template proxies + - bash: | + echo "Deploying $service_name artifact $(SERVICE_ARTIFACT_NAME) as $fully_qualified_service_name to $service_base_path on $(ENV_URL)" + env: + service_name: ${{ parameters.service_name }} + fully_qualified_service_name: ${{ parameters.fully_qualified_service_name }} + service_base_path: ${{ parameters.service_base_path }} + displayName: "Deploy Info" + + - ${{ each pre_template_step in parameters.pre_template }}: + - ${{ pre_template_step }} + + # pre_template steps might have been doing cross account stuff + # make sure we bring everything back to the correct AWS role here + - template: ../components/aws-assume-role.yml + parameters: + role: "auto-ops" + profile: "apm_${{ parameters.aws_account }}" + aws_account: "${{ parameters.aws_account }}" + + - ${{ if parameters.jinja_templates }}: + - bash: mkdir -p group_vars/all && touch jinja_templates.yml + workingDirectory: "$(UTILS_DIR)/ansible/" + displayName: Prepare extra template-proxy vars + + - ${{ each jinja_template in parameters.jinja_templates }}: + - bash: 'echo "${{ jinja_template.key }}: ${{ jinja_template.value }}" >> jinja_templates.yml' + workingDirectory: "$(UTILS_DIR)/ansible/group_vars/all" + displayName: "Set ${{ jinja_template.key }} to ${{ jinja_template.value }}" - - ${{ each post_template_step in parameters.post_template }}: - - ${{ post_template_step }} + - bash: | + set -euo pipefail + + export SERVICE_NAME="$service_name" + export PROXIES_DIR="$(SERVICE_DIR)/proxies" + export SERVICE_BASE_PATH="$service_base_path" + export APIGEE_ENVIRONMENT="$apigee_environment" + export HOSTED_TARGET_CONNECTION_PATH_SUFFIX="$hosted_target_connection_path_suffix" + export HOSTED_TARGET_HEALTHCHECK_SUFFIX="$hosted_target_healthcheck_suffix" + export SOURCE_COMMIT_ID=$(Build.SourceVersion) + export RELEASE_RELEASEID=$(Build.BuildId) + + export ASSUMED_VERSION=`echo $SERVICE_ARTIFACT_NAME | ( grep -E -o "v[0-9]+\.[0-9]+\.[0-9]+-?[a-z]*" || true ) | tail -1` + if [[ ! -z $ASSUMED_VERSION ]]; then + export DEPLOYED_VERSION=$ASSUMED_VERSION + else + export DEPLOYED_VERSION="$fully_qualified_service_name" + fi + + if [ -f $(SERVICE_DIR)/.build_env_vars ]; then + source $(SERVICE_DIR)/.build_env_vars + fi + + cd $(UTILS_DIR) + ANSIBLE_FORCE_COLOR=yes \ + make --no-print-directory -C ansible template-proxies + + if [ -f $(SERVICE_DIR)/manifest.yml ]; then + DIST_DIR=$(SERVICE_DIR) \ + PROXY_DIR=$proxy_path \ + ANSIBLE_FORCE_COLOR=yes \ + make --no-print-directory -C ansible add-apim-guids-policy + fi + env: + proxy_path: ${{ parameters.proxy_path }} + service_name: ${{ parameters.service_name }} + service_base_path: ${{ parameters.service_base_path }} + apigee_environment: ${{ parameters.apigee_environment }} + fully_qualified_service_name: ${{ parameters.fully_qualified_service_name }} + hosted_target_connection_path_suffix: ${{ parameters.hosted_target_connection_path_suffix }} + hosted_target_healthcheck_suffix: ${{ parameters.hosted_target_healthcheck_suffix }} + displayName: Template proxies + + - ${{ each post_template_step in parameters.post_template }}: + - ${{ post_template_step }} - ${{ each pre_deploy_step in parameters.pre_deploy }}: - - ${{ pre_deploy_step }} + - ${{ pre_deploy_step }} - bash: | set -e proxy_vars_file="" - if [ -f $(SERVICE_DIR)/ecs-deploy-${{ parameters.apigee_environment }}.yml ]; then - proxy_vars_file="$(SERVICE_DIR)/ecs-deploy-${{ parameters.apigee_environment }}.yml" + if [ -f $(SERVICE_DIR)/ecs-deploy-$apigee_environment.yml ]; then + proxy_vars_file="$(SERVICE_DIR)/ecs-deploy-$apigee_environment.yml" else if [ -f $(SERVICE_DIR)/ecs-deploy-all.yml ]; then proxy_vars_file="$(SERVICE_DIR)/ecs-deploy-all.yml" @@ -176,55 +190,60 @@ steps: if [[ ! -z "${proxy_vars_file}" ]]; then source $(SERVICE_DIR)/.build_env_vars - deploy_role="deploy-${{ parameters.apigee_environment }}-${service_id}" + deploy_role="deploy-$apigee_environment-${service_id}" - account=${{ parameters.aws_account }} \ - SERVICE_BASE_PATH=${{ parameters.service_base_path }} \ - APIGEE_ENVIRONMENT=${{ parameters.apigee_environment }} \ + account=$aws_account \ + SERVICE_BASE_PATH=$service_base_path \ + APIGEE_ENVIRONMENT=$apigee_environment \ make --no-print-directory -C $(UTILS_DIR)/ansible create-api-deployment-pre-reqs fi echo "##vso[task.setvariable variable=PROXY_VARS_FILE]${proxy_vars_file}" echo "##vso[task.setvariable variable=DEPLOY_ROLE]${deploy_role}" - + env: + apigee_environment: ${{ parameters.apigee_environment }} + service_base_path: ${{ parameters.service_base_path }} + aws_account: ${{ parameters.aws_account }} displayName: Create ECS Prerequisites condition: and(succeeded(), eq(variables['deploy_containers'], 'true')) - bash: | - echo "##vso[task.setvariable variable=SERVICE_BASE_PATH_FOR_CONDITION]${{ parameters.service_base_path }}" + echo "##vso[task.setvariable variable=SERVICE_BASE_PATH_FOR_CONDITION]$service_base_path" + env: + service_base_path: ${{ parameters.service_base_path }} displayName: Set SERVICE_BASE_PATH_FOR_CONDITION - - bash: | set -e export CONTAINER_VARS_FILE="$SERVICE_DIR/ecs-proxies-containers.yml" - + source $(SERVICE_DIR)/.build_env_vars - export SERVICE_BASE_PATH="${{ parameters.service_base_path }}" + export SERVICE_BASE_PATH="$service_base_path" echo "SERVICE_BASE_PATH='${SERVICE_BASE_PATH}'" export ASSUMED_VERSION=$(echo $SERVICE_ARTIFACT_NAME | grep -E -o "v[0-9]+\.[0-9]+\.[0-9]+-?[a-z]*" || true | tail -1) - export DEPLOYED_VERSION=${ASSUMED_VERSION:-${{ parameters.fully_qualified_service_name }}} + export DEPLOYED_VERSION=${ASSUMED_VERSION:-$fully_qualified_service_name} - account=${{ parameters.aws_account }} \ + account=$aws_account \ CONTAINER_VARS_FILE="${CONTAINER_VARS_FILE}" \ SOURCE_COMMIT_ID="$(Build.SourceVersion)" \ RELEASE_RELEASEID="$(Build.BuildId)" \ make --no-print-directory -C $(UTILS_DIR)/ansible deploy-ecs-proxies-retag - + env: + service_base_path: ${{ parameters.service_base_path }} + aws_account: ${{ parameters.aws_account }} + fully_qualified_service_name: ${{ parameters.fully_qualified_service_name }} displayName: Retag ECS image condition: and(succeeded(), ne(variables['DEPLOY_ROLE'], ''), contains(variables['SERVICE_BASE_PATH_FOR_CONDITION'], 'canary'), not(contains(variables['SERVICE_BASE_PATH_FOR_CONDITION'], '-pr-'))) - - - template: ../components/aws-assume-role.yml parameters: role: "$(DEPLOY_ROLE)" profile: "$(DEPLOY_ROLE)" aws_account: "${{ parameters.aws_account }}" - + - bash: | set -e proxy_vars_file="$(PROXY_VARS_FILE)" @@ -235,10 +254,10 @@ steps: if [[ ! -z $ASSUMED_VERSION ]]; then export DEPLOYED_VERSION=$ASSUMED_VERSION else - export DEPLOYED_VERSION="${{ parameters.fully_qualified_service_name }}" + export DEPLOYED_VERSION="$fully_qualified_service_name" fi - - + + if [[ "$SERVICE_BASE_PATH_FOR_CONDITION" == *"-pr-"* ]]; then export TF_VAR_use_ecs_tag=false else @@ -250,161 +269,175 @@ steps: export use_ecs_tag=$TF_VAR_use_ecs_tag - account=${{ parameters.aws_account }} \ + account=$aws_account \ PROXY_VARS_FILE="${proxy_vars_file}" \ SOURCE_COMMIT_ID="$(Build.SourceVersion)" \ RELEASE_RELEASEID="$(Build.BuildId)" \ - SERVICE_BASE_PATH=${{ parameters.service_base_path }} \ - APIGEE_ENVIRONMENT=${{ parameters.apigee_environment }} \ + SERVICE_BASE_PATH=$service_base_path \ + APIGEE_ENVIRONMENT=$apigee_environment \ use_ecs_tag="${use_ecs_tag}" \ make --no-print-directory -C $(UTILS_DIR)/ansible deploy-ecs-proxies - + env: + aws_account: ${{ parameters.aws_account }} + service_base_path: ${{ parameters.service_base_path }} + apigee_environment: ${{ parameters.apigee_environment }} + fully_qualified_service_name: ${{ parameters.fully_qualified_service_name }} displayName: Deploy ECS proxies condition: and(succeeded(), ne(variables['DEPLOY_ROLE'], '')) - ${{ if parameters.proxy_path }}: - - bash: | - set -euo pipefail - - export PROXY_DIR="$(SERVICE_DIR)/proxies/${{ parameters.proxy_path }}" - export SERVICE_NAME="${{ parameters.service_name }}" - export FULLY_QUALIFIED_SERVICE_NAME="${{ parameters.fully_qualified_service_name }}" - export SERVICE_BASE_PATH="${{ parameters.service_base_path }}" - export APIGEE_ACCESS_TOKEN="$(secret.AccessToken)" - export APIGEE_ENVIRONMENT="${{ parameters.apigee_environment }}" - export APIGEE_ORGANIZATION="nhsd-${{ parameters.apigee_organization }}" - export PING="${{ parameters.ping }}" - export ANSIBLE_FORCE_COLOR=yes - - make --no-print-directory -C $(UTILS_DIR)/ansible deploy-apigee-proxy - displayName: Deploy proxy + - bash: | + set -euo pipefail + + export PROXY_DIR="$(SERVICE_DIR)/proxies/$proxy_path" + export SERVICE_NAME="$service_name" + export FULLY_QUALIFIED_SERVICE_NAME="$fully_qualified_service_name" + export SERVICE_BASE_PATH="$service_base_path" + export APIGEE_ACCESS_TOKEN="$(secret.AccessToken)" + export APIGEE_ENVIRONMENT="$apigee_environment" + export APIGEE_ORGANIZATION="nhsd-$apigee_organization" + export PING="$ping" + export ANSIBLE_FORCE_COLOR=yes + make --no-print-directory -C $(UTILS_DIR)/ansible deploy-apigee-proxy + env: + service_name: ${{ parameters.service_name }} + fully_qualified_service_name: ${{ parameters.fully_qualified_service_name }} + service_base_path: ${{ parameters.service_base_path }} + apigee_environment: ${{ parameters.apigee_environment }} + apigee_organization: ${{ parameters.apigee_organization }} + proxy_path: ${{ parameters.proxy_path }} + ping: ${{ parameters.ping }} + displayName: Deploy proxy - bash: | set -euo pipefail export APIGEE_ACCESS_TOKEN="$(secret.AccessToken)" - export APIGEE_ENVIRONMENT="${{ parameters.apigee_environment }}" - export APIGEE_ORGANIZATION="nhsd-${{ parameters.apigee_organization }}" + export APIGEE_ENVIRONMENT="$apigee_environment" + export APIGEE_ORGANIZATION="nhsd-$apigee_organization" export ANSIBLE_FORCE_COLOR=yes export DIST_DIR=$(SERVICE_DIR) - export PULL_REQUEST="${{ parameters.pr_label }}" + export PULL_REQUEST="$pr_label" make --no-print-directory -C $(UTILS_DIR)/ansible deploy-manifest - displayName: 'Deploy Manifest' + env: + apigee_environment: ${{ parameters.apigee_environment }} + apigee_organization: ${{ parameters.apigee_organization }} + pr_label: ${{ parameters.pr_label }} + displayName: "Deploy Manifest" - ${{ if parameters.proxy_path }}: - - bash: | - set -euo pipefail - - echo "Setting stage name as snake case for monitoring" - export MONITORING_STAGE_NAME="${{ replace(parameters.stage_name, '_', '-') }}" - echo "MONITORING_STAGE_NAME=${MONITORING_STAGE_NAME}" - echo "##vso[task.setvariable variable=MONITORING_STAGE_NAME]$MONITORING_STAGE_NAME" - - if [[ "${{ parameters.apigee_environment }}" == "prod" ]]; then - export url="https://api.service.nhs.uk/monitoring-sd/service" - export status_body='{ "${{ parameters.service_name }}": { "${{ parameters.apigee_environment }}": [ "${{ parameters.service_name }}@$(MONITORING_STAGE_NAME)=http_2xx_with_api_key https://api.service.nhs.uk/${{ parameters.service_base_path }}/_status" ] } }' - export ping_body='{ "${{ parameters.service_name }}": { "${{ parameters.apigee_environment }}": [ "${{ parameters.service_name }}@$(MONITORING_STAGE_NAME)=http_2xx https://api.service.nhs.uk/${{ parameters.service_base_path }}/_ping" ] } }' - else - export url="https://internal-dev.api.service.nhs.uk/monitoring-sd/service" - export status_body='{ "${{ parameters.service_name }}": { "${{ parameters.apigee_environment }}": [ "${{ parameters.service_name }}@$(MONITORING_STAGE_NAME)=http_2xx_with_api_key https://${{ parameters.apigee_environment }}.api.service.nhs.uk/${{ parameters.service_base_path }}/_status" ] } }' - export ping_body='{ "${{ parameters.service_name }}": { "${{ parameters.apigee_environment }}": [ "${{ parameters.service_name }}@$(MONITORING_STAGE_NAME)=http_2xx https://${{ parameters.apigee_environment }}.api.service.nhs.uk/${{ parameters.service_base_path }}/_ping" ] } }' - fi - - echo "##vso[task.setvariable variable=is_pull_request]false" - echo "##vso[task.setvariable variable=check_and_enable_ping]${{ parameters.enable_monitoring }}" - echo "##vso[task.setvariable variable=check_and_enable_status]${{ parameters.enable_status_monitoring }}" - - if [[ "${{ parameters.service_base_path }}" == *"-pr-"* ]]; then - echo "##vso[task.setvariable variable=check_and_enable_ping]false" - echo "##vso[task.setvariable variable=check_and_enable_status]false" - echo "##vso[task.setvariable variable=is_pull_request]true" - fi - - echo "##vso[task.setvariable variable=url]$url" - echo "##vso[task.setvariable variable=status_body]$status_body" - echo "##vso[task.setvariable variable=ping_body]$ping_body" - displayName: 'Set monitoring variables' + - bash: | + set -euo pipefail + + echo "Setting stage name as snake case for monitoring" + export MONITORING_STAGE_NAME="${{ replace(parameters.stage_name, '_', '-') }}" + echo "MONITORING_STAGE_NAME=${MONITORING_STAGE_NAME}" + echo "##vso[task.setvariable variable=MONITORING_STAGE_NAME]$MONITORING_STAGE_NAME" + + if [[ "${{ parameters.apigee_environment }}" == "prod" ]]; then + export url="https://api.service.nhs.uk/monitoring-sd/service" + export status_body='{ "${{ parameters.service_name }}": { "${{ parameters.apigee_environment }}": [ "${{ parameters.service_name }}@$(MONITORING_STAGE_NAME)=http_2xx_with_api_key https://api.service.nhs.uk/${{ parameters.service_base_path }}/_status" ] } }' + export ping_body='{ "${{ parameters.service_name }}": { "${{ parameters.apigee_environment }}": [ "${{ parameters.service_name }}@$(MONITORING_STAGE_NAME)=http_2xx https://api.service.nhs.uk/${{ parameters.service_base_path }}/_ping" ] } }' + else + export url="https://internal-dev.api.service.nhs.uk/monitoring-sd/service" + export status_body='{ "${{ parameters.service_name }}": { "${{ parameters.apigee_environment }}": [ "${{ parameters.service_name }}@$(MONITORING_STAGE_NAME)=http_2xx_with_api_key https://${{ parameters.apigee_environment }}.api.service.nhs.uk/${{ parameters.service_base_path }}/_status" ] } }' + export ping_body='{ "${{ parameters.service_name }}": { "${{ parameters.apigee_environment }}": [ "${{ parameters.service_name }}@$(MONITORING_STAGE_NAME)=http_2xx https://${{ parameters.apigee_environment }}.api.service.nhs.uk/${{ parameters.service_base_path }}/_ping" ] } }' + fi + + echo "##vso[task.setvariable variable=is_pull_request]false" + echo "##vso[task.setvariable variable=check_and_enable_ping]${{ parameters.enable_monitoring }}" + echo "##vso[task.setvariable variable=check_and_enable_status]${{ parameters.enable_status_monitoring }}" + + if [[ "${{ parameters.service_base_path }}" == *"-pr-"* ]]; then + echo "##vso[task.setvariable variable=check_and_enable_ping]false" + echo "##vso[task.setvariable variable=check_and_enable_status]false" + echo "##vso[task.setvariable variable=is_pull_request]true" + fi + + echo "##vso[task.setvariable variable=url]$url" + echo "##vso[task.setvariable variable=status_body]$status_body" + echo "##vso[task.setvariable variable=ping_body]$ping_body" + displayName: "Set monitoring variables" - - bash: | - set -euo pipefail + - bash: | + set -euo pipefail - if [[ "${{ parameters.apigee_environment }}" == "prod" ]]; then - export ping_endpoint_response=`curl -s -o /dev/null -w '%{http_code}' -H "apikey: $(status-endpoint-api-key)" https://api.service.nhs.uk/${{ parameters.service_base_path }}/_ping` - else - export ping_endpoint_response=`curl -s -o /dev/null -w '%{http_code}' -H "apikey: $(status-endpoint-api-key)" https://${{ parameters.apigee_environment }}.api.service.nhs.uk/${{ parameters.service_base_path }}/_ping` - fi + if [[ "${{ parameters.apigee_environment }}" == "prod" ]]; then + export ping_endpoint_response=`curl -s -o /dev/null -w '%{http_code}' -H "apikey: $(status-endpoint-api-key)" https://api.service.nhs.uk/${{ parameters.service_base_path }}/_ping` + else + export ping_endpoint_response=`curl -s -o /dev/null -w '%{http_code}' -H "apikey: $(status-endpoint-api-key)" https://${{ parameters.apigee_environment }}.api.service.nhs.uk/${{ parameters.service_base_path }}/_ping` + fi - echo _ping_response=$ping_endpoint_response - echo "##vso[task.setvariable variable=ping_endpoint_response]$ping_endpoint_response" + echo _ping_response=$ping_endpoint_response + echo "##vso[task.setvariable variable=ping_endpoint_response]$ping_endpoint_response" - if [[ $ping_endpoint_response == "404" ]]; then - echo "##vso[task.logissue type=error]Your proxy doesn't have a _ping endpoint therefore we can't monitor this proxy and it should not be released, use the flag 'enable_monitoring=false' if your API doesn't support _ping healthcheck." - exit 1 - fi + if [[ $ping_endpoint_response == "404" ]]; then + echo "##vso[task.logissue type=error]Your proxy doesn't have a _ping endpoint therefore we can't monitor this proxy and it should not be released, use the flag 'enable_monitoring=false' if your API doesn't support _ping healthcheck." + exit 1 + fi - if [[ $ping_endpoint_response != "200" ]]; then - echo "##vso[task.logissue type=error]Your proxy's _ping endpoint has encounted an error and should not be released until it is resolved." - exit 1 - fi - displayName: 'Check _ping' - condition: and(succeeded(), eq(variables['check_and_enable_ping'], 'true')) + if [[ $ping_endpoint_response != "200" ]]; then + echo "##vso[task.logissue type=error]Your proxy's _ping endpoint has encounted an error and should not be released until it is resolved." + exit 1 + fi + displayName: "Check _ping" + condition: and(succeeded(), eq(variables['check_and_enable_ping'], 'true')) - - bash: | - set -euo pipefail + - bash: | + set -euo pipefail - if [[ "${{ parameters.apigee_environment }}" == "prod" ]]; then - export status_endpoint_response=`curl -s -o /dev/null -w '%{http_code}' -H "apikey: $(status-endpoint-api-key)" https://api.service.nhs.uk/${{ parameters.service_base_path }}/_status` - else - export status_endpoint_response=`curl -s -o /dev/null -w '%{http_code}' -H "apikey: $(status-endpoint-api-key)" https://${{ parameters.apigee_environment }}.api.service.nhs.uk/${{ parameters.service_base_path }}/_status` - fi + if [[ "${{ parameters.apigee_environment }}" == "prod" ]]; then + export status_endpoint_response=`curl -s -o /dev/null -w '%{http_code}' -H "apikey: $(status-endpoint-api-key)" https://api.service.nhs.uk/${{ parameters.service_base_path }}/_status` + else + export status_endpoint_response=`curl -s -o /dev/null -w '%{http_code}' -H "apikey: $(status-endpoint-api-key)" https://${{ parameters.apigee_environment }}.api.service.nhs.uk/${{ parameters.service_base_path }}/_status` + fi - echo _status_response=$status_endpoint_response - echo "##vso[task.setvariable variable=status_endpoint_response]$status_endpoint_response" + echo _status_response=$status_endpoint_response + echo "##vso[task.setvariable variable=status_endpoint_response]$status_endpoint_response" - if [[ $status_endpoint_response == "404" ]]; then - echo "##vso[task.logissue type=error]Your proxy doesn't have a _status endpoint therefore we can't monitor this proxy and it should not be released, use the flag 'enable_status_monitoring=false' if your API doesn't support _status healthcheck." - exit 1 - fi - - if [[ $status_endpoint_response != "200" ]]; then - echo "##vso[task.logissue type=error]Your proxy's _status endpoint has encounted an error and should not be released until it is resolved." - exit 1 - fi - displayName: 'Check _status' - condition: and(succeeded(), eq(variables['check_and_enable_status'], 'true')) + if [[ $status_endpoint_response == "404" ]]; then + echo "##vso[task.logissue type=error]Your proxy doesn't have a _status endpoint therefore we can't monitor this proxy and it should not be released, use the flag 'enable_status_monitoring=false' if your API doesn't support _status healthcheck." + exit 1 + fi - - bash: | - set -euo pipefail + if [[ $status_endpoint_response != "200" ]]; then + echo "##vso[task.logissue type=error]Your proxy's _status endpoint has encounted an error and should not be released until it is resolved." + exit 1 + fi + displayName: "Check _status" + condition: and(succeeded(), eq(variables['check_and_enable_status'], 'true')) - curl --fail -X 'POST' -H 'apikey: $(MONITORING_API_KEY)' -d '$(status_body)' $(url) - displayName: Enable _status monitoring - condition: and(succeeded(), eq(variables['check_and_enable_status'], 'true')) + - bash: | + set -euo pipefail - - bash: | - set -euo pipefail + curl --fail -X 'POST' -H 'apikey: $(MONITORING_API_KEY)' -d '$(status_body)' $(url) + displayName: Enable _status monitoring + condition: and(succeeded(), eq(variables['check_and_enable_status'], 'true')) - curl --fail -X 'DELETE' -H 'apikey: $(MONITORING_API_KEY)' -d '$(status_body)' $(url) - echo - echo "_status endpoint monitoring is disabled" - displayName: Disable _status monitoring - condition: and(succeeded(), eq(variables['check_and_enable_status'], 'false'), eq(variables['is_pull_request'], 'false')) + - bash: | + set -euo pipefail + curl --fail -X 'DELETE' -H 'apikey: $(MONITORING_API_KEY)' -d '$(status_body)' $(url) + echo + echo "_status endpoint monitoring is disabled" + displayName: Disable _status monitoring + condition: and(succeeded(), eq(variables['check_and_enable_status'], 'false'), eq(variables['is_pull_request'], 'false')) - - bash: | - set -euo pipefail + - bash: | + set -euo pipefail - curl --fail -X 'POST' -H 'apikey: $(MONITORING_API_KEY)' -d '$(ping_body)' $(url) - displayName: Enable _ping monitoring - condition: and(succeeded(), eq(variables['check_and_enable_ping'], 'true')) + curl --fail -X 'POST' -H 'apikey: $(MONITORING_API_KEY)' -d '$(ping_body)' $(url) + displayName: Enable _ping monitoring + condition: and(succeeded(), eq(variables['check_and_enable_ping'], 'true')) - - bash: | - set -euo pipefail + - bash: | + set -euo pipefail - curl --fail -X 'DELETE' -H 'apikey: $(MONITORING_API_KEY)' -d '$(ping_body)' $(url) - echo - echo "_ping endpoint monitoring is disabled" - displayName: Disable _ping monitoring - condition: and(succeeded(), eq(variables['check_and_enable_ping'], 'false'), eq(variables['is_pull_request'], 'false')) + curl --fail -X 'DELETE' -H 'apikey: $(MONITORING_API_KEY)' -d '$(ping_body)' $(url) + echo + echo "_ping endpoint monitoring is disabled" + displayName: Disable _ping monitoring + condition: and(succeeded(), eq(variables['check_and_enable_ping'], 'false'), eq(variables['is_pull_request'], 'false')) - ${{ each post_deploy_step in parameters.post_deploy }}: - - ${{ post_deploy_step }} + - ${{ post_deploy_step }} diff --git a/azure/templates/setup-build-name.yml b/azure/templates/setup-build-name.yml index 0d54059a..07258990 100644 --- a/azure/templates/setup-build-name.yml +++ b/azure/templates/setup-build-name.yml @@ -8,11 +8,15 @@ steps: if [ -n "$(System.PullRequest.SourceBranch)" ]; then export SOURCE_BRANCH="$(System.PullRequest.SourceBranch)" else - export SOURCE_BRANCH=`echo $(SERVICE_ARTIFACT_NAME) | cut -d'+' -f1 | sed -e "s/${{ parameters.service_name }}-//"` + export SOURCE_BRANCH=`echo $(SERVICE_ARTIFACT_NAME) | cut -d'+' -f1 | sed -e "s/$service_name-//"` fi echo "##vso[task.setvariable variable=BRANCH_NAME]`echo $SOURCE_BRANCH | sed -r 's/[/|\\@":<>?*]+/-/g'`" + env: + service_name: ${{ parameters.service_name }} displayName: Set and replace invalid characters in branch name - - bash: 'echo "##vso[build.updatebuildnumber]${{ parameters.service_name }}-$(BRANCH_NAME)+$(Build.BuildID)"' + - bash: 'echo "##vso[build.updatebuildnumber]$service_name-$(BRANCH_NAME)+$(Build.BuildID)"' condition: eq(variables['Build.SourceBranchName'], 'merge') displayName: Update build ID if this is a pull request + env: + service_name: ${{ parameters.service_name }} diff --git a/azure/utils-pr-pipeline.yml b/azure/utils-pr-pipeline.yml index 7fff0c5f..bfdf4915 100644 --- a/azure/utils-pr-pipeline.yml +++ b/azure/utils-pr-pipeline.yml @@ -10,7 +10,7 @@ trigger: pr: branches: - include: ['*'] + include: ["*"] resources: repositories: @@ -20,12 +20,12 @@ resources: ref: refs/heads/master endpoint: NHSDigital -jobs: +jobs: - job: test displayName: Test ansible timeoutInMinutes: 30 pool: - name: 'AWS-ECS' + name: "AWS-ECS" workspace: clean: all steps: @@ -38,20 +38,13 @@ jobs: echo "##vso[task.setvariable variable=PY_VER]$PATCH" echo "Resolved latest python version: $PATCH" echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/" - displayName: 'Query and set python tool cache path' - + displayName: "Query and set python tool cache path" - task: UsePythonVersion@0 name: UsePy - displayName: 'Use Python 3.13' + displayName: "Use Python 3.13" inputs: - versionSpec: '$(PY_VER)' - - # - bash: | - # echo "Checking the python version in use to set LD_LIBRARY_PATH" - # echo "Python location: $(UsePy.pythonLocation)" - # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" - # displayName: 'Set LD_LIBRARY_PATH' + versionSpec: "$(PY_VER)" - bash: | instance_id="$(curl -s http://169.254.169.254/latest/meta-data/instance-id)" @@ -78,36 +71,42 @@ jobs: - /ptl/azure-devops/GITHUB_USER - bash: | - echo "Build.SourceBranch: $(Build.SourceBranch)" - echo "Build.SourceBranchName: $(Build.SourceBranchName)" - echo "Build.SourceVersion: $(Build.SourceVersion)" - echo "Build.SourceVersionMessage: $(Build.SourceVersionMessage)" - - if [[ ! -z $(NOTIFY_COMMIT_SHA) ]]; then - echo "##[debug]Using already provided NOTIFY_COMMIT_SHA=$(NOTIFY_COMMIT_SHA)" + echo "Build.SourceBranch: $Build_SourceBranch" + echo "Build.SourceBranchName: $Build_SourceBranchName" + echo "Build.SourceVersion: $Build_SourceVersion" + echo "Build.SourceVersionMessage: $Build_SourceVersionMessage" + + if [[ ! -z $NOTIFY_COMMIT_SHA ]]; then + echo "##[debug]Using already provided NOTIFY_COMMIT_SHA=$NOTIFY_COMMIT_SHA" else NOTIFY_COMMIT_SHA="" - if [[ "$(Build.SourceBranch)" =~ ^refs/tags/.+$ ]]; then + if [[ "$Build_SourceBranch" =~ ^refs/tags/.+$ ]]; then echo "##[debug]Build appears to be a tag build" echo "##[debug]Using Build.SourceVersion as NOTIFY_COMMIT_SHA" - NOTIFY_COMMIT_SHA="$(Build.SourceVersion)" + NOTIFY_COMMIT_SHA="$Build_SourceVersion" fi - if [[ "$(Build.SourceBranch)" =~ ^refs/pull/.+$ ]]; then + if [[ "$Build_SourceBranch" =~ ^refs/pull/.+$ ]]; then echo "##[debug]Build appears to be a pull request build" echo "##[debug]Extracting NOTIFY_COMMIT_SHA from Build.SourceVersionMessage" - NOTIFY_COMMIT_SHA=`echo "$(Build.SourceVersionMessage)" | cut -d' ' -f2` + NOTIFY_COMMIT_SHA=`echo "$Build_SourceVersionMessage" | cut -d' ' -f2` fi if [[ -z $NOTIFY_COMMIT_SHA ]]; then echo "##[debug]Build does not appear to be pull or tag build" echo "##[debug]Using Build.SourceVersion as NOTIFY_COMMIT_SHA" - NOTIFY_COMMIT_SHA="$(Build.SourceVersion)" + NOTIFY_COMMIT_SHA="$Build.SourceVersion" fi echo "##vso[task.setvariable variable=NOTIFY_COMMIT_SHA]$NOTIFY_COMMIT_SHA" fi + env: + Build_SourceBranch: $(Build.SourceBranch) + Build_SourceBranchName: $(Build.SourceBranchName) + Build_SourceVersion: $(Build.SourceVersion) + Build_SourceVersionMessage: $(Build.SourceVersionMessage) + NOTIFY_COMMIT_SHA: $(NOTIFY_COMMIT_SHA) displayName: Set NOTIFY_COMMIT_SHA condition: always() @@ -115,7 +114,7 @@ jobs: parameters: state: pending description: "Testing ansible collection" - + - bash: | ls -R make install @@ -130,7 +129,7 @@ jobs: ANSIBLE_FORCE_COLOR=yes poetry run ansible-test integration --python=3.13 workingDirectory: ansible/collections/ansible_collections/nhsd/apigee displayName: Integration test ansible - + - template: ./components/update-github-status.yml parameters: state: success diff --git a/env_cleaner.py b/env_cleaner.py index cc912e69..59db4e96 100644 --- a/env_cleaner.py +++ b/env_cleaner.py @@ -19,13 +19,13 @@ --min-age= (NOT IMPLEMENTED) Minimum age in seconds --respect-prs Don't undeploy if there's a PR open for it """ + import re import requests from typing import Optional, Set from docopt import docopt from apigee_client import ApigeeClient - SPEC_MATCHER = re.compile("(personal-demographics|identity-service|hello-world)-.+") PROXY_MATCHER = re.compile( "(personal-demographics|identity-service|hello-world)-internal-dev-.+" @@ -49,7 +49,7 @@ def canonicalize(name: str) -> str: class GithubClient: def get_open_prs(self, env: str) -> Set[str]: - """ Returns names of open pull requests """ + """Returns names of open pull requests""" canonical_prs = set() for repo_name in REPO_NAMES.keys(): prs = requests.get( @@ -65,7 +65,7 @@ def get_open_prs(self, env: str) -> Set[str]: return canonical_prs -def clean_specs(client: ApigeeClient, env: str, dry_run: bool = False): +def clean_specs(client: ApigeeClient, dry_run: bool = False): specs = client.list_specs() spec_names = [spec["name"] for spec in specs["contents"]] @@ -85,10 +85,10 @@ def clean_proxies( env: str, dry_run: bool = False, sandboxes_only: bool = False, - min_age: Optional[int] = None, undeploy_only: bool = False, respect_prs: bool = False, ): + open_prs = set() if respect_prs: # Get open PRs, if there are any @@ -135,7 +135,7 @@ def clean_proxies( client.delete_proxy(proxy) -def clean_products(client: ApigeeClient, env: str, dry_run: bool = False): +def clean_products(client: ApigeeClient, dry_run: bool = False): products = client.list_products() pr_products = [product for product in products if PRODUCT_MATCHER.match(product)] @@ -154,12 +154,11 @@ def clean_env( should_clean_products: bool = False, sandboxes_only: bool = False, dry_run: bool = False, - min_age: Optional[int] = None, undeploy_only: bool = False, respect_prs: bool = False, ): if should_clean_specs: - clean_specs(client, env, dry_run) + clean_specs(client, dry_run) if should_clean_proxies: clean_proxies( @@ -168,13 +167,12 @@ def clean_env( env, dry_run, sandboxes_only, - min_age, undeploy_only, respect_prs, ) if should_clean_products: - clean_products(client, env, dry_run) + clean_products(client, dry_run) if __name__ == "__main__": @@ -190,7 +188,6 @@ def clean_env( should_clean_products=args["--products"], sandboxes_only=args["--sandbox-only"], dry_run=args["--dry-run"], - min_age=args["--min-age"], undeploy_only=args["--undeploy-only"], respect_prs=args["--respect-prs"], ) diff --git a/scripts/calculate_version.py b/scripts/calculate_version.py index 56c8dc7a..35c84a5d 100644 --- a/scripts/calculate_version.py +++ b/scripts/calculate_version.py @@ -21,7 +21,6 @@ import git import semver - SCRIPT_LOCATION = os.path.join(os.path.dirname(os.path.abspath(__file__))) REPO_ROOT = os.path.abspath(os.path.join(SCRIPT_LOCATION, "..")) REPO = git.Repo(REPO_ROOT) @@ -67,11 +66,10 @@ def without_empty(commits): yield commits[-1] -def calculate_version(base_major=1, base_minor=0, base_revision=0, base_pre="alpha"): +def calculate_version(base_major=1, base_minor=0, base_pre="alpha"): """Calculates a semver based on commit history and special flags in commit messages""" major = base_major minor = base_minor - patch = base_revision pre = base_pre commits = get_versionable_commits(REPO) @@ -96,7 +94,6 @@ def calculate_version(base_major=1, base_minor=0, base_revision=0, base_pre="alp if major_incs: major += len(major_incs) minor = 0 - patch = 0 # If there are any +minor in commit messages, increment the counter # We only care about commits after the last major increment @@ -105,7 +102,6 @@ def calculate_version(base_major=1, base_minor=0, base_revision=0, base_pre="alp if minor_incs: minor += len(minor_incs) - patch = 0 # Now increment patch number for every commit since the last patch commits = list(itertools.takewhile(lambda c: not is_minor_inc(c), commits)) diff --git a/scripts/githooks/scan-secrets.sh b/scripts/githooks/scan-secrets.sh index be4b7914..5cca0510 100644 --- a/scripts/githooks/scan-secrets.sh +++ b/scripts/githooks/scan-secrets.sh @@ -34,6 +34,7 @@ function main() { dir="/workdir" cmd="$(get-cmd-to-run)" run-gitleaks-in-docker fi + return 0 } # Get Gitleaks command to execute and configuration. @@ -52,15 +53,20 @@ function get-cmd-to-run() { "staged-changes") cmd="protect --source $dir --verbose --staged" ;; + *) + echo "ERROR: Unknown check type: $check" >&2 + return 1 + ;; esac # Include base line file if it exists - if [ -f "$dir/scripts/config/.gitleaks-baseline.json" ]; then + if [[ -f "$dir/scripts/config/.gitleaks-baseline.json" ]]; then cmd="$cmd --baseline-path $dir/scripts/config/.gitleaks-baseline.json" fi # Include the config file cmd="$cmd --config $dir/scripts/config/gitleaks.toml" echo "$cmd" + return 0 } # Run Gitleaks natively. @@ -70,6 +76,7 @@ function run-gitleaks-natively() { # shellcheck disable=SC2086 gitleaks $cmd + return 0 } # Run Gitleaks in a Docker container. @@ -89,13 +96,15 @@ function run-gitleaks-in-docker() { --workdir $dir \ "$image" \ $cmd + return 0 } # ============================================================================== function is-arg-true() { + local arg="$1" - if [[ "$1" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then + if [[ "$arg" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then return 0 else return 1 diff --git a/scripts/purge_incorrect_env_vars.py b/scripts/purge_incorrect_env_vars.py index c076a9d8..0d166369 100644 --- a/scripts/purge_incorrect_env_vars.py +++ b/scripts/purge_incorrect_env_vars.py @@ -18,25 +18,27 @@ -t | --token= Auth token (get_token) -c --check Will show the data due to be deleted """ + import json import requests from docopt import docopt + def find_offending_entires(json_data, offending_str): - if check_mode: - print("*************************** Check mode *****************************") - print("* The following items would be deleted when run outside of check mode *") - for api in json_data: - for env in json_data[api]: - to_keep = [] - for line in json_data[api][env]: - if offending_str not in line: - to_keep.append(line) - else: - if check_mode: - print(" -", line) - json_data[api][env] = to_keep - return json_data + if check_mode: + print("*************************** Check mode *****************************") + print("* The following items would be deleted when run outside of check mode *") + for api in json_data: + for env in json_data[api]: + to_keep = [] + for line in json_data[api][env]: + if offending_str not in line: + to_keep.append(line) + elif check_mode: + print(" -", line) + json_data[api][env] = to_keep + return json_data + def post_new_data(new_json_data): url = f"https://api.enterprise.apigee.com/v1/organizations/{org_name}/environments/{env_name}/keyvaluemaps/{map_name}/entries/{entry_name}" @@ -46,7 +48,8 @@ def post_new_data(new_json_data): post_new_json_data = requests.post(url, json=format_new_json, headers=auth) print(post_new_json_data.text) - + + def main(args): global map_name, entry_name, token, check_mode input_value = str(args["--value"]) @@ -63,12 +66,13 @@ def main(args): url = f"https://api.enterprise.apigee.com/v1/organizations/{org_name}/environments/{env_name}/keyvaluemaps/{map_name}/entries/{entry_name}" resp = requests.get(url, headers=auth) - json_data = json.loads(json.loads(resp.text).get('value')) + json_data = json.loads(json.loads(resp.text).get("value")) new_json_data = find_offending_entires(json_data, input_value) if not check_mode: - post_new_data(new_json_data) + post_new_data(new_json_data) + if __name__ == "__main__": - main(docopt(__doc__, version="1")) + main(docopt(__doc__, version="1")) diff --git a/scripts/terraform_force_unlock.py b/scripts/terraform_force_unlock.py index 7e631f97..9cbf1ef9 100644 --- a/scripts/terraform_force_unlock.py +++ b/scripts/terraform_force_unlock.py @@ -46,14 +46,14 @@ def main(min_age_hr, key_prefix, table_name, profile): filter_expr = "begins_with(#n0, :v0) AND attribute_exists(#n1)" - ExpressionAttributeNames = {"#n0": "LockID", "#n1": "Info"} - ExpressionAttributeValues = { + expression_attribute_names = {"#n0": "LockID", "#n1": "Info"} + expression_attribute_values = { ":v0": key_prefix, } items = terraform_lock_table.scan( FilterExpression=filter_expr, - ExpressionAttributeNames=ExpressionAttributeNames, - ExpressionAttributeValues=ExpressionAttributeValues, + ExpressionAttributeNames=expression_attribute_names, + ExpressionAttributeValues=expression_attribute_values, ) total_items = items["Items"] @@ -61,8 +61,8 @@ def main(min_age_hr, key_prefix, table_name, profile): while "LastEvaluatedKey" in items: items = terraform_lock_table.scan( FilterExpression=filter_expr, - ExpressionAttributeNames=ExpressionAttributeNames, - ExpressionAttributeValues=ExpressionAttributeValues, + ExpressionAttributeNames=expression_attribute_names, + ExpressionAttributeValues=expression_attribute_values, ExclusiveStartKey=items["LastEvaluatedKey"], ) total_items.extend(items["Items"]) diff --git a/scripts/test_pull_request_deployments.py b/scripts/test_pull_request_deployments.py index 39d56074..ed861dfb 100644 --- a/scripts/test_pull_request_deployments.py +++ b/scripts/test_pull_request_deployments.py @@ -3,13 +3,8 @@ from multiprocessing import Process from trigger_pipelines import AzureDevOps - PULL_REQUEST_PIPELINES = { - "canary-api": { - "build": 222, - "pr": 223, - "branch": "refs/heads/main" - } + "canary-api": {"build": 222, "pr": 223, "branch": "refs/heads/main"} } @@ -19,17 +14,10 @@ def trigger_pipelines(pipeline_ids: dict, service: str): service=service, pipeline_type="build", pipeline_id=pipeline_ids["build"], - pipeline_branch=pipeline_ids["branch"] + pipeline_branch=pipeline_ids["branch"], ) if build_status != "succeeded": sys.exit(1) - return - # azure_dev_ops.run_pipeline( - # service=service, - # pipeline_type="pr", - # pipeline_id=pipeline_ids["pr"], - # pipeline_branch=pipeline_ids["branch"] - # ) def main(): @@ -37,7 +25,10 @@ def main(): for service, pipeline_ids in PULL_REQUEST_PIPELINES.items(): process = Process( target=trigger_pipelines, - args=(pipeline_ids, service,) + args=( + pipeline_ids, + service, + ), ) process.start() jobs.append(process)