diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index ebe0477..b7378bb 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6007,6 +6007,91 @@ def apic_vmm_inventory_sync_faults_check(**kwargs): recommended_action=recommended_action, doc_url=doc_url) + +@check_wrapper(check_title='Snapshot files check') +def snapshot_files_check(fabric_nodes, cversion, username, password, **kwargs): + result = PASS + headers = ['apic_id', 'apic_name', 'snapshot_files'] + data = [] + recommended_action = 'Contact Cisco TAC for Support before upgrade' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#Snapshot-files-check' + if cversion.older_than('6.0(3d)'): + apics = [node for node in fabric_nodes if node["fabricNode"]["attributes"]["role"] == "controller"] + if not apics: + return Result(result=ERROR, msg="No fabricNode of APIC. Is the cluster healthy?", doc_url=doc_url) + # `fabricNode` in pre-4.0 does not have `address` + if not apics[0]["fabricNode"]["attributes"].get("address"): + apic1 = [apic for apic in apics if apic["fabricNode"]["attributes"]["id"] == "1"][0] + apic1_dn = apic1["fabricNode"]["attributes"]["dn"] + apics = icurl("class", "{}/infraWiNode.json".format(apic1_dn)) + has_error = False + for apic in apics: + if apic.get("fabricNode"): + apic_id = apic["fabricNode"]["attributes"]["id"] + apic_name = apic["fabricNode"]["attributes"]["name"] + apic_addr = apic["fabricNode"]["attributes"]["address"] + else: + apic_id = apic["infraWiNode"]["attributes"]["id"] + apic_name = apic["infraWiNode"]["attributes"]["nodeName"] + apic_addr = apic["infraWiNode"]["attributes"]["addr"] + try: + c = Connection(apic_addr) + c.username = username + c.password = password + c.log = LOG_FILE + c.connect() + except Exception as e: + data.append([apic_id, apic_name, str(e)]) + has_error = True + continue + try: + c.cmd('tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404') + access_logs = c.output.splitlines() + if len(access_logs) < 15 and any("No such file or directory" in line for line in access_logs): + data.append([apic_id, apic_name, '/var/log/dme/log/access.log not found']) + has_error = True + continue + + requests = [] + + for line in access_logs: + timestamp_match = re.search(r'\[(\d{1,2}/\w{3}/\d{4}):(\d{2}:\d{2}:\d{2})', line) + filename_match = re.search(r'GET /snapshots/([^\s]+)', line) + + if timestamp_match and filename_match: + timestamp_str = f"{timestamp_match.group(1)}:{timestamp_match.group(2)}" + filename = filename_match.group(1) + try: + timestamp = datetime.strptime(timestamp_str, "%d/%b/%Y:%H:%M:%S") + requests.append((timestamp, filename)) + except: + continue + + requests.sort() + + # Checking if any 10 consecutive requests are within 1 minute + if len(requests) >= 10: + for i in range(len(requests) - 9): + time_diff = (requests[i + 9][0] - requests[i][0]).total_seconds() + if time_diff <= 60: + window_files = [filename for _, filename in requests[i:i+10]] + for filename in window_files: + data.append([apic_id, apic_name, filename]) + break + + except Exception as e: + data.append([apic_id, apic_name, str(e)]) + has_error = True + continue + + if has_error: + result = ERROR + elif data: + result = FAIL_UF + + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + + # ---- Script Execution ---- @@ -6168,6 +6253,7 @@ class CheckManager: standby_sup_sync_check, isis_database_byte_check, configpush_shard_check, + snapshot_files_check, ] ssh_checks = [ diff --git a/docs/docs/validations.md b/docs/docs/validations.md index fa1fc0e..144371e 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -191,6 +191,7 @@ Items | Defect | This Script [Stale pconsRA Object][d26] | CSCwp22212 | :warning:{title="Deprecated"} | :no_entry_sign: [ISIS DTEPs Byte Size][d27] | CSCwp15375 | :white_check_mark: | :no_entry_sign: [Policydist configpushShardCont Crash][d28] | CSCwp95515 | :white_check_mark: | +[Snapshot files check][d29] | CSCwe07002 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -220,7 +221,7 @@ Items | Defect | This Script [d26]: #stale-pconsra-object [d27]: #isis-dteps-byte-size [d28]: #policydist-configpushshardcont-crash - +[d29]: #Snapshot-files-check ## General Check Details @@ -2614,6 +2615,18 @@ Due to [CSCwp95515][59], upgrading to an affected version while having any `conf If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade. +### Snapshot files check + +RCA: +Issue occured in 3 node apic cluster, AE process on one apic is busy. Reason is it's trying to fetch the snapshot files which was taken earlier is missing on all apics. + +IMPACT: +Access logs will be flooded with the GET calls. APIC Upgrade/downgrade will fail with message Installer Exited - Pre-upgrade callbacks were not completed. + +Suggestion: +Restart AE on each APIC one at a time. For Reference [CSCwe07002][62]. + + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html @@ -2676,3 +2689,4 @@ If any instances of `configpushShardCont` are flagged by this script, Cisco TAC [59]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp95515 [60]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#Inter [61]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#EnablePolicyCompression +[62]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwe07002 \ No newline at end of file diff --git a/tests/checks/snapshot_files_check/fabricNode.json b/tests/checks/snapshot_files_check/fabricNode.json new file mode 100644 index 0000000..962a4ad --- /dev/null +++ b/tests/checks/snapshot_files_check/fabricNode.json @@ -0,0 +1,93 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.1", + "dn": "topology/pod-1/node-1", + "fabricSt": "commissioned", + "id": "1", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic1", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.2", + "dn": "topology/pod-1/node-2", + "fabricSt": "commissioned", + "id": "2", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic2", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.3", + "dn": "topology/pod-2/node-3", + "fabricSt": "commissioned", + "id": "3", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic3", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.101", + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "id": "101", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf101", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.102", + "dn": "topology/pod-1/node-102", + "fabricSt": "active", + "id": "102", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf102", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.201", + "dn": "topology/pod-1/node-201", + "fabricSt": "active", + "id": "201", + "model": "N9K-C9504", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine201", + "nodeType": "unspecified", + "role": "spine" + } + } + } +] + diff --git a/tests/checks/snapshot_files_check/fabricNode_no_apic.json b/tests/checks/snapshot_files_check/fabricNode_no_apic.json new file mode 100644 index 0000000..254f40d --- /dev/null +++ b/tests/checks/snapshot_files_check/fabricNode_no_apic.json @@ -0,0 +1,48 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.101", + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "id": "101", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf101", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.102", + "dn": "topology/pod-1/node-102", + "fabricSt": "active", + "id": "102", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf102", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.201", + "dn": "topology/pod-1/node-201", + "fabricSt": "active", + "id": "201", + "model": "N9K-C9504", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine201", + "nodeType": "unspecified", + "role": "spine" + } + } + } +] + diff --git a/tests/checks/snapshot_files_check/fabricNode_old.json b/tests/checks/snapshot_files_check/fabricNode_old.json new file mode 100644 index 0000000..f71fb9f --- /dev/null +++ b/tests/checks/snapshot_files_check/fabricNode_old.json @@ -0,0 +1,62 @@ +[ + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-1", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "1", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic1", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-2", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "2", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic2", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-2/node-3", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "3", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic3", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "nodeType": "unspecified", + "id": "101", + "version": "", + "role": "leaf", + "adSt": "on", + "name": "leaf1", + "model": "N9K-C9396PX" + } + } + } +] diff --git a/tests/checks/snapshot_files_check/fabricNode_old_single_apic.json b/tests/checks/snapshot_files_check/fabricNode_old_single_apic.json new file mode 100644 index 0000000..5e5f451 --- /dev/null +++ b/tests/checks/snapshot_files_check/fabricNode_old_single_apic.json @@ -0,0 +1,32 @@ +[ + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-1", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "1", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic1", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "nodeType": "unspecified", + "id": "101", + "version": "A", + "role": "leaf", + "adSt": "on", + "name": "leaf101", + "model": "N9K-C93180YC-EX" + } + } + } +] diff --git a/tests/checks/snapshot_files_check/fabricNode_single_apic.json b/tests/checks/snapshot_files_check/fabricNode_single_apic.json new file mode 100644 index 0000000..cf35918 --- /dev/null +++ b/tests/checks/snapshot_files_check/fabricNode_single_apic.json @@ -0,0 +1,32 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.1", + "dn": "topology/pod-1/node-1", + "fabricSt": "commissioned", + "id": "1", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic1", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.101", + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "id": "101", + "model": "N9K-C93180YC-EX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf101", + "nodeType": "unspecified", + "role": "leaf" + } + } + } +] diff --git a/tests/checks/snapshot_files_check/infraWiNode_apic1.json b/tests/checks/snapshot_files_check/infraWiNode_apic1.json new file mode 100644 index 0000000..b6626d0 --- /dev/null +++ b/tests/checks/snapshot_files_check/infraWiNode_apic1.json @@ -0,0 +1,62 @@ +[ + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.1", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-1", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "1", + "mbSn": "FCH1234ABCD", + "name": "", + "nodeName": "apic1", + "operSt": "available", + "podId": "0", + "targetMbSn": "" + } + } + }, + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.2", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-2", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "2", + "mbSn": "FCH1235ABCD", + "name": "", + "nodeName": "apic2", + "operSt": "available", + "podId": "0", + "targetMbSn": "" + } + } + }, + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.3", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-3", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "3", + "mbSn": "FCH1236ABCD", + "name": "", + "nodeName": "apic3", + "operSt": "available", + "podId": "1", + "targetMbSn": "" + } + } + } +] diff --git a/tests/checks/snapshot_files_check/infraWiNode_single_apic.json b/tests/checks/snapshot_files_check/infraWiNode_single_apic.json new file mode 100644 index 0000000..fe17cdf --- /dev/null +++ b/tests/checks/snapshot_files_check/infraWiNode_single_apic.json @@ -0,0 +1,22 @@ +[ + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.1", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-1", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "1", + "mbSn": "FCH1234ABCD", + "name": "", + "nodeName": "apic1", + "operSt": "available", + "podId": "0", + "targetMbSn": "" + } + } + } +] diff --git a/tests/checks/snapshot_files_check/test_snapshot_files_check.py b/tests/checks/snapshot_files_check/test_snapshot_files_check.py new file mode 100644 index 0000000..b9f05f8 --- /dev/null +++ b/tests/checks/snapshot_files_check/test_snapshot_files_check.py @@ -0,0 +1,882 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "snapshot_files_check" +infraWiNode = "topology/pod-1/node-1/infraWiNode.json" + +apic_ips = [ + node["fabricNode"]["attributes"]["address"] + for node in read_data(dir, "fabricNode.json") + if node["fabricNode"]["attributes"]["role"] == "controller" +] +apic_single_ips = [ + node["fabricNode"]["attributes"]["address"] + for node in read_data(dir, "fabricNode_single_apic.json") + if node["fabricNode"]["attributes"]["role"] == "controller" +] + +grep_cmd = 'tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404' + +# Sample log output with 10+ requests within 2 minutes (issue detected) +grep_output_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:06 +0000] "GET /snapshots/ce2_backup_policy-2025-12-24T14-30-06.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:12 +0000] "GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-30-12.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:18 +0000] "GET /snapshots/ce2_ndi_up-2025-12-24T14-30-18.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:24 +0000] "GET /snapshots/ce2_hourly_backup-2025-12-24T14-30-24.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:30 +0000] "GET /snapshots/ce2_config_export-2025-12-24T14-30-30.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:36 +0000] "GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-36.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:42 +0000] "GET /snapshots/ce2_fabric_backup-2025-12-24T14-30-42.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:48 +0000] "GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-30-48.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:54 +0000] "GET /snapshots/ce2_scheduler_backup-2025-12-24T14-30-54.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:00 +0000] "GET /snapshots/ce2_ndi_up-2025-12-24T14-31-00.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:06 +0000] "GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-31-06.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +# Sample log output with less than 10 requests or spread over more than 2 minutes (no issue) +grep_output_no_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:15 +0000] "GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-05.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:35:20 +0000] "GET /snapshots/ce2_backup_policy-2025-12-24T14-35-10.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:40:30 +0000] "GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-40-20.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +grep_output_no_file = """\ +grep: /var/log/dme/log/access.log: No such file or directory +fabric-apic# +""" + +# Edge case: Empty log output (no 404 requests found) +grep_output_empty = "" + +# Edge case: Exactly 10 requests within 60 seconds +grep_output_boundary_fail = """10.0.0.3 (-) - - [24/Dec/2025:14:30:00 +0000] "GET /snapshots/ce2_snapshot1.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:06 +0000] "GET /snapshots/ce2_snapshot2.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:12 +0000] "GET /snapshots/ce2_snapshot3.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:18 +0000] "GET /snapshots/ce2_snapshot4.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:24 +0000] "GET /snapshots/ce2_snapshot5.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:30 +0000] "GET /snapshots/ce2_snapshot6.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:36 +0000] "GET /snapshots/ce2_snapshot7.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:42 +0000] "GET /snapshots/ce2_snapshot8.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:48 +0000] "GET /snapshots/ce2_snapshot9.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:54 +0000] "GET /snapshots/ce2_snapshot10.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +# Edge case: 9 requests within 60 seconds +grep_output_boundary_pass = """10.0.0.3 (-) - - [24/Dec/2025:14:30:00 +0000] "GET /snapshots/ce2_snapshot1.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:08 +0000] "GET /snapshots/ce2_snapshot2.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:16 +0000] "GET /snapshots/ce2_snapshot3.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:24 +0000] "GET /snapshots/ce2_snapshot4.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:32 +0000] "GET /snapshots/ce2_snapshot5.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:40 +0000] "GET /snapshots/ce2_snapshot6.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:48 +0000] "GET /snapshots/ce2_snapshot7.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:56 +0000] "GET /snapshots/ce2_snapshot8.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:04 +0000] "GET /snapshots/ce2_snapshot9.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +@pytest.mark.parametrize( + "icurl_outputs, conn_failure, conn_cmds, cversion, fabric_nodes, expected_result", + [ + # Version not affected (6.0(3d) or newer) + ( + {}, + False, + [], + "6.0(3d)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Version affected, but no issues found + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Version affected, issue detected (10+ requests in 2 minutes) + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Version affected, issue detected (10+ requests in 2 minutes) + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Connection failure + ( + {}, + True, + [], + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Exception during grep command execution + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Simulated exception at grep command"), + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # error (no such file or directory for cmd execution) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Pass (pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + }, + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.PASS, + ), + # fail (pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + }, + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # connection failure(pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + True, + [], + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # exception during grep command execution (pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Simulated exception at grep command"), + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # error (pre-4.0 with infraWiNode and no such file or directory for cmd execution) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # Edge case: Single APIC with issue + ( + {}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + }, + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.FAIL_UF, + ), + # Edge case: Single APIC without issue + ( + {}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ] + }, + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.PASS, + ), + # Edge case: Single APIC with connection failure + ( + {}, + True, + [], + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.ERROR, + ), + # Edge case: Single APIC with file not found + ( + {}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + }, + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.ERROR, + ), + # Edge case: Multi-APIC with first APIC having issue, others clean + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "5.2(8g)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case: Multi-APIC with last APIC having issue, others clean + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + }, + "5.2(8g)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case: Multi-APIC with middle APIC having connection issue + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Connection timeout"), + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Edge case: Multi-APIC with one file not found, others clean + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "5.2(8g)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Edge case: Empty log output (no 404 requests) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_empty]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Edge case: Exactly 10 requests within 60 seconds (boundary - should fail) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_fail]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case: 9 requests within 60 seconds (boundary - should pass) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_pass]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Edge case: Multi-APIC with all having issues + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "5.2(8g)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case: Multi-APIC with mixed issues (issue + file not found + clean) + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Single APIC with issue + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + }, + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Single APIC without issue + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ] + }, + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.PASS, + ), + # Edge case (pre-4.0): Single APIC with connection failure + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + True, + [], + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Single APIC with file not found + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + }, + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Multi-APIC with first APIC having issue, others clean + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Multi-APIC with last APIC having issue, others clean + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Multi-APIC with middle APIC having connection issue + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Connection timeout"), + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Multi-APIC with one file not found, others clean + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Empty log output (no 404 requests) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_empty]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.PASS, + ), + # Edge case (pre-4.0): Exactly 10 requests within 60 seconds (boundary - should fail) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_fail]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): 9 requests within 60 seconds + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_pass]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.PASS, + ), + # Edge case (pre-4.0): Multi-APIC with all having issues + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Multi-APIC with mixed issues (issue + file not found + clean) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + ], +) +def test_snapshot_files_check(run_check, mock_icurl, mock_conn, cversion, fabric_nodes, expected_result): + result = run_check( + cversion=script.AciVersion(cversion), + username="fake_username", + password="fake_password", + fabric_nodes=fabric_nodes, + ) + assert result.result == expected_result \ No newline at end of file