From 0e1aa32b8a28e9c2545e8b063843a47d7ffe1864 Mon Sep 17 00:00:00 2001 From: sudharson-soundrapandiyan Date: Wed, 24 Dec 2025 20:01:23 +0000 Subject: [PATCH 1/3] just for backup --- aci-preupgrade-validation-script.py | 88 ++++++++- .../fabricNode.json | 93 +++++++++ .../fabricNode_no_apic.json | 48 +++++ .../fabricNode_old.json | 62 ++++++ .../infraWiNode_apic1.json | 62 ++++++ ...est_infinite_snapshot_file_access_check.py | 176 ++++++++++++++++++ 6 files changed, 528 insertions(+), 1 deletion(-) create mode 100644 tests/checks/infinite_snapshot_file_access_check/fabricNode.json create mode 100644 tests/checks/infinite_snapshot_file_access_check/fabricNode_no_apic.json create mode 100644 tests/checks/infinite_snapshot_file_access_check/fabricNode_old.json create mode 100644 tests/checks/infinite_snapshot_file_access_check/infraWiNode_apic1.json create mode 100644 tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index ebe04773..1d9b2fea 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -65,7 +65,7 @@ tz = time.strftime('%z') ts = datetime.now().strftime('%Y-%m-%dT%H-%M-%S') -BUNDLE_NAME = 'preupgrade_validator_%s%s.tgz' % (ts, tz) +BUNDLE_NAME = 'preupgrade_validator_sudharson1_%s%s.tgz' % (ts, tz) DIR = 'preupgrade_validator_logs/' JSON_DIR = os.path.join(DIR, 'json_results/') META_FILE = os.path.join(DIR, 'meta.json') @@ -6007,6 +6007,91 @@ def apic_vmm_inventory_sync_faults_check(**kwargs): recommended_action=recommended_action, doc_url=doc_url) + +@check_wrapper(check_title='infinite snapshot file access check') +def infinite_snapshot_file_access_check(fabric_nodes, cversion, username, password, **kwargs): + result = PASS + headers = ['apic_id', 'apic_name', 'snapshot_files'] + data = [] + recommended_action = 'Contact Cisco TAC for Support before upgrade' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#infinite-snapshot-file-access-check' + if cversion.older_than('6.0(3d)'): + apics = [node for node in fabric_nodes if node["fabricNode"]["attributes"]["role"] == "controller"] + log.debug("APIC nodes found: %s", apics) + if not apics: + return Result(result=ERROR, msg="No fabricNode of APIC. Is the cluster healthy?", doc_url=doc_url) + # `fabricNode` in pre-4.0 does not have `address` + if not apics[0]["fabricNode"]["attributes"].get("address"): + apic1 = [apic for apic in apics if apic["fabricNode"]["attributes"]["id"] == "1"][0] + log.debug("Re-fetching infraWiNode.json from APIC-1 using : %s", apic1) + apic1_dn = apic1["fabricNode"]["attributes"]["dn"] + apics = icurl("class", "{}/infraWiNode.json".format(apic1_dn)) + log.debug("Re-fetched infraWiNode.json from APIC-1: %s", apics) + has_error = False + for apic in apics: + if apic.get("fabricNode"): + apic_id = apic["fabricNode"]["attributes"]["id"] + apic_name = apic["fabricNode"]["attributes"]["name"] + apic_addr = apic["fabricNode"]["attributes"]["address"] + else: + apic_id = apic["infraWiNode"]["attributes"]["id"] + apic_name = apic["infraWiNode"]["attributes"]["nodeName"] + apic_addr = apic["infraWiNode"]["attributes"]["addr"] + try: + c = Connection(apic_addr) + c.username = username + c.password = password + c.log = LOG_FILE + c.connect() + except Exception as e: + data.append([apic_id, apic_name]) + has_error = True + continue + try: + """ c.cmd('tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404') """ + c.cmd('tail -n 1000 /data/techsupport/snapshotfile.txt | grep "GET /snapshots" | grep 404') + access_logs = c.output.splitlines() + + requests = [] # [(timestamp, filename), ...] + + for line in access_logs: + # Extract timestamp: [22/Dec/2025:04:05:02 +0000] + timestamp_match = re.search(r'\[(\d{1,2}/\w{3}/\d{4}):(\d{2}:\d{2}:\d{2})', line) + # Extract filename: ce2_NDI_EXPORT_POLICY-2025-12-22T10-04-36.tar.gz + filename_match = re.search(r'GET /snapshots/([^\s]+)', line) + + if timestamp_match and filename_match: + timestamp_str = f"{timestamp_match.group(1)}:{timestamp_match.group(2)}" + filename = filename_match.group(1) + try: + timestamp = datetime.strptime(timestamp_str, "%d/%b/%Y:%H:%M:%S") + requests.append((timestamp, filename)) + except: + continue + + requests.sort() + + # Checking if any 10 consecutive requests are within 2 minutes + if len(requests) >= 10: + for i in range(len(requests) - 9): + time_diff = (requests[i + 9][0] - requests[i][0]).total_seconds() + if time_diff <= 120: + window_files = [filename for _, filename in requests[i:i+10]] + for filename in window_files: + data.append([apic_id, apic_name, filename]) + break + + except Exception as e: + data.append([apic_id, apic_name, str(e)]) + has_error = True + continue + if has_error: + result = ERROR + elif data: + result = FAIL_UF + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + + # ---- Script Execution ---- @@ -6168,6 +6253,7 @@ class CheckManager: standby_sup_sync_check, isis_database_byte_check, configpush_shard_check, + infinite_snapshot_file_access_check, ] ssh_checks = [ diff --git a/tests/checks/infinite_snapshot_file_access_check/fabricNode.json b/tests/checks/infinite_snapshot_file_access_check/fabricNode.json new file mode 100644 index 00000000..962a4ad9 --- /dev/null +++ b/tests/checks/infinite_snapshot_file_access_check/fabricNode.json @@ -0,0 +1,93 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.1", + "dn": "topology/pod-1/node-1", + "fabricSt": "commissioned", + "id": "1", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic1", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.2", + "dn": "topology/pod-1/node-2", + "fabricSt": "commissioned", + "id": "2", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic2", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.3", + "dn": "topology/pod-2/node-3", + "fabricSt": "commissioned", + "id": "3", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic3", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.101", + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "id": "101", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf101", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.102", + "dn": "topology/pod-1/node-102", + "fabricSt": "active", + "id": "102", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf102", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.201", + "dn": "topology/pod-1/node-201", + "fabricSt": "active", + "id": "201", + "model": "N9K-C9504", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine201", + "nodeType": "unspecified", + "role": "spine" + } + } + } +] + diff --git a/tests/checks/infinite_snapshot_file_access_check/fabricNode_no_apic.json b/tests/checks/infinite_snapshot_file_access_check/fabricNode_no_apic.json new file mode 100644 index 00000000..254f40d0 --- /dev/null +++ b/tests/checks/infinite_snapshot_file_access_check/fabricNode_no_apic.json @@ -0,0 +1,48 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.101", + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "id": "101", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf101", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.102", + "dn": "topology/pod-1/node-102", + "fabricSt": "active", + "id": "102", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf102", + "nodeType": "unspecified", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.201", + "dn": "topology/pod-1/node-201", + "fabricSt": "active", + "id": "201", + "model": "N9K-C9504", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine201", + "nodeType": "unspecified", + "role": "spine" + } + } + } +] + diff --git a/tests/checks/infinite_snapshot_file_access_check/fabricNode_old.json b/tests/checks/infinite_snapshot_file_access_check/fabricNode_old.json new file mode 100644 index 00000000..f71fb9fc --- /dev/null +++ b/tests/checks/infinite_snapshot_file_access_check/fabricNode_old.json @@ -0,0 +1,62 @@ +[ + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-1", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "1", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic1", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-2", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "2", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic2", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-2/node-3", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "3", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic3", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "nodeType": "unspecified", + "id": "101", + "version": "", + "role": "leaf", + "adSt": "on", + "name": "leaf1", + "model": "N9K-C9396PX" + } + } + } +] diff --git a/tests/checks/infinite_snapshot_file_access_check/infraWiNode_apic1.json b/tests/checks/infinite_snapshot_file_access_check/infraWiNode_apic1.json new file mode 100644 index 00000000..b6626d02 --- /dev/null +++ b/tests/checks/infinite_snapshot_file_access_check/infraWiNode_apic1.json @@ -0,0 +1,62 @@ +[ + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.1", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-1", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "1", + "mbSn": "FCH1234ABCD", + "name": "", + "nodeName": "apic1", + "operSt": "available", + "podId": "0", + "targetMbSn": "" + } + } + }, + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.2", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-2", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "2", + "mbSn": "FCH1235ABCD", + "name": "", + "nodeName": "apic2", + "operSt": "available", + "podId": "0", + "targetMbSn": "" + } + } + }, + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.3", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-3", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "3", + "mbSn": "FCH1236ABCD", + "name": "", + "nodeName": "apic3", + "operSt": "available", + "podId": "1", + "targetMbSn": "" + } + } + } +] diff --git a/tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py b/tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py new file mode 100644 index 00000000..1c01ed83 --- /dev/null +++ b/tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py @@ -0,0 +1,176 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "infinite_snapshot_file_access_check" +infraWiNode = "topology/pod-1/node-1/infraWiNode.json" + +apic_ips = [ + node["fabricNode"]["attributes"]["address"] + for node in read_data(dir, "fabricNode.json") + if node["fabricNode"]["attributes"]["role"] == "controller" +] + +grep_cmd = 'tail -n 1000 /data/techsupport/snapshotfile.txt | grep "GET /snapshots" | grep 404' + +# Sample log output with 10+ requests within 2 minutes (issue detected) +grep_output_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:15 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-05.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:25 +0000]"GET /snapshots/ce2_backup_policy-2025-12-24T14-30-15.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:35 +0000]"GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-30-25.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:45 +0000]"GET /snapshots/ce2_ndi_up-2025-12-24T14-30-35.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:55 +0000]"GET /snapshots/ce2_hourly_backup-2025-12-24T14-30-45.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:05 +0000]"GET /snapshots/ce2_config_export-2025-12-24T14-30-55.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:15 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-31-05.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:25 +0000]"GET /snapshots/ce2_fabric_backup-2025-12-24T14-31-15.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:35 +0000]"GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-31-25.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:45 +0000]"GET /snapshots/ce2_scheduler_backup-2025-12-24T14-31-35.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:55 +0000]"GET /snapshots/ce2_ndi_up-2025-12-24T14-31-45.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:32:05 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-31-55.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +# Sample log output with less than 10 requests or spread over more than 2 minutes (no issue) +grep_output_no_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:15 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-05.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:35:20 +0000]"GET /snapshots/ce2_backup_policy-2025-12-24T14-35-10.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:40:30 +0000]"GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-40-20.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +@pytest.mark.parametrize( + "icurl_outputs, conn_failure, conn_cmds, cversion, fabric_nodes, expected_result", + [ + # Version not affected (6.0(3d) or newer) + ( + {}, + False, + [], + "6.0(3d)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Version affected, but no issues found + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Version affected, issue detected (10+ requests in 2 minutes) + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Connection failure + ( + {}, + True, + [], + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Exception during grep command execution + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Simulated exception at grep command"), + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Pass (pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + }, + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.PASS, + ), + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + }, + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_no_apic.json"), + script.ERROR, + ) + ], +) +def test_infinite_snapshot_file_access_check(run_check, mock_icurl, mock_conn, cversion, fabric_nodes, expected_result): + result = run_check( + cversion=script.AciVersion(cversion), + username="fake_username", + password="fake_password", + fabric_nodes=fabric_nodes, + ) + assert result.result == expected_result \ No newline at end of file From 5b417656b7523b1cb9ae1eb4a63e5e1ab30ac5b2 Mon Sep 17 00:00:00 2001 From: sudharson-soundrapandiyan Date: Tue, 6 Jan 2026 12:38:46 +0000 Subject: [PATCH 2/3] changed after shutdown holiday --- aci-preupgrade-validation-script.py | 32 +- docs/docs/validations.md | 16 +- ...est_infinite_snapshot_file_access_check.py | 176 ---- .../fabricNode.json | 0 .../fabricNode_no_apic.json | 0 .../fabricNode_old.json | 0 .../fabricNode_old_single_apic.json | 32 + .../fabricNode_single_apic.json | 32 + .../infraWiNode_apic1.json | 0 .../infraWiNode_single_apic.json | 22 + .../test_snapshot_files_check.py | 882 ++++++++++++++++++ 11 files changed, 999 insertions(+), 193 deletions(-) delete mode 100644 tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py rename tests/checks/{infinite_snapshot_file_access_check => snapshot_files_check}/fabricNode.json (100%) rename tests/checks/{infinite_snapshot_file_access_check => snapshot_files_check}/fabricNode_no_apic.json (100%) rename tests/checks/{infinite_snapshot_file_access_check => snapshot_files_check}/fabricNode_old.json (100%) create mode 100644 tests/checks/snapshot_files_check/fabricNode_old_single_apic.json create mode 100644 tests/checks/snapshot_files_check/fabricNode_single_apic.json rename tests/checks/{infinite_snapshot_file_access_check => snapshot_files_check}/infraWiNode_apic1.json (100%) create mode 100644 tests/checks/snapshot_files_check/infraWiNode_single_apic.json create mode 100644 tests/checks/snapshot_files_check/test_snapshot_files_check.py diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 1d9b2fea..b6e41e3a 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -65,7 +65,7 @@ tz = time.strftime('%z') ts = datetime.now().strftime('%Y-%m-%dT%H-%M-%S') -BUNDLE_NAME = 'preupgrade_validator_sudharson1_%s%s.tgz' % (ts, tz) +BUNDLE_NAME = 'preupgrade_validator_%s%s.tgz' % (ts, tz) DIR = 'preupgrade_validator_logs/' JSON_DIR = os.path.join(DIR, 'json_results/') META_FILE = os.path.join(DIR, 'meta.json') @@ -6008,25 +6008,22 @@ def apic_vmm_inventory_sync_faults_check(**kwargs): doc_url=doc_url) -@check_wrapper(check_title='infinite snapshot file access check') -def infinite_snapshot_file_access_check(fabric_nodes, cversion, username, password, **kwargs): +@check_wrapper(check_title='Snapshot files check') +def snapshot_files_check(fabric_nodes, cversion, username, password, **kwargs): result = PASS headers = ['apic_id', 'apic_name', 'snapshot_files'] data = [] recommended_action = 'Contact Cisco TAC for Support before upgrade' - doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#infinite-snapshot-file-access-check' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#Snapshot-files-check' if cversion.older_than('6.0(3d)'): apics = [node for node in fabric_nodes if node["fabricNode"]["attributes"]["role"] == "controller"] - log.debug("APIC nodes found: %s", apics) if not apics: return Result(result=ERROR, msg="No fabricNode of APIC. Is the cluster healthy?", doc_url=doc_url) # `fabricNode` in pre-4.0 does not have `address` if not apics[0]["fabricNode"]["attributes"].get("address"): apic1 = [apic for apic in apics if apic["fabricNode"]["attributes"]["id"] == "1"][0] - log.debug("Re-fetching infraWiNode.json from APIC-1 using : %s", apic1) apic1_dn = apic1["fabricNode"]["attributes"]["dn"] apics = icurl("class", "{}/infraWiNode.json".format(apic1_dn)) - log.debug("Re-fetched infraWiNode.json from APIC-1: %s", apics) has_error = False for apic in apics: if apic.get("fabricNode"): @@ -6048,16 +6045,17 @@ def infinite_snapshot_file_access_check(fabric_nodes, cversion, username, passwo has_error = True continue try: - """ c.cmd('tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404') """ - c.cmd('tail -n 1000 /data/techsupport/snapshotfile.txt | grep "GET /snapshots" | grep 404') + c.cmd('tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404') access_logs = c.output.splitlines() - - requests = [] # [(timestamp, filename), ...] + if len(access_logs)<15 and any("No such file or directory" in line for line in access_logs): + data.append([apic_id, apic_name, '/var/log/dme/log/access.log not found']) + has_error = True + continue + + requests = [] for line in access_logs: - # Extract timestamp: [22/Dec/2025:04:05:02 +0000] timestamp_match = re.search(r'\[(\d{1,2}/\w{3}/\d{4}):(\d{2}:\d{2}:\d{2})', line) - # Extract filename: ce2_NDI_EXPORT_POLICY-2025-12-22T10-04-36.tar.gz filename_match = re.search(r'GET /snapshots/([^\s]+)', line) if timestamp_match and filename_match: @@ -6071,11 +6069,11 @@ def infinite_snapshot_file_access_check(fabric_nodes, cversion, username, passwo requests.sort() - # Checking if any 10 consecutive requests are within 2 minutes + # Checking if any 10 consecutive requests are within 1 minute if len(requests) >= 10: for i in range(len(requests) - 9): time_diff = (requests[i + 9][0] - requests[i][0]).total_seconds() - if time_diff <= 120: + if time_diff <= 60: window_files = [filename for _, filename in requests[i:i+10]] for filename in window_files: data.append([apic_id, apic_name, filename]) @@ -6085,10 +6083,12 @@ def infinite_snapshot_file_access_check(fabric_nodes, cversion, username, passwo data.append([apic_id, apic_name, str(e)]) has_error = True continue + if has_error: result = ERROR elif data: result = FAIL_UF + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) @@ -6253,7 +6253,7 @@ class CheckManager: standby_sup_sync_check, isis_database_byte_check, configpush_shard_check, - infinite_snapshot_file_access_check, + snapshot_files_check, ] ssh_checks = [ diff --git a/docs/docs/validations.md b/docs/docs/validations.md index fa1fc0e4..144371e4 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -191,6 +191,7 @@ Items | Defect | This Script [Stale pconsRA Object][d26] | CSCwp22212 | :warning:{title="Deprecated"} | :no_entry_sign: [ISIS DTEPs Byte Size][d27] | CSCwp15375 | :white_check_mark: | :no_entry_sign: [Policydist configpushShardCont Crash][d28] | CSCwp95515 | :white_check_mark: | +[Snapshot files check][d29] | CSCwe07002 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -220,7 +221,7 @@ Items | Defect | This Script [d26]: #stale-pconsra-object [d27]: #isis-dteps-byte-size [d28]: #policydist-configpushshardcont-crash - +[d29]: #Snapshot-files-check ## General Check Details @@ -2614,6 +2615,18 @@ Due to [CSCwp95515][59], upgrading to an affected version while having any `conf If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade. +### Snapshot files check + +RCA: +Issue occured in 3 node apic cluster, AE process on one apic is busy. Reason is it's trying to fetch the snapshot files which was taken earlier is missing on all apics. + +IMPACT: +Access logs will be flooded with the GET calls. APIC Upgrade/downgrade will fail with message Installer Exited - Pre-upgrade callbacks were not completed. + +Suggestion: +Restart AE on each APIC one at a time. For Reference [CSCwe07002][62]. + + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html @@ -2676,3 +2689,4 @@ If any instances of `configpushShardCont` are flagged by this script, Cisco TAC [59]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp95515 [60]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#Inter [61]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#EnablePolicyCompression +[62]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwe07002 \ No newline at end of file diff --git a/tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py b/tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py deleted file mode 100644 index 1c01ed83..00000000 --- a/tests/checks/infinite_snapshot_file_access_check/test_infinite_snapshot_file_access_check.py +++ /dev/null @@ -1,176 +0,0 @@ -import os -import pytest -import logging -import importlib -from helpers.utils import read_data - -script = importlib.import_module("aci-preupgrade-validation-script") - -log = logging.getLogger(__name__) -dir = os.path.dirname(os.path.abspath(__file__)) - -test_function = "infinite_snapshot_file_access_check" -infraWiNode = "topology/pod-1/node-1/infraWiNode.json" - -apic_ips = [ - node["fabricNode"]["attributes"]["address"] - for node in read_data(dir, "fabricNode.json") - if node["fabricNode"]["attributes"]["role"] == "controller" -] - -grep_cmd = 'tail -n 1000 /data/techsupport/snapshotfile.txt | grep "GET /snapshots" | grep 404' - -# Sample log output with 10+ requests within 2 minutes (issue detected) -grep_output_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:15 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-05.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:30:25 +0000]"GET /snapshots/ce2_backup_policy-2025-12-24T14-30-15.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:30:35 +0000]"GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-30-25.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:30:45 +0000]"GET /snapshots/ce2_ndi_up-2025-12-24T14-30-35.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:30:55 +0000]"GET /snapshots/ce2_hourly_backup-2025-12-24T14-30-45.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:31:05 +0000]"GET /snapshots/ce2_config_export-2025-12-24T14-30-55.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:31:15 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-31-05.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:31:25 +0000]"GET /snapshots/ce2_fabric_backup-2025-12-24T14-31-15.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:31:35 +0000]"GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-31-25.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:31:45 +0000]"GET /snapshots/ce2_scheduler_backup-2025-12-24T14-31-35.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:31:55 +0000]"GET /snapshots/ce2_ndi_up-2025-12-24T14-31-45.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:32:05 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-31-55.tar.gz HTTP/1.1" 404 151 "-" "-" -""" - -# Sample log output with less than 10 requests or spread over more than 2 minutes (no issue) -grep_output_no_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:15 +0000]"GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-05.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:35:20 +0000]"GET /snapshots/ce2_backup_policy-2025-12-24T14-35-10.tar.gz HTTP/1.1" 404 151 "-" "-" -10.0.0.3 (-) - - [24/Dec/2025:14:40:30 +0000]"GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-40-20.tar.gz HTTP/1.1" 404 151 "-" "-" -""" - -@pytest.mark.parametrize( - "icurl_outputs, conn_failure, conn_cmds, cversion, fabric_nodes, expected_result", - [ - # Version not affected (6.0(3d) or newer) - ( - {}, - False, - [], - "6.0(3d)", - read_data(dir, "fabricNode.json"), - script.PASS, - ), - # Version affected, but no issues found - ( - {}, - False, - { - apic_ip: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ] - for apic_ip in apic_ips - }, - "5.2(1h)", - read_data(dir, "fabricNode.json"), - script.PASS, - ), - # Version affected, issue detected (10+ requests in 2 minutes) - ( - {}, - False, - { - apic_ips[0]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_issue]), - "exception": None, - } - ], - apic_ips[1]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ], - apic_ips[2]: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - } - ], - }, - "6.0(2a)", - read_data(dir, "fabricNode.json"), - script.FAIL_UF, - ), - # Connection failure - ( - {}, - True, - [], - "5.2(1h)", - read_data(dir, "fabricNode.json"), - script.ERROR, - ), - # Exception during grep command execution - ( - {}, - False, - { - apic_ip: [ - { - "cmd": grep_cmd, - "output": "", - "exception": Exception("Simulated exception at grep command"), - } - ] - for apic_ip in apic_ips - }, - "6.0(2a)", - read_data(dir, "fabricNode.json"), - script.ERROR, - ), - # Pass (pre-4.0 with infraWiNode) - ( - {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, - False, - { - apic_ip: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - }, - ] - for apic_ip in apic_ips - }, - "3.2(6o)", - read_data(dir, "fabricNode_old.json"), - script.PASS, - ), - ( - {}, - False, - { - apic_ip: [ - { - "cmd": grep_cmd, - "output": "\n".join([grep_cmd, grep_output_no_issue]), - "exception": None, - }, - ] - for apic_ip in apic_ips - }, - "3.2(6o)", - read_data(dir, "fabricNode_no_apic.json"), - script.ERROR, - ) - ], -) -def test_infinite_snapshot_file_access_check(run_check, mock_icurl, mock_conn, cversion, fabric_nodes, expected_result): - result = run_check( - cversion=script.AciVersion(cversion), - username="fake_username", - password="fake_password", - fabric_nodes=fabric_nodes, - ) - assert result.result == expected_result \ No newline at end of file diff --git a/tests/checks/infinite_snapshot_file_access_check/fabricNode.json b/tests/checks/snapshot_files_check/fabricNode.json similarity index 100% rename from tests/checks/infinite_snapshot_file_access_check/fabricNode.json rename to tests/checks/snapshot_files_check/fabricNode.json diff --git a/tests/checks/infinite_snapshot_file_access_check/fabricNode_no_apic.json b/tests/checks/snapshot_files_check/fabricNode_no_apic.json similarity index 100% rename from tests/checks/infinite_snapshot_file_access_check/fabricNode_no_apic.json rename to tests/checks/snapshot_files_check/fabricNode_no_apic.json diff --git a/tests/checks/infinite_snapshot_file_access_check/fabricNode_old.json b/tests/checks/snapshot_files_check/fabricNode_old.json similarity index 100% rename from tests/checks/infinite_snapshot_file_access_check/fabricNode_old.json rename to tests/checks/snapshot_files_check/fabricNode_old.json diff --git a/tests/checks/snapshot_files_check/fabricNode_old_single_apic.json b/tests/checks/snapshot_files_check/fabricNode_old_single_apic.json new file mode 100644 index 00000000..5e5f4518 --- /dev/null +++ b/tests/checks/snapshot_files_check/fabricNode_old_single_apic.json @@ -0,0 +1,32 @@ +[ + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-1", + "fabricSt": "unknown", + "nodeType": "unspecified", + "id": "1", + "version": "A", + "role": "controller", + "adSt": "on", + "name": "apic1", + "model": "APIC-SERVER-M1" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "nodeType": "unspecified", + "id": "101", + "version": "A", + "role": "leaf", + "adSt": "on", + "name": "leaf101", + "model": "N9K-C93180YC-EX" + } + } + } +] diff --git a/tests/checks/snapshot_files_check/fabricNode_single_apic.json b/tests/checks/snapshot_files_check/fabricNode_single_apic.json new file mode 100644 index 00000000..cf359184 --- /dev/null +++ b/tests/checks/snapshot_files_check/fabricNode_single_apic.json @@ -0,0 +1,32 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.1", + "dn": "topology/pod-1/node-1", + "fabricSt": "commissioned", + "id": "1", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic1", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.101", + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "id": "101", + "model": "N9K-C93180YC-EX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf101", + "nodeType": "unspecified", + "role": "leaf" + } + } + } +] diff --git a/tests/checks/infinite_snapshot_file_access_check/infraWiNode_apic1.json b/tests/checks/snapshot_files_check/infraWiNode_apic1.json similarity index 100% rename from tests/checks/infinite_snapshot_file_access_check/infraWiNode_apic1.json rename to tests/checks/snapshot_files_check/infraWiNode_apic1.json diff --git a/tests/checks/snapshot_files_check/infraWiNode_single_apic.json b/tests/checks/snapshot_files_check/infraWiNode_single_apic.json new file mode 100644 index 00000000..fe17cdf8 --- /dev/null +++ b/tests/checks/snapshot_files_check/infraWiNode_single_apic.json @@ -0,0 +1,22 @@ +[ + { + "infraWiNode": { + "attributes": { + "addr": "10.0.0.1", + "adminSt": "in-service", + "apicMode": "active", + "cntrlSbstState": "approved", + "dn": "topology/pod-1/node-1/av/node-1", + "failoverStatus": "idle", + "health": "fully-fit", + "id": "1", + "mbSn": "FCH1234ABCD", + "name": "", + "nodeName": "apic1", + "operSt": "available", + "podId": "0", + "targetMbSn": "" + } + } + } +] diff --git a/tests/checks/snapshot_files_check/test_snapshot_files_check.py b/tests/checks/snapshot_files_check/test_snapshot_files_check.py new file mode 100644 index 00000000..c76c8c5a --- /dev/null +++ b/tests/checks/snapshot_files_check/test_snapshot_files_check.py @@ -0,0 +1,882 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "snapshot_files_check" +infraWiNode = "topology/pod-1/node-1/infraWiNode.json" + +apic_ips = [ + node["fabricNode"]["attributes"]["address"] + for node in read_data(dir, "fabricNode.json") + if node["fabricNode"]["attributes"]["role"] == "controller" +] +apic_single_ips = [ + node["fabricNode"]["attributes"]["address"] + for node in read_data(dir, "fabricNode_single_apic.json") + if node["fabricNode"]["attributes"]["role"] == "controller" +] + +grep_cmd = 'tail -n 1000 /data/techsupport/snapshotfile1.txt | grep "GET /snapshots" | grep 404' + +# Sample log output with 10+ requests within 2 minutes (issue detected) +grep_output_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:06 +0000] "GET /snapshots/ce2_backup_policy-2025-12-24T14-30-06.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:12 +0000] "GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-30-12.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:18 +0000] "GET /snapshots/ce2_ndi_up-2025-12-24T14-30-18.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:24 +0000] "GET /snapshots/ce2_hourly_backup-2025-12-24T14-30-24.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:30 +0000] "GET /snapshots/ce2_config_export-2025-12-24T14-30-30.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:36 +0000] "GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-36.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:42 +0000] "GET /snapshots/ce2_fabric_backup-2025-12-24T14-30-42.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:48 +0000] "GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-30-48.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:54 +0000] "GET /snapshots/ce2_scheduler_backup-2025-12-24T14-30-54.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:00 +0000] "GET /snapshots/ce2_ndi_up-2025-12-24T14-31-00.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:06 +0000] "GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-31-06.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +# Sample log output with less than 10 requests or spread over more than 2 minutes (no issue) +grep_output_no_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:15 +0000] "GET /snapshots/ce2_NDI_EXPORT_POLICY-2025-12-24T14-30-05.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:35:20 +0000] "GET /snapshots/ce2_backup_policy-2025-12-24T14-35-10.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:40:30 +0000] "GET /snapshots/ce2_DailyAutoBackup-2025-12-24T14-40-20.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +grep_output_no_file = """\ +grep: /data/techsupport/snapshotfile1.txt: No such file or directory +fabric-apic# +""" + +# Edge case: Empty log output (no 404 requests found) +grep_output_empty = "" + +# Edge case: Exactly 10 requests within 60 seconds (boundary - should trigger) +grep_output_boundary_fail = """10.0.0.3 (-) - - [24/Dec/2025:14:30:00 +0000] "GET /snapshots/ce2_snapshot1.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:06 +0000] "GET /snapshots/ce2_snapshot2.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:12 +0000] "GET /snapshots/ce2_snapshot3.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:18 +0000] "GET /snapshots/ce2_snapshot4.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:24 +0000] "GET /snapshots/ce2_snapshot5.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:30 +0000] "GET /snapshots/ce2_snapshot6.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:36 +0000] "GET /snapshots/ce2_snapshot7.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:42 +0000] "GET /snapshots/ce2_snapshot8.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:48 +0000] "GET /snapshots/ce2_snapshot9.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:54 +0000] "GET /snapshots/ce2_snapshot10.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +# Edge case: 9 requests within 60 seconds (should pass - below threshold) +grep_output_boundary_pass = """10.0.0.3 (-) - - [24/Dec/2025:14:30:00 +0000] "GET /snapshots/ce2_snapshot1.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:08 +0000] "GET /snapshots/ce2_snapshot2.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:16 +0000] "GET /snapshots/ce2_snapshot3.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:24 +0000] "GET /snapshots/ce2_snapshot4.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:32 +0000] "GET /snapshots/ce2_snapshot5.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:40 +0000] "GET /snapshots/ce2_snapshot6.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:48 +0000] "GET /snapshots/ce2_snapshot7.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:30:56 +0000] "GET /snapshots/ce2_snapshot8.tar.gz HTTP/1.1" 404 151 "-" "-" +10.0.0.3 (-) - - [24/Dec/2025:14:31:04 +0000] "GET /snapshots/ce2_snapshot9.tar.gz HTTP/1.1" 404 151 "-" "-" +""" + +@pytest.mark.parametrize( + "icurl_outputs, conn_failure, conn_cmds, cversion, fabric_nodes, expected_result", + [ + # Version not affected (6.0(3d) or newer) + ( + {}, + False, + [], + "6.0(3d)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Version affected, but no issues found + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Version affected, issue detected (10+ requests in 2 minutes) + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Version affected, issue detected (10+ requests in 2 minutes) + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Connection failure + ( + {}, + True, + [], + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Exception during grep command execution + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Simulated exception at grep command"), + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # error (no such file or directory for cmd execution) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "5.2(1h)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Pass (pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + }, + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.PASS, + ), + # fail (pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + }, + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # connection failure(pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + True, + [], + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # exception during grep command execution (pre-4.0 with infraWiNode) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Simulated exception at grep command"), + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # error (pre-4.0 with infraWiNode and no such file or directory for cmd execution) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # Edge case: Single APIC with issue + ( + {}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + }, + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.FAIL_UF, + ), + # Edge case: Single APIC without issue + ( + {}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ] + }, + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.PASS, + ), + # Edge case: Single APIC with connection failure + ( + {}, + True, + [], + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.ERROR, + ), + # Edge case: Single APIC with file not found + ( + {}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + }, + "6.0(2a)", + read_data(dir, "fabricNode_single_apic.json"), + script.ERROR, + ), + # Edge case: Multi-APIC with first APIC having issue, others clean + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "5.2(8g)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case: Multi-APIC with last APIC having issue, others clean + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + }, + "5.2(8g)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case: Multi-APIC with middle APIC having connection issue + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Connection timeout"), + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Edge case: Multi-APIC with one file not found, others clean + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "5.2(8g)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Edge case: Empty log output (no 404 requests) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_empty]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Edge case: Exactly 10 requests within 60 seconds (boundary - should fail) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_fail]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case: 9 requests within 60 seconds (boundary - should pass) + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_pass]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.PASS, + ), + # Edge case: Multi-APIC with all having issues + ( + {}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "5.2(8g)", + read_data(dir, "fabricNode.json"), + script.FAIL_UF, + ), + # Edge case: Multi-APIC with mixed issues (issue + file not found + clean) + ( + {}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "6.0(2a)", + read_data(dir, "fabricNode.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Single APIC with issue + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + }, + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Single APIC without issue + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ] + }, + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.PASS, + ), + # Edge case (pre-4.0): Single APIC with connection failure + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + True, + [], + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Single APIC with file not found + ( + {infraWiNode: read_data(dir, "infraWiNode_single_apic.json")}, + False, + { + apic_single_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ] + }, + "3.2(6o)", + read_data(dir, "fabricNode_old_single_apic.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Multi-APIC with first APIC having issue, others clean + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Multi-APIC with last APIC having issue, others clean + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Multi-APIC with middle APIC having connection issue + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "", + "exception": Exception("Connection timeout"), + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Multi-APIC with one file not found, others clean + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + # Edge case (pre-4.0): Empty log output (no 404 requests) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_empty]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.PASS, + ), + # Edge case (pre-4.0): Exactly 10 requests within 60 seconds (boundary - should fail) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_fail]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): 9 requests within 60 seconds + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_boundary_pass]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.PASS, + ), + # Edge case (pre-4.0): Multi-APIC with all having issues + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ip: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ] + for apic_ip in apic_ips + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.FAIL_UF, + ), + # Edge case (pre-4.0): Multi-APIC with mixed issues (issue + file not found + clean) + ( + {infraWiNode: read_data(dir, "infraWiNode_apic1.json")}, + False, + { + apic_ips[0]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_issue]), + "exception": None, + } + ], + apic_ips[1]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_file]), + "exception": None, + } + ], + apic_ips[2]: [ + { + "cmd": grep_cmd, + "output": "\n".join([grep_cmd, grep_output_no_issue]), + "exception": None, + } + ], + }, + "3.2(6o)", + read_data(dir, "fabricNode_old.json"), + script.ERROR, + ), + ], +) +def test_snapshot_files_check(run_check, mock_icurl, mock_conn, cversion, fabric_nodes, expected_result): + result = run_check( + cversion=script.AciVersion(cversion), + username="fake_username", + password="fake_password", + fabric_nodes=fabric_nodes, + ) + assert result.result == expected_result \ No newline at end of file From ae55164d1861dcaf7ea51c3afbf836276daba2af Mon Sep 17 00:00:00 2001 From: sudharson-soundrapandiyan Date: Tue, 6 Jan 2026 12:48:02 +0000 Subject: [PATCH 3/3] latest changes --- aci-preupgrade-validation-script.py | 4 ++-- .../snapshot_files_check/test_snapshot_files_check.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index b6e41e3a..b7378bb3 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6041,13 +6041,13 @@ def snapshot_files_check(fabric_nodes, cversion, username, password, **kwargs): c.log = LOG_FILE c.connect() except Exception as e: - data.append([apic_id, apic_name]) + data.append([apic_id, apic_name, str(e)]) has_error = True continue try: c.cmd('tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404') access_logs = c.output.splitlines() - if len(access_logs)<15 and any("No such file or directory" in line for line in access_logs): + if len(access_logs) < 15 and any("No such file or directory" in line for line in access_logs): data.append([apic_id, apic_name, '/var/log/dme/log/access.log not found']) has_error = True continue diff --git a/tests/checks/snapshot_files_check/test_snapshot_files_check.py b/tests/checks/snapshot_files_check/test_snapshot_files_check.py index c76c8c5a..b9f05f8e 100644 --- a/tests/checks/snapshot_files_check/test_snapshot_files_check.py +++ b/tests/checks/snapshot_files_check/test_snapshot_files_check.py @@ -23,7 +23,7 @@ if node["fabricNode"]["attributes"]["role"] == "controller" ] -grep_cmd = 'tail -n 1000 /data/techsupport/snapshotfile1.txt | grep "GET /snapshots" | grep 404' +grep_cmd = 'tail -n 1000 /var/log/dme/log/access.log | grep "GET /snapshots" | grep 404' # Sample log output with 10+ requests within 2 minutes (issue detected) grep_output_issue = """10.0.0.3 (-) - - [24/Dec/2025:14:30:06 +0000] "GET /snapshots/ce2_backup_policy-2025-12-24T14-30-06.tar.gz HTTP/1.1" 404 151 "-" "-" @@ -46,14 +46,14 @@ """ grep_output_no_file = """\ -grep: /data/techsupport/snapshotfile1.txt: No such file or directory +grep: /var/log/dme/log/access.log: No such file or directory fabric-apic# """ # Edge case: Empty log output (no 404 requests found) grep_output_empty = "" -# Edge case: Exactly 10 requests within 60 seconds (boundary - should trigger) +# Edge case: Exactly 10 requests within 60 seconds grep_output_boundary_fail = """10.0.0.3 (-) - - [24/Dec/2025:14:30:00 +0000] "GET /snapshots/ce2_snapshot1.tar.gz HTTP/1.1" 404 151 "-" "-" 10.0.0.3 (-) - - [24/Dec/2025:14:30:06 +0000] "GET /snapshots/ce2_snapshot2.tar.gz HTTP/1.1" 404 151 "-" "-" 10.0.0.3 (-) - - [24/Dec/2025:14:30:12 +0000] "GET /snapshots/ce2_snapshot3.tar.gz HTTP/1.1" 404 151 "-" "-" @@ -66,7 +66,7 @@ 10.0.0.3 (-) - - [24/Dec/2025:14:30:54 +0000] "GET /snapshots/ce2_snapshot10.tar.gz HTTP/1.1" 404 151 "-" "-" """ -# Edge case: 9 requests within 60 seconds (should pass - below threshold) +# Edge case: 9 requests within 60 seconds grep_output_boundary_pass = """10.0.0.3 (-) - - [24/Dec/2025:14:30:00 +0000] "GET /snapshots/ce2_snapshot1.tar.gz HTTP/1.1" 404 151 "-" "-" 10.0.0.3 (-) - - [24/Dec/2025:14:30:08 +0000] "GET /snapshots/ce2_snapshot2.tar.gz HTTP/1.1" 404 151 "-" "-" 10.0.0.3 (-) - - [24/Dec/2025:14:30:16 +0000] "GET /snapshots/ce2_snapshot3.tar.gz HTTP/1.1" 404 151 "-" "-"