diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index ebe0477..90c0816 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6007,6 +6007,63 @@ def apic_vmm_inventory_sync_faults_check(**kwargs): recommended_action=recommended_action, doc_url=doc_url) + +@check_wrapper(check_title='AAA Provider DNS Name Configuration check') +def aaa_snmpd_dns_provider_check(tversion, **kwargs): + result = PASS + headers = ["Configuration Type", "Name"] + data = [] + recommended_action = 'Contact Cisco TAC for Support before upgrade' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#aaa-provider-dns-name-configuration-check' + + if not tversion: + return Result(result=MANUAL, msg=TVER_MISSING) + + if tversion.older_than("6.1(5e)"): + + # Query for all AAA providers (TACACS+, RADIUS, and LDAP) + provider_api = 'uni.json?query-target=subtree&target-subtree-class=aaaTacacsPlusProvider,aaaRadiusProvider,aaaLdapProvider' + providers = icurl('mo', provider_api) + + # Regular expression to detect DNS names (not IP addresses) + dns_name_pattern = r'[a-zA-Z]' + + for provider in providers: + provider_type = None + provider_name = None + + if 'aaaRadiusProvider' in provider: + provider_type = 'RADIUS' + provider_name = provider['aaaRadiusProvider']['attributes']['name'] + elif 'aaaLdapProvider' in provider: + provider_type = 'LDAP' + provider_name = provider['aaaLdapProvider']['attributes']['name'] + elif 'aaaTacacsPlusProvider' in provider: + provider_type = 'TACACS+' + provider_name = provider['aaaTacacsPlusProvider']['attributes']['name'] + + # Check if the provider name contains DNS name + if provider_name and re.search(dns_name_pattern, provider_name): + data.append([provider_type, provider_name]) + + snmp_api = 'snmpPol.json?query-target-filter=and(eq(snmpPol.adminSt,"enabled"))' + snmp_policy = icurl('class', snmp_api) + + if snmp_policy: + for policy in snmp_policy: + policy_name = policy['snmpPol']['attributes']['name'] + data.append(["SNMP Policy", policy_name]) + + if providers and snmp_policy: + result = MANUAL + return Result(result=result, headers=headers, data=data, msg="AAA providers are configured using hostnames and SNMP policies are enabled. If SNMP policies are deployed, please contact TAC for further investigation before proceeding with the upgrade. Otherwise, this warning can be safely ignored.") + + else: + return Result(result=PASS, msg=VER_NOT_AFFECTED) + + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + + # ---- Script Execution ---- @@ -6168,6 +6225,7 @@ class CheckManager: standby_sup_sync_check, isis_database_byte_check, configpush_shard_check, + aaa_snmpd_dns_provider_check, ] ssh_checks = [ diff --git a/docs/docs/validations.md b/docs/docs/validations.md index fa1fc0e..35980c2 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -191,6 +191,7 @@ Items | Defect | This Script [Stale pconsRA Object][d26] | CSCwp22212 | :warning:{title="Deprecated"} | :no_entry_sign: [ISIS DTEPs Byte Size][d27] | CSCwp15375 | :white_check_mark: | :no_entry_sign: [Policydist configpushShardCont Crash][d28] | CSCwp95515 | :white_check_mark: | +[AAA Provider DNS Name Configuration check][d29] | CSCwq57598 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -220,6 +221,7 @@ Items | Defect | This Script [d26]: #stale-pconsra-object [d27]: #isis-dteps-byte-size [d28]: #policydist-configpushshardcont-crash +[d29]: #aaa-provider-dns-name-configuration-check ## General Check Details @@ -2614,6 +2616,14 @@ Due to [CSCwp95515][59], upgrading to an affected version while having any `conf If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade. +### AAA Provider DNS Name Configuration check + +Due to `CSCwq57598`, spines hit kernel panic after the upgrade. SNMPd main process acquires a lock for memory operations at the same time, child SNMP process is spawned by the DNS Cache to resolve the AAA server name & inherently acquires parent mutex_lock state & eventually ends up in blocked state. + +This script identifies if APIC is configured with AAA Authentication providers (TACACS+, RADIUS, and LDAP) using hostname and enabled SNMP policies. If detected, contact cisco TAC support and upgrade to the fix version. + + + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html diff --git a/tests/checks/aaa_snmpd_dns_provider_check/README.md b/tests/checks/aaa_snmpd_dns_provider_check/README.md new file mode 100644 index 0000000..d42063e --- /dev/null +++ b/tests/checks/aaa_snmpd_dns_provider_check/README.md @@ -0,0 +1,118 @@ +# Test Cases for aaa_snmpd_dns_provider_check + +## Overview +This directory contains test cases for the `aaa_snmpd_dns_provider_check` function, which validates AAA provider configurations to prevent snmpd memory exhaustion issues (CSCwq57598). + +## Test Function +**Function Name:** `aaa_snmpd_dns_provider_check` +**Bug Reference:** CSCwq57598 - hundreds of snmpd process exhaust memory and lead kernel panic with oom + +## Test Scenarios + +### Test Case 1: No Target Version Provided +- **Input:** No target version (None) +- **JSON File:** `providers_with_dns.json` +- **Expected Result:** `MANUAL CHECK REQUIRED` +- **Expected Data Count:** 0 +- **Description:** When target version is not provided, the check should return MANUAL and indicate that target version is missing. + +### Test Case 2: Target Version 6.1(5e) - Not Affected +- **Input:** Target version `6.1(5e)` +- **JSON File:** `providers_with_dns.json` +- **Expected Result:** `PASS` +- **Expected Data Count:** 0 +- **Description:** Version 6.1(5e) includes the fix, so the check should pass with "Version not affected" message. + +### Test Case 3: Target Version Newer Than 6.1(5e) - Not Affected +- **Input:** Target version `6.2(1a)` +- **JSON File:** `providers_with_dns.json` +- **Expected Result:** `PASS` +- **Expected Data Count:** 0 +- **Description:** Versions newer than 6.1(5e) are not affected, so the check should pass. + +### Test Case 4: Target Version Older Than 6.1(5e) with DNS Names - FAIL +- **Input:** Target version `6.1(4a)` +- **JSON File:** `providers_with_dns.json` +- **Expected Result:** `FAIL_O` (FAIL - OUTAGE WARNING!!) +- **Expected Data Count:** 2 +- **Description:** When target version is older than 6.1(5e) and DNS names are found in AAA providers, the check should fail. +- **Flagged Providers:** + - RADIUS provider: `rad.cisco.com` + - LDAP provider: `aaa.domain.com` + +### Test Case 5: Target Version Older Than 6.1(5e) with Only IP Addresses - PASS +- **Input:** Target version `6.1(3a)` +- **JSON File:** `providers_with_ips.json` +- **Expected Result:** `PASS` +- **Expected Data Count:** 0 +- **Description:** When only IP addresses are configured, the check should pass even on affected versions. +- **Providers (all IPs):** + - TACACS+ provider: `10.0.0.1` + - RADIUS provider: `192.168.1.100` + - LDAP provider: `10.10.10.50` + +### Test Case 6: Target Version Older Than 6.1(5e) with Mixed (IP and DNS) - FAIL +- **Input:** Target version `6.0(5a)` +- **JSON File:** `providers_mixed.json` +- **Expected Result:** `FAIL_O` (FAIL - OUTAGE WARNING!!) +- **Expected Data Count:** 2 +- **Description:** When a mix of IP and DNS names is configured, only DNS entries should be flagged. +- **Flagged Providers:** + - RADIUS provider: `radius.example.com` + - LDAP provider: `ldap-server.local` +- **Not Flagged:** + - TACACS+ provider: `10.0.0.1` (IP address) + +### Test Case 7: Target Version Older Than 6.1(5e) with No Providers - PASS +- **Input:** Target version `5.2(8a)` +- **JSON File:** `providers_empty.json` +- **Expected Result:** `PASS` +- **Expected Data Count:** 0 +- **Description:** When no AAA providers are configured, the check should pass. + +## JSON Test Data Files + +### providers_with_dns.json +Contains AAA providers with DNS names: +- RADIUS provider: `rad.cisco.com` +- LDAP provider: `aaa.domain.com` + +### providers_with_ips.json +Contains AAA providers with only IP addresses: +- TACACS+ provider: `10.0.0.1` +- RADIUS provider: `192.168.1.100` +- LDAP provider: `10.10.10.50` + +### providers_mixed.json +Contains a mix of IP and DNS configurations: +- TACACS+ provider: `10.0.0.1` (IP) +- RADIUS provider: `radius.example.com` (DNS) +- LDAP provider: `ldap-server.local` (DNS) + +### providers_empty.json +Empty array - no AAA providers configured. + +## Running the Tests + +```bash +# Run all tests +cd /data/ssd/dhaselva/repo/ACI-Escalation/ACI-Pre-Upgrade-Validation-Script +python3 -m pytest tests/checks/aaa_snmpd_dns_provider_check/test_aaa_snmpd_dns_provider_check.py -v + +# Run with detailed output +python3 -m pytest tests/checks/aaa_snmpd_dns_provider_check/test_aaa_snmpd_dns_provider_check.py -v -s +``` + +## Test Results +All 7 test cases pass successfully, validating: +1. Version checking logic (missing version, affected versions, not affected versions) +2. DNS name detection (alphabetic characters in provider names) +3. Proper handling of IP addresses (should not be flagged) +4. Mixed configurations (only DNS names should be flagged) +5. Empty configurations (no providers) + +## Expected Output Format +When DNS names are detected in affected versions: +- **Headers:** ["Provider Type", "Provider Name", "Provider DN"] +- **Data:** List of providers with DNS names +- **Recommended Action:** "Replace DNS names with IP addresses in AAA provider configurations to prevent snmpd memory exhaustion" diff --git a/tests/checks/aaa_snmpd_dns_provider_check/providers_empty.json b/tests/checks/aaa_snmpd_dns_provider_check/providers_empty.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/tests/checks/aaa_snmpd_dns_provider_check/providers_empty.json @@ -0,0 +1 @@ +[] diff --git a/tests/checks/aaa_snmpd_dns_provider_check/providers_mixed.json b/tests/checks/aaa_snmpd_dns_provider_check/providers_mixed.json new file mode 100644 index 0000000..625ab00 --- /dev/null +++ b/tests/checks/aaa_snmpd_dns_provider_check/providers_mixed.json @@ -0,0 +1,106 @@ +[ + { + "aaaTacacsPlusProvider": { + "attributes": { + "SSLValidationLevel": "permissive", + "annotation": "", + "authProtocol": "pap", + "childAction": "", + "descr": "", + "dn": "uni/userext/tacacsext/tacacsplusprovider-10.0.0.1", + "enableTLS": "no", + "epgDn": "", + "extMngdBy": "", + "keyring": "", + "lcOwn": "local", + "modTs": "2025-12-17T05:26:53.215+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "10.0.0.1", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "port": "49", + "retries": "1", + "snmpIndex": "1", + "status": "", + "timeout": "5", + "tp": "", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + }, + { + "aaaRadiusProvider": { + "attributes": { + "annotation": "", + "authPort": "1812", + "authProtocol": "pap", + "childAction": "", + "descr": "", + "dn": "uni/userext/radiusext/radiusprovider-radius.example.com", + "epgDn": "", + "extMngdBy": "", + "includeMsgAuth": "no", + "lcOwn": "local", + "modTs": "2025-12-23T05:40:49.902+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "radius.example.com", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "retries": "1", + "snmpIndex": "3", + "status": "", + "timeout": "5", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + }, + { + "aaaLdapProvider": { + "attributes": { + "SSLValidationLevel": "strict", + "annotation": "", + "attribute": "CiscoAVPair", + "authMethod": "LdapBind", + "basedn": "", + "childAction": "", + "descr": "test", + "dn": "uni/userext/ldapext/ldapprovider-ldap-server.local", + "enableSSL": "no", + "epgDn": "", + "extMngdBy": "", + "filter": "cn=$userid", + "lcOwn": "local", + "modTs": "2025-12-23T05:39:47.423+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "ldap-server.local", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "port": "389", + "retries": "1", + "rootdn": "", + "snmpIndex": "2", + "status": "", + "timeout": "30", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + } +] diff --git a/tests/checks/aaa_snmpd_dns_provider_check/providers_with_dns.json b/tests/checks/aaa_snmpd_dns_provider_check/providers_with_dns.json new file mode 100644 index 0000000..74ddc9d --- /dev/null +++ b/tests/checks/aaa_snmpd_dns_provider_check/providers_with_dns.json @@ -0,0 +1,71 @@ +[ + { + "aaaRadiusProvider": { + "attributes": { + "annotation": "", + "authPort": "1812", + "authProtocol": "pap", + "childAction": "", + "descr": "", + "dn": "uni/userext/radiusext/radiusprovider-rad.cisco.com", + "epgDn": "", + "extMngdBy": "", + "includeMsgAuth": "no", + "lcOwn": "local", + "modTs": "2025-12-23T05:40:49.902+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "rad.cisco.com", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "retries": "1", + "snmpIndex": "3", + "status": "", + "timeout": "5", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + }, + { + "aaaLdapProvider": { + "attributes": { + "SSLValidationLevel": "strict", + "annotation": "", + "attribute": "CiscoAVPair", + "authMethod": "LdapBind", + "basedn": "", + "childAction": "", + "descr": "test", + "dn": "uni/userext/ldapext/ldapprovider-aaa.domain.com", + "enableSSL": "no", + "epgDn": "", + "extMngdBy": "", + "filter": "cn=$userid", + "lcOwn": "local", + "modTs": "2025-12-23T05:39:47.423+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "aaa.domain.com", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "port": "389", + "retries": "1", + "rootdn": "", + "snmpIndex": "2", + "status": "", + "timeout": "30", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + } +] diff --git a/tests/checks/aaa_snmpd_dns_provider_check/providers_with_ips.json b/tests/checks/aaa_snmpd_dns_provider_check/providers_with_ips.json new file mode 100644 index 0000000..bf3b215 --- /dev/null +++ b/tests/checks/aaa_snmpd_dns_provider_check/providers_with_ips.json @@ -0,0 +1,106 @@ +[ + { + "aaaTacacsPlusProvider": { + "attributes": { + "SSLValidationLevel": "permissive", + "annotation": "", + "authProtocol": "pap", + "childAction": "", + "descr": "", + "dn": "uni/userext/tacacsext/tacacsplusprovider-10.0.0.1", + "enableTLS": "no", + "epgDn": "", + "extMngdBy": "", + "keyring": "", + "lcOwn": "local", + "modTs": "2025-12-17T05:26:53.215+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "10.0.0.1", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "port": "49", + "retries": "1", + "snmpIndex": "1", + "status": "", + "timeout": "5", + "tp": "", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + }, + { + "aaaRadiusProvider": { + "attributes": { + "annotation": "", + "authPort": "1812", + "authProtocol": "pap", + "childAction": "", + "descr": "", + "dn": "uni/userext/radiusext/radiusprovider-192.168.1.100", + "epgDn": "", + "extMngdBy": "", + "includeMsgAuth": "no", + "lcOwn": "local", + "modTs": "2025-12-23T05:40:49.902+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "192.168.1.100", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "retries": "1", + "snmpIndex": "3", + "status": "", + "timeout": "5", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + }, + { + "aaaLdapProvider": { + "attributes": { + "SSLValidationLevel": "strict", + "annotation": "", + "attribute": "CiscoAVPair", + "authMethod": "LdapBind", + "basedn": "", + "childAction": "", + "descr": "test", + "dn": "uni/userext/ldapext/ldapprovider-10.10.10.50", + "enableSSL": "no", + "epgDn": "", + "extMngdBy": "", + "filter": "cn=$userid", + "lcOwn": "local", + "modTs": "2025-12-23T05:39:47.423+00:00", + "monPolDn": "uni/fabric/monfab-default", + "monitorServer": "disabled", + "monitoringUser": "default", + "name": "10.10.10.50", + "nameAlias": "", + "operState": "unknown", + "ownerKey": "", + "ownerTag": "", + "port": "389", + "retries": "1", + "rootdn": "", + "snmpIndex": "2", + "status": "", + "timeout": "30", + "uid": "15374", + "userdom": ":all:", + "vrfName": "" + } + } + } +] diff --git a/tests/checks/aaa_snmpd_dns_provider_check/snmp_disabled.json b/tests/checks/aaa_snmpd_dns_provider_check/snmp_disabled.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/tests/checks/aaa_snmpd_dns_provider_check/snmp_disabled.json @@ -0,0 +1 @@ +[] diff --git a/tests/checks/aaa_snmpd_dns_provider_check/snmp_enabled.json b/tests/checks/aaa_snmpd_dns_provider_check/snmp_enabled.json new file mode 100644 index 0000000..f729977 --- /dev/null +++ b/tests/checks/aaa_snmpd_dns_provider_check/snmp_enabled.json @@ -0,0 +1,25 @@ +[ + { + "snmpPol": { + "attributes": { + "adminSt": "enabled", + "annotation": "", + "childAction": "", + "contact": "", + "descr": "", + "dn": "uni/fabric/snmppol-default", + "extMngdBy": "", + "lcOwn": "local", + "location": "", + "modTs": "2025-01-09T10:15:30.123+00:00", + "monPolDn": "uni/fabric/monfab-default", + "name": "default", + "nameAlias": "", + "ownerKey": "", + "ownerTag": "", + "status": "", + "userdom": ":all:" + } + } + } +] diff --git a/tests/checks/aaa_snmpd_dns_provider_check/test_aaa_snmpd_dns_provider_check.py b/tests/checks/aaa_snmpd_dns_provider_check/test_aaa_snmpd_dns_provider_check.py new file mode 100644 index 0000000..c6cb9d8 --- /dev/null +++ b/tests/checks/aaa_snmpd_dns_provider_check/test_aaa_snmpd_dns_provider_check.py @@ -0,0 +1,118 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "aaa_snmpd_dns_provider_check" + +# icurl queries +providers_api = 'uni.json?query-target=subtree&target-subtree-class=aaaTacacsPlusProvider,aaaRadiusProvider,aaaLdapProvider' +snmp_api = 'snmpPol.json?query-target-filter=and(eq(snmpPol.adminSt,"enabled"))' + + +@pytest.mark.parametrize( + "icurl_outputs, tversion, expected_result, expected_data_count", + [ + # Test case 1: No target version provided - should return MANUAL + ( + { + providers_api: read_data(dir, "providers_with_dns.json"), + snmp_api: read_data(dir, "snmp_disabled.json"), + }, + None, + script.MANUAL, + 0, + ), + # Test case 2: Target version 6.1(5e) or newer - should return PASS (not affected) + ( + { + providers_api: read_data(dir, "providers_with_dns.json"), + snmp_api: read_data(dir, "snmp_enabled.json"), + }, + "6.1(5e)", + script.PASS, + 0, + ), + # Test case 3: Target version newer than 6.1(5e) - should return PASS (not affected) + ( + { + providers_api: read_data(dir, "providers_with_dns.json"), + snmp_api: read_data(dir, "snmp_enabled.json"), + }, + "6.2(1a)", + script.PASS, + 0, + ), + # Test case 4: Target version older than 6.1(5e) with DNS names but no SNMP - should return PASS + ( + { + providers_api: read_data(dir, "providers_with_dns.json"), + snmp_api: read_data(dir, "snmp_disabled.json"), + }, + "6.1(4h)", + script.PASS, + 2, + ), + # Test case 5: Target version older than 6.1(5e) with DNS names and SNMP enabled - should return MANUAL + ( + { + providers_api: read_data(dir, "providers_with_dns.json"), + snmp_api: read_data(dir, "snmp_enabled.json"), + }, + "6.1(4h)", + script.MANUAL, + 3, + ), + # Test case 6: Target version older than 6.1(5e) with only IP addresses - should return PASS + ( + { + providers_api: read_data(dir, "providers_with_ips.json"), + snmp_api: read_data(dir, "snmp_disabled.json"), + }, + "6.1(3g)", + script.PASS, + 0, + ), + # Test case 7: Target version older than 6.1(5e) with mixed (IP and DNS) and SNMP enabled - should return MANUAL + ( + { + providers_api: read_data(dir, "providers_mixed.json"), + snmp_api: read_data(dir, "snmp_enabled.json"), + }, + "6.0(9e)", + script.MANUAL, + 3, + ), + # Test case 8: Target version older than 6.1(5e) with no providers but SNMP enabled - should return PASS + ( + { + providers_api: read_data(dir, "providers_empty.json"), + snmp_api: read_data(dir, "snmp_enabled.json"), + }, + "5.2(8i)", + script.PASS, + 0, + ), + # Test case 9: Target version older than 6.1(5e) with no providers and no SNMP - should return PASS + ( + { + providers_api: read_data(dir, "providers_empty.json"), + snmp_api: read_data(dir, "snmp_disabled.json"), + }, + "5.2(8i)", + script.PASS, + 0, + ), + ], +) +def test_logic(run_check, mock_icurl, tversion, expected_result, expected_data_count): + result = run_check(tversion=script.AciVersion(tversion) if tversion else None) + assert result.result == expected_result + if expected_data_count > 0: + assert len(result.data) == expected_data_count