Added pre-upgrade validation for N9K-C9408 with more than 6 N9K-X9400-16W LEM's for the bug CSCws82819

aci-preupgrade-validation-script.py

@@ -6026,6 +6026,53 @@ def apic_downgrade_compat_warning_check(cversion, tversion, **kwargs):
     return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
 
+@check_wrapper(check_title='N9K-C9408 with 6 or more N9K-X9400-16W LEMs')
+def c9408_boot_loop_lem_count_check(tversion, fabric_nodes, **kwargs):
+    result = PASS
+    headers = ["Node ID", "Switch Model", "LEM Model", "LEM Count"]
+    data = []
+    recommended_action = (
+        "Upgrade from pre-16.1(2f) to 16.1(2f) or later on N9K-C9408 with 6 or more LEMs will result in boot loop. Do NOT proceed. Use fewer LEMs or choose a different version.  "
+    )
+    doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#n9k-c9408-with-6-n9k-x9400-16w-lems'
+
+    if tversion.older_than("6.1(2f)") or tversion.newer_than("6.2(1g)"):
+        return Result(result=NA, msg=VER_NOT_AFFECTED)
+
+    c9408_nodes = {}
+    for node in fabric_nodes:
+        node_id = node['fabricNode']['attributes']['id']
+        model = node['fabricNode']['attributes']['model']
+        if model == "N9K-C9408":
+            c9408_nodes[node_id] = "N9K-C9408"
+
+    if not c9408_nodes:
+        return Result(result=NA, msg='No N9K-C9408 nodes found. Skipping.')
+
+    eqptLC_api = 'eqptLC.json?query-target-filter=eq(eqptLC.model,"N9K-X9400-16W")'
+    eqptLCs = icurl('class', eqptLC_api)
+
+    lem_count_per_node = defaultdict(int)
+    for eqptLC in eqptLCs:
+        dn = eqptLC['eqptLC']['attributes']['dn']
+        dn_match = re.search(node_regex, dn)
+        if not dn_match:
+            continue
+        node_id = dn_match.group("node")
+        if node_id in c9408_nodes:
+            lem_count_per_node[node_id] += 1
+
+    for node_id in sorted(c9408_nodes, key=int):
+        lem_count = lem_count_per_node[node_id]
+        if lem_count > 5:
+            data.append([node_id, c9408_nodes[node_id], "N9K-X9400-16W", lem_count])
+
+    if data:
+        result = FAIL_O
+
+    return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
+
+
 # ---- Script Execution ----
 
 
@@ -6184,6 +6231,7 @@ class CheckManager:
         clock_signal_component_failure_check,
         stale_decomissioned_spine_check,
         n9408_model_check,
+        c9408_lem_count_check,
         pbr_high_scale_check,
         standby_sup_sync_check,
         isis_database_byte_check,

docs/docs/validations.md

@@ -187,6 +187,7 @@ Items                                           | Defect       | This Script
 [Nexus 950X FM or LC Might Fail to boot after reload][d20] | CSCvg26013   | :white_check_mark: | :no_entry_sign:
 [Stale Decommissioned Spine][d21]               | CSCwf58763   | :white_check_mark: | :no_entry_sign:
 [N9K-C9408 Platform Model][d22]                 | CSCwk77800   | :white_check_mark: | :no_entry_sign:
+[N9K-C9408 with 6+ N9K-X9400-16W LEMs][d29]     | CSCws82819   | :white_check_mark: | :no_entry_sign:
 [PBR High Scale][d23]                           | CSCwi66348   | :white_check_mark: | :no_entry_sign:
 [Standby Sup Image Sync][d24]                   | CSCwi66348   | :white_check_mark: | :no_entry_sign:
 [Observer Database Size][d25]                   | CSCvw45531   | :white_check_mark: | :no_entry_sign:
@@ -222,6 +223,7 @@ Items                                           | Defect       | This Script
 [d26]: #stale-pconsra-object
 [d27]: #isis-dteps-byte-size
 [d28]: #policydist-configpushshardcont-crash
+[d29]: #n9k-c9408-with-6-n9k-x9400-16w-lems
 
 
 ## General Check Details
@@ -2647,6 +2649,17 @@ Due to [CSCwp95515][59], upgrading to an affected version while having any `conf
 
 If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade.
 
+### N9K-C9408 with 6 or more N9K-X9400-16W LEMs
+
+Due to defect [CSCws82819][62], upgrading from pre-16.1(2f) to 16.1(2f) or later can result in a boot loop when a `N9K-C9408` node has 6 or more installed N9K-X9400-16W` LEMs.
+
+Recommended action:
+
+* Do **not** proceed with the upgrade when flagged.
+* Reduce the number of `N9K-X9400-16W` LEMs to 5 or fewer on each affected `N9K-C9408` node, or choose a different target version.
+* If reducing LEM count is not operationally feasible, open a TAC case for an upgrade path recommendation.
+
+
 
 [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script
 [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html
@@ -2710,3 +2723,4 @@ If any instances of `configpushShardCont` are flagged by this script, Cisco TAC
 [59]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp95515
 [60]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#Inter
 [61]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#EnablePolicyCompression
+[62]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCws82819

tests/checks/c9408_lem_count_check/eqptLC_5_node101.json

@@ -0,0 +1,7 @@
+[
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-2/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-3/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-4/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-5/lc", "model": "N9K-X9400-16W"}}}
+]

tests/checks/c9408_lem_count_check/eqptLC_6_node101.json

@@ -0,0 +1,8 @@
+[
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-2/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-3/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-4/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-5/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-6/lc", "model": "N9K-X9400-16W"}}}
+]

tests/checks/c9408_lem_count_check/eqptLC_7_node101.json

@@ -0,0 +1,58 @@
+[
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-1/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-2/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-3/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-4/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-5/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-6/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  },
+  {
+    "eqptLC": {
+      "attributes": {
+        "dn": "topology/pod-1/node-101/sys/ch/lcslot-7/lc",
+        "model": "N9K-X9400-16W"
+      }
+    }
+  }
+]

tests/checks/c9408_lem_count_check/eqptLC_empty.json

@@ -0,0 +1 @@
+[]

tests/checks/c9408_lem_count_check/eqptLC_mixed_101_102_201.json

@@ -0,0 +1,15 @@
+[
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-2/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-3/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-4/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-5/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-101/sys/ch/lcslot-6/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-2/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-3/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-4/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-5/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-102/sys/ch/lcslot-6/lc", "model": "N9K-X9400-16W"}}},
+  {"eqptLC": {"attributes": {"dn": "topology/pod-1/node-201/sys/ch/lcslot-1/lc", "model": "N9K-X9400-16W"}}}
+]

tests/checks/c9408_lem_count_check/fabric_nodes_c9408_101.json

@@ -0,0 +1,10 @@
+[
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "101",
+        "model": "N9K-C9408"
+      }
+    }
+  }
+]

tests/checks/c9408_lem_count_check/fabric_nodes_mixed.json

@@ -0,0 +1,26 @@
+[
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "101",
+        "model": "N9K-C9408"
+      }
+    }
+  },
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "102",
+        "model": "N9K-C9408"
+      }
+    }
+  },
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "201",
+        "model": "N9K-C93180YC-FX"
+      }
+    }
+  }
+]

tests/checks/c9408_lem_count_check/fabric_nodes_no_c9408.json

@@ -0,0 +1,10 @@
+[
+  {
+    "fabricNode": {
+      "attributes": {
+        "id": "101",
+        "model": "N9K-C93180YC-FX"
+      }
+    }
+  }
+]

tests/checks/c9408_lem_count_check/test_c9408_lem_count_check.py

@@ -0,0 +1,136 @@
+import os
+import pytest
+import logging
+import importlib
+from helpers.utils import read_data
+
+script = importlib.import_module("aci-preupgrade-validation-script")
+
+log = logging.getLogger(__name__)
+dir = os.path.dirname(os.path.abspath(__file__))
+
+test_function = "c9408_lem_count_check"
+
+
+eqptLC_api = 'eqptLC.json?query-target-filter=eq(eqptLC.model,"N9K-X9400-16W")'
+
+
+@pytest.mark.parametrize(
+    "icurl_outputs, tversion, fabric_nodes, expected_result, expected_data, expected_msg",
+    [
+        # Version not affected (lower than 6.1(2f))
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_empty.json")},
+            "6.1(2e)",
+            read_data(dir, "fabric_nodes_c9408_101.json"),
+            script.NA,
+            [],
+            script.VER_NOT_AFFECTED,
+        ),
+        # Version not affected (higher than 6.2(1g))
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_empty.json")},
+            "6.2(1h)",
+            read_data(dir, "fabric_nodes_c9408_101.json"),
+            script.NA,
+            [],
+            script.VER_NOT_AFFECTED,
+        ),
+        # Applicable version but no C9408 node
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_6_node101.json")},
+            "6.1(2f)",
+            read_data(dir, "fabric_nodes_no_c9408.json"),
+            script.NA,
+            [],
+            "No N9K-C9408 nodes found. Skipping.",
+        ),
+        # Applicable version, C9408 exists, no LEM entries
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_empty.json")},
+            "6.1(2f)",
+            read_data(dir, "fabric_nodes_c9408_101.json"),
+            script.PASS,
+            [],
+            "",
+        ),
+        # Applicable version, exactly 5 LEMs -> PASS
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_5_node101.json")},
+            "6.2(1g)",
+            read_data(dir, "fabric_nodes_c9408_101.json"),
+            script.PASS,
+            [],
+            "",
+        ),
+        # Applicable mid-train version 6.1(5e), less than 6 LEMs on C9408 -> PASS
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_5_node101.json")},
+            "6.1(5e)",
+            read_data(dir, "fabric_nodes_c9408_101.json"),
+            script.PASS,
+            [],
+            "",
+        ),
+        # Applicable mid-train version 6.1(5e), 6 LEMs on C9408 -> FAIL_O
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_6_node101.json")},
+            "6.1(5e)",
+            read_data(dir, "fabric_nodes_c9408_101.json"),
+            script.FAIL_O,
+            [["101", "N9K-C9408", "N9K-X9400-16W", 6]],
+            "",
+        ),
+        # Applicable mid-train version 6.1(5e), more than 6 LEMs on C9408 -> FAIL_O
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_7_node101.json")},
+            "6.1(5e)",
+            read_data(dir, "fabric_nodes_c9408_101.json"),
+            script.FAIL_O,
+            [["101", "N9K-C9408", "N9K-X9400-16W", 7]],
+            "",
+        ),
+        # Applicable version, 6 LEMs on C9408 -> FAIL_O
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_6_node101.json")},
+            "6.1(2f)",
+            read_data(dir, "fabric_nodes_c9408_101.json"),
+            script.FAIL_O,
+            [["101", "N9K-C9408", "N9K-X9400-16W", 6]],
+            "",
+        ),
+        # Applicable version, more than 6 LEMs on C9408 -> FAIL_O
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_7_node101.json")},
+            "6.1(2f)",
+            read_data(dir, "fabric_nodes_c9408_101.json"),
+            script.FAIL_O,
+            [["101", "N9K-C9408", "N9K-X9400-16W", 7]],
+            "",
+        ),
+        # Count only C9408 nodes and only matching LEM model
+        (
+            {eqptLC_api: read_data(dir, "eqptLC_mixed_101_102_201.json")},
+            "6.1(3a)",
+            read_data(dir, "fabric_nodes_mixed.json"),
+            script.FAIL_O,
+            [
+                ["101", "N9K-C9408", "N9K-X9400-16W", 6],
+                ["102", "N9K-C9408", "N9K-X9400-16W", 6],
+            ],
+            "",
+        ),
+    ],
+)
+def test_logic(run_check, mock_icurl, icurl_outputs, tversion, fabric_nodes, expected_result, expected_data, expected_msg):
+    result = run_check(
+        tversion=script.AciVersion(tversion),
+        fabric_nodes=fabric_nodes,
+    )
+
+    assert result.result == expected_result
+    assert result.data == expected_data
+    assert result.msg == expected_msg
+
+    if expected_result == script.FAIL_O:
+        assert "boot loop" in result.recommended_action