Release v1.17.0

docs/admin/release_notes/version_1.17.md

@@ -0,0 +1,23 @@
+
+# v1.17 Release Notes
+
+This document describes all new features and changes in the release. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [v1.17.0 (2026-01-30)](https://github.com/networktocode/netutils/releases/tag/v1.17.0)
+
+### Added
+
+- [#752](https://github.com/networktocode/netutils/issues/752) - Added custom parsing of HP Network OS devices.
+- [#793](https://github.com/networktocode/netutils/issues/793) - Added hp_comware running configuration command to the RUNNING_CONFIG_MAPPER.
+
+### Deprecated
+
+- Deprecated the public HPEConfigParser class in lieu of a private class that should be subclassed for specific HP platforms.
+
+### Fixed
+
+- [#780](https://github.com/networktocode/netutils/issues/780) - Fixed parsing of login banner in Palo Alto Networks config.
+
+### Housekeeping
+
+- Added `--pattern` and `--label` options to the `invoke pytest` task.

docs/user/lib_mapper/running_config.md

@@ -19,6 +19,7 @@
 | extreme_slx | → | show running-config |
 | extreme_vsp | → | show running-config |
 | fortinet | → | show full-configuration |
+| hp_comware | → | display current-configuration |
 | hp_procurve | → | show running-config |
 | juniper_junos | → | show configuration | display set |
 | mikrotik_routeros | → | /export |

mkdocs.yml

@@ -124,6 +124,7 @@ nav:
       - Uninstall: "admin/uninstall.md"
       - Release Notes:
           - "admin/release_notes/index.md"
+          - v1.17: "admin/release_notes/version_1.17.md"
           - v1.16: "admin/release_notes/version_1.16.md"
           - v1.15: "admin/release_notes/version_1.15.md"
           - v1.14: "admin/release_notes/version_1.14.md"

netutils/config/compliance.py

@@ -39,6 +39,12 @@
 }
 
 
+# Network OSes that we do not strip leading whitespace from the config lines.
+NON_STRIP_NETWORK_OS = [
+    "hp_comware",
+]
+
+
 # TODO: Once support for 3.7 is dropped, there should be a typing.TypedDict for this which should then also be used
 # as the return type for a bunch of the following methods.
 default_feature: t.Dict[str, t.Union[str, bool, None]] = {
@@ -465,7 +471,9 @@ def section_config(
             else:
                 match = False
         for line_start in section_starts_with:  # type: ignore
-            if not match and line.config_line.startswith(line_start):
+            if not match and not line.parents and line.config_line.startswith(line_start):
                 section_config_list.append(line.config_line)
                 match = True
+    if network_os in NON_STRIP_NETWORK_OS:
+        return "\n".join(section_config_list)
     return "\n".join(section_config_list).strip()

netutils/config/conversion.py

@@ -125,6 +125,7 @@ def paloalto_panos_clean_newlines(cfg: str) -> str:
     return newlines_cleaned_cfg
 
 
+# pylint: disable=too-many-branches
 def paloalto_panos_brace_to_set(cfg: str, cfg_type: str = "file") -> str:
     r"""Convert Palo Alto Brace format configuration to set format.
 
@@ -182,21 +183,44 @@ def paloalto_panos_brace_to_set(cfg: str, cfg_type: str = "file") -> str:
         cfg_raw = paloalto_panos_clean_newlines(cfg=cfg)
         cfg_list = cfg_raw.splitlines()
 
-    for i, line in enumerate(cfg_list):
+    def cfg_generator(cfg_list: t.List[str]) -> t.Generator[str, None, None]:
+        """We use a generator to avoid parsing the banner lines twice."""
+        yield from cfg_list
+
+    cfg_gen = cfg_generator(cfg_list)
+
+    for line in cfg_gen:
         line = line.strip()
         if line.endswith(";") and not line.startswith('";'):
-            line = line.split(";", 1)[0]
+            line = line[:-1]
             line = "".join(str(s) for s in stack) + line
             line = line.split("config ", 1)[1]
             line = "set " + line
             cfg_value.append(line.strip())
-        elif line.endswith('login-banner "') or line.endswith('content "'):
+        elif "login-banner" in line or line.endswith('content "'):
             _first_banner_line = "".join(str(s) for s in stack) + line
             cfg_value.append("set " + _first_banner_line.split("config ", 1)[1])
-
-            for banner_line in cfg_list[i + 1:]:  # fmt: skip
-                if '"' in banner_line:
-                    banner_line = banner_line.split(";", 1)[0]
+            # Palo Alto uses either double or single quotes for the banner delimiter,
+            # but only if there are certain characters or spaces in the banner.
+            if 'login-banner "' in line:
+                delimiter = '"'
+            elif "login-banner '" in line:
+                delimiter = "'"
+            else:
+                delimiter = ""
+
+            # Deal with single line banners first
+            if line.endswith(f"{delimiter};"):
+                line = line[:-1]
+                cfg_value.append(line.strip())
+                continue
+
+            # Multi-line banners
+            for banner_line in cfg_gen:  # fmt: skip
+                # This is a little brittle and will break if any line in the middle of the banner
+                # ends with the expected delimiter and semicolon.
+                if banner_line.endswith(f"{delimiter};"):
+                    banner_line = banner_line[:-1]
                     cfg_value.append(banner_line.strip())
                     break
                 cfg_value.append(banner_line.strip())

netutils/config/parser.py

@@ -8,6 +8,7 @@
 
 from netutils.banner import normalise_delimiter_caret_c
 from netutils.config.conversion import paloalto_panos_brace_to_set
+from netutils.config.utils import _deprecated
 
 ConfigLine = namedtuple("ConfigLine", "config_line,parents")
 
@@ -1491,17 +1492,11 @@ class PaloAltoNetworksConfigParser(BaseSpaceConfigParser):
 
     comment_chars: t.List[str] = []
     banner_start: t.List[str] = [
-        'set system login-banner "',
-        'login-banner "',
-        'set deviceconfig system login-banner "',
+        "set system login-banner",
+        "set deviceconfig system login-banner",
     ]
-    banner_end = '"'
-
-    def is_banner_end(self, line: str) -> bool:
-        """Determine if end of banner."""
-        if line.endswith('"') or line.startswith('";') or line.startswith("set") or line.endswith(self.banner_end):
-            return True
-        return False
+    # Not used, but must be defined
+    banner_end = ""
 
     def _build_banner(self, config_line: str) -> t.Optional[str]:
         """Handle banner config lines.
@@ -1517,24 +1512,24 @@ def _build_banner(self, config_line: str) -> t.Optional[str]:
         """
         self._update_config_lines(config_line)
         self._current_parents += (config_line,)
-        banner_config = []
+        banner_config: t.List[str] = []
         for line in self.generator_config:
-            if not self.is_banner_end(line):
-                banner_config.append(line)
-            else:
-                banner_config.append(line.strip())
-                line = "\n".join(banner_config)
-                if line.endswith('"'):
-                    banner, end, _ = line.rpartition('"')
-                    line = banner + end
-                self._update_config_lines(line.strip())
+            # Note, this is a little fragile and will cause false positives if any line in
+            # the middle of a multi-line banner starts with "set ".
+            if line.startswith("set "):
+                # New command, save the banner and return the next line
+                if banner_config:
+                    banner_string = "\n".join(banner_config)
+                    self._update_config_lines(banner_string)
                 self._current_parents = self._current_parents[:-1]
-                try:
-                    return next(self.generator_config)
-                except StopIteration:
-                    return None
+                return line
+            banner_config.append(line)
 
-        raise ValueError("Unable to parse banner end.")
+        # Edge case, the last line of the config is the banner
+        banner_string = "\n".join(banner_config)
+        self._update_config_lines(banner_string)
+        self._current_parents = self._current_parents[:-1]
+        return None
 
     def build_config_relationship(self) -> t.List[ConfigLine]:  # pylint: disable=too-many-branches
         r"""Parse text of config lines and find their parents.
@@ -1557,42 +1552,27 @@ def build_config_relationship(self) -> t.List[ConfigLine]:  # pylint: disable=to
             ... ]
             True
         """
-        # assume configuration does not need conversion
-        _needs_conversion = False
-
-        # if config is in palo brace format, convert to set
-        if self.config_lines_only is not None:
-            for line in self.config_lines_only.splitlines():
-                if line.endswith("{"):
-                    _needs_conversion = True
-        if _needs_conversion:
+        if self.config_lines_only is None:
+            raise ValueError("Config is empty.")
+
+        if "@dirtyId" in self.config_lines_only:
+            # We have to specifically check for JSON format because it can be confused with the brace format
+            raise ValueError("Found 'json' configuration format. Please provide in 'set' or 'default' (brace) format.")
+        config_lines = self.config_lines_only.splitlines()
+        if any(line.endswith("{") for line in config_lines):
             converted_config = paloalto_panos_brace_to_set(cfg=self.config, cfg_type="string")
             list_config = converted_config.splitlines()
             self.generator_config = (line for line in list_config)
+        elif not any(line.startswith("set ") for line in config_lines):
+            raise ValueError("Unexpected configuration format. Please provide in 'set' or 'default' (brace) format.")
 
         # build config relationships
         for line in self.generator_config:
-            if not line[0].isspace():
-                self._current_parents = ()
-                if self.is_banner_start(line):
-                    line = self._build_banner(line)  # type: ignore
-            else:
-                previous_config = self.config_lines[-1]
-                self._current_parents = (previous_config.config_line,)
-                self.indent_level = self.get_leading_space_count(line)
-                if not self.is_banner_start(line):
-                    line = self._build_nested_config(line)  # type: ignore
-                else:
-                    line = self._build_banner(line)  # type: ignore
-                    if line is not None and line[0].isspace():
-                        line = self._build_nested_config(line)  # type: ignore
-                    else:
-                        self._current_parents = ()
+            if self.is_banner_start(line):
+                line = self._build_banner(line)  # type: ignore
 
             if line is None:
                 break
-            elif self.is_banner_start(line):
-                line = self._build_banner(line)  # type: ignore
 
             self._update_config_lines(line)
         return self.config_lines
@@ -1675,16 +1655,91 @@ def config_lines_only(self) -> str:
         return "\n".join(config_lines)
 
 
-class HPEConfigParser(BaseSpaceConfigParser):
+class _HPEConfigParser(BaseSpaceConfigParser):
     """HPE Implementation of ConfigParser Class."""
 
-    regex_banner = re.compile(r"^header\s(\w+)\s+(?P<banner_delimiter>\^C|\S?)")
+    regex_banner = re.compile(r"^\s*header\s(\w+)\s+(?P<banner_delimiter>\^C|\S?)")
+    banner_start: t.List[str] = ["header "]
+    comment_chars: t.List[str] = ["#"]
 
     def __init__(self, config: str):
-        """Initialize the HPEConfigParser object."""
+        """Initialize the _HPEConfigParser object."""
         self.delimiter = ""
         self._banner_end: t.Optional[str] = None
-        super(HPEConfigParser, self).__init__(config)
+        super(_HPEConfigParser, self).__init__(config)
+
+    @property
+    def config_lines_only(self) -> str:
+        """Remove spaces and unwanted lines from config lines, but leave comments.
+
+        Returns:
+            The non-space lines from ``config``.
+        """
+        if self._config is None:
+            config_lines = (line.rstrip() for line in self.config.splitlines() if line and not line.isspace())
+            self._config = "\n".join(config_lines)
+        return self._config
+
+    def build_config_relationship(self) -> t.List[ConfigLine]:
+        r"""This is a custom build method for HPE Network OS.
+
+        HP config is a bit different from other network operating systems.
+        It uses comments (#) to demarcate sections of the config.
+        Each new section that starts without a leading space is a new section.
+        That new section may or may not have children.
+        Each config line that has a leading space but not a parent is just a single config line.
+        Single lines that have leading spaces also sometimes differs between models (e.g., 59XX vs 79XX series).
+
+        Examples:
+            >>> from netutils.config.parser import _HPEConfigParser, ConfigLine
+            >>> config = '''#
+            ... version 7.1.045, Release 2418P06
+            ... #
+            ...  sysname NTC123456
+            ... #
+            ... vlan 101
+            ...  name Test-Vlan-101
+            ...  description Test Vlan 101
+            ... #'''
+            >>> config_tree = _HPEConfigParser(config)
+            >>> config_tree.build_config_relationship() == \
+            ... [
+            ...     ConfigLine(config_line="version 7.1.045, Release 2418P06", parents=()),
+            ...     ConfigLine(config_line=" sysname NTC123456", parents=()),
+            ...     ConfigLine(config_line="vlan 101", parents=()),
+            ...     ConfigLine(config_line=" name Test-Vlan-101", parents=("vlan 101",)),
+            ...     ConfigLine(config_line=" description Test Vlan 101", parents=("vlan 101",)),
+            ... ]
+            True
+            >>>
+        """
+        new_section = True
+        for line in self.generator_config:
+            if line.startswith(tuple(self.comment_chars)):
+                # Closing any previous sections
+                self._current_parents = ()
+                self.indent_level = 0
+                new_section = True
+                continue
+            if line.strip().startswith(tuple(self.comment_chars)):
+                # Just ignore comments inside sections
+                continue
+            if self.is_banner_start(line):
+                # Special case for banners
+                self._build_banner(line)
+                continue
+
+            current_spaces = self.get_leading_space_count(line) if line[0].isspace() else 0
+            if current_spaces > self.indent_level and not new_section:
+                previous_config = self.config_lines[-1]
+                self._current_parents += (previous_config.config_line,)
+            elif current_spaces < self.indent_level:
+                self._current_parents = self._remove_parents(line, current_spaces)
+
+            new_section = False
+            self.indent_level = current_spaces
+            self._update_config_lines(line)
+        return self.config_lines
 
     def _build_banner(self, config_line: str) -> t.Optional[str]:
         """
@@ -1744,7 +1799,7 @@ def is_banner_one_line(self, config_line: str) -> bool:
 
     def is_banner_start(self, line: str) -> bool:
         """Checks if the given line is the start of a banner."""
-        state = super(HPEConfigParser, self).is_banner_start(line)
+        state = super(_HPEConfigParser, self).is_banner_start(line)
         if state:
             self.banner_end = line
         return state
@@ -1763,15 +1818,15 @@ def banner_end(self, banner_start_line: str) -> None:
         self._banner_end = self.delimiter
 
 
-class HPComwareConfigParser(HPEConfigParser, BaseSpaceConfigParser):
+class HPComwareConfigParser(_HPEConfigParser):
     """HP Comware Implementation of ConfigParser Class."""
 
-    banner_start: t.List[str] = ["header "]
-    comment_chars: t.List[str] = ["#"]
 
-    def _build_banner(self, config_line: str) -> t.Optional[str]:
-        """Build a banner from the given config line."""
-        return super(HPComwareConfigParser, self)._build_banner(config_line)
+@_deprecated(
+    "HPEConfigParser is deprecated and will be removed in a future version. Use subclasses like HPComwareConfigParser instead."
+)
+class HPEConfigParser(_HPEConfigParser):
+    """Deprecated in favor of internal class _HPEConfigParser."""
 
 
 class NvidiaOnyxConfigParser(BaseConfigParser):  # pylint: disable=abstract-method