fead: add OpenTelemetry observability support for modular inputs

.github/workflows/build-test-release.yml

@@ -55,7 +55,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.7"
+          python-version: "3.9"
       - uses: pre-commit/action@v3.0.1
 
   semgrep:
@@ -71,8 +71,6 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.7"
-          - "3.8"
           - "3.9"
           - "3.10"
           - "3.11"
@@ -100,7 +98,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.7
+          python-version: "3.9"
       - run: curl -sSL https://install.python-poetry.org | python3 - --version 1.5.1
       - name: Install Splunk
         run: |
@@ -135,7 +133,7 @@ jobs:
           echo -e "[user_info]\nUSERNAME=Admin\nPASSWORD=Chang3d"'!' | tee -a $SPLUNK_HOME/etc/system/local/user-seed.conf
           echo 'OPTIMISTIC_ABOUT_FILE_LOCKING=1' | tee -a $SPLUNK_HOME/etc/splunk-launch.conf
           $SPLUNK_HOME/bin/splunk start --accept-license
-          $SPLUNK_HOME/bin/splunk cmd python -m pip install solnlib
+          $SPLUNK_HOME/bin/splunk cmd python -m pip install .
           $SPLUNK_HOME/bin/splunk set servername custom-servername -auth admin:Chang3d!
           $SPLUNK_HOME/bin/splunk restart
           until curl -k -s -u admin:Chang3d! https://localhost:8089/services/server/info\?output_mode\=json | jq '.entry[0].content.kvStoreStatus' | grep -o "ready" ; do echo -n "Waiting for KVStore to become ready-" && sleep 5 ; done
@@ -167,7 +165,7 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.7"
+          python-version: "3.9"
       - run: curl -sSL https://install.python-poetry.org | python3 - --version 1.5.1
       - run: |
           poetry install

.licenserc.yaml

@@ -17,6 +17,7 @@ header:
   license:
     spdx-id: Apache-2.0
     copyright-owner: Splunk Inc.
+    pattern: "Copyright \\d{4} Splunk Inc"
 
   paths-ignore:
     - ".github/"

.pre-commit-config.yaml

@@ -3,16 +3,17 @@ repos:
     rev: v3.1.0
     hooks:
     -   id: pyupgrade
-        args: [--py37-plus]
+        args: [--py39-plus]
 -   repo: https://github.com/psf/black
     rev: 22.10.0
     hooks:
     -   id: black
--   repo: https://github.com/myint/docformatter
-    rev: v1.5.0
+-   repo: https://github.com/PyCQA/docformatter
+    rev: v1.7.7
     hooks:
       - id: docformatter
         args: [--in-place]
+        language_version: python3
 -   repo: https://github.com/PyCQA/flake8
     rev: 5.0.4
     hooks:

docs/index.md

@@ -4,4 +4,4 @@ Splunk Solutions SDK is an open source packaged solution for getting data into S
 This SDK is used by Splunk Add-on builder, and Splunk UCC based add-ons and is intended for use by partner
 developers. This SDK/Library extends the Splunk SDK for Python.
 
-> Note: this project uses `poetry` 1.5.1.
+> Note: this project uses `poetry` 2.1.2.

docs/observability.md

@@ -0,0 +1,3 @@
+# observability.py
+
+::: solnlib.observability

mkdocs.yml

@@ -45,6 +45,7 @@ nav:
       - "hec_config.py": hec_config.md
       - "log.py": log.md
       - "net_utils.py": net_utils.md
+      - "observability.py": observability.md
       - "orphan_process_monitor.py": orphan_process_monitor.md
       - "pattern.py": pattern.md
       - "server_info.py": server_info.md

pyproject.toml

@@ -29,8 +29,6 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Topic :: Software Development :: Code Generators",
     "License :: OSI Approved :: Apache Software License",
-    "Programming Language :: Python :: 3.7",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
@@ -39,10 +37,14 @@ classifiers = [
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.7,<3.14"
+python = ">=3.9,<3.14"
 sortedcontainers = ">=2"
 defusedxml = ">=0.7"
 splunk-sdk = ">=2.0.2"
+opentelemetry-api = "1.39.1"
+opentelemetry-sdk = "1.39.1"
+opentelemetry-exporter-otlp-proto-grpc = "1.39.1"
+grpcio = "1.74.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7"
@@ -51,6 +53,10 @@ mkdocs-material = ">=9"
 mkdocstrings = {version=">=0", extras=["python"]}
 mkdocs-print-site-plugin = "^2.3.6"
 
+[tool.docformatter]
+wrap-summaries = 0
+wrap-descriptions = 0
+
 [build-system]
 requires = ["poetry>=1.0.0"]
 build-backend = "poetry.masonry.api"

solnlib/__init__.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
 """The Splunk Software Development Kit for Solutions."""
 
 from . import (

solnlib/_utils.py

@@ -16,7 +16,7 @@
 """This module provide utils that are private to solnlib."""
 
 import re
-from typing import Any, Dict, Optional, Union
+from typing import Any, Optional, Union
 
 from splunklib import binding, client
 
@@ -33,7 +33,7 @@ def get_collection_data(
     scheme: Optional[str] = None,
     host: Optional[str] = None,
     port: Optional[Union[str, int]] = None,
-    fields: Optional[Dict] = None,
+    fields: Optional[dict] = None,
     **context: Any,
 ) -> client.KVStoreCollectionData:
     """Get collection data, if there is no such collection - creates one.

solnlib/acl.py

@@ -13,11 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
 """This module contains interfaces that support CRUD operations on ACL."""
 
 import json
-from typing import List
 
 from splunklib import binding
 
@@ -108,8 +106,8 @@ def update(
         self,
         path: str,
         owner: str = None,
-        perms_read: List = None,
-        perms_write: List = None,
+        perms_read: list = None,
+        perms_write: list = None,
     ) -> dict:
         """Update ACL of /servicesNS/{`owner`}/{`app`}/{`path`}.
 

solnlib/alerts_rest_client.py

@@ -15,7 +15,7 @@
 #
 import json
 from enum import Enum
-from typing import Tuple, Union, Optional
+from typing import Union, Optional
 
 from solnlib import splunk_rest_client as rest_client
 
@@ -89,7 +89,7 @@ def create_search_alert(
         alert_condition: str = "",
         alert_comparator: AlertComparator = AlertComparator.GREATER_THAN,
         alert_threshold: Union[int, float, str] = 0,
-        time_window: Tuple[str, str] = ("-15m", "now"),
+        time_window: tuple[str, str] = ("-15m", "now"),
         alert_severity: AlertSeverity = AlertSeverity.WARN,
         cron_schedule: str = "* * * * *",
         expires: Union[int, str] = "24h",
@@ -188,7 +188,7 @@ def update_search_alert(
         alert_condition: Optional[str] = None,
         alert_comparator: Optional[AlertComparator] = None,
         alert_threshold: Optional[Union[int, float, str]] = None,
-        time_window: Optional[Tuple[str, str]] = None,
+        time_window: Optional[tuple[str, str]] = None,
         alert_severity: Optional[AlertSeverity] = None,
         cron_schedule: Optional[str] = None,
         expires: Optional[Union[int, str]] = None,

solnlib/bulletin_rest_client.py

@@ -15,7 +15,7 @@
 #
 
 from solnlib import splunk_rest_client as rest_client
-from typing import Optional, List
+from typing import Optional
 import json
 
 __all__ = ["BulletinRestClient"]
@@ -72,8 +72,8 @@ def create_message(
         self,
         msg: str,
         severity: Severity = Severity.WARNING,
-        capabilities: Optional[List[str]] = None,
-        roles: Optional[List] = None,
+        capabilities: Optional[list[str]] = None,
+        roles: Optional[list] = None,
     ):
         """Creates a message in the Splunk's bulletin. Calling this method
         multiple times for the same instance will overwrite existing message.
@@ -144,7 +144,7 @@ def delete_message(self):
         self._rest_client.delete(endpoint)
 
     @staticmethod
-    def _validate_and_get_body_value(arg, error_msg) -> List:
+    def _validate_and_get_body_value(arg, error_msg) -> list:
         if type(arg) is list and (all(isinstance(el, str) for el in arg)):
             return [el for el in arg]
         else:

solnlib/concurrent/concurrent_executor.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
 """Concurrent executor provides concurrent executing function either in a
 thread pool or a process pool."""
 
@@ -69,11 +68,11 @@ def run_io_func_async(self, func, args=(), kwargs=None, callback=None):
         return self._io_executor.apply_async(func, args, kwargs, callback)
 
     def enqueue_io_funcs(self, funcs, block=True):
-        """run jobs in a fire and forget way, no result will be handled over to
+        """Run jobs in a fire and forget way, no result will be handled over to
         clients.
 
-        :param funcs: tuple/list-like or generator like object, func shall be
-                      callable
+        :param funcs: tuple/list-like or generator like object, func
+            shall be callable
         """
 
         return self._io_executor.enqueue_funcs(funcs, block)

solnlib/concurrent/process_pool.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
 """A wrapper of multiprocessing.pool."""
 
 import multiprocessing

solnlib/concurrent/thread_pool.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
 """A simple thread pool implementation."""
 
 import multiprocessing
@@ -94,11 +93,11 @@ def tear_down(self):
         logging.info("ThreadPool stopped.")
 
     def enqueue_funcs(self, funcs, block=True):
-        """run jobs in a fire and forget way, no result will be handled over to
+        """Run jobs in a fire and forget way, no result will be handled over to
         clients.
 
-        :param funcs: tuple/list-like or generator like object, func shall be
-                      callable
+        :param funcs: tuple/list-like or generator like object, func
+            shall be callable
         """
 
         if not self._started:

solnlib/conf_manager.py

@@ -13,15 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
 """This module contains simple interfaces for Splunk config file management,
 you can update/get/delete stanzas and encrypt/decrypt some fields of stanza
 automatically."""
 
 import json
 import logging
 import traceback
-from typing import List, Union, Dict, NoReturn
+from typing import Union, NoReturn
 
 from splunklib import binding, client
 
@@ -280,7 +279,7 @@ def get_all(self, only_current_app: bool = False) -> dict:
         return res
 
     @retry(exceptions=[binding.HTTPError])
-    def update(self, stanza_name: str, stanza: dict, encrypt_keys: List[str] = None):
+    def update(self, stanza_name: str, stanza: dict, encrypt_keys: list[str] = None):
         """Update stanza.
 
         It will try to encrypt the credential automatically fist if
@@ -559,7 +558,7 @@ def get_proxy_dict(
     conf_name: str,
     proxy_stanza: str = "proxy",
     **kwargs,
-) -> Union[Dict[str, str], NoReturn]:
+) -> Union[dict[str, str], NoReturn]:
     """This function returns the proxy settings for the addon from
     configuration file.
 

solnlib/credentials.py

@@ -13,12 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
 """This module contains Splunk credential related interfaces."""
 
 import re
 import warnings
-from typing import Dict, List
 
 from splunklib import binding, client
 
@@ -232,21 +230,21 @@ def delete_password(self, user: str):
                 f"Failed to delete password of realm={self._realm}, user={user}"
             )
 
-    def get_raw_passwords(self) -> List[client.StoragePassword]:
+    def get_raw_passwords(self) -> list[client.StoragePassword]:
         """Returns all passwords in the "raw" format."""
         warnings.warn(
             "Please pass realm to the CredentialManager, "
             "so it can utilize get_raw_passwords_in_realm method instead."
         )
         return self._storage_passwords.list(count=-1)
 
-    def get_raw_passwords_in_realm(self) -> List[client.StoragePassword]:
+    def get_raw_passwords_in_realm(self) -> list[client.StoragePassword]:
         """Returns all passwords within the realm in the "raw" format."""
         if self._realm is None:
             raise ValueError("No realm was specified")
         return self._storage_passwords.list(count=-1, search=f"realm={self._realm}")
 
-    def get_clear_passwords(self) -> List[Dict[str, str]]:
+    def get_clear_passwords(self) -> list[dict[str, str]]:
         """Returns all passwords in the "clear" format."""
         warnings.warn(
             "Please pass realm to the CredentialManager, "
@@ -255,14 +253,14 @@ def get_clear_passwords(self) -> List[Dict[str, str]]:
         raw_passwords = self.get_raw_passwords()
         return self._get_clear_passwords(raw_passwords)
 
-    def get_clear_passwords_in_realm(self) -> List[Dict[str, str]]:
+    def get_clear_passwords_in_realm(self) -> list[dict[str, str]]:
         """Returns all passwords within the realm in the "clear" format."""
         if self._realm is None:
             raise ValueError("No realm was specified")
         raw_passwords = self.get_raw_passwords_in_realm()
         return self._get_clear_passwords(raw_passwords)
 
-    def _get_all_passwords_in_realm(self) -> List[client.StoragePassword]:
+    def _get_all_passwords_in_realm(self) -> list[client.StoragePassword]:
         warnings.warn(
             "_get_all_passwords_in_realm is deprecated, "
             "please use get_raw_passwords_in_realm instead.",
@@ -277,8 +275,8 @@ def _get_all_passwords_in_realm(self) -> List[client.StoragePassword]:
         return all_passwords
 
     def _get_clear_passwords(
-        self, passwords: List[client.StoragePassword]
-    ) -> List[Dict[str, str]]:
+        self, passwords: list[client.StoragePassword]
+    ) -> list[dict[str, str]]:
         results = {}
         ptn = re.compile(rf"(.+){self.SEP}(\d+)")
         for password in passwords:
@@ -327,7 +325,7 @@ def _get_clear_passwords(
         return list(results.values())
 
     @retry(exceptions=[binding.HTTPError])
-    def _get_all_passwords(self) -> List[Dict[str, str]]:
+    def _get_all_passwords(self) -> list[dict[str, str]]:
         warnings.warn(
             "_get_all_passwords is deprecated, "
             "please use get_all_passwords_in_realm instead.",