diff --git a/services/spine-exporter/README.md b/services/spine-exporter/README.md index 004e66a8..032f72ed 100644 --- a/services/spine-exporter/README.md +++ b/services/spine-exporter/README.md @@ -111,14 +111,15 @@ Note: date interpretation is using UTC. #### Environment variables Configuration is achieved via the following environment variables: -| Environment variable | Description | -|-----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| SPLUNK_URL | URL of the Splunk API | -| SPLUNK_API_TOKEN_PARAM_NAME | AWS Parameter store name which contains the Splunk API token | -| OUTPUT_SPINE_DATA_BUCKET | Output S3 Bucket to write the Spine logs | -| BUILD_TAG | Optional - Unique identifier for version of code build tag (e.g. short git hash) | -| START_DATETIME | Optional - The start date and time for the search results date range from Splunk API (must be at midnight) | -| END_DATETIME | Optional - The end date and time for the search results date range from Splunk API (must be at midnight) | +| Environment variable | Description | +|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| SPLUNK_URL | URL of the Splunk API | +| SPLUNK_INDEX | Name of the Splunk index to communicate with. Prod & Non-Prod use different indexes | +| SPLUNK_API_TOKEN_PARAM_NAME | AWS Parameter store name which contains the Splunk API token | +| OUTPUT_SPINE_DATA_BUCKET | Output S3 Bucket to write the Spine logs | +| BUILD_TAG | Optional - Unique identifier for version of code build tag (e.g. short git hash) | +| START_DATETIME | Optional - The start date and time for the search results date range from Splunk API (must be at midnight) | +| END_DATETIME | Optional - The end date and time for the search results date range from Splunk API (must be at midnight) | | SEARCH_WAIT_TIME_IN_SECONDS | Optional - Wait time before making another request to Splunk API (default is 0). When running Spine Exporter task manually for more than 1 day, set to 30 to make sure Splunk is not overloaded with requests | diff --git a/services/spine-exporter/src/prmexporter/config.py b/services/spine-exporter/src/prmexporter/config.py index 304446ae..ca17c6de 100644 --- a/services/spine-exporter/src/prmexporter/config.py +++ b/services/spine-exporter/src/prmexporter/config.py @@ -55,6 +55,7 @@ def read_optional_datetime(self, name) -> datetime: @dataclass class SpineExporterConfig: splunk_url: str + splunk_index: str splunk_api_token_param_name: str output_spine_data_bucket: str build_tag: str @@ -68,6 +69,7 @@ def from_environment_variables(cls, env_vars): env = EnvConfig(env_vars) return cls( splunk_url=env.read_str("SPLUNK_URL"), + splunk_index=env.read_str("SPLUNK_INDEX"), splunk_api_token_param_name=env.read_str("SPLUNK_API_TOKEN_PARAM_NAME"), output_spine_data_bucket=env.read_str("OUTPUT_SPINE_DATA_BUCKET"), build_tag=env.read_str("BUILD_TAG"), diff --git a/services/spine-exporter/src/prmexporter/spine_exporter.py b/services/spine-exporter/src/prmexporter/spine_exporter.py index 04e5e32d..22525fad 100644 --- a/services/spine-exporter/src/prmexporter/spine_exporter.py +++ b/services/spine-exporter/src/prmexporter/spine_exporter.py @@ -42,14 +42,21 @@ def _construct_json_log_date_range_info(search_dates: SearchDates) -> dict: def _get_api_auth_token(self) -> str: return self._ssm_secret_manager.get_secret(self._config.splunk_api_token_param_name) - def _fetch_spine_data(self, search_start_time: str, search_end_time: str) -> bytes: + def _fetch_spine_data( + self, + search_start_time: str, + search_end_time: str, + splunk_index: str, + ) -> bytes: request_body = { "output_mode": "csv", "earliest_time": search_start_time, "latest_time": search_end_time, - "search": """search index=\"spine2vfmmonitor\" service=\"gp2gp\" logReference=\"MPS0053d\" - | table _time, conversationID, GUID, interactionID, messageSender, - messageRecipient, messageRef, jdiEvent, toSystem, fromSystem""", + "search": ( + f'search index="{splunk_index}" service="gp2gp" logReference="MPS0053d"\n' + " | table _time, conversationID, GUID, interactionID, messageSender,\n" + " messageRecipient, messageRef, jdiEvent, toSystem, fromSystem" + ), } splunk_api_token = self._get_api_auth_token() @@ -96,7 +103,9 @@ def run(self): search_end_datetime = convert_to_datetime_string(date + timedelta(days=1)) spine_data = self._fetch_spine_data( - search_start_time=search_start_datetime, search_end_time=search_end_datetime + search_start_time=search_start_datetime, + search_end_time=search_end_datetime, + splunk_index=self._config.splunk_index, ) s3_key = self._create_s3_key(date) diff --git a/services/spine-exporter/tests/e2e/test_spine_exporter.py b/services/spine-exporter/tests/e2e/test_spine_exporter.py index 0630a581..2ef9f9ed 100644 --- a/services/spine-exporter/tests/e2e/test_spine_exporter.py +++ b/services/spine-exporter/tests/e2e/test_spine_exporter.py @@ -20,6 +20,7 @@ FAKE_SPLUNK_HOST = "127.0.0.1" FAKE_SPLUNK_PORT = 9000 FAKE_SPLUNK_URL = f"http://{FAKE_SPLUNK_HOST}:{FAKE_SPLUNK_PORT}" +FAKE_SPLUNK_INDEX = "fakeindex" FAKE_AWS_HOST = "127.0.0.1" FAKE_AWS_PORT = 8887 @@ -263,6 +264,7 @@ def _setup(): environ["AWS_SECRET_ACCESS_KEY"] = "testing" environ["AWS_DEFAULT_REGION"] = "us-west-1" environ["SPLUNK_URL"] = FAKE_SPLUNK_URL + environ["SPLUNK_INDEX"] = FAKE_SPLUNK_INDEX environ["OUTPUT_SPINE_DATA_BUCKET"] = OUTPUT_BUCKET_NAME environ["SPLUNK_API_TOKEN_PARAM_NAME"] = API_TOKEN_PARAM_NAME environ["AWS_ENDPOINT_URL"] = FAKE_AWS_URL diff --git a/services/spine-exporter/tests/unit/test_config_environment_variables.py b/services/spine-exporter/tests/unit/test_config_environment_variables.py index eda42b91..636ae50a 100644 --- a/services/spine-exporter/tests/unit/test_config_environment_variables.py +++ b/services/spine-exporter/tests/unit/test_config_environment_variables.py @@ -13,6 +13,7 @@ def test_reads_from_environment_variables(): environment = { "SPLUNK_URL": "https://test.com", + "SPLUNK_INDEX": "testindex", "SPLUNK_API_TOKEN_PARAM_NAME": "/param/name/api-token", "OUTPUT_SPINE_DATA_BUCKET": "output-spine-data-bucket", "BUILD_TAG": "61ad1e1c", @@ -24,6 +25,7 @@ def test_reads_from_environment_variables(): expected_config = SpineExporterConfig( splunk_url="https://test.com", + splunk_index="testindex", splunk_api_token_param_name="/param/name/api-token", output_spine_data_bucket="output-spine-data-bucket", build_tag="61ad1e1c", @@ -50,6 +52,7 @@ def test_error_from_environment_when_required_fields_are_not_set(): def test_reads_from_environment_variables_when_optional_fields_are_not_set(): environment = { "SPLUNK_URL": "https://test.com", + "SPLUNK_INDEX": "testindex", "SPLUNK_API_TOKEN_PARAM_NAME": "/param/name/api-token", "OUTPUT_SPINE_DATA_BUCKET": "output-spine-data-bucket", "BUILD_TAG": "61ad1e1c", @@ -57,6 +60,7 @@ def test_reads_from_environment_variables_when_optional_fields_are_not_set(): expected_config = SpineExporterConfig( splunk_url="https://test.com", + splunk_index="testindex", splunk_api_token_param_name="/param/name/api-token", output_spine_data_bucket="output-spine-data-bucket", build_tag="61ad1e1c",