From 13f9ac445585aa9bcbc48c25502051e53a120a73 Mon Sep 17 00:00:00 2001 From: moxarth-rathod Date: Tue, 23 Dec 2025 14:44:51 +0530 Subject: [PATCH 1/3] Improve system test coverage for all inputs - 2 --- .../_dev/deploy/docker/docker-compose.yml | 15 + .../_dev/deploy/docker/files/config.yml | 111 +++ .../_dev/test/system/test-httpjson-config.yml | 10 + .../data_stream/alerts/sample_event.json | 74 +- packages/blacklens/docs/README.md | 72 +- .../_dev/deploy/docker/files/config.yml | 644 ++++++++++++++++++ .../data_stream/antibot/sample_event.json | 12 +- .../_dev/test/system/test-default-config.yml | 12 + .../data_stream/antimalware/sample_event.json | 20 +- .../_dev/test/system/test-default-config.yml | 12 + .../data_stream/forensics/sample_event.json | 60 +- .../_dev/test/system/test-default-config.yml | 12 + .../threatemulation/sample_event.json | 20 +- .../_dev/test/system/test-default-config.yml | 12 + .../threatextraction/sample_event.json | 36 +- .../_dev/test/system/test-default-config.yml | 12 + .../urlfiltering/sample_event.json | 20 +- .../_dev/test/system/test-default-config.yml | 12 + .../zerophishing/sample_event.json | 23 +- .../docs/README.md | 179 +++-- .../_dev/deploy/docker/docker-compose.yml | 34 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++++++ .../deploy/docker/sample_logs/test-events.log | 1 + .../_dev/test/system/test-azure-config.yml | 16 + .../_dev/test/system/test-default-config.yml | 1 + .../_dev/test/system/test-gcs-config.yml | 16 + .../event/agent/stream/gcs.yml.hbs | 3 + .../data_stream/event/fields/beats.yml | 40 +- .../data_stream/event/manifest.yml | 7 + .../data_stream/event/sample_event.json | 47 +- .../symantec_endpoint_security/docs/README.md | 59 +- 34 files changed, 1566 insertions(+), 337 deletions(-) create mode 100644 packages/blacklens/data_stream/alerts/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/blacklens/data_stream/alerts/_dev/deploy/docker/files/config.yml create mode 100644 packages/blacklens/data_stream/alerts/_dev/test/system/test-httpjson-config.yml create mode 100644 packages/checkpoint_harmony_endpoint/data_stream/antimalware/_dev/test/system/test-default-config.yml create mode 100644 packages/checkpoint_harmony_endpoint/data_stream/forensics/_dev/test/system/test-default-config.yml create mode 100644 packages/checkpoint_harmony_endpoint/data_stream/threatemulation/_dev/test/system/test-default-config.yml create mode 100644 packages/checkpoint_harmony_endpoint/data_stream/threatextraction/_dev/test/system/test-default-config.yml create mode 100644 packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/_dev/test/system/test-default-config.yml create mode 100644 packages/checkpoint_harmony_endpoint/data_stream/zerophishing/_dev/test/system/test-default-config.yml create mode 100644 packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/gcs-mock-service/main.go create mode 100644 packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/sample_logs/test-events.log create mode 100644 packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-azure-config.yml create mode 100644 packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-gcs-config.yml diff --git a/packages/blacklens/data_stream/alerts/_dev/deploy/docker/docker-compose.yml b/packages/blacklens/data_stream/alerts/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..de397b79e58 --- /dev/null +++ b/packages/blacklens/data_stream/alerts/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,15 @@ +version: '2.3' +services: + blacklens: + image: docker.elastic.co/observability/stream:v0.20.0 + hostname: blacklens + ports: + - 8090 + volumes: + - ./files:/files:ro + environment: + PORT: '8090' + command: + - http-server + - --addr=:8090 + - --config=/files/config.yml diff --git a/packages/blacklens/data_stream/alerts/_dev/deploy/docker/files/config.yml b/packages/blacklens/data_stream/alerts/_dev/deploy/docker/files/config.yml new file mode 100644 index 00000000000..b0be9fde761 --- /dev/null +++ b/packages/blacklens/data_stream/alerts/_dev/deploy/docker/files/config.yml @@ -0,0 +1,111 @@ +rules: + - path: /api/v1/test_ws_id/alert + methods: [GET] + query_params: + page: 2 + per_page: 50 + sort_by: created_date + sort: asc + responses: + - status_code: 200 + body: |- + {{ minify_json ` + { + "results": [ + { + "updated_date": "2024-11-12T09:39:58.489Z", + "created_date": "2024-11-12T09:39:58.489Z", + "id": 1003, + "details": { + "id": 100, + "engine": "Port Scanner", + "title": "New Open Port" + }, + "severity": "medium", + "affected_entities": 2, + "alert_outcome": "affected", + "alert_status": "resolved", + "customer_state": "open", + "alert_payload": [], + "type_id": 100 + } + ], + "pagination": { + "next": "page3", + "page": "2" + } + } + `}} + - path: /api/v1/test_ws_id/alert + methods: [GET] + query_params: + page: 3 + per_page: 50 + sort_by: created_date + sort: asc + responses: + - status_code: 200 + body: |- + {{ minify_json ` + { + "results": [ + { + "updated_date": "2024-11-12T09:39:58.489Z", + "created_date": "2024-11-12T09:39:58.489Z", + "id": 1002, + "details": { + "id": 100, + "engine": "Port Scanner", + "title": "New Open Port" + }, + "severity": "medium", + "affected_entities": 2, + "alert_outcome": "affected", + "alert_status": "resolved", + "customer_state": "open", + "alert_payload": [], + "type_id": 100 + } + ], + "pagination": { + "next": "", + "page": "3" + } + } + `}} + - path: /api/v1/test_ws_id/alert + methods: [GET] + query_params: + per_page: 50 + sort_by: created_date + sort: asc + responses: + - status_code: 200 + body: |- + {{ minify_json ` + { + "results": [ + { + "updated_date": "2024-11-12T09:39:58.489Z", + "created_date": "2024-11-12T09:39:58.489Z", + "id": 1001, + "details": { + "id": 100, + "engine": "Port Scanner", + "title": "New Open Port" + }, + "severity": "medium", + "affected_entities": 2, + "alert_outcome": "affected", + "alert_status": "resolved", + "customer_state": "open", + "alert_payload": [], + "type_id": 100 + } + ], + "pagination": { + "next": "page2", + "page": "1" + } + } + `}} diff --git a/packages/blacklens/data_stream/alerts/_dev/test/system/test-httpjson-config.yml b/packages/blacklens/data_stream/alerts/_dev/test/system/test-httpjson-config.yml new file mode 100644 index 00000000000..9edc51f1974 --- /dev/null +++ b/packages/blacklens/data_stream/alerts/_dev/test/system/test-httpjson-config.yml @@ -0,0 +1,10 @@ +input: httpjson +service: blacklens +vars: + url: http://{{Hostname}}:{{Port}} + tenant_id: test_tenant_id + ws_id: test_ws_id + api_key: test_api_key + request.tracer: true +assert: + hit_count: 3 diff --git a/packages/blacklens/data_stream/alerts/sample_event.json b/packages/blacklens/data_stream/alerts/sample_event.json index 02778c80033..19a5426d98c 100644 --- a/packages/blacklens/data_stream/alerts/sample_event.json +++ b/packages/blacklens/data_stream/alerts/sample_event.json @@ -1,61 +1,55 @@ { - "input": { - "type": "httpjson" - }, + "@timestamp": "2024-11-12T09:39:58.489Z", "agent": { - "name": "example-agent", - "id": "example-agent-id-12345", + "ephemeral_id": "33939e93-54ef-4184-b92b-bc8f02e179a6", + "id": "f98f4444-6fca-4500-83b6-a8c5e8f32bf1", + "name": "elastic-agent-49577", "type": "filebeat", - "ephemeral_id": "ephemeral-id-12345", "version": "8.15.2" }, - "@timestamp": "2024-11-07T08:09:22.094Z", - "ecs": { - "version": "8.11.0" + "blacklens": { + "alert": { + "id": 1001, + "outcome": "affected", + "severity": "medium", + "status": "resolved", + "title": "New Open Port", + "type": "Port Scanner", + "type_id": 100, + "updated_date": "2024-11-12T09:39:58.489Z" + } }, "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "blacklens.alerts" + "dataset": "blacklens.alerts", + "namespace": "41265", + "type": "logs" }, - "elastic_agent": { - "id": "example-agent-id-12345", - "version": "8.15.2", - "snapshot": false - }, - "host": { - "name": "example-host" + "ecs": { + "version": "8.11.0" }, - "blacklens": { - "alert": { - "severity": "info", - "type_id": 1001, - "details": [], - "updated_date": "2024-08-14T15:06:13.151Z", - "id": 12345, - "type": "Example Threat System (ETS)", - "title": "Example Threat Scan Notification", - "outcome": "undefined", - "status": "resolved" - } + "elastic_agent": { + "id": "f98f4444-6fca-4500-83b6-a8c5e8f32bf1", + "snapshot": false, + "version": "8.15.2" }, - "message": "{\"affected_entities\":null,\"alert_outcome\":\"undefined\",\"alert_payload\":[],\"reference\":\"https://example.com/reference123\"}],\"alert_status\":\"resolved\",\"created_date\":\"2024-11-07T08:09:22.094028Z\",\"customer_state\":\"open\",\"details\":{\"engine\":\"Example Threat System (ETS)\",\"id\":1001,\"title\":\"Example Threat Scan Notification\"},\"id\":12345,\"severity\":\"info\",\"type_id\":1001,\"updated_date\":\"2024-08-14T15:06:13.151728Z\"}", "event": { - "id": 12345, "agent_id_status": "verified", - "ingested": "2024-11-07T09:45:30Z", - "created": "2024-11-07T09:45:29.354Z", - "kind": "alert", "category": [ "threat" ], + "created": "2025-12-09T05:45:05.855Z", + "dataset": "blacklens.alerts", + "id": "1001", + "ingested": "2025-12-09T05:45:08Z", "type": [ "indicator" - ], - "dataset": "blacklens.alerts" + ] + }, + "input": { + "type": "httpjson" }, "tags": [ "forwarded", - "example-alert" + "blacklens-alert" ] -} \ No newline at end of file +} diff --git a/packages/blacklens/docs/README.md b/packages/blacklens/docs/README.md index 50582ede402..dec0fd03fd7 100644 --- a/packages/blacklens/docs/README.md +++ b/packages/blacklens/docs/README.md @@ -54,64 +54,58 @@ An example event for `alerts` looks as following: ```json { - "input": { - "type": "httpjson" - }, + "@timestamp": "2024-11-12T09:39:58.489Z", "agent": { - "name": "example-agent", - "id": "example-agent-id-12345", + "ephemeral_id": "33939e93-54ef-4184-b92b-bc8f02e179a6", + "id": "f98f4444-6fca-4500-83b6-a8c5e8f32bf1", + "name": "elastic-agent-49577", "type": "filebeat", - "ephemeral_id": "ephemeral-id-12345", "version": "8.15.2" }, - "@timestamp": "2024-11-07T08:09:22.094Z", - "ecs": { - "version": "8.11.0" + "blacklens": { + "alert": { + "id": 1001, + "outcome": "affected", + "severity": "medium", + "status": "resolved", + "title": "New Open Port", + "type": "Port Scanner", + "type_id": 100, + "updated_date": "2024-11-12T09:39:58.489Z" + } }, "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "blacklens.alerts" + "dataset": "blacklens.alerts", + "namespace": "41265", + "type": "logs" }, - "elastic_agent": { - "id": "example-agent-id-12345", - "version": "8.15.2", - "snapshot": false - }, - "host": { - "name": "example-host" + "ecs": { + "version": "8.11.0" }, - "blacklens": { - "alert": { - "severity": "info", - "type_id": 1001, - "details": [], - "updated_date": "2024-08-14T15:06:13.151Z", - "id": 12345, - "type": "Example Threat System (ETS)", - "title": "Example Threat Scan Notification", - "outcome": "undefined", - "status": "resolved" - } + "elastic_agent": { + "id": "f98f4444-6fca-4500-83b6-a8c5e8f32bf1", + "snapshot": false, + "version": "8.15.2" }, - "message": "{\"affected_entities\":null,\"alert_outcome\":\"undefined\",\"alert_payload\":[],\"reference\":\"https://example.com/reference123\"}],\"alert_status\":\"resolved\",\"created_date\":\"2024-11-07T08:09:22.094028Z\",\"customer_state\":\"open\",\"details\":{\"engine\":\"Example Threat System (ETS)\",\"id\":1001,\"title\":\"Example Threat Scan Notification\"},\"id\":12345,\"severity\":\"info\",\"type_id\":1001,\"updated_date\":\"2024-08-14T15:06:13.151728Z\"}", "event": { - "id": 12345, "agent_id_status": "verified", - "ingested": "2024-11-07T09:45:30Z", - "created": "2024-11-07T09:45:29.354Z", - "kind": "alert", "category": [ "threat" ], + "created": "2025-12-09T05:45:05.855Z", + "dataset": "blacklens.alerts", + "id": "1001", + "ingested": "2025-12-09T05:45:08Z", "type": [ "indicator" - ], - "dataset": "blacklens.alerts" + ] + }, + "input": { + "type": "httpjson" }, "tags": [ "forwarded", - "example-alert" + "blacklens-alert" ] } ``` diff --git a/packages/checkpoint_harmony_endpoint/_dev/deploy/docker/files/config.yml b/packages/checkpoint_harmony_endpoint/_dev/deploy/docker/files/config.yml index e6363558472..1f2bc493d90 100644 --- a/packages/checkpoint_harmony_endpoint/_dev/deploy/docker/files/config.yml +++ b/packages/checkpoint_harmony_endpoint/_dev/deploy/docker/files/config.yml @@ -139,3 +139,647 @@ rules: "nextPageToken": "NULL" } } + +# Anti-Malware + - path: /app/laas-logs-api/api/logs_query + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"cloudService\":\"Harmony Endpoint\",\"filter\":\"product:\\\"Anti-Malware\\\"\",\"limit\":[0-9]+,\"pageLimit\":[0-9]+,\"timeframe\":\{\"endTime\":\"[0-9T\:\-Z]+\",\"startTime\":\"[0-9T\:\-Z]+\"\}\}/ + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "taskId": "testtaskid2" + } + } + - path: /app/laas-logs-api/api/logs_query/testtaskid2 + methods: ["GET"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "state": "Processing", + "pageTokens": [] + } + } + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "state": "Ready", + "pageTokens": [ + "testpagetoken1" + ] + } + } + - path: /app/laas-logs-api/api/logs_query/retrieve + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"pageToken\":\"testpagetoken1\",\"taskId\":\"testtaskid2\"\}/ + responses: + - status_code: 200 + body: | + { + "success": true, + "data": { + "records": [ + {"policy_date":"2024-08-29T13:12:46.0000000Z","severity":"Low","time":"2024-09-02T10:37:24Z","id":"a4640108-91b1-0f19-66d5-967600000000","orig":"164.100.1.8","sequencenum":16777215,"product":"Anti-Malware","domain":"SMC User","product_family":"Endpoint","type":"Log","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"event_type":"Scan Stop","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_name":"Default Anti-Malware settings for the entire organization","policy_number":3,"action_details":"Finished","connectivity_state":"Connected","engine_ver":"3.90","sig_ver":"202409011444","integrity_av_invoke_type":"Scheduled","duration":2787,"items_detected":0,"items_scanned":61330,"items_treated":0}, + {"policy_date":"2024-09-03T10:46:10.0000000Z","severity":"Low","time":"2024-09-03T11:27:05.000002Z","id":"a4640108-91b1-0f19-66d6-f30b00000004","orig":"164.100.1.8","sequencenum":2,"product":"Anti-Malware","domain":"SMC User","product_family":"Endpoint","type":"Log","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"event_type":"Scan Start","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_name":"Default Anti-Malware settings for the entire organization","policy_number":4,"action_details":"Started","connectivity_state":"Connected","engine_ver":"3.90","sig_ver":"202409021821","integrity_av_invoke_type":"Unknown"}, + {"policy_date":"2024-09-06T10:21:35.0000000Z","severity":"Low","time":"2024-09-09T12:38:48Z","id":"a4640108-91b1-0f19-66de-ed0700000013","orig":"164.100.1.8","sequencenum":16777215,"product":"Anti-Malware","domain":"SMC User","product_family":"Endpoint","type":"Log","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"event_type":"Update","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_name":"Default Anti-Malware settings for the entire organization","policy_number":7,"action_details":"Successfully","connectivity_state":"Connected","engine_ver":"3.90","failed_updates":0,"result":"Finished","sig_ver":"202409090014"} + ], + "recordsCount": 3, + "nextPageToken": "testpagetoken2" + } + } + - path: /app/laas-logs-api/api/logs_query/retrieve + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"pageToken\":\"testpagetoken2\",\"taskId\":\"testtaskid2\"\}/ + responses: + - status_code: 200 + body: | + { + "success": true, + "data": { + "records": [ + {"confidence_level":"High","policy_date":"2024-08-29T13:12:46.0000000Z","severity":"High","time":"2024-09-02T09:09:07Z","id":"a4640108-91b1-0f19-66d5-815d0000000f","orig":"164.100.1.8","sequencenum":16777215,"product":"Anti-Malware","domain":"SMC User","product_family":"Endpoint","type":"Log","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"event_type":"Infection","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_name":"Default Anti-Malware settings for the entire organization","policy_number":3,"action_details":"Infected","connectivity_state":"Connected","engine_ver":"3.90","sig_ver":"202409011444","action":"Detect","protection_type":"Protection","file_name":["9e68140d-22bb-4e96-8aaa-70ec80eb2dc4.tmp"],"packet_capture_unique_id":"31dc576b-7192-49bf-b2fc-b40c93f84b7c","protection_name":"Mal/ShellDl-A","packet_capture":"Packet Capture","advanced_info":"\"exclusions\":[{\"exclusion_engine_type\":\"File & Folder exclusions (system, scheduled and on-demand)\",\"exclusion_type\":\"Path\",\"exclusion_value\":{\"default_value\":\"md5:\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"md5 taken from file C:\\\\Users\\\\admin\\\\AppData\\\\Local\\\\Temp\\\\9e68140d-22bb-4e96-8aaa-70ec80eb2dc4.tmp\"}}]","infection_category":"Malware"}, + {"confidence_level":"High","policy_date":"2024-09-06T10:21:35.0000000Z","severity":"High","time":"2024-09-09T12:38:42Z","id":"a4640108-91b1-0f19-66de-ed0700000012","orig":"164.100.1.8","sequencenum":16777215,"product":"Anti-Malware","domain":"SMC User","product_family":"Endpoint","type":"Log","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"event_type":"Infection","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_name":"Default Anti-Malware settings for the entire organization","policy_number":7,"action_details":"Infected","connectivity_state":"Connected","engine_ver":"3.90","sig_ver":"202409090014","action":"Detect","protection_type":"Protection","file_md5":"44d88612fea8a8f36de82e1278abb02f","file_name":["eicar.txt"],"packet_capture_unique_id":"bb6c808c-b96e-4f12-9749-1dfae7f6f8c2","protection_name":"EICAR-AV-Test","packet_capture":"Packet Capture","advanced_info":"\"exclusions\":[{\"exclusion_engine_type\":\"File & Folder exclusions (system, scheduled and on-demand)\",\"exclusion_type\":\"Path\",\"exclusion_value\":{\"default_value\":\"md5:44d88612fea8a8f36de82e1278abb02f\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"md5 taken from file C:\\\\Users\\\\admin\\\\Downloads\\\\eicar.txt\"}}]","infection_category":"Virus"} + ], + "recordsCount": 2, + "nextPageToken": "NULL" + } + } + +# Forensics + - path: /app/laas-logs-api/api/logs_query + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"cloudService\":\"Harmony Endpoint\",\"filter\":\"product:\\\"Forensics\\\"\",\"limit\":[0-9]+,\"pageLimit\":[0-9]+,\"timeframe\":\{\"endTime\":\"[0-9T\:\-Z]+\",\"startTime\":\"[0-9T\:\-Z]+\"\}\}/ + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "taskId": "testtaskid3" + } + } + - path: /app/laas-logs-api/api/logs_query/testtaskid3 + methods: ["GET"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "state": "Processing", + "pageTokens": [] + } + } + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "state": "Ready", + "pageTokens": [ + "testpagetoken1" + ] + } + } + - path: /app/laas-logs-api/api/logs_query/retrieve + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"pageToken\":\"testpagetoken1\",\"taskId\":\"testtaskid3\"\}/ + responses: + - status_code: 200 + body: | + { + "success": true, + "data": { + "records": [ + {"confidence_level":"High","policy_date":"2024-09-02T06:23:25.0000000Z","severity":"Critical","time":"2024-09-03T08:53:12.000001Z","id":"a4640108-91b1-0f19-66d6-ceb500000000","orig":"164.100.1.8","sequencenum":1,"action":"Detect","product":"Forensics","domain":"SMC User","product_family":"Endpoint","type":"Log","description":"To exclude the file: On the Harmony Endpoint Management add this sha1 exclusion: 62f0bd56-b0e1235b-99940b34-916c19ec-fac8e80c Attack status: Dormant.","protection_type":"File Reputation","attack_status":"Dormant","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"detected_by":"Endpoint File Reputation","event_type":"Forensics Case Analysis","file_md5":"1468c1908845ef238f7f196809946288","file_name":["malz5.zip"],"file_sha1":"62f0bd56b0e1235b99940b34916c19ecfac8e80c","file_size":12707198,"file_type":"zip","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","malware_action":[" "],"os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"packet_capture_unique_id":"0acd55a9-f241-4097-a699-6b7e41cd26af","policy_name":"Default Forensics settings","policy_number":3,"protection_name":"Gen.Rep.zip","remediated_files":"malz5.zip(Remediation disabled in policy)","resource":["c:\\users\\admin\\downloads\\malz5.zip"],"service_domain":"ep-demo","src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"suspicious_events":"System Shutdown / Reboot: ; ","tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","packet_capture":"Packet Capture"}, + {"confidence_level":"Medium","policy_date":"2024-09-02T06:23:25.0000000Z","severity":"Critical","time":"2024-09-02T08:58:05.000001Z","id":"a4640108-91b1-0f19-66d5-7e5100000001","orig":"164.100.1.8","sequencenum":1,"action":"Detect","product":"Forensics","domain":"SMC User","product_family":"Endpoint","type":"Log","description":"Endpoint Anti-Bot has detected :http://www.threat-cloud.com/test/files/MediumConfidenceBot.html. Attack status: Dormant.","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"event_type":"Forensics Case Analysis","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_name":"Default Forensics settings","policy_number":3,"protection_type":"URL Reputation","attack_status":"Dormant","detected_by":"Endpoint Anti-Bot","file_name":["chrome.exe"],"file_size":0,"file_type":"exe","malware_action":["Communication with C&C"],"packet_capture_unique_id":"ef93cc38-215a-4fba-b571-d71ccdace123","protection_name":"Anti-Bot test.TC.e","resource":["http://www.threat-cloud.com/test/files/MediumConfidenceBot.html"],"service_domain":"ep-demo","packet_capture":"Packet Capture"} + ], + "recordsCount": 2, + "nextPageToken": "testpagetoken2" + } + } + - path: /app/laas-logs-api/api/logs_query/retrieve + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"pageToken\":\"testpagetoken2\",\"taskId\":\"testtaskid3\"\}/ + responses: + - status_code: 200 + body: | + { + "success": true, + "data": { + "records": [ + {"confidence_level":"High","policy_date":"2024-09-02T06:23:25.0000000Z","severity":"High","time":"2024-09-03T05:36:43.000001Z","id":"a4640108-91b1-0f19-66d6-a08700000000","orig":"164.100.1.8","sequencenum":1,"action":"Detect","product":"Forensics","domain":"SMC User","product_family":"Endpoint","type":"Log","description":"To exclude the file: On the Harmony Endpoint Management add this sha1 exclusion: d2726507-4c9eac2e-2122ed69-294dbc4d-7cce9141 Attack status: Dormant.","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"event_type":"Forensics Case Analysis","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_name":"Default Forensics settings","policy_number":3,"protection_type":"HTTP Emulation","attack_status":"Dormant","detected_by":"Harmony Agent for Browsers","file_name":["chrome.exe"],"file_sha1":"d27265074c9eac2e2122ed69294dbc4d7cce9141","file_size":0,"file_type":"exe","malware_action":[" "],"packet_capture_unique_id":"d33f791c-004c-412d-b2dc-5dc1ddf5cd2b","protection_name":"gen.ba.sb.zip","resource":["https://www.ikarussecurity.com/wp-content/downloads/eicar_com.zip"],"service_domain":"ep-demo","packet_capture":"Packet Capture"} + ], + "recordsCount": 1, + "nextPageToken": "NULL" + } + } + +# Threat Emulation + - path: /app/laas-logs-api/api/logs_query + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"cloudService\":\"Harmony Endpoint\",\"filter\":\"product:\\\"Threat Emulation\\\"\",\"limit\":[0-9]+,\"pageLimit\":[0-9]+,\"timeframe\":\{\"endTime\":\"[0-9T\:\-Z]+\",\"startTime\":\"[0-9T\:\-Z]+\"\}\}/ + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "taskId": "testtaskid4" + } + } + - path: /app/laas-logs-api/api/logs_query/testtaskid4 + methods: ["GET"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "state": "Processing", + "pageTokens": [] + } + } + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "state": "Ready", + "pageTokens": [ + "testpagetoken1" + ] + } + } + - path: /app/laas-logs-api/api/logs_query/retrieve + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"pageToken\":\"testpagetoken1\",\"taskId\":\"testtaskid4\"\}/ + responses: + - status_code: 200 + body: | + { + "success": true, + "data": { + "records": [ + {"policy_date":"2024-09-06T10:21:38.0000000Z","severity":"Informational","time":"2024-09-09T12:46:36Z","id":"a4640108-91b1-0f19-66df-dc9a00000004","orig":"164.100.1.8","sequencenum":16777215,"client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"description":"Threat Emulation is enabled due to valid license. ","event_type":"TE Info Event","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"policy_name":"Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization","policy_number":7,"product":"Threat Emulation","product_family":"Endpoint","reason":"Valid_TE_License","src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","domain":"SMC User","type":"Control"}, + {"confidence_level":"High","policy_date":"2024-08-29T13:12:50.0000000Z","severity":"High","time":"2024-09-03T08:52:45Z","id":"a4640108-91b1-0f19-66d6-cf6900000000","orig":"164.100.1.8","sequencenum":16777215,"client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"description":"Endpoint FR detected malicious file (malz5.zip) . To exclude the file: On the Harmony Endpoint Management add this sha1 exclusion: 62f0bd56-b0e1235b-99940b34-916c19ec-fac8e80c","event_type":"TE Event","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"policy_name":"Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization","policy_number":3,"product":"Threat Emulation","product_family":"Endpoint","src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","domain":"SMC User","type":"Log","action":"Detect","protection_type":"File Reputation","file_md5":"1468c1908845ef238f7f196809946288","file_name":["malz5.zip"],"file_sha1":"62f0bd56b0e1235b99940b34916c19ecfac8e80c","file_size":12707198,"file_type":"zip","malware_action":[" "],"packet_capture_unique_id":"ee12249f-b2bd-48aa-80a4-9f92745caba1","protection_name":"Gen.Rep.zip","resource":["C:\\Users\\admin\\Downloads\\malz5.zip"],"packet_capture":"Packet Capture","advanced_info":"\"exclusions\":[{\"exclusion_engine_type\":\"Threat Emulation, Extraction and Zero Phishing Exclusions\",\"exclusion_type\":\"SHA1\",\"exclusion_value\":{\"default_value\":\"62f0bd56b0e1235b99940b34916c19ecfac8e80c\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]","analyzed_on":"Harmony Local Cache","incident_uid":"0acd55a9-f241-4097-a699-6b7e41cd26af","verdict":"Malicious","web_client_type":[" "]}, + {"confidence_level":"High","policy_date":"2024-08-29T13:12:50.0000000Z","severity":"Critical","time":"2024-09-02T09:04:54Z","id":"a4640108-91b1-0f19-66d5-803100000012","orig":"164.100.1.8","sequencenum":16777215,"client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"description":"Endpoint TE detected malicious file (681573a2-414a-4f7d-9683-177df4f8ca7f.tmp) . To exclude the file: On the Harmony Endpoint Management add this sha1 exclusion: 9d3395d9-4c6bbba5-2abf0e6a-fcbf4ca3-12597c21","event_type":"TE Event","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"policy_name":"Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization","policy_number":3,"product":"Threat Emulation","product_family":"Endpoint","src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","domain":"SMC User","type":"Log","action":"Detect","protection_type":"File System Emulation","file_md5":"ebe8b633d231bbfee9543d744a2ab59d","file_name":["681573a2-414a-4f7d-9683-177df4f8ca7f.tmp"],"file_sha1":"9d3395d94c6bbba52abf0e6afcbf4ca312597c21","file_size":139648,"file_type":"zip","malware_action":["Adware\",\"Solimba\",\"Trojan\",\"behavior"],"packet_capture_unique_id":"5e3302e5-3f73-4b77-beec-2849003e9d47","protection_name":"Gen.SB.zip","resource":["C:\\Users\\admin\\Downloads\\681573a2-414a-4f7d-9683-177df4f8ca7f.tmp"],"packet_capture":"Packet Capture","advanced_info":"\"exclusions\":[{\"exclusion_engine_type\":\"Threat Emulation, Extraction and Zero Phishing Exclusions\",\"exclusion_type\":\"SHA1\",\"exclusion_value\":{\"default_value\":\"9d3395d94c6bbba52abf0e6afcbf4ca312597c21\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]","analyzed_on":"Check Point Threat Emulation Cloud","incident_uid":"74a33ecb-1b91-4c25-a136-1989eb175638","verdict":"Malicious","web_client_type":[" "]} + ], + "recordsCount": 3, + "nextPageToken": "testpagetoken2" + } + } + - path: /app/laas-logs-api/api/logs_query/retrieve + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"pageToken\":\"testpagetoken2\",\"taskId\":\"testtaskid4\"\}/ + responses: + - status_code: 200 + body: | + { + "success": true, + "data": { + "records": [ + {"confidence_level":"High","policy_date":"2024-09-03T10:46:13.0000000Z","severity":"Low","time":"2024-09-06T09:06:55Z","id":"a4640108-91b1-0f19-66da-c6a900000004","orig":"164.100.1.8","sequencenum":16777215,"client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"description":"Download of malicious file eicar_com.zip was prevented . To exclude the file: On the Harmony Endpoint Management add this sha1 exclusion: d2726507-4c9eac2e-2122ed69-294dbc4d-7cce9141","event_type":"TE Event","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"policy_name":"Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization","policy_number":4,"product":"Threat Emulation","product_family":"Endpoint","src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","domain":"SMC User","type":"Log","action":"Prevent","protection_type":"HTTP Emulation","file_name":["eicar_com.zip"],"file_sha1":"d27265074c9eac2e2122ed69294dbc4d7cce9141","file_size":184,"file_type":"zip","malware_action":[" "],"packet_capture_unique_id":"9672a696-fdcd-43be-a8c0-7b50b5efc6c4","protection_name":"gen.ba.sb.zip","resource":["https://secure.eicar.org/eicar_com.zip"],"packet_capture":"Packet Capture","advanced_info":"\"exclusions\":[{\"exclusion_engine_type\":\"Threat Emulation, Extraction and Zero Phishing Exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"secure.eicar.org\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]","analyzed_on":"Harmony Local Cache","incident_uid":"c40d72fb-533b-423a-9749-d239a9c4ea3f","verdict":"Malicious","web_client_type":["Chrome"]} + ], + "recordsCount": 1, + "nextPageToken": "NULL" + } + } + +# Threat Extraction + - path: /app/laas-logs-api/api/logs_query + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"cloudService\":\"Harmony Endpoint\",\"filter\":\"product:\\\"Threat Extraction\\\"\",\"limit\":[0-9]+,\"pageLimit\":[0-9]+,\"timeframe\":\{\"endTime\":\"[0-9T\:\-Z]+\",\"startTime\":\"[0-9T\:\-Z]+\"\}\}/ + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "taskId": "testtaskid5" + } + } + - path: /app/laas-logs-api/api/logs_query/testtaskid5 + methods: ["GET"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "state": "Processing", + "pageTokens": [] + } + } + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "state": "Ready", + "pageTokens": [ + "testpagetoken1" + ] + } + } + - path: /app/laas-logs-api/api/logs_query/retrieve + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"pageToken\":\"testpagetoken1\",\"taskId\":\"testtaskid5\"\}/ + responses: + - status_code: 200 + body: | + { + "success": true, + "data": { + "records": [ + {"confidence_level":"High","policy_date":"2024-09-03T10:46:13.0000000Z","severity":"Informational","time":"2024-09-04T09:08:30Z","id":"a4640108-91b1-0f19-66d8-247c00000100","orig":"164.100.1.8","sequencenum":16777215,"action":"Extract","product":"Threat Extraction","domain":"SMC User","product_family":"Endpoint","type":"Log","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"description":"File is not supported for extraction","event_type":"TEX Event","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_name":"Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization","policy_number":4,"protection_type":"Content Removal","advanced_info":" \"disable_exclusion\": true ","extension_version":" 990.99.486 ","file_name":["ransomware.go"],"file_sha1":"no-sha1","file_size":0,"file_type":"go","malware_action":["Not Supported"],"protection_name":"Extract potentially malicious content","resource":["blob:https://github.com/9d747e42-c19c-43d8-b77d-7258ab12a233"],"web_client_type":["Chrome"]}, + {"confidence_level":"High","policy_date":"2024-09-03T10:46:13.0000000Z","severity":"Informational","time":"2024-09-04T09:08:30Z","id":"a4640108-91b1-0f19-66d8-247c00000000","orig":"164.100.1.8","sequencenum":16777215,"action":"Extract","product":"Threat Extraction","domain":"SMC User","product_family":"Endpoint","type":"Log","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"description":"File is not supported for extraction","event_type":"TEX Event","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_name":"Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization","policy_number":4,"protection_type":"Content Removal","advanced_info":" \"disable_exclusion\": true ","extension_version":" 990.99.486 ","file_name":["ransomware.go"],"file_sha1":"no-sha1","file_size":0,"file_type":"go","malware_action":["Not Supported"],"protection_name":"Extract potentially malicious content","resource":["blob:https://github.com/9d747e42-c19c-43d8-b77d-7258ab12a233"],"web_client_type":["Chrome"]} + ], + "recordsCount": 2, + "nextPageToken": "testpagetoken2" + } + } + - path: /app/laas-logs-api/api/logs_query/retrieve + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"pageToken\":\"testpagetoken2\",\"taskId\":\"testtaskid5\"\}/ + responses: + - status_code: 200 + body: | + { + "success": true, + "data": { + "records": [ + {"confidence_level":"High","policy_date":"2024-09-03T10:46:13.0000000Z","severity":"Informational","time":"2024-09-04T09:08:30Z","id":"a4640108-91b1-0f19-66d8-247c00000001","orig":"164.100.1.8","sequencenum":16777215,"action":"Extract","product":"Threat Extraction","domain":"SMC User","product_family":"Endpoint","type":"Log","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"description":"File is not supported for extraction","event_type":"TEX Event","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_name":"Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization","policy_number":4,"protection_type":"Content Removal","advanced_info":" \"disable_exclusion\": true ","extension_version":" 990.99.486 ","file_name":["ransomware.go"],"file_sha1":"no-sha1","file_size":0,"file_type":"go","malware_action":["Not Supported"],"protection_name":"Extract potentially malicious content","resource":["blob:https://github.com/9d747e42-c19c-43d8-b77d-7258ab12a233"],"web_client_type":["Chrome"]} + ], + "recordsCount": 1, + "nextPageToken": "NULL" + } + } + +# Threat Extraction + - path: /app/laas-logs-api/api/logs_query + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"cloudService\":\"Harmony Endpoint\",\"filter\":\"product:\\\"URL Filtering\\\"\",\"limit\":[0-9]+,\"pageLimit\":[0-9]+,\"timeframe\":\{\"endTime\":\"[0-9T\:\-Z]+\",\"startTime\":\"[0-9T\:\-Z]+\"\}\}/ + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "taskId": "testtaskid6" + } + } + - path: /app/laas-logs-api/api/logs_query/testtaskid6 + methods: ["GET"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "state": "Processing", + "pageTokens": [] + } + } + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "state": "Ready", + "pageTokens": [ + "testpagetoken1" + ] + } + } + - path: /app/laas-logs-api/api/logs_query/retrieve + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"pageToken\":\"testpagetoken1\",\"taskId\":\"testtaskid6\"\}/ + responses: + - status_code: 200 + body: | + { + "success": true, + "data": { + "records": [ + {"policy_date":"2024-09-06T10:08:44.0000000Z","severity":"Informational","time":"2024-09-06T10:19:56.000001Z","id":"a4640108-91b1-0f19-66da-d78900000009","orig":"164.100.1.8","sequencenum":1,"action":"Prevent","product":"URL Filtering","domain":"SMC User","product_family":"Endpoint","type":"Log","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"description":"To exclude: Open the Harmony Management -> POLICY -> Threat Prevention -> EXCLUSION CENTER -> Web and Files Protection -> URL Filtering exclusions -> + -> paste this: livingliquidz.com","event_type":"URLF Info Event","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"policy_name":"Default Anti-Bot settings","src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_number":5,"protection_type":"URL Filtering","protection_name":"gen.urlf","resource":["https://livingliquidz.com/"],"advanced_info":"\"exclusions\":[{\"exclusion_engine_type\":\"URL Filtering exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"livingliquidz.com\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]","dst":"0.0.0.0","web_client_type":["Chrome"],"app_id":"0","app_properties":["Alcohol & Tobacco"],"appi_name":"livingliquidz.com","matched_category":"Alcohol & Tobacco","process_exe_path":"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe","usercheck_incident_uid":"7fcc5b44"}, + {"policy_date":"2024-09-06T09:57:28.0000000Z","severity":"Informational","time":"2024-09-06T10:07:43Z","id":"a4640108-91b1-0f19-66da-d62100000013","orig":"164.100.1.8","sequencenum":16777215,"action":"Detect","product":"URL Filtering","domain":"SMC User","product_family":"Endpoint","type":"Log","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"description":"URLF Info Event","event_type":"URLF Info Event","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"policy_name":"Default Anti-Bot settings","src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_number":4,"protection_type":"URL Filtering","protection_name":"gen.urlf","resource":["https://secure.indeed.com/auth?branding=save-profile-modal&tmpl=inline&from=act_zeroauth_profile_tst&iframe_tk=9a019527-a6f1-4b3d-b803-2b25bb46b1db&hl=en_IN&co=IN"],"advanced_info":"\"exclusions\":[{\"exclusion_engine_type\":\"URL Filtering exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"secure.indeed.com\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]","dst":"0.0.0.0","web_client_type":["Chrome"],"app_id":"0","app_properties":["Job Search / Careers, Business / Economy"],"appi_name":"secure.indeed.com","matched_category":"Job Search / Careers","process_exe_path":"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe","usercheck_incident_uid":"b04d8940"} + ], + "recordsCount": 2, + "nextPageToken": "testpagetoken2" + } + } + - path: /app/laas-logs-api/api/logs_query/retrieve + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"pageToken\":\"testpagetoken2\",\"taskId\":\"testtaskid6\"\}/ + responses: + - status_code: 200 + body: | + { + "success": true, + "data": { + "records": [ + {"policy_date":"2024-09-06T10:08:44.0000000Z","severity":"Informational","time":"2024-09-06T10:19:56.000001Z","id":"a4640108-91b1-0f19-66da-d78900000019","orig":"164.100.1.8","sequencenum":1,"action":"Prevent","product":"URL Filtering","domain":"SMC User","product_family":"Endpoint","type":"Log","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"description":"To exclude: Open the Harmony Management -> POLICY -> Threat Prevention -> EXCLUSION CENTER -> Web and Files Protection -> URL Filtering exclusions -> + -> paste this: livingliquidz.com","event_type":"URLF Info Event","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"policy_name":"Default Anti-Bot settings","src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_number":5,"protection_type":"URL Filtering","protection_name":"gen.urlf","resource":["https://livingliquidz.com/"],"advanced_info":"\"exclusions\":[{\"exclusion_engine_type\":\"URL Filtering exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"livingliquidz.com\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]","dst":"0.0.0.0","web_client_type":["Chrome"],"app_id":"0","app_properties":["Alcohol & Tobacco"],"appi_name":"livingliquidz.com","matched_category":"Alcohol & Tobacco","process_exe_path":"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe","usercheck_incident_uid":"7fcc5b44"} + ], + "recordsCount": 1, + "nextPageToken": "NULL" + } + } + +# Zero Phishing + - path: /app/laas-logs-api/api/logs_query + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"cloudService\":\"Harmony Endpoint\",\"filter\":\"product:\\\"Zero Phishing\\\"\",\"limit\":[0-9]+,\"pageLimit\":[0-9]+,\"timeframe\":\{\"endTime\":\"[0-9T\:\-Z]+\",\"startTime\":\"[0-9T\:\-Z]+\"\}\}/ + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "taskId": "testtaskid7" + } + } + - path: /app/laas-logs-api/api/logs_query/testtaskid7 + methods: ["GET"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "state": "Processing", + "pageTokens": [] + } + } + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: | + { + "success": true, + "data": { + "state": "Ready", + "pageTokens": [ + "testpagetoken1" + ] + } + } + - path: /app/laas-logs-api/api/logs_query/retrieve + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"pageToken\":\"testpagetoken1\",\"taskId\":\"testtaskid7\"\}/ + responses: + - status_code: 200 + body: | + { + "success": true, + "data": { + "records": [ + {"confidence_level":"High","policy_date":"2024-08-29T13:12:50.0000000Z","severity":"High","time":"2024-09-02T08:51:08Z","id":"a4640108-91b1-0f19-66d5-7d6100000114","orig":"164.100.1.8","sequencenum":16777215,"action":"Detect","product":"Zero Phishing","domain":"SMC User","product_family":"Endpoint","type":"Log","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"description":"Deceptive site (https://main.sbm-demo.xyz/zero-phishing) was detected.","event_type":"Phishing Event","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_name":"Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization","policy_number":3,"protection_type":"Phishing","malware_action":[" "],"protection_name":"gen.ba.phishing","resource":["https://main.sbm-demo.xyz/zero-phishing"],"advanced_info":"\"exclusions\":[{\"exclusion_engine_type\":\"Threat Emulation, Extraction and Zero Phishing Exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"main.sbm-demo.xyz\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]","extension_version":"Check Point Endpoint Security Client","web_client_type":["Chrome"]}, + {"confidence_level":"High","policy_date":"2024-08-29T13:12:50.0000000Z","severity":"High","time":"2024-09-02T08:51:08Z","id":"a4640108-91b1-0f19-66d5-7d6100000224","orig":"164.100.1.8","sequencenum":16777215,"action":"Detect","product":"Zero Phishing","domain":"SMC User","product_family":"Endpoint","type":"Log","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"description":"Deceptive site (https://main.sbm-demo.xyz/zero-phishing) was detected.","event_type":"Phishing Event","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_name":"Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization","policy_number":3,"protection_type":"Phishing","malware_action":[" "],"protection_name":"gen.ba.phishing","resource":["https://main.sbm-demo.xyz/zero-phishing"],"advanced_info":"\"exclusions\":[{\"exclusion_engine_type\":\"Threat Emulation, Extraction and Zero Phishing Exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"main.sbm-demo.xyz\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]","extension_version":"Check Point Endpoint Security Client","web_client_type":["Chrome"]} + ], + "recordsCount": 2, + "nextPageToken": "testpagetoken2" + } + } + - path: /app/laas-logs-api/api/logs_query/retrieve + methods: ["POST"] + request_headers: + Accept: + - "application/json" + Content-Type: + - "application/json" + Authorization: + - "Bearer testauthtoken" + request_body: /^\{\"pageToken\":\"testpagetoken2\",\"taskId\":\"testtaskid7\"\}/ + responses: + - status_code: 200 + body: | + { + "success": true, + "data": { + "records": [ + {"confidence_level":"High","policy_date":"2024-08-29T13:12:50.0000000Z","severity":"High","time":"2024-09-02T08:51:08Z","id":"a4640108-91b1-0f19-66d5-7d6100000223","orig":"164.100.1.8","sequencenum":16777215,"action":"Detect","product":"Zero Phishing","domain":"SMC User","product_family":"Endpoint","type":"Log","client_name":"Check Point Endpoint Security Client","client_version":["88.50.0213"],"description":"Deceptive site (https://main.sbm-demo.xyz/zero-phishing) was detected.","event_type":"Phishing Event","host_type":["Desktop"],"installed_products":"Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation","os_name":["Microsoft Windows 10 Pro"],"os_version":["10.0-19045-SP0.0-SMP"],"src":"10.35.38.102","src_machine_name":"DESKTOP-E2P4OL0","src_user_name":["admin"],"tenant_id":"3e15ed24-89ff-4986-a204-c425cee4ba48","user_sid":"S-1-5-21-3766288932-3295778425-2939962592-1001","policy_name":"Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization","policy_number":3,"protection_type":"Phishing","malware_action":[" "],"protection_name":"gen.ba.phishing","resource":["https://main.sbm-demo.xyz/zero-phishing"],"advanced_info":"\"exclusions\":[{\"exclusion_engine_type\":\"Threat Emulation, Extraction and Zero Phishing Exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"main.sbm-demo.xyz\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]","extension_version":"Check Point Endpoint Security Client","web_client_type":["Chrome"]} ], + "recordsCount": 1, + "nextPageToken": "NULL" + } + } diff --git a/packages/checkpoint_harmony_endpoint/data_stream/antibot/sample_event.json b/packages/checkpoint_harmony_endpoint/data_stream/antibot/sample_event.json index 01ac868337e..035eaae6754 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/antibot/sample_event.json +++ b/packages/checkpoint_harmony_endpoint/data_stream/antibot/sample_event.json @@ -1,9 +1,9 @@ { "@timestamp": "2024-09-02T08:53:44.000Z", "agent": { - "ephemeral_id": "f96c672e-e52f-49f4-a098-fb8e47463002", - "id": "a17315c6-08c5-498c-a743-f2125ce9bf1b", - "name": "elastic-agent-77268", + "ephemeral_id": "395d0bda-166c-4800-9c38-ec1455f30548", + "id": "9e02b661-1ee6-4686-9926-edc63a7f4b02", + "name": "elastic-agent-27616", "type": "filebeat", "version": "8.14.0" }, @@ -42,7 +42,7 @@ }, "data_stream": { "dataset": "checkpoint_harmony_endpoint.antibot", - "namespace": "67908", + "namespace": "18419", "type": "logs" }, "destination": { @@ -55,7 +55,7 @@ "version": "8.17.0" }, "elastic_agent": { - "id": "a17315c6-08c5-498c-a743-f2125ce9bf1b", + "id": "9e02b661-1ee6-4686-9926-edc63a7f4b02", "snapshot": false, "version": "8.14.0" }, @@ -67,7 +67,7 @@ ], "dataset": "checkpoint_harmony_endpoint.antibot", "id": "a4640108-91b1-0f19-66d5-7d9d00000003", - "ingested": "2025-06-05T09:35:27Z", + "ingested": "2025-12-16T11:56:39Z", "kind": "event", "module": "checkpoint_harmony_endpoint", "type": [ diff --git a/packages/checkpoint_harmony_endpoint/data_stream/antimalware/_dev/test/system/test-default-config.yml b/packages/checkpoint_harmony_endpoint/data_stream/antimalware/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..b204fdcd4f6 --- /dev/null +++ b/packages/checkpoint_harmony_endpoint/data_stream/antimalware/_dev/test/system/test-default-config.yml @@ -0,0 +1,12 @@ +input: cel +service: harmony +vars: + base_url: http://{{Hostname}}:{{Port}} + client_id: testclientid + access_key: testaccesskey + initial_interval: 720h + interval: 5m + limit: 3 + page_limit: 10 +assert: + hit_count: 5 diff --git a/packages/checkpoint_harmony_endpoint/data_stream/antimalware/sample_event.json b/packages/checkpoint_harmony_endpoint/data_stream/antimalware/sample_event.json index 40b362af03d..36a740fe909 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/antimalware/sample_event.json +++ b/packages/checkpoint_harmony_endpoint/data_stream/antimalware/sample_event.json @@ -1,11 +1,11 @@ { "@timestamp": "2024-09-02T09:09:07.000Z", "agent": { - "ephemeral_id": "972620ca-77a9-4305-991a-5bd475860580", - "id": "4cdb965a-db2a-4ec3-9abf-6e20dbb120c9", - "name": "elastic-agent-21918", + "ephemeral_id": "ce69a1cc-2624-4c1b-abea-4c65c408b88c", + "id": "59648861-728d-4e8a-9109-5d16c682af18", + "name": "elastic-agent-75194", "type": "filebeat", - "version": "8.15.1" + "version": "8.14.0" }, "checkpoint_harmony_endpoint": { "antimalware": { @@ -45,16 +45,16 @@ }, "data_stream": { "dataset": "checkpoint_harmony_endpoint.antimalware", - "namespace": "85578", + "namespace": "52446", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "8.17.0" }, "elastic_agent": { - "id": "4cdb965a-db2a-4ec3-9abf-6e20dbb120c9", + "id": "59648861-728d-4e8a-9109-5d16c682af18", "snapshot": false, - "version": "8.15.1" + "version": "8.14.0" }, "event": { "action": "Detect", @@ -64,7 +64,7 @@ ], "dataset": "checkpoint_harmony_endpoint.antimalware", "id": "a4640108-91b1-0f19-66d5-815d0000000f", - "ingested": "2024-10-24T05:32:25Z", + "ingested": "2025-12-16T11:57:31Z", "kind": "alert", "module": "checkpoint_harmony_endpoint", "type": [ @@ -120,4 +120,4 @@ "admin" ] } -} \ No newline at end of file +} diff --git a/packages/checkpoint_harmony_endpoint/data_stream/forensics/_dev/test/system/test-default-config.yml b/packages/checkpoint_harmony_endpoint/data_stream/forensics/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..8192ded1be8 --- /dev/null +++ b/packages/checkpoint_harmony_endpoint/data_stream/forensics/_dev/test/system/test-default-config.yml @@ -0,0 +1,12 @@ +input: cel +service: harmony +vars: + base_url: http://{{Hostname}}:{{Port}} + client_id: testclientid + access_key: testaccesskey + initial_interval: 720h + interval: 5m + limit: 2 + page_limit: 10 +assert: + hit_count: 3 diff --git a/packages/checkpoint_harmony_endpoint/data_stream/forensics/sample_event.json b/packages/checkpoint_harmony_endpoint/data_stream/forensics/sample_event.json index 8214d372337..b103adae087 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/forensics/sample_event.json +++ b/packages/checkpoint_harmony_endpoint/data_stream/forensics/sample_event.json @@ -1,11 +1,11 @@ { - "@timestamp": "2024-09-03T08:53:12.000Z", + "@timestamp": "2024-09-02T08:58:05.000Z", "agent": { - "ephemeral_id": "76820ab1-9086-4fc7-975c-2e7cda1f601c", - "id": "3df1f948-9917-4dc4-a724-f2b5934a6652", - "name": "elastic-agent-71957", + "ephemeral_id": "4d32ca32-d2fd-4e13-84dc-5c69ba84bd1e", + "id": "10e7cd33-c414-404f-800a-5878962095d8", + "name": "elastic-agent-75832", "type": "filebeat", - "version": "8.15.1" + "version": "8.14.0" }, "checkpoint_harmony_endpoint": { "forensics": { @@ -14,14 +14,16 @@ "name": "Check Point Endpoint Security Client", "version": "88.50.0213" }, - "confidence_level": "High", - "description": "To exclude the file: On the Harmony Endpoint Management add this sha1 exclusion: 62f0bd56-b0e1235b-99940b34-916c19ec-fac8e80c Attack status: Dormant.", - "detected_by": "Endpoint File Reputation", + "confidence_level": "Medium", + "description": "Endpoint Anti-Bot has detected :http://www.threat-cloud.com/test/files/MediumConfidenceBot.html. Attack status: Dormant.", + "detected_by": "Endpoint Anti-Bot", "event_type": "Forensics Case Analysis", "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", - "malware": {}, + "malware": { + "action": "Communication with C&C" + }, "packet_capture": "Packet Capture", - "packet_capture_unique_id": "0acd55a9-f241-4097-a699-6b7e41cd26af", + "packet_capture_unique_id": "ef93cc38-215a-4fba-b571-d71ccdace123", "policy": { "date": "2024-09-02T06:23:25.0000000Z", "name": "Default Forensics settings", @@ -31,29 +33,27 @@ "family": "Endpoint", "name": "Forensics" }, - "protection_type": "File Reputation", - "remediated_files": "malz5.zip(Remediation disabled in policy)", + "protection_type": "URL Reputation", "sequencenum": 1, "service_domain": "ep-demo", "severity": "Critical", "src": "10.35.38.102", - "suspicious_events": "System Shutdown / Reboot: ; ", "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "type": "Log" } }, "data_stream": { "dataset": "checkpoint_harmony_endpoint.forensics", - "namespace": "38429", + "namespace": "79277", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "8.17.0" }, "elastic_agent": { - "id": "3df1f948-9917-4dc4-a724-f2b5934a6652", + "id": "10e7cd33-c414-404f-800a-5878962095d8", "snapshot": false, - "version": "8.15.1" + "version": "8.14.0" }, "event": { "action": "Detect", @@ -62,8 +62,8 @@ "malware" ], "dataset": "checkpoint_harmony_endpoint.forensics", - "id": "a4640108-91b1-0f19-66d6-ceb500000000", - "ingested": "2024-10-24T05:33:21Z", + "id": "a4640108-91b1-0f19-66d5-7e5100000001", + "ingested": "2025-12-16T11:59:48Z", "kind": "alert", "module": "checkpoint_harmony_endpoint", "type": [ @@ -71,14 +71,9 @@ ] }, "file": { - "hash": { - "md5": "1468c1908845ef238f7f196809946288", - "sha1": "62f0bd56b0e1235b99940b34916c19ecfac8e80c" - }, - "name": "malz5.zip", - "path": "c:\\users\\admin\\downloads\\malz5.zip", - "size": 12707198, - "type": "zip" + "name": "chrome.exe", + "size": 0, + "type": "exe" }, "host": { "hostname": "DESKTOP-E2P4OL0", @@ -98,10 +93,6 @@ "type": "cel" }, "related": { - "hash": [ - "1468c1908845ef238f7f196809946288", - "62f0bd56b0e1235b99940b34916c19ecfac8e80c" - ], "hosts": [ "DESKTOP-E2P4OL0" ], @@ -113,11 +104,14 @@ ] }, "rule": { - "name": "Gen.Rep.zip" + "name": "Anti-Bot test.TC.e" }, "tags": [ "forwarded" ], + "url": { + "original": "http://www.threat-cloud.com/test/files/MediumConfidenceBot.html" + }, "user": { "domain": "SMC User", "id": "S-1-5-21-3766288932-3295778425-2939962592-1001", @@ -125,4 +119,4 @@ "admin" ] } -} \ No newline at end of file +} diff --git a/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/_dev/test/system/test-default-config.yml b/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..96d4fa91ebc --- /dev/null +++ b/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/_dev/test/system/test-default-config.yml @@ -0,0 +1,12 @@ +input: cel +service: harmony +vars: + base_url: http://{{Hostname}}:{{Port}} + client_id: testclientid + access_key: testaccesskey + initial_interval: 720h + interval: 5m + limit: 3 + page_limit: 10 +assert: + hit_count: 4 diff --git a/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/sample_event.json b/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/sample_event.json index 48babeb6b91..574c4c074a9 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/sample_event.json +++ b/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/sample_event.json @@ -1,11 +1,11 @@ { "@timestamp": "2024-09-02T09:04:54.000Z", "agent": { - "ephemeral_id": "8723e6bf-0b1a-4a95-95b6-d5e11a0380a7", - "id": "9f7d3384-0b1f-462c-9d71-0e0580545765", - "name": "elastic-agent-95748", + "ephemeral_id": "11d05133-3249-4c3d-99c4-e8b1453536ce", + "id": "dfd3982d-22a8-4682-99f7-a8c6820fcccd", + "name": "elastic-agent-46934", "type": "filebeat", - "version": "8.15.1" + "version": "8.14.0" }, "checkpoint_harmony_endpoint": { "threatemulation": { @@ -45,16 +45,16 @@ }, "data_stream": { "dataset": "checkpoint_harmony_endpoint.threatemulation", - "namespace": "43839", + "namespace": "20772", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "8.17.0" }, "elastic_agent": { - "id": "9f7d3384-0b1f-462c-9d71-0e0580545765", + "id": "dfd3982d-22a8-4682-99f7-a8c6820fcccd", "snapshot": false, - "version": "8.15.1" + "version": "8.14.0" }, "event": { "action": "Detect", @@ -64,7 +64,7 @@ ], "dataset": "checkpoint_harmony_endpoint.threatemulation", "id": "a4640108-91b1-0f19-66d5-803100000012", - "ingested": "2024-10-24T05:34:17Z", + "ingested": "2025-12-16T12:00:39Z", "kind": "alert", "module": "checkpoint_harmony_endpoint", "type": [ @@ -126,4 +126,4 @@ "admin" ] } -} \ No newline at end of file +} diff --git a/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/_dev/test/system/test-default-config.yml b/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..8192ded1be8 --- /dev/null +++ b/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/_dev/test/system/test-default-config.yml @@ -0,0 +1,12 @@ +input: cel +service: harmony +vars: + base_url: http://{{Hostname}}:{{Port}} + client_id: testclientid + access_key: testaccesskey + initial_interval: 720h + interval: 5m + limit: 2 + page_limit: 10 +assert: + hit_count: 3 diff --git a/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/sample_event.json b/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/sample_event.json index 28e80d1efe2..e472ee1d49f 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/sample_event.json +++ b/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/sample_event.json @@ -1,11 +1,11 @@ { - "@timestamp": "2024-09-02T09:21:42.000Z", + "@timestamp": "2024-09-04T09:08:30.000Z", "agent": { - "ephemeral_id": "b2ca27d2-5544-4cc2-9491-f91097060c1c", - "id": "82b03ad0-7025-436d-9e81-8a39705e0152", - "name": "elastic-agent-30042", + "ephemeral_id": "6d89d8b4-3779-422b-855b-1d4d67b59ea9", + "id": "3c976c78-8b22-4eec-88ff-353c39411304", + "name": "elastic-agent-22411", "type": "filebeat", - "version": "8.15.1" + "version": "8.14.0" }, "checkpoint_harmony_endpoint": { "threatextraction": { @@ -22,16 +22,16 @@ "action": "Not Supported" }, "policy": { - "date": "2024-08-29T13:12:50.0000000Z", + "date": "2024-09-03T10:46:13.0000000Z", "name": "Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization", - "number": 3 + "number": 4 }, "product": { "family": "Endpoint", "name": "Threat Extraction" }, "protection_type": "Content Removal", - "sequencenum": 1, + "sequencenum": 16777215, "severity": "Informational", "src": "10.35.38.102", "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", @@ -41,16 +41,16 @@ }, "data_stream": { "dataset": "checkpoint_harmony_endpoint.threatextraction", - "namespace": "81720", + "namespace": "12108", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "8.17.0" }, "elastic_agent": { - "id": "82b03ad0-7025-436d-9e81-8a39705e0152", + "id": "3c976c78-8b22-4eec-88ff-353c39411304", "snapshot": false, - "version": "8.15.1" + "version": "8.14.0" }, "event": { "action": "Extract", @@ -59,8 +59,8 @@ "malware" ], "dataset": "checkpoint_harmony_endpoint.threatextraction", - "id": "a4640108-91b1-0f19-66d5-83f100000019", - "ingested": "2024-10-24T05:35:11Z", + "id": "a4640108-91b1-0f19-66d8-247c00000100", + "ingested": "2025-12-16T12:02:59Z", "kind": "alert", "module": "checkpoint_harmony_endpoint", "type": [ @@ -71,10 +71,10 @@ "hash": { "sha1": "no-sha1" }, - "name": "mirai.sh4", - "path": "blob:https://github.com/6bd30ea7-29a8-4dd2-9056-f5077632e110", + "name": "ransomware.go", + "path": "blob:https://github.com/9d747e42-c19c-43d8-b77d-7258ab12a233", "size": 0, - "type": "sh4" + "type": "go" }, "host": { "hostname": "DESKTOP-E2P4OL0", @@ -120,4 +120,4 @@ "admin" ] } -} \ No newline at end of file +} diff --git a/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/_dev/test/system/test-default-config.yml b/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..8192ded1be8 --- /dev/null +++ b/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/_dev/test/system/test-default-config.yml @@ -0,0 +1,12 @@ +input: cel +service: harmony +vars: + base_url: http://{{Hostname}}:{{Port}} + client_id: testclientid + access_key: testaccesskey + initial_interval: 720h + interval: 5m + limit: 2 + page_limit: 10 +assert: + hit_count: 3 diff --git a/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/sample_event.json b/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/sample_event.json index 8ace514474b..98d51b645e2 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/sample_event.json +++ b/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/sample_event.json @@ -1,11 +1,11 @@ { "@timestamp": "2024-09-06T10:07:43.000Z", "agent": { - "ephemeral_id": "95fc55ec-9d54-4116-87cc-a4fe3767eba0", - "id": "3c23eeec-fde0-4811-91a1-6bc5b403c95e", - "name": "elastic-agent-18777", + "ephemeral_id": "a6d1e92c-d3fd-44f3-a3fb-a54f236373a4", + "id": "f24c437b-50f2-4092-9b39-e2fb95287852", + "name": "elastic-agent-96253", "type": "filebeat", - "version": "8.15.1" + "version": "8.14.0" }, "checkpoint_harmony_endpoint": { "urlfiltering": { @@ -45,16 +45,16 @@ }, "data_stream": { "dataset": "checkpoint_harmony_endpoint.urlfiltering", - "namespace": "69408", + "namespace": "85769", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "8.17.0" }, "elastic_agent": { - "id": "3c23eeec-fde0-4811-91a1-6bc5b403c95e", + "id": "f24c437b-50f2-4092-9b39-e2fb95287852", "snapshot": false, - "version": "8.15.1" + "version": "8.14.0" }, "event": { "action": "Detect", @@ -64,7 +64,7 @@ ], "dataset": "checkpoint_harmony_endpoint.urlfiltering", "id": "a4640108-91b1-0f19-66da-d62100000013", - "ingested": "2024-10-24T05:36:11Z", + "ingested": "2025-12-16T12:03:49Z", "kind": "alert", "module": "checkpoint_harmony_endpoint", "type": [ @@ -119,4 +119,4 @@ "admin" ] } -} \ No newline at end of file +} diff --git a/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/_dev/test/system/test-default-config.yml b/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..8192ded1be8 --- /dev/null +++ b/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/_dev/test/system/test-default-config.yml @@ -0,0 +1,12 @@ +input: cel +service: harmony +vars: + base_url: http://{{Hostname}}:{{Port}} + client_id: testclientid + access_key: testaccesskey + initial_interval: 720h + interval: 5m + limit: 2 + page_limit: 10 +assert: + hit_count: 3 diff --git a/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/sample_event.json b/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/sample_event.json index 651d0c8d255..620d093038a 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/sample_event.json +++ b/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/sample_event.json @@ -1,11 +1,11 @@ { "@timestamp": "2024-09-02T08:51:08.000Z", "agent": { - "ephemeral_id": "9fc6c363-e390-492c-bfdf-684e4d20aff8", - "id": "64f03e47-f005-4ecd-8d91-e63af37617a3", - "name": "elastic-agent-34074", + "ephemeral_id": "0e399915-3c5d-46f4-8be7-8e28468123ca", + "id": "aa0b9c7f-94f4-4a81-95cc-acb7f805d24f", + "name": "elastic-agent-30654", "type": "filebeat", - "version": "8.15.1" + "version": "8.14.0" }, "checkpoint_harmony_endpoint": { "zerophishing": { @@ -19,7 +19,6 @@ "event_type": "Phishing Event", "extension_version": "Check Point Endpoint Security Client", "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", - "malware": {}, "policy": { "date": "2024-08-29T13:12:50.0000000Z", "name": "Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization", @@ -40,16 +39,16 @@ }, "data_stream": { "dataset": "checkpoint_harmony_endpoint.zerophishing", - "namespace": "39288", + "namespace": "93890", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "8.17.0" }, "elastic_agent": { - "id": "64f03e47-f005-4ecd-8d91-e63af37617a3", + "id": "aa0b9c7f-94f4-4a81-95cc-acb7f805d24f", "snapshot": false, - "version": "8.15.1" + "version": "8.14.0" }, "event": { "action": "Detect", @@ -58,8 +57,8 @@ "malware" ], "dataset": "checkpoint_harmony_endpoint.zerophishing", - "id": "a4640108-91b1-0f19-66d5-7d6100000004", - "ingested": "2024-10-24T05:37:11Z", + "id": "a4640108-91b1-0f19-66d5-7d6100000114", + "ingested": "2025-12-16T12:04:39Z", "kind": "alert", "module": "checkpoint_harmony_endpoint", "type": [ @@ -113,4 +112,4 @@ "admin" ] } -} \ No newline at end of file +} diff --git a/packages/checkpoint_harmony_endpoint/docs/README.md b/packages/checkpoint_harmony_endpoint/docs/README.md index 05ac2abf3f3..e113b3a5322 100644 --- a/packages/checkpoint_harmony_endpoint/docs/README.md +++ b/packages/checkpoint_harmony_endpoint/docs/README.md @@ -56,9 +56,9 @@ An example event for `antibot` looks as following: { "@timestamp": "2024-09-02T08:53:44.000Z", "agent": { - "ephemeral_id": "f96c672e-e52f-49f4-a098-fb8e47463002", - "id": "a17315c6-08c5-498c-a743-f2125ce9bf1b", - "name": "elastic-agent-77268", + "ephemeral_id": "395d0bda-166c-4800-9c38-ec1455f30548", + "id": "9e02b661-1ee6-4686-9926-edc63a7f4b02", + "name": "elastic-agent-27616", "type": "filebeat", "version": "8.14.0" }, @@ -97,7 +97,7 @@ An example event for `antibot` looks as following: }, "data_stream": { "dataset": "checkpoint_harmony_endpoint.antibot", - "namespace": "67908", + "namespace": "18419", "type": "logs" }, "destination": { @@ -110,7 +110,7 @@ An example event for `antibot` looks as following: "version": "8.17.0" }, "elastic_agent": { - "id": "a17315c6-08c5-498c-a743-f2125ce9bf1b", + "id": "9e02b661-1ee6-4686-9926-edc63a7f4b02", "snapshot": false, "version": "8.14.0" }, @@ -122,7 +122,7 @@ An example event for `antibot` looks as following: ], "dataset": "checkpoint_harmony_endpoint.antibot", "id": "a4640108-91b1-0f19-66d5-7d9d00000003", - "ingested": "2025-06-05T09:35:27Z", + "ingested": "2025-12-16T11:56:39Z", "kind": "event", "module": "checkpoint_harmony_endpoint", "type": [ @@ -243,11 +243,11 @@ An example event for `antimalware` looks as following: { "@timestamp": "2024-09-02T09:09:07.000Z", "agent": { - "ephemeral_id": "972620ca-77a9-4305-991a-5bd475860580", - "id": "4cdb965a-db2a-4ec3-9abf-6e20dbb120c9", - "name": "elastic-agent-21918", + "ephemeral_id": "ce69a1cc-2624-4c1b-abea-4c65c408b88c", + "id": "59648861-728d-4e8a-9109-5d16c682af18", + "name": "elastic-agent-75194", "type": "filebeat", - "version": "8.15.1" + "version": "8.14.0" }, "checkpoint_harmony_endpoint": { "antimalware": { @@ -287,16 +287,16 @@ An example event for `antimalware` looks as following: }, "data_stream": { "dataset": "checkpoint_harmony_endpoint.antimalware", - "namespace": "85578", + "namespace": "52446", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "8.17.0" }, "elastic_agent": { - "id": "4cdb965a-db2a-4ec3-9abf-6e20dbb120c9", + "id": "59648861-728d-4e8a-9109-5d16c682af18", "snapshot": false, - "version": "8.15.1" + "version": "8.14.0" }, "event": { "action": "Detect", @@ -306,7 +306,7 @@ An example event for `antimalware` looks as following: ], "dataset": "checkpoint_harmony_endpoint.antimalware", "id": "a4640108-91b1-0f19-66d5-815d0000000f", - "ingested": "2024-10-24T05:32:25Z", + "ingested": "2025-12-16T11:57:31Z", "kind": "alert", "module": "checkpoint_harmony_endpoint", "type": [ @@ -423,13 +423,13 @@ An example event for `forensics` looks as following: ```json { - "@timestamp": "2024-09-03T08:53:12.000Z", + "@timestamp": "2024-09-02T08:58:05.000Z", "agent": { - "ephemeral_id": "76820ab1-9086-4fc7-975c-2e7cda1f601c", - "id": "3df1f948-9917-4dc4-a724-f2b5934a6652", - "name": "elastic-agent-71957", + "ephemeral_id": "4d32ca32-d2fd-4e13-84dc-5c69ba84bd1e", + "id": "10e7cd33-c414-404f-800a-5878962095d8", + "name": "elastic-agent-75832", "type": "filebeat", - "version": "8.15.1" + "version": "8.14.0" }, "checkpoint_harmony_endpoint": { "forensics": { @@ -438,14 +438,16 @@ An example event for `forensics` looks as following: "name": "Check Point Endpoint Security Client", "version": "88.50.0213" }, - "confidence_level": "High", - "description": "To exclude the file: On the Harmony Endpoint Management add this sha1 exclusion: 62f0bd56-b0e1235b-99940b34-916c19ec-fac8e80c Attack status: Dormant.", - "detected_by": "Endpoint File Reputation", + "confidence_level": "Medium", + "description": "Endpoint Anti-Bot has detected :http://www.threat-cloud.com/test/files/MediumConfidenceBot.html. Attack status: Dormant.", + "detected_by": "Endpoint Anti-Bot", "event_type": "Forensics Case Analysis", "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", - "malware": {}, + "malware": { + "action": "Communication with C&C" + }, "packet_capture": "Packet Capture", - "packet_capture_unique_id": "0acd55a9-f241-4097-a699-6b7e41cd26af", + "packet_capture_unique_id": "ef93cc38-215a-4fba-b571-d71ccdace123", "policy": { "date": "2024-09-02T06:23:25.0000000Z", "name": "Default Forensics settings", @@ -455,29 +457,27 @@ An example event for `forensics` looks as following: "family": "Endpoint", "name": "Forensics" }, - "protection_type": "File Reputation", - "remediated_files": "malz5.zip(Remediation disabled in policy)", + "protection_type": "URL Reputation", "sequencenum": 1, "service_domain": "ep-demo", "severity": "Critical", "src": "10.35.38.102", - "suspicious_events": "System Shutdown / Reboot: ; ", "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "type": "Log" } }, "data_stream": { "dataset": "checkpoint_harmony_endpoint.forensics", - "namespace": "38429", + "namespace": "79277", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "8.17.0" }, "elastic_agent": { - "id": "3df1f948-9917-4dc4-a724-f2b5934a6652", + "id": "10e7cd33-c414-404f-800a-5878962095d8", "snapshot": false, - "version": "8.15.1" + "version": "8.14.0" }, "event": { "action": "Detect", @@ -486,8 +486,8 @@ An example event for `forensics` looks as following: "malware" ], "dataset": "checkpoint_harmony_endpoint.forensics", - "id": "a4640108-91b1-0f19-66d6-ceb500000000", - "ingested": "2024-10-24T05:33:21Z", + "id": "a4640108-91b1-0f19-66d5-7e5100000001", + "ingested": "2025-12-16T11:59:48Z", "kind": "alert", "module": "checkpoint_harmony_endpoint", "type": [ @@ -495,14 +495,9 @@ An example event for `forensics` looks as following: ] }, "file": { - "hash": { - "md5": "1468c1908845ef238f7f196809946288", - "sha1": "62f0bd56b0e1235b99940b34916c19ecfac8e80c" - }, - "name": "malz5.zip", - "path": "c:\\users\\admin\\downloads\\malz5.zip", - "size": 12707198, - "type": "zip" + "name": "chrome.exe", + "size": 0, + "type": "exe" }, "host": { "hostname": "DESKTOP-E2P4OL0", @@ -522,10 +517,6 @@ An example event for `forensics` looks as following: "type": "cel" }, "related": { - "hash": [ - "1468c1908845ef238f7f196809946288", - "62f0bd56b0e1235b99940b34916c19ecfac8e80c" - ], "hosts": [ "DESKTOP-E2P4OL0" ], @@ -537,11 +528,14 @@ An example event for `forensics` looks as following: ] }, "rule": { - "name": "Gen.Rep.zip" + "name": "Anti-Bot test.TC.e" }, "tags": [ "forwarded" ], + "url": { + "original": "http://www.threat-cloud.com/test/files/MediumConfidenceBot.html" + }, "user": { "domain": "SMC User", "id": "S-1-5-21-3766288932-3295778425-2939962592-1001", @@ -599,11 +593,11 @@ An example event for `threatemulation` looks as following: { "@timestamp": "2024-09-02T09:04:54.000Z", "agent": { - "ephemeral_id": "8723e6bf-0b1a-4a95-95b6-d5e11a0380a7", - "id": "9f7d3384-0b1f-462c-9d71-0e0580545765", - "name": "elastic-agent-95748", + "ephemeral_id": "11d05133-3249-4c3d-99c4-e8b1453536ce", + "id": "dfd3982d-22a8-4682-99f7-a8c6820fcccd", + "name": "elastic-agent-46934", "type": "filebeat", - "version": "8.15.1" + "version": "8.14.0" }, "checkpoint_harmony_endpoint": { "threatemulation": { @@ -643,16 +637,16 @@ An example event for `threatemulation` looks as following: }, "data_stream": { "dataset": "checkpoint_harmony_endpoint.threatemulation", - "namespace": "43839", + "namespace": "20772", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "8.17.0" }, "elastic_agent": { - "id": "9f7d3384-0b1f-462c-9d71-0e0580545765", + "id": "dfd3982d-22a8-4682-99f7-a8c6820fcccd", "snapshot": false, - "version": "8.15.1" + "version": "8.14.0" }, "event": { "action": "Detect", @@ -662,7 +656,7 @@ An example event for `threatemulation` looks as following: ], "dataset": "checkpoint_harmony_endpoint.threatemulation", "id": "a4640108-91b1-0f19-66d5-803100000012", - "ingested": "2024-10-24T05:34:17Z", + "ingested": "2025-12-16T12:00:39Z", "kind": "alert", "module": "checkpoint_harmony_endpoint", "type": [ @@ -774,13 +768,13 @@ An example event for `threatextraction` looks as following: ```json { - "@timestamp": "2024-09-02T09:21:42.000Z", + "@timestamp": "2024-09-04T09:08:30.000Z", "agent": { - "ephemeral_id": "b2ca27d2-5544-4cc2-9491-f91097060c1c", - "id": "82b03ad0-7025-436d-9e81-8a39705e0152", - "name": "elastic-agent-30042", + "ephemeral_id": "6d89d8b4-3779-422b-855b-1d4d67b59ea9", + "id": "3c976c78-8b22-4eec-88ff-353c39411304", + "name": "elastic-agent-22411", "type": "filebeat", - "version": "8.15.1" + "version": "8.14.0" }, "checkpoint_harmony_endpoint": { "threatextraction": { @@ -797,16 +791,16 @@ An example event for `threatextraction` looks as following: "action": "Not Supported" }, "policy": { - "date": "2024-08-29T13:12:50.0000000Z", + "date": "2024-09-03T10:46:13.0000000Z", "name": "Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization", - "number": 3 + "number": 4 }, "product": { "family": "Endpoint", "name": "Threat Extraction" }, "protection_type": "Content Removal", - "sequencenum": 1, + "sequencenum": 16777215, "severity": "Informational", "src": "10.35.38.102", "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", @@ -816,16 +810,16 @@ An example event for `threatextraction` looks as following: }, "data_stream": { "dataset": "checkpoint_harmony_endpoint.threatextraction", - "namespace": "81720", + "namespace": "12108", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "8.17.0" }, "elastic_agent": { - "id": "82b03ad0-7025-436d-9e81-8a39705e0152", + "id": "3c976c78-8b22-4eec-88ff-353c39411304", "snapshot": false, - "version": "8.15.1" + "version": "8.14.0" }, "event": { "action": "Extract", @@ -834,8 +828,8 @@ An example event for `threatextraction` looks as following: "malware" ], "dataset": "checkpoint_harmony_endpoint.threatextraction", - "id": "a4640108-91b1-0f19-66d5-83f100000019", - "ingested": "2024-10-24T05:35:11Z", + "id": "a4640108-91b1-0f19-66d8-247c00000100", + "ingested": "2025-12-16T12:02:59Z", "kind": "alert", "module": "checkpoint_harmony_endpoint", "type": [ @@ -846,10 +840,10 @@ An example event for `threatextraction` looks as following: "hash": { "sha1": "no-sha1" }, - "name": "mirai.sh4", - "path": "blob:https://github.com/6bd30ea7-29a8-4dd2-9056-f5077632e110", + "name": "ransomware.go", + "path": "blob:https://github.com/9d747e42-c19c-43d8-b77d-7258ab12a233", "size": 0, - "type": "sh4" + "type": "go" }, "host": { "hostname": "DESKTOP-E2P4OL0", @@ -944,11 +938,11 @@ An example event for `urlfiltering` looks as following: { "@timestamp": "2024-09-06T10:07:43.000Z", "agent": { - "ephemeral_id": "95fc55ec-9d54-4116-87cc-a4fe3767eba0", - "id": "3c23eeec-fde0-4811-91a1-6bc5b403c95e", - "name": "elastic-agent-18777", + "ephemeral_id": "a6d1e92c-d3fd-44f3-a3fb-a54f236373a4", + "id": "f24c437b-50f2-4092-9b39-e2fb95287852", + "name": "elastic-agent-96253", "type": "filebeat", - "version": "8.15.1" + "version": "8.14.0" }, "checkpoint_harmony_endpoint": { "urlfiltering": { @@ -988,16 +982,16 @@ An example event for `urlfiltering` looks as following: }, "data_stream": { "dataset": "checkpoint_harmony_endpoint.urlfiltering", - "namespace": "69408", + "namespace": "85769", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "8.17.0" }, "elastic_agent": { - "id": "3c23eeec-fde0-4811-91a1-6bc5b403c95e", + "id": "f24c437b-50f2-4092-9b39-e2fb95287852", "snapshot": false, - "version": "8.15.1" + "version": "8.14.0" }, "event": { "action": "Detect", @@ -1007,7 +1001,7 @@ An example event for `urlfiltering` looks as following: ], "dataset": "checkpoint_harmony_endpoint.urlfiltering", "id": "a4640108-91b1-0f19-66da-d62100000013", - "ingested": "2024-10-24T05:36:11Z", + "ingested": "2025-12-16T12:03:49Z", "kind": "alert", "module": "checkpoint_harmony_endpoint", "type": [ @@ -1117,11 +1111,11 @@ An example event for `zerophishing` looks as following: { "@timestamp": "2024-09-02T08:51:08.000Z", "agent": { - "ephemeral_id": "9fc6c363-e390-492c-bfdf-684e4d20aff8", - "id": "64f03e47-f005-4ecd-8d91-e63af37617a3", - "name": "elastic-agent-34074", + "ephemeral_id": "0e399915-3c5d-46f4-8be7-8e28468123ca", + "id": "aa0b9c7f-94f4-4a81-95cc-acb7f805d24f", + "name": "elastic-agent-30654", "type": "filebeat", - "version": "8.15.1" + "version": "8.14.0" }, "checkpoint_harmony_endpoint": { "zerophishing": { @@ -1135,7 +1129,6 @@ An example event for `zerophishing` looks as following: "event_type": "Phishing Event", "extension_version": "Check Point Endpoint Security Client", "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", - "malware": {}, "policy": { "date": "2024-08-29T13:12:50.0000000Z", "name": "Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization", @@ -1156,16 +1149,16 @@ An example event for `zerophishing` looks as following: }, "data_stream": { "dataset": "checkpoint_harmony_endpoint.zerophishing", - "namespace": "39288", + "namespace": "93890", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "8.17.0" }, "elastic_agent": { - "id": "64f03e47-f005-4ecd-8d91-e63af37617a3", + "id": "aa0b9c7f-94f4-4a81-95cc-acb7f805d24f", "snapshot": false, - "version": "8.15.1" + "version": "8.14.0" }, "event": { "action": "Detect", @@ -1174,8 +1167,8 @@ An example event for `zerophishing` looks as following: "malware" ], "dataset": "checkpoint_harmony_endpoint.zerophishing", - "id": "a4640108-91b1-0f19-66d5-7d6100000004", - "ingested": "2024-10-24T05:37:11Z", + "id": "a4640108-91b1-0f19-66d5-7d6100000114", + "ingested": "2025-12-16T12:04:39Z", "kind": "alert", "module": "checkpoint_harmony_endpoint", "type": [ diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/docker-compose.yml b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..8a6b88d312d --- /dev/null +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,34 @@ +version: "2.3" +services: + azure-blob-storage-emulator: + image: mcr.microsoft.com/azure-storage/azurite + command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 + ports: + - "10000/tcp" + uploader: + image: mcr.microsoft.com/azure-cli + depends_on: + - azure-blob-storage-emulator + volumes: + - ./sample_logs:/sample_logs + entrypoint: > + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/test-events.log --name test-events.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/files/manifest.yml b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..b106d87cc99 --- /dev/null +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/test-events.log + content-type: application/x-ndjson diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/gcs-mock-service/main.go b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/sample_logs/test-events.log b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/sample_logs/test-events.log new file mode 100644 index 00000000000..2a631c602e4 --- /dev/null +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/sample_logs/test-events.log @@ -0,0 +1 @@ +{"category_id":3,"collector_device_ip":"175.16.199.0","collector_device_name":"Collector_Device_Name","collector_name":"Collection12","collector_uid":"TT1456","composite":1,"container":{"host_name":"azure-us.local","image_name":"Image-sp2133","image_uid":"SH4322323","name":"User12345","networks":[{"bssid":"Container-Networks-BSSID 1","gateway_ip":"89.160.20.112","gateway_mac":"00:B0:D0:63:C2:01","ipv4":"81.2.69.144","ipv6":"2a02:cf40::","is_public":true,"mac":"00:B0:D0:63:C2:02","rep_score_id":0,"ssid":"SSID-4326451","type_id":0},{"bssid":"HN0845435","gateway_ip":"81.2.69.142","gateway_mac":"00:B0:D0:63:C2:03","ipv4":"81.2.69.144","ipv6":"2a02:cf40::","is_public":true,"mac":"00:B0:D0:63:C2:04","rep_score_id":1,"ssid":"Container-Networks-SSID 2","type_id":1}],"os_name":"Windows","uid":"UU35r3454"},"correlation_uid":"DD78666","count":563,"customer_registry_uid":"CP45254","customer_uid":"CH32534","cybox":{"domains":["example.com","abc.com"],"emails":[{"direction_id":0,"header_from":"abc@example.com","header_message_id":"Cybox-Emails-Header_Message_ID 1","header_reply_to":"Cybox-Emails-Header_Reply_To 1","header_subject":"Cybox-Emails-Header_Subject 1","header_to":["Cybox-Emails-Header_To 1","Cybox-Emails-Header_Tos 1"],"sender_ip":"81.2.69.144","size":12345678901,"smtp_from":"Cybox-Emails-SMTP_From 1","smtp_hello":"Cybox-Emails-SMTP_Hello 1","smtp_to":"Cybox-Emails-SMTP_To 1"},{"direction_id":1,"header_from":"abc@example.com","header_message_id":"Cybox-Emails-Header_Message_ID 2","header_reply_to":"Cybox-Emails-Header_Reply_To 2","header_subject":"Cybox-Emails-Header_Subject 2","header_to":["Cybox-Emails-Header_To 2","Cybox-Emails-Header_Tos 2"],"sender_ip":"81.2.69.144","size":12345678902,"smtp_from":"Cybox-Emails-SMTP_From 2","smtp_hello":"Cybox-Emails-SMTP_Hello 2","smtp_to":"Cybox-Emails-SMTP_To 2"}],"files":[{"accessed":1613021404000,"accessor":"Cybox-Files-Accessor 1","attribute_ids":[1,2,3,4,5,6,7,8,9,10],"attributes":12345678901,"company_name":"Microsoft Corporation","confidentiality_id":0,"content_type":{"family_id":0,"subtype":"SubType 1","type_id":0},"created":1613021404000,"creator":"Creator 1","creator_process":"Cybox-Files-Creator_Process 1","desc":"Cybox-Files-Desc 1","folder":"c:\\windows\\system32\\cybox\\files\\folder\\1","folder_uid":"Cybox-Files-Folder_UID 1","is_system":true,"md5":"HFDajsdf3254345436","mime_type":"Cybox-Files-MIME_Type 1","modified":1613021404000,"modifier":"Cybox-Files-Modifier 1","name":"cybox_files_name_1.exe","normalized_path":"CSIDL_SYSTEM\\cybox_files_normalized_path_1.exe","original_name":"Cybox-Files-Original_Name 1","owner":"Cybox-Files-Owner 1","parent_name":"Cybox-Files-Parent_Name 1","parent_sha2":"Cybox-Files-Parent_SHA2 1","path":"c:\\windows\\system32\\cybox_files_path_1.exe","product_name":"Windows Internet Explorer 1","product_path":"Cybox-Files-Product_Path 1","rep_discovered_band":0,"rep_discovered_date":1613021404000,"rep_prevalence":12345678901,"rep_prevalence_band":0,"rep_score":12345678901,"rep_score_band":0,"security_descriptor":"Cybox-Files-Security_Descriptor 1","sha1":"Cybox-Files-SHA1 1","sha2":"Cybox-Files-SHA2 1","signature_company_name":"Cybox-Files-Signature_Company_Name 1","signature_created_date":1613021404000,"signature_developer_uid":"Cybox-Files-Signature_Developer_UID 1","signature_fingerprints":[{"algorithm":"Cybox-Files-Signature_Fingerprints-Algorithm 1","value":"Cybox-Files-Signature_Fingerprints-Value 1"},{"algorithm":"Cybox-Files-Signature_Fingerprints-Algorithms 1","value":"Cybox-Files-Signature_Fingerprints-Values 1"}],"signature_issuer":"Cybox-Files-Signature_Issuer 1","signature_level_id":0,"signature_serial_number":"Cybox-Files-Signature_Serial_Number 1","signature_value":12345678901,"signature_value_ids":[0,1,2,3,4,5,6,7,8,9,10],"size":12345678901,"size_compressed":12345678901,"src_ip":"81.2.69.142","src_name":"Cybox-Files-SRC_Name 1","type_id":1,"uid":"Cybox-Files-UID 1","url":{"categories":["Cybox-Files-URL-Category 1","Cybox-Files-URL-Categories 1"],"category_ids":[1,3,4,5,6,7,9,11,14,15,16,17,18,20,21,22,23,24,25,26,27,29,30,31,32,33,34,35,36,37,38,40,43,44,45,46,47,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,65,66,67,68,71,83,84,85,86,87,88,89,90,92,93,95,96,97,98],"extension":"Cybox-Files-URL-Extension 1","host":"www.files-url-host-1.com","method":"Cybox-Files-URL-Method 1","parent_categories":["Cybox-Files-URL-Parent_Category 1","Cybox-Files-URL-Parent_Categories 1"],"path":"/download/trouble/cybox/files/url/path/1","port":80,"provider":"Cybox-Files-URL-Provider 1","query":"q=bad&sort=date_1","referrer":"Cybox-Files-URL-Referrer 1","referrer_categories":["Cybox-Files-URL-Referrer_Category 1","Cybox-Files-URL-Referrer_Categories 1"],"referrer_category_ids":[12345678901,67890123451],"rep_score_id":0,"scheme":"Cybox-Files-URL-Scheme 1","text":"www.files-url-text-1.com/download/trouble"},"version":"Cybox-Files-Version 1","xattributes":{"ads_name":"Cybox-Files-XAttributes-ADS_Name 1","ads_size":"Cybox-Files-XAttributes-ADS_Size 1","dacl":"Cybox-Files-XAttributes-DACL 1","owner":"Cybox-Files-XAttributes-Owner 1","primary_group":"Cybox-Files-XAttributes-Primary_Group 1","link_name":"Cybox-Files-XAttributes-Link_Name 1","hard_link_count":"Cybox-Files-XAttributes-Hard_Link_Count 1","Unix_permissions":"Cybox-Files-XAttributes-Unix_Permissions 1"}},{"accessed":1613021404000,"accessor":"Cybox-Files-Accessor 2","attribute_ids":[11,12,13,14,15,16,17],"attributes":12345678902,"company_name":"Microsoft Corporation 2","confidentiality_id":1,"content_type":{"family_id":1,"subtype":"Cybox-Files-Content_Type-SubType 2","type_id":1},"created":1613021404000,"creator":"Cybox-Files-Creator 2","creator_process":"Cybox-Files-Creator_Process 2","desc":"Cybox-Files-Desc 2","folder":"c:\\windows\\system32\\cybox\\files\\folder\\2","folder_uid":"Cybox-Files-Folder_UID 2","is_system":true,"md5":"Cybox-Files-MD5 2","mime_type":"Cybox-Files-MIME_Type 2","modified":1613021404000,"modifier":"Cybox-Files-Modifier 2","name":"cybox_files_name_2.exe","normalized_path":"CSIDL_SYSTEM\\cybox_files_normalized_path_2.exe","original_name":"Cybox-Files-Original_Name 2","owner":"Cybox-Files-Owner 2","parent_name":"Cybox-Files-Parent_Name 2","parent_sha2":"Cybox-Files-Parent_SHA2 2","path":"c:\\windows\\system32\\cybox_files_path_2.exe","product_name":"Windows Internet Explorer 2","product_path":"Cybox-Files-Product_Path 2","rep_discovered_band":1,"rep_discovered_date":1613021404000,"rep_prevalence":12345678902,"rep_prevalence_band":1,"rep_score":12345678902,"rep_score_band":1,"security_descriptor":"Cybox-Files-Security_Descriptor 2","sha1":"Cybox-Files-SHA1 2","sha2":"Cybox-Files-SHA2 2","signature_company_name":"Cybox-Files-Signature_Company_Name 2","signature_created_date":1613021404000,"signature_developer_uid":"Cybox-Files-Signature_Developer_UID 2","signature_fingerprints":[{"algorithm":"Cybox-Files-Signature_Fingerprints-Algorithm 2","value":"Cybox-Files-Signature_Fingerprints-Value 2"},{"algorithm":"Cybox-Files-Signature_Fingerprints-Algorithms 2","value":"Cybox-Files-Signature_Fingerprints-Values 2"}],"signature_issuer":"Cybox-Files-Signature_Issuer 2","signature_level_id":1,"signature_serial_number":"Cybox-Files-Signature_Serial_Number 2","signature_value":12345678902,"signature_value_ids":[11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],"size":12345678902,"size_compressed":12345678902,"src_ip":"81.2.69.144","src_name":"Cybox-Files-SRC_Name 2","type_id":1,"uid":"Cybox-Files-UID 2","url":{"categories":["Cybox-Files-URL-Category 2","Cybox-Files-URL-Categories 2"],"category_ids":[101,102,103,104,105,106,107,108,109,110,111,112,113,114,116,117,118,121,124],"extension":"Cybox-Files-URL-Extension 2","host":"www.files-url-host-2.com","method":"Cybox-Files-URL-Method 2","parent_categories":["Cybox-Files-URL-Parent_Category 2","Cybox-Files-URL-Parent_Categories 2"],"path":"/download/trouble/cybox/files/url/path/2","port":81,"provider":"Cybox-Files-URL-Provider 2","query":"q=bad&sort=date_2","referrer":"Cybox-Files-URL-Referrer 2","referrer_categories":["Cybox-Files-URL-Referrer_Category 2","Cybox-Files-URL-Referrer_Categories 2"],"referrer_category_ids":[12345678902,67890123452],"rep_score_id":1,"scheme":"Cybox-Files-URL-Scheme 2","text":"www.files-url-text-2.com/download/trouble"},"version":"Cybox-Files-Version 2","xattributes":{"ads_name":"Cybox-Files-XAttributes-ADS_Name 2","ads_size":"Cybox-Files-XAttributes-ADS_Size 2","dacl":"Cybox-Files-XAttributes-DACL 2","owner":"Cybox-Files-XAttributes-Owner 2","primary_group":"Cybox-Files-XAttributes-Primary_Group 2","link_name":"Cybox-Files-XAttributes-Link_Name 2","hard_link_count":"Cybox-Files-XAttributes-Hard_Link_Count 2","Unix_permissions":"Cybox-Files-XAttributes-Unix_Permissions 2"}}],"hostnames":["Cybox-Hostname 1","Cybox-Hostnames 1"],"icap_reqmod":[{"metadata":{"field1_keyword":"Cybox-ICAP_ReqMod-field1_Keyword","field1_number":12345678901,"field1_boolean":true,"field1_ip":"175.16.199.0"},"service":"Cybox-ICAP_ReqMod-Service 1","status":"Cybox-ICAP_ReqMod-Status 1","status_detail":"Cybox-ICAP_ReqMod-Status_Detail 1"},{"metadata":{"field2_keyword":"Cybox-ICAP_ReqMod-field2_Keyword","field2_number":12345678902,"field2_boolean":true,"field2_ip":"175.16.199.0"},"service":"Cybox-ICAP_ReqMod-Service 2","status":"Cybox-ICAP_ReqMod-Status 2","status_detail":"Cybox-ICAP_ReqMod-Status_Detail 2"}],"icap_respmod":[{"metadata":{"field1_keyword":"Cybox-ICAP_RespMod-field1_Keyword","field1_number":12345678901,"field1_boolean":true,"field1_ip":"175.16.199.0"},"service":"Cybox-ICAP_RespMod-Service 1","status":"Cybox-ICAP_RespMod-Status 1","status_detail":"Cybox-ICAP_RespMod-Status_Detail 1"},{"metadata":{"field2_keyword":"Cybox-ICAP_RespMod-field2_Keyword","field2_number":12345678902,"field2_boolean":true,"field2_ip":"175.16.199.0"},"service":"Cybox-ICAP_RespMod-Service 2","status":"Cybox-ICAP_RespMod-Status 2","status_detail":"Cybox-ICAP_RespMod-Status_Detail 2"}],"ipv4s":["175.16.199.0","175.16.199.0"],"ipv6s":["2a02:cf40::","2a02:cf40::"],"macs":["00:B0:D0:63:C2:05","00:B0:D0:63:C2:06"],"urls":[{"categories":["Cybox-URLs-Category 1","Cybox-URLs-Categories 1"],"category_ids":[1,3,4,5,6,7,9,11,14,15,16,17,18,20,21,22,23,24,25,26,27,29,30,31,32,33,34,35,36,37,38,40,43,44,45,46,47,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,65,66,67,68,71,83,84,85,86,87,88,89,90,92,93,95,96,97,98],"extension":"Cybox-URLs-Extension 1","host":"www.urls-host-1.com","method":"Cybox-URLs-Method 1","parent_categories":["Cybox-URLs-Parent_Category 1","Cybox-URLs-Parent_Categories 1"],"path":"/download/trouble/cybox/urls/path/1","port":80,"provider":"Cybox-URLs-Provider 1","query":"q=bad&sort=date_1","referrer":"Cybox-URLs-Referrer 1","referrer_categories":["Cybox-URLs-Referrer_Category 1","Cybox-URLs-Referrer_Categories 1"],"referrer_category_ids":[12345678901,67890123451],"rep_score_id":0,"scheme":"Cybox-URLs-Scheme 1","text":"www.urls-text-1.com/download/trouble"},{"categories":["Cybox-URLs-Category 2","Cybox-URLs-Categories 2"],"category_ids":[101,102,103,104,105,106,107,108,109,110,111,112,113,114,116,117,118,121,124],"extension":"Cybox-URLs-Extension 2","host":"www.urls-host-2.com","method":"Cybox-URLs-Method 2","parent_categories":["Cybox-URLs-Parent_Category 2","Cybox-URLs-Parent_Categories 2"],"path":"/download/trouble/cybox/urls/path/2","port":81,"provider":"Cybox-URLs-Provider 2","query":"q=bad&sort=date_2","referrer":"Cybox-URLs-Referrer 2","referrer_categories":["Cybox-URLs-Referrer_Category 2","Cybox-URLs-Referrer_Categories 2"],"referrer_category_ids":[12345678902,67890123452],"rep_score_id":1,"scheme":"Cybox-URLs-Scheme 2","text":"www.urls-text-2.com/download/trouble"}]},"device_alias_name":"Device_Alias_Name","device_cap":"Device_Cap","device_cloud_vm":{"autoscale_uid":"Device_Cloud_VM-Autoscale_UID","dc_region":"Device_Cloud_VM-DC_Region","instance_uid":"Device_Cloud_VM-Instance_UID","subnet_uid":"Device_Cloud_VM-Subnet_UID","vpc_uid":"Device_Cloud_VM-VPC_UID"},"device_desc":"Device_Desc","device_domain":"device.domain.internal.somecompany.com","device_domain_uid":"Device_Domain_UID","device_end_time":1613021404000,"device_gateway":"175.16.199.0","device_group":"Device_Group","device_group_name":"Device_Group_Name","device_hw_bios_date":"03/31/16","device_hw_bios_manufacturer":"LENOVO","device_hw_bios_ver":"LENOVO G5ETA2WW (2.62)","device_hw_cpu_type":"x86 Family 6 Model 37 Stepping 5","device_imei":"Device_IMEI","device_ip":"175.16.199.0","device_is_compliant":true,"device_is_personal":true,"device_is_trusted":true,"device_is_unmanaged":true,"device_location":{"city":"Device_Location-City","continent":"Device_Location-Continent","coordinates":[-12.345,56.789],"country":"US","desc":"Device_Location-Desc","isp":"Device_Location-ISP","on_premises":true,"region":"US-CA"},"device_mac":"00:B0:D0:63:C2:07","device_name":"device.name.computer.domain","device_name_md5":"4ED962DDBF17E2BBA7B14EBC00F3162E","device_networks":[{"bssid":"Device_Networks-BSSID 1","gateway_ip":"175.16.199.0","gateway_mac":"00:B0:D0:63:C2:08","ipv4":"175.16.199.0","ipv6":"2a02:cf40::","is_public":true,"mac":"00:B0:D0:63:C2:09","rep_score_id":0,"ssid":"Device_Networks-SSID 1","type_id":0},{"bssid":"Device_Networks-BSSID 2","gateway_ip":"89.160.20.112","gateway_mac":"00:B0:D0:63:C2:10","ipv4":"89.160.20.112","ipv6":"2a02:cf40::","is_public":true,"mac":"00:B0:D0:63:C2:11","rep_score_id":1,"ssid":"Device_Networks-SSID 2","type_id":1}],"device_org_unit":"Device_Org_Unit","device_os_bits":12345678901,"device_os_build":"Device_OS_Build","device_os_country":"IN","device_os_edition":"Professional","device_os_lang":"en","device_os_name":"Windows Server 2019 Standard Edition","device_os_sp_name":"Device_OS_SP_Name","device_os_sp_ver":"Device_OS_SP_Ver","device_os_type_id":0,"device_os_ver":"Windows 10","device_proxy_ip":"89.160.20.112","device_proxy_name":"Device_Proxy_Name","device_public_ip":"89.160.20.112","device_ref_uid":"Device_Ref_UID","device_site":"Device_Site","device_subnet":"81.2.69.144","device_time":1613021404000,"device_type":"server","device_uid":"Device_UID","device_vhost":"Device_VHost","device_vhost_id":0,"domain_uid":"Domain_UID","end_time":"2024-02-29T01:00:00.000Z","entity":{"data":{"field1_keyword":"Entity-Data-field1_Keyword","field1_number":12345678901,"field1_boolean":true},"name":"Entity-Name","type":"Entity-Type","uid":"Entity-UID","version":"Entity-Version"},"event_id":2001,"events":[{"connection":{"direction_id":1,"dst_service":"C:\\Windows\\system32\\NTOSKRNL.EXE","src_ip":"159.19.163.218"},"count":1,"device_end_time":1709225074618,"device_time":1709225074618}],"feature_name":"Feature_Name","feature_path":"Feature_Path","feature_type":"Feature_Type","feature_uid":"Feature_UID","feature_ver":"2014.1.4.25","id":12345678901,"impersonator_customer_uid":"Impersonator_Customer_UID","impersonator_domain_uid":"Impersonator_Domain_UID","impersonator_user_uid":"Impersonator_User_UID","is_user_present":true,"log_level":"Log Level","log_name":"Log_Name","log_time":"2024-02-29T01:00:00.000Z","logging_device_ip":"89.160.20.112","logging_device_name":"Logging_Device_Name","logging_device_post_time":1613021404000,"logging_device_ref_uid":"Logging_Device_Ref_UID","message":"Message","message_code":"Message_Code","message_id":0,"org_unit_uid":"Org_Unit_UID","orig_data":"Orig_Data","product_data":{"sep_domain_uid":"Product_Data-Sep_Domain_UID","sep_hw_uid":"Product_Data-Sep_HW_UID"},"product_lang":"en","product_name":"Symantec Endpoint Security","product_uid":"Product_UID","product_ver":"2014.1.4.25-beta","proxy_device_ip":"89.160.20.112","proxy_device_name":"Proxy_Device_Name","raw_data":{"assetID":"vc9DagprQYyLZ23SEY1APw","assetOpstateDTO":{"productUuid":"31B0C880-0229-49E8-94C5-48D56B1BD7B9","features":[{"uuid":"1DF0351C-146D-4F07-B155-BF5C7077FF40","featureStatus":"SECURE","opstate":{"EDRContentSequence":"20231128005","EDREngineVersion":"4.11.0.10","EDRFramworkVersion":"4.10.0.59","FDRStatus":true,"LowDiskSpace":false,"MaxDBSizeHonored":true,"applied_policy":{"effective_date":1709219437080,"sha2":"ee6b0bebbc4575b507ac616d2c362f2c54d462b92cf4068cb6681ae3187d4de3","uid":"7dc29d40-f303-477a-9012-287ef252a391","version":"16"},"disk_usage_mb":1546,"fdr_first_event_date":"20240227","fdr_state":1},"state":"ENABLED","statusReason":["-107","0"],"prevention_state":"1"}],"products_active":0,"blades":0}},"ref_log_name":"Ref_Log_Name","ref_log_time":"2024-02-29T01:00:00.000Z","ref_orig_uid":"Ref_Orig_UID","ref_uid":"Ref_UID","remediated":true,"remediation":"Remediation","remediation_ref":"Remediation_Ref","remediation_uid":0,"seq_num":12345678901,"sessions":[{"auth_protocol_id":0,"cleartext_credentials":true,"direction_id":0,"id":12345678901,"is_admin":true,"logon_type_id":1,"port":80,"previous_users":["Sessions-Previous_User 1","Sessions-Previous_Users 1"],"remote":true,"remote_host":"Sessions-Remote_Host 1","remote_ip":"89.160.20.112","user":{"account_disabled":true,"cloud_resource_uid":"Sessions-User-Cloud_Resource_UID 1","domain":"Sessions-User-Domain 1","external_account_uid":"Sessions-User-External_Account_UID 1","external_uid":"Sessions-User-External_UID 1","full_name":"Sessions-User-Full_Name 1","groups":["Sessions-User-Group 1","Sessions-User-Groups 1"],"home":"Sessions-User-Home 1","is_admin":true,"logon_name":"Sessions-User-Logon_Name 1","name":"session-User-Name 1","password_expires":true,"shell":"Sessions-User-Shell 1","sid":"Sessions-User-SID 1","uid":"Sessions-User-UID 1"}},{"auth_protocol_id":1,"cleartext_credentials":true,"direction_id":1,"id":67890123451,"is_admin":true,"logon_type_id":2,"port":81,"previous_users":["Sessions-Previous_User 2","Sessions-Previous_Users 2"],"remote":true,"remote_host":"Sessions-Remote_Host 2","remote_ip":"89.160.20.112","user":{"account_disabled":true,"cloud_resource_uid":"Sessions-User-Cloud_Resource_UID 2","domain":"Sessions-User-Domain 2","external_account_uid":"Sessions-User-External_Account_UID 2","external_uid":"Sessions-User-External_UID 2","full_name":"Sessions-User-Full_Name 2","groups":["Sessions-User-Group 2","Sessions-User-Groups 2"],"home":"Sessions-User-Home 2","is_admin":true,"logon_name":"Sessions-User-Logon_Name 2","name":"session-User-Name 2","password_expires":true,"shell":"Sessions-User-Shell 2","sid":"Sessions-User-SID 2","uid":"Sessions-User-UID 2"}}],"severity_id":0,"source":{"facility":"Source-Facility","facility_detail":"Source-Facility_Detail","facility_uid":"Source-Facility_UID","type_id":1},"status_detail":"Status_Detail","status_id":0,"status_os":"Status_OS","status_os_src":12345678901,"status_stack_trace":"Status_Stack_Trace","status_thread_name":"Status_Thread_Name","stic_has_pii":true,"stic_hw_uid":"STIC_HW_UID","stic_ip_hash":"STIC_IP_Hash","stic_legacy_ent_uids":["STIC_Legacy_Ent_UIDs 1","STIC_Legacy_Ent_UIDs 2"],"stic_legacy_hw_uids":["STIC_Legacy_HW_UIDs 1","STIC_Legacy_HW_UIDs 2"],"stic_legacy_uids":["STIC_Legacy_UIDs 1","STIC_Legacy_UIDs 2"],"stic_schema_id":"STIC_Schema_ID","stic_uid":"STIC_UID","stic_version":"STIC_Version","subfeature_name":"Subfeature_Name","time":"2024-02-29T02:00:00Z","timezone":12345678901,"type":"Type","type_id":2,"user":{"account_disabled":true,"cloud_resource_uid":"User-Cloud_Resource_UID","domain":"User-Domain","external_account_uid":"User-External_Account_UID","external_uid":"User-External_UID","full_name":"User-Full_Name","groups":["User-Group 1","User-Groups 1"],"home":"User-Home","is_admin":true,"logon_name":"User-Logon_Name","name":"User123","password_expires":true,"shell":"User-Shell","sid":"TT23009","uid":"UU34899825"},"user_name":"Mohit","user_uid":"AB45698","uuid":"SR-1565234545","version":"1.4"} diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-azure-config.yml b/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-azure-config.yml new file mode 100644 index 00000000000..83156fe327e --- /dev/null +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-azure-config.yml @@ -0,0 +1,16 @@ +deployer: docker +service: azure-blob-storage-emulator +input: azure-blob-storage +vars: +data_stream: + vars: + account_name: devstoreaccount1 + service_account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + storage_url: "http://{{Hostname}}:{{Port}}/devstoreaccount1/" + containers: | + - name: test-container + max_workers: 3 + poll: true + poll_interval: 15s +assert: + hit_count: 1 diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-default-config.yml b/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-default-config.yml index 245ae4131a7..3dcb0bd2caa 100644 --- a/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-default-config.yml +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-default-config.yml @@ -1,3 +1,4 @@ +deployer: tf input: aws-s3 wait_for_data_timeout: 20m data_stream: diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-gcs-config.yml b/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-gcs-config.yml new file mode 100644 index 00000000000..26a85f7bb25 --- /dev/null +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-gcs-config.yml @@ -0,0 +1,16 @@ +deployer: docker +service: gcs-mock-service +input: gcs +vars: +data_stream: + vars: + project_id: fake-gcs-project + alternative_host: "http://{{Hostname}}:{{Port}}" + number_of_workers: 1 + poll: true + poll_interval: 15s + service_account_key: "{\"type\":\"service_account\",\"project_id\":\"fake-gcs-project\"}" + buckets: | + - name: testbucket +assert: + hit_count: 1 diff --git a/packages/symantec_endpoint_security/data_stream/event/agent/stream/gcs.yml.hbs b/packages/symantec_endpoint_security/data_stream/event/agent/stream/gcs.yml.hbs index 274ee35b8ec..82988fc0747 100644 --- a/packages/symantec_endpoint_security/data_stream/event/agent/stream/gcs.yml.hbs +++ b/packages/symantec_endpoint_security/data_stream/event/agent/stream/gcs.yml.hbs @@ -1,6 +1,9 @@ {{#if project_id}} project_id: {{project_id}} {{/if}} +{{#if alternative_host}} +alternative_host: {{alternative_host}} +{{/if}} {{#if service_account_key}} auth.credentials_json.account_key: {{service_account_key}} {{/if}} diff --git a/packages/symantec_endpoint_security/data_stream/event/fields/beats.yml b/packages/symantec_endpoint_security/data_stream/event/fields/beats.yml index fff1b3f1b6b..9a99e9c81eb 100644 --- a/packages/symantec_endpoint_security/data_stream/event/fields/beats.yml +++ b/packages/symantec_endpoint_security/data_stream/event/fields/beats.yml @@ -8,17 +8,31 @@ type: group fields: - name: bucket - type: group - fields: - - name: name - type: keyword - description: The AWS S3 bucket name. - - name: arn - type: keyword - description: The AWS S3 bucket ARN. + type: flattened - name: object - type: group - fields: - - name: key - type: keyword - description: The AWS S3 Object key. + type: flattened +- name: gcs.storage + type: group + fields: + - name: bucket.name + type: keyword + description: The name of the Google Cloud Storage Bucket. + - name: object.json_data + type: keyword + description: When parse_json is true, the resulting JSON data is stored in this field. + - name: object.name + type: keyword + description: The content type of the Google Cloud Storage object. + - name: object.content_type + type: keyword + description: The content type of the Google Cloud Storage object. +- name: azure + type: group + fields: + - name: resource + type: flattened + - name: storage + type: flattened + - name: subscription_id + type: keyword + description: Azure subscription ID. diff --git a/packages/symantec_endpoint_security/data_stream/event/manifest.yml b/packages/symantec_endpoint_security/data_stream/event/manifest.yml index 8fcbcd17a41..d4e862331ea 100644 --- a/packages/symantec_endpoint_security/data_stream/event/manifest.yml +++ b/packages/symantec_endpoint_security/data_stream/event/manifest.yml @@ -331,6 +331,13 @@ streams: required: false show_user: false description: Defines the epoch time in seconds, which is used to filter out objects/files that are older than the specified timestamp. + - name: alternative_host + type: text + title: Alternative Host + description: Used to override the default host for the storage client (default is storage.googleapis.com) + required: false + multi: false + show_user: false - name: preserve_original_event required: false show_user: true diff --git a/packages/symantec_endpoint_security/data_stream/event/sample_event.json b/packages/symantec_endpoint_security/data_stream/event/sample_event.json index dacf7525be5..fa007c56dee 100644 --- a/packages/symantec_endpoint_security/data_stream/event/sample_event.json +++ b/packages/symantec_endpoint_security/data_stream/event/sample_event.json @@ -1,22 +1,11 @@ { "@timestamp": "2024-02-29T02:00:00.000Z", "agent": { - "ephemeral_id": "1e13efa5-b9f8-46fa-a694-2c68a106525f", - "id": "ed4de143-3b92-4c05-89dc-dd1b6574fd80", - "name": "elastic-agent-95953", + "ephemeral_id": "2979533b-cdb6-49c0-8665-3223e6922f90", + "id": "68d148de-0c02-47af-b9dd-210b7682ece0", + "name": "elastic-agent-86709", "type": "filebeat", - "version": "8.13.0" - }, - "aws": { - "s3": { - "bucket": { - "arn": "arn:aws:s3:::elastic-package-symantec-endpoint-security-bucket-96277", - "name": "elastic-package-symantec-endpoint-security-bucket-96277" - }, - "object": { - "key": "events.log" - } - } + "version": "8.16.5" }, "client": { "domain": "device.domain.internal.somecompany.com", @@ -25,11 +14,11 @@ } }, "cloud": { - "region": "us-east-1" + "provider": "google cloud" }, "data_stream": { "dataset": "symantec_endpoint_security.event", - "namespace": "34134", + "namespace": "56890", "type": "logs" }, "destination": { @@ -47,9 +36,9 @@ "version": "8.11.0" }, "elastic_agent": { - "id": "ed4de143-3b92-4c05-89dc-dd1b6574fd80", + "id": "68d148de-0c02-47af-b9dd-210b7682ece0", "snapshot": false, - "version": "8.13.0" + "version": "8.16.5" }, "email": { "direction": [ @@ -82,10 +71,9 @@ "2021-02-11T05:30:04.000Z" ], "id": "SR-1565234545", - "ingested": "2025-04-24T12:35:04Z", + "ingested": "2025-10-28T11:45:21Z", "kind": "event", "module": "Feature_Name", - "original": "{\"category_id\":3,\"collector_device_ip\":\"175.16.199.0\",\"collector_device_name\":\"Collector_Device_Name\",\"collector_name\":\"Collection12\",\"collector_uid\":\"TT1456\",\"composite\":1,\"container\":{\"host_name\":\"azure-us.local\",\"image_name\":\"Image-sp2133\",\"image_uid\":\"SH4322323\",\"name\":\"User12345\",\"networks\":[{\"bssid\":\"Container-Networks-BSSID 1\",\"gateway_ip\":\"89.160.20.112\",\"gateway_mac\":\"00:B0:D0:63:C2:01\",\"ipv4\":\"81.2.69.144\",\"ipv6\":\"2a02:cf40::\",\"is_public\":true,\"mac\":\"00:B0:D0:63:C2:02\",\"rep_score_id\":0,\"ssid\":\"SSID-4326451\",\"type_id\":0},{\"bssid\":\"HN0845435\",\"gateway_ip\":\"81.2.69.142\",\"gateway_mac\":\"00:B0:D0:63:C2:03\",\"ipv4\":\"81.2.69.144\",\"ipv6\":\"2a02:cf40::\",\"is_public\":true,\"mac\":\"00:B0:D0:63:C2:04\",\"rep_score_id\":1,\"ssid\":\"Container-Networks-SSID 2\",\"type_id\":1}],\"os_name\":\"Windows\",\"uid\":\"UU35r3454\"},\"correlation_uid\":\"DD78666\",\"count\":563,\"customer_registry_uid\":\"CP45254\",\"customer_uid\":\"CH32534\",\"cybox\":{\"domains\":[\"example.com\",\"abc.com\"],\"emails\":[{\"direction_id\":0,\"header_from\":\"abc@example.com\",\"header_message_id\":\"Cybox-Emails-Header_Message_ID 1\",\"header_reply_to\":\"Cybox-Emails-Header_Reply_To 1\",\"header_subject\":\"Cybox-Emails-Header_Subject 1\",\"header_to\":[\"Cybox-Emails-Header_To 1\",\"Cybox-Emails-Header_Tos 1\"],\"sender_ip\":\"81.2.69.144\",\"size\":12345678901,\"smtp_from\":\"Cybox-Emails-SMTP_From 1\",\"smtp_hello\":\"Cybox-Emails-SMTP_Hello 1\",\"smtp_to\":\"Cybox-Emails-SMTP_To 1\"},{\"direction_id\":1,\"header_from\":\"abc@example.com\",\"header_message_id\":\"Cybox-Emails-Header_Message_ID 2\",\"header_reply_to\":\"Cybox-Emails-Header_Reply_To 2\",\"header_subject\":\"Cybox-Emails-Header_Subject 2\",\"header_to\":[\"Cybox-Emails-Header_To 2\",\"Cybox-Emails-Header_Tos 2\"],\"sender_ip\":\"81.2.69.144\",\"size\":12345678902,\"smtp_from\":\"Cybox-Emails-SMTP_From 2\",\"smtp_hello\":\"Cybox-Emails-SMTP_Hello 2\",\"smtp_to\":\"Cybox-Emails-SMTP_To 2\"}],\"files\":[{\"accessed\":1613021404000,\"accessor\":\"Cybox-Files-Accessor 1\",\"attribute_ids\":[1,2,3,4,5,6,7,8,9,10],\"attributes\":12345678901,\"company_name\":\"Microsoft Corporation\",\"confidentiality_id\":0,\"content_type\":{\"family_id\":0,\"subtype\":\"SubType 1\",\"type_id\":0},\"created\":1613021404000,\"creator\":\"Creator 1\",\"creator_process\":\"Cybox-Files-Creator_Process 1\",\"desc\":\"Cybox-Files-Desc 1\",\"folder\":\"c:\\\\windows\\\\system32\\\\cybox\\\\files\\\\folder\\\\1\",\"folder_uid\":\"Cybox-Files-Folder_UID 1\",\"is_system\":true,\"md5\":\"HFDajsdf3254345436\",\"mime_type\":\"Cybox-Files-MIME_Type 1\",\"modified\":1613021404000,\"modifier\":\"Cybox-Files-Modifier 1\",\"name\":\"cybox_files_name_1.exe\",\"normalized_path\":\"CSIDL_SYSTEM\\\\cybox_files_normalized_path_1.exe\",\"original_name\":\"Cybox-Files-Original_Name 1\",\"owner\":\"Cybox-Files-Owner 1\",\"parent_name\":\"Cybox-Files-Parent_Name 1\",\"parent_sha2\":\"Cybox-Files-Parent_SHA2 1\",\"path\":\"c:\\\\windows\\\\system32\\\\cybox_files_path_1.exe\",\"product_name\":\"Windows Internet Explorer 1\",\"product_path\":\"Cybox-Files-Product_Path 1\",\"rep_discovered_band\":0,\"rep_discovered_date\":1613021404000,\"rep_prevalence\":12345678901,\"rep_prevalence_band\":0,\"rep_score\":12345678901,\"rep_score_band\":0,\"security_descriptor\":\"Cybox-Files-Security_Descriptor 1\",\"sha1\":\"Cybox-Files-SHA1 1\",\"sha2\":\"Cybox-Files-SHA2 1\",\"signature_company_name\":\"Cybox-Files-Signature_Company_Name 1\",\"signature_created_date\":1613021404000,\"signature_developer_uid\":\"Cybox-Files-Signature_Developer_UID 1\",\"signature_fingerprints\":[{\"algorithm\":\"Cybox-Files-Signature_Fingerprints-Algorithm 1\",\"value\":\"Cybox-Files-Signature_Fingerprints-Value 1\"},{\"algorithm\":\"Cybox-Files-Signature_Fingerprints-Algorithms 1\",\"value\":\"Cybox-Files-Signature_Fingerprints-Values 1\"}],\"signature_issuer\":\"Cybox-Files-Signature_Issuer 1\",\"signature_level_id\":0,\"signature_serial_number\":\"Cybox-Files-Signature_Serial_Number 1\",\"signature_value\":12345678901,\"signature_value_ids\":[0,1,2,3,4,5,6,7,8,9,10],\"size\":12345678901,\"size_compressed\":12345678901,\"src_ip\":\"81.2.69.142\",\"src_name\":\"Cybox-Files-SRC_Name 1\",\"type_id\":1,\"uid\":\"Cybox-Files-UID 1\",\"url\":{\"categories\":[\"Cybox-Files-URL-Category 1\",\"Cybox-Files-URL-Categories 1\"],\"category_ids\":[1,3,4,5,6,7,9,11,14,15,16,17,18,20,21,22,23,24,25,26,27,29,30,31,32,33,34,35,36,37,38,40,43,44,45,46,47,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,65,66,67,68,71,83,84,85,86,87,88,89,90,92,93,95,96,97,98],\"extension\":\"Cybox-Files-URL-Extension 1\",\"host\":\"www.files-url-host-1.com\",\"method\":\"Cybox-Files-URL-Method 1\",\"parent_categories\":[\"Cybox-Files-URL-Parent_Category 1\",\"Cybox-Files-URL-Parent_Categories 1\"],\"path\":\"/download/trouble/cybox/files/url/path/1\",\"port\":80,\"provider\":\"Cybox-Files-URL-Provider 1\",\"query\":\"q=bad&sort=date_1\",\"referrer\":\"Cybox-Files-URL-Referrer 1\",\"referrer_categories\":[\"Cybox-Files-URL-Referrer_Category 1\",\"Cybox-Files-URL-Referrer_Categories 1\"],\"referrer_category_ids\":[12345678901,67890123451],\"rep_score_id\":0,\"scheme\":\"Cybox-Files-URL-Scheme 1\",\"text\":\"www.files-url-text-1.com/download/trouble\"},\"version\":\"Cybox-Files-Version 1\",\"xattributes\":{\"ads_name\":\"Cybox-Files-XAttributes-ADS_Name 1\",\"ads_size\":\"Cybox-Files-XAttributes-ADS_Size 1\",\"dacl\":\"Cybox-Files-XAttributes-DACL 1\",\"owner\":\"Cybox-Files-XAttributes-Owner 1\",\"primary_group\":\"Cybox-Files-XAttributes-Primary_Group 1\",\"link_name\":\"Cybox-Files-XAttributes-Link_Name 1\",\"hard_link_count\":\"Cybox-Files-XAttributes-Hard_Link_Count 1\",\"Unix_permissions\":\"Cybox-Files-XAttributes-Unix_Permissions 1\"}},{\"accessed\":1613021404000,\"accessor\":\"Cybox-Files-Accessor 2\",\"attribute_ids\":[11,12,13,14,15,16,17],\"attributes\":12345678902,\"company_name\":\"Microsoft Corporation 2\",\"confidentiality_id\":1,\"content_type\":{\"family_id\":1,\"subtype\":\"Cybox-Files-Content_Type-SubType 2\",\"type_id\":1},\"created\":1613021404000,\"creator\":\"Cybox-Files-Creator 2\",\"creator_process\":\"Cybox-Files-Creator_Process 2\",\"desc\":\"Cybox-Files-Desc 2\",\"folder\":\"c:\\\\windows\\\\system32\\\\cybox\\\\files\\\\folder\\\\2\",\"folder_uid\":\"Cybox-Files-Folder_UID 2\",\"is_system\":true,\"md5\":\"Cybox-Files-MD5 2\",\"mime_type\":\"Cybox-Files-MIME_Type 2\",\"modified\":1613021404000,\"modifier\":\"Cybox-Files-Modifier 2\",\"name\":\"cybox_files_name_2.exe\",\"normalized_path\":\"CSIDL_SYSTEM\\\\cybox_files_normalized_path_2.exe\",\"original_name\":\"Cybox-Files-Original_Name 2\",\"owner\":\"Cybox-Files-Owner 2\",\"parent_name\":\"Cybox-Files-Parent_Name 2\",\"parent_sha2\":\"Cybox-Files-Parent_SHA2 2\",\"path\":\"c:\\\\windows\\\\system32\\\\cybox_files_path_2.exe\",\"product_name\":\"Windows Internet Explorer 2\",\"product_path\":\"Cybox-Files-Product_Path 2\",\"rep_discovered_band\":1,\"rep_discovered_date\":1613021404000,\"rep_prevalence\":12345678902,\"rep_prevalence_band\":1,\"rep_score\":12345678902,\"rep_score_band\":1,\"security_descriptor\":\"Cybox-Files-Security_Descriptor 2\",\"sha1\":\"Cybox-Files-SHA1 2\",\"sha2\":\"Cybox-Files-SHA2 2\",\"signature_company_name\":\"Cybox-Files-Signature_Company_Name 2\",\"signature_created_date\":1613021404000,\"signature_developer_uid\":\"Cybox-Files-Signature_Developer_UID 2\",\"signature_fingerprints\":[{\"algorithm\":\"Cybox-Files-Signature_Fingerprints-Algorithm 2\",\"value\":\"Cybox-Files-Signature_Fingerprints-Value 2\"},{\"algorithm\":\"Cybox-Files-Signature_Fingerprints-Algorithms 2\",\"value\":\"Cybox-Files-Signature_Fingerprints-Values 2\"}],\"signature_issuer\":\"Cybox-Files-Signature_Issuer 2\",\"signature_level_id\":1,\"signature_serial_number\":\"Cybox-Files-Signature_Serial_Number 2\",\"signature_value\":12345678902,\"signature_value_ids\":[11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],\"size\":12345678902,\"size_compressed\":12345678902,\"src_ip\":\"81.2.69.144\",\"src_name\":\"Cybox-Files-SRC_Name 2\",\"type_id\":1,\"uid\":\"Cybox-Files-UID 2\",\"url\":{\"categories\":[\"Cybox-Files-URL-Category 2\",\"Cybox-Files-URL-Categories 2\"],\"category_ids\":[101,102,103,104,105,106,107,108,109,110,111,112,113,114,116,117,118,121,124],\"extension\":\"Cybox-Files-URL-Extension 2\",\"host\":\"www.files-url-host-2.com\",\"method\":\"Cybox-Files-URL-Method 2\",\"parent_categories\":[\"Cybox-Files-URL-Parent_Category 2\",\"Cybox-Files-URL-Parent_Categories 2\"],\"path\":\"/download/trouble/cybox/files/url/path/2\",\"port\":81,\"provider\":\"Cybox-Files-URL-Provider 2\",\"query\":\"q=bad&sort=date_2\",\"referrer\":\"Cybox-Files-URL-Referrer 2\",\"referrer_categories\":[\"Cybox-Files-URL-Referrer_Category 2\",\"Cybox-Files-URL-Referrer_Categories 2\"],\"referrer_category_ids\":[12345678902,67890123452],\"rep_score_id\":1,\"scheme\":\"Cybox-Files-URL-Scheme 2\",\"text\":\"www.files-url-text-2.com/download/trouble\"},\"version\":\"Cybox-Files-Version 2\",\"xattributes\":{\"ads_name\":\"Cybox-Files-XAttributes-ADS_Name 2\",\"ads_size\":\"Cybox-Files-XAttributes-ADS_Size 2\",\"dacl\":\"Cybox-Files-XAttributes-DACL 2\",\"owner\":\"Cybox-Files-XAttributes-Owner 2\",\"primary_group\":\"Cybox-Files-XAttributes-Primary_Group 2\",\"link_name\":\"Cybox-Files-XAttributes-Link_Name 2\",\"hard_link_count\":\"Cybox-Files-XAttributes-Hard_Link_Count 2\",\"Unix_permissions\":\"Cybox-Files-XAttributes-Unix_Permissions 2\"}}],\"hostnames\":[\"Cybox-Hostname 1\",\"Cybox-Hostnames 1\"],\"icap_reqmod\":[{\"metadata\":{\"field1_keyword\":\"Cybox-ICAP_ReqMod-field1_Keyword\",\"field1_number\":12345678901,\"field1_boolean\":true,\"field1_ip\":\"175.16.199.0\"},\"service\":\"Cybox-ICAP_ReqMod-Service 1\",\"status\":\"Cybox-ICAP_ReqMod-Status 1\",\"status_detail\":\"Cybox-ICAP_ReqMod-Status_Detail 1\"},{\"metadata\":{\"field2_keyword\":\"Cybox-ICAP_ReqMod-field2_Keyword\",\"field2_number\":12345678902,\"field2_boolean\":true,\"field2_ip\":\"175.16.199.0\"},\"service\":\"Cybox-ICAP_ReqMod-Service 2\",\"status\":\"Cybox-ICAP_ReqMod-Status 2\",\"status_detail\":\"Cybox-ICAP_ReqMod-Status_Detail 2\"}],\"icap_respmod\":[{\"metadata\":{\"field1_keyword\":\"Cybox-ICAP_RespMod-field1_Keyword\",\"field1_number\":12345678901,\"field1_boolean\":true,\"field1_ip\":\"175.16.199.0\"},\"service\":\"Cybox-ICAP_RespMod-Service 1\",\"status\":\"Cybox-ICAP_RespMod-Status 1\",\"status_detail\":\"Cybox-ICAP_RespMod-Status_Detail 1\"},{\"metadata\":{\"field2_keyword\":\"Cybox-ICAP_RespMod-field2_Keyword\",\"field2_number\":12345678902,\"field2_boolean\":true,\"field2_ip\":\"175.16.199.0\"},\"service\":\"Cybox-ICAP_RespMod-Service 2\",\"status\":\"Cybox-ICAP_RespMod-Status 2\",\"status_detail\":\"Cybox-ICAP_RespMod-Status_Detail 2\"}],\"ipv4s\":[\"175.16.199.0\",\"175.16.199.0\"],\"ipv6s\":[\"2a02:cf40::\",\"2a02:cf40::\"],\"macs\":[\"00:B0:D0:63:C2:05\",\"00:B0:D0:63:C2:06\"],\"urls\":[{\"categories\":[\"Cybox-URLs-Category 1\",\"Cybox-URLs-Categories 1\"],\"category_ids\":[1,3,4,5,6,7,9,11,14,15,16,17,18,20,21,22,23,24,25,26,27,29,30,31,32,33,34,35,36,37,38,40,43,44,45,46,47,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,65,66,67,68,71,83,84,85,86,87,88,89,90,92,93,95,96,97,98],\"extension\":\"Cybox-URLs-Extension 1\",\"host\":\"www.urls-host-1.com\",\"method\":\"Cybox-URLs-Method 1\",\"parent_categories\":[\"Cybox-URLs-Parent_Category 1\",\"Cybox-URLs-Parent_Categories 1\"],\"path\":\"/download/trouble/cybox/urls/path/1\",\"port\":80,\"provider\":\"Cybox-URLs-Provider 1\",\"query\":\"q=bad&sort=date_1\",\"referrer\":\"Cybox-URLs-Referrer 1\",\"referrer_categories\":[\"Cybox-URLs-Referrer_Category 1\",\"Cybox-URLs-Referrer_Categories 1\"],\"referrer_category_ids\":[12345678901,67890123451],\"rep_score_id\":0,\"scheme\":\"Cybox-URLs-Scheme 1\",\"text\":\"www.urls-text-1.com/download/trouble\"},{\"categories\":[\"Cybox-URLs-Category 2\",\"Cybox-URLs-Categories 2\"],\"category_ids\":[101,102,103,104,105,106,107,108,109,110,111,112,113,114,116,117,118,121,124],\"extension\":\"Cybox-URLs-Extension 2\",\"host\":\"www.urls-host-2.com\",\"method\":\"Cybox-URLs-Method 2\",\"parent_categories\":[\"Cybox-URLs-Parent_Category 2\",\"Cybox-URLs-Parent_Categories 2\"],\"path\":\"/download/trouble/cybox/urls/path/2\",\"port\":81,\"provider\":\"Cybox-URLs-Provider 2\",\"query\":\"q=bad&sort=date_2\",\"referrer\":\"Cybox-URLs-Referrer 2\",\"referrer_categories\":[\"Cybox-URLs-Referrer_Category 2\",\"Cybox-URLs-Referrer_Categories 2\"],\"referrer_category_ids\":[12345678902,67890123452],\"rep_score_id\":1,\"scheme\":\"Cybox-URLs-Scheme 2\",\"text\":\"www.urls-text-2.com/download/trouble\"}]},\"device_alias_name\":\"Device_Alias_Name\",\"device_cap\":\"Device_Cap\",\"device_cloud_vm\":{\"autoscale_uid\":\"Device_Cloud_VM-Autoscale_UID\",\"dc_region\":\"Device_Cloud_VM-DC_Region\",\"instance_uid\":\"Device_Cloud_VM-Instance_UID\",\"subnet_uid\":\"Device_Cloud_VM-Subnet_UID\",\"vpc_uid\":\"Device_Cloud_VM-VPC_UID\"},\"device_desc\":\"Device_Desc\",\"device_domain\":\"device.domain.internal.somecompany.com\",\"device_domain_uid\":\"Device_Domain_UID\",\"device_end_time\":1613021404000,\"device_gateway\":\"175.16.199.0\",\"device_group\":\"Device_Group\",\"device_group_name\":\"Device_Group_Name\",\"device_hw_bios_date\":\"03/31/16\",\"device_hw_bios_manufacturer\":\"LENOVO\",\"device_hw_bios_ver\":\"LENOVO G5ETA2WW (2.62)\",\"device_hw_cpu_type\":\"x86 Family 6 Model 37 Stepping 5\",\"device_imei\":\"Device_IMEI\",\"device_ip\":\"175.16.199.0\",\"device_is_compliant\":true,\"device_is_personal\":true,\"device_is_trusted\":true,\"device_is_unmanaged\":true,\"device_location\":{\"city\":\"Device_Location-City\",\"continent\":\"Device_Location-Continent\",\"coordinates\":[-12.345,56.789],\"country\":\"US\",\"desc\":\"Device_Location-Desc\",\"isp\":\"Device_Location-ISP\",\"on_premises\":true,\"region\":\"US-CA\"},\"device_mac\":\"00:B0:D0:63:C2:07\",\"device_name\":\"device.name.computer.domain\",\"device_name_md5\":\"4ED962DDBF17E2BBA7B14EBC00F3162E\",\"device_networks\":[{\"bssid\":\"Device_Networks-BSSID 1\",\"gateway_ip\":\"175.16.199.0\",\"gateway_mac\":\"00:B0:D0:63:C2:08\",\"ipv4\":\"175.16.199.0\",\"ipv6\":\"2a02:cf40::\",\"is_public\":true,\"mac\":\"00:B0:D0:63:C2:09\",\"rep_score_id\":0,\"ssid\":\"Device_Networks-SSID 1\",\"type_id\":0},{\"bssid\":\"Device_Networks-BSSID 2\",\"gateway_ip\":\"89.160.20.112\",\"gateway_mac\":\"00:B0:D0:63:C2:10\",\"ipv4\":\"89.160.20.112\",\"ipv6\":\"2a02:cf40::\",\"is_public\":true,\"mac\":\"00:B0:D0:63:C2:11\",\"rep_score_id\":1,\"ssid\":\"Device_Networks-SSID 2\",\"type_id\":1}],\"device_org_unit\":\"Device_Org_Unit\",\"device_os_bits\":12345678901,\"device_os_build\":\"Device_OS_Build\",\"device_os_country\":\"IN\",\"device_os_edition\":\"Professional\",\"device_os_lang\":\"en\",\"device_os_name\":\"Windows Server 2019 Standard Edition\",\"device_os_sp_name\":\"Device_OS_SP_Name\",\"device_os_sp_ver\":\"Device_OS_SP_Ver\",\"device_os_type_id\":0,\"device_os_ver\":\"Windows 10\",\"device_proxy_ip\":\"89.160.20.112\",\"device_proxy_name\":\"Device_Proxy_Name\",\"device_public_ip\":\"89.160.20.112\",\"device_ref_uid\":\"Device_Ref_UID\",\"device_site\":\"Device_Site\",\"device_subnet\":\"81.2.69.144\",\"device_time\":1613021404000,\"device_type\":\"server\",\"device_uid\":\"Device_UID\",\"device_vhost\":\"Device_VHost\",\"device_vhost_id\":0,\"domain_uid\":\"Domain_UID\",\"end_time\":\"2024-02-29T01:00:00.000Z\",\"entity\":{\"data\":{\"field1_keyword\":\"Entity-Data-field1_Keyword\",\"field1_number\":12345678901,\"field1_boolean\":true},\"name\":\"Entity-Name\",\"type\":\"Entity-Type\",\"uid\":\"Entity-UID\",\"version\":\"Entity-Version\"},\"event_id\":2001,\"events\":[{\"connection\":{\"direction_id\":1,\"dst_service\":\"C:\\\\Windows\\\\system32\\\\NTOSKRNL.EXE\",\"src_ip\":\"159.19.163.218\"},\"count\":1,\"device_end_time\":1709225074618,\"device_time\":1709225074618}],\"feature_name\":\"Feature_Name\",\"feature_path\":\"Feature_Path\",\"feature_type\":\"Feature_Type\",\"feature_uid\":\"Feature_UID\",\"feature_ver\":\"2014.1.4.25\",\"id\":12345678901,\"impersonator_customer_uid\":\"Impersonator_Customer_UID\",\"impersonator_domain_uid\":\"Impersonator_Domain_UID\",\"impersonator_user_uid\":\"Impersonator_User_UID\",\"is_user_present\":true,\"log_level\":\"Log Level\",\"log_name\":\"Log_Name\",\"log_time\":\"2024-02-29T01:00:00.000Z\",\"logging_device_ip\":\"89.160.20.112\",\"logging_device_name\":\"Logging_Device_Name\",\"logging_device_post_time\":1613021404000,\"logging_device_ref_uid\":\"Logging_Device_Ref_UID\",\"message\":\"Message\",\"message_code\":\"Message_Code\",\"message_id\":0,\"org_unit_uid\":\"Org_Unit_UID\",\"orig_data\":\"Orig_Data\",\"product_data\":{\"sep_domain_uid\":\"Product_Data-Sep_Domain_UID\",\"sep_hw_uid\":\"Product_Data-Sep_HW_UID\"},\"product_lang\":\"en\",\"product_name\":\"Symantec Endpoint Security\",\"product_uid\":\"Product_UID\",\"product_ver\":\"2014.1.4.25-beta\",\"proxy_device_ip\":\"89.160.20.112\",\"proxy_device_name\":\"Proxy_Device_Name\",\"raw_data\":{\"assetID\":\"vc9DagprQYyLZ23SEY1APw\",\"assetOpstateDTO\":{\"productUuid\":\"31B0C880-0229-49E8-94C5-48D56B1BD7B9\",\"features\":[{\"uuid\":\"1DF0351C-146D-4F07-B155-BF5C7077FF40\",\"featureStatus\":\"SECURE\",\"opstate\":{\"EDRContentSequence\":\"20231128005\",\"EDREngineVersion\":\"4.11.0.10\",\"EDRFramworkVersion\":\"4.10.0.59\",\"FDRStatus\":true,\"LowDiskSpace\":false,\"MaxDBSizeHonored\":true,\"applied_policy\":{\"effective_date\":1709219437080,\"sha2\":\"ee6b0bebbc4575b507ac616d2c362f2c54d462b92cf4068cb6681ae3187d4de3\",\"uid\":\"7dc29d40-f303-477a-9012-287ef252a391\",\"version\":\"16\"},\"disk_usage_mb\":1546,\"fdr_first_event_date\":\"20240227\",\"fdr_state\":1},\"state\":\"ENABLED\",\"statusReason\":[\"-107\",\"0\"],\"prevention_state\":\"1\"}],\"products_active\":0,\"blades\":0}},\"ref_log_name\":\"Ref_Log_Name\",\"ref_log_time\":\"2024-02-29T01:00:00.000Z\",\"ref_orig_uid\":\"Ref_Orig_UID\",\"ref_uid\":\"Ref_UID\",\"remediated\":true,\"remediation\":\"Remediation\",\"remediation_ref\":\"Remediation_Ref\",\"remediation_uid\":0,\"seq_num\":12345678901,\"sessions\":[{\"auth_protocol_id\":0,\"cleartext_credentials\":true,\"direction_id\":0,\"id\":12345678901,\"is_admin\":true,\"logon_type_id\":1,\"port\":80,\"previous_users\":[\"Sessions-Previous_User 1\",\"Sessions-Previous_Users 1\"],\"remote\":true,\"remote_host\":\"Sessions-Remote_Host 1\",\"remote_ip\":\"89.160.20.112\",\"user\":{\"account_disabled\":true,\"cloud_resource_uid\":\"Sessions-User-Cloud_Resource_UID 1\",\"domain\":\"Sessions-User-Domain 1\",\"external_account_uid\":\"Sessions-User-External_Account_UID 1\",\"external_uid\":\"Sessions-User-External_UID 1\",\"full_name\":\"Sessions-User-Full_Name 1\",\"groups\":[\"Sessions-User-Group 1\",\"Sessions-User-Groups 1\"],\"home\":\"Sessions-User-Home 1\",\"is_admin\":true,\"logon_name\":\"Sessions-User-Logon_Name 1\",\"name\":\"session-User-Name 1\",\"password_expires\":true,\"shell\":\"Sessions-User-Shell 1\",\"sid\":\"Sessions-User-SID 1\",\"uid\":\"Sessions-User-UID 1\"}},{\"auth_protocol_id\":1,\"cleartext_credentials\":true,\"direction_id\":1,\"id\":67890123451,\"is_admin\":true,\"logon_type_id\":2,\"port\":81,\"previous_users\":[\"Sessions-Previous_User 2\",\"Sessions-Previous_Users 2\"],\"remote\":true,\"remote_host\":\"Sessions-Remote_Host 2\",\"remote_ip\":\"89.160.20.112\",\"user\":{\"account_disabled\":true,\"cloud_resource_uid\":\"Sessions-User-Cloud_Resource_UID 2\",\"domain\":\"Sessions-User-Domain 2\",\"external_account_uid\":\"Sessions-User-External_Account_UID 2\",\"external_uid\":\"Sessions-User-External_UID 2\",\"full_name\":\"Sessions-User-Full_Name 2\",\"groups\":[\"Sessions-User-Group 2\",\"Sessions-User-Groups 2\"],\"home\":\"Sessions-User-Home 2\",\"is_admin\":true,\"logon_name\":\"Sessions-User-Logon_Name 2\",\"name\":\"session-User-Name 2\",\"password_expires\":true,\"shell\":\"Sessions-User-Shell 2\",\"sid\":\"Sessions-User-SID 2\",\"uid\":\"Sessions-User-UID 2\"}}],\"severity_id\":0,\"source\":{\"facility\":\"Source-Facility\",\"facility_detail\":\"Source-Facility_Detail\",\"facility_uid\":\"Source-Facility_UID\",\"type_id\":1},\"status_detail\":\"Status_Detail\",\"status_id\":0,\"status_os\":\"Status_OS\",\"status_os_src\":12345678901,\"status_stack_trace\":\"Status_Stack_Trace\",\"status_thread_name\":\"Status_Thread_Name\",\"stic_has_pii\":true,\"stic_hw_uid\":\"STIC_HW_UID\",\"stic_ip_hash\":\"STIC_IP_Hash\",\"stic_legacy_ent_uids\":[\"STIC_Legacy_Ent_UIDs 1\",\"STIC_Legacy_Ent_UIDs 2\"],\"stic_legacy_hw_uids\":[\"STIC_Legacy_HW_UIDs 1\",\"STIC_Legacy_HW_UIDs 2\"],\"stic_legacy_uids\":[\"STIC_Legacy_UIDs 1\",\"STIC_Legacy_UIDs 2\"],\"stic_schema_id\":\"STIC_Schema_ID\",\"stic_uid\":\"STIC_UID\",\"stic_version\":\"STIC_Version\",\"subfeature_name\":\"Subfeature_Name\",\"time\":\"2024-02-29T02:00:00Z\",\"timezone\":12345678901,\"type\":\"Type\",\"type_id\":2,\"user\":{\"account_disabled\":true,\"cloud_resource_uid\":\"User-Cloud_Resource_UID\",\"domain\":\"User-Domain\",\"external_account_uid\":\"User-External_Account_UID\",\"external_uid\":\"User-External_UID\",\"full_name\":\"User-Full_Name\",\"groups\":[\"User-Group 1\",\"User-Groups 1\"],\"home\":\"User-Home\",\"is_admin\":true,\"logon_name\":\"User-Logon_Name\",\"name\":\"User123\",\"password_expires\":true,\"shell\":\"User-Shell\",\"sid\":\"TT23009\",\"uid\":\"UU34899825\"},\"user_name\":\"Mohit\",\"user_uid\":\"AB45698\",\"uuid\":\"SR-1565234545\",\"version\":\"1.4\"}", "sequence": [ 12345678901 ], @@ -158,6 +146,17 @@ ] } }, + "gcs": { + "storage": { + "bucket": { + "name": "testbucket" + }, + "object": { + "content_type": "application/x-ndjson", + "name": "test-events.log" + } + } + }, "group": { "name": "Device_Group" }, @@ -190,11 +189,11 @@ ] }, "input": { - "type": "aws-s3" + "type": "gcs" }, "log": { "file": { - "path": "https://elastic-package-symantec-endpoint-security-bucket-96277.s3.us-east-1.amazonaws.com/events.log" + "path": "gs://testbucket/test-events.log" }, "level": [ "Log Level" @@ -1163,8 +1162,6 @@ "mac": "00-B0-D0-63-C2-07" }, "tags": [ - "collect_sqs_logs", - "preserve_original_event", "forwarded", "symantec_endpoint_security-event" ], diff --git a/packages/symantec_endpoint_security/docs/README.md b/packages/symantec_endpoint_security/docs/README.md index f01d75c6bfe..f4b873ffc80 100644 --- a/packages/symantec_endpoint_security/docs/README.md +++ b/packages/symantec_endpoint_security/docs/README.md @@ -229,22 +229,11 @@ An example event for `event` looks as following: { "@timestamp": "2024-02-29T02:00:00.000Z", "agent": { - "ephemeral_id": "1e13efa5-b9f8-46fa-a694-2c68a106525f", - "id": "ed4de143-3b92-4c05-89dc-dd1b6574fd80", - "name": "elastic-agent-95953", + "ephemeral_id": "2979533b-cdb6-49c0-8665-3223e6922f90", + "id": "68d148de-0c02-47af-b9dd-210b7682ece0", + "name": "elastic-agent-86709", "type": "filebeat", - "version": "8.13.0" - }, - "aws": { - "s3": { - "bucket": { - "arn": "arn:aws:s3:::elastic-package-symantec-endpoint-security-bucket-96277", - "name": "elastic-package-symantec-endpoint-security-bucket-96277" - }, - "object": { - "key": "events.log" - } - } + "version": "8.16.5" }, "client": { "domain": "device.domain.internal.somecompany.com", @@ -253,11 +242,11 @@ An example event for `event` looks as following: } }, "cloud": { - "region": "us-east-1" + "provider": "google cloud" }, "data_stream": { "dataset": "symantec_endpoint_security.event", - "namespace": "34134", + "namespace": "56890", "type": "logs" }, "destination": { @@ -275,9 +264,9 @@ An example event for `event` looks as following: "version": "8.11.0" }, "elastic_agent": { - "id": "ed4de143-3b92-4c05-89dc-dd1b6574fd80", + "id": "68d148de-0c02-47af-b9dd-210b7682ece0", "snapshot": false, - "version": "8.13.0" + "version": "8.16.5" }, "email": { "direction": [ @@ -310,10 +299,9 @@ An example event for `event` looks as following: "2021-02-11T05:30:04.000Z" ], "id": "SR-1565234545", - "ingested": "2025-04-24T12:35:04Z", + "ingested": "2025-10-28T11:45:21Z", "kind": "event", "module": "Feature_Name", - "original": "{\"category_id\":3,\"collector_device_ip\":\"175.16.199.0\",\"collector_device_name\":\"Collector_Device_Name\",\"collector_name\":\"Collection12\",\"collector_uid\":\"TT1456\",\"composite\":1,\"container\":{\"host_name\":\"azure-us.local\",\"image_name\":\"Image-sp2133\",\"image_uid\":\"SH4322323\",\"name\":\"User12345\",\"networks\":[{\"bssid\":\"Container-Networks-BSSID 1\",\"gateway_ip\":\"89.160.20.112\",\"gateway_mac\":\"00:B0:D0:63:C2:01\",\"ipv4\":\"81.2.69.144\",\"ipv6\":\"2a02:cf40::\",\"is_public\":true,\"mac\":\"00:B0:D0:63:C2:02\",\"rep_score_id\":0,\"ssid\":\"SSID-4326451\",\"type_id\":0},{\"bssid\":\"HN0845435\",\"gateway_ip\":\"81.2.69.142\",\"gateway_mac\":\"00:B0:D0:63:C2:03\",\"ipv4\":\"81.2.69.144\",\"ipv6\":\"2a02:cf40::\",\"is_public\":true,\"mac\":\"00:B0:D0:63:C2:04\",\"rep_score_id\":1,\"ssid\":\"Container-Networks-SSID 2\",\"type_id\":1}],\"os_name\":\"Windows\",\"uid\":\"UU35r3454\"},\"correlation_uid\":\"DD78666\",\"count\":563,\"customer_registry_uid\":\"CP45254\",\"customer_uid\":\"CH32534\",\"cybox\":{\"domains\":[\"example.com\",\"abc.com\"],\"emails\":[{\"direction_id\":0,\"header_from\":\"abc@example.com\",\"header_message_id\":\"Cybox-Emails-Header_Message_ID 1\",\"header_reply_to\":\"Cybox-Emails-Header_Reply_To 1\",\"header_subject\":\"Cybox-Emails-Header_Subject 1\",\"header_to\":[\"Cybox-Emails-Header_To 1\",\"Cybox-Emails-Header_Tos 1\"],\"sender_ip\":\"81.2.69.144\",\"size\":12345678901,\"smtp_from\":\"Cybox-Emails-SMTP_From 1\",\"smtp_hello\":\"Cybox-Emails-SMTP_Hello 1\",\"smtp_to\":\"Cybox-Emails-SMTP_To 1\"},{\"direction_id\":1,\"header_from\":\"abc@example.com\",\"header_message_id\":\"Cybox-Emails-Header_Message_ID 2\",\"header_reply_to\":\"Cybox-Emails-Header_Reply_To 2\",\"header_subject\":\"Cybox-Emails-Header_Subject 2\",\"header_to\":[\"Cybox-Emails-Header_To 2\",\"Cybox-Emails-Header_Tos 2\"],\"sender_ip\":\"81.2.69.144\",\"size\":12345678902,\"smtp_from\":\"Cybox-Emails-SMTP_From 2\",\"smtp_hello\":\"Cybox-Emails-SMTP_Hello 2\",\"smtp_to\":\"Cybox-Emails-SMTP_To 2\"}],\"files\":[{\"accessed\":1613021404000,\"accessor\":\"Cybox-Files-Accessor 1\",\"attribute_ids\":[1,2,3,4,5,6,7,8,9,10],\"attributes\":12345678901,\"company_name\":\"Microsoft Corporation\",\"confidentiality_id\":0,\"content_type\":{\"family_id\":0,\"subtype\":\"SubType 1\",\"type_id\":0},\"created\":1613021404000,\"creator\":\"Creator 1\",\"creator_process\":\"Cybox-Files-Creator_Process 1\",\"desc\":\"Cybox-Files-Desc 1\",\"folder\":\"c:\\\\windows\\\\system32\\\\cybox\\\\files\\\\folder\\\\1\",\"folder_uid\":\"Cybox-Files-Folder_UID 1\",\"is_system\":true,\"md5\":\"HFDajsdf3254345436\",\"mime_type\":\"Cybox-Files-MIME_Type 1\",\"modified\":1613021404000,\"modifier\":\"Cybox-Files-Modifier 1\",\"name\":\"cybox_files_name_1.exe\",\"normalized_path\":\"CSIDL_SYSTEM\\\\cybox_files_normalized_path_1.exe\",\"original_name\":\"Cybox-Files-Original_Name 1\",\"owner\":\"Cybox-Files-Owner 1\",\"parent_name\":\"Cybox-Files-Parent_Name 1\",\"parent_sha2\":\"Cybox-Files-Parent_SHA2 1\",\"path\":\"c:\\\\windows\\\\system32\\\\cybox_files_path_1.exe\",\"product_name\":\"Windows Internet Explorer 1\",\"product_path\":\"Cybox-Files-Product_Path 1\",\"rep_discovered_band\":0,\"rep_discovered_date\":1613021404000,\"rep_prevalence\":12345678901,\"rep_prevalence_band\":0,\"rep_score\":12345678901,\"rep_score_band\":0,\"security_descriptor\":\"Cybox-Files-Security_Descriptor 1\",\"sha1\":\"Cybox-Files-SHA1 1\",\"sha2\":\"Cybox-Files-SHA2 1\",\"signature_company_name\":\"Cybox-Files-Signature_Company_Name 1\",\"signature_created_date\":1613021404000,\"signature_developer_uid\":\"Cybox-Files-Signature_Developer_UID 1\",\"signature_fingerprints\":[{\"algorithm\":\"Cybox-Files-Signature_Fingerprints-Algorithm 1\",\"value\":\"Cybox-Files-Signature_Fingerprints-Value 1\"},{\"algorithm\":\"Cybox-Files-Signature_Fingerprints-Algorithms 1\",\"value\":\"Cybox-Files-Signature_Fingerprints-Values 1\"}],\"signature_issuer\":\"Cybox-Files-Signature_Issuer 1\",\"signature_level_id\":0,\"signature_serial_number\":\"Cybox-Files-Signature_Serial_Number 1\",\"signature_value\":12345678901,\"signature_value_ids\":[0,1,2,3,4,5,6,7,8,9,10],\"size\":12345678901,\"size_compressed\":12345678901,\"src_ip\":\"81.2.69.142\",\"src_name\":\"Cybox-Files-SRC_Name 1\",\"type_id\":1,\"uid\":\"Cybox-Files-UID 1\",\"url\":{\"categories\":[\"Cybox-Files-URL-Category 1\",\"Cybox-Files-URL-Categories 1\"],\"category_ids\":[1,3,4,5,6,7,9,11,14,15,16,17,18,20,21,22,23,24,25,26,27,29,30,31,32,33,34,35,36,37,38,40,43,44,45,46,47,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,65,66,67,68,71,83,84,85,86,87,88,89,90,92,93,95,96,97,98],\"extension\":\"Cybox-Files-URL-Extension 1\",\"host\":\"www.files-url-host-1.com\",\"method\":\"Cybox-Files-URL-Method 1\",\"parent_categories\":[\"Cybox-Files-URL-Parent_Category 1\",\"Cybox-Files-URL-Parent_Categories 1\"],\"path\":\"/download/trouble/cybox/files/url/path/1\",\"port\":80,\"provider\":\"Cybox-Files-URL-Provider 1\",\"query\":\"q=bad&sort=date_1\",\"referrer\":\"Cybox-Files-URL-Referrer 1\",\"referrer_categories\":[\"Cybox-Files-URL-Referrer_Category 1\",\"Cybox-Files-URL-Referrer_Categories 1\"],\"referrer_category_ids\":[12345678901,67890123451],\"rep_score_id\":0,\"scheme\":\"Cybox-Files-URL-Scheme 1\",\"text\":\"www.files-url-text-1.com/download/trouble\"},\"version\":\"Cybox-Files-Version 1\",\"xattributes\":{\"ads_name\":\"Cybox-Files-XAttributes-ADS_Name 1\",\"ads_size\":\"Cybox-Files-XAttributes-ADS_Size 1\",\"dacl\":\"Cybox-Files-XAttributes-DACL 1\",\"owner\":\"Cybox-Files-XAttributes-Owner 1\",\"primary_group\":\"Cybox-Files-XAttributes-Primary_Group 1\",\"link_name\":\"Cybox-Files-XAttributes-Link_Name 1\",\"hard_link_count\":\"Cybox-Files-XAttributes-Hard_Link_Count 1\",\"Unix_permissions\":\"Cybox-Files-XAttributes-Unix_Permissions 1\"}},{\"accessed\":1613021404000,\"accessor\":\"Cybox-Files-Accessor 2\",\"attribute_ids\":[11,12,13,14,15,16,17],\"attributes\":12345678902,\"company_name\":\"Microsoft Corporation 2\",\"confidentiality_id\":1,\"content_type\":{\"family_id\":1,\"subtype\":\"Cybox-Files-Content_Type-SubType 2\",\"type_id\":1},\"created\":1613021404000,\"creator\":\"Cybox-Files-Creator 2\",\"creator_process\":\"Cybox-Files-Creator_Process 2\",\"desc\":\"Cybox-Files-Desc 2\",\"folder\":\"c:\\\\windows\\\\system32\\\\cybox\\\\files\\\\folder\\\\2\",\"folder_uid\":\"Cybox-Files-Folder_UID 2\",\"is_system\":true,\"md5\":\"Cybox-Files-MD5 2\",\"mime_type\":\"Cybox-Files-MIME_Type 2\",\"modified\":1613021404000,\"modifier\":\"Cybox-Files-Modifier 2\",\"name\":\"cybox_files_name_2.exe\",\"normalized_path\":\"CSIDL_SYSTEM\\\\cybox_files_normalized_path_2.exe\",\"original_name\":\"Cybox-Files-Original_Name 2\",\"owner\":\"Cybox-Files-Owner 2\",\"parent_name\":\"Cybox-Files-Parent_Name 2\",\"parent_sha2\":\"Cybox-Files-Parent_SHA2 2\",\"path\":\"c:\\\\windows\\\\system32\\\\cybox_files_path_2.exe\",\"product_name\":\"Windows Internet Explorer 2\",\"product_path\":\"Cybox-Files-Product_Path 2\",\"rep_discovered_band\":1,\"rep_discovered_date\":1613021404000,\"rep_prevalence\":12345678902,\"rep_prevalence_band\":1,\"rep_score\":12345678902,\"rep_score_band\":1,\"security_descriptor\":\"Cybox-Files-Security_Descriptor 2\",\"sha1\":\"Cybox-Files-SHA1 2\",\"sha2\":\"Cybox-Files-SHA2 2\",\"signature_company_name\":\"Cybox-Files-Signature_Company_Name 2\",\"signature_created_date\":1613021404000,\"signature_developer_uid\":\"Cybox-Files-Signature_Developer_UID 2\",\"signature_fingerprints\":[{\"algorithm\":\"Cybox-Files-Signature_Fingerprints-Algorithm 2\",\"value\":\"Cybox-Files-Signature_Fingerprints-Value 2\"},{\"algorithm\":\"Cybox-Files-Signature_Fingerprints-Algorithms 2\",\"value\":\"Cybox-Files-Signature_Fingerprints-Values 2\"}],\"signature_issuer\":\"Cybox-Files-Signature_Issuer 2\",\"signature_level_id\":1,\"signature_serial_number\":\"Cybox-Files-Signature_Serial_Number 2\",\"signature_value\":12345678902,\"signature_value_ids\":[11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],\"size\":12345678902,\"size_compressed\":12345678902,\"src_ip\":\"81.2.69.144\",\"src_name\":\"Cybox-Files-SRC_Name 2\",\"type_id\":1,\"uid\":\"Cybox-Files-UID 2\",\"url\":{\"categories\":[\"Cybox-Files-URL-Category 2\",\"Cybox-Files-URL-Categories 2\"],\"category_ids\":[101,102,103,104,105,106,107,108,109,110,111,112,113,114,116,117,118,121,124],\"extension\":\"Cybox-Files-URL-Extension 2\",\"host\":\"www.files-url-host-2.com\",\"method\":\"Cybox-Files-URL-Method 2\",\"parent_categories\":[\"Cybox-Files-URL-Parent_Category 2\",\"Cybox-Files-URL-Parent_Categories 2\"],\"path\":\"/download/trouble/cybox/files/url/path/2\",\"port\":81,\"provider\":\"Cybox-Files-URL-Provider 2\",\"query\":\"q=bad&sort=date_2\",\"referrer\":\"Cybox-Files-URL-Referrer 2\",\"referrer_categories\":[\"Cybox-Files-URL-Referrer_Category 2\",\"Cybox-Files-URL-Referrer_Categories 2\"],\"referrer_category_ids\":[12345678902,67890123452],\"rep_score_id\":1,\"scheme\":\"Cybox-Files-URL-Scheme 2\",\"text\":\"www.files-url-text-2.com/download/trouble\"},\"version\":\"Cybox-Files-Version 2\",\"xattributes\":{\"ads_name\":\"Cybox-Files-XAttributes-ADS_Name 2\",\"ads_size\":\"Cybox-Files-XAttributes-ADS_Size 2\",\"dacl\":\"Cybox-Files-XAttributes-DACL 2\",\"owner\":\"Cybox-Files-XAttributes-Owner 2\",\"primary_group\":\"Cybox-Files-XAttributes-Primary_Group 2\",\"link_name\":\"Cybox-Files-XAttributes-Link_Name 2\",\"hard_link_count\":\"Cybox-Files-XAttributes-Hard_Link_Count 2\",\"Unix_permissions\":\"Cybox-Files-XAttributes-Unix_Permissions 2\"}}],\"hostnames\":[\"Cybox-Hostname 1\",\"Cybox-Hostnames 1\"],\"icap_reqmod\":[{\"metadata\":{\"field1_keyword\":\"Cybox-ICAP_ReqMod-field1_Keyword\",\"field1_number\":12345678901,\"field1_boolean\":true,\"field1_ip\":\"175.16.199.0\"},\"service\":\"Cybox-ICAP_ReqMod-Service 1\",\"status\":\"Cybox-ICAP_ReqMod-Status 1\",\"status_detail\":\"Cybox-ICAP_ReqMod-Status_Detail 1\"},{\"metadata\":{\"field2_keyword\":\"Cybox-ICAP_ReqMod-field2_Keyword\",\"field2_number\":12345678902,\"field2_boolean\":true,\"field2_ip\":\"175.16.199.0\"},\"service\":\"Cybox-ICAP_ReqMod-Service 2\",\"status\":\"Cybox-ICAP_ReqMod-Status 2\",\"status_detail\":\"Cybox-ICAP_ReqMod-Status_Detail 2\"}],\"icap_respmod\":[{\"metadata\":{\"field1_keyword\":\"Cybox-ICAP_RespMod-field1_Keyword\",\"field1_number\":12345678901,\"field1_boolean\":true,\"field1_ip\":\"175.16.199.0\"},\"service\":\"Cybox-ICAP_RespMod-Service 1\",\"status\":\"Cybox-ICAP_RespMod-Status 1\",\"status_detail\":\"Cybox-ICAP_RespMod-Status_Detail 1\"},{\"metadata\":{\"field2_keyword\":\"Cybox-ICAP_RespMod-field2_Keyword\",\"field2_number\":12345678902,\"field2_boolean\":true,\"field2_ip\":\"175.16.199.0\"},\"service\":\"Cybox-ICAP_RespMod-Service 2\",\"status\":\"Cybox-ICAP_RespMod-Status 2\",\"status_detail\":\"Cybox-ICAP_RespMod-Status_Detail 2\"}],\"ipv4s\":[\"175.16.199.0\",\"175.16.199.0\"],\"ipv6s\":[\"2a02:cf40::\",\"2a02:cf40::\"],\"macs\":[\"00:B0:D0:63:C2:05\",\"00:B0:D0:63:C2:06\"],\"urls\":[{\"categories\":[\"Cybox-URLs-Category 1\",\"Cybox-URLs-Categories 1\"],\"category_ids\":[1,3,4,5,6,7,9,11,14,15,16,17,18,20,21,22,23,24,25,26,27,29,30,31,32,33,34,35,36,37,38,40,43,44,45,46,47,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,65,66,67,68,71,83,84,85,86,87,88,89,90,92,93,95,96,97,98],\"extension\":\"Cybox-URLs-Extension 1\",\"host\":\"www.urls-host-1.com\",\"method\":\"Cybox-URLs-Method 1\",\"parent_categories\":[\"Cybox-URLs-Parent_Category 1\",\"Cybox-URLs-Parent_Categories 1\"],\"path\":\"/download/trouble/cybox/urls/path/1\",\"port\":80,\"provider\":\"Cybox-URLs-Provider 1\",\"query\":\"q=bad&sort=date_1\",\"referrer\":\"Cybox-URLs-Referrer 1\",\"referrer_categories\":[\"Cybox-URLs-Referrer_Category 1\",\"Cybox-URLs-Referrer_Categories 1\"],\"referrer_category_ids\":[12345678901,67890123451],\"rep_score_id\":0,\"scheme\":\"Cybox-URLs-Scheme 1\",\"text\":\"www.urls-text-1.com/download/trouble\"},{\"categories\":[\"Cybox-URLs-Category 2\",\"Cybox-URLs-Categories 2\"],\"category_ids\":[101,102,103,104,105,106,107,108,109,110,111,112,113,114,116,117,118,121,124],\"extension\":\"Cybox-URLs-Extension 2\",\"host\":\"www.urls-host-2.com\",\"method\":\"Cybox-URLs-Method 2\",\"parent_categories\":[\"Cybox-URLs-Parent_Category 2\",\"Cybox-URLs-Parent_Categories 2\"],\"path\":\"/download/trouble/cybox/urls/path/2\",\"port\":81,\"provider\":\"Cybox-URLs-Provider 2\",\"query\":\"q=bad&sort=date_2\",\"referrer\":\"Cybox-URLs-Referrer 2\",\"referrer_categories\":[\"Cybox-URLs-Referrer_Category 2\",\"Cybox-URLs-Referrer_Categories 2\"],\"referrer_category_ids\":[12345678902,67890123452],\"rep_score_id\":1,\"scheme\":\"Cybox-URLs-Scheme 2\",\"text\":\"www.urls-text-2.com/download/trouble\"}]},\"device_alias_name\":\"Device_Alias_Name\",\"device_cap\":\"Device_Cap\",\"device_cloud_vm\":{\"autoscale_uid\":\"Device_Cloud_VM-Autoscale_UID\",\"dc_region\":\"Device_Cloud_VM-DC_Region\",\"instance_uid\":\"Device_Cloud_VM-Instance_UID\",\"subnet_uid\":\"Device_Cloud_VM-Subnet_UID\",\"vpc_uid\":\"Device_Cloud_VM-VPC_UID\"},\"device_desc\":\"Device_Desc\",\"device_domain\":\"device.domain.internal.somecompany.com\",\"device_domain_uid\":\"Device_Domain_UID\",\"device_end_time\":1613021404000,\"device_gateway\":\"175.16.199.0\",\"device_group\":\"Device_Group\",\"device_group_name\":\"Device_Group_Name\",\"device_hw_bios_date\":\"03/31/16\",\"device_hw_bios_manufacturer\":\"LENOVO\",\"device_hw_bios_ver\":\"LENOVO G5ETA2WW (2.62)\",\"device_hw_cpu_type\":\"x86 Family 6 Model 37 Stepping 5\",\"device_imei\":\"Device_IMEI\",\"device_ip\":\"175.16.199.0\",\"device_is_compliant\":true,\"device_is_personal\":true,\"device_is_trusted\":true,\"device_is_unmanaged\":true,\"device_location\":{\"city\":\"Device_Location-City\",\"continent\":\"Device_Location-Continent\",\"coordinates\":[-12.345,56.789],\"country\":\"US\",\"desc\":\"Device_Location-Desc\",\"isp\":\"Device_Location-ISP\",\"on_premises\":true,\"region\":\"US-CA\"},\"device_mac\":\"00:B0:D0:63:C2:07\",\"device_name\":\"device.name.computer.domain\",\"device_name_md5\":\"4ED962DDBF17E2BBA7B14EBC00F3162E\",\"device_networks\":[{\"bssid\":\"Device_Networks-BSSID 1\",\"gateway_ip\":\"175.16.199.0\",\"gateway_mac\":\"00:B0:D0:63:C2:08\",\"ipv4\":\"175.16.199.0\",\"ipv6\":\"2a02:cf40::\",\"is_public\":true,\"mac\":\"00:B0:D0:63:C2:09\",\"rep_score_id\":0,\"ssid\":\"Device_Networks-SSID 1\",\"type_id\":0},{\"bssid\":\"Device_Networks-BSSID 2\",\"gateway_ip\":\"89.160.20.112\",\"gateway_mac\":\"00:B0:D0:63:C2:10\",\"ipv4\":\"89.160.20.112\",\"ipv6\":\"2a02:cf40::\",\"is_public\":true,\"mac\":\"00:B0:D0:63:C2:11\",\"rep_score_id\":1,\"ssid\":\"Device_Networks-SSID 2\",\"type_id\":1}],\"device_org_unit\":\"Device_Org_Unit\",\"device_os_bits\":12345678901,\"device_os_build\":\"Device_OS_Build\",\"device_os_country\":\"IN\",\"device_os_edition\":\"Professional\",\"device_os_lang\":\"en\",\"device_os_name\":\"Windows Server 2019 Standard Edition\",\"device_os_sp_name\":\"Device_OS_SP_Name\",\"device_os_sp_ver\":\"Device_OS_SP_Ver\",\"device_os_type_id\":0,\"device_os_ver\":\"Windows 10\",\"device_proxy_ip\":\"89.160.20.112\",\"device_proxy_name\":\"Device_Proxy_Name\",\"device_public_ip\":\"89.160.20.112\",\"device_ref_uid\":\"Device_Ref_UID\",\"device_site\":\"Device_Site\",\"device_subnet\":\"81.2.69.144\",\"device_time\":1613021404000,\"device_type\":\"server\",\"device_uid\":\"Device_UID\",\"device_vhost\":\"Device_VHost\",\"device_vhost_id\":0,\"domain_uid\":\"Domain_UID\",\"end_time\":\"2024-02-29T01:00:00.000Z\",\"entity\":{\"data\":{\"field1_keyword\":\"Entity-Data-field1_Keyword\",\"field1_number\":12345678901,\"field1_boolean\":true},\"name\":\"Entity-Name\",\"type\":\"Entity-Type\",\"uid\":\"Entity-UID\",\"version\":\"Entity-Version\"},\"event_id\":2001,\"events\":[{\"connection\":{\"direction_id\":1,\"dst_service\":\"C:\\\\Windows\\\\system32\\\\NTOSKRNL.EXE\",\"src_ip\":\"159.19.163.218\"},\"count\":1,\"device_end_time\":1709225074618,\"device_time\":1709225074618}],\"feature_name\":\"Feature_Name\",\"feature_path\":\"Feature_Path\",\"feature_type\":\"Feature_Type\",\"feature_uid\":\"Feature_UID\",\"feature_ver\":\"2014.1.4.25\",\"id\":12345678901,\"impersonator_customer_uid\":\"Impersonator_Customer_UID\",\"impersonator_domain_uid\":\"Impersonator_Domain_UID\",\"impersonator_user_uid\":\"Impersonator_User_UID\",\"is_user_present\":true,\"log_level\":\"Log Level\",\"log_name\":\"Log_Name\",\"log_time\":\"2024-02-29T01:00:00.000Z\",\"logging_device_ip\":\"89.160.20.112\",\"logging_device_name\":\"Logging_Device_Name\",\"logging_device_post_time\":1613021404000,\"logging_device_ref_uid\":\"Logging_Device_Ref_UID\",\"message\":\"Message\",\"message_code\":\"Message_Code\",\"message_id\":0,\"org_unit_uid\":\"Org_Unit_UID\",\"orig_data\":\"Orig_Data\",\"product_data\":{\"sep_domain_uid\":\"Product_Data-Sep_Domain_UID\",\"sep_hw_uid\":\"Product_Data-Sep_HW_UID\"},\"product_lang\":\"en\",\"product_name\":\"Symantec Endpoint Security\",\"product_uid\":\"Product_UID\",\"product_ver\":\"2014.1.4.25-beta\",\"proxy_device_ip\":\"89.160.20.112\",\"proxy_device_name\":\"Proxy_Device_Name\",\"raw_data\":{\"assetID\":\"vc9DagprQYyLZ23SEY1APw\",\"assetOpstateDTO\":{\"productUuid\":\"31B0C880-0229-49E8-94C5-48D56B1BD7B9\",\"features\":[{\"uuid\":\"1DF0351C-146D-4F07-B155-BF5C7077FF40\",\"featureStatus\":\"SECURE\",\"opstate\":{\"EDRContentSequence\":\"20231128005\",\"EDREngineVersion\":\"4.11.0.10\",\"EDRFramworkVersion\":\"4.10.0.59\",\"FDRStatus\":true,\"LowDiskSpace\":false,\"MaxDBSizeHonored\":true,\"applied_policy\":{\"effective_date\":1709219437080,\"sha2\":\"ee6b0bebbc4575b507ac616d2c362f2c54d462b92cf4068cb6681ae3187d4de3\",\"uid\":\"7dc29d40-f303-477a-9012-287ef252a391\",\"version\":\"16\"},\"disk_usage_mb\":1546,\"fdr_first_event_date\":\"20240227\",\"fdr_state\":1},\"state\":\"ENABLED\",\"statusReason\":[\"-107\",\"0\"],\"prevention_state\":\"1\"}],\"products_active\":0,\"blades\":0}},\"ref_log_name\":\"Ref_Log_Name\",\"ref_log_time\":\"2024-02-29T01:00:00.000Z\",\"ref_orig_uid\":\"Ref_Orig_UID\",\"ref_uid\":\"Ref_UID\",\"remediated\":true,\"remediation\":\"Remediation\",\"remediation_ref\":\"Remediation_Ref\",\"remediation_uid\":0,\"seq_num\":12345678901,\"sessions\":[{\"auth_protocol_id\":0,\"cleartext_credentials\":true,\"direction_id\":0,\"id\":12345678901,\"is_admin\":true,\"logon_type_id\":1,\"port\":80,\"previous_users\":[\"Sessions-Previous_User 1\",\"Sessions-Previous_Users 1\"],\"remote\":true,\"remote_host\":\"Sessions-Remote_Host 1\",\"remote_ip\":\"89.160.20.112\",\"user\":{\"account_disabled\":true,\"cloud_resource_uid\":\"Sessions-User-Cloud_Resource_UID 1\",\"domain\":\"Sessions-User-Domain 1\",\"external_account_uid\":\"Sessions-User-External_Account_UID 1\",\"external_uid\":\"Sessions-User-External_UID 1\",\"full_name\":\"Sessions-User-Full_Name 1\",\"groups\":[\"Sessions-User-Group 1\",\"Sessions-User-Groups 1\"],\"home\":\"Sessions-User-Home 1\",\"is_admin\":true,\"logon_name\":\"Sessions-User-Logon_Name 1\",\"name\":\"session-User-Name 1\",\"password_expires\":true,\"shell\":\"Sessions-User-Shell 1\",\"sid\":\"Sessions-User-SID 1\",\"uid\":\"Sessions-User-UID 1\"}},{\"auth_protocol_id\":1,\"cleartext_credentials\":true,\"direction_id\":1,\"id\":67890123451,\"is_admin\":true,\"logon_type_id\":2,\"port\":81,\"previous_users\":[\"Sessions-Previous_User 2\",\"Sessions-Previous_Users 2\"],\"remote\":true,\"remote_host\":\"Sessions-Remote_Host 2\",\"remote_ip\":\"89.160.20.112\",\"user\":{\"account_disabled\":true,\"cloud_resource_uid\":\"Sessions-User-Cloud_Resource_UID 2\",\"domain\":\"Sessions-User-Domain 2\",\"external_account_uid\":\"Sessions-User-External_Account_UID 2\",\"external_uid\":\"Sessions-User-External_UID 2\",\"full_name\":\"Sessions-User-Full_Name 2\",\"groups\":[\"Sessions-User-Group 2\",\"Sessions-User-Groups 2\"],\"home\":\"Sessions-User-Home 2\",\"is_admin\":true,\"logon_name\":\"Sessions-User-Logon_Name 2\",\"name\":\"session-User-Name 2\",\"password_expires\":true,\"shell\":\"Sessions-User-Shell 2\",\"sid\":\"Sessions-User-SID 2\",\"uid\":\"Sessions-User-UID 2\"}}],\"severity_id\":0,\"source\":{\"facility\":\"Source-Facility\",\"facility_detail\":\"Source-Facility_Detail\",\"facility_uid\":\"Source-Facility_UID\",\"type_id\":1},\"status_detail\":\"Status_Detail\",\"status_id\":0,\"status_os\":\"Status_OS\",\"status_os_src\":12345678901,\"status_stack_trace\":\"Status_Stack_Trace\",\"status_thread_name\":\"Status_Thread_Name\",\"stic_has_pii\":true,\"stic_hw_uid\":\"STIC_HW_UID\",\"stic_ip_hash\":\"STIC_IP_Hash\",\"stic_legacy_ent_uids\":[\"STIC_Legacy_Ent_UIDs 1\",\"STIC_Legacy_Ent_UIDs 2\"],\"stic_legacy_hw_uids\":[\"STIC_Legacy_HW_UIDs 1\",\"STIC_Legacy_HW_UIDs 2\"],\"stic_legacy_uids\":[\"STIC_Legacy_UIDs 1\",\"STIC_Legacy_UIDs 2\"],\"stic_schema_id\":\"STIC_Schema_ID\",\"stic_uid\":\"STIC_UID\",\"stic_version\":\"STIC_Version\",\"subfeature_name\":\"Subfeature_Name\",\"time\":\"2024-02-29T02:00:00Z\",\"timezone\":12345678901,\"type\":\"Type\",\"type_id\":2,\"user\":{\"account_disabled\":true,\"cloud_resource_uid\":\"User-Cloud_Resource_UID\",\"domain\":\"User-Domain\",\"external_account_uid\":\"User-External_Account_UID\",\"external_uid\":\"User-External_UID\",\"full_name\":\"User-Full_Name\",\"groups\":[\"User-Group 1\",\"User-Groups 1\"],\"home\":\"User-Home\",\"is_admin\":true,\"logon_name\":\"User-Logon_Name\",\"name\":\"User123\",\"password_expires\":true,\"shell\":\"User-Shell\",\"sid\":\"TT23009\",\"uid\":\"UU34899825\"},\"user_name\":\"Mohit\",\"user_uid\":\"AB45698\",\"uuid\":\"SR-1565234545\",\"version\":\"1.4\"}", "sequence": [ 12345678901 ], @@ -386,6 +374,17 @@ An example event for `event` looks as following: ] } }, + "gcs": { + "storage": { + "bucket": { + "name": "testbucket" + }, + "object": { + "content_type": "application/x-ndjson", + "name": "test-events.log" + } + } + }, "group": { "name": "Device_Group" }, @@ -418,11 +417,11 @@ An example event for `event` looks as following: ] }, "input": { - "type": "aws-s3" + "type": "gcs" }, "log": { "file": { - "path": "https://elastic-package-symantec-endpoint-security-bucket-96277.s3.us-east-1.amazonaws.com/events.log" + "path": "gs://testbucket/test-events.log" }, "level": [ "Log Level" @@ -1391,8 +1390,6 @@ An example event for `event` looks as following: "mac": "00-B0-D0-63-C2-07" }, "tags": [ - "collect_sqs_logs", - "preserve_original_event", "forwarded", "symantec_endpoint_security-event" ], @@ -1433,13 +1430,19 @@ An example event for `event` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | -| aws.s3.bucket.name | The AWS S3 bucket name. | keyword | -| aws.s3.object.key | The AWS S3 Object key. | keyword | +| aws.s3.bucket | | flattened | +| aws.s3.object | | flattened | +| azure.resource | | flattened | +| azure.storage | | flattened | +| azure.subscription_id | Azure subscription ID. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | event.dataset | Event dataset. | constant_keyword | +| gcs.storage.bucket.name | The name of the Google Cloud Storage Bucket. | keyword | +| gcs.storage.object.content_type | The content type of the Google Cloud Storage object. | keyword | +| gcs.storage.object.json_data | When parse_json is true, the resulting JSON data is stored in this field. | keyword | +| gcs.storage.object.name | The content type of the Google Cloud Storage object. | keyword | | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | | ses.access_mask | The access mask in platform-native format. | long | From 602e8527707c037c026268d628b5d32a678497eb Mon Sep 17 00:00:00 2001 From: moxarth-rathod Date: Tue, 23 Dec 2025 14:50:21 +0530 Subject: [PATCH 2/3] improve coverage for cloudflare_logpush --- .../_dev/deploy/docker/Dockerfile | 20 - .../_dev/deploy/docker/docker-compose.yml | 202 --- .../_dev/deploy/docker/files/emulator.sh | 35 - packages/cloudflare_logpush/changelog.yml | 5 + .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../docker/sample_logs/access_request.log | 0 .../access_request/_dev/deploy/tf/env.yml | 9 + .../deploy/tf/files/test-access-request.log | 1 + .../access_request/_dev/deploy/tf/main.tf | 57 + .../_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../access_request/agent/stream/gcs.yml.hbs | 3 + .../access_request/fields/beats.yml | 33 + .../data_stream/access_request/manifest.yml | 7 + .../access_request/sample_event.json | 56 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../_dev/deploy/docker/sample_logs/audit.log | 0 .../data_stream/audit/_dev/deploy/tf/env.yml | 9 + .../audit/_dev/deploy/tf/files/test-audit.log | 1 + .../data_stream/audit/_dev/deploy/tf/main.tf | 57 + .../audit/_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../audit/agent/stream/gcs.yml.hbs | 3 + .../data_stream/audit/fields/beats.yml | 33 + .../data_stream/audit/manifest.yml | 7 + .../data_stream/audit/sample_event.json | 53 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../_dev/deploy/docker/sample_logs/casb.log | 0 .../data_stream/casb/_dev/deploy/tf/env.yml | 9 + .../casb/_dev/deploy/tf/files/test-casb.log | 1 + .../data_stream/casb/_dev/deploy/tf/main.tf | 57 + .../casb/_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../casb/_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../data_stream/casb/agent/stream/gcs.yml.hbs | 3 + .../data_stream/casb/fields/beats.yml | 33 + .../data_stream/casb/manifest.yml | 7 + .../data_stream/casb/sample_event.json | 51 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../docker/sample_logs/device_posture.log | 0 .../device_posture/_dev/deploy/tf/env.yml | 9 + .../deploy/tf/files/test-device-posture.log | 1 + .../device_posture/_dev/deploy/tf/main.tf | 57 + .../_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../device_posture/agent/stream/gcs.yml.hbs | 3 + .../device_posture/fields/beats.yml | 33 + .../data_stream/device_posture/manifest.yml | 7 + .../device_posture/sample_event.json | 61 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../sample_logs/dlp_forensic_copies.log | 0 .../_dev/deploy/tf/env.yml | 9 + .../tf/files/test-dlp-forensic-copies.log | 1 + .../_dev/deploy/tf/main.tf | 57 + .../_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../agent/stream/gcs.yml.hbs | 3 + .../dlp_forensic_copies/fields/beats.yml | 33 + .../dlp_forensic_copies/manifest.yml | 7 + .../dlp_forensic_copies/sample_event.json | 45 +- .../dns/_dev/deploy/docker/docker-compose.yml | 43 + .../dns/_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../_dev/deploy/docker/sample_logs/dns.log | 0 .../data_stream/dns/_dev/deploy/tf/env.yml | 9 + .../dns/_dev/deploy/tf/files/test-dns.log | 1 + .../data_stream/dns/_dev/deploy/tf/main.tf | 57 + .../dns/_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../dns/_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../data_stream/dns/agent/stream/gcs.yml.hbs | 3 + .../data_stream/dns/fields/beats.yml | 33 + .../data_stream/dns/manifest.yml | 7 + .../data_stream/dns/sample_event.json | 48 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../docker/sample_logs/dns_firewall.log | 0 .../dns_firewall/_dev/deploy/tf/env.yml | 9 + .../deploy/tf/files/test-dns-firewall.log | 1 + .../dns_firewall/_dev/deploy/tf/main.tf | 57 + .../dns_firewall/_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../dns_firewall/agent/stream/gcs.yml.hbs | 3 + .../data_stream/dns_firewall/fields/beats.yml | 33 + .../data_stream/dns_firewall/manifest.yml | 7 + .../dns_firewall/sample_event.json | 49 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../sample_logs/email_security_alerts.log | 0 .../_dev/deploy/tf/env.yml | 9 + .../tf/files/test-email-security-alerts.log | 1 + .../_dev/deploy/tf/main.tf | 57 + .../_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../agent/stream/gcs.yml.hbs | 3 + .../email_security_alerts/fields/beats.yml | 33 + .../email_security_alerts/manifest.yml | 7 + .../email_security_alerts/sample_event.json | 43 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../docker/sample_logs/firewall_event.log | 0 .../firewall_event/_dev/deploy/tf/env.yml | 9 + .../deploy/tf/files/test-firewall-event.log | 1 + .../firewall_event/_dev/deploy/tf/main.tf | 57 + .../_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../firewall_event/agent/stream/gcs.yml.hbs | 3 + .../firewall_event/fields/beats.yml | 33 + .../data_stream/firewall_event/manifest.yml | 7 + .../firewall_event/sample_event.json | 58 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../deploy/docker/sample_logs/gateway_dns.log | 0 .../gateway_dns/_dev/deploy/tf/env.yml | 9 + .../_dev/deploy/tf/files/test-gateway-dns.log | 1 + .../gateway_dns/_dev/deploy/tf/main.tf | 57 + .../gateway_dns/_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../gateway_dns/agent/stream/gcs.yml.hbs | 3 + .../data_stream/gateway_dns/fields/beats.yml | 33 + .../data_stream/gateway_dns/manifest.yml | 7 + .../data_stream/gateway_dns/sample_event.json | 84 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../docker/sample_logs/gateway_http.log | 0 .../gateway_http/_dev/deploy/tf/env.yml | 9 + .../deploy/tf/files/test-gateway-http.log | 1 + .../gateway_http/_dev/deploy/tf/main.tf | 57 + .../gateway_http/_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../gateway_http/agent/stream/gcs.yml.hbs | 3 + .../data_stream/gateway_http/fields/beats.yml | 33 + .../data_stream/gateway_http/manifest.yml | 7 + .../gateway_http/sample_event.json | 74 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../docker/sample_logs/gateway_network.log | 0 .../gateway_network/_dev/deploy/tf/env.yml | 9 + .../deploy/tf/files/test-gateway-network.log | 1 + .../gateway_network/_dev/deploy/tf/main.tf | 57 + .../_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../gateway_network/agent/stream/gcs.yml.hbs | 3 + .../gateway_network/fields/beats.yml | 33 + .../data_stream/gateway_network/manifest.yml | 7 + .../gateway_network/sample_event.json | 65 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../docker/sample_logs/http_request.log | 0 .../http_request/_dev/deploy/tf/env.yml | 9 + .../deploy/tf/files/test-http-request.log | 2 + .../http_request/_dev/deploy/tf/main.tf | 57 + .../http_request/_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../http_request/agent/stream/gcs.yml.hbs | 3 + .../data_stream/http_request/fields/beats.yml | 33 + .../data_stream/http_request/manifest.yml | 7 + .../http_request/sample_event.json | 56 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../deploy/docker/sample_logs/magic_ids.log | 0 .../magic_ids/_dev/deploy/tf/env.yml | 9 + .../_dev/deploy/tf/files/test-magic-ids.log | 1 + .../magic_ids/_dev/deploy/tf/main.tf | 57 + .../magic_ids/_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../magic_ids/agent/stream/gcs.yml.hbs | 3 + .../data_stream/magic_ids/fields/beats.yml | 33 + .../data_stream/magic_ids/manifest.yml | 7 + .../data_stream/magic_ids/sample_event.json | 54 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../deploy/docker/sample_logs/nel_report.log | 0 .../nel_report/_dev/deploy/tf/env.yml | 9 + .../_dev/deploy/tf/files/test-nel-report.log | 1 + .../nel_report/_dev/deploy/tf/main.tf | 57 + .../nel_report/_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../nel_report/agent/stream/gcs.yml.hbs | 3 + .../data_stream/nel_report/fields/beats.yml | 33 + .../data_stream/nel_report/manifest.yml | 7 + .../data_stream/nel_report/sample_event.json | 47 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../docker/sample_logs/network_analytics.log | 0 .../network_analytics/_dev/deploy/tf/env.yml | 9 + .../tf/files/test-network-analytics.log | 1 + .../network_analytics/_dev/deploy/tf/main.tf | 57 + .../_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../agent/stream/gcs.yml.hbs | 3 + .../network_analytics/fields/beats.yml | 33 + .../network_analytics/manifest.yml | 7 + .../network_analytics/sample_event.json | 60 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../docker/sample_logs/network_session.log | 0 .../network_session/_dev/deploy/tf/env.yml | 9 + .../deploy/tf/files/test-network-session.log | 1 + .../network_session/_dev/deploy/tf/main.tf | 57 + .../_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../network_session/agent/stream/gcs.yml.hbs | 3 + .../network_session/fields/beats.yml | 33 + .../data_stream/network_session/manifest.yml | 7 + .../network_session/sample_event.json | 68 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../docker/sample_logs/page_shield_events.log | 0 .../page_shield_events/_dev/deploy/tf/env.yml | 9 + .../tf/files/test-page-shield-events.log | 1 + .../page_shield_events/_dev/deploy/tf/main.tf | 57 + .../_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../agent/stream/gcs.yml.hbs | 3 + .../page_shield_events/fields/beats.yml | 33 + .../page_shield_events/manifest.yml | 7 + .../page_shield_events/sample_event.json | 52 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../docker/sample_logs/sinkhole_http.log | 0 .../sinkhole_http/_dev/deploy/tf/env.yml | 9 + .../deploy/tf/files/test-sinkhole-http.log | 1 + .../sinkhole_http/_dev/deploy/tf/main.tf | 57 + .../sinkhole_http/_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../sinkhole_http/agent/stream/gcs.yml.hbs | 3 + .../sinkhole_http/fields/beats.yml | 33 + .../data_stream/sinkhole_http/manifest.yml | 7 + .../sinkhole_http/sample_event.json | 67 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../docker/sample_logs/spectrum_event.log | 0 .../spectrum_event/_dev/deploy/tf/env.yml | 9 + .../deploy/tf/files/test-spectrum-event.log | 1 + .../spectrum_event/_dev/deploy/tf/main.tf | 57 + .../_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../spectrum_event/agent/stream/gcs.yml.hbs | 3 + .../spectrum_event/fields/beats.yml | 33 + .../data_stream/spectrum_event/manifest.yml | 7 + .../spectrum_event/sample_event.json | 54 +- .../_dev/deploy/docker/docker-compose.yml | 43 + .../_dev/deploy/docker/files/manifest.yml | 5 + .../deploy/docker/gcs-mock-service/go.mod | 5 + .../deploy/docker/gcs-mock-service/go.sum | 4 + .../deploy/docker/gcs-mock-service/main.go | 297 ++++ .../docker/sample_logs/workers_trace.log | 0 .../workers_trace/_dev/deploy/tf/env.yml | 9 + .../deploy/tf/files/test-workers-trace.log | 1 + .../workers_trace/_dev/deploy/tf/main.tf | 57 + .../workers_trace/_dev/deploy/tf/variables.tf | 26 + .../_dev/test/system/test-aws-s3-config.yml | 15 + .../_dev/test/system/test-gcs-config.yml | 16 + .../test/system/test-http-endpoint-config.yml | 1 + .../workers_trace/agent/stream/gcs.yml.hbs | 3 + .../workers_trace/fields/beats.yml | 33 + .../data_stream/workers_trace/manifest.yml | 7 + .../workers_trace/sample_event.json | 56 +- packages/cloudflare_logpush/docs/README.md | 1348 ++++++++++------- packages/cloudflare_logpush/manifest.yml | 2 +- 363 files changed, 12428 insertions(+), 1348 deletions(-) delete mode 100644 packages/cloudflare_logpush/_dev/deploy/docker/Dockerfile delete mode 100644 packages/cloudflare_logpush/_dev/deploy/docker/docker-compose.yml delete mode 100644 packages/cloudflare_logpush/_dev/deploy/docker/files/emulator.sh create mode 100644 packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/access_request}/_dev/deploy/docker/sample_logs/access_request.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/tf/files/test-access-request.log create mode 100644 packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/access_request/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/access_request/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/audit}/_dev/deploy/docker/sample_logs/audit.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/audit/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/audit/_dev/deploy/tf/files/test-audit.log create mode 100644 packages/cloudflare_logpush/data_stream/audit/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/audit/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/audit/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/audit/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/casb}/_dev/deploy/docker/sample_logs/casb.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/casb/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/casb/_dev/deploy/tf/files/test-casb.log create mode 100644 packages/cloudflare_logpush/data_stream/casb/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/casb/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/casb/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/casb/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/device_posture}/_dev/deploy/docker/sample_logs/device_posture.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/tf/files/test-device-posture.log create mode 100644 packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/device_posture/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/device_posture/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/dlp_forensic_copies}/_dev/deploy/docker/sample_logs/dlp_forensic_copies.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/tf/files/test-dlp-forensic-copies.log create mode 100644 packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/dns}/_dev/deploy/docker/sample_logs/dns.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/dns/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/dns/_dev/deploy/tf/files/test-dns.log create mode 100644 packages/cloudflare_logpush/data_stream/dns/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/dns/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/dns/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/dns/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/dns_firewall}/_dev/deploy/docker/sample_logs/dns_firewall.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/tf/files/test-dns-firewall.log create mode 100644 packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/dns_firewall/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/dns_firewall/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/email_security_alerts}/_dev/deploy/docker/sample_logs/email_security_alerts.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/tf/files/test-email-security-alerts.log create mode 100644 packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/firewall_event}/_dev/deploy/docker/sample_logs/firewall_event.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/tf/files/test-firewall-event.log create mode 100644 packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/firewall_event/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/firewall_event/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/gateway_dns}/_dev/deploy/docker/sample_logs/gateway_dns.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/tf/files/test-gateway-dns.log create mode 100644 packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/gateway_dns/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_dns/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/gateway_http}/_dev/deploy/docker/sample_logs/gateway_http.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/tf/files/test-gateway-http.log create mode 100644 packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/gateway_http/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_http/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/gateway_network}/_dev/deploy/docker/sample_logs/gateway_network.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/tf/files/test-gateway-network.log create mode 100644 packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/gateway_network/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/gateway_network/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/http_request}/_dev/deploy/docker/sample_logs/http_request.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/tf/files/test-http-request.log create mode 100644 packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/http_request/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/http_request/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/magic_ids}/_dev/deploy/docker/sample_logs/magic_ids.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/tf/files/test-magic-ids.log create mode 100644 packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/magic_ids/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/magic_ids/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/nel_report}/_dev/deploy/docker/sample_logs/nel_report.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/tf/files/test-nel-report.log create mode 100644 packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/nel_report/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/nel_report/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/network_analytics}/_dev/deploy/docker/sample_logs/network_analytics.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/tf/files/test-network-analytics.log create mode 100644 packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/network_analytics/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/network_analytics/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/network_session}/_dev/deploy/docker/sample_logs/network_session.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/tf/files/test-network-session.log create mode 100644 packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/network_session/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/network_session/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/page_shield_events}/_dev/deploy/docker/sample_logs/page_shield_events.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/tf/files/test-page-shield-events.log create mode 100644 packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/page_shield_events/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/page_shield_events/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/sinkhole_http}/_dev/deploy/docker/sample_logs/sinkhole_http.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/tf/files/test-sinkhole-http.log create mode 100644 packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/spectrum_event}/_dev/deploy/docker/sample_logs/spectrum_event.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/tf/files/test-spectrum-event.log create mode 100644 packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/spectrum_event/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/spectrum_event/_dev/test/system/test-gcs-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/files/manifest.yml create mode 100644 packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/gcs-mock-service/go.mod create mode 100644 packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/gcs-mock-service/go.sum create mode 100644 packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/gcs-mock-service/main.go rename packages/cloudflare_logpush/{ => data_stream/workers_trace}/_dev/deploy/docker/sample_logs/workers_trace.log (100%) create mode 100644 packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/tf/env.yml create mode 100644 packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/tf/files/test-workers-trace.log create mode 100644 packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/tf/main.tf create mode 100644 packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/tf/variables.tf create mode 100644 packages/cloudflare_logpush/data_stream/workers_trace/_dev/test/system/test-aws-s3-config.yml create mode 100644 packages/cloudflare_logpush/data_stream/workers_trace/_dev/test/system/test-gcs-config.yml diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/Dockerfile b/packages/cloudflare_logpush/_dev/deploy/docker/Dockerfile deleted file mode 100644 index 40d0c938497..00000000000 --- a/packages/cloudflare_logpush/_dev/deploy/docker/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM node:current-bullseye-slim - -# Install dependencies -RUN apt-get update && apt-get install -y \ - curl \ - netcat \ - ca-certificates \ - lsb-release \ - && rm -rf /var/lib/apt/lists/* -# Install Azurite -RUN npm install -g azurite - -# Install Azure CLI -RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash - -# Copy emulator script -COPY files/emulator.sh /emulator.sh -RUN chmod +x /emulator.sh - -ENTRYPOINT ["/bin/sh", "/emulator.sh"] diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/docker-compose.yml b/packages/cloudflare_logpush/_dev/deploy/docker/docker-compose.yml deleted file mode 100644 index a884fb896d5..00000000000 --- a/packages/cloudflare_logpush/_dev/deploy/docker/docker-compose.yml +++ /dev/null @@ -1,202 +0,0 @@ -version: '2.3' -services: - cloudflare-logpush-audit-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/audit - command: log --start-signal=SIGHUP --delay=5s /sample_logs/audit.log - cloudflare-logpush-dlp-forensic-copies-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/dlp_forensic_copies - command: log --start-signal=SIGHUP --delay=5s /sample_logs/dlp_forensic_copies.log - cloudflare-logpush-dns-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/dns - command: log --start-signal=SIGHUP --delay=5s /sample_logs/dns.log - cloudflare-logpush-email-security-alerts-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/email_security_alerts - command: log --start-signal=SIGHUP --delay=5s /sample_logs/email_security_alerts.log - cloudflare-logpush-firewall-event-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/firewall_event - command: log --start-signal=SIGHUP --delay=5s /sample_logs/firewall_event.log - cloudflare-logpush-http-request-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/http_request - command: log --start-signal=SIGHUP --delay=5s /sample_logs/http_request.log - cloudflare-logpush-nel-report-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/nel_report - command: log --start-signal=SIGHUP --delay=5s /sample_logs/nel_report.log - cloudflare-logpush-network-analytics-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/network_analytics - command: log --start-signal=SIGHUP --delay=5s /sample_logs/network_analytics.log - cloudflare-logpush-spectrum-event-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/spectrum_event - command: log --start-signal=SIGHUP --delay=5s /sample_logs/spectrum_event.log - cloudflare-logpush-gateway-dns-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/gateway_dns - command: log --start-signal=SIGHUP --delay=5s /sample_logs/gateway_dns.log - cloudflare-logpush-gateway-http-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/gateway_http - command: log --start-signal=SIGHUP --delay=5s /sample_logs/gateway_http.log - cloudflare-logpush-gateway-network-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/gateway_network - command: log --start-signal=SIGHUP --delay=5s /sample_logs/gateway_network.log - cloudflare-logpush-network-session-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/network_session - command: log --start-signal=SIGHUP --delay=5s /sample_logs/network_session.log - cloudflare-logpush-page-shield-events-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/page_shield_events - command: log --start-signal=SIGHUP --delay=5s /sample_logs/page_shield_events.log - cloudflare-logpush-casb-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/casb - command: log --start-signal=SIGHUP --delay=5s /sample_logs/casb.log - cloudflare-logpush-access-request-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/access_request - command: log --start-signal=SIGHUP --delay=5s /sample_logs/access_request.log - cloudflare-logpush-device-posture-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/device_posture - command: log --start-signal=SIGHUP --delay=5s /sample_logs/device_posture.log - cloudflare-logpush-workers-trace-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/workers_trace - command: log --start-signal=SIGHUP --delay=5s /sample_logs/workers_trace.log - cloudflare-logpush-magic-ids-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/magic_ids - command: log --start-signal=SIGHUP --delay=5s /sample_logs/magic_ids.log - cloudflare-logpush-sinkhole-http-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/sinkhole_http - command: log --start-signal=SIGHUP --delay=5s /sample_logs/sinkhole_http.log - cloudflare-logpush-dns-firewall-http-endpoint: - image: docker.elastic.co/observability/stream:v0.20.0 - volumes: - - ./sample_logs:/sample_logs:ro - environment: - - STREAM_PROTOCOL=webhook - - STREAM_WEBHOOK_PROBE=false - - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/dns_firewall - command: log --start-signal=SIGHUP --delay=5s /sample_logs/dns_firewall.log - - azure-blob-storage-emulator: - build: . - volumes: - - ./sample_logs:/sample_logs - ports: - - "10000/tcp" - healthcheck: - test: "curl --fail --silent --output /dev/null http://localhost:4443 || exit 1" - interval: 10s - timeout: 5s diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/files/emulator.sh b/packages/cloudflare_logpush/_dev/deploy/docker/files/emulator.sh deleted file mode 100644 index e484ec55a88..00000000000 --- a/packages/cloudflare_logpush/_dev/deploy/docker/files/emulator.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -set -e - -# Set environment variables for Azure cli -export AZURE_STORAGE_ACCOUNT=devstoreaccount1 -export AZURE_STORAGE_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" -export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=$AZURE_STORAGE_ACCOUNT;AccountKey=$AZURE_STORAGE_KEY;BlobEndpoint=http://127.0.0.1:10000/$AZURE_STORAGE_ACCOUNT;" - -# Start Azurite in background -azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --skipApiVersionCheck & -AZURITE_PID=$! - -# Wait until Azurite is ready -while ! nc -z 127.0.0.1 10000; do sleep 0.5; done - -# Create container -az storage container create --name test-container - -# Upload all files -for f in /sample_logs/*; do - if [ -f "$f" ]; then - az storage blob upload --container-name test-container --file "$f" --name "$(basename "$f")" - fi -done - -node -e " -const http = require('http'); -const server = http.createServer((req, res) => { res.writeHead(200); res.end('OK'); }); -server.listen(4443, () => console.log('Healthcheck API available on port 4443')); -" & - -echo "All files uploaded. Healthcheck API started." - -# Keep Azurite running -wait $AZURITE_PID diff --git a/packages/cloudflare_logpush/changelog.yml b/packages/cloudflare_logpush/changelog.yml index a4a924bcb79..f443610fd3d 100644 --- a/packages/cloudflare_logpush/changelog.yml +++ b/packages/cloudflare_logpush/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.42.0" + changes: + - description: Add alternative host configuration option support in gcs input. + type: enhancement + link: https://github.com/elastic/integrations/pull/1 - version: "1.41.0" changes: - description: Add Cloudflare `SourceInternalIP` field values to `related.ip` for the gateway_http and gateway_network data streams. diff --git a/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/docker-compose.yml b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..47c263c0849 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,43 @@ +version: '2.3' +services: + cloudflare-logpush-access-request-http-endpoint: + image: docker.elastic.co/observability/stream:v0.20.0 + volumes: + - ./sample_logs:/sample_logs:ro + environment: + - STREAM_PROTOCOL=webhook + - STREAM_WEBHOOK_PROBE=false + - STREAM_ADDR=http://elastic-agent:9560/cloudflare_logpush/access_request + command: log --start-signal=SIGHUP --delay=5s /sample_logs/access_request.log + azure-blob-storage-emulator: + image: mcr.microsoft.com/azure-storage/azurite + command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 + ports: + - "10000/tcp" + uploader: + image: mcr.microsoft.com/azure-cli + depends_on: + - azure-blob-storage-emulator + volumes: + - ./sample_logs:/sample_logs + entrypoint: > + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/access_request.log --name access_request.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..96ca872bc78 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/access_request.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/access_request.log b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/sample_logs/access_request.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/access_request.log rename to packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/docker/sample_logs/access_request.log diff --git a/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/tf/files/test-access-request.log b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/tf/files/test-access-request.log new file mode 100644 index 00000000000..f98a0cd9bf2 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/tf/files/test-access-request.log @@ -0,0 +1 @@ +{"Action":"login","Allowed":true,"AppDomain":"partner-zt-logs.cloudflareaccess.com/warp","AppUUID":"123e4567-e89b-12d3-a456-426614174000","Connection":"onetimepin","Country":"us","CreatedAt":1684862313000000000,"Email":"user@example.com","IPAddress":"67.43.156.93","PurposeJustificationPrompt":"Please provide your reason for accessing the application.","PurposeJustificationResponse":"I need to access the application for work purposes.","RayID":"00c0ffeeabc12345","TemporaryAccessApprovers":["approver1@example.com","approver2@example.com"],"TemporaryAccessDuration":7200,"UserUID":"166befbb-00e3-5e20-bd6e-27245333949f"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..3af68e0275d --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/access_request/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-access-request-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-access-request-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/audit.log --name audit.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..2cefed75ea3 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/audit.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/audit.log b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/sample_logs/audit.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/audit.log rename to packages/cloudflare_logpush/data_stream/audit/_dev/deploy/docker/sample_logs/audit.log diff --git a/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/tf/files/test-audit.log b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/tf/files/test-audit.log new file mode 100644 index 00000000000..15a74d28958 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/tf/files/test-audit.log @@ -0,0 +1 @@ +{"ActionResult":true,"ActionType":"token_create","ActorEmail":"user@example.com","ActorID":"enl3j9du8rnx2swwd9l32qots7l54t9s","ActorIP":"81.2.69.142","ActorType":"user","ID":"73fd39ed-5aab-4a2a-b93c-c9a4abf0c425","Interface":"UI","Metadata":{"token_name":"test","token_tag":"b7261c49a793a82678d12285f0bc1401"},"NewValue":{"key1":"value1","key2":"value2"},"OldValue":{"key3":"value4","key4":"value4"},"OwnerID":"enl3j9du8rnx2swwd9l32qots7l54t9s","ResourceID":"enl3j9du8rnx2swwd9l32qots7l54t9s","ResourceType":"account","When":"2021-11-30T20:19:48Z"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..38d7194909a --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/audit/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-audit-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-audit-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/casb.log --name casb.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..37d0e91c53b --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/casb.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/casb.log b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/sample_logs/casb.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/casb.log rename to packages/cloudflare_logpush/data_stream/casb/_dev/deploy/docker/sample_logs/casb.log diff --git a/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/tf/files/test-casb.log b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/tf/files/test-casb.log new file mode 100644 index 00000000000..389eb2d912d --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/tf/files/test-casb.log @@ -0,0 +1 @@ +{"AssetDisplayName":"John Doe","AssetExternalID":"0051N000004mG2LAAA","AssetLink":"https://example.com/resource","AssetMetadata":{"Id":"0051N000004mG2LAAA","Fax":null,"Name":"John Doe","Alias":"JDoe","Email":"user@example.com","Phone":"+3460000000","Title":"Customer Solutions Engineer","Address":{"city":"Singapore","state":null,"street":null,"country":"Singapore","latitude":null,"longitude":null,"stateCode":null,"postalCode":null,"countryCode":"SG","geocodeAccuracy":null},"Division":null,"IsActive":false,"LastName":"Doe","UserType":"Standard","AccountId":null,"BadgeText":"","ContactId":null,"Extension":null,"FirstName":"John","Signature":null,"Department":"521","SenderName":null,"UserRoleId":"00E2G000001E","attributes":{"url":"/services/data/userID","type":"User"},"CompanyName":"MyCompany","MobilePhone":null,"SenderEmail":"sender@example.com","CallCenterId":null,"FullPhotoUrl":"https://photos.com/profilephoto/001","LocaleSidKey":"en_SG","LastLoginDate":"2021-10-06T06:32:09.000+0000","SmallPhotoUrl":"https://photos.com/photo/001","BannerPhotoUrl":"/profilephoto/001","EmployeeNumber":"18124","LastViewedDate":null,"TimeZoneSidKey":"Asia/Singapore","DigestFrequency":"D","ForecastEnabled":false,"EmailEncodingKey":"UTF-8","CommunityNickname":"Doe.John","LanguageLocaleKey":"en_US","LastReferencedDate":null,"ReceivesInfoEmails":true,"SmallBannerPhotoUrl":"/profilephoto/001/D","FederationIdentifier":null,"IsProfilePhotoActive":false,"MediumBannerPhotoUrl":"/profilephoto/001/E","EmailPreferencesAutoBcc":true,"ReceivesAdminInfoEmails":true,"OfflineTrialExpirationDate":null,"UserPermissionsOfflineUser":false,"UserPermissionsSupportUser":false,"UserPermissionsMarketingUser":false,"UserPermissionsInteractionUser":true,"DefaultGroupNotificationFrequency":"N","UserPermissionsCallCenterAutoLogin":false},"DetectedTimestamp":"2023-05-16T10:00:00Z","FindingTypeDisplayName":"Salesforce User Sending Email with Different Email Address","FindingTypeID":"a2790c4f-03f5-449f-b209-5f4447f417aa","FindingTypeSeverity":"Medium","InstanceID":"6b187be4-2dd5-42c5-a37b-111111111111","IntegrationDisplayName":"Salesforce Testing","IntegrationID":"c772678d-5cf1-4c73-bf3f-111111111111","IntegrationPolicyVendor":"Salesforce Connection"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..eb1df5997e5 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/casb/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-casb-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-casb-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/device_posture.log --name device_posture.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..fa515a38fa7 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/device_posture.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/device_posture.log b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/sample_logs/device_posture.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/device_posture.log rename to packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/docker/sample_logs/device_posture.log diff --git a/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/tf/files/test-device-posture.log b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/tf/files/test-device-posture.log new file mode 100644 index 00000000000..8d43739206c --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/tf/files/test-device-posture.log @@ -0,0 +1 @@ +{"ClientVersion":"2023.3.258","DeviceID":"083a8354-d56c-11ed-9771-111111111","DeviceManufacturer":"Google Compute Engine","DeviceModel":"Google Compute Engine","DeviceName":"zt-test-vm1","DeviceSerialNumber":"GoogleCloud-ABCD1234567890","DeviceType":"linux","Email":"user@example.com","OSVersion":"5.15.0","PolicyID":"policy-abcdefgh","PostureCheckName":"Ubuntu","PostureCheckType":"os_version","PostureEvaluatedResult":true,"PostureExpectedJSON":{"version":"5.15.0-1025-gcp","operator":"==","os_distro_name":"ubuntu","os_distro_revision":"20.04"},"PostureReceivedJSON":{"version":"5.15.0-1025-gcp","operator":"==","os_distro_name":"ubuntu","os_distro_revision":"20.04"},"Timestamp":"2023-05-17T12:00:00Z","UserUID":"user-abcdefgh"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..e21af695869 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/device_posture/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-device-posture-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-device-posture-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/dlp_forensic_copies.log --name dlp_forensic_copies.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..0f50db2926a --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/dlp_forensic_copies.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/dlp_forensic_copies.log b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/sample_logs/dlp_forensic_copies.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/dlp_forensic_copies.log rename to packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/docker/sample_logs/dlp_forensic_copies.log diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/tf/files/test-dlp-forensic-copies.log b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/tf/files/test-dlp-forensic-copies.log new file mode 100644 index 00000000000..ba0556a1b20 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/tf/files/test-dlp-forensic-copies.log @@ -0,0 +1 @@ +{"AccountID":"acc-id","ForensicCopyID":"copy-id","GatewayRequestID":"req-id","Payload":"Tm90aGluZyB0byBzZWUgaGVyZS4gTW92ZSBhbG9uZy4K","Phase":"request","TriggeredRuleID":"9","Datetime":"2023-05-04T11:29:14Z","Headers":{"key1":"val1","key2":"val2"}} diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..824ab0d1cbb --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-dlp-forensic-copies-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-dlp-forensic-copies-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/dns.log --name dns.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..fe1d4a300fc --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/dns.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/dns.log b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/sample_logs/dns.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/dns.log rename to packages/cloudflare_logpush/data_stream/dns/_dev/deploy/docker/sample_logs/dns.log diff --git a/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/tf/files/test-dns.log b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/tf/files/test-dns.log new file mode 100644 index 00000000000..9c4d06de1b5 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/tf/files/test-dns.log @@ -0,0 +1 @@ +{"ColoCode":"MRS","EDNSSubnet":"1.128.0.0","EDNSSubnetLength":0,"QueryName":"example.com","QueryType":65535,"ResponseCached":false,"ResponseCode":0,"SourceIP":"175.16.199.0","Timestamp":"2022-05-26T09:23:54Z"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..bb21255cc94 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dns/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-dns-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-dns-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/dns_firewall.log --name dns_firewall.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..96350784b06 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/dns_firewall.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/dns_firewall.log b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/sample_logs/dns_firewall.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/dns_firewall.log rename to packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/docker/sample_logs/dns_firewall.log diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/tf/files/test-dns-firewall.log b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/tf/files/test-dns-firewall.log new file mode 100644 index 00000000000..e34e078eae2 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/tf/files/test-dns-firewall.log @@ -0,0 +1 @@ +{"ClientResponseCode":0,"ClusterID":"CLUSTER-001","ColoCode":"SFO","EDNSSubnet":"67.43.156.0","EDNSSubnetLength":24,"QueryDO":true,"QueryName":"example.com","QueryRD":true,"QuerySize":60,"QueryTCP":false,"QueryType":1,"ResponseCached":true,"ResponseCachedStale":false,"SourceIP":"67.43.156.2","Timestamp":"2023-09-19T12:30:00Z","UpstreamIP":"81.2.69.144","UpstreamResponseCode":0,"UpstreamResponseTimeMs":30} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..dbe5501906b --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/dns_firewall/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-dns-firewall-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-dns-firewall-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/email_security_alerts.log --name email_security_alerts.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..d8ef5b5e159 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/email_security_alerts.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/email_security_alerts.log b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/sample_logs/email_security_alerts.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/email_security_alerts.log rename to packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/docker/sample_logs/email_security_alerts.log diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/tf/files/test-email-security-alerts.log b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/tf/files/test-email-security-alerts.log new file mode 100644 index 00000000000..8434a765dbf --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/tf/files/test-email-security-alerts.log @@ -0,0 +1 @@ +{"AlertID":"4WtWkr6nlBz9sNH-2024-08-28T15:32:35","AlertReasons":["because","said-so"],"Attachments":[{"Md5":"91f073bd208689ddbd248e8989ecae90","Sha1":"62b77e14e2c43049c45b5725018e78d0f9986930","Sha256":"3b57505305e7162141fd898ed87d08f92fc42579b5047495859e56b3275a6c06","Ssdeep":"McAQ8tPlH25e85Q2OiYpD08NvHmjJ97UfPMO47sekO:uN9M553OiiN/OJ9MM+e3","Name":"attachment.gif","ContentTypeProvided":"image/gif","ContentTypeComputed":"application/x-msi","Encrypted":true,"Decrypted":true}],"CC":["firstlast+cc@cloudflare.com"],"CCName":["First Last (cc)"],"FinalDisposition":"malicious","From":"firstlast+from@cloudflare.com","FromName":"First Last (from)","Links":["https://example.com"],"MessageDeliveryMode":"unset","MessageID":"","Origin":"unset","OriginalSender":"firstlast+origin@cloudflare.com","ReplyTo":"firstlast+reply@cloudflare.com","ReplyToName":"First Last (reply)","SMTPEnvelopeFrom":"firstlast+env_from@cloudflare.com","SMTPEnvelopeTo":["firstlast+env_to@cloudflare.com"],"SMTPHeloServerIP":"81.2.69.144","SMTPHeloServerIPAsName":"asn","SMTPHeloServerIPAsNumber":"42","SMTPHeloServerIPGeo":"US/NV/Las Vegas","SMTPHeloServerName":"servername","Subject":"innocuous message: please read","ThreatCategories":["CredentialHarvester","Dropper"],"Timestamp":"2024-08-28T15:32:35Z","To":"firstlast+to@cloudflare.com","ToName":"First Last (to)"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..90978b900ab --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/email_security_alerts/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-email-security-alerts-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-email-security-alerts-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/firewall_event.log --name firewall_event.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..aea39441279 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/firewall_event.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/firewall_event.log b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/sample_logs/firewall_event.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/firewall_event.log rename to packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/docker/sample_logs/firewall_event.log diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/tf/files/test-firewall-event.log b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/tf/files/test-firewall-event.log new file mode 100644 index 00000000000..16af53c082c --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/tf/files/test-firewall-event.log @@ -0,0 +1 @@ +{"ClientRequestScheme":"https","MatchIndex":1,"ClientRefererHost":"abc.example.com","Source":"firewallrules","ClientRequestUserAgent":"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","ClientRefererPath":"/abc/checkout","Metadata":{"filter":"1ced07e066a34abf8b14f2a99593bc8d","type":"customer"},"EdgeResponseStatus":403,"ClientRequestProtocol":"HTTP/1.1","OriginatorRayID":"00","RayID":"713d477539b55c29","ClientRequestMethod":"GET","ClientIP":"175.16.199.0","ClientRequestPath":"/abc/checkout","Action":"block","Kind":"firewall","RuleID":"7dc666e026974dab84884c73b3e2afe1","ClientIPClass":"searchEngine","ClientASNDescription":"CLOUDFLARENET","ClientCountry":"us","ClientRefererQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","ClientRequestQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","OriginResponseStatus":0,"EdgeColoCode":"IAD","ClientRefererScheme":"referer URL scheme","Datetime":"2022-05-31T05:23:43Z","ClientRequestHost":"xyz.example.com","ClientASN":15169} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..5ff85889ef9 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-firewall-event-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-firewall-event-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/gateway_dns.log --name gateway_dns.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..0399ae4d8da --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/gateway_dns.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/gateway_dns.log b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/sample_logs/gateway_dns.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/gateway_dns.log rename to packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/docker/sample_logs/gateway_dns.log diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/tf/files/test-gateway-dns.log b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/tf/files/test-gateway-dns.log new file mode 100644 index 00000000000..ff72e6cb2e5 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/tf/files/test-gateway-dns.log @@ -0,0 +1 @@ +{"ApplicationID":0,"ColoCode":"ORD","ColoID":14,"Datetime":"2023-05-02T22:49:53Z","DeviceID":"083a8354-d56c-11ed-9771-6a842b111aaa","DeviceName":"zt-test-vm1","DstIP":"89.160.20.129","DstPort":443,"Email":"user@test.com","Location":"GCP default","LocationID":"f233bd67-78c7-4050-9aff-ad63cce25732","MatchedCategoryIDs":[7,163],"MatchedCategoryNames":["Photography","Weather"],"Policy":"7bdc7a9c-81d3-4816-8e56-de1acad3dec5","PolicyID":"1412","Protocol":"https","QueryCategoryIDs":[26,155],"QueryCategoryNames":["Technology","Technology"],"QueryName":"security.ubuntu.com","QueryNameReversed":"com.ubuntu.security","QuerySize":48,"QueryType":1,"QueryTypeName":"A","RCode":0,"RData":[{"type":"1","data":"CHNlY3VyaXR5BnVidW50dQMjb20AAAEAAQAAAAgABLl9vic="},{"type":"1","data":"CHNlY3VyaXR5BnVidW50dQNjb20AAAEAABAAAAgABLl9viQ="},{"type":"1","data":"CHNlT3VyaXR5BnVidW50dQNjb20AAAEAAQAAAAgABFu9Wyc="}],"ResolvedIPs":["67.43.156.1","67.43.156.2","67.43.156.3"],"ResolverDecision":"allowedOnNoPolicyMatch","SrcIP":"67.43.156.2","SrcPort":0,"TimeZone":"UTC","TimeZoneInferredMethod":"fromLocalTime","UserID":"166befbb-00e3-5e20-bd6e-27245000000"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..05a2a5a439d --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-gateway-dns-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-gateway-dns-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/gateway_http.log --name gateway_http.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..a2a48180b95 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/gateway_http.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/gateway_http.log b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/sample_logs/gateway_http.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/gateway_http.log rename to packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/docker/sample_logs/gateway_http.log diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/tf/files/test-gateway-http.log b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/tf/files/test-gateway-http.log new file mode 100644 index 00000000000..3e8de98efc6 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/tf/files/test-gateway-http.log @@ -0,0 +1 @@ +{"AccountID":"e1836771179f98aabb828da5ea69a348","Action":"block","BlockedFileHash":"91dc1db739a705105e1c763bfdbdaa84c0de8","BlockedFileName":"downloaded_test","BlockedFileReason":"malware","BlockedFileSize":43,"BlockedFileType":"bin","Datetime":"2023-05-03T20:55:05Z","DestinationIP":"89.160.20.129","DestinationPort":443,"DeviceID":"083a8354-d56c-11ed-9771-6a842b100cff","DeviceName":"zt-test-vm1","DownloadedFileNames":["downloaded_file","downloaded_test"],"Email":"user@example.com","FileInfo":{"files":[{"name":"downloaded_file","size":43},{"name":"downloaded_test","size":341}]},"HTTPHost":"guce.yahoo.com","HTTPMethod":"GET","HTTPStatusCode":302,"HTTPVersion":"HTTP/2","IsIsolated":false,"PolicyID":"85063bec-74cb-4546-85a3-e0cde2cdfda2","PolicyName":"Block Yahoo","Referer":"https://www.example.com/","RequestID":"1884fec9b600007fb06a299400000001","SourceInternalIP":"192.168.1.123","SourceIP":"67.43.156.2","SourcePort":47924,"UntrustedCertificateAction":"none","UploadedFileNames":["uploaded_file","uploaded_test"],"URL":"https://test.com","UserAgent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64) Firefox/112.0","UserID":"166befbb-00e3-5e20-bd6e-27245723949f"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..0dd360ed42b --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-gateway-http-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-gateway-http-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/gateway_network.log --name gateway_network.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..143f02ca5b6 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/gateway_network.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/gateway_network.log b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/sample_logs/gateway_network.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/gateway_network.log rename to packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/docker/sample_logs/gateway_network.log diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/tf/files/test-gateway-network.log b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/tf/files/test-gateway-network.log new file mode 100644 index 00000000000..62317302895 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/tf/files/test-gateway-network.log @@ -0,0 +1 @@ +{"AccountID":"e1836771179f98aabb828da5ea69a111","Action":"allowedOnNoRuleMatch","Datetime":1684444377058000000,"DestinationIP":"89.160.20.129","DestinationPort":443,"DeviceID":"083a8354-d56c-11ed-9771-6a842b100cff","DeviceName":"zt-test-vm1","Email":"user@test.com","OverrideIP":"175.16.199.4","OverridePort":8080,"PolicyID":"85063bec-74cb-4546-85a3-e0cde2cdfda2","PolicyName":"My policy","SNI":"www.elastic.co","SessionID":"5f2d04be-3512-11e8-b467-0ed5f89f718b","SourceIP":"67.43.156.2","SourceInternalIP":"192.168.1.3","SourcePort":47924,"Transport":"tcp","UserID":"166befbb-00e3-5e20-bd6e-27245723949f"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..ede70c98e6e --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/gateway_network/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-gateway-network-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-gateway-network-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/http_request.log --name http_request.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..c59a6670da5 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/http_request.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/http_request.log b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/sample_logs/http_request.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/http_request.log rename to packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/docker/sample_logs/http_request.log diff --git a/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/tf/files/test-http-request.log b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/tf/files/test-http-request.log new file mode 100644 index 00000000000..bda066db44c --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/tf/files/test-http-request.log @@ -0,0 +1,2 @@ +{"BotDetectionIDs":[7,8,9],"BotScore":20,"BotScoreSrc":"Verified Bot","BotTags":["bing","api"],"CacheCacheStatus":"dynamic","CacheResponseBytes":983828,"CacheResponseStatus":200,"CacheTieredFill":false,"ClientASN":43766,"ClientCountry":"sa","ClientDeviceType":"desktop","ClientIP":"175.16.199.0","ClientIPClass":"noRecord","ClientMTLSAuthCertFingerprint":"Fingerprint","ClientMTLSAuthStatus":"unknown","ClientRequestBytes":5800,"ClientRequestHost":"xyz.example.com","ClientRequestMethod":"POST","ClientRequestPath":"/xyz/checkout","ClientRequestProtocol":"HTTP/1.1","ClientRequestReferer":"https://example.com/s/example/default?sourcerer=(default:(id:!n,selectedPatterns:!(example,%27logs-endpoint.*-example%27,%27logs-system.*-example%27,%27logs-windows.*-example%27)))&timerange=(global:(linkTo:!(),timerange:(from:%272022-05-16T06:26:36.340Z%27,fromStr:now-24h,kind:relative,to:%272022-05-17T06:26:36.340Z%27,toStr:now)),timeline:(linkTo:!(),timerange:(from:%272022-04-17T22:00:00.000Z%27,kind:absolute,to:%272022-04-18T21:59:59.999Z%27)))&timeline=(activeTab:notes,graphEventId:%27%27,id:%279844bdd4-4dd6-5b22-ab40-3cd46fce8d6b%27,isOpen:!t)","ClientRequestScheme":"https","ClientRequestSource":"edgeWorkerFetch","ClientRequestURI":"/s/example/api/telemetry/v2/clusters/_stats","ClientRequestUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36","ClientSrcPort":0,"ClientSSLCipher":"NONE","ClientSSLProtocol":"TLSv1.2","ClientTCPRTTMs":0,"ClientXRequestedWith":"Request With","Cookies":{"key":"value"},"EdgeCFConnectingO2O":false,"EdgeColoCode":"RUH","EdgeColoID":339,"EdgeEndTimestamp":"2022-05-25T13:25:32Z","EdgePathingOp":"wl","EdgePathingSrc":"macro","EdgePathingStatus":"nr","EdgeRateLimitAction":"unknown","EdgeRateLimitID":0,"EdgeRequestHost":"abc.example.com","EdgeResponseBodyBytes":980397,"EdgeResponseBytes":981308,"EdgeResponseCompressionRatio":0,"EdgeResponseContentType":"application/json","EdgeResponseStatus":200,"EdgeServerIP":"1.128.0.0","EdgeStartTimestamp":"2022-05-25T13:25:26Z","EdgeTimeToFirstByteMs":5333,"OriginDNSResponseTimeMs":3,"OriginIP":"67.43.156.0","OriginRequestHeaderSendDurationMs":0,"OriginResponseBytes":0,"OriginResponseDurationMs":5319,"OriginResponseHeaderReceiveDurationMs":5155,"OriginResponseHTTPExpires":"2022-05-27T13:25:26Z","OriginResponseHTTPLastModified":"2022-05-26T13:25:26Z","OriginResponseStatus":200,"OriginResponseTime":5232000000,"OriginSSLProtocol":"TLSv1.2","OriginTCPHandshakeDurationMs":24,"OriginTLSHandshakeDurationMs":53,"ParentRayID":"710e98d93d50357d","RayID":"710e98d9367f357d","SecurityLevel":"off","SmartRouteColoID":20,"UpperTierColoID":0,"SecurityAction":"unknown","WAFAttackScore":50,"WAFRCEAttackScore":1,"WAFSQLiAttackScore":99,"WAFXSSAttackScore":90,"WAFFlags":"0","WAFMatchedVar":"example","WAFProfile":"unknown","SecurityRuleID":"98d93d5","SecurityRuleDescription":"matchad variable message","WorkerCPUTime":0,"WorkerStatus":"unknown","WorkerSubrequest":true,"WorkerSubrequestCount":0,"ZoneID":393347122,"ZoneName":"example.com"} +{"ClientASN":8075,"ClientCountry":"NL","ClientDeviceType":"desktop","ClientIP":"89.160.20.156","ClientIPClass":"noRecord","ClientRequestBytes":7495,"ClientRequestHost":"api.deals-referral.com","ClientRequestMethod":"GET","ClientRequestPath":"/referralapi/api/deals/find","ClientRequestProtocol":"HTTP/1.1","ClientRequestReferer":"https://www.main-site.com/user/dashboard","ClientRequestURI":"/referralapi/api/deals/find?pricingCaseId=68c15fccf03482a4cb20460b","ClientRequestUserAgent":"got (https://github.com/sindresorhus/got)","ClientSSLCipher":"AEAD-AES256-GCM-SHA384","ClientSSLProtocol":"TLSv1.3","ClientSrcPort":32462,"ClientXRequestedWith":"XMLHttpRequest","EdgeColoCode":"AMS","EdgeColoID":996,"EdgeEndTimestamp":"2025-09-17T08:37:18.000Z","EdgePathingOp":"wl","EdgePathingSrc":"undef","EdgePathingStatus":"nr","EdgeRateLimitAction":"log","EdgeRateLimitID":12345,"EdgeRequestHost":"api.deals-referral.com","EdgeResponseBytes":1063,"EdgeResponseCompressionRatio":1,"EdgeResponseContentType":"application/json; charset=utf-8","EdgeResponseStatus":200,"EdgeServerIP":"89.160.20.156","EdgeStartTimestamp":"2025-09-17T08:37:18.000Z","EdgeTimeToFirstByteMs":65,"FirewallMatchesActions":"skip","FirewallMatchesRuleIDs":"fddd53f90acd49a89aea84e1c665269c","FirewallMatchesSources":"firewallManaged","OriginDNSResponseTimeMs":2,"OriginIP":"172.16.10.100","OriginResponseBytes":980,"OriginResponseDurationMs":62,"OriginResponseHTTPExpires":"Wed, 17 Sep 2025 08:42:18 GMT","OriginResponseHTTPLastModified":"Wed, 17 Sep 2025 08:30:00 GMT","OriginResponseStatus":200,"OriginResponseTime":62543210,"OriginSSLProtocol":"TLSv1.3","ParentRayID":"980747e7ad8f0001","RayID":"980747e7ad8fbdf3","SecurityLevel":"high","WAFAction":"log","WAFFlags":"0","WAFMatchedVar":"ARGS:pricingCaseId","WAFProfile":"default","WAFRuleID":"942100","WAFRuleMessage":"SQL Injection Attack Detected","WorkerCPUTime":1601,"WorkerStatus":"ok","WorkerSubrequest":false,"WorkerSubrequestCount":1,"WorkerWallTimeUs":36953,"ZoneID":296364127} diff --git a/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..3b9235be766 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/http_request/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-http-request-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-http-request-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/magic_ids.log --name magic_ids.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..e378118a04d --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/magic_ids.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/magic_ids.log b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/sample_logs/magic_ids.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/magic_ids.log rename to packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/docker/sample_logs/magic_ids.log diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/tf/files/test-magic-ids.log b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/tf/files/test-magic-ids.log new file mode 100644 index 00000000000..213b278ab5d --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/tf/files/test-magic-ids.log @@ -0,0 +1 @@ +{"Action":"pass","ColoCode":"NRT","DestinationIP":"89.160.20.129","DestinationPort":80,"Protocol":"tcp","SignatureID":2031296,"SignatureMessage":"ET CURRENT_EVENTS [Fireeye] POSSIBLE HackTool.TCP.Rubeus.[User32LogonProcesss]","SignatureRevision":1,"SourceIP":"67.43.156.2","SourcePort":44667,"Timestamp":"2023-09-11T03:02:57Z","ColoCity":"Tokyo"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..72626f53c67 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/magic_ids/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-magic-ids-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-magic-ids-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/nel_report.log --name nel_report.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..467b69e471a --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/nel_report.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/nel_report.log b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/sample_logs/nel_report.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/nel_report.log rename to packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/docker/sample_logs/nel_report.log diff --git a/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/tf/files/test-nel-report.log b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/tf/files/test-nel-report.log new file mode 100644 index 00000000000..5583050f828 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/tf/files/test-nel-report.log @@ -0,0 +1 @@ +{"ClientIPASN":"13335","ClientIPASNDescription":"CLOUDFLARENET","ClientIPCountry":"US","LastKnownGoodColoCode":"SJC","Phase":"connection","Timestamp":"2021-07-27T00:01:07Z","Type":"network-error"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..18e9062c89a --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/nel_report/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-nel-report-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-nel-report-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/network_analytics.log --name network_analytics.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..38ed6ee1b64 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/network_analytics.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/network_analytics.log b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/sample_logs/network_analytics.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/network_analytics.log rename to packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/docker/sample_logs/network_analytics.log diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/tf/files/test-network-analytics.log b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/tf/files/test-network-analytics.log new file mode 100644 index 00000000000..4196c40179b --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/tf/files/test-network-analytics.log @@ -0,0 +1 @@ +{"AttackCampaignID":"xyz987","AttackID":"abc777","ColoCountry":"AD","ColoGeoHash":"gbuun","ColoID":46,"ColoName":"SJC","Datetime":"2021-07-27T00:01:07Z","DestinationASN":1900,"DestinationASNDescription":"asn description","DestinationCountry":"AD","DestinationGeoHash":"gbuun","DestinationPort":0,"Direction":"ingress","GREChecksum":10,"GREEthertype":10,"GREHeaderLength":1024,"GREKey":10,"GRESequenceNumber":10,"GREVersion":10,"ICMPChecksum":10,"ICMPCode":10,"ICMPType":10,"IPDestinationAddress":"175.16.199.0","IPDestinationSubnet":"/24","IPFragmentOffset":1480,"IPHeaderLength":20,"IPMoreFragments":1480,"IPProtocol":6,"IPProtocolName":"tcp","IPSourceAddress":"67.43.156.0","IPSourceSubnet":"/24","IPTotalLength":1024,"IPTotalLengthBuckets":10,"IPTtl":240,"IPTtlBuckets":2,"IPv4Checksum":0,"IPv4DontFragment":0,"IPv4Dscp":46,"IPv4Ecn":1,"IPv4Identification":1,"IPv4Options":1,"IPv6Dscp":46,"IPv6Ecn":1,"IPv6ExtensionHeaders":"header","IPv6FlowLabel":1,"IPv6Identification":1,"MitigationReason":"BLOCKED","MitigationScope":"local","MitigationSystem":"flowtrackd","Outcome":"pass","ProtocolState":"OPEN","RuleID":"rule1","RulesetID":"3b64149bfa6e4220bbbc2bd6db589552","RulesetOverrideID":"id1","SampleInterval":1,"SourceASN":1500,"SourceASNDescription":"Source ASN Description","SourceCountry":"AD","SourceGeoHash":"gbuun","SourcePort":0,"TCPAcknowledgementNumber":1000,"TCPChecksum":10,"TCPDataOffset":0,"TCPFlags":1,"TCPFlagsString":"Human-readable flags string","TCPMss":512,"TCPOptions":"mss","TCPSackBlocks":1,"TCPSacksPermitted":1,"TCPSequenceNumber":100,"TCPTimestampEcr":100,"TCPTimestampValue":100,"TCPUrgentPointer":10,"TCPWindowScale":10,"TCPWindowSize":10,"UDPChecksum":10,"UDPPayloadLength":10,"Verdict":"pass"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..08959f3f35c --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/network_analytics/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-network-analytics-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-network-analytics-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/network_session.log --name network_session.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..54a85de1193 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/network_session.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/network_session.log b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/sample_logs/network_session.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/network_session.log rename to packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/docker/sample_logs/network_session.log diff --git a/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/tf/files/test-network-session.log b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/tf/files/test-network-session.log new file mode 100644 index 00000000000..25d8c3d328b --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/tf/files/test-network-session.log @@ -0,0 +1 @@ +{"AccountID":"e1836771179f98aabb828da5ea69a111","BytesReceived":679,"BytesSent":2333,"ClientTCPHandshakeDurationMs":12,"ClientTLSCipher":"TLS_AES_128_GCM_SHA256","ClientTLSHandshakeDurationMs":125,"ClientTLSVersion":"TLS 1.3","ConnectionCloseReason":"CLIENT_CLOSED","ConnectionReuse":false,"DestinationTunnelID":"00000000-0000-0000-0000-000000000000","DeviceID":"083a8354-d56c-11ed-9771-6a842b100cff","DeviceName":"zt-test-vm1","EgressColoName":"ORD","EgressIP":"2a02:cf40::23","EgressPort":41052,"EgressRuleID":"00000000-0000-0000-0000-000000000000","EgressRuleName":"Egress Rule 1","Email":"user@test.com","IngressColoName":"ORD","Offramp":"INTERNET","OriginIP":"89.160.20.129","OriginPort":80,"OriginTLSCertificateIssuer":"DigiCert Inc","OriginTLSCertificateValidationResult":"VALID","OriginTLSCipher":"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","OriginTLSHandshakeDurationMs":130,"OriginTLSVersion":"TLS 1.2","Protocol":"TCP","RuleEvaluationDurationMs":10,"SessionEndTime":"2023-05-04T11:29:14Z","SessionID":"18881f179300007fb0d06d6400000001","SessionStartTime":"2023-05-04T11:29:14Z","SourceInternalIP":"1.128.0.1","SourceIP":"67.43.156.2","SourcePort":52994,"UserID":"166befbb-00e3-5e20-bd6e-27245723949f","VirtualNetworkID":"0ce99869-63d3-4d5d-bdaf-d4f33df964aa"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..4662700b745 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/network_session/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-network-session-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-network-session-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/page_shield_events.log --name page_shield_events.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..a82e9eff32f --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/page_shield_events.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/page_shield_events.log b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/sample_logs/page_shield_events.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/page_shield_events.log rename to packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/docker/sample_logs/page_shield_events.log diff --git a/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/tf/files/test-page-shield-events.log b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/tf/files/test-page-shield-events.log new file mode 100644 index 00000000000..9b88538ad0f --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/tf/files/test-page-shield-events.log @@ -0,0 +1 @@ +{"Action":"log","CSPDirective":"directive","Host":"hostymchost.face","PageURL":"http://example.com/?query=42","PolicyID":"9","ResourceType":"other","Timestamp":"2023-05-04T11:29:14Z","URL":"https://example.com/?query=hog","URLContainsCDNCGIPath":true,"URLHost":"example.com"} diff --git a/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..ccb2e7684ae --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/page_shield_events/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-page-shield-events-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-page-shield-events-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/sinkhole_http.log --name sinkhole_http.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..2b32adb2d88 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/sinkhole_http.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/sinkhole_http.log b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/sample_logs/sinkhole_http.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/sinkhole_http.log rename to packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/docker/sample_logs/sinkhole_http.log diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/tf/files/test-sinkhole-http.log b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/tf/files/test-sinkhole-http.log new file mode 100644 index 00000000000..a368eb7d09a --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/tf/files/test-sinkhole-http.log @@ -0,0 +1 @@ +{"AccountID":"AC123456","Body":"{\"action\": \"login\", \"user\": \"john_doe\"}","BodyLength":39,"DestAddr":"89.160.20.129","Headers":"Host: example.com\nUser-Agent: Mozilla/5.0\nAccept: */*\nConnection: keep-alive","Host":"example.com","Method":"POST","Password":"password123","R2Path":"","Referrer":"https://searchengine.com/","SinkholeID":"SH001","SrcAddr":"67.43.156.2","Timestamp":"2023-09-19T12:00:00Z","URI":"/api/v1/login","URL":"https://example.com/api/v1/login","UserAgent":"Mozilla/5.0","Username":"john_doe"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..cd8f0fd03e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-sinkhole-http-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-sinkhole-http-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/spectrum_event.log --name spectrum_event.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..dbb580973f7 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/spectrum_event.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/spectrum_event.log b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/sample_logs/spectrum_event.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/spectrum_event.log rename to packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/docker/sample_logs/spectrum_event.log diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/tf/files/test-spectrum-event.log b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/tf/files/test-spectrum-event.log new file mode 100644 index 00000000000..9e7a0fac7ca --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/tf/files/test-spectrum-event.log @@ -0,0 +1 @@ +{"Application":"7ef659a2f8ef4810a9bade96fdad7c75","ClientAsn":200391,"ClientBytes":0,"ClientCountry":"bg","ClientIP":"67.43.156.0","ClientMatchedIpFirewall":"UNKNOWN","ClientPort":40456,"ClientProto":"tcp","ClientTcpRtt":0,"ClientTlsCipher":"UNK","ClientTlsClientHelloServerName":"server name","ClientTlsProtocol":"unknown","ClientTlsStatus":"UNKNOWN","ColoCode":"SOF","ConnectTimestamp":"2022-05-26T09:24:00Z","DisconnectTimestamp":"1970-01-01T00:00:00Z","Event":"connect","IpFirewall":false,"OriginBytes":0,"OriginIP":"175.16.199.0","OriginPort":3389,"OriginProto":"tcp","OriginTcpRtt":0,"OriginTlsCipher":"UNK","OriginTlsFingerprint":"0000000000000000000000000000000000000000000000000000000000000000.","OriginTlsMode":"off","OriginTlsProtocol":"unknown","OriginTlsStatus":"UNKNOWN","ProxyProtocol":"off","Status":0,"Timestamp":"2022-05-26T09:24:00Z"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..2e19ce09e89 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-spectrum-event-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-spectrum-event-queue-${var.TEST_RUN_ID}" + policy = < + sh -c " + sleep 5 && + export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-blob-storage-emulator:10000/devstoreaccount1;' && + az storage container create --name test-container && + az storage blob upload --container-name test-container --file /sample_logs/workers_trace.log --name workers_trace.log + " + gcs-mock-service: + image: golang:1.24.7-alpine + working_dir: /app + volumes: + - ./gcs-mock-service:/app + - ./files/manifest.yml:/files/manifest.yml:ro + - ./sample_logs/:/data + ports: + - "4443/tcp" + healthcheck: + test: "wget --no-verbose --tries=1 --spider http://localhost:4443/health || exit 1" + interval: 10s + timeout: 5s + command: go run main.go -manifest /files/manifest.yml diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/files/manifest.yml b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/files/manifest.yml new file mode 100644 index 00000000000..49d99812552 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/files/manifest.yml @@ -0,0 +1,5 @@ +buckets: + testbucket: + files: + - path: /data/workers_trace.log + content-type: application/json diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/gcs-mock-service/go.mod b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/gcs-mock-service/go.mod new file mode 100644 index 00000000000..df08767f7e8 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/gcs-mock-service/go.mod @@ -0,0 +1,5 @@ +module gcs-mock-service + +go 1.24.7 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/gcs-mock-service/go.sum b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/gcs-mock-service/go.sum new file mode 100644 index 00000000000..a62c313c5b0 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/gcs-mock-service/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/gcs-mock-service/main.go b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/gcs-mock-service/main.go new file mode 100644 index 00000000000..391e75c7efe --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/gcs-mock-service/main.go @@ -0,0 +1,297 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func main() { + host := flag.String("host", "0.0.0.0", "host to listen on") + port := flag.String("port", "4443", "port to listen on") + manifest := flag.String("manifest", "", "path to YAML manifest file for preloading buckets and objects") + flag.Parse() + + addr := fmt.Sprintf("%s:%s", *host, *port) + + fmt.Printf("Starting mock GCS server on http://%s\n", addr) + if *manifest != "" { + m, err := readManifest(*manifest) + if err != nil { + log.Fatalf("error reading manifest: %v", err) + } + if err := processManifest(m); err != nil { + log.Fatalf("error processing manifest: %v", err) + } + } else { + fmt.Println("Store is empty. Create buckets and objects via API calls.") + } + + // setup HTTP handlers + mux := http.NewServeMux() + // health check + mux.HandleFunc("/health", healthHandler) + // standard gcs api calls + mux.HandleFunc("GET /storage/v1/b/{bucket}/o", handleListObjects) + mux.HandleFunc("GET /storage/v1/b/{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("POST /storage/v1/b", handleCreateBucket) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o", handleUploadObject) + mux.HandleFunc("POST /upload/storage/v1/b/{bucket}/o/{object...}", handleUploadObject) + // direct path-style gcs sdk calls + mux.HandleFunc("GET /{bucket}/o/{object...}", handleGetObject) + mux.HandleFunc("GET /{bucket}/{object...}", handleGetObject) + // debug: log all requests + loggedMux := loggingMiddleware(mux) + + if err := http.ListenAndServe(addr, loggedMux); err != nil { + log.Fatalf("failed to start server: %v", err) + } +} + +// loggingMiddleware logs incoming HTTP requests. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s %s\n", r.Method, r.URL.Path) + next.ServeHTTP(w, r) + }) +} + +// readManifest reads and parses the YAML manifest file. +func readManifest(path string) (*Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read manifest: %w", err) + } + + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil +} + +// processManifest creates buckets and uploads objects as specified in the manifest. +func processManifest(manifest *Manifest) error { + for bucketName, bucket := range manifest.Buckets { + for _, file := range bucket.Files { + fmt.Printf("preloading data for bucket: %s | path: %s | content-type: %s...\n", + bucketName, file.Path, file.ContentType) + + if err := createBucket(bucketName); err != nil { + return fmt.Errorf("failed to create bucket '%s': %w", bucketName, err) + } + data, err := os.ReadFile(file.Path) + if err != nil { + return fmt.Errorf("failed to read bucket data file '%s': %w", file.Path, err) + } + pathParts := strings.Split(file.Path, "/") + if _, err := uploadObject(bucketName, pathParts[len(pathParts)-1], data, file.ContentType); err != nil { + return fmt.Errorf("failed to create object '%s' in bucket '%s': %w", file.Path, bucketName, err) + } + } + } + return nil +} + +// healthHandler responds with a simple "OK" message for health checks. +func healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "OK") +} + +// handleListObjects lists all objects in the specified bucket. +func handleListObjects(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + + if bucket, ok := inMemoryStore[bucketName]; ok { + response := GCSListResponse{ + Kind: "storage#objects", + Items: []GCSObject{}, + } + for name, object := range bucket { + item := GCSObject{ + Kind: "storage#object", + Name: name, + Bucket: bucketName, + Size: strconv.Itoa(len(object.Data)), + ContentType: object.ContentType, + } + response.Items = append(response.Items, item) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleGetObject retrieves a specific object from a bucket. +func handleGetObject(w http.ResponseWriter, r *http.Request) { + bucketName := r.PathValue("bucket") + objectName := r.PathValue("object") + + if bucketName == "" || objectName == "" { + http.Error(w, "not found: invalid URL format", http.StatusNotFound) + return + } + + if bucket, ok := inMemoryStore[bucketName]; ok { + if object, ok := bucket[objectName]; ok { + w.Header().Set("Content-Type", object.ContentType) + w.Write(object.Data) + return + } + } + http.Error(w, "not found", http.StatusNotFound) +} + +// handleCreateBucket creates a new bucket. +func handleCreateBucket(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + var bucketInfo struct { + Name string `json:"name"` + } + if err := json.NewDecoder(r.Body).Decode(&bucketInfo); err != nil { + http.Error(w, "invalid JSON body", http.StatusBadRequest) + return + } + if bucketInfo.Name == "" { + http.Error(w, "bucket name is required", http.StatusBadRequest) + return + } + + if err := createBucket(bucketInfo.Name); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(bucketInfo) +} + +// handleUploadObject uploads an object to a specified bucket. +func handleUploadObject(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + + bucketName := r.PathValue("bucket") + objectName := r.URL.Query().Get("name") + if objectName == "" { + objectName = r.PathValue("object") + } + + if bucketName == "" || objectName == "" { + http.Error(w, "missing bucket or object name", http.StatusBadRequest) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "failed to read request body", http.StatusInternalServerError) + return + } + defer r.Body.Close() + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + + response, err := uploadObject(bucketName, objectName, data, contentType) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func createBucket(bucketName string) error { + if _, exists := inMemoryStore[bucketName]; exists { + return fmt.Errorf("bucket already exists") + } + inMemoryStore[bucketName] = make(map[string]ObjectData) + log.Printf("created bucket: %s", bucketName) + return nil +} + +func uploadObject(bucketName, objectName string, data []byte, contentType string) (*GCSObject, error) { + if _, ok := inMemoryStore[bucketName]; !ok { + return nil, fmt.Errorf("bucket not found") + } + + inMemoryStore[bucketName][objectName] = ObjectData{ + Data: data, + ContentType: contentType, + } + log.Printf("created object '%s' in bucket '%s' with Content-Type '%s'", + objectName, bucketName, contentType) + + return &GCSObject{ + Kind: "storage#object", + Name: objectName, + Bucket: bucketName, + Size: strconv.Itoa(len(data)), + ContentType: contentType, + }, nil +} + +// The in-memory store to hold ObjectData structs. +var inMemoryStore = make(map[string]map[string]ObjectData) + +// ObjectData stores the raw data and its content type. +type ObjectData struct { + Data []byte + ContentType string +} + +// GCSListResponse mimics the structure of a real GCS object list response. +type GCSListResponse struct { + Kind string `json:"kind"` + Items []GCSObject `json:"items"` +} + +// GCSObject mimics the structure of a GCS object resource with ContentType. +type GCSObject struct { + Kind string `json:"kind"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size string `json:"size"` + ContentType string `json:"contentType"` +} + +// Manifest represents the top-level structure of the YAML file +type Manifest struct { + Buckets map[string]Bucket `yaml:"buckets"` +} + +// Bucket represents each bucket and its files +type Bucket struct { + Files []File `yaml:"files"` +} + +// File represents each file entry inside a bucket +type File struct { + Path string `yaml:"path"` + ContentType string `yaml:"content-type"` +} diff --git a/packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/workers_trace.log b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/sample_logs/workers_trace.log similarity index 100% rename from packages/cloudflare_logpush/_dev/deploy/docker/sample_logs/workers_trace.log rename to packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/docker/sample_logs/workers_trace.log diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/tf/env.yml b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/tf/env.yml new file mode 100644 index 00000000000..aee5f1c5900 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/tf/env.yml @@ -0,0 +1,9 @@ +version: '2.3' +services: + terraform: + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_DEFAULT_PROFILE=${AWS_DEFAULT_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/tf/files/test-workers-trace.log b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/tf/files/test-workers-trace.log new file mode 100644 index 00000000000..c92856e56e2 --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/tf/files/test-workers-trace.log @@ -0,0 +1 @@ +{"Event":{"RayID":"7e9ae7157ac0c33a","Request":{"URL":"http://chat-gpt-little-butterfly-0c3d.example.workers.dev/v2/_catalog","Method":"GET"},"Response":{"Status":404}},"EventTimestampMs":1689852946804,"EventType":"fetch","Exceptions":[{"Message":"Uncaught TypeError: Cannot read property 'x' of undefined","Stack":"TypeError: Cannot read property 'x' of undefined\n at fetchHandler (/workers/script.js:12:27)\n at handleRequest (/workers/script.js:6:13)"}],"Logs":[{"level":"info","message":"Request received for /api/data"},{"level":"error","message":"Something went wrong"}],"Outcome":"exception","ScriptName":"chat-gpt-little-butterfly-0c3d","ScriptTags":["api","chatgpt"],"DispatchNamespace":"my-worker-dispatch"} \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/tf/main.tf b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/tf/main.tf new file mode 100644 index 00000000000..5573dc7fd3c --- /dev/null +++ b/packages/cloudflare_logpush/data_stream/workers_trace/_dev/deploy/tf/main.tf @@ -0,0 +1,57 @@ +provider "aws" { + region = "us-east-1" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } +} + +resource "aws_s3_bucket" "bucket" { + bucket = "elastic-package-workers-trace-bucket-${var.TEST_RUN_ID}" +} + +resource "aws_sqs_queue" "queue" { + name = "elastic-package-workers-trace-queue-${var.TEST_RUN_ID}" + policy = < Date: Wed, 31 Dec 2025 17:45:41 +0530 Subject: [PATCH 3/3] Address PR comments --- .../deploy/docker/sample_logs/test-events.log | 1 + .../_dev/deploy/tf/files/test-events.log | 1 + .../_dev/test/system/test-azure-config.yml | 2 +- .../_dev/test/system/test-default-config.yml | 4 +- .../_dev/test/system/test-gcs-config.yml | 2 +- .../data_stream/event/sample_event.json | 40 +++++++++---------- .../symantec_endpoint_security/docs/README.md | 40 +++++++++---------- 7 files changed, 46 insertions(+), 44 deletions(-) diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/sample_logs/test-events.log b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/sample_logs/test-events.log index 2a631c602e4..25f796d769d 100644 --- a/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/sample_logs/test-events.log +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/docker/sample_logs/test-events.log @@ -1 +1,2 @@ {"category_id":3,"collector_device_ip":"175.16.199.0","collector_device_name":"Collector_Device_Name","collector_name":"Collection12","collector_uid":"TT1456","composite":1,"container":{"host_name":"azure-us.local","image_name":"Image-sp2133","image_uid":"SH4322323","name":"User12345","networks":[{"bssid":"Container-Networks-BSSID 1","gateway_ip":"89.160.20.112","gateway_mac":"00:B0:D0:63:C2:01","ipv4":"81.2.69.144","ipv6":"2a02:cf40::","is_public":true,"mac":"00:B0:D0:63:C2:02","rep_score_id":0,"ssid":"SSID-4326451","type_id":0},{"bssid":"HN0845435","gateway_ip":"81.2.69.142","gateway_mac":"00:B0:D0:63:C2:03","ipv4":"81.2.69.144","ipv6":"2a02:cf40::","is_public":true,"mac":"00:B0:D0:63:C2:04","rep_score_id":1,"ssid":"Container-Networks-SSID 2","type_id":1}],"os_name":"Windows","uid":"UU35r3454"},"correlation_uid":"DD78666","count":563,"customer_registry_uid":"CP45254","customer_uid":"CH32534","cybox":{"domains":["example.com","abc.com"],"emails":[{"direction_id":0,"header_from":"abc@example.com","header_message_id":"Cybox-Emails-Header_Message_ID 1","header_reply_to":"Cybox-Emails-Header_Reply_To 1","header_subject":"Cybox-Emails-Header_Subject 1","header_to":["Cybox-Emails-Header_To 1","Cybox-Emails-Header_Tos 1"],"sender_ip":"81.2.69.144","size":12345678901,"smtp_from":"Cybox-Emails-SMTP_From 1","smtp_hello":"Cybox-Emails-SMTP_Hello 1","smtp_to":"Cybox-Emails-SMTP_To 1"},{"direction_id":1,"header_from":"abc@example.com","header_message_id":"Cybox-Emails-Header_Message_ID 2","header_reply_to":"Cybox-Emails-Header_Reply_To 2","header_subject":"Cybox-Emails-Header_Subject 2","header_to":["Cybox-Emails-Header_To 2","Cybox-Emails-Header_Tos 2"],"sender_ip":"81.2.69.144","size":12345678902,"smtp_from":"Cybox-Emails-SMTP_From 2","smtp_hello":"Cybox-Emails-SMTP_Hello 2","smtp_to":"Cybox-Emails-SMTP_To 2"}],"files":[{"accessed":1613021404000,"accessor":"Cybox-Files-Accessor 1","attribute_ids":[1,2,3,4,5,6,7,8,9,10],"attributes":12345678901,"company_name":"Microsoft Corporation","confidentiality_id":0,"content_type":{"family_id":0,"subtype":"SubType 1","type_id":0},"created":1613021404000,"creator":"Creator 1","creator_process":"Cybox-Files-Creator_Process 1","desc":"Cybox-Files-Desc 1","folder":"c:\\windows\\system32\\cybox\\files\\folder\\1","folder_uid":"Cybox-Files-Folder_UID 1","is_system":true,"md5":"HFDajsdf3254345436","mime_type":"Cybox-Files-MIME_Type 1","modified":1613021404000,"modifier":"Cybox-Files-Modifier 1","name":"cybox_files_name_1.exe","normalized_path":"CSIDL_SYSTEM\\cybox_files_normalized_path_1.exe","original_name":"Cybox-Files-Original_Name 1","owner":"Cybox-Files-Owner 1","parent_name":"Cybox-Files-Parent_Name 1","parent_sha2":"Cybox-Files-Parent_SHA2 1","path":"c:\\windows\\system32\\cybox_files_path_1.exe","product_name":"Windows Internet Explorer 1","product_path":"Cybox-Files-Product_Path 1","rep_discovered_band":0,"rep_discovered_date":1613021404000,"rep_prevalence":12345678901,"rep_prevalence_band":0,"rep_score":12345678901,"rep_score_band":0,"security_descriptor":"Cybox-Files-Security_Descriptor 1","sha1":"Cybox-Files-SHA1 1","sha2":"Cybox-Files-SHA2 1","signature_company_name":"Cybox-Files-Signature_Company_Name 1","signature_created_date":1613021404000,"signature_developer_uid":"Cybox-Files-Signature_Developer_UID 1","signature_fingerprints":[{"algorithm":"Cybox-Files-Signature_Fingerprints-Algorithm 1","value":"Cybox-Files-Signature_Fingerprints-Value 1"},{"algorithm":"Cybox-Files-Signature_Fingerprints-Algorithms 1","value":"Cybox-Files-Signature_Fingerprints-Values 1"}],"signature_issuer":"Cybox-Files-Signature_Issuer 1","signature_level_id":0,"signature_serial_number":"Cybox-Files-Signature_Serial_Number 1","signature_value":12345678901,"signature_value_ids":[0,1,2,3,4,5,6,7,8,9,10],"size":12345678901,"size_compressed":12345678901,"src_ip":"81.2.69.142","src_name":"Cybox-Files-SRC_Name 1","type_id":1,"uid":"Cybox-Files-UID 1","url":{"categories":["Cybox-Files-URL-Category 1","Cybox-Files-URL-Categories 1"],"category_ids":[1,3,4,5,6,7,9,11,14,15,16,17,18,20,21,22,23,24,25,26,27,29,30,31,32,33,34,35,36,37,38,40,43,44,45,46,47,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,65,66,67,68,71,83,84,85,86,87,88,89,90,92,93,95,96,97,98],"extension":"Cybox-Files-URL-Extension 1","host":"www.files-url-host-1.com","method":"Cybox-Files-URL-Method 1","parent_categories":["Cybox-Files-URL-Parent_Category 1","Cybox-Files-URL-Parent_Categories 1"],"path":"/download/trouble/cybox/files/url/path/1","port":80,"provider":"Cybox-Files-URL-Provider 1","query":"q=bad&sort=date_1","referrer":"Cybox-Files-URL-Referrer 1","referrer_categories":["Cybox-Files-URL-Referrer_Category 1","Cybox-Files-URL-Referrer_Categories 1"],"referrer_category_ids":[12345678901,67890123451],"rep_score_id":0,"scheme":"Cybox-Files-URL-Scheme 1","text":"www.files-url-text-1.com/download/trouble"},"version":"Cybox-Files-Version 1","xattributes":{"ads_name":"Cybox-Files-XAttributes-ADS_Name 1","ads_size":"Cybox-Files-XAttributes-ADS_Size 1","dacl":"Cybox-Files-XAttributes-DACL 1","owner":"Cybox-Files-XAttributes-Owner 1","primary_group":"Cybox-Files-XAttributes-Primary_Group 1","link_name":"Cybox-Files-XAttributes-Link_Name 1","hard_link_count":"Cybox-Files-XAttributes-Hard_Link_Count 1","Unix_permissions":"Cybox-Files-XAttributes-Unix_Permissions 1"}},{"accessed":1613021404000,"accessor":"Cybox-Files-Accessor 2","attribute_ids":[11,12,13,14,15,16,17],"attributes":12345678902,"company_name":"Microsoft Corporation 2","confidentiality_id":1,"content_type":{"family_id":1,"subtype":"Cybox-Files-Content_Type-SubType 2","type_id":1},"created":1613021404000,"creator":"Cybox-Files-Creator 2","creator_process":"Cybox-Files-Creator_Process 2","desc":"Cybox-Files-Desc 2","folder":"c:\\windows\\system32\\cybox\\files\\folder\\2","folder_uid":"Cybox-Files-Folder_UID 2","is_system":true,"md5":"Cybox-Files-MD5 2","mime_type":"Cybox-Files-MIME_Type 2","modified":1613021404000,"modifier":"Cybox-Files-Modifier 2","name":"cybox_files_name_2.exe","normalized_path":"CSIDL_SYSTEM\\cybox_files_normalized_path_2.exe","original_name":"Cybox-Files-Original_Name 2","owner":"Cybox-Files-Owner 2","parent_name":"Cybox-Files-Parent_Name 2","parent_sha2":"Cybox-Files-Parent_SHA2 2","path":"c:\\windows\\system32\\cybox_files_path_2.exe","product_name":"Windows Internet Explorer 2","product_path":"Cybox-Files-Product_Path 2","rep_discovered_band":1,"rep_discovered_date":1613021404000,"rep_prevalence":12345678902,"rep_prevalence_band":1,"rep_score":12345678902,"rep_score_band":1,"security_descriptor":"Cybox-Files-Security_Descriptor 2","sha1":"Cybox-Files-SHA1 2","sha2":"Cybox-Files-SHA2 2","signature_company_name":"Cybox-Files-Signature_Company_Name 2","signature_created_date":1613021404000,"signature_developer_uid":"Cybox-Files-Signature_Developer_UID 2","signature_fingerprints":[{"algorithm":"Cybox-Files-Signature_Fingerprints-Algorithm 2","value":"Cybox-Files-Signature_Fingerprints-Value 2"},{"algorithm":"Cybox-Files-Signature_Fingerprints-Algorithms 2","value":"Cybox-Files-Signature_Fingerprints-Values 2"}],"signature_issuer":"Cybox-Files-Signature_Issuer 2","signature_level_id":1,"signature_serial_number":"Cybox-Files-Signature_Serial_Number 2","signature_value":12345678902,"signature_value_ids":[11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],"size":12345678902,"size_compressed":12345678902,"src_ip":"81.2.69.144","src_name":"Cybox-Files-SRC_Name 2","type_id":1,"uid":"Cybox-Files-UID 2","url":{"categories":["Cybox-Files-URL-Category 2","Cybox-Files-URL-Categories 2"],"category_ids":[101,102,103,104,105,106,107,108,109,110,111,112,113,114,116,117,118,121,124],"extension":"Cybox-Files-URL-Extension 2","host":"www.files-url-host-2.com","method":"Cybox-Files-URL-Method 2","parent_categories":["Cybox-Files-URL-Parent_Category 2","Cybox-Files-URL-Parent_Categories 2"],"path":"/download/trouble/cybox/files/url/path/2","port":81,"provider":"Cybox-Files-URL-Provider 2","query":"q=bad&sort=date_2","referrer":"Cybox-Files-URL-Referrer 2","referrer_categories":["Cybox-Files-URL-Referrer_Category 2","Cybox-Files-URL-Referrer_Categories 2"],"referrer_category_ids":[12345678902,67890123452],"rep_score_id":1,"scheme":"Cybox-Files-URL-Scheme 2","text":"www.files-url-text-2.com/download/trouble"},"version":"Cybox-Files-Version 2","xattributes":{"ads_name":"Cybox-Files-XAttributes-ADS_Name 2","ads_size":"Cybox-Files-XAttributes-ADS_Size 2","dacl":"Cybox-Files-XAttributes-DACL 2","owner":"Cybox-Files-XAttributes-Owner 2","primary_group":"Cybox-Files-XAttributes-Primary_Group 2","link_name":"Cybox-Files-XAttributes-Link_Name 2","hard_link_count":"Cybox-Files-XAttributes-Hard_Link_Count 2","Unix_permissions":"Cybox-Files-XAttributes-Unix_Permissions 2"}}],"hostnames":["Cybox-Hostname 1","Cybox-Hostnames 1"],"icap_reqmod":[{"metadata":{"field1_keyword":"Cybox-ICAP_ReqMod-field1_Keyword","field1_number":12345678901,"field1_boolean":true,"field1_ip":"175.16.199.0"},"service":"Cybox-ICAP_ReqMod-Service 1","status":"Cybox-ICAP_ReqMod-Status 1","status_detail":"Cybox-ICAP_ReqMod-Status_Detail 1"},{"metadata":{"field2_keyword":"Cybox-ICAP_ReqMod-field2_Keyword","field2_number":12345678902,"field2_boolean":true,"field2_ip":"175.16.199.0"},"service":"Cybox-ICAP_ReqMod-Service 2","status":"Cybox-ICAP_ReqMod-Status 2","status_detail":"Cybox-ICAP_ReqMod-Status_Detail 2"}],"icap_respmod":[{"metadata":{"field1_keyword":"Cybox-ICAP_RespMod-field1_Keyword","field1_number":12345678901,"field1_boolean":true,"field1_ip":"175.16.199.0"},"service":"Cybox-ICAP_RespMod-Service 1","status":"Cybox-ICAP_RespMod-Status 1","status_detail":"Cybox-ICAP_RespMod-Status_Detail 1"},{"metadata":{"field2_keyword":"Cybox-ICAP_RespMod-field2_Keyword","field2_number":12345678902,"field2_boolean":true,"field2_ip":"175.16.199.0"},"service":"Cybox-ICAP_RespMod-Service 2","status":"Cybox-ICAP_RespMod-Status 2","status_detail":"Cybox-ICAP_RespMod-Status_Detail 2"}],"ipv4s":["175.16.199.0","175.16.199.0"],"ipv6s":["2a02:cf40::","2a02:cf40::"],"macs":["00:B0:D0:63:C2:05","00:B0:D0:63:C2:06"],"urls":[{"categories":["Cybox-URLs-Category 1","Cybox-URLs-Categories 1"],"category_ids":[1,3,4,5,6,7,9,11,14,15,16,17,18,20,21,22,23,24,25,26,27,29,30,31,32,33,34,35,36,37,38,40,43,44,45,46,47,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,65,66,67,68,71,83,84,85,86,87,88,89,90,92,93,95,96,97,98],"extension":"Cybox-URLs-Extension 1","host":"www.urls-host-1.com","method":"Cybox-URLs-Method 1","parent_categories":["Cybox-URLs-Parent_Category 1","Cybox-URLs-Parent_Categories 1"],"path":"/download/trouble/cybox/urls/path/1","port":80,"provider":"Cybox-URLs-Provider 1","query":"q=bad&sort=date_1","referrer":"Cybox-URLs-Referrer 1","referrer_categories":["Cybox-URLs-Referrer_Category 1","Cybox-URLs-Referrer_Categories 1"],"referrer_category_ids":[12345678901,67890123451],"rep_score_id":0,"scheme":"Cybox-URLs-Scheme 1","text":"www.urls-text-1.com/download/trouble"},{"categories":["Cybox-URLs-Category 2","Cybox-URLs-Categories 2"],"category_ids":[101,102,103,104,105,106,107,108,109,110,111,112,113,114,116,117,118,121,124],"extension":"Cybox-URLs-Extension 2","host":"www.urls-host-2.com","method":"Cybox-URLs-Method 2","parent_categories":["Cybox-URLs-Parent_Category 2","Cybox-URLs-Parent_Categories 2"],"path":"/download/trouble/cybox/urls/path/2","port":81,"provider":"Cybox-URLs-Provider 2","query":"q=bad&sort=date_2","referrer":"Cybox-URLs-Referrer 2","referrer_categories":["Cybox-URLs-Referrer_Category 2","Cybox-URLs-Referrer_Categories 2"],"referrer_category_ids":[12345678902,67890123452],"rep_score_id":1,"scheme":"Cybox-URLs-Scheme 2","text":"www.urls-text-2.com/download/trouble"}]},"device_alias_name":"Device_Alias_Name","device_cap":"Device_Cap","device_cloud_vm":{"autoscale_uid":"Device_Cloud_VM-Autoscale_UID","dc_region":"Device_Cloud_VM-DC_Region","instance_uid":"Device_Cloud_VM-Instance_UID","subnet_uid":"Device_Cloud_VM-Subnet_UID","vpc_uid":"Device_Cloud_VM-VPC_UID"},"device_desc":"Device_Desc","device_domain":"device.domain.internal.somecompany.com","device_domain_uid":"Device_Domain_UID","device_end_time":1613021404000,"device_gateway":"175.16.199.0","device_group":"Device_Group","device_group_name":"Device_Group_Name","device_hw_bios_date":"03/31/16","device_hw_bios_manufacturer":"LENOVO","device_hw_bios_ver":"LENOVO G5ETA2WW (2.62)","device_hw_cpu_type":"x86 Family 6 Model 37 Stepping 5","device_imei":"Device_IMEI","device_ip":"175.16.199.0","device_is_compliant":true,"device_is_personal":true,"device_is_trusted":true,"device_is_unmanaged":true,"device_location":{"city":"Device_Location-City","continent":"Device_Location-Continent","coordinates":[-12.345,56.789],"country":"US","desc":"Device_Location-Desc","isp":"Device_Location-ISP","on_premises":true,"region":"US-CA"},"device_mac":"00:B0:D0:63:C2:07","device_name":"device.name.computer.domain","device_name_md5":"4ED962DDBF17E2BBA7B14EBC00F3162E","device_networks":[{"bssid":"Device_Networks-BSSID 1","gateway_ip":"175.16.199.0","gateway_mac":"00:B0:D0:63:C2:08","ipv4":"175.16.199.0","ipv6":"2a02:cf40::","is_public":true,"mac":"00:B0:D0:63:C2:09","rep_score_id":0,"ssid":"Device_Networks-SSID 1","type_id":0},{"bssid":"Device_Networks-BSSID 2","gateway_ip":"89.160.20.112","gateway_mac":"00:B0:D0:63:C2:10","ipv4":"89.160.20.112","ipv6":"2a02:cf40::","is_public":true,"mac":"00:B0:D0:63:C2:11","rep_score_id":1,"ssid":"Device_Networks-SSID 2","type_id":1}],"device_org_unit":"Device_Org_Unit","device_os_bits":12345678901,"device_os_build":"Device_OS_Build","device_os_country":"IN","device_os_edition":"Professional","device_os_lang":"en","device_os_name":"Windows Server 2019 Standard Edition","device_os_sp_name":"Device_OS_SP_Name","device_os_sp_ver":"Device_OS_SP_Ver","device_os_type_id":0,"device_os_ver":"Windows 10","device_proxy_ip":"89.160.20.112","device_proxy_name":"Device_Proxy_Name","device_public_ip":"89.160.20.112","device_ref_uid":"Device_Ref_UID","device_site":"Device_Site","device_subnet":"81.2.69.144","device_time":1613021404000,"device_type":"server","device_uid":"Device_UID","device_vhost":"Device_VHost","device_vhost_id":0,"domain_uid":"Domain_UID","end_time":"2024-02-29T01:00:00.000Z","entity":{"data":{"field1_keyword":"Entity-Data-field1_Keyword","field1_number":12345678901,"field1_boolean":true},"name":"Entity-Name","type":"Entity-Type","uid":"Entity-UID","version":"Entity-Version"},"event_id":2001,"events":[{"connection":{"direction_id":1,"dst_service":"C:\\Windows\\system32\\NTOSKRNL.EXE","src_ip":"159.19.163.218"},"count":1,"device_end_time":1709225074618,"device_time":1709225074618}],"feature_name":"Feature_Name","feature_path":"Feature_Path","feature_type":"Feature_Type","feature_uid":"Feature_UID","feature_ver":"2014.1.4.25","id":12345678901,"impersonator_customer_uid":"Impersonator_Customer_UID","impersonator_domain_uid":"Impersonator_Domain_UID","impersonator_user_uid":"Impersonator_User_UID","is_user_present":true,"log_level":"Log Level","log_name":"Log_Name","log_time":"2024-02-29T01:00:00.000Z","logging_device_ip":"89.160.20.112","logging_device_name":"Logging_Device_Name","logging_device_post_time":1613021404000,"logging_device_ref_uid":"Logging_Device_Ref_UID","message":"Message","message_code":"Message_Code","message_id":0,"org_unit_uid":"Org_Unit_UID","orig_data":"Orig_Data","product_data":{"sep_domain_uid":"Product_Data-Sep_Domain_UID","sep_hw_uid":"Product_Data-Sep_HW_UID"},"product_lang":"en","product_name":"Symantec Endpoint Security","product_uid":"Product_UID","product_ver":"2014.1.4.25-beta","proxy_device_ip":"89.160.20.112","proxy_device_name":"Proxy_Device_Name","raw_data":{"assetID":"vc9DagprQYyLZ23SEY1APw","assetOpstateDTO":{"productUuid":"31B0C880-0229-49E8-94C5-48D56B1BD7B9","features":[{"uuid":"1DF0351C-146D-4F07-B155-BF5C7077FF40","featureStatus":"SECURE","opstate":{"EDRContentSequence":"20231128005","EDREngineVersion":"4.11.0.10","EDRFramworkVersion":"4.10.0.59","FDRStatus":true,"LowDiskSpace":false,"MaxDBSizeHonored":true,"applied_policy":{"effective_date":1709219437080,"sha2":"ee6b0bebbc4575b507ac616d2c362f2c54d462b92cf4068cb6681ae3187d4de3","uid":"7dc29d40-f303-477a-9012-287ef252a391","version":"16"},"disk_usage_mb":1546,"fdr_first_event_date":"20240227","fdr_state":1},"state":"ENABLED","statusReason":["-107","0"],"prevention_state":"1"}],"products_active":0,"blades":0}},"ref_log_name":"Ref_Log_Name","ref_log_time":"2024-02-29T01:00:00.000Z","ref_orig_uid":"Ref_Orig_UID","ref_uid":"Ref_UID","remediated":true,"remediation":"Remediation","remediation_ref":"Remediation_Ref","remediation_uid":0,"seq_num":12345678901,"sessions":[{"auth_protocol_id":0,"cleartext_credentials":true,"direction_id":0,"id":12345678901,"is_admin":true,"logon_type_id":1,"port":80,"previous_users":["Sessions-Previous_User 1","Sessions-Previous_Users 1"],"remote":true,"remote_host":"Sessions-Remote_Host 1","remote_ip":"89.160.20.112","user":{"account_disabled":true,"cloud_resource_uid":"Sessions-User-Cloud_Resource_UID 1","domain":"Sessions-User-Domain 1","external_account_uid":"Sessions-User-External_Account_UID 1","external_uid":"Sessions-User-External_UID 1","full_name":"Sessions-User-Full_Name 1","groups":["Sessions-User-Group 1","Sessions-User-Groups 1"],"home":"Sessions-User-Home 1","is_admin":true,"logon_name":"Sessions-User-Logon_Name 1","name":"session-User-Name 1","password_expires":true,"shell":"Sessions-User-Shell 1","sid":"Sessions-User-SID 1","uid":"Sessions-User-UID 1"}},{"auth_protocol_id":1,"cleartext_credentials":true,"direction_id":1,"id":67890123451,"is_admin":true,"logon_type_id":2,"port":81,"previous_users":["Sessions-Previous_User 2","Sessions-Previous_Users 2"],"remote":true,"remote_host":"Sessions-Remote_Host 2","remote_ip":"89.160.20.112","user":{"account_disabled":true,"cloud_resource_uid":"Sessions-User-Cloud_Resource_UID 2","domain":"Sessions-User-Domain 2","external_account_uid":"Sessions-User-External_Account_UID 2","external_uid":"Sessions-User-External_UID 2","full_name":"Sessions-User-Full_Name 2","groups":["Sessions-User-Group 2","Sessions-User-Groups 2"],"home":"Sessions-User-Home 2","is_admin":true,"logon_name":"Sessions-User-Logon_Name 2","name":"session-User-Name 2","password_expires":true,"shell":"Sessions-User-Shell 2","sid":"Sessions-User-SID 2","uid":"Sessions-User-UID 2"}}],"severity_id":0,"source":{"facility":"Source-Facility","facility_detail":"Source-Facility_Detail","facility_uid":"Source-Facility_UID","type_id":1},"status_detail":"Status_Detail","status_id":0,"status_os":"Status_OS","status_os_src":12345678901,"status_stack_trace":"Status_Stack_Trace","status_thread_name":"Status_Thread_Name","stic_has_pii":true,"stic_hw_uid":"STIC_HW_UID","stic_ip_hash":"STIC_IP_Hash","stic_legacy_ent_uids":["STIC_Legacy_Ent_UIDs 1","STIC_Legacy_Ent_UIDs 2"],"stic_legacy_hw_uids":["STIC_Legacy_HW_UIDs 1","STIC_Legacy_HW_UIDs 2"],"stic_legacy_uids":["STIC_Legacy_UIDs 1","STIC_Legacy_UIDs 2"],"stic_schema_id":"STIC_Schema_ID","stic_uid":"STIC_UID","stic_version":"STIC_Version","subfeature_name":"Subfeature_Name","time":"2024-02-29T02:00:00Z","timezone":12345678901,"type":"Type","type_id":2,"user":{"account_disabled":true,"cloud_resource_uid":"User-Cloud_Resource_UID","domain":"User-Domain","external_account_uid":"User-External_Account_UID","external_uid":"User-External_UID","full_name":"User-Full_Name","groups":["User-Group 1","User-Groups 1"],"home":"User-Home","is_admin":true,"logon_name":"User-Logon_Name","name":"User123","password_expires":true,"shell":"User-Shell","sid":"TT23009","uid":"UU34899825"},"user_name":"Mohit","user_uid":"AB45698","uuid":"SR-1565234545","version":"1.4"} +{"category_id":3,"device_ip":"81.2.69.144","device_location":{"desc":"Remote","on_premises":false},"device_time":1727988510685,"feature_name":"AGENT_FRAMEWORK","feature_uid":"AFD9F423-8A5B-4899-B34C-6E67FDC29D4E","id":6,"message":"User is attempting to restart the endpoint service.","message_id":"0x1307021A","raw_data":"","severity_id":3,"status_detail":"Restarting","status_id":2,"type":"APPLICATION_LIFECYCLE","type_id":2,"version":"1.2","composite":3,"device_domain":"DEF.LOCAL","device_group":"Default/WGS/3480-US-Remote Group/3480-Clients","device_name":"45GHFS4","device_networks":[{"ipv4":"175.16.199.0","ipv6":"2a02:cf40::29","mac":"12:9F:E3:43:CD:7E"}],"device_os_name":"Windows 11 Enterprise","device_uid":"zRn4vL63UbLwi8kMLTXYzw","org_unit_uid":"dz4-LTGrBdXJ6e9AeckeOq","product_data":"{\"sep_domain_uid\":\"\",\"sep_hw_uid\":\"6CC8DFF19B9C8B01EF053210D1E519A7\"}" ,"product_name":"Symantec Endpoint Security","product_uid":"51C2F990-2234-43F9-87D5-39C76C1BD8C9","product_ver":"14.4.21216.9001","stic_hw_uid":"B114AFE9-594E-784D-599D-4C28392F963G","stic_uid":"8F73119D-D169-464F-98CD-F2E9EFFB96F9","timezone":300,"user_name":"JSmith","customer_uid":"ftp6xpg7UyS8W2n7FgzMrB","device_public_ip":"10.125.45.33","domain_uid":"cEF6Eq6mRYgfl9mJK2TspC","event_data_type":"sep","user":{"name":"JSmith"},"device_os_type_id":102,"time":"2024-10-03T21:10:30.685Z","log_time":"2024-10-04T12:15:50.069Z","js_canary_cust":"false","uuid":"2:e81ee8e1-82d9-11ef-e92d-0000078g231f"} diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/tf/files/test-events.log b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/tf/files/test-events.log index 2a631c602e4..25f796d769d 100644 --- a/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/tf/files/test-events.log +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/deploy/tf/files/test-events.log @@ -1 +1,2 @@ {"category_id":3,"collector_device_ip":"175.16.199.0","collector_device_name":"Collector_Device_Name","collector_name":"Collection12","collector_uid":"TT1456","composite":1,"container":{"host_name":"azure-us.local","image_name":"Image-sp2133","image_uid":"SH4322323","name":"User12345","networks":[{"bssid":"Container-Networks-BSSID 1","gateway_ip":"89.160.20.112","gateway_mac":"00:B0:D0:63:C2:01","ipv4":"81.2.69.144","ipv6":"2a02:cf40::","is_public":true,"mac":"00:B0:D0:63:C2:02","rep_score_id":0,"ssid":"SSID-4326451","type_id":0},{"bssid":"HN0845435","gateway_ip":"81.2.69.142","gateway_mac":"00:B0:D0:63:C2:03","ipv4":"81.2.69.144","ipv6":"2a02:cf40::","is_public":true,"mac":"00:B0:D0:63:C2:04","rep_score_id":1,"ssid":"Container-Networks-SSID 2","type_id":1}],"os_name":"Windows","uid":"UU35r3454"},"correlation_uid":"DD78666","count":563,"customer_registry_uid":"CP45254","customer_uid":"CH32534","cybox":{"domains":["example.com","abc.com"],"emails":[{"direction_id":0,"header_from":"abc@example.com","header_message_id":"Cybox-Emails-Header_Message_ID 1","header_reply_to":"Cybox-Emails-Header_Reply_To 1","header_subject":"Cybox-Emails-Header_Subject 1","header_to":["Cybox-Emails-Header_To 1","Cybox-Emails-Header_Tos 1"],"sender_ip":"81.2.69.144","size":12345678901,"smtp_from":"Cybox-Emails-SMTP_From 1","smtp_hello":"Cybox-Emails-SMTP_Hello 1","smtp_to":"Cybox-Emails-SMTP_To 1"},{"direction_id":1,"header_from":"abc@example.com","header_message_id":"Cybox-Emails-Header_Message_ID 2","header_reply_to":"Cybox-Emails-Header_Reply_To 2","header_subject":"Cybox-Emails-Header_Subject 2","header_to":["Cybox-Emails-Header_To 2","Cybox-Emails-Header_Tos 2"],"sender_ip":"81.2.69.144","size":12345678902,"smtp_from":"Cybox-Emails-SMTP_From 2","smtp_hello":"Cybox-Emails-SMTP_Hello 2","smtp_to":"Cybox-Emails-SMTP_To 2"}],"files":[{"accessed":1613021404000,"accessor":"Cybox-Files-Accessor 1","attribute_ids":[1,2,3,4,5,6,7,8,9,10],"attributes":12345678901,"company_name":"Microsoft Corporation","confidentiality_id":0,"content_type":{"family_id":0,"subtype":"SubType 1","type_id":0},"created":1613021404000,"creator":"Creator 1","creator_process":"Cybox-Files-Creator_Process 1","desc":"Cybox-Files-Desc 1","folder":"c:\\windows\\system32\\cybox\\files\\folder\\1","folder_uid":"Cybox-Files-Folder_UID 1","is_system":true,"md5":"HFDajsdf3254345436","mime_type":"Cybox-Files-MIME_Type 1","modified":1613021404000,"modifier":"Cybox-Files-Modifier 1","name":"cybox_files_name_1.exe","normalized_path":"CSIDL_SYSTEM\\cybox_files_normalized_path_1.exe","original_name":"Cybox-Files-Original_Name 1","owner":"Cybox-Files-Owner 1","parent_name":"Cybox-Files-Parent_Name 1","parent_sha2":"Cybox-Files-Parent_SHA2 1","path":"c:\\windows\\system32\\cybox_files_path_1.exe","product_name":"Windows Internet Explorer 1","product_path":"Cybox-Files-Product_Path 1","rep_discovered_band":0,"rep_discovered_date":1613021404000,"rep_prevalence":12345678901,"rep_prevalence_band":0,"rep_score":12345678901,"rep_score_band":0,"security_descriptor":"Cybox-Files-Security_Descriptor 1","sha1":"Cybox-Files-SHA1 1","sha2":"Cybox-Files-SHA2 1","signature_company_name":"Cybox-Files-Signature_Company_Name 1","signature_created_date":1613021404000,"signature_developer_uid":"Cybox-Files-Signature_Developer_UID 1","signature_fingerprints":[{"algorithm":"Cybox-Files-Signature_Fingerprints-Algorithm 1","value":"Cybox-Files-Signature_Fingerprints-Value 1"},{"algorithm":"Cybox-Files-Signature_Fingerprints-Algorithms 1","value":"Cybox-Files-Signature_Fingerprints-Values 1"}],"signature_issuer":"Cybox-Files-Signature_Issuer 1","signature_level_id":0,"signature_serial_number":"Cybox-Files-Signature_Serial_Number 1","signature_value":12345678901,"signature_value_ids":[0,1,2,3,4,5,6,7,8,9,10],"size":12345678901,"size_compressed":12345678901,"src_ip":"81.2.69.142","src_name":"Cybox-Files-SRC_Name 1","type_id":1,"uid":"Cybox-Files-UID 1","url":{"categories":["Cybox-Files-URL-Category 1","Cybox-Files-URL-Categories 1"],"category_ids":[1,3,4,5,6,7,9,11,14,15,16,17,18,20,21,22,23,24,25,26,27,29,30,31,32,33,34,35,36,37,38,40,43,44,45,46,47,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,65,66,67,68,71,83,84,85,86,87,88,89,90,92,93,95,96,97,98],"extension":"Cybox-Files-URL-Extension 1","host":"www.files-url-host-1.com","method":"Cybox-Files-URL-Method 1","parent_categories":["Cybox-Files-URL-Parent_Category 1","Cybox-Files-URL-Parent_Categories 1"],"path":"/download/trouble/cybox/files/url/path/1","port":80,"provider":"Cybox-Files-URL-Provider 1","query":"q=bad&sort=date_1","referrer":"Cybox-Files-URL-Referrer 1","referrer_categories":["Cybox-Files-URL-Referrer_Category 1","Cybox-Files-URL-Referrer_Categories 1"],"referrer_category_ids":[12345678901,67890123451],"rep_score_id":0,"scheme":"Cybox-Files-URL-Scheme 1","text":"www.files-url-text-1.com/download/trouble"},"version":"Cybox-Files-Version 1","xattributes":{"ads_name":"Cybox-Files-XAttributes-ADS_Name 1","ads_size":"Cybox-Files-XAttributes-ADS_Size 1","dacl":"Cybox-Files-XAttributes-DACL 1","owner":"Cybox-Files-XAttributes-Owner 1","primary_group":"Cybox-Files-XAttributes-Primary_Group 1","link_name":"Cybox-Files-XAttributes-Link_Name 1","hard_link_count":"Cybox-Files-XAttributes-Hard_Link_Count 1","Unix_permissions":"Cybox-Files-XAttributes-Unix_Permissions 1"}},{"accessed":1613021404000,"accessor":"Cybox-Files-Accessor 2","attribute_ids":[11,12,13,14,15,16,17],"attributes":12345678902,"company_name":"Microsoft Corporation 2","confidentiality_id":1,"content_type":{"family_id":1,"subtype":"Cybox-Files-Content_Type-SubType 2","type_id":1},"created":1613021404000,"creator":"Cybox-Files-Creator 2","creator_process":"Cybox-Files-Creator_Process 2","desc":"Cybox-Files-Desc 2","folder":"c:\\windows\\system32\\cybox\\files\\folder\\2","folder_uid":"Cybox-Files-Folder_UID 2","is_system":true,"md5":"Cybox-Files-MD5 2","mime_type":"Cybox-Files-MIME_Type 2","modified":1613021404000,"modifier":"Cybox-Files-Modifier 2","name":"cybox_files_name_2.exe","normalized_path":"CSIDL_SYSTEM\\cybox_files_normalized_path_2.exe","original_name":"Cybox-Files-Original_Name 2","owner":"Cybox-Files-Owner 2","parent_name":"Cybox-Files-Parent_Name 2","parent_sha2":"Cybox-Files-Parent_SHA2 2","path":"c:\\windows\\system32\\cybox_files_path_2.exe","product_name":"Windows Internet Explorer 2","product_path":"Cybox-Files-Product_Path 2","rep_discovered_band":1,"rep_discovered_date":1613021404000,"rep_prevalence":12345678902,"rep_prevalence_band":1,"rep_score":12345678902,"rep_score_band":1,"security_descriptor":"Cybox-Files-Security_Descriptor 2","sha1":"Cybox-Files-SHA1 2","sha2":"Cybox-Files-SHA2 2","signature_company_name":"Cybox-Files-Signature_Company_Name 2","signature_created_date":1613021404000,"signature_developer_uid":"Cybox-Files-Signature_Developer_UID 2","signature_fingerprints":[{"algorithm":"Cybox-Files-Signature_Fingerprints-Algorithm 2","value":"Cybox-Files-Signature_Fingerprints-Value 2"},{"algorithm":"Cybox-Files-Signature_Fingerprints-Algorithms 2","value":"Cybox-Files-Signature_Fingerprints-Values 2"}],"signature_issuer":"Cybox-Files-Signature_Issuer 2","signature_level_id":1,"signature_serial_number":"Cybox-Files-Signature_Serial_Number 2","signature_value":12345678902,"signature_value_ids":[11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],"size":12345678902,"size_compressed":12345678902,"src_ip":"81.2.69.144","src_name":"Cybox-Files-SRC_Name 2","type_id":1,"uid":"Cybox-Files-UID 2","url":{"categories":["Cybox-Files-URL-Category 2","Cybox-Files-URL-Categories 2"],"category_ids":[101,102,103,104,105,106,107,108,109,110,111,112,113,114,116,117,118,121,124],"extension":"Cybox-Files-URL-Extension 2","host":"www.files-url-host-2.com","method":"Cybox-Files-URL-Method 2","parent_categories":["Cybox-Files-URL-Parent_Category 2","Cybox-Files-URL-Parent_Categories 2"],"path":"/download/trouble/cybox/files/url/path/2","port":81,"provider":"Cybox-Files-URL-Provider 2","query":"q=bad&sort=date_2","referrer":"Cybox-Files-URL-Referrer 2","referrer_categories":["Cybox-Files-URL-Referrer_Category 2","Cybox-Files-URL-Referrer_Categories 2"],"referrer_category_ids":[12345678902,67890123452],"rep_score_id":1,"scheme":"Cybox-Files-URL-Scheme 2","text":"www.files-url-text-2.com/download/trouble"},"version":"Cybox-Files-Version 2","xattributes":{"ads_name":"Cybox-Files-XAttributes-ADS_Name 2","ads_size":"Cybox-Files-XAttributes-ADS_Size 2","dacl":"Cybox-Files-XAttributes-DACL 2","owner":"Cybox-Files-XAttributes-Owner 2","primary_group":"Cybox-Files-XAttributes-Primary_Group 2","link_name":"Cybox-Files-XAttributes-Link_Name 2","hard_link_count":"Cybox-Files-XAttributes-Hard_Link_Count 2","Unix_permissions":"Cybox-Files-XAttributes-Unix_Permissions 2"}}],"hostnames":["Cybox-Hostname 1","Cybox-Hostnames 1"],"icap_reqmod":[{"metadata":{"field1_keyword":"Cybox-ICAP_ReqMod-field1_Keyword","field1_number":12345678901,"field1_boolean":true,"field1_ip":"175.16.199.0"},"service":"Cybox-ICAP_ReqMod-Service 1","status":"Cybox-ICAP_ReqMod-Status 1","status_detail":"Cybox-ICAP_ReqMod-Status_Detail 1"},{"metadata":{"field2_keyword":"Cybox-ICAP_ReqMod-field2_Keyword","field2_number":12345678902,"field2_boolean":true,"field2_ip":"175.16.199.0"},"service":"Cybox-ICAP_ReqMod-Service 2","status":"Cybox-ICAP_ReqMod-Status 2","status_detail":"Cybox-ICAP_ReqMod-Status_Detail 2"}],"icap_respmod":[{"metadata":{"field1_keyword":"Cybox-ICAP_RespMod-field1_Keyword","field1_number":12345678901,"field1_boolean":true,"field1_ip":"175.16.199.0"},"service":"Cybox-ICAP_RespMod-Service 1","status":"Cybox-ICAP_RespMod-Status 1","status_detail":"Cybox-ICAP_RespMod-Status_Detail 1"},{"metadata":{"field2_keyword":"Cybox-ICAP_RespMod-field2_Keyword","field2_number":12345678902,"field2_boolean":true,"field2_ip":"175.16.199.0"},"service":"Cybox-ICAP_RespMod-Service 2","status":"Cybox-ICAP_RespMod-Status 2","status_detail":"Cybox-ICAP_RespMod-Status_Detail 2"}],"ipv4s":["175.16.199.0","175.16.199.0"],"ipv6s":["2a02:cf40::","2a02:cf40::"],"macs":["00:B0:D0:63:C2:05","00:B0:D0:63:C2:06"],"urls":[{"categories":["Cybox-URLs-Category 1","Cybox-URLs-Categories 1"],"category_ids":[1,3,4,5,6,7,9,11,14,15,16,17,18,20,21,22,23,24,25,26,27,29,30,31,32,33,34,35,36,37,38,40,43,44,45,46,47,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,65,66,67,68,71,83,84,85,86,87,88,89,90,92,93,95,96,97,98],"extension":"Cybox-URLs-Extension 1","host":"www.urls-host-1.com","method":"Cybox-URLs-Method 1","parent_categories":["Cybox-URLs-Parent_Category 1","Cybox-URLs-Parent_Categories 1"],"path":"/download/trouble/cybox/urls/path/1","port":80,"provider":"Cybox-URLs-Provider 1","query":"q=bad&sort=date_1","referrer":"Cybox-URLs-Referrer 1","referrer_categories":["Cybox-URLs-Referrer_Category 1","Cybox-URLs-Referrer_Categories 1"],"referrer_category_ids":[12345678901,67890123451],"rep_score_id":0,"scheme":"Cybox-URLs-Scheme 1","text":"www.urls-text-1.com/download/trouble"},{"categories":["Cybox-URLs-Category 2","Cybox-URLs-Categories 2"],"category_ids":[101,102,103,104,105,106,107,108,109,110,111,112,113,114,116,117,118,121,124],"extension":"Cybox-URLs-Extension 2","host":"www.urls-host-2.com","method":"Cybox-URLs-Method 2","parent_categories":["Cybox-URLs-Parent_Category 2","Cybox-URLs-Parent_Categories 2"],"path":"/download/trouble/cybox/urls/path/2","port":81,"provider":"Cybox-URLs-Provider 2","query":"q=bad&sort=date_2","referrer":"Cybox-URLs-Referrer 2","referrer_categories":["Cybox-URLs-Referrer_Category 2","Cybox-URLs-Referrer_Categories 2"],"referrer_category_ids":[12345678902,67890123452],"rep_score_id":1,"scheme":"Cybox-URLs-Scheme 2","text":"www.urls-text-2.com/download/trouble"}]},"device_alias_name":"Device_Alias_Name","device_cap":"Device_Cap","device_cloud_vm":{"autoscale_uid":"Device_Cloud_VM-Autoscale_UID","dc_region":"Device_Cloud_VM-DC_Region","instance_uid":"Device_Cloud_VM-Instance_UID","subnet_uid":"Device_Cloud_VM-Subnet_UID","vpc_uid":"Device_Cloud_VM-VPC_UID"},"device_desc":"Device_Desc","device_domain":"device.domain.internal.somecompany.com","device_domain_uid":"Device_Domain_UID","device_end_time":1613021404000,"device_gateway":"175.16.199.0","device_group":"Device_Group","device_group_name":"Device_Group_Name","device_hw_bios_date":"03/31/16","device_hw_bios_manufacturer":"LENOVO","device_hw_bios_ver":"LENOVO G5ETA2WW (2.62)","device_hw_cpu_type":"x86 Family 6 Model 37 Stepping 5","device_imei":"Device_IMEI","device_ip":"175.16.199.0","device_is_compliant":true,"device_is_personal":true,"device_is_trusted":true,"device_is_unmanaged":true,"device_location":{"city":"Device_Location-City","continent":"Device_Location-Continent","coordinates":[-12.345,56.789],"country":"US","desc":"Device_Location-Desc","isp":"Device_Location-ISP","on_premises":true,"region":"US-CA"},"device_mac":"00:B0:D0:63:C2:07","device_name":"device.name.computer.domain","device_name_md5":"4ED962DDBF17E2BBA7B14EBC00F3162E","device_networks":[{"bssid":"Device_Networks-BSSID 1","gateway_ip":"175.16.199.0","gateway_mac":"00:B0:D0:63:C2:08","ipv4":"175.16.199.0","ipv6":"2a02:cf40::","is_public":true,"mac":"00:B0:D0:63:C2:09","rep_score_id":0,"ssid":"Device_Networks-SSID 1","type_id":0},{"bssid":"Device_Networks-BSSID 2","gateway_ip":"89.160.20.112","gateway_mac":"00:B0:D0:63:C2:10","ipv4":"89.160.20.112","ipv6":"2a02:cf40::","is_public":true,"mac":"00:B0:D0:63:C2:11","rep_score_id":1,"ssid":"Device_Networks-SSID 2","type_id":1}],"device_org_unit":"Device_Org_Unit","device_os_bits":12345678901,"device_os_build":"Device_OS_Build","device_os_country":"IN","device_os_edition":"Professional","device_os_lang":"en","device_os_name":"Windows Server 2019 Standard Edition","device_os_sp_name":"Device_OS_SP_Name","device_os_sp_ver":"Device_OS_SP_Ver","device_os_type_id":0,"device_os_ver":"Windows 10","device_proxy_ip":"89.160.20.112","device_proxy_name":"Device_Proxy_Name","device_public_ip":"89.160.20.112","device_ref_uid":"Device_Ref_UID","device_site":"Device_Site","device_subnet":"81.2.69.144","device_time":1613021404000,"device_type":"server","device_uid":"Device_UID","device_vhost":"Device_VHost","device_vhost_id":0,"domain_uid":"Domain_UID","end_time":"2024-02-29T01:00:00.000Z","entity":{"data":{"field1_keyword":"Entity-Data-field1_Keyword","field1_number":12345678901,"field1_boolean":true},"name":"Entity-Name","type":"Entity-Type","uid":"Entity-UID","version":"Entity-Version"},"event_id":2001,"events":[{"connection":{"direction_id":1,"dst_service":"C:\\Windows\\system32\\NTOSKRNL.EXE","src_ip":"159.19.163.218"},"count":1,"device_end_time":1709225074618,"device_time":1709225074618}],"feature_name":"Feature_Name","feature_path":"Feature_Path","feature_type":"Feature_Type","feature_uid":"Feature_UID","feature_ver":"2014.1.4.25","id":12345678901,"impersonator_customer_uid":"Impersonator_Customer_UID","impersonator_domain_uid":"Impersonator_Domain_UID","impersonator_user_uid":"Impersonator_User_UID","is_user_present":true,"log_level":"Log Level","log_name":"Log_Name","log_time":"2024-02-29T01:00:00.000Z","logging_device_ip":"89.160.20.112","logging_device_name":"Logging_Device_Name","logging_device_post_time":1613021404000,"logging_device_ref_uid":"Logging_Device_Ref_UID","message":"Message","message_code":"Message_Code","message_id":0,"org_unit_uid":"Org_Unit_UID","orig_data":"Orig_Data","product_data":{"sep_domain_uid":"Product_Data-Sep_Domain_UID","sep_hw_uid":"Product_Data-Sep_HW_UID"},"product_lang":"en","product_name":"Symantec Endpoint Security","product_uid":"Product_UID","product_ver":"2014.1.4.25-beta","proxy_device_ip":"89.160.20.112","proxy_device_name":"Proxy_Device_Name","raw_data":{"assetID":"vc9DagprQYyLZ23SEY1APw","assetOpstateDTO":{"productUuid":"31B0C880-0229-49E8-94C5-48D56B1BD7B9","features":[{"uuid":"1DF0351C-146D-4F07-B155-BF5C7077FF40","featureStatus":"SECURE","opstate":{"EDRContentSequence":"20231128005","EDREngineVersion":"4.11.0.10","EDRFramworkVersion":"4.10.0.59","FDRStatus":true,"LowDiskSpace":false,"MaxDBSizeHonored":true,"applied_policy":{"effective_date":1709219437080,"sha2":"ee6b0bebbc4575b507ac616d2c362f2c54d462b92cf4068cb6681ae3187d4de3","uid":"7dc29d40-f303-477a-9012-287ef252a391","version":"16"},"disk_usage_mb":1546,"fdr_first_event_date":"20240227","fdr_state":1},"state":"ENABLED","statusReason":["-107","0"],"prevention_state":"1"}],"products_active":0,"blades":0}},"ref_log_name":"Ref_Log_Name","ref_log_time":"2024-02-29T01:00:00.000Z","ref_orig_uid":"Ref_Orig_UID","ref_uid":"Ref_UID","remediated":true,"remediation":"Remediation","remediation_ref":"Remediation_Ref","remediation_uid":0,"seq_num":12345678901,"sessions":[{"auth_protocol_id":0,"cleartext_credentials":true,"direction_id":0,"id":12345678901,"is_admin":true,"logon_type_id":1,"port":80,"previous_users":["Sessions-Previous_User 1","Sessions-Previous_Users 1"],"remote":true,"remote_host":"Sessions-Remote_Host 1","remote_ip":"89.160.20.112","user":{"account_disabled":true,"cloud_resource_uid":"Sessions-User-Cloud_Resource_UID 1","domain":"Sessions-User-Domain 1","external_account_uid":"Sessions-User-External_Account_UID 1","external_uid":"Sessions-User-External_UID 1","full_name":"Sessions-User-Full_Name 1","groups":["Sessions-User-Group 1","Sessions-User-Groups 1"],"home":"Sessions-User-Home 1","is_admin":true,"logon_name":"Sessions-User-Logon_Name 1","name":"session-User-Name 1","password_expires":true,"shell":"Sessions-User-Shell 1","sid":"Sessions-User-SID 1","uid":"Sessions-User-UID 1"}},{"auth_protocol_id":1,"cleartext_credentials":true,"direction_id":1,"id":67890123451,"is_admin":true,"logon_type_id":2,"port":81,"previous_users":["Sessions-Previous_User 2","Sessions-Previous_Users 2"],"remote":true,"remote_host":"Sessions-Remote_Host 2","remote_ip":"89.160.20.112","user":{"account_disabled":true,"cloud_resource_uid":"Sessions-User-Cloud_Resource_UID 2","domain":"Sessions-User-Domain 2","external_account_uid":"Sessions-User-External_Account_UID 2","external_uid":"Sessions-User-External_UID 2","full_name":"Sessions-User-Full_Name 2","groups":["Sessions-User-Group 2","Sessions-User-Groups 2"],"home":"Sessions-User-Home 2","is_admin":true,"logon_name":"Sessions-User-Logon_Name 2","name":"session-User-Name 2","password_expires":true,"shell":"Sessions-User-Shell 2","sid":"Sessions-User-SID 2","uid":"Sessions-User-UID 2"}}],"severity_id":0,"source":{"facility":"Source-Facility","facility_detail":"Source-Facility_Detail","facility_uid":"Source-Facility_UID","type_id":1},"status_detail":"Status_Detail","status_id":0,"status_os":"Status_OS","status_os_src":12345678901,"status_stack_trace":"Status_Stack_Trace","status_thread_name":"Status_Thread_Name","stic_has_pii":true,"stic_hw_uid":"STIC_HW_UID","stic_ip_hash":"STIC_IP_Hash","stic_legacy_ent_uids":["STIC_Legacy_Ent_UIDs 1","STIC_Legacy_Ent_UIDs 2"],"stic_legacy_hw_uids":["STIC_Legacy_HW_UIDs 1","STIC_Legacy_HW_UIDs 2"],"stic_legacy_uids":["STIC_Legacy_UIDs 1","STIC_Legacy_UIDs 2"],"stic_schema_id":"STIC_Schema_ID","stic_uid":"STIC_UID","stic_version":"STIC_Version","subfeature_name":"Subfeature_Name","time":"2024-02-29T02:00:00Z","timezone":12345678901,"type":"Type","type_id":2,"user":{"account_disabled":true,"cloud_resource_uid":"User-Cloud_Resource_UID","domain":"User-Domain","external_account_uid":"User-External_Account_UID","external_uid":"User-External_UID","full_name":"User-Full_Name","groups":["User-Group 1","User-Groups 1"],"home":"User-Home","is_admin":true,"logon_name":"User-Logon_Name","name":"User123","password_expires":true,"shell":"User-Shell","sid":"TT23009","uid":"UU34899825"},"user_name":"Mohit","user_uid":"AB45698","uuid":"SR-1565234545","version":"1.4"} +{"category_id":3,"device_ip":"81.2.69.144","device_location":{"desc":"Remote","on_premises":false},"device_time":1727988510685,"feature_name":"AGENT_FRAMEWORK","feature_uid":"AFD9F423-8A5B-4899-B34C-6E67FDC29D4E","id":6,"message":"User is attempting to restart the endpoint service.","message_id":"0x1307021A","raw_data":"","severity_id":3,"status_detail":"Restarting","status_id":2,"type":"APPLICATION_LIFECYCLE","type_id":2,"version":"1.2","composite":3,"device_domain":"DEF.LOCAL","device_group":"Default/WGS/3480-US-Remote Group/3480-Clients","device_name":"45GHFS4","device_networks":[{"ipv4":"175.16.199.0","ipv6":"2a02:cf40::29","mac":"12:9F:E3:43:CD:7E"}],"device_os_name":"Windows 11 Enterprise","device_uid":"zRn4vL63UbLwi8kMLTXYzw","org_unit_uid":"dz4-LTGrBdXJ6e9AeckeOq","product_data":"{\"sep_domain_uid\":\"\",\"sep_hw_uid\":\"6CC8DFF19B9C8B01EF053210D1E519A7\"}" ,"product_name":"Symantec Endpoint Security","product_uid":"51C2F990-2234-43F9-87D5-39C76C1BD8C9","product_ver":"14.4.21216.9001","stic_hw_uid":"B114AFE9-594E-784D-599D-4C28392F963G","stic_uid":"8F73119D-D169-464F-98CD-F2E9EFFB96F9","timezone":300,"user_name":"JSmith","customer_uid":"ftp6xpg7UyS8W2n7FgzMrB","device_public_ip":"10.125.45.33","domain_uid":"cEF6Eq6mRYgfl9mJK2TspC","event_data_type":"sep","user":{"name":"JSmith"},"device_os_type_id":102,"time":"2024-10-03T21:10:30.685Z","log_time":"2024-10-04T12:15:50.069Z","js_canary_cust":"false","uuid":"2:e81ee8e1-82d9-11ef-e92d-0000078g231f"} diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-azure-config.yml b/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-azure-config.yml index 83156fe327e..1a6d23a0abb 100644 --- a/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-azure-config.yml +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-azure-config.yml @@ -13,4 +13,4 @@ data_stream: poll: true poll_interval: 15s assert: - hit_count: 1 + hit_count: 2 diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-default-config.yml b/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-default-config.yml index 3dcb0bd2caa..7b2f40f3cc2 100644 --- a/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-default-config.yml +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-default-config.yml @@ -10,5 +10,5 @@ data_stream: preserve_original_event: true file_selectors: | - regex: '^(.+?)\.log' - assert: - hit_count: 1 +assert: + hit_count: 2 diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-gcs-config.yml b/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-gcs-config.yml index 26a85f7bb25..919e2001e48 100644 --- a/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-gcs-config.yml +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/test/system/test-gcs-config.yml @@ -13,4 +13,4 @@ data_stream: buckets: | - name: testbucket assert: - hit_count: 1 + hit_count: 2 diff --git a/packages/symantec_endpoint_security/data_stream/event/sample_event.json b/packages/symantec_endpoint_security/data_stream/event/sample_event.json index fa007c56dee..fe0dba0fa68 100644 --- a/packages/symantec_endpoint_security/data_stream/event/sample_event.json +++ b/packages/symantec_endpoint_security/data_stream/event/sample_event.json @@ -1,12 +1,23 @@ { "@timestamp": "2024-02-29T02:00:00.000Z", "agent": { - "ephemeral_id": "2979533b-cdb6-49c0-8665-3223e6922f90", - "id": "68d148de-0c02-47af-b9dd-210b7682ece0", - "name": "elastic-agent-86709", + "ephemeral_id": "75140a75-8187-43dd-ab8e-a5913552125e", + "id": "47c6af15-e32a-45f2-822b-9c41f7482560", + "name": "elastic-agent-67776", "type": "filebeat", "version": "8.16.5" }, + "azure": { + "storage": { + "blob": { + "content_type": "application/octet-stream", + "name": "test-events.log" + }, + "container": { + "name": "test-container" + } + } + }, "client": { "domain": "device.domain.internal.somecompany.com", "geo": { @@ -14,11 +25,11 @@ } }, "cloud": { - "provider": "google cloud" + "provider": "azure" }, "data_stream": { "dataset": "symantec_endpoint_security.event", - "namespace": "56890", + "namespace": "14632", "type": "logs" }, "destination": { @@ -36,7 +47,7 @@ "version": "8.11.0" }, "elastic_agent": { - "id": "68d148de-0c02-47af-b9dd-210b7682ece0", + "id": "47c6af15-e32a-45f2-822b-9c41f7482560", "snapshot": false, "version": "8.16.5" }, @@ -71,7 +82,7 @@ "2021-02-11T05:30:04.000Z" ], "id": "SR-1565234545", - "ingested": "2025-10-28T11:45:21Z", + "ingested": "2025-12-29T06:04:10Z", "kind": "event", "module": "Feature_Name", "sequence": [ @@ -146,17 +157,6 @@ ] } }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/x-ndjson", - "name": "test-events.log" - } - } - }, "group": { "name": "Device_Group" }, @@ -189,11 +189,11 @@ ] }, "input": { - "type": "gcs" + "type": "azure-blob-storage" }, "log": { "file": { - "path": "gs://testbucket/test-events.log" + "path": "http://svc-azure-blob-storage-emulator:10000/devstoreaccount1/test-container/test-events.log" }, "level": [ "Log Level" diff --git a/packages/symantec_endpoint_security/docs/README.md b/packages/symantec_endpoint_security/docs/README.md index f4b873ffc80..5a23093c463 100644 --- a/packages/symantec_endpoint_security/docs/README.md +++ b/packages/symantec_endpoint_security/docs/README.md @@ -229,12 +229,23 @@ An example event for `event` looks as following: { "@timestamp": "2024-02-29T02:00:00.000Z", "agent": { - "ephemeral_id": "2979533b-cdb6-49c0-8665-3223e6922f90", - "id": "68d148de-0c02-47af-b9dd-210b7682ece0", - "name": "elastic-agent-86709", + "ephemeral_id": "75140a75-8187-43dd-ab8e-a5913552125e", + "id": "47c6af15-e32a-45f2-822b-9c41f7482560", + "name": "elastic-agent-67776", "type": "filebeat", "version": "8.16.5" }, + "azure": { + "storage": { + "blob": { + "content_type": "application/octet-stream", + "name": "test-events.log" + }, + "container": { + "name": "test-container" + } + } + }, "client": { "domain": "device.domain.internal.somecompany.com", "geo": { @@ -242,11 +253,11 @@ An example event for `event` looks as following: } }, "cloud": { - "provider": "google cloud" + "provider": "azure" }, "data_stream": { "dataset": "symantec_endpoint_security.event", - "namespace": "56890", + "namespace": "14632", "type": "logs" }, "destination": { @@ -264,7 +275,7 @@ An example event for `event` looks as following: "version": "8.11.0" }, "elastic_agent": { - "id": "68d148de-0c02-47af-b9dd-210b7682ece0", + "id": "47c6af15-e32a-45f2-822b-9c41f7482560", "snapshot": false, "version": "8.16.5" }, @@ -299,7 +310,7 @@ An example event for `event` looks as following: "2021-02-11T05:30:04.000Z" ], "id": "SR-1565234545", - "ingested": "2025-10-28T11:45:21Z", + "ingested": "2025-12-29T06:04:10Z", "kind": "event", "module": "Feature_Name", "sequence": [ @@ -374,17 +385,6 @@ An example event for `event` looks as following: ] } }, - "gcs": { - "storage": { - "bucket": { - "name": "testbucket" - }, - "object": { - "content_type": "application/x-ndjson", - "name": "test-events.log" - } - } - }, "group": { "name": "Device_Group" }, @@ -417,11 +417,11 @@ An example event for `event` looks as following: ] }, "input": { - "type": "gcs" + "type": "azure-blob-storage" }, "log": { "file": { - "path": "gs://testbucket/test-events.log" + "path": "http://svc-azure-blob-storage-emulator:10000/devstoreaccount1/test-container/test-events.log" }, "level": [ "Log Level"