From 942cfe518f87f6e1fcc30f028276575bf7a7bb8c Mon Sep 17 00:00:00 2001 From: Greg Shepherd Date: Wed, 18 Feb 2026 16:18:46 -0500 Subject: [PATCH 1/5] Updated pipeline to lowercase all hashes --- packages/ti_threatconnect/changelog.yml | 5 +++++ .../test/pipeline/test-indicator.log-expected.json | 10 +++++----- ...ssociated-groups-and-attributes.log-expected.json | 10 +++++----- .../elasticsearch/ingest_pipeline/default.yml | 12 ++++++++++++ packages/ti_threatconnect/manifest.yml | 2 +- 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/packages/ti_threatconnect/changelog.yml b/packages/ti_threatconnect/changelog.yml index 1ac8ad6370a..16f6ea62883 100644 --- a/packages/ti_threatconnect/changelog.yml +++ b/packages/ti_threatconnect/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.0.0" + changes: + - description: Update all hash types to be lowercase + type: breaking-change + link: https://github.com/elastic/integrations/pull/10000 - version: "1.12.0" changes: - description: Allow transforms to run in unattended mode. diff --git a/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json b/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json index a3175bb73d6..3670ed124f6 100644 --- a/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json +++ b/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json @@ -39,9 +39,9 @@ }, "related": { "hash": [ - "F5A2496CF66CXXCFFE66CXXB27D7XXXX", + "f5a2496cf66cxxcffe66cxxb27d7xxxx", "samplesha1", - "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX" + "7d5fffbfe8d098e369466164f705b4d692517a2b4659a03901daf67cf78xxxxx" ], "hosts": [ "samplehost" @@ -455,7 +455,7 @@ "last_modified": "2023-12-01T08:26:48.000Z", "last_seen": "2023-10-04T12:34:56.000Z", "legacy_link": "https://partnerstage-intel.threatconnect.com/auth/indicators/details/emailaddress.xhtml?emailaddress=misoyil388%40poverts.com&owner=Elastic", - "md5": "F5A2496CF66CXXCFFE66CXXB27D7XXXX", + "md5": "f5a2496cf66cxxcffe66cxxb27d7xxxx", "mutex": "Test.Mutex()", "observations": { "data": [ @@ -480,7 +480,7 @@ ] }, "sha1": "samplesha1", - "sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX", + "sha256": "7d5fffbfe8d098e369466164f705b4d692517a2b4659a03901daf67cf78xxxxx", "size": "123", "source": "https://fp.tools/api/v4/indicators/attribute/pN0psYjPUQ6a_sxXXXXX", "subject": "Spam", @@ -540,4 +540,4 @@ } } ] -} +} \ No newline at end of file diff --git a/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-with-associated-groups-and-attributes.log-expected.json b/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-with-associated-groups-and-attributes.log-expected.json index 4974817138b..7537b86c8d0 100644 --- a/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-with-associated-groups-and-attributes.log-expected.json +++ b/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-with-associated-groups-and-attributes.log-expected.json @@ -39,9 +39,9 @@ }, "related": { "hash": [ - "F5A2496CF66CXXCFFE66CXXB27D7XXXX", + "f5a2496cf66cxxcffe66cxxb27d7xxxx", "samplesha1", - "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX" + "7d5fffbfe8d098e369466164f705b4d692517a2b4659a03901daf67cf78xxxxx" ], "hosts": [ "samplehost" @@ -473,7 +473,7 @@ "last_modified": "2023-12-01T08:26:48.000Z", "last_seen": "2023-10-04T12:34:56.000Z", "legacy_link": "https://partnerstage-intel.threatconnect.com/auth/indicators/details/emailaddress.xhtml?emailaddress=misoyil388%40poverts.com&owner=Elastic", - "md5": "F5A2496CF66CXXCFFE66CXXB27D7XXXX", + "md5": "f5a2496cf66cxxcffe66cxxb27d7xxxx", "mutex": "Test.Mutex()", "observations": { "data": [ @@ -498,7 +498,7 @@ ] }, "sha1": "samplesha1", - "sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX", + "sha256": "7d5fffbfe8d098e369466164f705b4d692517a2b4659a03901daf67cf78xxxxx", "size": "123", "source": "https://fp.tools/api/v4/indicators/attribute/pN0psYjPUQ6a_sxXXXXX", "subject": "Spam", @@ -561,4 +561,4 @@ } } ] -} +} \ No newline at end of file diff --git a/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml index 9fd35f4ee04..613b593df2a 100644 --- a/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -1815,6 +1815,10 @@ processors: copy_from: threat_connect.indicator.md5 ignore_empty_value: true if: ctx.threat_connect?.indicator?.type instanceof String && ctx.threat_connect.indicator.type.contains('File') + - lowercase: + field: threat_connect.indicator.md5 + tag: lowercase_md5 + ignore_missing: true - rename: field: json.Key Name tag: rename_Key Name @@ -1945,6 +1949,10 @@ processors: copy_from: threat_connect.indicator.sha1 ignore_empty_value: true if: ctx.threat_connect?.indicator?.type instanceof String && ctx.threat_connect.indicator.type.contains('File') + - lowercase: + field: threat_connect.indicator.sha1 + tag: lowercase_sha1 + ignore_missing: true - append: field: related.hash tag: convert_sha1_address_to_related_hash @@ -1962,6 +1970,10 @@ processors: copy_from: threat_connect.indicator.sha256 ignore_empty_value: true if: ctx.threat_connect?.indicator?.type instanceof String && ctx.threat_connect.indicator.type.contains('File') + - lowercase: + field: threat_connect.indicator.sha256 + tag: lowercase_sha256 + ignore_missing: true - append: field: related.hash tag: convert_sha256_address_to_related_hash diff --git a/packages/ti_threatconnect/manifest.yml b/packages/ti_threatconnect/manifest.yml index db1ffd1a00a..e18caef0817 100644 --- a/packages/ti_threatconnect/manifest.yml +++ b/packages/ti_threatconnect/manifest.yml @@ -2,7 +2,7 @@ format_version: 3.0.3 name: ti_threatconnect title: ThreatConnect -version: "1.12.0" +version: "2.0.0" description: Collects Indicators from ThreatConnect using the Elastic Agent and saves them as logs inside Elastic type: integration categories: From 6396108de7999ded92fbdb9d7916c3a73d676787 Mon Sep 17 00:00:00 2001 From: Greg Shepherd Date: Wed, 18 Feb 2026 16:27:13 -0500 Subject: [PATCH 2/5] Updated pull request number --- packages/ti_threatconnect/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_threatconnect/changelog.yml b/packages/ti_threatconnect/changelog.yml index 16f6ea62883..7a6b65ab117 100644 --- a/packages/ti_threatconnect/changelog.yml +++ b/packages/ti_threatconnect/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Update all hash types to be lowercase type: breaking-change - link: https://github.com/elastic/integrations/pull/10000 + link: https://github.com/elastic/integrations/pull/17455 - version: "1.12.0" changes: - description: Allow transforms to run in unattended mode. From 6ea3d1cd9687737ce5d2d898e522bab359835558 Mon Sep 17 00:00:00 2001 From: Greg Shepherd Date: Thu, 19 Feb 2026 13:43:55 -0500 Subject: [PATCH 3/5] Update packages/ti_threatconnect/changelog.yml ti_threatconnect: lowercase all hash values in indicator processing Hash values should be normalized to lowercase for consistent matching and deduplication across threat intelligence sources. Co-authored-by: Dan Kortschak --- packages/ti_threatconnect/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_threatconnect/changelog.yml b/packages/ti_threatconnect/changelog.yml index 7a6b65ab117..9804129345c 100644 --- a/packages/ti_threatconnect/changelog.yml +++ b/packages/ti_threatconnect/changelog.yml @@ -1,7 +1,7 @@ # newer versions go on top - version: "2.0.0" changes: - - description: Update all hash types to be lowercase + - description: Normalize all hash fields to be lowercase. type: breaking-change link: https://github.com/elastic/integrations/pull/17455 - version: "1.12.0" From d200ae7a37f44d67fead2feca705f9918382e9b6 Mon Sep 17 00:00:00 2001 From: Greg Shepherd Date: Thu, 19 Feb 2026 16:14:49 -0500 Subject: [PATCH 4/5] Incorporate review comments and map file size. --- .../elasticsearch/ingest_pipeline/default.yml | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml index 613b593df2a..29e7726ff53 100644 --- a/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -1804,6 +1804,22 @@ processors: value: '{{{threat_connect.indicator.legacy_link}}}' allow_duplicates: false if: ctx.threat_connect?.indicator?.legacy_link != null +# handle file size if exists before hash mappings + - convert: + field: json.size + tag: convert_size_to_string + target_field: threat_connect.indicator.size + type: string + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: threat.indicator.file.size + tag: set_threat_indicator_file_size_from_indicator_size + copy_from: threat_connect.indicator.size + ignore_empty_value: true - rename: field: json.md5 tag: rename_md5 @@ -1980,16 +1996,6 @@ processors: value: '{{{threat_connect.indicator.sha256}}}' allow_duplicates: false if: ctx.threat_connect?.indicator?.md5 != null - - convert: - field: json.size - tag: convert_size_to_string - target_field: threat_connect.indicator.size - type: string - ignore_missing: true - on_failure: - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: field: json.source tag: rename_source From 7b55ef1afeaa5b7cdc6295b2bb16296aec3d8df3 Mon Sep 17 00:00:00 2001 From: Greg Shepherd Date: Thu, 19 Feb 2026 17:12:33 -0500 Subject: [PATCH 5/5] Update to have pipeline tests pass --- .../_dev/test/pipeline/test-indicator.log | 2 +- .../pipeline/test-indicator.log-expected.json | 5 ++++- ...ed-groups-and-attributes.log-expected.json | 3 +++ .../elasticsearch/ingest_pipeline/default.yml | 20 +++++++++++-------- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log b/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log index f8e43d472d2..159d08605d9 100644 --- a/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log +++ b/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log @@ -1 +1 @@ -{"id": 891599,"dateAdded": "2023-08-25T12:57:24Z","description": "bad email","securityLabels": {"data": [{"id": 3,"name": "TLP:AMBER","source": "https://fp.tools/api/v4/indicators/attribute/pN0psYjPUQ6a_sxPSW5XjQ","description": "Thissecuritylabelisusedforinformationthatrequiressupporttobeeffectivelyactedupon,yetcarriesriskstoprivacy,reputation,oroperationsifsharedoutsideoftheorganizationsinvolved.Informationwiththislabelcanbesharedwithmembersofanorganizationanditsclients.","color": "FFC000","owner": "System","dateAdded": "2016-08-31T00:00:00Z"}]},"ownerId": 51,"ownerName": "Elastic","webLink": "https://partnerstage-intel.threatconnect.com/","tags": {"data": [{"id": 1,"name": "userexecution:maliciouslink","lastUsed": "2023-08-25T13:15:30Z","description": "ApplythisTagtoobjectsrelatedtoransomwareattacks","owner": "Demoorganization","techniqueId": "T1055.005","platforms": {"data": ["Windows"],"count": 1}}]},"type": "EmailAddress","lastModified": "2023-12-01T08:26:48Z","rating": 3,"confidence": 61,"threatAssessRating": 3,"threatAssessConfidence": 61,"threatAssessScore": 382,"threatAssessScoreObserved": 0,"threatAssessScoreFalsePositive": 0,"summary": "johnbae@poverts.com","privateFlag": false,"active": true,"activeLocked": false,"Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","md5": "F5A2496CF66CXXCFFE66CXXB27D7XXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","hostName": "samplehost","size": 123,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/tes_pc.html","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z","Hashtag": "#testabc","Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"Subject": "Spam","source": "https://fp.tools/api/v4/indicators/attribute/pN0psYjPUQ6a_sxXXXXX","externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","Block": "0.0.0.0","User Agent String": "PostmanRuntime/7.32.3","associatedGroups": {"data": [{"id": 6,"ownerId": 51,"ownerName": "DemoOrganization","dateAdded": "2021-11-03T14:57:45Z","webLink": "https://app.threatconnect.com/#/details/groups/3/overview","type": "Incident","name": "BadIncident","createdBy": {"userName": "johnsmithxyz@gmail.com","firstName": "john","lastName": "smith","pseudonym": "jsmithAPI","owner": "DemoOrganization","id": 3},"upVoteCount": "0","downVoteCount": "0","generatedReport": true,"password": "duwyhfsjhsi","malware": true,"lastModified": "2021-10-21T19:54:59Z","legacyLink": "https://app.threatconnect.com/auth/document/document.xhtml?document=10","to": "demo@sample.com","from": "auto-confirm@bad.com","subject": "YourAmazon.comorderfordemo@sample.com","header": "emailheadergoeshere","body": "Pleasevisitbad.comtoseeyourorderandgiveusallyourmoney","scoreIncludesBody": true,"emailDate": "2021-09-17T12:50:19Z","scoreBreakdown": "RuleSPFNeutralwasmatchedagainst'neutral'.","eventDate": "2021-09-17T12:50:19Z","status": "New","publishDate": "2021-09-17T12:50:19Z","fileText": "Filetext","assignments": {"data": [{"type": "Assigned","user": {"id": 12}}]},"dueDate": "2021-09-17T12:50:19Z","escalationDate": "2021-09-17T12:50:19Z","reminderDate": "2021-09-17T12:50:19Z","externalDateAdded": "2021-09-17T12:50:19Z","externalDateExpires": "2021-09-17T12:50:19Z","externalLastModified": "2021-09-17T12:50:19Z","firstSeen": "2021-09-17T12:50:19Z","lastSeen": "2021-09-17T12:50:19Z","xid": "a1a1a1a1-a1a1-a1a1-a1a1-a1a1a1a1a1a1","upVote": false,"fileName": "indicators.txt","fileSize": 36,"documentType": "Text","documentDateAdded": "2021-10-21T19:54:59Z","fileType": "Hash"}]},"associatedIndicators": {"data": [{"lastModified": "2021-11-02T13:07:08Z","description": "A bad email found","Subject": "Spam","id": 10,"md5": "F5A2496CF66CB8CFFE66CB1B27DXXXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","size": 124,"Block": "0.0.0.0","hostName": "samplehost","type": "File","summary": "F5A2496CF66CB8CFFE66CB1B27D7DEDE","confidence": 20,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/test_pc.html","Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","address": "johnxyz@newnime.com","User Agent String": "PostmanRuntime/7.32.3","ownerId": 1,"ownerName": "DemoOrganization","dateAdded": "2021-11-02T13:07:08Z","webLink": "https://app.threatconnect.com/#/details/indicators/10/overview","privateFlag": false,"active": true,"activeLocked": false,"legacyLink": "https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization","Hashtag": "#testabc","rating": 3,"Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z"},{"lastModified": "2021-11-02T13:07:08Z","description": "A bad email found","Subject": "Spam","id": 11,"md5": "F5A2496CF66CB8CFFE66CB1B27DXXXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","size": "124","Block": "0.0.0.0/8","hostName": "samplehost","type": "File","summary": "F5A2496CF66CB8CFFE66CB1B27D7DEDE","confidence": 20,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/test_pc.html","Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","address": "johnxyz@newnime.com","User Agent String": "PostmanRuntime/7.32.3","ownerId": 1,"ownerName": "DemoOrganization","dateAdded": "2021-11-02T13:07:08Z","webLink": "https://app.threatconnect.com/#/details/indicators/10/overview","privateFlag": false,"active": true,"activeLocked": false,"legacyLink": "https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization","Hashtag": "#testabc","rating": 3,"Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z"},{"lastModified": "2021-11-02T13:07:08Z","description": "A bad email found","Subject": "Spam","id": 12,"md5": "F5A2496CF66CB8CFFE66CB1B27DXXXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","size": "124","Block": "0.0.0.0/125","hostName": "samplehost","type": "File","summary": "F5A2496CF66CB8CFFE66CB1B27D7DEDE","confidence": 20,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/test_pc.html","Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","address": "johnxyz@newnime.com","User Agent String": "PostmanRuntime/7.32.3","ownerId": 1,"ownerName": "DemoOrganization","dateAdded": "2021-11-02T13:07:08Z","webLink": "https://app.threatconnect.com/#/details/indicators/10/overview","privateFlag": false,"active": true,"activeLocked": false,"legacyLink": "https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization","Hashtag": "#testabc","rating": 3,"Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z"}]},"attributes": {"data": [{"id": 6843246,"dateAdded": "2023-08-25T13:16:12Z","type": "EmailAddressUsage","value": "PhishingEmailSender","createdBy": {"id": 69,"userName": "johnxys@abc.co","firstName": "John","lastName": "Smith","pseudonym": "JohnS","owner": "Elastic"},"lastModified": "2023-08-25T13:16:12Z","pinned": false,"default": false}]},"address": "hohnabc@xyz.com","legacyLink": "https://partnerstage-intel.threatconnect.com/auth/indicators/details/emailaddress.xhtml?emailaddress=misoyil388%40poverts.com&owner=Elastic","associatedArtifacts": {"data": [{"id": 12345}]},"associatedCases": {"data": [{"id": 123457}]},"fileActions": {"data": [{"id": 123456}]},"fileOccurrences": {"data": [{"fileName": "win999301.dll","path": "C:\\Windows\\System","date": "2022-06-14T10:00:00Z"}]},"customAssociations": {"data": [{"id": 123458}]},"dnsResolution": {"data": [{"id": 123459}]},"enrichment": {"data": [{"id": 123455}]},"falsePositives": 1,"lastFalsePositive": "2023-10-04T12:34:56Z","falsePositiveReportedByUser": false,"genericCustomIndicatorValues": {"data": [{"id": 1234551}]},"geoLocation": {"data": [{"id": 1234552}]},"investigationLinks": {"data": [{"id": 1234553}]},"observations": {"data": [{"id": 1234556}]},"trackedUsers": {"data": [{"id": 1234557}]},"whoIs": {"data": [{"id": 1234558}]}} +{"id": 891599,"dateAdded": "2023-08-25T12:57:24Z","description": "bad email","securityLabels": {"data": [{"id": 3,"name": "TLP:AMBER","source": "https://fp.tools/api/v4/indicators/attribute/pN0psYjPUQ6a_sxPSW5XjQ","description": "Thissecuritylabelisusedforinformationthatrequiressupporttobeeffectivelyactedupon,yetcarriesriskstoprivacy,reputation,oroperationsifsharedoutsideoftheorganizationsinvolved.Informationwiththislabelcanbesharedwithmembersofanorganizationanditsclients.","color": "FFC000","owner": "System","dateAdded": "2016-08-31T00:00:00Z"}]},"ownerId": 51,"ownerName": "Elastic","webLink": "https://partnerstage-intel.threatconnect.com/","tags": {"data": [{"id": 1,"name": "userexecution:maliciouslink","lastUsed": "2023-08-25T13:15:30Z","description": "ApplythisTagtoobjectsrelatedtoransomwareattacks","owner": "Demoorganization","techniqueId": "T1055.005","platforms": {"data": ["Windows"],"count": 1}}]},"type": "EmailAddress","lastModified": "2023-12-01T08:26:48Z","rating": 3,"confidence": 61,"threatAssessRating": 3,"threatAssessConfidence": 61,"threatAssessScore": 382,"threatAssessScoreObserved": 0,"threatAssessScoreFalsePositive": 0,"summary": "johnbae@poverts.com","privateFlag": false,"active": true,"activeLocked": false,"Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","md5": "F5A2496CF66CXXCFFE66CXXB27D7XXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","hostName": "samplehost","size": 123,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/tes_pc.html","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z","Hashtag": "#testabc","Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"Subject": "Spam","source": "https://fp.tools/api/v4/indicators/attribute/pN0psYjPUQ6a_sxXXXXX","externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","Block": "0.0.0.0","User Agent String": "PostmanRuntime/7.32.3","associatedGroups": {"data": [{"id": 6,"ownerId": 51,"ownerName": "DemoOrganization","dateAdded": "2021-11-03T14:57:45Z","webLink": "https://app.threatconnect.com/#/details/groups/3/overview","type": "Incident","name": "BadIncident","createdBy": {"userName": "johnsmithxyz@gmail.com","firstName": "john","lastName": "smith","pseudonym": "jsmithAPI","owner": "DemoOrganization","id": 3},"upVoteCount": "0","downVoteCount": "0","generatedReport": true,"password": "duwyhfsjhsi","malware": true,"lastModified": "2021-10-21T19:54:59Z","legacyLink": "https://app.threatconnect.com/auth/document/document.xhtml?document=10","to": "demo@sample.com","from": "auto-confirm@bad.com","subject": "YourAmazon.comorderfordemo@sample.com","header": "emailheadergoeshere","body": "Pleasevisitbad.comtoseeyourorderandgiveusallyourmoney","scoreIncludesBody": true,"emailDate": "2021-09-17T12:50:19Z","scoreBreakdown": "RuleSPFNeutralwasmatchedagainst'neutral'.","eventDate": "2021-09-17T12:50:19Z","status": "New","publishDate": "2021-09-17T12:50:19Z","fileText": "Filetext","assignments": {"data": [{"type": "Assigned","user": {"id": 12}}]},"dueDate": "2021-09-17T12:50:19Z","escalationDate": "2021-09-17T12:50:19Z","reminderDate": "2021-09-17T12:50:19Z","externalDateAdded": "2021-09-17T12:50:19Z","externalDateExpires": "2021-09-17T12:50:19Z","externalLastModified": "2021-09-17T12:50:19Z","firstSeen": "2021-09-17T12:50:19Z","lastSeen": "2021-09-17T12:50:19Z","xid": "a1a1a1a1-a1a1-a1a1-a1a1-a1a1a1a1a1a1","upVote": false,"fileName": "indicators.txt","fileSize": 36,"documentType": "Text","documentDateAdded": "2021-10-21T19:54:59Z","fileType": "Hash"}]},"associatedIndicators": {"data": [{"lastModified": "2021-11-02T13:07:08Z","description": "A bad email found","Subject": "Spam","id": 10,"md5": "F5A2496CF66CB8CFFE66CB1B27DXXXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","size": 124,"Block": "0.0.0.0","hostName": "samplehost","type": "File","summary": "F5A2496CF66CB8CFFE66CB1B27D7DEDE","confidence": 20,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/test_pc.html","Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","address": "johnxyz@newnime.com","User Agent String": "PostmanRuntime/7.32.3","ownerId": 1,"ownerName": "DemoOrganization","dateAdded": "2021-11-02T13:07:08Z","webLink": "https://app.threatconnect.com/#/details/indicators/10/overview","privateFlag": false,"active": true,"activeLocked": false,"legacyLink": "https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization","Hashtag": "#testabc","rating": 3,"Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z"},{"lastModified": "2021-11-02T13:07:08Z","description": "A bad email found","Subject": "Spam","id": 11,"md5": "F5A2496CF66CB8CFFE66CB1B27DXXXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","size": 124,"Block": "0.0.0.0/8","hostName": "samplehost","type": "File","summary": "F5A2496CF66CB8CFFE66CB1B27D7DEDE","confidence": 20,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/test_pc.html","Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","address": "johnxyz@newnime.com","User Agent String": "PostmanRuntime/7.32.3","ownerId": 1,"ownerName": "DemoOrganization","dateAdded": "2021-11-02T13:07:08Z","webLink": "https://app.threatconnect.com/#/details/indicators/10/overview","privateFlag": false,"active": true,"activeLocked": false,"legacyLink": "https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization","Hashtag": "#testabc","rating": 3,"Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z"},{"lastModified": "2021-11-02T13:07:08Z","description": "A bad email found","Subject": "Spam","id": 12,"md5": "F5A2496CF66CB8CFFE66CB1B27DXXXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","size": 124,"Block": "0.0.0.0/125","hostName": "samplehost","type": "File","summary": "F5A2496CF66CB8CFFE66CB1B27D7DEDE","confidence": 20,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/test_pc.html","Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","address": "johnxyz@newnime.com","User Agent String": "PostmanRuntime/7.32.3","ownerId": 1,"ownerName": "DemoOrganization","dateAdded": "2021-11-02T13:07:08Z","webLink": "https://app.threatconnect.com/#/details/indicators/10/overview","privateFlag": false,"active": true,"activeLocked": false,"legacyLink": "https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization","Hashtag": "#testabc","rating": 3,"Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z"}]},"attributes": {"data": [{"id": 6843246,"dateAdded": "2023-08-25T13:16:12Z","type": "EmailAddressUsage","value": "PhishingEmailSender","createdBy": {"id": 69,"userName": "johnxys@abc.co","firstName": "John","lastName": "Smith","pseudonym": "JohnS","owner": "Elastic"},"lastModified": "2023-08-25T13:16:12Z","pinned": false,"default": false}]},"address": "hohnabc@xyz.com","legacyLink": "https://partnerstage-intel.threatconnect.com/auth/indicators/details/emailaddress.xhtml?emailaddress=misoyil388%40poverts.com&owner=Elastic","associatedArtifacts": {"data": [{"id": 12345}]},"associatedCases": {"data": [{"id": 123457}]},"fileActions": {"data": [{"id": 123456}]},"fileOccurrences": {"data": [{"fileName": "win999301.dll","path": "C:\\Windows\\System","date": "2022-06-14T10:00:00Z"}]},"customAssociations": {"data": [{"id": 123458}]},"dnsResolution": {"data": [{"id": 123459}]},"enrichment": {"data": [{"id": 123455}]},"falsePositives": 1,"lastFalsePositive": "2023-10-04T12:34:56Z","falsePositiveReportedByUser": false,"genericCustomIndicatorValues": {"data": [{"id": 1234551}]},"geoLocation": {"data": [{"id": 1234552}]},"investigationLinks": {"data": [{"id": 1234553}]},"observations": {"data": [{"id": 1234556}]},"trackedUsers": {"data": [{"id": 1234557}]},"whoIs": {"data": [{"id": 1234558}]}} diff --git a/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json b/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json index 3670ed124f6..fa2284b2119 100644 --- a/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json +++ b/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json @@ -24,7 +24,7 @@ ], "id": "johnbae@poverts.com", "kind": "enrichment", - "original": "{\"id\": 891599,\"dateAdded\": \"2023-08-25T12:57:24Z\",\"description\": \"bad email\",\"securityLabels\": {\"data\": [{\"id\": 3,\"name\": \"TLP:AMBER\",\"source\": \"https://fp.tools/api/v4/indicators/attribute/pN0psYjPUQ6a_sxPSW5XjQ\",\"description\": \"Thissecuritylabelisusedforinformationthatrequiressupporttobeeffectivelyactedupon,yetcarriesriskstoprivacy,reputation,oroperationsifsharedoutsideoftheorganizationsinvolved.Informationwiththislabelcanbesharedwithmembersofanorganizationanditsclients.\",\"color\": \"FFC000\",\"owner\": \"System\",\"dateAdded\": \"2016-08-31T00:00:00Z\"}]},\"ownerId\": 51,\"ownerName\": \"Elastic\",\"webLink\": \"https://partnerstage-intel.threatconnect.com/\",\"tags\": {\"data\": [{\"id\": 1,\"name\": \"userexecution:maliciouslink\",\"lastUsed\": \"2023-08-25T13:15:30Z\",\"description\": \"ApplythisTagtoobjectsrelatedtoransomwareattacks\",\"owner\": \"Demoorganization\",\"techniqueId\": \"T1055.005\",\"platforms\": {\"data\": [\"Windows\"],\"count\": 1}}]},\"type\": \"EmailAddress\",\"lastModified\": \"2023-12-01T08:26:48Z\",\"rating\": 3,\"confidence\": 61,\"threatAssessRating\": 3,\"threatAssessConfidence\": 61,\"threatAssessScore\": 382,\"threatAssessScoreObserved\": 0,\"threatAssessScoreFalsePositive\": 0,\"summary\": \"johnbae@poverts.com\",\"privateFlag\": false,\"active\": true,\"activeLocked\": false,\"Key Name\": \"HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\Setup\\\\Status\\\\ChildCompletion\",\"Value Name\": \"0\",\"Value Type\": \"REG_QWORD\",\"AS Number\": \"ASN1234\",\"md5\": \"F5A2496CF66CXXCFFE66CXXB27D7XXXX\",\"sha256\": \"7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX\",\"sha1\": \"samplesha1\",\"hostName\": \"samplehost\",\"size\": 123,\"ip\": \"0.0.0.0\",\"text\": \"http://www.testingmcafeesites.com/tes_pc.html\",\"firstSeen\": \"2023-10-04T12:34:56Z\",\"lastSeen\": \"2023-10-04T12:34:56Z\",\"Hashtag\": \"#testabc\",\"Mutex\": \"Test.Mutex()\",\"dnsActive\": false,\"whoisActive\": true,\"Subject\": \"Spam\",\"source\": \"https://fp.tools/api/v4/indicators/attribute/pN0psYjPUQ6a_sxXXXXX\",\"externalDateAdded\": \"2023-10-04T12:34:56Z\",\"externalDateExpires\": \"2023-10-04T12:34:56Z\",\"externalLastModified\": \"2023-10-04T12:34:56Z\",\"Block\": \"0.0.0.0\",\"User Agent String\": \"PostmanRuntime/7.32.3\",\"associatedGroups\": {\"data\": [{\"id\": 6,\"ownerId\": 51,\"ownerName\": \"DemoOrganization\",\"dateAdded\": \"2021-11-03T14:57:45Z\",\"webLink\": \"https://app.threatconnect.com/#/details/groups/3/overview\",\"type\": \"Incident\",\"name\": \"BadIncident\",\"createdBy\": {\"userName\": \"johnsmithxyz@gmail.com\",\"firstName\": \"john\",\"lastName\": \"smith\",\"pseudonym\": \"jsmithAPI\",\"owner\": \"DemoOrganization\",\"id\": 3},\"upVoteCount\": \"0\",\"downVoteCount\": \"0\",\"generatedReport\": true,\"password\": \"duwyhfsjhsi\",\"malware\": true,\"lastModified\": \"2021-10-21T19:54:59Z\",\"legacyLink\": \"https://app.threatconnect.com/auth/document/document.xhtml?document=10\",\"to\": \"demo@sample.com\",\"from\": \"auto-confirm@bad.com\",\"subject\": \"YourAmazon.comorderfordemo@sample.com\",\"header\": \"emailheadergoeshere\",\"body\": \"Pleasevisitbad.comtoseeyourorderandgiveusallyourmoney\",\"scoreIncludesBody\": true,\"emailDate\": \"2021-09-17T12:50:19Z\",\"scoreBreakdown\": \"RuleSPFNeutralwasmatchedagainst'neutral'.\",\"eventDate\": \"2021-09-17T12:50:19Z\",\"status\": \"New\",\"publishDate\": \"2021-09-17T12:50:19Z\",\"fileText\": \"Filetext\",\"assignments\": {\"data\": [{\"type\": \"Assigned\",\"user\": {\"id\": 12}}]},\"dueDate\": \"2021-09-17T12:50:19Z\",\"escalationDate\": \"2021-09-17T12:50:19Z\",\"reminderDate\": \"2021-09-17T12:50:19Z\",\"externalDateAdded\": \"2021-09-17T12:50:19Z\",\"externalDateExpires\": \"2021-09-17T12:50:19Z\",\"externalLastModified\": \"2021-09-17T12:50:19Z\",\"firstSeen\": \"2021-09-17T12:50:19Z\",\"lastSeen\": \"2021-09-17T12:50:19Z\",\"xid\": \"a1a1a1a1-a1a1-a1a1-a1a1-a1a1a1a1a1a1\",\"upVote\": false,\"fileName\": \"indicators.txt\",\"fileSize\": 36,\"documentType\": \"Text\",\"documentDateAdded\": \"2021-10-21T19:54:59Z\",\"fileType\": \"Hash\"}]},\"associatedIndicators\": {\"data\": [{\"lastModified\": \"2021-11-02T13:07:08Z\",\"description\": \"A bad email found\",\"Subject\": \"Spam\",\"id\": 10,\"md5\": \"F5A2496CF66CB8CFFE66CB1B27DXXXXX\",\"sha256\": \"7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX\",\"sha1\": \"samplesha1\",\"size\": 124,\"Block\": \"0.0.0.0\",\"hostName\": \"samplehost\",\"type\": \"File\",\"summary\": \"F5A2496CF66CB8CFFE66CB1B27D7DEDE\",\"confidence\": 20,\"ip\": \"0.0.0.0\",\"text\": \"http://www.testingmcafeesites.com/test_pc.html\",\"Key Name\": \"HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\Setup\\\\Status\\\\ChildCompletion\",\"Value Name\": \"0\",\"Value Type\": \"REG_QWORD\",\"AS Number\": \"ASN1234\",\"address\": \"johnxyz@newnime.com\",\"User Agent String\": \"PostmanRuntime/7.32.3\",\"ownerId\": 1,\"ownerName\": \"DemoOrganization\",\"dateAdded\": \"2021-11-02T13:07:08Z\",\"webLink\": \"https://app.threatconnect.com/#/details/indicators/10/overview\",\"privateFlag\": false,\"active\": true,\"activeLocked\": false,\"legacyLink\": \"https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization\",\"Hashtag\": \"#testabc\",\"rating\": 3,\"Mutex\": \"Test.Mutex()\",\"dnsActive\": false,\"whoisActive\": true,\"externalDateAdded\": \"2023-10-04T12:34:56Z\",\"externalDateExpires\": \"2023-10-04T12:34:56Z\",\"externalLastModified\": \"2023-10-04T12:34:56Z\",\"firstSeen\": \"2023-10-04T12:34:56Z\",\"lastSeen\": \"2023-10-04T12:34:56Z\"},{\"lastModified\": \"2021-11-02T13:07:08Z\",\"description\": \"A bad email found\",\"Subject\": \"Spam\",\"id\": 11,\"md5\": \"F5A2496CF66CB8CFFE66CB1B27DXXXXX\",\"sha256\": \"7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX\",\"sha1\": \"samplesha1\",\"size\": \"124\",\"Block\": \"0.0.0.0/8\",\"hostName\": \"samplehost\",\"type\": \"File\",\"summary\": \"F5A2496CF66CB8CFFE66CB1B27D7DEDE\",\"confidence\": 20,\"ip\": \"0.0.0.0\",\"text\": \"http://www.testingmcafeesites.com/test_pc.html\",\"Key Name\": \"HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\Setup\\\\Status\\\\ChildCompletion\",\"Value Name\": \"0\",\"Value Type\": \"REG_QWORD\",\"AS Number\": \"ASN1234\",\"address\": \"johnxyz@newnime.com\",\"User Agent String\": \"PostmanRuntime/7.32.3\",\"ownerId\": 1,\"ownerName\": \"DemoOrganization\",\"dateAdded\": \"2021-11-02T13:07:08Z\",\"webLink\": \"https://app.threatconnect.com/#/details/indicators/10/overview\",\"privateFlag\": false,\"active\": true,\"activeLocked\": false,\"legacyLink\": \"https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization\",\"Hashtag\": \"#testabc\",\"rating\": 3,\"Mutex\": \"Test.Mutex()\",\"dnsActive\": false,\"whoisActive\": true,\"externalDateAdded\": \"2023-10-04T12:34:56Z\",\"externalDateExpires\": \"2023-10-04T12:34:56Z\",\"externalLastModified\": \"2023-10-04T12:34:56Z\",\"firstSeen\": \"2023-10-04T12:34:56Z\",\"lastSeen\": \"2023-10-04T12:34:56Z\"},{\"lastModified\": \"2021-11-02T13:07:08Z\",\"description\": \"A bad email found\",\"Subject\": \"Spam\",\"id\": 12,\"md5\": \"F5A2496CF66CB8CFFE66CB1B27DXXXXX\",\"sha256\": \"7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX\",\"sha1\": \"samplesha1\",\"size\": \"124\",\"Block\": \"0.0.0.0/125\",\"hostName\": \"samplehost\",\"type\": \"File\",\"summary\": \"F5A2496CF66CB8CFFE66CB1B27D7DEDE\",\"confidence\": 20,\"ip\": \"0.0.0.0\",\"text\": \"http://www.testingmcafeesites.com/test_pc.html\",\"Key Name\": \"HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\Setup\\\\Status\\\\ChildCompletion\",\"Value Name\": \"0\",\"Value Type\": \"REG_QWORD\",\"AS Number\": \"ASN1234\",\"address\": \"johnxyz@newnime.com\",\"User Agent String\": \"PostmanRuntime/7.32.3\",\"ownerId\": 1,\"ownerName\": \"DemoOrganization\",\"dateAdded\": \"2021-11-02T13:07:08Z\",\"webLink\": \"https://app.threatconnect.com/#/details/indicators/10/overview\",\"privateFlag\": false,\"active\": true,\"activeLocked\": false,\"legacyLink\": \"https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization\",\"Hashtag\": \"#testabc\",\"rating\": 3,\"Mutex\": \"Test.Mutex()\",\"dnsActive\": false,\"whoisActive\": true,\"externalDateAdded\": \"2023-10-04T12:34:56Z\",\"externalDateExpires\": \"2023-10-04T12:34:56Z\",\"externalLastModified\": \"2023-10-04T12:34:56Z\",\"firstSeen\": \"2023-10-04T12:34:56Z\",\"lastSeen\": \"2023-10-04T12:34:56Z\"}]},\"attributes\": {\"data\": [{\"id\": 6843246,\"dateAdded\": \"2023-08-25T13:16:12Z\",\"type\": \"EmailAddressUsage\",\"value\": \"PhishingEmailSender\",\"createdBy\": {\"id\": 69,\"userName\": \"johnxys@abc.co\",\"firstName\": \"John\",\"lastName\": \"Smith\",\"pseudonym\": \"JohnS\",\"owner\": \"Elastic\"},\"lastModified\": \"2023-08-25T13:16:12Z\",\"pinned\": false,\"default\": false}]},\"address\": \"hohnabc@xyz.com\",\"legacyLink\": \"https://partnerstage-intel.threatconnect.com/auth/indicators/details/emailaddress.xhtml?emailaddress=misoyil388%40poverts.com&owner=Elastic\",\"associatedArtifacts\": {\"data\": [{\"id\": 12345}]},\"associatedCases\": {\"data\": [{\"id\": 123457}]},\"fileActions\": {\"data\": [{\"id\": 123456}]},\"fileOccurrences\": {\"data\": [{\"fileName\": \"win999301.dll\",\"path\": \"C:\\\\Windows\\\\System\",\"date\": \"2022-06-14T10:00:00Z\"}]},\"customAssociations\": {\"data\": [{\"id\": 123458}]},\"dnsResolution\": {\"data\": [{\"id\": 123459}]},\"enrichment\": {\"data\": [{\"id\": 123455}]},\"falsePositives\": 1,\"lastFalsePositive\": \"2023-10-04T12:34:56Z\",\"falsePositiveReportedByUser\": false,\"genericCustomIndicatorValues\": {\"data\": [{\"id\": 1234551}]},\"geoLocation\": {\"data\": [{\"id\": 1234552}]},\"investigationLinks\": {\"data\": [{\"id\": 1234553}]},\"observations\": {\"data\": [{\"id\": 1234556}]},\"trackedUsers\": {\"data\": [{\"id\": 1234557}]},\"whoIs\": {\"data\": [{\"id\": 1234558}]}}", + "original": "{\"id\": 891599,\"dateAdded\": \"2023-08-25T12:57:24Z\",\"description\": \"bad email\",\"securityLabels\": {\"data\": [{\"id\": 3,\"name\": \"TLP:AMBER\",\"source\": \"https://fp.tools/api/v4/indicators/attribute/pN0psYjPUQ6a_sxPSW5XjQ\",\"description\": \"Thissecuritylabelisusedforinformationthatrequiressupporttobeeffectivelyactedupon,yetcarriesriskstoprivacy,reputation,oroperationsifsharedoutsideoftheorganizationsinvolved.Informationwiththislabelcanbesharedwithmembersofanorganizationanditsclients.\",\"color\": \"FFC000\",\"owner\": \"System\",\"dateAdded\": \"2016-08-31T00:00:00Z\"}]},\"ownerId\": 51,\"ownerName\": \"Elastic\",\"webLink\": \"https://partnerstage-intel.threatconnect.com/\",\"tags\": {\"data\": [{\"id\": 1,\"name\": \"userexecution:maliciouslink\",\"lastUsed\": \"2023-08-25T13:15:30Z\",\"description\": \"ApplythisTagtoobjectsrelatedtoransomwareattacks\",\"owner\": \"Demoorganization\",\"techniqueId\": \"T1055.005\",\"platforms\": {\"data\": [\"Windows\"],\"count\": 1}}]},\"type\": \"EmailAddress\",\"lastModified\": \"2023-12-01T08:26:48Z\",\"rating\": 3,\"confidence\": 61,\"threatAssessRating\": 3,\"threatAssessConfidence\": 61,\"threatAssessScore\": 382,\"threatAssessScoreObserved\": 0,\"threatAssessScoreFalsePositive\": 0,\"summary\": \"johnbae@poverts.com\",\"privateFlag\": false,\"active\": true,\"activeLocked\": false,\"Key Name\": \"HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\Setup\\\\Status\\\\ChildCompletion\",\"Value Name\": \"0\",\"Value Type\": \"REG_QWORD\",\"AS Number\": \"ASN1234\",\"md5\": \"F5A2496CF66CXXCFFE66CXXB27D7XXXX\",\"sha256\": \"7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX\",\"sha1\": \"samplesha1\",\"hostName\": \"samplehost\",\"size\": 123,\"ip\": \"0.0.0.0\",\"text\": \"http://www.testingmcafeesites.com/tes_pc.html\",\"firstSeen\": \"2023-10-04T12:34:56Z\",\"lastSeen\": \"2023-10-04T12:34:56Z\",\"Hashtag\": \"#testabc\",\"Mutex\": \"Test.Mutex()\",\"dnsActive\": false,\"whoisActive\": true,\"Subject\": \"Spam\",\"source\": \"https://fp.tools/api/v4/indicators/attribute/pN0psYjPUQ6a_sxXXXXX\",\"externalDateAdded\": \"2023-10-04T12:34:56Z\",\"externalDateExpires\": \"2023-10-04T12:34:56Z\",\"externalLastModified\": \"2023-10-04T12:34:56Z\",\"Block\": \"0.0.0.0\",\"User Agent String\": \"PostmanRuntime/7.32.3\",\"associatedGroups\": {\"data\": [{\"id\": 6,\"ownerId\": 51,\"ownerName\": \"DemoOrganization\",\"dateAdded\": \"2021-11-03T14:57:45Z\",\"webLink\": \"https://app.threatconnect.com/#/details/groups/3/overview\",\"type\": \"Incident\",\"name\": \"BadIncident\",\"createdBy\": {\"userName\": \"johnsmithxyz@gmail.com\",\"firstName\": \"john\",\"lastName\": \"smith\",\"pseudonym\": \"jsmithAPI\",\"owner\": \"DemoOrganization\",\"id\": 3},\"upVoteCount\": \"0\",\"downVoteCount\": \"0\",\"generatedReport\": true,\"password\": \"duwyhfsjhsi\",\"malware\": true,\"lastModified\": \"2021-10-21T19:54:59Z\",\"legacyLink\": \"https://app.threatconnect.com/auth/document/document.xhtml?document=10\",\"to\": \"demo@sample.com\",\"from\": \"auto-confirm@bad.com\",\"subject\": \"YourAmazon.comorderfordemo@sample.com\",\"header\": \"emailheadergoeshere\",\"body\": \"Pleasevisitbad.comtoseeyourorderandgiveusallyourmoney\",\"scoreIncludesBody\": true,\"emailDate\": \"2021-09-17T12:50:19Z\",\"scoreBreakdown\": \"RuleSPFNeutralwasmatchedagainst'neutral'.\",\"eventDate\": \"2021-09-17T12:50:19Z\",\"status\": \"New\",\"publishDate\": \"2021-09-17T12:50:19Z\",\"fileText\": \"Filetext\",\"assignments\": {\"data\": [{\"type\": \"Assigned\",\"user\": {\"id\": 12}}]},\"dueDate\": \"2021-09-17T12:50:19Z\",\"escalationDate\": \"2021-09-17T12:50:19Z\",\"reminderDate\": \"2021-09-17T12:50:19Z\",\"externalDateAdded\": \"2021-09-17T12:50:19Z\",\"externalDateExpires\": \"2021-09-17T12:50:19Z\",\"externalLastModified\": \"2021-09-17T12:50:19Z\",\"firstSeen\": \"2021-09-17T12:50:19Z\",\"lastSeen\": \"2021-09-17T12:50:19Z\",\"xid\": \"a1a1a1a1-a1a1-a1a1-a1a1-a1a1a1a1a1a1\",\"upVote\": false,\"fileName\": \"indicators.txt\",\"fileSize\": 36,\"documentType\": \"Text\",\"documentDateAdded\": \"2021-10-21T19:54:59Z\",\"fileType\": \"Hash\"}]},\"associatedIndicators\": {\"data\": [{\"lastModified\": \"2021-11-02T13:07:08Z\",\"description\": \"A bad email found\",\"Subject\": \"Spam\",\"id\": 10,\"md5\": \"F5A2496CF66CB8CFFE66CB1B27DXXXXX\",\"sha256\": \"7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX\",\"sha1\": \"samplesha1\",\"size\": 124,\"Block\": \"0.0.0.0\",\"hostName\": \"samplehost\",\"type\": \"File\",\"summary\": \"F5A2496CF66CB8CFFE66CB1B27D7DEDE\",\"confidence\": 20,\"ip\": \"0.0.0.0\",\"text\": \"http://www.testingmcafeesites.com/test_pc.html\",\"Key Name\": \"HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\Setup\\\\Status\\\\ChildCompletion\",\"Value Name\": \"0\",\"Value Type\": \"REG_QWORD\",\"AS Number\": \"ASN1234\",\"address\": \"johnxyz@newnime.com\",\"User Agent String\": \"PostmanRuntime/7.32.3\",\"ownerId\": 1,\"ownerName\": \"DemoOrganization\",\"dateAdded\": \"2021-11-02T13:07:08Z\",\"webLink\": \"https://app.threatconnect.com/#/details/indicators/10/overview\",\"privateFlag\": false,\"active\": true,\"activeLocked\": false,\"legacyLink\": \"https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization\",\"Hashtag\": \"#testabc\",\"rating\": 3,\"Mutex\": \"Test.Mutex()\",\"dnsActive\": false,\"whoisActive\": true,\"externalDateAdded\": \"2023-10-04T12:34:56Z\",\"externalDateExpires\": \"2023-10-04T12:34:56Z\",\"externalLastModified\": \"2023-10-04T12:34:56Z\",\"firstSeen\": \"2023-10-04T12:34:56Z\",\"lastSeen\": \"2023-10-04T12:34:56Z\"},{\"lastModified\": \"2021-11-02T13:07:08Z\",\"description\": \"A bad email found\",\"Subject\": \"Spam\",\"id\": 11,\"md5\": \"F5A2496CF66CB8CFFE66CB1B27DXXXXX\",\"sha256\": \"7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX\",\"sha1\": \"samplesha1\",\"size\": 124,\"Block\": \"0.0.0.0/8\",\"hostName\": \"samplehost\",\"type\": \"File\",\"summary\": \"F5A2496CF66CB8CFFE66CB1B27D7DEDE\",\"confidence\": 20,\"ip\": \"0.0.0.0\",\"text\": \"http://www.testingmcafeesites.com/test_pc.html\",\"Key Name\": \"HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\Setup\\\\Status\\\\ChildCompletion\",\"Value Name\": \"0\",\"Value Type\": \"REG_QWORD\",\"AS Number\": \"ASN1234\",\"address\": \"johnxyz@newnime.com\",\"User Agent String\": \"PostmanRuntime/7.32.3\",\"ownerId\": 1,\"ownerName\": \"DemoOrganization\",\"dateAdded\": \"2021-11-02T13:07:08Z\",\"webLink\": \"https://app.threatconnect.com/#/details/indicators/10/overview\",\"privateFlag\": false,\"active\": true,\"activeLocked\": false,\"legacyLink\": \"https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization\",\"Hashtag\": \"#testabc\",\"rating\": 3,\"Mutex\": \"Test.Mutex()\",\"dnsActive\": false,\"whoisActive\": true,\"externalDateAdded\": \"2023-10-04T12:34:56Z\",\"externalDateExpires\": \"2023-10-04T12:34:56Z\",\"externalLastModified\": \"2023-10-04T12:34:56Z\",\"firstSeen\": \"2023-10-04T12:34:56Z\",\"lastSeen\": \"2023-10-04T12:34:56Z\"},{\"lastModified\": \"2021-11-02T13:07:08Z\",\"description\": \"A bad email found\",\"Subject\": \"Spam\",\"id\": 12,\"md5\": \"F5A2496CF66CB8CFFE66CB1B27DXXXXX\",\"sha256\": \"7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX\",\"sha1\": \"samplesha1\",\"size\": 124,\"Block\": \"0.0.0.0/125\",\"hostName\": \"samplehost\",\"type\": \"File\",\"summary\": \"F5A2496CF66CB8CFFE66CB1B27D7DEDE\",\"confidence\": 20,\"ip\": \"0.0.0.0\",\"text\": \"http://www.testingmcafeesites.com/test_pc.html\",\"Key Name\": \"HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\Setup\\\\Status\\\\ChildCompletion\",\"Value Name\": \"0\",\"Value Type\": \"REG_QWORD\",\"AS Number\": \"ASN1234\",\"address\": \"johnxyz@newnime.com\",\"User Agent String\": \"PostmanRuntime/7.32.3\",\"ownerId\": 1,\"ownerName\": \"DemoOrganization\",\"dateAdded\": \"2021-11-02T13:07:08Z\",\"webLink\": \"https://app.threatconnect.com/#/details/indicators/10/overview\",\"privateFlag\": false,\"active\": true,\"activeLocked\": false,\"legacyLink\": \"https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization\",\"Hashtag\": \"#testabc\",\"rating\": 3,\"Mutex\": \"Test.Mutex()\",\"dnsActive\": false,\"whoisActive\": true,\"externalDateAdded\": \"2023-10-04T12:34:56Z\",\"externalDateExpires\": \"2023-10-04T12:34:56Z\",\"externalLastModified\": \"2023-10-04T12:34:56Z\",\"firstSeen\": \"2023-10-04T12:34:56Z\",\"lastSeen\": \"2023-10-04T12:34:56Z\"}]},\"attributes\": {\"data\": [{\"id\": 6843246,\"dateAdded\": \"2023-08-25T13:16:12Z\",\"type\": \"EmailAddressUsage\",\"value\": \"PhishingEmailSender\",\"createdBy\": {\"id\": 69,\"userName\": \"johnxys@abc.co\",\"firstName\": \"John\",\"lastName\": \"Smith\",\"pseudonym\": \"JohnS\",\"owner\": \"Elastic\"},\"lastModified\": \"2023-08-25T13:16:12Z\",\"pinned\": false,\"default\": false}]},\"address\": \"hohnabc@xyz.com\",\"legacyLink\": \"https://partnerstage-intel.threatconnect.com/auth/indicators/details/emailaddress.xhtml?emailaddress=misoyil388%40poverts.com&owner=Elastic\",\"associatedArtifacts\": {\"data\": [{\"id\": 12345}]},\"associatedCases\": {\"data\": [{\"id\": 123457}]},\"fileActions\": {\"data\": [{\"id\": 123456}]},\"fileOccurrences\": {\"data\": [{\"fileName\": \"win999301.dll\",\"path\": \"C:\\\\Windows\\\\System\",\"date\": \"2022-06-14T10:00:00Z\"}]},\"customAssociations\": {\"data\": [{\"id\": 123458}]},\"dnsResolution\": {\"data\": [{\"id\": 123459}]},\"enrichment\": {\"data\": [{\"id\": 123455}]},\"falsePositives\": 1,\"lastFalsePositive\": \"2023-10-04T12:34:56Z\",\"falsePositiveReportedByUser\": false,\"genericCustomIndicatorValues\": {\"data\": [{\"id\": 1234551}]},\"geoLocation\": {\"data\": [{\"id\": 1234552}]},\"investigationLinks\": {\"data\": [{\"id\": 1234553}]},\"observations\": {\"data\": [{\"id\": 1234556}]},\"trackedUsers\": {\"data\": [{\"id\": 1234557}]},\"whoIs\": {\"data\": [{\"id\": 1234558}]}}", "type": [ "indicator" ] @@ -72,6 +72,9 @@ "email": { "address": "hohnabc@xyz.com" }, + "file": { + "size": 123 + }, "first_seen": "2023-10-04T12:34:56.000Z", "last_seen": "2023-10-04T12:34:56.000Z", "marking": { diff --git a/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-with-associated-groups-and-attributes.log-expected.json b/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-with-associated-groups-and-attributes.log-expected.json index 7537b86c8d0..7f43c545d92 100644 --- a/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-with-associated-groups-and-attributes.log-expected.json +++ b/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-with-associated-groups-and-attributes.log-expected.json @@ -72,6 +72,9 @@ "email": { "address": "hohnabc@xyz.com" }, + "file": { + "size": 123 + }, "first_seen": "2023-10-04T12:34:56.000Z", "last_seen": "2023-10-04T12:34:56.000Z", "marking": { diff --git a/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml index 29e7726ff53..797f9f828a8 100644 --- a/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -1804,17 +1804,11 @@ processors: value: '{{{threat_connect.indicator.legacy_link}}}' allow_duplicates: false if: ctx.threat_connect?.indicator?.legacy_link != null -# handle file size if exists before hash mappings - - convert: + - rename: field: json.size - tag: convert_size_to_string + tag: rename_size target_field: threat_connect.indicator.size - type: string ignore_missing: true - on_failure: - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: threat.indicator.file.size tag: set_threat_indicator_file_size_from_indicator_size @@ -1996,6 +1990,16 @@ processors: value: '{{{threat_connect.indicator.sha256}}}' allow_duplicates: false if: ctx.threat_connect?.indicator?.md5 != null + - convert: + field: threat_connect.indicator.size + tag: convert_size_to_string + target_field: threat_connect.indicator.size + type: string + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: field: json.source tag: rename_source