ti_threatconnect: update pipeline to lowercase all hashes

packages/ti_threatconnect/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.0.0"
+  changes:
+    - description: Normalize all hash fields to be lowercase.
+      type: breaking-change
+      link: https://github.com/elastic/integrations/pull/17455
 - version: "1.12.0"
   changes:
     - description: Allow transforms to run in unattended mode.

packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log

@@ -1 +1 @@
-{"id": 891599,"dateAdded": "2023-08-25T12:57:24Z","description": "bad email","securityLabels": {"data": [{"id": 3,"name": "TLP:AMBER","source": "https://fp.tools/api/v4/indicators/attribute/pN0psYjPUQ6a_sxPSW5XjQ","description": "Thissecuritylabelisusedforinformationthatrequiressupporttobeeffectivelyactedupon,yetcarriesriskstoprivacy,reputation,oroperationsifsharedoutsideoftheorganizationsinvolved.Informationwiththislabelcanbesharedwithmembersofanorganizationanditsclients.","color": "FFC000","owner": "System","dateAdded": "2016-08-31T00:00:00Z"}]},"ownerId": 51,"ownerName": "Elastic","webLink": "https://partnerstage-intel.threatconnect.com/","tags": {"data": [{"id": 1,"name": "userexecution:maliciouslink","lastUsed": "2023-08-25T13:15:30Z","description": "ApplythisTagtoobjectsrelatedtoransomwareattacks","owner": "Demoorganization","techniqueId": "T1055.005","platforms": {"data": ["Windows"],"count": 1}}]},"type": "EmailAddress","lastModified": "2023-12-01T08:26:48Z","rating": 3,"confidence": 61,"threatAssessRating": 3,"threatAssessConfidence": 61,"threatAssessScore": 382,"threatAssessScoreObserved": 0,"threatAssessScoreFalsePositive": 0,"summary": "johnbae@poverts.com","privateFlag": false,"active": true,"activeLocked": false,"Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","md5": "F5A2496CF66CXXCFFE66CXXB27D7XXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","hostName": "samplehost","size": 123,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/tes_pc.html","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z","Hashtag": "#testabc","Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"Subject": "Spam","source": "https://fp.tools/api/v4/indicators/attribute/pN0psYjPUQ6a_sxXXXXX","externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","Block": "0.0.0.0","User Agent String": "PostmanRuntime/7.32.3","associatedGroups": {"data": [{"id": 6,"ownerId": 51,"ownerName": "DemoOrganization","dateAdded": "2021-11-03T14:57:45Z","webLink": "https://app.threatconnect.com/#/details/groups/3/overview","type": "Incident","name": "BadIncident","createdBy": {"userName": "johnsmithxyz@gmail.com","firstName": "john","lastName": "smith","pseudonym": "jsmithAPI","owner": "DemoOrganization","id": 3},"upVoteCount": "0","downVoteCount": "0","generatedReport": true,"password": "duwyhfsjhsi","malware": true,"lastModified": "2021-10-21T19:54:59Z","legacyLink": "https://app.threatconnect.com/auth/document/document.xhtml?document=10","to": "demo@sample.com","from": "auto-confirm@bad.com","subject": "YourAmazon.comorderfordemo@sample.com","header": "emailheadergoeshere","body": "Pleasevisitbad.comtoseeyourorderandgiveusallyourmoney","scoreIncludesBody": true,"emailDate": "2021-09-17T12:50:19Z","scoreBreakdown": "RuleSPFNeutralwasmatchedagainst'neutral'.","eventDate": "2021-09-17T12:50:19Z","status": "New","publishDate": "2021-09-17T12:50:19Z","fileText": "Filetext","assignments": {"data": [{"type": "Assigned","user": {"id": 12}}]},"dueDate": "2021-09-17T12:50:19Z","escalationDate": "2021-09-17T12:50:19Z","reminderDate": "2021-09-17T12:50:19Z","externalDateAdded": "2021-09-17T12:50:19Z","externalDateExpires": "2021-09-17T12:50:19Z","externalLastModified": "2021-09-17T12:50:19Z","firstSeen": "2021-09-17T12:50:19Z","lastSeen": "2021-09-17T12:50:19Z","xid": "a1a1a1a1-a1a1-a1a1-a1a1-a1a1a1a1a1a1","upVote": false,"fileName": "indicators.txt","fileSize": 36,"documentType": "Text","documentDateAdded": "2021-10-21T19:54:59Z","fileType": "Hash"}]},"associatedIndicators": {"data": [{"lastModified": "2021-11-02T13:07:08Z","description": "A bad email found","Subject": "Spam","id": 10,"md5": "F5A2496CF66CB8CFFE66CB1B27DXXXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","size": 124,"Block": "0.0.0.0","hostName": "samplehost","type": "File","summary": "F5A2496CF66CB8CFFE66CB1B27D7DEDE","confidence": 20,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/test_pc.html","Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","address": "johnxyz@newnime.com","User Agent String": "PostmanRuntime/7.32.3","ownerId": 1,"ownerName": "DemoOrganization","dateAdded": "2021-11-02T13:07:08Z","webLink": "https://app.threatconnect.com/#/details/indicators/10/overview","privateFlag": false,"active": true,"activeLocked": false,"legacyLink": "https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization","Hashtag": "#testabc","rating": 3,"Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z"},{"lastModified": "2021-11-02T13:07:08Z","description": "A bad email found","Subject": "Spam","id": 11,"md5": "F5A2496CF66CB8CFFE66CB1B27DXXXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","size": "124","Block": "0.0.0.0/8","hostName": "samplehost","type": "File","summary": "F5A2496CF66CB8CFFE66CB1B27D7DEDE","confidence": 20,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/test_pc.html","Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","address": "johnxyz@newnime.com","User Agent String": "PostmanRuntime/7.32.3","ownerId": 1,"ownerName": "DemoOrganization","dateAdded": "2021-11-02T13:07:08Z","webLink": "https://app.threatconnect.com/#/details/indicators/10/overview","privateFlag": false,"active": true,"activeLocked": false,"legacyLink": "https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization","Hashtag": "#testabc","rating": 3,"Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z"},{"lastModified": "2021-11-02T13:07:08Z","description": "A bad email found","Subject": "Spam","id": 12,"md5": "F5A2496CF66CB8CFFE66CB1B27DXXXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","size": "124","Block": "0.0.0.0/125","hostName": "samplehost","type": "File","summary": "F5A2496CF66CB8CFFE66CB1B27D7DEDE","confidence": 20,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/test_pc.html","Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","address": "johnxyz@newnime.com","User Agent String": "PostmanRuntime/7.32.3","ownerId": 1,"ownerName": "DemoOrganization","dateAdded": "2021-11-02T13:07:08Z","webLink": "https://app.threatconnect.com/#/details/indicators/10/overview","privateFlag": false,"active": true,"activeLocked": false,"legacyLink": "https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization","Hashtag": "#testabc","rating": 3,"Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z"}]},"attributes": {"data": [{"id": 6843246,"dateAdded": "2023-08-25T13:16:12Z","type": "EmailAddressUsage","value": "PhishingEmailSender","createdBy": {"id": 69,"userName": "johnxys@abc.co","firstName": "John","lastName": "Smith","pseudonym": "JohnS","owner": "Elastic"},"lastModified": "2023-08-25T13:16:12Z","pinned": false,"default": false}]},"address": "hohnabc@xyz.com","legacyLink": "https://partnerstage-intel.threatconnect.com/auth/indicators/details/emailaddress.xhtml?emailaddress=misoyil388%40poverts.com&owner=Elastic","associatedArtifacts": {"data": [{"id": 12345}]},"associatedCases": {"data": [{"id": 123457}]},"fileActions": {"data": [{"id": 123456}]},"fileOccurrences": {"data": [{"fileName": "win999301.dll","path": "C:\\Windows\\System","date": "2022-06-14T10:00:00Z"}]},"customAssociations": {"data": [{"id": 123458}]},"dnsResolution": {"data": [{"id": 123459}]},"enrichment": {"data": [{"id": 123455}]},"falsePositives": 1,"lastFalsePositive": "2023-10-04T12:34:56Z","falsePositiveReportedByUser": false,"genericCustomIndicatorValues": {"data": [{"id": 1234551}]},"geoLocation": {"data": [{"id": 1234552}]},"investigationLinks": {"data": [{"id": 1234553}]},"observations": {"data": [{"id": 1234556}]},"trackedUsers": {"data": [{"id": 1234557}]},"whoIs": {"data": [{"id": 1234558}]}}
+{"id": 891599,"dateAdded": "2023-08-25T12:57:24Z","description": "bad email","securityLabels": {"data": [{"id": 3,"name": "TLP:AMBER","source": "https://fp.tools/api/v4/indicators/attribute/pN0psYjPUQ6a_sxPSW5XjQ","description": "Thissecuritylabelisusedforinformationthatrequiressupporttobeeffectivelyactedupon,yetcarriesriskstoprivacy,reputation,oroperationsifsharedoutsideoftheorganizationsinvolved.Informationwiththislabelcanbesharedwithmembersofanorganizationanditsclients.","color": "FFC000","owner": "System","dateAdded": "2016-08-31T00:00:00Z"}]},"ownerId": 51,"ownerName": "Elastic","webLink": "https://partnerstage-intel.threatconnect.com/","tags": {"data": [{"id": 1,"name": "userexecution:maliciouslink","lastUsed": "2023-08-25T13:15:30Z","description": "ApplythisTagtoobjectsrelatedtoransomwareattacks","owner": "Demoorganization","techniqueId": "T1055.005","platforms": {"data": ["Windows"],"count": 1}}]},"type": "EmailAddress","lastModified": "2023-12-01T08:26:48Z","rating": 3,"confidence": 61,"threatAssessRating": 3,"threatAssessConfidence": 61,"threatAssessScore": 382,"threatAssessScoreObserved": 0,"threatAssessScoreFalsePositive": 0,"summary": "johnbae@poverts.com","privateFlag": false,"active": true,"activeLocked": false,"Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","md5": "F5A2496CF66CXXCFFE66CXXB27D7XXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","hostName": "samplehost","size": 123,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/tes_pc.html","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z","Hashtag": "#testabc","Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"Subject": "Spam","source": "https://fp.tools/api/v4/indicators/attribute/pN0psYjPUQ6a_sxXXXXX","externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","Block": "0.0.0.0","User Agent String": "PostmanRuntime/7.32.3","associatedGroups": {"data": [{"id": 6,"ownerId": 51,"ownerName": "DemoOrganization","dateAdded": "2021-11-03T14:57:45Z","webLink": "https://app.threatconnect.com/#/details/groups/3/overview","type": "Incident","name": "BadIncident","createdBy": {"userName": "johnsmithxyz@gmail.com","firstName": "john","lastName": "smith","pseudonym": "jsmithAPI","owner": "DemoOrganization","id": 3},"upVoteCount": "0","downVoteCount": "0","generatedReport": true,"password": "duwyhfsjhsi","malware": true,"lastModified": "2021-10-21T19:54:59Z","legacyLink": "https://app.threatconnect.com/auth/document/document.xhtml?document=10","to": "demo@sample.com","from": "auto-confirm@bad.com","subject": "YourAmazon.comorderfordemo@sample.com","header": "emailheadergoeshere","body": "Pleasevisitbad.comtoseeyourorderandgiveusallyourmoney","scoreIncludesBody": true,"emailDate": "2021-09-17T12:50:19Z","scoreBreakdown": "RuleSPFNeutralwasmatchedagainst'neutral'.","eventDate": "2021-09-17T12:50:19Z","status": "New","publishDate": "2021-09-17T12:50:19Z","fileText": "Filetext","assignments": {"data": [{"type": "Assigned","user": {"id": 12}}]},"dueDate": "2021-09-17T12:50:19Z","escalationDate": "2021-09-17T12:50:19Z","reminderDate": "2021-09-17T12:50:19Z","externalDateAdded": "2021-09-17T12:50:19Z","externalDateExpires": "2021-09-17T12:50:19Z","externalLastModified": "2021-09-17T12:50:19Z","firstSeen": "2021-09-17T12:50:19Z","lastSeen": "2021-09-17T12:50:19Z","xid": "a1a1a1a1-a1a1-a1a1-a1a1-a1a1a1a1a1a1","upVote": false,"fileName": "indicators.txt","fileSize": 36,"documentType": "Text","documentDateAdded": "2021-10-21T19:54:59Z","fileType": "Hash"}]},"associatedIndicators": {"data": [{"lastModified": "2021-11-02T13:07:08Z","description": "A bad email found","Subject": "Spam","id": 10,"md5": "F5A2496CF66CB8CFFE66CB1B27DXXXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","size": 124,"Block": "0.0.0.0","hostName": "samplehost","type": "File","summary": "F5A2496CF66CB8CFFE66CB1B27D7DEDE","confidence": 20,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/test_pc.html","Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","address": "johnxyz@newnime.com","User Agent String": "PostmanRuntime/7.32.3","ownerId": 1,"ownerName": "DemoOrganization","dateAdded": "2021-11-02T13:07:08Z","webLink": "https://app.threatconnect.com/#/details/indicators/10/overview","privateFlag": false,"active": true,"activeLocked": false,"legacyLink": "https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization","Hashtag": "#testabc","rating": 3,"Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z"},{"lastModified": "2021-11-02T13:07:08Z","description": "A bad email found","Subject": "Spam","id": 11,"md5": "F5A2496CF66CB8CFFE66CB1B27DXXXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","size": 124,"Block": "0.0.0.0/8","hostName": "samplehost","type": "File","summary": "F5A2496CF66CB8CFFE66CB1B27D7DEDE","confidence": 20,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/test_pc.html","Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","address": "johnxyz@newnime.com","User Agent String": "PostmanRuntime/7.32.3","ownerId": 1,"ownerName": "DemoOrganization","dateAdded": "2021-11-02T13:07:08Z","webLink": "https://app.threatconnect.com/#/details/indicators/10/overview","privateFlag": false,"active": true,"activeLocked": false,"legacyLink": "https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization","Hashtag": "#testabc","rating": 3,"Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z"},{"lastModified": "2021-11-02T13:07:08Z","description": "A bad email found","Subject": "Spam","id": 12,"md5": "F5A2496CF66CB8CFFE66CB1B27DXXXXX","sha256": "7D5FFFBFE8D098E369466164F705B4D692517A2B4659A03901DAF67CF78XXXXX","sha1": "samplesha1","size": 124,"Block": "0.0.0.0/125","hostName": "samplehost","type": "File","summary": "F5A2496CF66CB8CFFE66CB1B27D7DEDE","confidence": 20,"ip": "0.0.0.0","text": "http://www.testingmcafeesites.com/test_pc.html","Key Name": "HKEY_LOCAL_MACHINE\\SYSTEM\\Setup\\Status\\ChildCompletion","Value Name": "0","Value Type": "REG_QWORD","AS Number": "ASN1234","address": "johnxyz@newnime.com","User Agent String": "PostmanRuntime/7.32.3","ownerId": 1,"ownerName": "DemoOrganization","dateAdded": "2021-11-02T13:07:08Z","webLink": "https://app.threatconnect.com/#/details/indicators/10/overview","privateFlag": false,"active": true,"activeLocked": false,"legacyLink": "https://app.threatconnect.com/auth/indicators/details/file.xhtml?file=F5A2496CF66CB8CFFE66CB1B27D7DEDE&owner=Demo+Organization","Hashtag": "#testabc","rating": 3,"Mutex": "Test.Mutex()","dnsActive": false,"whoisActive": true,"externalDateAdded": "2023-10-04T12:34:56Z","externalDateExpires": "2023-10-04T12:34:56Z","externalLastModified": "2023-10-04T12:34:56Z","firstSeen": "2023-10-04T12:34:56Z","lastSeen": "2023-10-04T12:34:56Z"}]},"attributes": {"data": [{"id": 6843246,"dateAdded": "2023-08-25T13:16:12Z","type": "EmailAddressUsage","value": "PhishingEmailSender","createdBy": {"id": 69,"userName": "johnxys@abc.co","firstName": "John","lastName": "Smith","pseudonym": "JohnS","owner": "Elastic"},"lastModified": "2023-08-25T13:16:12Z","pinned": false,"default": false}]},"address": "hohnabc@xyz.com","legacyLink": "https://partnerstage-intel.threatconnect.com/auth/indicators/details/emailaddress.xhtml?emailaddress=misoyil388%40poverts.com&owner=Elastic","associatedArtifacts": {"data": [{"id": 12345}]},"associatedCases": {"data": [{"id": 123457}]},"fileActions": {"data": [{"id": 123456}]},"fileOccurrences": {"data": [{"fileName": "win999301.dll","path": "C:\\Windows\\System","date": "2022-06-14T10:00:00Z"}]},"customAssociations": {"data": [{"id": 123458}]},"dnsResolution": {"data": [{"id": 123459}]},"enrichment": {"data": [{"id": 123455}]},"falsePositives": 1,"lastFalsePositive": "2023-10-04T12:34:56Z","falsePositiveReportedByUser": false,"genericCustomIndicatorValues": {"data": [{"id": 1234551}]},"geoLocation": {"data": [{"id": 1234552}]},"investigationLinks": {"data": [{"id": 1234553}]},"observations": {"data": [{"id": 1234556}]},"trackedUsers": {"data": [{"id": 1234557}]},"whoIs": {"data": [{"id": 1234558}]}}