[pull] master from kevoreilly:master

analyzer/windows/modules/auxiliary/amsi_etw.py

@@ -0,0 +1,197 @@
+"""
+This module captures AMSI events via ETW, uploading script contents (powershell, WMI, macros, etc)
+to aux/amsi_etw and saving trace details to be reported by the amsi_etw processing module.
+
+It is a reimplementation of the SecureWorks amsi_collector and amsi modules, adapted to
+use the CCCS event tracing module format.
+
+Installation of the pywintrace python library on the guest is mandatory.
+Setting the option 'amsi_etw_assemblies=1' during tasking will cause full CLR assemblies
+to be collected as well.
+"""
+import json
+import logging
+import os
+import tempfile
+import binascii
+
+from lib.common.abstracts import Auxiliary
+from lib.common.results import upload_buffer_to_host, upload_to_host
+from lib.core.config import Config
+
+log = logging.getLogger(__name__)
+
+ETW = False
+HAVE_ETW = False
+try:
+    from etw import ETW, ProviderInfo
+    from etw.GUID import GUID
+
+    HAVE_ETW = True
+except ImportError as e:
+    log.debug(
+        "Could not load auxiliary module AMSI_ETW due to '%s'\nIn order to use AMSI_ETW functionality, it "
+        "is required to have pywintrace setup in python", str(e)
+    )
+
+if HAVE_ETW:
+
+    class ETW_provider(ETW):
+        def __init__(
+            self,
+            ring_buf_size=1024,
+            max_str_len=1024,
+            min_buffers=0,
+            max_buffers=0,
+            filters=None,
+            event_callback=None,
+            logfile=None,
+            upload_prefix="aux/amsi_etw",
+            upload_assemblies=False
+        ):
+            """
+            Initializes an instance of AMSI_ETW. The default parameters represent a very typical use case and should not be
+            overridden unless the user knows what they are doing.
+
+            :param ring_buf_size: The size of the ring buffer used for capturing events.
+            :param max_str_len: The maximum length of the strings the proceed the structure.
+                                Unless you know what you are doing, do not modify this value.
+            :param min_buffers: The minimum number of buffers for an event tracing session.
+                                Unless you know what you are doing, do not modify this value.
+            :param max_buffers: The maximum number of buffers for an event tracing session.
+                                Unless you know what you are doing, do not modify this value.
+            :param filters: List of filters to apply to capture.
+            :param logfile: Path to logfile.
+            :param upload_prefix: Path to upload results to. Must be approved in resultserver.py.
+            :param upload_assemblies: Whether to also upload the content of dotnet assemblies.
+            """
+            self.upload_prefix = upload_prefix
+            self.log_file = logfile
+            self.event_callback = self.on_event
+            self.upload_assemblies = upload_assemblies
+
+            providers = [
+                ProviderInfo(
+                    "AMSI",
+                    GUID("{2A576B87-09A7-520E-C21A-4942F0271D67}"),
+                    level=255,
+                    any_keywords=None,
+                    all_keywords=None,
+                )
+            ]
+            self.event_id_filters = [1101]
+            super().__init__(
+                session_name="ETW_AMSI",
+                ring_buf_size=ring_buf_size,
+                max_str_len=max_str_len,
+                min_buffers=min_buffers,
+                max_buffers=max_buffers,
+                event_callback=self.event_callback,
+                task_name_filters=filters,
+                providers=providers,
+                event_id_filters=self.event_id_filters,
+            )
+
+        def on_event(self, event_tufo):
+            """
+            Starts the capture using ETW.
+            :param event_tufo: tufo containing event information
+            :param logfile: Path to logfile.
+            :return: Does not return anything.
+            """
+            event_id, event = event_tufo
+            content = event.pop("content", None)
+            if content:
+                dump_path = f"{self.upload_prefix}/{event['hash'][2:].lower()}"
+                decoded_content = binascii.unhexlify(content[2:])
+                if event.get("appname", "") in ("DotNet", "coreclr"):
+                    # The content is the full in-memory .NET assembly PE.
+                    if self.upload_assemblies:
+                        event['dump_path'] =  dump_path+".bin"
+                        upload_buffer_to_host(decoded_content, event['dump_path'])
+                    else:
+                        log.debug("Skipping upload of %d byte CLR assembly - amsi_etw_assemblies option was not set", len(decoded_content))
+                else:
+                    # The content is UTF-16 encoded text. We'll store it as utf-8, just like all other text files.
+                    decoded_content = decoded_content.decode("utf-16", errors="replace").encode("utf-8")
+                    event['dump_path'] =  dump_path+".txt"
+                    upload_buffer_to_host(decoded_content, event['dump_path'])
+
+            if self.log_file:
+                # Write the event metadata as a line in the jsonl log file.
+                json.dump(event, self.log_file)
+                self.log_file.write("\n")
+
+        def start(self):
+            # do pre-capture setup
+            self.do_capture_setup()
+            super().start()
+
+        def stop(self):
+            super().stop()
+            # do post-capture teardown
+            self.do_capture_teardown()
+
+        def do_capture_setup(self):
+            # do whatever setup for capture here
+            pass
+
+        def do_capture_teardown(self):
+            # do whatever for capture teardown here
+            pass
+
+    class AMSI_ETW(Auxiliary):
+        """ETW logging"""
+
+        def __init__(self, options, config):
+            Auxiliary.__init__(self, options, config)
+
+            self.config = Config(cfg="analysis.conf")
+            self.enabled = self.config.amsi_etw
+            self.do_run = self.enabled
+            self.upload_prefix = "aux/amsi_etw"
+            self.upload_assemblies = options.get("amsi_etw_assemblies", False)
+            if self.upload_assemblies:
+                log.debug("Will upload Dotnet assembly content")
+            else:
+                log.debug("Will discard Dotnet assembly content")
+
+            if HAVE_ETW:
+                self.log_file = tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False)
+                self.capture = ETW_provider(logfile=self.log_file, upload_prefix=self.upload_prefix,
+                    upload_assemblies=self.upload_assemblies)
+
+        def start(self):
+            if not self.enabled or not HAVE_ETW:
+                return False
+            try:
+                log.debug("Starting AMSI ETW")
+                # Start AMSI_ETW_provider in the background
+                self.capture.start()
+            except Exception as e:
+                log.exception("An error occurred while starting AMSI ETW: %s", e)
+            return True
+
+        def stop(self):
+            if not HAVE_ETW:
+                return
+            log.debug("Stopping AMSI_ETW...")
+            self.capture.stop()
+
+            """Upload the file that contains the metadata for all of the events."""
+            if not self.log_file or not os.path.exists(self.log_file.name):
+                log.debug("No logfile to upload")
+                return
+            self.log_file.close()
+
+            try:
+                if os.stat(self.log_file.name).st_size > 0:
+                    upload_to_host(self.log_file.name, f"{self.upload_prefix}/amsi.jsonl")
+                else:
+                    log.debug("No AMSI events were collected.")
+            except Exception:
+                log.exception("Exception was raised while uploading amsi.jsonl")
+                raise
+            finally:
+                os.unlink(self.log_file.name)
+                self.log_file = None

analyzer/windows/modules/auxiliary/dns_etw.py

@@ -182,7 +182,13 @@ def __init__(self, options, config):
             self.config = Config(cfg="analysis.conf")
             self.enabled = self.config.dns_etw
             self.do_run = self.enabled
-            self.output_dir = "C:\\\\etw_dns"
+
+            self.output_dir = "C:\\etw_dns\\"
+            try:
+                os.mkdir(self.output_dir)
+            except FileExistsError:
+                pass
+
             self.log_file = os.path.join(self.output_dir, "dns_provider.log")
             if HAVE_ETW:
                 self.capture = ETW_provider(logfile=self.log_file, level=255, no_conout=True)
@@ -204,7 +210,7 @@ def start(self):
         def stop(self):
             if not HAVE_ETW:
                 return
-            log.debug("Stopping!!!")
+            log.debug("Stopping DNS_ETW...")
             self.capture.stop()
             files_to_upload = set()
 

conf/default/auxiliary.conf.default

@@ -17,7 +17,6 @@
 
 # Modules to be enabled or not inside of the VM
 [auxiliary_modules]
-amsi = no
 browser = yes
 curtain = no
 digisig = yes
@@ -46,8 +45,10 @@ tracee_linux = no
 sslkeylogfile = no
 # Requires setting up browser extension, check extra/browser_extension
 browsermonitor = no
+# ETW logging modules require pywintrace to be installed on the guest VM
 wmi_etw = no
 dns_etw = no
+amsi_etw = no
 watchdownloads = no
 
 [AzSniffer]

lib/cuckoo/core/resultserver.py

@@ -61,7 +61,7 @@
 RESULT_UPLOADABLE = (
     b"CAPE",
     b"aux",
-    b"aux/amsi",
+    b"aux/amsi_etw",
     b"browser",
     b"curtain",
     b"debugger",

modules/processing/amsi.py → modules/processing/amsi_etw.py

@@ -8,11 +8,11 @@
 log = logging.getLogger(__name__)
 
 
-class Amsi(Processing):
-    key = "amsi"
+class AMSI_ETW(Processing):
+    key = "amsi_etw"
 
     def run(self):
-        jsonl_file = os.path.join(self.aux_path, "amsi", "amsi.jsonl")
+        jsonl_file = os.path.join(self.aux_path, "amsi_etw", "amsi.jsonl")
         if not os.path.exists(jsonl_file) or os.stat(jsonl_file).st_size == 0:
             return None
 
@@ -46,10 +46,12 @@ def decode_event(cls, event):
             "kernel_time": header["KernelTime"],
             "user_time": header["UserTime"],
             "activity_id": header["ActivityId"],
-            "scan_result": cls.scan_result_to_str(event["scanResult"]),
+            "scan_result": cls.scan_result_to_str(int(event["scanResult"])),
             "app_name": event["appname"],
             "content_name": event["contentname"],
             "content_filtered": event["contentFiltered"],
+            "content_size": int(event["contentsize"]),
+            "dump_path": event.get("dump_path", ""),
             "hash": event["hash"][2:].lower(),
         }
 

utils/go-fetcher/go.mod

@@ -13,7 +13,8 @@ require (
 	github.com/go-sql-driver/mysql v1.7.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
-	github.com/jackc/pgx/v5 v5.4.3 // indirect
+	github.com/jackc/pgx/v5 v5.5.4 // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/mattn/go-sqlite3 v1.14.17 // indirect

utils/go-fetcher/go.sum

@@ -7,8 +7,10 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
 github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY=
-github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
+github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8=
+github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=