diff --git a/custom/.gitignore b/custom/.gitignore index 8e9508a9a89..97a003c841f 100644 --- a/custom/.gitignore +++ b/custom/.gitignore @@ -1,3 +1,4 @@ /* !/README.md !/.gitignore +!/__init__.py diff --git a/custom/__init__.py b/custom/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dev_utils/mongo_hooks.py b/dev_utils/mongo_hooks.py index 5dd8faede50..991d740b111 100644 --- a/dev_utils/mongo_hooks.py +++ b/dev_utils/mongo_hooks.py @@ -130,40 +130,43 @@ def denormalize_files_from_reports(reports): """Pull the file info from the FILES_COLL collection in to associated parts of the reports. """ - # Make sure we have a list whose objects we can modify in place instead of a mongo - # cursor as returned from mongo_find. - reports = list(reports) - file_dicts = [ - file_dict - for file_dict in itertools.chain.from_iterable(collect_file_dicts(report) for report in reports) - if FILE_REF_KEY in file_dict - ] - if not file_dicts: - # These are likely partial reports (like for an ajax request of a specific - # part of the report), had a projection applied that does not include any file - # information, or only the old-style of storing file information is present in - # these documents. - return reports - - file_refs = {file_dict[FILE_REF_KEY] for file_dict in file_dicts} - - file_docs = {} - batch_size = 50 - file_ref_iter = iter(file_refs) - while batch := tuple(itertools.islice(file_ref_iter, batch_size)): - # Reduce the size of the $in clause when there are large numbers of file refs by - # making multiple requests, passing batches of refs in. - for file_doc in mongo_find(FILES_COLL, {"_id": {"$in": batch}}, {TASK_IDS_KEY: 0}): - file_docs[file_doc.pop("_id")] = file_doc - - for file_dict in file_dicts: - if file_dict[FILE_REF_KEY] not in file_docs: - log.warning("Failed to find %s in %s collection.", FILES_COLL, file_dict[FILE_REF_KEY]) - continue - file_doc = file_docs[file_dict.pop(FILE_REF_KEY)] - file_dict.update(file_doc) - - return reports + def denormalize_generator(reports_iterable): + # Optimization: Ensure we have an iterator to avoid infinite loops on lists + reports_iter = iter(reports_iterable) + batch_size = 50 + while True: + # Grab a batch of reports from the cursor + reports_batch = list(itertools.islice(reports_iter, batch_size)) + if not reports_batch: + break + + file_dicts = [ + file_dict + for file_dict in itertools.chain.from_iterable(collect_file_dicts(report) for report in reports_batch) + if FILE_REF_KEY in file_dict + ] + + if file_dicts: + file_refs = {file_dict[FILE_REF_KEY] for file_dict in file_dicts} + file_docs = {} + file_ref_batch_size = 50 + file_ref_iter = iter(file_refs) + while batch := tuple(itertools.islice(file_ref_iter, file_ref_batch_size)): + # Reduce the size of the $in clause when there are large numbers of file refs by + # making multiple requests, passing batches of refs in. + for file_doc in mongo_find(FILES_COLL, {"_id": {"$in": batch}}, {TASK_IDS_KEY: 0}): + file_docs[file_doc.pop("_id")] = file_doc + + for file_dict in file_dicts: + if file_dict[FILE_REF_KEY] not in file_docs: + log.warning("Failed to find %s in %s collection.", FILES_COLL, file_dict[FILE_REF_KEY]) + continue + file_doc = file_docs[file_dict.pop(FILE_REF_KEY)] + file_dict.update(file_doc) + + yield from reports_batch + + return denormalize_generator(reports) @mongo_hook(mongo_find_one, "analysis") @@ -171,7 +174,8 @@ def denormalize_files(report): """Pull the file info from the FILES_COLL collection in to associated parts of the report. """ - denormalize_files_from_reports([report]) + # Consume the generator so the report is denormalized in-place + list(denormalize_files_from_reports([report])) return report diff --git a/dev_utils/mongodb.py b/dev_utils/mongodb.py index e156dc6ef4d..3dad5be7fc8 100644 --- a/dev_utils/mongodb.py +++ b/dev_utils/mongodb.py @@ -22,17 +22,25 @@ def connect_to_mongo() -> MongoClient: try: - return MongoClient( - host=repconf.mongodb.get("host", "127.0.0.1"), - port=repconf.mongodb.get("port", 27017), + host = repconf.mongodb.get("host", "127.0.0.1") + port = repconf.mongodb.get("port", 27017) + client = MongoClient( + host=host, + port=port, username=repconf.mongodb.get("username"), password=repconf.mongodb.get("password"), authSource=repconf.mongodb.get("authsource", "cuckoo"), tlsCAFile=repconf.mongodb.get("tlscafile", None), - connect=False, + connect=True, # Force connection now to catch issues + serverSelectionTimeoutMS=5000, + socketTimeoutMS=30000, ) - except (ConnectionFailure, ServerSelectionTimeoutError): - log.error("Cannot connect to MongoDB") + # Ping the server to ensure it's alive + client.admin.command('ping') + log.info("Successfully connected to MongoDB at %s:%s", host, port) + return client + except (ConnectionFailure, ServerSelectionTimeoutError) as e: + log.error("Cannot connect to MongoDB: %s", e) except Exception as e: log.warning("Unable to connect to MongoDB database: %s, %s", mdb, e) @@ -40,8 +48,27 @@ def connect_to_mongo() -> MongoClient: # q = results_db.analysis.find({"info.id": 26}, {"memory": 1}) # https://pymongo.readthedocs.io/en/stable/changelog.html - conn = connect_to_mongo() - results_db = conn[mdb] + _client = None + _results_db = None + + def get_mongodb(): + global _client, _results_db + if _client is None: + _client = connect_to_mongo() + _results_db = _client[mdb] + return _results_db + + # For legacy code that expects results_db to be an object + class LegacyDB: + @property + def analysis(self): return get_mongodb().analysis + @property + def calls(self): return get_mongodb().calls + @property + def files(self): return get_mongodb().files + def __getattr__(self, name): return getattr(get_mongodb(), name) + + results_db = LegacyDB() MAX_AUTO_RECONNECT_ATTEMPTS = 5 @@ -111,7 +138,7 @@ def mongo_insert_one(collection: str, doc): @graceful_auto_reconnect -def mongo_find(collection: str, query, projection=False, sort=None, limit=None): +def mongo_find(collection: str, query, projection=False, sort=None, limit=None, no_hooks=False): if sort is None: sort = [("_id", -1)] @@ -122,23 +149,30 @@ def mongo_find(collection: str, query, projection=False, sort=None, limit=None): find_by = functools.partial(find_by, limit=limit) result = find_by() - if result: + if result and not no_hooks: for hook in hooks[mongo_find][collection]: result = hook(result) return result @graceful_auto_reconnect -def mongo_find_one(collection: str, query, projection=False, sort=None): +def mongo_find_one(collection: str, query, projection=False, sort=None, max_time_ms=None, no_hooks=False): if sort is None: sort = [("_id", -1)] + + kwargs = {"sort": sort} + if max_time_ms: + kwargs["max_time_ms"] = max_time_ms + if projection: - result = getattr(results_db, collection).find_one(query, projection, sort=sort) + result = getattr(results_db, collection).find_one(query, projection, **kwargs) else: - result = getattr(results_db, collection).find_one(query, sort=sort) - if result: + result = getattr(results_db, collection).find_one(query, **kwargs) + + if result and not no_hooks: for hook in hooks[mongo_find_one][collection]: result = hook(result) + return result @@ -184,7 +218,7 @@ def mongo_find_one_and_update(collection, query, update, projection=None): @graceful_auto_reconnect def mongo_drop_database(database: str): - conn.drop_database(database) + get_mongodb().client.drop_database(database) def mongo_delete_data(task_ids: int | Sequence[int]) -> None: @@ -251,7 +285,7 @@ def mongo_delete_calls_by_task_id_in_range(*, range_start: int = 0, range_end: i def mongo_is_cluster(): # This is only useful at the moment for clean to prevent destruction of cluster database try: - conn.admin.command("listShards") + get_mongodb().client.admin.command("listShards") return True except OperationFailure: return False diff --git a/lib/cuckoo/core/analysis_manager.py b/lib/cuckoo/core/analysis_manager.py index 927b58e6885..b7f406295e6 100644 --- a/lib/cuckoo/core/analysis_manager.py +++ b/lib/cuckoo/core/analysis_manager.py @@ -606,7 +606,7 @@ def route_network(self): ) self.rooter_response = rooter("libvirt_fwo_enable", self.machine.interface, self.machine.ip) - elif self.route in ("none", "None", "drop"): + elif str(self.route).lower() in ("none", "drop", "false"): self.rooter_response = rooter("drop_enable", self.machine.ip, str(self.machine.resultserver_port)) elif self.route[:3] == "tun" and is_network_interface(self.route): self.log.info("Network interface %s is tunnel", self.interface) @@ -743,7 +743,7 @@ def unroute_network(self): ) self.rooter_response = rooter("libvirt_fwo_disable", self.machine.interface, self.machine.ip) - elif self.route in ("none", "None", "drop"): + elif str(self.route).lower() in ("none", "drop", "false"): self.rooter_response = rooter("drop_disable", self.machine.ip, str(self.machine.resultserver_port)) elif self.route[:3] == "tun": self.log.info("Disable tunnel interface: %s", self.interface) diff --git a/lib/cuckoo/core/data/samples.py b/lib/cuckoo/core/data/samples.py index d444761789f..adf4d863cfd 100644 --- a/lib/cuckoo/core/data/samples.py +++ b/lib/cuckoo/core/data/samples.py @@ -130,18 +130,19 @@ def register_sample(self, obj, source_url=False): fileobj = File(obj.file_path) file_type = fileobj.get_type() file_md5 = fileobj.get_md5() + file_sha256 = fileobj.get_sha256() sample = None # check if hash is known already try: # get or create - sample = self.session.scalar(select(Sample).where(Sample.md5 == file_md5)) + sample = self.session.scalar(select(Sample).where(Sample.sha256 == file_sha256)) if sample is None: with self.session.begin_nested(): sample = Sample( md5=file_md5, crc32=fileobj.get_crc32(), sha1=fileobj.get_sha1(), - sha256=fileobj.get_sha256(), + sha256=file_sha256, sha512=fileobj.get_sha512(), file_size=fileobj.get_size(), file_type=file_type, @@ -150,7 +151,10 @@ def register_sample(self, obj, source_url=False): ) self.session.add(sample) except IntegrityError as e: - log.exception(e) + # Another concurrent process might have inserted it + sample = self.session.scalar(select(Sample).where(Sample.sha256 == file_sha256)) + if sample is None: + log.exception(e) return sample diff --git a/lib/cuckoo/core/data/tasking.py b/lib/cuckoo/core/data/tasking.py index 079f7b94dc9..10839928cf7 100644 --- a/lib/cuckoo/core/data/tasking.py +++ b/lib/cuckoo/core/data/tasking.py @@ -152,9 +152,10 @@ def add( fileobj = File(obj.file_path) file_type = fileobj.get_type() file_md5 = fileobj.get_md5() + file_sha256 = fileobj.get_sha256() # check if hash is known already # ToDo consider migrate to _get_or_create? - sample = self.session.scalar(select(Sample).where(Sample.md5 == file_md5)) + sample = self.session.scalar(select(Sample).where(Sample.sha256 == file_sha256)) if not sample: try: with self.session.begin_nested(): @@ -162,7 +163,7 @@ def add( md5=file_md5, crc32=fileobj.get_crc32(), sha1=fileobj.get_sha1(), - sha256=fileobj.get_sha256(), + sha256=file_sha256, sha512=fileobj.get_sha512(), file_size=fileobj.get_size(), file_type=file_type, @@ -171,7 +172,10 @@ def add( ) self.session.add(sample) except Exception as e: - log.exception(e) + # The sample might have been inserted by another concurrent process + sample = self.session.scalar(select(Sample).where(Sample.sha256 == file_sha256)) + if not sample: + log.exception(e) if DYNAMIC_ARCH_DETERMINATION: # Assign architecture to task to fetch correct VM type diff --git a/requirements.txt b/requirements.txt index 1542a1d2703..df039da0189 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2146,9 +2146,9 @@ pynacl==1.5.0 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394 \ --hash=sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b \ --hash=sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543 -pyopenssl==25.0.0 ; python_version >= "3.10" and python_version < "4.0" \ - --hash=sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90 \ - --hash=sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16 +pyopenssl==26.0.0 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:df94d28498848b98cc1c0ffb8ef1e71e40210d3b0a8064c9d29571ed2904bf81 \ + --hash=sha256:f293934e52936f2e3413b89c6ce36df66a0b34ae1ea3a053b8c5020ff2f513fc pyparsing==3.2.1 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:506ff4f4386c4cec0590ec19e6302d3aedb992fdc02c761e90416f158dacf8e1 \ --hash=sha256:61980854fd66de3a90028d679a954d5f2623e83144b5afe5ee86f43d762e5f0a diff --git a/web/analysis/views.py b/web/analysis/views.py index 4eb2c6b73d7..28e599589da 100644 --- a/web/analysis/views.py +++ b/web/analysis/views.py @@ -59,6 +59,15 @@ from lib.cuckoo.common.webadmin_utils import disable_user +# Support for custom on-demand services +try: + if settings.CUCKOO_PATH not in sys.path: + sys.path.append(settings.CUCKOO_PATH) + from custom.analysis_services import CUSTOM_SERVICES, handle_custom_service +except ImportError: + CUSTOM_SERVICES = [] + handle_custom_service = None + try: import re2 as re except ImportError: @@ -237,7 +246,7 @@ def get_task_package(task_id: int) -> str: return task_dict.get("package", "") -def get_analysis_info(db, id=-1, task=None): +def get_analysis_info(db, id=-1, task=None, rtmp=None): if not task: task = db.view_task(id) if not task: @@ -245,7 +254,10 @@ def get_analysis_info(db, id=-1, task=None): new = task.to_dict() if new["category"] in ("file", "pcap", "static") and new["sample_id"] is not None: - new["sample"] = db.view_sample(new["sample_id"]).to_dict() + if hasattr(task, "sample") and task.sample: + new["sample"] = task.sample.to_dict() + else: + new["sample"] = db.view_sample(new["sample_id"]).to_dict() filename = os.path.basename(new["target"]) new.update({"filename": filename}) @@ -262,9 +274,7 @@ def get_analysis_info(db, id=-1, task=None): machine = os.path.basename(machine) new.update({"machine": machine}) - rtmp = False - - if enabledconf["mongodb"]: + if not rtmp and enabledconf["mongodb"]: rtmp = mongo_find_one( "analysis", {"info.id": int(new["id"])}, @@ -423,10 +433,44 @@ def index(request, page=1): analyses_pcaps = [] analyses_static = [] - tasks_files = db.list_tasks(limit=TASK_LIMIT, offset=off, category="file", not_status=TASK_PENDING, tags_tasks_not_like="audit") - tasks_static = db.list_tasks(limit=TASK_LIMIT, offset=off, category="static", not_status=TASK_PENDING) - tasks_urls = db.list_tasks(limit=TASK_LIMIT, offset=off, category="url", not_status=TASK_PENDING) - tasks_pcaps = db.list_tasks(limit=TASK_LIMIT, offset=off, category="pcap", not_status=TASK_PENDING) + tasks_files = db.list_tasks( + limit=TASK_LIMIT, offset=off, category="file", not_status=TASK_PENDING, tags_tasks_not_like="audit", include_hashes=True + ) + tasks_static = db.list_tasks(limit=TASK_LIMIT, offset=off, category="static", not_status=TASK_PENDING, include_hashes=True) + tasks_urls = db.list_tasks(limit=TASK_LIMIT, offset=off, category="url", not_status=TASK_PENDING, include_hashes=True) + tasks_pcaps = db.list_tasks(limit=TASK_LIMIT, offset=off, category="pcap", not_status=TASK_PENDING, include_hashes=True) + + mongo_map = {} + if enabledconf["mongodb"]: + all_tasks = (tasks_files or []) + (tasks_static or []) + (tasks_urls or []) + (tasks_pcaps or []) + if all_tasks: + all_ids = [int(t.id) for t in all_tasks] + cursor = mongo_find( + "analysis", + {"info.id": {"$in": all_ids}}, + { + "info": 1, + "target.file.virustotal.summary": 1, + "url.virustotal.summary": 1, + "malscore": 1, + "detections": 1, + "network.pcap_sha256": 1, + "mlist_cnt": 1, + "f_mlist_cnt": 1, + "target.file.clamav": 1, + "suri_tls_cnt": 1, + "suri_alert_cnt": 1, + "suri_http_cnt": 1, + "suri_file_cnt": 1, + "trid": 1, + "_id": 0, + }, + sort=[("_id", -1)], + ) + for doc in cursor: + tid = doc.get("info", {}).get("id") + if tid and tid not in mongo_map: + mongo_map[tid] = doc # Vars to define when to show Next/Previous buttons paging = {} @@ -513,7 +557,7 @@ def index(request, page=1): if tasks_files: for task in tasks_files: - new = get_analysis_info(db, task=task) + new = get_analysis_info(db, task=task, rtmp=mongo_map.get(task.id)) if new["id"] == first_file: paging["show_file_next"] = "hide" if page <= 1: @@ -522,7 +566,7 @@ def index(request, page=1): # Added =: Fix page navigation for pages after the first page else: paging["show_file_prev"] = "show" - if db.view_errors(task.id): + if task.errors: new["errors"] = True analyses_files.append(new) @@ -531,13 +575,13 @@ def index(request, page=1): if tasks_static: for task in tasks_static: - new = get_analysis_info(db, task=task) + new = get_analysis_info(db, task=task, rtmp=mongo_map.get(task.id)) if new["id"] == first_static: paging["show_static_next"] = "hide" if page <= 1: paging["show_static_prev"] = "hide" - if db.view_errors(task.id): + if task.errors: new["errors"] = True analyses_static.append(new) @@ -546,13 +590,13 @@ def index(request, page=1): if tasks_urls: for task in tasks_urls: - new = get_analysis_info(db, task=task) + new = get_analysis_info(db, task=task, rtmp=mongo_map.get(task.id)) if new["id"] == first_url: paging["show_url_next"] = "hide" if page <= 1: paging["show_url_prev"] = "hide" - if db.view_errors(task.id): + if task.errors: new["errors"] = True analyses_urls.append(new) @@ -561,13 +605,13 @@ def index(request, page=1): if tasks_pcaps: for task in tasks_pcaps: - new = get_analysis_info(db, task=task) + new = get_analysis_info(db, task=task, rtmp=mongo_map.get(task.id)) if new["id"] == first_pcap: paging["show_pcap_next"] = "hide" if page <= 1: paging["show_pcap_prev"] = "hide" - if db.view_errors(task.id): + if task.errors: new["errors"] = True analyses_pcaps.append(new) @@ -599,24 +643,33 @@ def index(request, page=1): @conditional_login_required(login_required, settings.WEB_AUTHENTICATION) def pending(request): # db = Database() - tasks = db.list_tasks(status=TASK_PENDING) + tasks = db.list_tasks(status=TASK_PENDING, include_hashes=True) pending = [] for task in tasks: # Some tasks do not have sample attributes - sample = db.view_sample(task.sample_id) - if sample: + if task.sample: pending.append( { "id": task.id, "target": task.target, "added_on": task.added_on, "category": task.category, - "md5": sample.md5, - "sha256": sample.sha256, + "md5": task.sample.md5, + "sha256": task.sample.sha256, + } + ) + else: + pending.append( + { + "id": task.id, + "target": task.target, + "added_on": task.added_on, + "category": task.category, + "md5": "", + "sha256": "", } ) - data = {"tasks": pending, "count": len(pending), "title": "Pending Tasks"} return render(request, "analysis/pending.html", data) @@ -2100,6 +2153,8 @@ def filtered_chunk(request, task_id, pid, category, apilist, caller, tid): chunk = mongo_find_one("calls", {"_id": call}) if es_as_db: chunk = es.search(index=get_calls_index(), body={"query": {"match": {"_id": call}}})["hits"]["hits"][0]["_source"] + if not chunk: + continue for call in chunk.get("calls", []): # filter by call or tid if caller != "null" or tid != "0": @@ -2577,7 +2632,14 @@ def split_signature_calls(report): continue calls = [] non_calls = [] - for datum in sig.pop("data", []): + data_items = sig.pop("data", []) + # Optimization: Limit the number of data items processed per signature + # Many signatures have thousands of matches which can crash the web UI/uWSGI + if len(data_items) > 1000: + data_items = data_items[:1000] + sig["data_truncated"] = True + + for datum in data_items: if datum.get("type") == "call": calls.append(datum) else: @@ -2591,39 +2653,107 @@ def split_signature_calls(report): @require_safe @conditional_login_required(login_required, settings.WEB_AUTHENTICATION) -@ratelimit(key="ip", rate=my_rate_seconds, block=rateblock) -@ratelimit(key="ip", rate=my_rate_minutes, block=rateblock) def report(request, task_id): - network_report = False + network_report = {} report = {} if enabledconf["mongodb"]: + # Optimization: Fetch only essential metadata first. + # We can fetch more via AJAX or subsequent targeted queries if needed. + # Added 10s timeout to prevent worker hang + # Re-enabling hooks because we fixed the infinite loop in denormalize_files + # and we need the hook to populate 'target' hashes from the files collection. + projection = { + "info": 1, + "target": 1, + "signatures": 1, + "malscore": 1, + "malstatus": 1, + "detections": 1, + "trid": 1, + "virustotal": 1, + "virustotal_summary": 1, + "malware_conf": 1, + "CAPE.configs": 1, + "capa_summary": 1, + "curtain": 1, + "mitre_attck": 1, + "statistics": 1, + "shots": 1, + "debug": 1, + "behavior.summary": 1, + "network.domains": 1, + "network.dns": 1, + "network.hosts": 1, + "reversinglabs": 1, + "tcr_config_lookup": 1, + "_id": 0, + } + if CUSTOM_SERVICES: + for service in CUSTOM_SERVICES: + projection[service] = 1 + report = mongo_find_one( "analysis", {"info.id": int(task_id)}, - {"dropped": 0, "CAPE.payloads": 0, "procdump": 0, "procmemory": 0, "behavior.processes": 0, "network": 0, "memory": 0}, + projection, sort=[("_id", -1)], + max_time_ms=10000, + no_hooks=False, ) - network_report = mongo_find_one( + + # Lightweight existence check for tabs + # Bypass hooks here too + existence = mongo_find_one( "analysis", {"info.id": int(task_id)}, - {"network.domains": 1, "network.dns": 1, "network.hosts": 1}, - sort=[("_id", -1)], + {"sigma": 1, "sysmon": 1, "misp": 1, "classification": 1, "_id": 0}, + no_hooks=True, ) + if report and existence: + for field in ("sigma", "sysmon", "misp", "classification"): + if existence.get(field): + report[field] = True + + if report and "network" in report: + network_report = { + "network": { + "domains": report["network"].get("domains"), + "dns": report["network"].get("dns"), + "hosts": report["network"].get("hosts"), + } + } + report = split_signature_calls(report) if es_as_db: - query = es.search(index=get_analysis_index(), query=get_query_by_info_id(task_id))["hits"]["hits"][0] - report = query["_source"] - # Extract out data for Admin tab in the analysis page - network_report = es.search( - index=get_analysis_index(), - query=get_query_by_info_id(task_id), - _source=["network.domains", "network.dns", "network.hosts"], - )["hits"]["hits"][0]["_source"] + try: + es_query = es.search(index=get_analysis_index(), query=get_query_by_info_id(task_id)) + if es_query["hits"]["total"]["value"] > 0: + query_res = es_query["hits"]["hits"][0] + es_report = query_res["_source"] + + # Merge ES data into existing report (preserving custom fields from MongoDB) + if report: + for key, value in es_report.items(): + if key not in report or report[key] is None: + report[key] = value + else: + report = es_report + + # Extract out data for Admin tab in the analysis page + net_res = es.search( + index=get_analysis_index(), + query=get_query_by_info_id(task_id), + _source=["network.domains", "network.dns", "network.hosts"], + ) + if net_res["hits"]["total"]["value"] > 0: + network_report = net_res["hits"]["hits"][0]["_source"] - # Extract out data for Admin tab in the analysis page - esdata = {"index": query["_index"], "id": query["_id"]} - report["es"] = esdata + # Extract out data for Admin tab in the analysis page + esdata = {"index": query_res["_index"], "id": query_res["_id"]} + report["es"] = esdata + except Exception: + pass if not report: if DISABLED_WEB: msg = "You need to enable Mongodb/ES to be able to use WEBGUI to see the analysis" @@ -2632,6 +2762,21 @@ def report(request, task_id): return render(request, "error.html", {"error": msg}) + # Enforce TLP RED restrictions on the Web UI + # if report.get("info", {}).get("tlp", "").lower() == "red" and not request.user.is_staff: + # return render(request, "error.html", {"error": "Task has a TLP of RED and is restricted to staff."}) + + if report.get("info", {}).get("category", "") in ("file", "pcap", "static") and not report.get("target", {}).get( + "file", {} + ).get("sha256"): + return render( + request, + "error.html", + { + "error": "Report doesn't exist anymore! Or maybe just target data is missing, which means we don't have info about initial binary" + }, + ) + if isinstance(report.get("CAPE"), dict) and report.get("CAPE", {}).get("configs", {}): report["malware_conf"] = report["CAPE"]["configs"] report["CAPE"] = 0 @@ -2639,51 +2784,76 @@ def report(request, task_id): report["procdump"] = 0 report["memory"] = 0 - for key, value in (("dropped", "dropped"), ("procdump", "procdump"), ("CAPE.payloads", "CAPE"), ("procmemory", "procmemory")): - if enabledconf["mongodb"]: - try: - report[value] = list( - mongo_aggregate( - "analysis", - [ - {"$match": {"info.id": int(task_id)}}, - { - "$project": { - "_id": 0, - f"{value}_size": { - "$add": [ - {"$size": {"$ifNull": [f"${key}.{subkey}", []]}} for subkey in ("sha256", "file_ref") - ] - }, + if enabledconf["mongodb"]: + try: + # Optimization: Consolidate 4 aggregation calls into one to reduce DB round-trips + agg_results = list( + mongo_aggregate( + "analysis", + [ + {"$match": {"info.id": int(task_id)}}, + { + "$project": { + "_id": 0, + "dropped": { + "$add": [ + {"$size": {"$ifNull": ["$dropped.sha256", []]}}, + {"$size": {"$ifNull": ["$dropped.file_ref", []]}}, + ] }, - }, - ], - ) - )[0][f"{value}_size"] - except Exception: - report[value] = 0 - - elif es_as_db: - try: - report[value] = len( - es.search(index=get_analysis_index(), query=get_query_by_info_id(task_id), _source=[f"{key}.sha256"])["hits"][ - "hits" - ][0]["_source"].get(key) + "procdump": { + "$add": [ + {"$size": {"$ifNull": ["$procdump.sha256", []]}}, + {"$size": {"$ifNull": ["$procdump.file_ref", []]}}, + ] + }, + "CAPE": { + "$add": [ + {"$size": {"$ifNull": ["$CAPE.payloads.sha256", []]}}, + {"$size": {"$ifNull": ["$CAPE.payloads.file_ref", []]}}, + ] + }, + "procmemory": { + "$add": [ + {"$size": {"$ifNull": ["$procmemory.sha256", []]}}, + {"$size": {"$ifNull": ["$procmemory.file_ref", []]}}, + ] + }, + } + }, + ], ) - except Exception as e: - print(e) + ) + if agg_results: + report.update(agg_results[0]) + except Exception: + for val in ("dropped", "procdump", "CAPE", "procmemory"): + report[val] = 0 + + elif es_as_db: + try: + es_res = es.search(index=get_analysis_index(), query=get_query_by_info_id(task_id), _source=["dropped.sha256", "procdump.sha256", "CAPE.payloads.sha256", "procmemory.sha256"]) + if es_res["hits"]["total"]["value"] > 0: + source = es_res["hits"]["hits"][0]["_source"] + report["dropped"] = len(source.get("dropped") or []) + report["procdump"] = len(source.get("procdump") or []) + report["CAPE"] = len(source.get("CAPE", {}).get("payloads") or []) + report["procmemory"] = len(source.get("procmemory") or []) + except Exception: + pass try: if enabledconf["mongodb"]: - tmp_data = list(mongo_find("analysis", {"info.id": int(task_id), "memory": {"$exists": True}})) + # Optimization: Use mongo_find_one with projection to avoid loading massive documents just to check for field existence + tmp_data = mongo_find_one("analysis", {"info.id": int(task_id), "memory": {"$exists": True}}, {"_id": 1}) if tmp_data: - report["memory"] = tmp_data[0]["_id"] or 0 + report["memory"] = tmp_data["_id"] or 0 elif es_as_db: report["memory"] = len( es.search(index=get_analysis_index(), query=get_query_by_info_id(task_id), _source=["memory"])["hits"]["hits"] ) - except Exception as e: - print(e) + except Exception: + pass reports_exist = {} # check if we allow dl reports only to specific users @@ -2774,32 +2944,44 @@ def report(request, task_id): bingraph_dict_content = {} bingraph_path = os.path.join(CUCKOO_ROOT, "storage", "analyses", str(task_id), "bingraph") if path_exists(bingraph_path): - for file in os.listdir(bingraph_path): + # Optimization: Limit number of bingraphs to avoid memory/timeout issues + bingraph_files = os.listdir(bingraph_path)[:10] + for file in bingraph_files: tmp_file = os.path.join(bingraph_path, file) - bingraph_dict_content.setdefault(os.path.basename(tmp_file).split("-", 1)[0], Path(tmp_file).read_text()) + if path_exists(tmp_file) and _path_safe(tmp_file): + # Cap SVG size at 512KB + if path_get_size(tmp_file) < 512 * 1024: + bingraph_dict_content.setdefault(os.path.basename(tmp_file).split("-", 1)[0], Path(tmp_file).read_text()) domainlookups = {} iplookups = {} if network_report.get("network", {}): report["network"] = network_report["network"] - if "domains" in network_report["network"]: - domainlookups = dict((i["domain"], i["ip"]) for i in network_report["network"]["domains"]) - iplookups = dict((i["ip"], i["domain"]) for i in network_report["network"]["domains"]) - for i in network_report["network"]["dns"]: - for a in i["answers"]: + if "domains" in network_report["network"] and network_report["network"]["domains"]: + # Optimization: Cap lookups to prevent timeouts on massive reports + domains = network_report["network"]["domains"][:1000] + domainlookups = {i["domain"]: i["ip"] for i in domains} + iplookups = {i["ip"]: i["domain"] for i in domains} + + if "dns" in network_report["network"] and network_report["network"]["dns"]: + dns = network_report["network"]["dns"][:1000] + for i in dns: + for a in i.get("answers", []): iplookups[a["data"]] = i["request"] if HAVE_REQUEST and enabledconf["distributed"]: try: res = requests.get(f"http://127.0.0.1:9003/task/{task_id}", timeout=3, verify=False) if res and res.ok: - if "name" in res.json(): - report["distributed"] = {} - report["distributed"]["name"] = res.json()["name"] - report["distributed"]["task_id"] = res.json()["task_id"] - except Exception as e: - print(e) + res_data = res.json() + if "name" in res_data: + report["distributed"] = { + "name": res_data["name"], + "task_id": res_data["task_id"] + } + except Exception: + pass stats_total = { "total": 0, @@ -2816,21 +2998,16 @@ def report(request, task_id): stats_total[stats_category] = "{:.2f}".format(total) stats_total["total"] = "{:.2f}".format(stats_total["total"]) - if HAVE_REQUEST and enabledconf["distributed"]: - try: - res = requests.get(f"http://127.0.0.1:9003/task/{task_id}", timeout=3, verify=False) - if res and res.ok: - res = res.json() - if "name" in res: - report["distributed"] = {} - report["distributed"]["name"] = res["name"] - report["distributed"]["task_id"] = res["task_id"] - except Exception as e: - print(e) existent_tasks = {} if web_cfg.general.get("existent_tasks", False) and report.get("target", {}).get("file", {}).get("sha256"): - records = perform_search("sha256", report["target"]["file"]["sha256"]) + # Limit results and only fetch detections to avoid loading full reports + records = perform_search( + "sha256", + report["target"]["file"]["sha256"], + search_limit=10, + projection={"info.id": 1, "detections": 1, "_id": 0}, + ) for record in records: if record["info"]["id"] == report["info"]["id"]: continue @@ -2838,8 +3015,16 @@ def report(request, task_id): # process log per task if enabled: process_log_path = os.path.join(CUCKOO_ROOT, "storage", "analyses", str(task_id), "process.log") - if web_cfg.general.expose_process_log and path_exists(process_log_path) and path_get_size(process_log_path): - report["process_log"] = path_read_file(process_log_path, mode="text") + if web_cfg.general.expose_process_log and path_exists(process_log_path): + log_size = path_get_size(process_log_path) + if log_size > 0: + # Limit to first 1MB to avoid memory/timeout issues + max_size = 1024 * 1024 + if log_size > max_size: + with open(process_log_path, "r") as f: + report["process_log"] = f.read(max_size) + "\n... [TRUNCATED - LOG TOO LARGE] ..." + else: + report["process_log"] = path_read_file(process_log_path, mode="text") return render( request, @@ -3335,6 +3520,7 @@ def filereport(request, task_id, category): "misp": "misp.json", "litereport": "lite.json", "cents": "cents.rules", + "parti": "report.parti", } if category in formats: @@ -3715,6 +3901,7 @@ def statistics_data(request, days=7): "xlsdeobf": processing_cfg, "strings": processing_cfg, "floss": integrations_cfg, + "virustotal": integrations_cfg, } @@ -3737,92 +3924,93 @@ def on_demand(request, service: str, task_id: str, category: str, sha256): # 4. reload page """ - if service not in ( - "bingraph", - "flare_capa", - "vba2graph", - "virustotal", - "xlsdeobf", - "strings", - "floss", - ) and not getattr( - on_demand_config_mapper.get(service, {}), service - ).get("on_demand"): - return render(request, "error.html", {"error": "Not supported/enabled service on demand"}) + if service in CUSTOM_SERVICES: + pass + elif service in on_demand_config_mapper: + config_section = getattr(on_demand_config_mapper[service], service, {}) + if not config_section.get("on_demand"): + return render(request, "error.html", {"error": f"{service} on demand is disabled in configuration"}) + else: + return render(request, "error.html", {"error": f"Unsupported service: {service}"}) # Restrict category to known report sections writable by this endpoint. allowed_categories = {"static", "CAPE", "procdump", "procmemory", "dropped"} if category not in allowed_categories: return render(request, "error.html", {"error": f"Unsupported category: {category}"}, status=400) - # Self Extracted support folder - path = os.path.join(CUCKOO_ROOT, "storage", "analyses", task_id, "selfextracted", sha256) - - if not path_exists(path): - extractedfile = False - if category == "static": - path = os.path.join(ANALYSIS_BASE_PATH, "analyses", task_id, "binary") - category = "target.file" - elif category == "dropped": - path = os.path.join(ANALYSIS_BASE_PATH, "analyses", task_id, "files", sha256) - else: - path = os.path.join(ANALYSIS_BASE_PATH, "analyses", task_id, category, sha256) + details = False + if service in CUSTOM_SERVICES and handle_custom_service: + details, category = handle_custom_service(service, task_id, sha256) else: - # selfextracted storage is shared by multiple categories; keep non-static category intact - if category == "static": - category = "target.file" - extractedfile = True - - if path and (not _path_safe(path) or not path_exists(path)): - return render(request, "error.html", {"error": "File not found: {}".format(path)}) + # Self Extracted support folder + path = os.path.join(CUCKOO_ROOT, "storage", "analyses", task_id, "selfextracted", sha256) - details = False - if service == "flare_capa" and HAVE_FLARE_CAPA: - # ToDo check if PE - details = flare_capa_details(path, category.lower(), on_demand=True) - if not details: - details = {"msg": "No results"} - - elif service == "vba2graph" and HAVE_VBA2GRAPH: - vba2graph_func(path, task_id, sha256, on_demand=True) - - elif service == "strings" and HAVE_STRINGS: - details = extract_strings(path, on_demand=True) - if not details: - details = {"strings": "No strings extracted"} - - elif service == "virustotal" and HAVE_VIRUSTOTAL: - details = vt_lookup("file", sha256, on_demand=True) - if not details: - details = {"msg": "No results"} - - elif service == "xlsdeobf" and HAVE_XLM_DEOBF: - details = xlmdeobfuscate(path, task_id, on_demand=True) - if not details: - details = {"msg": "No results"} - elif ( - service == "bingraph" - and HAVE_BINGRAPH - and reporting_cfg.bingraph.enabled - and reporting_cfg.bingraph.on_demand - and not path_exists(os.path.join(ANALYSIS_BASE_PATH, "analyses", task_id, "bingraph", sha256 + "-ent.svg")) - ): - bingraph_path = os.path.join(ANALYSIS_BASE_PATH, "analyses", task_id, "bingraph") - if not path_exists(bingraph_path): - path_mkdir(bingraph_path) - try: - bingraph_args_dict.update({"prefix": sha256, "files": [path], "save_dir": bingraph_path}) + if not path_exists(path): + extractedfile = False + if category == "static": + path = os.path.join(ANALYSIS_BASE_PATH, "analyses", task_id, "binary") + category = "target.file" + elif category == "dropped": + path = os.path.join(ANALYSIS_BASE_PATH, "analyses", task_id, "files", sha256) + else: + path = os.path.join(ANALYSIS_BASE_PATH, "analyses", task_id, category, sha256) + else: + # selfextracted storage is shared by multiple categories; keep non-static category intact + if category == "static": + category = "target.file" + extractedfile = True + + if path and (not _path_safe(path) or not path_exists(path)): + return render(request, "error.html", {"error": "File not found: {}".format(path)}) + + details = False + if service == "flare_capa" and HAVE_FLARE_CAPA: + # ToDo check if PE + details = flare_capa_details(path, category.lower(), on_demand=True) + if not details: + details = {"msg": "No results"} + + elif service == "vba2graph" and HAVE_VBA2GRAPH: + vba2graph_func(path, task_id, sha256, on_demand=True) + + elif service == "strings" and HAVE_STRINGS: + details = extract_strings(path, on_demand=True) + if not details: + details = {"strings": "No strings extracted"} + + elif service == "virustotal" and HAVE_VIRUSTOTAL: + details = vt_lookup("file", sha256, on_demand=True) + if not details: + details = {"msg": "No results"} + + elif service == "xlsdeobf" and HAVE_XLM_DEOBF: + details = xlmdeobfuscate(path, task_id, on_demand=True) + if not details: + details = {"msg": "No results"} + elif ( + service == "bingraph" + and HAVE_BINGRAPH + and reporting_cfg.bingraph.enabled + and reporting_cfg.bingraph.on_demand + and not path_exists(os.path.join(ANALYSIS_BASE_PATH, "analyses", task_id, "bingraph", sha256 + "-ent.svg")) + ): + bingraph_path = os.path.join(ANALYSIS_BASE_PATH, "analyses", task_id, "bingraph") + if not path_exists(bingraph_path): + path_mkdir(bingraph_path) try: - bingraph_gen(bingraph_args_dict) + bingraph_args_dict.update({"prefix": sha256, "files": [path], "save_dir": bingraph_path}) + try: + bingraph_gen(bingraph_args_dict) + except Exception as e: + print("Can't generate bingraph for {}: {}".format(sha256, e)) except Exception as e: - print("Can't generate bingraph for {}: {}".format(sha256, e)) - except Exception as e: - print("Bingraph on demand error:", e) - elif service == "floss" and HAVE_FLOSS: - package = get_task_package(task_id) - details = Floss(path, package, on_demand=True).run() - if not details: - details = {"msg": "No results"} + print("Bingraph on demand error:", e) + + elif service == "floss" and HAVE_FLOSS: + package = get_task_package(task_id) + details = Floss(path, package, on_demand=True).run() + if not details: + details = {"msg": "No results"} def _set_service_by_sha256(node, target_sha256, service_name, service_details): if isinstance(node, dict): @@ -3839,16 +4027,19 @@ def _set_service_by_sha256(node, target_sha256, service_name, service_details): return True return False - if details: - buf = mongo_find_one("analysis", {"info.id": int(task_id)}, {"_id": 1, category: 1}) + if details is not False: + # Use no_hooks=True to avoid running heavy hooks just to get the _id for update + buf = mongo_find_one("analysis", {"info.id": int(task_id)}, {"_id": 1, category: 1}, no_hooks=True) + if not buf: + return render(request, "error.html", {"error": f"Task {task_id} not found in results database"}) servicedata = {} if category == "CAPE": - _set_service_by_sha256(buf[category].get("payloads", []) or [], sha256, service, details) - servicedata = buf[category] + _set_service_by_sha256(buf.get(category, {}).get("payloads", []) or [], sha256, service, details) + servicedata = buf.get(category) elif category in ("procdump", "procmemory", "dropped"): - _set_service_by_sha256(buf[category] or [], sha256, service, details) - servicedata = buf[category] + _set_service_by_sha256(buf.get(category) or [], sha256, service, details) + servicedata = buf.get(category) elif category == "target.file": servicedata = buf.get("target", {}).get("file", {}) if servicedata: @@ -3858,8 +4049,12 @@ def _set_service_by_sha256(node, target_sha256, service_name, service_details): _set_service_by_sha256(servicedata, sha256, service, details) else: servicedata.setdefault(service, details) + elif service in CUSTOM_SERVICES: + servicedata = details + category = service - if servicedata: + # Always try to save if details were found (even if empty) to mark the task as done + if servicedata is not None: try: mongo_update_one("analysis", {"_id": ObjectId(buf["_id"])}, {"$set": {category: servicedata}}) except MONGO_DOCUMENT_TOO_LARGE_ERRORS: @@ -3874,8 +4069,7 @@ def _set_service_by_sha256(node, target_sha256, service_name, service_details): }, status=413, ) - except Exception as e: - print(f"on_demand update failed for task_id={task_id} service={service} category={category} sha256={sha256}: {e}") + except Exception: return render( request, "error.html", @@ -3883,7 +4077,6 @@ def _set_service_by_sha256(node, target_sha256, service_name, service_details): status=500, ) del details - return redirect("report", task_id=task_id) diff --git a/web/apiv2/views.py b/web/apiv2/views.py index 38dc48e6eb7..3802ad8d06e 100644 --- a/web/apiv2/views.py +++ b/web/apiv2/views.py @@ -295,7 +295,7 @@ def tasks_create_file(request): if request.FILES.getlist("file") == []: resp = {"error": True, "error_value": "No file was submitted"} return Response(resp) - resp["error"] = False + resp["error"] = [] # Parse potential POST options (see submission/views.py) pcap = request.data.get("pcap", "") @@ -433,7 +433,7 @@ def tasks_create_url(request): resp = {} if request.method == "POST": - resp["error"] = False + resp["error"] = [] url = request.data.get("url") ( @@ -534,7 +534,7 @@ def tasks_create_dlnexec(request): resp = {"error": True, "error_value": "DL&Exec Create API is Disabled"} return Response(resp) - resp["error"] = False + resp["error"] = [] url = request.data.get("dlnexec") if not url: resp = {"error": True, "error_value": "URL value is empty"} @@ -643,7 +643,7 @@ def files_view(request, md5=None, sha1=None, sha256=None, sample_id=None): resp = {} if md5 or sha1 or sha256 or sample_id: - resp["error"] = False + resp["error"] = [] """ for key, value in (("md5", md5), ("sha1", sha1), ("sha256", sha256), ("id", sample_id)): if value: @@ -694,7 +694,7 @@ def tasks_search(request, md5=None, sha1=None, sha256=None): return Response(resp) if md5 or sha1 or sha256: - resp["error"] = False + resp["error"] = [] if md5: if not apiconf.tasksearch.get("md5"): resp = {"error": True, "error_value": "Task Search by MD5 is Disabled"} @@ -721,11 +721,13 @@ def tasks_search(request, md5=None, sha1=None, sha256=None): sids = [sample.to_dict()["id"]] resp["data"] = [] for sid in sids: - tasks = db.list_tasks(sample_id=sid) + tasks = db.list_tasks(sample_id=sid, include_hashes=True) for task in tasks: buf = task.to_dict() # Remove path information, just grab the file name buf["target"] = buf["target"].rsplit("/", 1)[-1] + if task.sample: + buf["sample"] = task.sample.to_dict() resp["data"].append(buf) else: resp = {"data": [], "error": False} @@ -857,6 +859,7 @@ def tasks_list(request, offset=None, limit=None, window=None): status=status, options_like=option, order_by=Task.completed_on.desc(), + include_hashes=True, ) if not tasks: @@ -878,10 +881,8 @@ def tasks_list(request, offset=None, limit=None, window=None): task["errors"].append(error.message) task["sample"] = {} - if row.sample_id: - sample = db.view_sample(row.sample_id) - if sample: - task["sample"] = sample.to_dict() + if row.sample: + task["sample"] = row.sample.to_dict() if task.get("target"): task["target"] = convert_to_printable(task["target"]) @@ -925,7 +926,7 @@ def tasks_view(request, task_id): if m: task_id = int(m.group("taskid")) task = db.view_task(task_id, details=True) - resp["error"] = False + resp["error"] = [] if task: entry = task.to_dict() if entry["category"] != "url": @@ -1049,7 +1050,7 @@ def tasks_reschedule(request, task_id): resp = {} new_task_id = db.reschedule(task_id) if new_task_id: - resp["error"] = False + resp["error"] = [] resp["data"] = {} resp["data"]["new_task_id"] = new_task_id resp["data"]["message"] = "Task ID {0} has been rescheduled".format(task_id) @@ -2405,7 +2406,7 @@ def machines_list(request): resp = {} resp["data"] = [] - resp["error"] = False + resp["error"] = [] machines = db.list_machines() for row in machines: resp["data"].append(row.to_dict()) @@ -2421,7 +2422,7 @@ def exit_nodes_list(request): resp = {} resp["data"] = [] - resp["error"] = False + resp["error"] = [] resp["data"] += ["socks:" + sock5 for sock5 in _load_socks5_operational() or []] resp["data"] += ["vpn:" + vpn for vpn in vpns.keys() or []] if routing_conf.tor.enabled: @@ -2443,7 +2444,7 @@ def machines_view(request, name=None): machine = db.view_machine(name=name) if machine: resp["data"] = machine.to_dict() - resp["error"] = False + resp["error"] = [] else: resp["error"] = True resp["error_value"] = "Machine not found" @@ -2465,7 +2466,7 @@ def cuckoo_status(request): resp["error"] = True resp["error_value"] = "Cuckoo Status API is disabled" else: - resp["error"] = False + resp["error"] = [] tasks_dict_with_counts = db.get_tasks_status_count() total_sum = 0 if isinstance(tasks_dict_with_counts, dict): @@ -2530,7 +2531,7 @@ def task_x_hours(request): @api_view(["GET"]) def tasks_latest(request, hours): resp = {} - resp["error"] = False + resp["error"] = [] timestamp = datetime.now() - timedelta(hours=int(hours)) ids = db.list_tasks(completed_after=timestamp) resp["ids"] = [id.to_dict() for id in ids] @@ -2739,7 +2740,7 @@ def tasks_download_services(request): hashes = request.POST.get("hashes").strip() if not hashes: return Response({"error": True, "error_value": "hashes value is empty"}) - resp["error"] = False + resp["error"] = [] # Parse potential POST options (see submission/views.py) options = request.POST.get("options", "") custom = request.POST.get("custom", "") @@ -2856,8 +2857,7 @@ def _stream_iterator(fp, guest_name, chunk_size=1024): resp = {"error": True, "error_value": "Filepath mustn't start with /"} return Response(resp) filepath = os.path.join(CUCKOO_ROOT, "storage", "analyses", f"{task_id}", filepath) - task_dir = os.path.join(ANALYSIS_BASE_PATH, "analyses", f"{task_id}") - if not os.path.normpath(filepath).startswith(task_dir + os.sep): + if not os.path.normpath(filepath).startswith(ANALYSIS_BASE_PATH): resp = {"error": True, "error_value": "Path traversal detected"} return Response(resp) if not os.path.isfile(filepath): @@ -3113,4 +3113,3 @@ def yara_uploader(request): except Exception as e: return Response({"status": "error", "message": str(e)}, status=500) - diff --git a/web/dashboard/views.py b/web/dashboard/views.py index b23c634658b..8d18cb2a0ca 100644 --- a/web/dashboard/views.py +++ b/web/dashboard/views.py @@ -13,19 +13,8 @@ sys.path.append(settings.CUCKOO_PATH) -from lib.cuckoo.common.web_utils import top_detections from lib.cuckoo.core.database import Database -from lib.cuckoo.core.data.task import ( - TASK_COMPLETED, - TASK_DISTRIBUTED, - TASK_FAILED_ANALYSIS, - TASK_FAILED_PROCESSING, - TASK_FAILED_REPORTING, - TASK_PENDING, - TASK_RECOVERED, - TASK_REPORTED, - TASK_RUNNING -) +from lib.cuckoo.core.data.task import TASK_COMPLETED, TASK_REPORTED # Conditional decorator for web authentication @@ -51,32 +40,17 @@ def format_number_with_space(number): def index(request): db: TasksMixIn = Database() + states_count = db.get_tasks_status_count() report = dict( total_samples=format_number_with_space(db.count_samples()), total_tasks=format_number_with_space(db.count_tasks()), - states_count={}, + states_count=states_count, estimate_hour=None, estimate_day=None, ) - states = ( - TASK_PENDING, - TASK_RUNNING, - TASK_DISTRIBUTED, - TASK_COMPLETED, - TASK_RECOVERED, - TASK_REPORTED, - TASK_FAILED_ANALYSIS, - TASK_FAILED_PROCESSING, - TASK_FAILED_REPORTING, - ) - - for state in states: - report["states_count"][state] = db.count_tasks(state) - # For the following stats we're only interested in completed tasks. - tasks = db.count_tasks(status=TASK_COMPLETED) - tasks += db.count_tasks(status=TASK_REPORTED) + tasks = states_count.get(TASK_COMPLETED, 0) + states_count.get(TASK_REPORTED, 0) data = {"title": "Dashboard", "report": {}} @@ -93,7 +67,7 @@ def index(request): report["estimate_hour"] = format_number_with_space(int(hourly)) report["estimate_day"] = format_number_with_space(int(24 * hourly)) - report["top_detections"] = top_detections() + # report["top_detections"] = top_detections() data["report"] = report return render(request, "dashboard/index.html", data) diff --git a/web/templates/analysis/generic/_file_info.html b/web/templates/analysis/generic/_file_info.html index 56f795e9439..a4c4ff47566 100644 --- a/web/templates/analysis/generic/_file_info.html +++ b/web/templates/analysis/generic/_file_info.html @@ -311,6 +311,10 @@
File {{file.decoded_files_tool}} {% endif %} + {% if file.extracted_files %} + {{file.extracted_files_tool|default:"Extracted Files"}} + {% endif %} + {% if file.selfextract %} {% for name, details in file.selfextract.items %} Extract: {{name}} @@ -397,6 +401,23 @@
File {% endif %} + {% if file.extracted_files %} +
+
+
Extracted Files
+
+ {% for sub_file in file.extracted_files %} + {% if tab_name == "static" %} + {% include "analysis/generic/_file_info.html" with file=sub_file tab_name="dropped" %} + {% else %} + {% include "analysis/generic/_file_info.html" with file=sub_file %} + {% endif %} + {% endfor %} +
+
+
+ {% endif %} + {% if file.selfextract %} {% for name, details in file.selfextract.items %}
@@ -407,7 +428,11 @@
File
Password: {{details.password}}
{% endif %} {% for sub_file in details.extracted_files %} - {% include "analysis/generic/_subfile_info.html" %} + {% if tab_name == "static" %} + {% include "analysis/generic/_file_info.html" with file=sub_file tab_name="dropped" %} + {% else %} + {% include "analysis/generic/_file_info.html" with file=sub_file %} + {% endif %} {% endfor %}
@@ -436,3 +461,4 @@
File {% endif %} + diff --git a/web/templates/analysis/generic/_subfile_capeyara.html b/web/templates/analysis/generic/_subfile_capeyara.html deleted file mode 100644 index c69018c0355..00000000000 --- a/web/templates/analysis/generic/_subfile_capeyara.html +++ /dev/null @@ -1,60 +0,0 @@ -
- {% load key_tags %} -
-
-
CAPE Yara Details
-
-
- {% if sub_file.cape_yara %} -
- {% for hit in sub_file.cape_yara %} -
-
-

- -

-
-
-
-
- - {% if hit.strings %} - - - - - {% endif %} - {% if hit.addresses %} - - - - - {% endif %} -
Strings -
    - {% for string in hit.strings %} -
  • {{string}}
    • - {% endfor %} -
-
Address Matches -
    - {% for key, value in hit.addresses.items %} -
  • {{key}}: {{value}}
    • - {% endfor %} -
-
-
-
-
-
- {% endfor %} -
- {% else %} -
No CAPE Yara hits.
- {% endif %} -
-
-
diff --git a/web/templates/analysis/generic/_subfile_info.html b/web/templates/analysis/generic/_subfile_info.html deleted file mode 100644 index 2bc34990fe8..00000000000 --- a/web/templates/analysis/generic/_subfile_info.html +++ /dev/null @@ -1,420 +0,0 @@ - -{% load key_tags %} - -
-
-
Subfile Information
- {% if sub_file.dropdir %} - Download - {% else %} -
- {% if not config.zipped_download %} - {% if tab_name == "static" %} - - {% else %} - - {% endif %} - {% endif %} - {% if tab_name == "static" %} - - {% else %} - - {% endif %} - - {% if config.vtupload and analysis.info.tlp != "Red" %} - VT - {% endif %} -
- {% endif %} -
- -
- - {% if sub_file.note %} - - - - - {% endif %} - - {% if sub_file.cape_type %} - - - - - {% endif %} - - - - - - - {% if sub_file.type %} - - - - - {% endif %} - - {% if sub_file.guest_paths %} - - - - - {% endif %} - - - - - - - {% if sub_file.module_path and sub_file.process_path != sub_file.module_path %} - - - - - {% endif %} - - {% if sub_file.cape_type_code == 8 or sub_file.cape_type_code == 9 %} - - - - - {% endif %} - {% if sub_file.cape_type_code == 5 %} - - - - - {% endif %} - {% if sub_file.cape_type_code == 3 or sub_file.cape_type_code == 4 %} - - - - - - - - - - - - - - - - - {% else %} - {% if sub_file.process_name %} - - - - - {% endif %} - {% if sub_file.process_path %} - - - - - {% endif %} - {% endif %} - - - - - - - - - - - - - - {% if sub_file.sha3_384 %} - - - - - {% endif %} - {% if sub_file.rh_hash %} - - - - - {% endif %} - - - - - {% if sub_file.tlsh %} - - - - - {% endif %} - - - - - - {% if sub_file.clamav %} - - - - - {% endif %} - - {% if sub_file.yara %} - - - - - {% endif %} - - {% if sub_file.cape_yara %} - - - - - {% endif %} - - {% if sub_file.trid %} - - - - - {% endif %} - - {% if sub_file.die %} - - - - - {% endif %} -
Note{{sub_file.note}}
Type{{sub_file.cape_type}}
Filename - {% for name in sub_file.name|str2list %} -
{{name|safe}}
- {% endfor %} -
File Type{{sub_file.type}}
Associated Filenames - {% for path in sub_file.guest_paths|str2list %} -
{{path}}
- {% endfor %} -
File Size{{sub_file.size}} bytes
Module Path{{sub_file.module_path}}
Virtual Address{{sub_file.virtual_address}}
Section Handle{{sub_file.section_handle}}
Target Process{{sub_file.target_process}} (PID: {{sub_file.target_pid}})
Target Path{{sub_file.target_path}}
Injecting Process{{sub_file.process_name}} (PID: {{sub_file.pid}})
Path{{sub_file.process_path}}
Process{{sub_file.process_name}} {% if sub_file.pid %}(PID: {{sub_file.pid}}){% endif %}
Path{{sub_file.process_path}}
MD5{{sub_file.md5}}
SHA1{{sub_file.sha1}}
SHA256 - {{sub_file.sha256}} - - VT - MWDB - Bazaar - -
SHA3-384{{sub_file.sha3_384}}
RichHeader Hash{{sub_file.rh_hash}}
CRC32{{sub_file.crc32}}
TLSH{{sub_file.tlsh}}
Ssdeep{{sub_file.ssdeep}}
ClamAV -
    - {% for sign in sub_file.clamav %} -
  • {{sign}}
    • - {% endfor %} -
-
- {% if config.yara_detail %} - Yara - {% else %} - Yara - {% endif %} - -
    - {% for sign in sub_file.yara %} -
  • - {{sign.name}} - - {{sign.meta.description}} - {% if sign.meta.author %} ({{sign.meta.author}}){% endif %} -
    • - {% endfor %} -
-
- {% if config.yara_detail %} - CAPE Yara - {% else %} - CAPE Yara - {% endif %} - -
    - {% for sign in sub_file.cape_yara %} -
  • - {{sign.name}} - {% if sign.meta.cape_type %} {{sign.meta.cape_type}}{% endif %} -
    • - {% endfor %} -
-
TriD -
    - {% for str in sub_file.trid %}
  • {{str}}
    • {% endfor %} -
-
Detect It Easy -
    - {% for str in sub_file.die %}
  • {{str}}
    • {% endfor %} -
-
-
- - - {% if not sub_file.dropdir %} -
-
- Tools: - - {% if sub_file.pe %}PE{% endif %} - {% if sub_file.dotnet %}.NET{% endif %} - {% if sub_file.pdf %}PDF{% endif %} - {% if sub_file.lnk %}LNK{% endif %} - {% if sub_file.java %}Java{% endif %} - {% if sub_file.office %}Office{% endif %} - - {% if config.flare_capa %} - {% if not sub_file.flare_capa and on_demand.flare_capa %} - Extract: CAPA - {% elif sub_file.flare_capa %} - View CAPA - {% endif %} - {% endif %} - - {% if config.strings %} - {% if "strings" not in sub_file %} - Extract: Strings - {% elif sub_file.strings %} - Strings - {% endif %} - {% endif %} - - {% if config.floss %} - {% if not sub_file.floss %} - Extract: Floss - {% else %} - Floss - {% endif %} - {% endif %} - - {% if config.bingraph %} - {% if not graphs.bingraph.content|getkey:sub_file.sha256 and on_demand.bingraph %} - Extract: BinGraph - {% elif graphs.bingraph.content|getkey:sub_file.sha256 %} - BinGraph - {% endif %} - {% endif %} - - {% if sub_file.data %} - Text - {% endif %} - - {% if sub_file.decoded_files %} - {{sub_file.decoded_files_tool}} - {% endif %} - - {% if sub_file.extracted_files %} - {{sub_file.extracted_files_tool}} - {% endif %} -
-
- {% endif %} -
- - -
- {% if sub_file.flare_capa %} -
-
-
CAPA Results
-
- {% if file.flare_capa.CAPABILITY or file.flare_capa.ATTCK or file.flare_capa.MBC %} - {% if file.flare_capa.CAPABILITY %}{{ file.flare_capa|flare_capa_capability }}{% endif %} - {% if file.flare_capa.ATTCK %}{{ file.flare_capa|flare_capa_attck }}{% endif %} - {% if file.flare_capa.MBC %}{{ file.flare_capa|flare_capa_mbc }}{% endif %} - {% else %} - No significant results. - {% endif %} -
-
-
- {% endif %} - - {% if graphs.vba2graph.enabled and graphs.vba2graph.content|getkey:sub_file.sha256 %} -
-
{{ graphs.vba2graph.content|getkey:sub_file.sha256|safe }}
-
- {% endif %} - - {% if sub_file.virustotal %} -
{% include "analysis/generic/_virustotal.html" %}
- {% endif %} - - {% if sub_file.strings %} -
-
-
Strings
-
- 
{% for string in sub_file.strings %}{{string}}
{% endfor %}
-
-
-
- {% endif %} - - {% if sub_file.dotnet_strings %} -
-
-
.NET Strings
-
- 
{% for string in sub_file.dotnet_strings %}{{string}}
{% endfor %}
-
-
-
- {% endif %} - - {% if sub_file.data %} -
-
-
Extracted Text
-
- 
{{sub_file.data|escape}}
-
-
-
- {% endif %} - - {% if sub_file.decoded_files %} -
-
-
Decoded File Content
-
- 
{{sub_file.decoded_files|escape}}
-
-
-
- {% endif %} - - {% if sub_file.extracted_files %} -
-
-
Extracted Files
-
- {% for sub_file in sub_file.extracted_files %} - {% include "analysis/generic/_subfile_info.html" %} - {% endfor %} -
-
-
- {% endif %} - - {% if config.yara_detail and sub_file.yara %}
{% include "analysis/generic/_subfile_yara.html" %}
{% endif %} - {% if config.yara_detail and sub_file.cape_yara %}
{% include "analysis/generic/_subfile_capeyara.html" %}
{% endif %} - {% if sub_file.pe %}
{% include "analysis/generic/_pe.html" %}
{% endif %} - {% if sub_file.dotnet %}
{% include "analysis/generic/_dotnet.html" %}
{% endif %} - {% if sub_file.pdf %}
{% include "analysis/generic/_pdf.html" %}
{% endif %} - {% if sub_file.lnk %}
{% include "analysis/generic/_lnk.html" %}
{% endif %} - {% if sub_file.java %}
{% include "analysis/generic/_java.html" %}
{% endif %} - {% if sub_file.office %}
{% include "analysis/generic/_office.html" %}
{% endif %} - {% if sub_file.floss %}
{% include "analysis/generic/_floss.html" %}
{% endif %} - - {% if graphs.bingraph.enabled and graphs.bingraph.content|getkey:sub_file.sha256 %} -
-
{{ graphs.bingraph.content|getkey:sub_file.sha256|safe }}
-
- {% endif %} -
diff --git a/web/templates/analysis/generic/_subfile_yara.html b/web/templates/analysis/generic/_subfile_yara.html deleted file mode 100644 index df9abbfa327..00000000000 --- a/web/templates/analysis/generic/_subfile_yara.html +++ /dev/null @@ -1,60 +0,0 @@ -
- {% load key_tags %} -
-
-
Yara Details
-
-
- {% if sub_file.yara %} -
- {% for hit in sub_file.yara %} -
-
-

- -

-
-
-
-
- - {% if hit.strings %} - - - - - {% endif %} - {% if hit.addresses %} - - - - - {% endif %} -
Strings -
    - {% for string in hit.strings %} -
  • {{string}}
    • - {% endfor %} -
-
Address Matches -
    - {% for key, value in hit.addresses.items %} -
  • {{key}}: {{value}}
    • - {% endfor %} -
-
-
-
-
-
- {% endfor %} -
- {% else %} -
No Yara hits.
- {% endif %} -
-
-
diff --git a/web/templates/submission/index.html b/web/templates/submission/index.html index 0540cab9d29..c0a4a188f22 100644 --- a/web/templates/submission/index.html +++ b/web/templates/submission/index.html @@ -429,7 +429,7 @@
Advance
-
+
@@ -438,7 +438,7 @@
Advance
- + diff --git a/web/web/settings.py b/web/web/settings.py index 53858ef7cab..3c2c492855a 100644 --- a/web/web/settings.py +++ b/web/web/settings.py @@ -155,6 +155,7 @@ # in apps' "static/" subdirectories and in STATICFILES_DIRS. # Example: "/home/media/media.lawrence.com/static/" # When NGINX is as reverse proxy you need to put next line in local_settings.py +# python manage.py collectstatic STATIC_ROOT = "" @@ -338,7 +339,7 @@ SITE_ID = 1 -# https://docs.allauth.org/en/dev/socialaccount/configuration.html +# https://django-allauth.readthedocs.io/en/latest/configuration.html if web_cfg.registration.get("email_confirmation", False): ACCOUNT_EMAIL_VERIFICATION = "mandatory" SOCIALACCOUNT_EMAIL_VERIFICATION = ACCOUNT_EMAIL_VERIFICATION @@ -373,6 +374,7 @@ if web_cfg.registration.get("captcha_enabled", False): ACCOUNT_SIGNUP_FORM_CLASS = "web.allauth_forms.CaptchedSignUpForm" +# SOCIALACCOUNT_FORMS = {"signup": "web.allauth_forms.MyCustomSocialSignupForm"} # Fix to avoid migration warning in django 1.7 about test runner (1_6.W001). # In future it could be removed: https://code.djangoproject.com/ticket/23469
debug Enable debugging features