diff --git a/.github/CHANGELOG.md b/.github/CHANGELOG.md index 89d54dcf84..de646230cc 100644 --- a/.github/CHANGELOG.md +++ b/.github/CHANGELOG.md @@ -2,6 +2,15 @@ [**Upgrade Guide**](https://intelowlproject.github.io/docs/IntelOwl/installation/#update-to-the-most-recent-version) +## [v6.7.0](https://github.com/intelowlproject/IntelOwl/releases/tag/v6.7.0) +We welcome the first working AI-based integration in IntelOwl! :tada: + +We invite all the users to test our new awesome Chatbot! :sunglasses: Check the [official doc](https://intelowlproject.github.io/docs/IntelOwl/chatbot/) for more info. + +Mainly this release merges all the developments performed by our Google Summer of Code contributors during the last month: +* [Francesco Berardi](https://github.com/berardifra): "Integrating a Self-Deployed LLM Chatbot for Threat Intelligence" +* [Sanjib Behera](https://github.com/sanjib2006): "Integration Ecosystem & Connector Optimization" + ## [v6.6.1](https://github.com/intelowlproject/IntelOwl/releases/tag/v6.6.1) A lot of minor contributions to fix bugs and improve maintenance. diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 0486db6351..aa6b347ec5 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -44,7 +44,7 @@ jobs: fetch-depth: 2 - name: Set up Python - uses: actions/setup-python@v6.2.0 + uses: actions/setup-python@v6.3.0 with: python-version: '3.11' diff --git a/.github/workflows/dependency_review.yml b/.github/workflows/dependency_review.yml index af748665bf..ceadc5d833 100644 --- a/.github/workflows/dependency_review.yml +++ b/.github/workflows/dependency_review.yml @@ -13,4 +13,11 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@v6.0.2 - name: 'Dependency Review' - uses: actions/dependency-review-action@v4 \ No newline at end of file + uses: actions/dependency-review-action@v4 + with: + # GHSA-gr75-jv2w-4656: LangChain path traversal in file-search middleware/loaders. + # Patched only in langchain 1.3.9 (breaking major upgrade). The chatbot uses no + # document loaders, file-search middleware, or hub prompt pulls (only langchain.agents, + # langchain_core.*, langchain_ollama), so the vulnerable code paths are unreachable. + # Accepted until the planned langchain 1.x migration. Approved by @mlodic. + allow-ghsas: GHSA-gr75-jv2w-4656 \ No newline at end of file diff --git a/.github/workflows/pull_request_automation.yml b/.github/workflows/pull_request_automation.yml index d75d7f2573..42ed75de07 100644 --- a/.github/workflows/pull_request_automation.yml +++ b/.github/workflows/pull_request_automation.yml @@ -37,7 +37,7 @@ jobs: uses: actions/checkout@v6.0.2 - name: Set up Python - uses: actions/setup-python@v6.2.0 + uses: actions/setup-python@v6.3.0 with: python-version: 3.11 @@ -87,10 +87,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Startup script launch (Slow) - if: contains(github.base_ref, 'master') + # Only the develop -> master release PR runs the full build: extra analyzers plus the + # chatbot stack (--ollama), which smoke-tests the ollama + celery_worker_chatbot topology + # on the :ci image. Kept off every other PR because the model pull is heavy. + if: github.base_ref == 'master' && github.head_ref == 'develop' run: | cp docker/env_file_integrations_template docker/env_file_integrations - ./start ci up --malware_tools_analyzers --phishing_analyzers -- --build -d + ./start ci up --malware_tools_analyzers --phishing_analyzers --ollama -- --build -d env: DOCKER_BUILDKIT: 1 BUILDKIT_PROGRESS: "plain" @@ -98,7 +101,9 @@ jobs: REPO_DOWNLOADER_ENABLED: false - name: Startup script launch (Fast) - if: "!contains(github.base_ref, 'master')" + # Exact complement of the Slow step so every other PR (incl. a non-develop head into + # master) still gets a container up for the test steps below. + if: ${{ !(github.base_ref == 'master' && github.head_ref == 'develop') }} run: | ./start ci up -- --build -d env: @@ -137,7 +142,7 @@ jobs: with: node-version: 18 - name: Cache node modules - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: ~/.npm key: npm-build-${{ hashFiles('frontend/package-lock.json') }} diff --git a/api_app/analyzers_manager/file_analyzers/ipqsfile.py b/api_app/analyzers_manager/file_analyzers/ipqsfile.py new file mode 100644 index 0000000000..d87968e7bd --- /dev/null +++ b/api_app/analyzers_manager/file_analyzers/ipqsfile.py @@ -0,0 +1,54 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +"""IPQualityScore file analyzer. + +Provides `IPQSFileScan`, a file-scanning analyzer that uploads files to +IPQualityScore for malware detection and polls for results. +""" + +import logging + +from api_app.analyzers_manager.classes import FileAnalyzer +from api_app.mixins import IPQualityScoreMixin + +logger = logging.getLogger(__name__) + + +class IPQSFileScan(FileAnalyzer, IPQualityScoreMixin): + """ + Scan a binary file using IPQualityScore malware detection service. + """ + + @classmethod + def update(cls): + pass + + def run(self): + binary = self.read_file_bytes() + files = {"files": (self.filename, binary)} + # lookup endpoint check for cached result + lookup_result = self._make_request( + self.lookup_endpoint, + method="POST", + _api_key=self._ipqs_api_key, + files=files, + ) + if lookup_result.get("status", False) == "cached": + lookup_result.pop("update_url", None) + return lookup_result + # sending file to ipqs for scan + scan_result = self._make_request( + self.scan_endpoint, + method="POST", + _api_key=self._ipqs_api_key, + files=files, + ) + # waiting for scan result with help of request id + result = self._poll_for_report( + endpoint=self.postback_endpoint, + _api_key=self._ipqs_api_key, + request_id=scan_result.get("request_id"), + ) + result.pop("update_url", None) + return result diff --git a/api_app/analyzers_manager/migrations/0191_analyzer_config_ipqs_malware_file_scanner.py b/api_app/analyzers_manager/migrations/0191_analyzer_config_ipqs_malware_file_scanner.py new file mode 100644 index 0000000000..531d39f938 --- /dev/null +++ b/api_app/analyzers_manager/migrations/0191_analyzer_config_ipqs_malware_file_scanner.py @@ -0,0 +1,218 @@ +from django.db import migrations +from django.db.models.fields.related_descriptors import ( + ForwardManyToOneDescriptor, + ForwardOneToOneDescriptor, + ManyToManyDescriptor, + ReverseManyToOneDescriptor, + ReverseOneToOneDescriptor, +) + +plugin = { + "python_module": { + "health_check_schedule": None, + "update_schedule": None, + "module": "ipqsfile.IPQSFileScan", + "base_path": "api_app.analyzers_manager.file_analyzers", + }, + "name": "IPQS_Malware_File_Scanner", + "description": "Scan files for malware, viruses, and malicious payloads in real-time using [IPQualityScore](https://www.ipqualityscore.com/)'s advanced file scanning engine.", + "disabled": False, + "soft_time_limit": 140, + "routing_key": "default", + "health_check_status": True, + "type": "file", + "docker_based": False, + "maximum_tlp": "AMBER", + "observable_supported": [], + "supported_filetypes": [ + "application/w-script-file", + "application/javascript", + "application/x-javascript", + "text/javascript", + "application/x-vbscript", + "text/x-ms-iqy", + "application/vnd.android.package-archive", + "application/x-dex", + "application/onenote", + "application/zip", + "multipart/x-zip", + "application/java-archive", + "text/rtf", + "application/rtf", + "application/x-sharedlib", + "application/vnd.microsoft.portable-executable", + "application/x-elf", + "application/octet-stream", + "application/vnd.tcpdump.pcap", + "application/pdf", + "text/html", + "application/x-mspublisher", + "application/vnd.ms-excel.addin.macroEnabled", + "application/vnd.ms-excel.sheet.macroEnabled.12", + "application/vnd.ms-excel", + "application/excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/xml", + "text/xml", + "application/encrypted", + "text/plain", + "text/csv", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.ms-powerpoint", + "application/vnd.ms-office", + "application/x-binary", + "application/x-macbinary", + "application/mac-binary", + "application/x-mach-binary", + "application/x-zip-compressed", + "application/x-compressed", + "application/vnd.ms-outlook", + "message/rfc822", + "application/pkcs7-signature", + "application/x-pkcs7-signature", + "multipart/mixed", + "text/x-shellscript", + "application/x-chrome-extension", + "application/json", + "application/x-executable", + "text/x-java", + "text/x-kotlin", + "text/x-swift", + "text/x-objective-c", + "application/x-ms-shortcut", + "application/gzip", + ], + "run_hash": False, + "run_hash_type": "", + "not_supported_filetypes": [], + "mapping_data_model": {}, + "model": "analyzers_manager.AnalyzerConfig", +} + +params = [ + { + "python_module": { + "module": "ipqsfile.IPQSFileScan", + "base_path": "api_app.analyzers_manager.file_analyzers", + }, + "name": "ipqs_api_key", + "type": "str", + "description": "Please provide the IPQS API key.", + "is_secret": True, + "required": True, + }, + { + "python_module": { + "module": "ipqsfile.IPQSFileScan", + "base_path": "api_app.analyzers_manager.file_analyzers", + }, + "name": "polling_interval", + "type": "int", + "description": "Recommended polling interval: 10 seconds.", + "is_secret": False, + "required": True, + }, + { + "python_module": { + "module": "ipqsfile.IPQSFileScan", + "base_path": "api_app.analyzers_manager.file_analyzers", + }, + "name": "max_retries", + "type": "int", + "description": "Recommended max retries: 8.", + "is_secret": False, + "required": True, + }, +] + +values = [] + + +def _get_real_obj(Model, field, value): + def _get_obj(Model, other_model, value): + if isinstance(value, dict): + real_vals = {} + for key, real_val in value.items(): + real_vals[key] = _get_real_obj(other_model, key, real_val) + value = other_model.objects.get_or_create(**real_vals)[0] + # it is just the primary key serialized + else: + if isinstance(value, int): + if Model.__name__ == "PluginConfig": + value = other_model.objects.get(name=plugin["name"]) + else: + value = other_model.objects.get(pk=value) + else: + value = other_model.objects.get(name=value) + return value + + if ( + type(getattr(Model, field)) + in [ + ForwardManyToOneDescriptor, + ReverseManyToOneDescriptor, + ReverseOneToOneDescriptor, + ForwardOneToOneDescriptor, + ] + and value + ): + other_model = getattr(Model, field).get_queryset().model + value = _get_obj(Model, other_model, value) + elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value: + other_model = getattr(Model, field).rel.model + value = [_get_obj(Model, other_model, val) for val in value] + return value + + +def _create_object(Model, data): + mtm, no_mtm = {}, {} + for field, value in data.items(): + value = _get_real_obj(Model, field, value) + if type(getattr(Model, field)) is ManyToManyDescriptor: + mtm[field] = value + else: + no_mtm[field] = value + try: + o = Model.objects.get(**no_mtm) + except Model.DoesNotExist: + o = Model(**no_mtm) + o.full_clean() + o.save() + for field, value in mtm.items(): + attribute = getattr(o, field) + if value is not None: + attribute.set(value) + return False + return True + + +def migrate(apps, schema_editor): + Parameter = apps.get_model("api_app", "Parameter") + PluginConfig = apps.get_model("api_app", "PluginConfig") + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + if not Model.objects.filter(name=plugin["name"]).exists(): + exists = _create_object(Model, plugin) + if not exists: + for param in params: + _create_object(Parameter, param) + for value in values: + _create_object(PluginConfig, value) + + +def reverse_migrate(apps, schema_editor): + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + Model.objects.get(name=plugin["name"]).delete() + + +class Migration(migrations.Migration): + atomic = False + dependencies = [ + ("api_app", "0073_alter_updatecheckstatus_last_checked_at_and_more"), + ("analyzers_manager", "0190_remove_greynoise_labs_analyzer"), + ] + + operations = [migrations.RunPython(migrate, reverse_migrate)] diff --git a/api_app/analyzers_manager/migrations/0192_analyzer_config_ipqs_url_file_scanner.py b/api_app/analyzers_manager/migrations/0192_analyzer_config_ipqs_url_file_scanner.py new file mode 100644 index 0000000000..a157b92fc5 --- /dev/null +++ b/api_app/analyzers_manager/migrations/0192_analyzer_config_ipqs_url_file_scanner.py @@ -0,0 +1,159 @@ +from django.db import migrations +from django.db.models.fields.related_descriptors import ( + ForwardManyToOneDescriptor, + ForwardOneToOneDescriptor, + ManyToManyDescriptor, + ReverseManyToOneDescriptor, + ReverseOneToOneDescriptor, +) + +plugin = { + "python_module": { + "health_check_schedule": None, + "update_schedule": None, + "module": "ipqsurl.IPQSUrlScan", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "IPQS_URL_File_Scanner", + "description": "Scans files hosted or accessible via a URL using [IPQualityScore](https://www.ipqualityscore.com/) malware detection API.", + "disabled": False, + "soft_time_limit": 140, + "routing_key": "default", + "health_check_status": True, + "type": "observable", + "docker_based": False, + "maximum_tlp": "AMBER", + "observable_supported": ["url"], + "supported_filetypes": [], + "run_hash": False, + "run_hash_type": "", + "not_supported_filetypes": [], + "mapping_data_model": {}, + "model": "analyzers_manager.AnalyzerConfig", +} + +params = [ + { + "python_module": { + "module": "ipqsurl.IPQSUrlScan", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "ipqs_api_key", + "type": "str", + "description": "Please provide the IPQS API key.", + "is_secret": True, + "required": True, + }, + { + "python_module": { + "module": "ipqsurl.IPQSUrlScan", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "polling_interval", + "type": "int", + "description": "Recommended polling interval: 10 seconds.", + "is_secret": False, + "required": True, + }, + { + "python_module": { + "module": "ipqsurl.IPQSUrlScan", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "max_retries", + "type": "int", + "description": "Recommended max retries: 8.", + "is_secret": False, + "required": True, + }, +] + +values = [] + + +def _get_real_obj(Model, field, value): + def _get_obj(Model, other_model, value): + if isinstance(value, dict): + real_vals = {} + for key, real_val in value.items(): + real_vals[key] = _get_real_obj(other_model, key, real_val) + value = other_model.objects.get_or_create(**real_vals)[0] + # it is just the primary key serialized + else: + if isinstance(value, int): + if Model.__name__ == "PluginConfig": + value = other_model.objects.get(name=plugin["name"]) + else: + value = other_model.objects.get(pk=value) + else: + value = other_model.objects.get(name=value) + return value + + if ( + type(getattr(Model, field)) + in [ + ForwardManyToOneDescriptor, + ReverseManyToOneDescriptor, + ReverseOneToOneDescriptor, + ForwardOneToOneDescriptor, + ] + and value + ): + other_model = getattr(Model, field).get_queryset().model + value = _get_obj(Model, other_model, value) + elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value: + other_model = getattr(Model, field).rel.model + value = [_get_obj(Model, other_model, val) for val in value] + return value + + +def _create_object(Model, data): + mtm, no_mtm = {}, {} + for field, value in data.items(): + value = _get_real_obj(Model, field, value) + if type(getattr(Model, field)) is ManyToManyDescriptor: + mtm[field] = value + else: + no_mtm[field] = value + try: + o = Model.objects.get(**no_mtm) + except Model.DoesNotExist: + o = Model(**no_mtm) + o.full_clean() + o.save() + for field, value in mtm.items(): + attribute = getattr(o, field) + if value is not None: + attribute.set(value) + return False + return True + + +def migrate(apps, schema_editor): + Parameter = apps.get_model("api_app", "Parameter") + PluginConfig = apps.get_model("api_app", "PluginConfig") + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + if not Model.objects.filter(name=plugin["name"]).exists(): + exists = _create_object(Model, plugin) + if not exists: + for param in params: + _create_object(Parameter, param) + for value in values: + _create_object(PluginConfig, value) + + +def reverse_migrate(apps, schema_editor): + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + Model.objects.get(name=plugin["name"]).delete() + + +class Migration(migrations.Migration): + atomic = False + dependencies = [ + ("api_app", "0073_alter_updatecheckstatus_last_checked_at_and_more"), + ("analyzers_manager", "0191_analyzer_config_ipqs_malware_file_scanner"), + ] + + operations = [migrations.RunPython(migrate, reverse_migrate)] diff --git a/api_app/analyzers_manager/migrations/0193_analyzer_config_cve_exploitability.py b/api_app/analyzers_manager/migrations/0193_analyzer_config_cve_exploitability.py new file mode 100644 index 0000000000..190020be68 --- /dev/null +++ b/api_app/analyzers_manager/migrations/0193_analyzer_config_cve_exploitability.py @@ -0,0 +1,128 @@ +from django.db import migrations +from django.db.models.fields.related_descriptors import ( + ForwardManyToOneDescriptor, + ForwardOneToOneDescriptor, + ManyToManyDescriptor, + ReverseManyToOneDescriptor, + ReverseOneToOneDescriptor, +) + +plugin = { + "python_module": { + "health_check_schedule": None, + "update_schedule": None, + "module": "cve_exploitability.CVEExploitability", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "CVE_Exploitability", + "description": "Enriches a CVE with real-world exploitation signals: membership in the " + "[CISA Known Exploited Vulnerabilities (KEV)](https://www.cisa.gov/known-exploited-vulnerabilities-catalog) " + "catalog (including the date added and known ransomware campaign use) and the " + "[FIRST.org EPSS](https://www.first.org/epss/) score and percentile (the probability the CVE will be " + "exploited in the wild in the next 30 days). Both data sources are free and require no API key.", + "disabled": False, + "soft_time_limit": 60, + "routing_key": "default", + "health_check_status": True, + "type": "observable", + "docker_based": False, + "maximum_tlp": "AMBER", + "observable_supported": ["generic"], + "supported_filetypes": [], + "run_hash": False, + "run_hash_type": "", + "not_supported_filetypes": [], + "model": "analyzers_manager.AnalyzerConfig", +} + +params = [] + +values = [] + + +def _get_real_obj(Model, field, value): + def _get_obj(Model, other_model, value): + if isinstance(value, dict): + real_vals = {} + for key, real_val in value.items(): + real_vals[key] = _get_real_obj(other_model, key, real_val) + value = other_model.objects.get_or_create(**real_vals)[0] + # it is just the primary key serialized + else: + if isinstance(value, int): + if Model.__name__ == "PluginConfig": + value = other_model.objects.get(name=plugin["name"]) + else: + value = other_model.objects.get(pk=value) + else: + value = other_model.objects.get(name=value) + return value + + if ( + type(getattr(Model, field)) + in [ + ForwardManyToOneDescriptor, + ReverseManyToOneDescriptor, + ReverseOneToOneDescriptor, + ForwardOneToOneDescriptor, + ] + and value + ): + other_model = getattr(Model, field).get_queryset().model + value = _get_obj(Model, other_model, value) + elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value: + other_model = getattr(Model, field).rel.model + value = [_get_obj(Model, other_model, val) for val in value] + return value + + +def _create_object(Model, data): + mtm, no_mtm = {}, {} + for field, value in data.items(): + value = _get_real_obj(Model, field, value) + if type(getattr(Model, field)) is ManyToManyDescriptor: + mtm[field] = value + else: + no_mtm[field] = value + try: + o = Model.objects.get(**no_mtm) + except Model.DoesNotExist: + o = Model(**no_mtm) + o.full_clean() + o.save() + for field, value in mtm.items(): + attribute = getattr(o, field) + if value is not None: + attribute.set(value) + return False + return True + + +def migrate(apps, schema_editor): + Parameter = apps.get_model("api_app", "Parameter") + PluginConfig = apps.get_model("api_app", "PluginConfig") + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + if not Model.objects.filter(name=plugin["name"]).exists(): + exists = _create_object(Model, plugin) + if not exists: + for param in params: + _create_object(Parameter, param) + for value in values: + _create_object(PluginConfig, value) + + +def reverse_migrate(apps, schema_editor): + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + Model.objects.get(name=plugin["name"]).delete() + + +class Migration(migrations.Migration): + atomic = False + dependencies = [ + ("api_app", "0073_alter_updatecheckstatus_last_checked_at_and_more"), + ("analyzers_manager", "0192_analyzer_config_ipqs_url_file_scanner"), + ] + + operations = [migrations.RunPython(migrate, reverse_migrate)] diff --git a/api_app/analyzers_manager/migrations/0194_analyzer_config_rdap.py b/api_app/analyzers_manager/migrations/0194_analyzer_config_rdap.py new file mode 100644 index 0000000000..f43a0bae0f --- /dev/null +++ b/api_app/analyzers_manager/migrations/0194_analyzer_config_rdap.py @@ -0,0 +1,129 @@ +from django.db import migrations +from django.db.models.fields.related_descriptors import ( + ForwardManyToOneDescriptor, + ForwardOneToOneDescriptor, + ManyToManyDescriptor, + ReverseManyToOneDescriptor, + ReverseOneToOneDescriptor, +) + +plugin = { + "python_module": { + "health_check_schedule": None, + "update_schedule": None, + "module": "rdap.Rdap", + "base_path": "api_app.analyzers_manager.observable_analyzers", + }, + "name": "Rdap", + "description": "Query the public [RDAP](https://about.rdap.org/) bootstrap " + "(rdap.org) for registration data (RFC 9082); the free, unauthenticated WHOIS " + "successor. Supports IPs, domains and URLs.", + "disabled": False, + "soft_time_limit": 30, + "routing_key": "default", + "health_check_status": True, + "type": "observable", + "docker_based": False, + "maximum_tlp": "AMBER", + "observable_supported": ["ip", "domain", "url"], + "supported_filetypes": [], + "run_hash": False, + "run_hash_type": "", + "not_supported_filetypes": [], + "mapping_data_model": {}, + "model": "analyzers_manager.AnalyzerConfig", +} + +params = [] + +values = [] + + +def _get_real_obj(Model, field, value): + def _get_obj(Model, other_model, value): + if isinstance(value, dict): + real_vals = {} + for key, real_val in value.items(): + real_vals[key] = _get_real_obj(other_model, key, real_val) + value = other_model.objects.get_or_create(**real_vals)[0] + # it is just the primary key serialized + else: + if isinstance(value, int): + if Model.__name__ == "PluginConfig": + value = other_model.objects.get(name=plugin["name"]) + else: + value = other_model.objects.get(pk=value) + else: + value = other_model.objects.get(name=value) + return value + + if ( + type(getattr(Model, field)) + in [ + ForwardManyToOneDescriptor, + ReverseManyToOneDescriptor, + ReverseOneToOneDescriptor, + ForwardOneToOneDescriptor, + ] + and value + ): + other_model = getattr(Model, field).get_queryset().model + value = _get_obj(Model, other_model, value) + elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value: + other_model = getattr(Model, field).rel.model + value = [_get_obj(Model, other_model, val) for val in value] + return value + + +def _create_object(Model, data): + mtm, no_mtm = {}, {} + for field, value in data.items(): + value = _get_real_obj(Model, field, value) + if type(getattr(Model, field)) is ManyToManyDescriptor: + mtm[field] = value + else: + no_mtm[field] = value + try: + o = Model.objects.get(**no_mtm) + except Model.DoesNotExist: + o = Model(**no_mtm) + o.full_clean() + o.save() + for field, value in mtm.items(): + attribute = getattr(o, field) + if value is not None: + attribute.set(value) + return False + return True + + +def migrate(apps, schema_editor): + Parameter = apps.get_model("api_app", "Parameter") + PluginConfig = apps.get_model("api_app", "PluginConfig") + plugin_data = dict(plugin) + python_path = plugin_data.pop("model") + Model = apps.get_model(*python_path.split(".")) + if not Model.objects.filter(name=plugin_data["name"]).exists(): + exists = _create_object(Model, plugin_data) + if not exists: + for param in params: + _create_object(Parameter, param) + for value in values: + _create_object(PluginConfig, value) + + +def reverse_migrate(apps, schema_editor): + plugin_data = dict(plugin) + python_path = plugin_data.pop("model") + Model = apps.get_model(*python_path.split(".")) + Model.objects.get(name=plugin_data["name"]).delete() + + +class Migration(migrations.Migration): + atomic = False + dependencies = [ + ("api_app", "0073_alter_updatecheckstatus_last_checked_at_and_more"), + ("analyzers_manager", "0193_analyzer_config_cve_exploitability"), + ] + + operations = [migrations.RunPython(migrate, reverse_migrate)] diff --git a/api_app/analyzers_manager/observable_analyzers/bgp_ranking.py b/api_app/analyzers_manager/observable_analyzers/bgp_ranking.py index a7127a976a..8838761ed1 100644 --- a/api_app/analyzers_manager/observable_analyzers/bgp_ranking.py +++ b/api_app/analyzers_manager/observable_analyzers/bgp_ranking.py @@ -33,7 +33,11 @@ def run(self): ) response.raise_for_status() response = response.json() - asn = response.get("response", {}).popitem()[1].get("asn", None) + asn_history = response.get("response", {}) + # an IP with no ASN history returns an empty "response"; peek the latest + # entry without mutating it (popitem() would crash on the empty dict and, + # when populated, drop the entry from the error message below) + asn = list(asn_history.values())[-1].get("asn", None) if asn_history else None if not asn: raise AnalyzerRunException(f"ASN not found in {response}") logger.info(f"ASN {asn} extracted from {self.observable_name}") diff --git a/api_app/analyzers_manager/observable_analyzers/cve_exploitability.py b/api_app/analyzers_manager/observable_analyzers/cve_exploitability.py new file mode 100644 index 0000000000..1d32f83881 --- /dev/null +++ b/api_app/analyzers_manager/observable_analyzers/cve_exploitability.py @@ -0,0 +1,86 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import re + +import requests + +from api_app.analyzers_manager.classes import AnalyzerRunException, ObservableAnalyzer + + +class CVEExploitability(ObservableAnalyzer): + """ + Enrich a CVE observable with real-world exploitation signals instead of + relying on CVSS severity alone: + + - CISA Known Exploited Vulnerabilities (KEV) catalog membership, the date + the CVE was added, and whether it is linked to known ransomware campaigns. + - FIRST.org EPSS (Exploit Prediction Scoring System) score and percentile, + i.e. the probability that the CVE will be exploited in the wild in the + next 30 days. + + Both data sources are free and require no API key. + """ + + # `url` is used by IntelOwl health checks (HEAD HTTP request). + url: str = "https://api.first.org/data/v1/epss" + kev_url: str = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + cve_pattern: str = r"^CVE-\d{4}-\d{4,7}$" + http_timeout: int = 30 + + @classmethod + def update(cls) -> bool: + pass + + def _query_kev(self, cve_id: str) -> dict: + response = requests.get(self.kev_url, timeout=self.http_timeout) + response.raise_for_status() + catalog = response.json() + for vuln in catalog.get("vulnerabilities", []): + if vuln.get("cveID", "").upper() == cve_id: + return { + "in_kev": True, + "kev_date": vuln.get("dateAdded"), + "ransomware_use": vuln.get("knownRansomwareCampaignUse"), + "kev_vendor_project": vuln.get("vendorProject"), + "kev_product": vuln.get("product"), + "kev_vulnerability_name": vuln.get("vulnerabilityName"), + "kev_required_action": vuln.get("requiredAction"), + "kev_due_date": vuln.get("dueDate"), + } + return { + "in_kev": False, + "kev_date": None, + "ransomware_use": None, + } + + def _query_epss(self, cve_id: str) -> dict: + response = requests.get(self.url, params={"cve": cve_id}, timeout=self.http_timeout) + response.raise_for_status() + data = response.json().get("data", []) + if not data: + return {"epss_score": None, "epss_percentile": None, "epss_date": None} + entry = data[0] + epss = entry.get("epss") + percentile = entry.get("percentile") + return { + "epss_score": float(epss) if epss is not None else None, + "epss_percentile": float(percentile) if percentile is not None else None, + "epss_date": entry.get("date"), + } + + def run(self): + # Validate the CVE format (e.g. CVE-2021-44228 or cve-2021-44228) + # before issuing any network request. + if not re.match(self.cve_pattern, self.observable_name, flags=re.IGNORECASE): + raise AnalyzerRunException(f"Invalid CVE format: {self.observable_name}") + + cve_id = self.observable_name.upper() + result = {"cve": cve_id} + try: + result.update(self._query_kev(cve_id)) + result.update(self._query_epss(cve_id)) + except requests.RequestException as e: + raise AnalyzerRunException(e) + + return result diff --git a/api_app/analyzers_manager/observable_analyzers/ipqs.py b/api_app/analyzers_manager/observable_analyzers/ipqs.py index d80ebb5a65..e8db4c6a35 100644 --- a/api_app/analyzers_manager/observable_analyzers/ipqs.py +++ b/api_app/analyzers_manager/observable_analyzers/ipqs.py @@ -1,3 +1,9 @@ +"""IPQualityScore observable analyzer using the IPQualityScore API. + +This module implements the `IPQualityScore` analyzer which queries the +IPQualityScore service for URLs, IPs, emails, phones and credential leaks. +""" + import logging import re @@ -8,9 +14,12 @@ logger = logging.getLogger(__name__) +IP_REG = ( + r"^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}" + r"(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])$" +) -IP_REG = "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])$" -IPv6_REG = ( +IPV6_REG = ( r"\b(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|" r"(?:[0-9a-fA-F]{1,4}:){1,7}:|" r"(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|" @@ -28,20 +37,34 @@ r"((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}" r"(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\b" ) -EMAIL_REG = "[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}" + +EMAIL_REG = r"^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$" + DOMAIN_REG = re.compile( - r"^(?:[a-zA-Z0-9]" # First character of the domain - r"(?:[a-zA-Z0-9-_]{0,61}[A-Za-z0-9])?\.)" # Sub domain + hostname - r"+[A-Za-z0-9][A-Za-z0-9-_]{0,61}" # First 61 characters of the gTLD - r"[A-Za-z]$" # Last character of the gTLD + r"^(?:[a-zA-Z0-9]" + r"(?:[a-zA-Z0-9-_]{0,61}[A-Za-z0-9])?\.)" + r"+[A-Za-z0-9][A-Za-z0-9-_]{0,61}" + r"[A-Za-z]$" ) -PHONE_REG = "^\+?[1-9]\d{1,14}$" + +PHONE_REG = r"^\+?[0-9(). -]{7,20}$" + URL_REG = ( - "((http|https)://)(www.)?[a-zA-Z0-9@:%._\\+~#?&//=]{2,256}\\.[a-z]{2,6}\\b([-a-zA-Z0-9@:%._\\+~#?&//=]*)" + r"((http|https)://)" + r"(www\.)?" + r"[a-zA-Z0-9@:%._\\+~#?&//=]{2,256}" + r"\.[a-z]{2,6}\b" + r"([-a-zA-Z0-9@:%._\\+~#?&//=]*)" ) class IPQualityScore(classes.ObservableAnalyzer): + """Observable analyzer for IPQualityScore service. + + Uses regex-based detection of the observable type and queries the + corresponding IPQualityScore endpoint. + """ + _ipqs_api_key: str url_timeout: int = 2 url_strictness: int = 0 @@ -69,6 +92,13 @@ class IPQualityScore(classes.ObservableAnalyzer): IP_ENDPOINT = IPQS_BASE_URL + "ip?ip=" EMAIL_ENDPOINT = IPQS_BASE_URL + "email?email=" PHONE_ENDPOINT = IPQS_BASE_URL + "phone?phone=" + USERNAME_ENDPOINT = IPQS_BASE_URL + "leaked/username?username=" + PASSWORD_ENDPOINT = IPQS_BASE_URL + "leaked/password?password=" + LEAKED_EMAILENDPOINT = IPQS_BASE_URL + "leaked/email?email=" + + @classmethod + def update(cls) -> bool: + pass def _get_url_payload(self): return { @@ -80,9 +110,9 @@ def _get_url_payload(self): def _get_ip_payload(self): payload = { "strictness": self.ip_strictness, - "allow_public_access_points": str(self.allow_public_access_points).lower(), + "allow_public_access_points": (str(self.allow_public_access_points).lower()), "fast": str(self.ip_fast).lower(), - "lighter_penalties": str(self.lighter_penalties).lower(), + "lighter_penalties": (str(self.lighter_penalties).lower()), "mobile": str(self.mobile).lower(), "transaction_strictness": self.transaction_strictness, } @@ -110,33 +140,114 @@ def _get_phone_payload(self): } def _get_calling_endpoint(self): - if re.match(IP_REG, self.observable_name) or re.match(IPv6_REG, self.observable_name): - return self.IP_ENDPOINT, self._get_ip_payload() - elif re.match(DOMAIN_REG, self.observable_name) or re.match(URL_REG, self.observable_name): - return self.URL_ENDPOINT, self._get_url_payload() - elif re.match(EMAIL_REG, self.observable_name): - return self.EMAIL_ENDPOINT, self._get_email_payload() - elif re.match(PHONE_REG, self.observable_name): - return self.PHONE_ENDPOINT, self._get_phone_payload() - else: - return None, None + if re.match(IP_REG, self.observable_name) or re.match(IPV6_REG, self.observable_name): + return { + "type": "ip", + "endpoint": self.IP_ENDPOINT, + "payload": self._get_ip_payload(), + } + if re.match(DOMAIN_REG, self.observable_name) or re.match(URL_REG, self.observable_name): + return { + "type": "url", + "endpoint": self.URL_ENDPOINT, + "payload": self._get_url_payload(), + } + if re.match(EMAIL_REG, self.observable_name): + return { + "type": "email", + "leaked_email_endpoint": self.LEAKED_EMAILENDPOINT, + "email_endpoint": self.EMAIL_ENDPOINT, + "payload": self._get_email_payload(), + } + if re.match(PHONE_REG, self.observable_name): + return { + "type": "phone", + "endpoint": self.PHONE_ENDPOINT, + "payload": self._get_phone_payload(), + } + return { + "type": "credentials", + "username_endpoint": self.USERNAME_ENDPOINT, + "password_endpoint": self.PASSWORD_ENDPOINT, + } def run(self): - calling_endpoint, payload = self._get_calling_endpoint() + endpoints = self._get_calling_endpoint() ipqs_headers = {"IPQS-KEY": self._ipqs_api_key} try: - if calling_endpoint and payload is not None: + if endpoints.get("type") == "credentials": + return self._handle_credentials(endpoints, ipqs_headers) + + if endpoints.get("type") == "email": + return self._handle_email(endpoints, ipqs_headers) + + if endpoints.get("type") in ["url", "phone", "ip"]: + calling_endpoint = endpoints.get("endpoint") + payload = endpoints.get("payload") response = requests.get( calling_endpoint + self.observable_name, headers=ipqs_headers, params=payload, + timeout=60, ) response.raise_for_status() - result = response.json() - return result - else: - logger.warning("Invalid or unsupported observable type") - raise AnalyzerRunException("Invalid or unsupported observable type") + return response.json() + + msg = "Invalid or unsupported observable type" + logger.warning(msg) + raise AnalyzerRunException(msg) except requests.RequestException as e: - raise AnalyzerRunException(e) + raise AnalyzerRunException(e) from e + + def _handle_credentials(self, endpoints, headers): + username_endpoint = endpoints.get("username_endpoint") + password_endpoint = endpoints.get("password_endpoint") + + response_username = requests.get( + username_endpoint + self.observable_name, + headers=headers, + timeout=60, + ) + response_username.raise_for_status() + result_username = response_username.json() + + response_password = requests.get( + password_endpoint + self.observable_name, + headers=headers, + timeout=60, + ) + response_password.raise_for_status() + result_password = response_password.json() + + return { + "darkweb_leak_username_api_result": result_username, + "darkweb_leak_password_api_result": result_password, + } + + def _handle_email(self, endpoints, headers): + leaked_email_endpoint = endpoints.get("leaked_email_endpoint") + email_endpoint = endpoints.get("email_endpoint") + email_payload = endpoints.get("payload") + + response_leaked = requests.get( + leaked_email_endpoint + self.observable_name, + headers=headers, + timeout=60, + ) + response_leaked.raise_for_status() + result_leaked = response_leaked.json() + + response_email = requests.get( + email_endpoint + self.observable_name, + headers=headers, + params=email_payload, + timeout=60, + ) + response_email.raise_for_status() + result_email = response_email.json() + + return { + "darkweb_leak_email_api_result": result_leaked, + "email_reputation_api_result": result_email, + } diff --git a/api_app/analyzers_manager/observable_analyzers/ipqsurl.py b/api_app/analyzers_manager/observable_analyzers/ipqsurl.py new file mode 100644 index 0000000000..b018127b72 --- /dev/null +++ b/api_app/analyzers_manager/observable_analyzers/ipqsurl.py @@ -0,0 +1,50 @@ +"""IPQualityScore URL analyzer. + +Provides `IPQSUrlScan`, an observable analyzer that scans URLs via +the IPQualityScore service and polls for results. +""" + +import logging + +from api_app.analyzers_manager import classes +from api_app.mixins import IPQualityScoreMixin + +logger = logging.getLogger(__name__) + + +class IPQSUrlScan(classes.ObservableAnalyzer, IPQualityScoreMixin): + """ + Scan a URL using IPQualityScore service. + """ + + @classmethod + def update(cls): + pass + + def run(self): + # lookup check for url results into ipqs database + lookup_result = self._make_request( + endpoint=self.lookup_endpoint, + method="POST", + _api_key=self._ipqs_api_key, + data={"url": self.observable_name}, + ) + if lookup_result.get("status", False) == "cached": + lookup_result.pop("update_url", None) + return lookup_result + + # sending url for scan + scan_result = self._make_request( + endpoint=self.scan_endpoint, + method="POST", + _api_key=self._ipqs_api_key, + data={"url": self.observable_name}, + ) + # waiting for results for with request id of scanned results + result = self._poll_for_report( + endpoint=self.postback_endpoint, + _api_key=self._ipqs_api_key, + request_id=scan_result.get("request_id"), + ) + result.pop("update_url", None) + return result diff --git a/api_app/analyzers_manager/observable_analyzers/netlas.py b/api_app/analyzers_manager/observable_analyzers/netlas.py index 57fe7b5ee9..05de3f2498 100644 --- a/api_app/analyzers_manager/observable_analyzers/netlas.py +++ b/api_app/analyzers_manager/observable_analyzers/netlas.py @@ -32,5 +32,9 @@ def run(self): except requests.RequestException as e: raise AnalyzerRunException(e) - result = response.json()["items"][0]["data"] - return result + # an IP with no whois record returns an empty "items" list, which must + # not crash with an IndexError when accessing the first element + items = response.json().get("items") + if not items: + raise AnalyzerRunException(f"No Netlas whois data found for {self.observable_name}") + return items[0]["data"] diff --git a/api_app/analyzers_manager/observable_analyzers/rdap.py b/api_app/analyzers_manager/observable_analyzers/rdap.py new file mode 100644 index 0000000000..47cfafb833 --- /dev/null +++ b/api_app/analyzers_manager/observable_analyzers/rdap.py @@ -0,0 +1,76 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import ipaddress +from urllib.parse import urlparse + +import requests + +from api_app.analyzers_manager import classes +from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification + + +class Rdap(classes.ObservableAnalyzer): + """Query the public RDAP bootstrap (https://rdap.org) for an observable's + registration data. + + RDAP (Registration Data Access Protocol, RFC 9082/9083) is the IETF-standard, + free and unauthenticated successor to WHOIS. It returns structured JSON + describing the registration of IP addresses, domains, and URLs (resolved to + their host). The rdap.org bootstrap redirects each query to the authoritative + RDAP server for the object. + """ + + url: str = "https://rdap.org" + + def update(self) -> bool: + # No local data to refresh; rdap.org is queried live on every run. + return False + + @staticmethod + def _path_for_host(host: str) -> str: + """Pick the RDAP endpoint for a host: ``ip/`` for an IP literal + (the host of a URL can be one), ``domain/`` otherwise.""" + try: + ipaddress.ip_address(host) + except ValueError: + return f"domain/{host}" + return f"ip/{host}" + + def run(self): + if self.observable_classification == Classification.IP: + path = f"ip/{self.observable_name}" + elif self.observable_classification == Classification.DOMAIN: + path = f"domain/{self.observable_name}" + elif self.observable_classification == Classification.URL: + hostname = urlparse(self.observable_name).hostname + if not hostname: + raise AnalyzerRunException(f"unable to extract a hostname from URL {self.observable_name}") + path = self._path_for_host(hostname) + else: + raise AnalyzerRunException( + f"{self.observable_classification} is not a supported observable type " + "for RDAP (supported: ip, domain, url)" + ) + + try: + response = requests.get( + f"{self.url}/{path}", + headers={"Accept": "application/rdap+json"}, + timeout=10, + ) + # RDAP returns 404 when the registry holds no record for the object; + # treat that as a clean negative result rather than an error. + if response.status_code == 404: + return {"found": False} + response.raise_for_status() + except requests.RequestException as e: + raise AnalyzerRunException(str(e)) from e + + try: + result = response.json() + except ValueError as e: + raise AnalyzerRunException(f"RDAP server returned a non-JSON response: {e}") from e + result["found"] = True + return result diff --git a/api_app/analyzers_manager/observable_analyzers/yeti.py b/api_app/analyzers_manager/observable_analyzers/yeti.py index 3c5d8b7185..68a63a74db 100644 --- a/api_app/analyzers_manager/observable_analyzers/yeti.py +++ b/api_app/analyzers_manager/observable_analyzers/yeti.py @@ -4,6 +4,7 @@ import requests from api_app.analyzers_manager import classes +from api_app.analyzers_manager.exceptions import AnalyzerRunException class YETI(classes.ObservableAnalyzer): @@ -19,10 +20,30 @@ def run(self): "query": {"value": self._job.analyzable.name}, "count": self.results_count, } - headers = {"Accept": "application/json", "X-Api-Key": self._api_key_name} + + # auth + auth_url = f"{self._url_key_name}/api/v2/auth/api-token" + auth_headers = {"x-yeti-apikey": self._api_key_name, "User-Agent": "IntelOwl"} + + try: + auth_resp = requests.post( + url=auth_url, + headers=auth_headers, + verify=self.verify_ssl, + timeout=60, + ) + auth_resp.raise_for_status() + access_token = auth_resp.json().get("access_token") + + if not access_token: + raise AnalyzerRunException("Failed to obtain access token from YETI.") + except requests.RequestException as e: + raise AnalyzerRunException(f"YETI Auth Request failed: {e}") + + headers = {"Accept": "application/json", "Authorization": f"Bearer {access_token}"} if self._url_key_name and self._url_key_name.endswith("/"): self._url_key_name = self._url_key_name[:-1] - url = f"{self._url_key_name}/api/v2/observables/search/" + url = f"{self._url_key_name}/api/v2/observables/search" # search for observables resp = requests.post( diff --git a/api_app/analyzers_manager/repo_downloader.sh b/api_app/analyzers_manager/repo_downloader.sh index e953684ce0..684f7f2fa2 100755 --- a/api_app/analyzers_manager/repo_downloader.sh +++ b/api_app/analyzers_manager/repo_downloader.sh @@ -32,11 +32,38 @@ curl -fSL "$DNSTWIST_RAW_BASE/common_tlds.dict" \ # download exiftool # https://exiftool.org/install.html#Unix -mkdir exiftool_download -cd exiftool_download || exit -version=$(curl https://exiftool.org/ver.txt) +# Define directories for clarity +DOWNLOAD_DIR="exiftool_download" +DEPLOY_DIR="/opt/deploy/exiftool_download" + +# 1. Create and enter directory +mkdir -p "$DOWNLOAD_DIR" || { echo "Failed to create directory"; exit 1; } +cd "$DOWNLOAD_DIR" || { echo "Failed to enter directory"; exit 1; } + +# 2. Get version with error handling +version=$(curl -s https://exiftool.org/ver.txt) +if [[ -z "$version" ]]; then + echo "Error: Could not retrieve version number." + exit 1 +fi echo "$version" >> exiftool_version.txt -wget "https://exiftool.org/Image-ExifTool-$version.tar.gz" -gzip -dc "Image-ExifTool-$version.tar.gz" | tar -xf - -cd "Image-ExifTool-$version" || exit -chown -R www-data:www-data /opt/deploy/exiftool_download + +# 3. Handle wget errors +# -q: quiet, -O: output file. +# Checking if the file actually exists after the command +if ! wget -q "https://exiftool.org/Image-ExifTool-$version.tar.gz"; then + echo "Error: Failed to download ExifTool version $version." + exit 1 +fi + +# 4. Handle extraction errors +if ! gzip -dc "Image-ExifTool-$version.tar.gz" | tar -xf -; then + echo "Error: Failed to extract files." + exit 1 +fi + +# 5. Handle directory navigation and permissions +cd "Image-ExifTool-$version" || { echo "Failed to enter extracted directory"; exit 1; } +chown -R www-data:www-data "$DEPLOY_DIR" || { echo "Warning: chown failed (check sudo permissions)"; } + +echo "ExifTool $version deployed successfully." diff --git a/api_app/chatbot_manager/__init__.py b/api_app/chatbot_manager/__init__.py new file mode 100644 index 0000000000..acb99e9651 --- /dev/null +++ b/api_app/chatbot_manager/__init__.py @@ -0,0 +1,2 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. diff --git a/api_app/chatbot_manager/admin.py b/api_app/chatbot_manager/admin.py new file mode 100644 index 0000000000..1c3be26ecd --- /dev/null +++ b/api_app/chatbot_manager/admin.py @@ -0,0 +1,22 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from django.contrib import admin + +from api_app.chatbot_manager.models import ChatMessage, ChatSession + + +@admin.register(ChatSession) +class ChatSessionAdmin(admin.ModelAdmin): + list_display = ["pk", "user", "created_at", "updated_at"] + list_filter = ["user", "created_at"] + ordering = ["-created_at"] + search_fields = ["user__username"] + + +@admin.register(ChatMessage) +class ChatMessageAdmin(admin.ModelAdmin): + list_display = ["pk", "session", "role", "timestamp"] + list_filter = ["role", "timestamp"] + ordering = ["timestamp"] + search_fields = ["content"] diff --git a/api_app/chatbot_manager/agent/__init__.py b/api_app/chatbot_manager/agent/__init__.py new file mode 100644 index 0000000000..acb99e9651 --- /dev/null +++ b/api_app/chatbot_manager/agent/__init__.py @@ -0,0 +1,2 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. diff --git a/api_app/chatbot_manager/agent/agent.py b/api_app/chatbot_manager/agent/agent.py new file mode 100644 index 0000000000..88927d6e74 --- /dev/null +++ b/api_app/chatbot_manager/agent/agent.py @@ -0,0 +1,87 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from pathlib import Path + +from django.conf import settings +from langchain.agents import AgentExecutor, create_tool_calling_agent +from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder +from langchain_ollama import ChatOllama + +from .tools import build_tools + +# The system prompt lives in its own text file so it is readable, testable, and +# editable without touching Python code. {page_context} is appended separately by +# the prompt template below so the file stays self-contained (no interpolation). +_SYSTEM_PROMPT = Path(__file__).parent.joinpath("system_prompt.txt").read_text(encoding="utf-8").strip() + +# The agent uses Ollama's native tool-calling API (`llm.bind_tools`, wired by +# `create_tool_calling_agent`), so the prompt carries no rendered tool list and no ReAct +# Thought/Action/Final Answer text scaffolding: the model emits structured tool calls and the +# executor loops tool call -> observation under the hood. `chat_history` and `agent_scratchpad` +# are message lists (MessagesPlaceholder), not pre-rendered text. +PROMPT = ChatPromptTemplate.from_messages( + [ + ("system", _SYSTEM_PROMPT + "\n\n{page_context}"), + MessagesPlaceholder("chat_history"), + ("human", "{input}"), + MessagesPlaceholder("agent_scratchpad"), + ] +) + +# One iteration = one round of tool calls + observation. Real questions resolve in 1-3 rounds +# (analyze_observable's confirm flow spans two turns, not two iterations), so 6 leaves room for +# a retry after a failed call while force-stopping a looping model well before the Celery task's +# 300s soft time limit would on CPU. +_MAX_AGENT_ITERATIONS = 6 + +# What AgentExecutor returns as "output" when max_iterations force-stops the run (the +# multi-action agent's return_stopped_response, langchain 0.3.25). Callers compare against this +# to turn a forced stop into a user-facing error instead of persisting the canned framework +# string as an assistant message. +AGENT_STOPPED_OUTPUT = "Agent stopped due to max iterations." + +# The rendered prompt (system prompt + the 10 bound tool schemas) is already ~2.2k tokens +# before any history: Ollama's default 2048-token context window silently truncates it from +# the start (observed live: "truncating input prompt" limit=2048 prompt=2174), dropping the +# system prompt and most tool schemas and wrecking tool selection. 8192 fits prompt + history +# + tool observations comfortably and keeps the prompt prefix stable across iterations, so +# follow-up rounds hit Ollama's KV prefix cache instead of re-evaluating everything. +_NUM_CTX = 8192 + + +def build_agent_executor(user, streaming: bool = False) -> AgentExecutor: + """Build a tool-calling agent executor scoped to `user`. + + `ChatOllama` is the local LLM; `create_tool_calling_agent` binds the tools to the model + through the native tool-calling API — there is no text format to parse, which is what made + the previous string-ReAct loop unreliable on small local models (they rarely emit a + parseable `Final Answer:` line). `AgentExecutor` runs the tool-call -> observation loop + until the model replies with plain text. `handle_parsing_errors=True` feeds an unparseable + model output back as an observation instead of raising (it does NOT cover schema-invalid + tool *arguments*, which raise from the tool itself and surface through the caller's generic + error handling); `max_iterations` bounds a looping model (`early_stopping_method` stays at + its default "force" — runnable agents support no other value in langchain 0.3, "generate" + raises ValueError). + + `streaming=True` makes `ChatOllama` emit token-level callbacks so the WebSocket path can + stream the answer live. No callbacks are bound to the model here: the caller attaches them + per run (`executor.invoke(..., config={"callbacks": [...]})`) so they also receive the + agent's tool actions, which originate from the executor and not from the LLM. + """ + llm = ChatOllama( + model=settings.OLLAMA_MODEL, + base_url=settings.OLLAMA_BASE_URL, + temperature=0, + num_ctx=_NUM_CTX, + streaming=streaming, + ) + tools = build_tools(user=user) + agent = create_tool_calling_agent(llm, tools, PROMPT) + return AgentExecutor( + agent=agent, + tools=tools, + verbose=False, + handle_parsing_errors=True, + max_iterations=_MAX_AGENT_ITERATIONS, + ) diff --git a/api_app/chatbot_manager/agent/context.py b/api_app/chatbot_manager/agent/context.py new file mode 100644 index 0000000000..c0c40d5f91 --- /dev/null +++ b/api_app/chatbot_manager/agent/context.py @@ -0,0 +1,47 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import re +from urllib.parse import urlparse + +# Fixed hint templates. context_url is client-supplied, so we render ONLY a validated integer id +# into these constants — the raw URL text never reaches the prompt. A crafted URL such as +# /jobs/42?x=ignore+previous+instructions therefore yields exactly "...job #42..." and nothing +# else, which neutralises prompt injection via the URL. +_JOB_CONTEXT = "The user is currently viewing job #{id} in the IntelOwl UI." +_INVESTIGATION_CONTEXT = "The user is currently viewing investigation #{id} in the IntelOwl UI." + +# These regexes mirror React Router path definitions in +# frontend/src/components/Routes.jsx. When those routes change, these patterns +# must be updated in lockstep — otherwise context injection silently stops +# recognising job/investigation pages. The coupling is enforced by +# test_regexes_match_frontend_route_definitions in test_context.py. +# +# /jobs/:id (Routes.jsx:157) +# /jobs/:id/:section (Routes.jsx:166) +# /jobs/:id/:section/:subSection (Routes.jsx:178) +# /jobs/:id/comments (Routes.jsx:186) +# /investigation/:id (Routes.jsx:243) +# +# Only the numeric id is captured; any trailing segments are ignored. +_JOB_RE = re.compile(r"^/jobs/(\d+)(?:/|$)") +_INVESTIGATION_RE = re.compile(r"^/investigation/(\d+)(?:/|$)") + + +def derive_page_context(context_url: str) -> str: + """Map the page the user is on (the WebSocket context_url) to a compact prompt hint. + + Returns a fixed hint for a job or investigation detail page, or "" for anything else + (non-entity page, empty, or unparseable). Only a validated integer id is extracted, so the + URL text cannot inject prompt instructions. + """ + if not context_url: + return "" + path = urlparse(context_url).path + match = _JOB_RE.match(path) + if match: + return _JOB_CONTEXT.format(id=match.group(1)) + match = _INVESTIGATION_RE.match(path) + if match: + return _INVESTIGATION_CONTEXT.format(id=match.group(1)) + return "" diff --git a/api_app/chatbot_manager/agent/memory.py b/api_app/chatbot_manager/agent/memory.py new file mode 100644 index 0000000000..8c3fadb828 --- /dev/null +++ b/api_app/chatbot_manager/agent/memory.py @@ -0,0 +1,44 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from langchain_core.chat_history import BaseChatMessageHistory +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage + + +class DjangoChatMessageHistory(BaseChatMessageHistory): + """LangChain chat history backed by the Django ChatMessage model. + + Implements the BaseChatMessageHistory contract so a ChatSession's messages can be + used as an agent's conversation memory: `messages` reads, `add_message` writes + (via the inherited add_user_message/add_ai_message helpers), `clear` resets. + """ + + def __init__(self, session): + """Bind this history to a single ChatSession (all reads/writes scope to it).""" + self.session = session + + @property + def messages(self) -> list[BaseMessage]: + """Return the session's messages, oldest first, as LangChain message objects.""" + from api_app.chatbot_manager.models import ChatMessage + + result = [] + for msg in ChatMessage.objects.filter(session=self.session).order_by("timestamp"): + if msg.role == ChatMessage.Role.USER: + result.append(HumanMessage(content=msg.content)) + else: + result.append(AIMessage(content=msg.content)) + return result + + def add_message(self, message: BaseMessage) -> None: + """Persist one message, mapping its LangChain type to the ChatMessage role.""" + from api_app.chatbot_manager.models import ChatMessage + + role = ChatMessage.Role.USER if isinstance(message, HumanMessage) else ChatMessage.Role.ASSISTANT + ChatMessage.objects.create(session=self.session, role=role, content=message.content) + + def clear(self) -> None: + """Delete every message in the session (resets the conversation).""" + from api_app.chatbot_manager.models import ChatMessage + + ChatMessage.objects.filter(session=self.session).delete() diff --git a/api_app/chatbot_manager/agent/streaming.py b/api_app/chatbot_manager/agent/streaming.py new file mode 100644 index 0000000000..91e35f745f --- /dev/null +++ b/api_app/chatbot_manager/agent/streaming.py @@ -0,0 +1,85 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +"""LangChain callback that streams a chat turn to the WebSocket as it runs. + +Attached at run level (``executor.invoke(..., config={"callbacks": [handler]})``) so it sees +both the LLM token stream and the agent's tool actions, then pushes them onto the per-user +channel group via ``group_send``. Running inside the (synchronous) Celery worker means there is +no event loop, so ``async_to_sync`` is the right bridge to the async channel layer — the same +pattern ``JobConsumer.serialize_and_send_job`` already uses. +""" + +import json +import logging + +from asgiref.sync import async_to_sync +from channels.layers import get_channel_layer +from langchain_core.callbacks.base import BaseCallbackHandler + +from api_app.chatbot_manager.events import ( + ActionRequiredEvent, + ChatEvent, + StatusEvent, + TokenEvent, + chat_group_for_user, +) + +logger = logging.getLogger(__name__) + + +class ChatStreamingCallbackHandler(BaseCallbackHandler): + """Streams the answer text token-by-token, plus a status event per tool call. + + With a tool-calling agent there is no ``Final Answer:`` marker to gate on: when the model + decides to call a tool, the streamed chunks carry structured tool-call deltas whose *text* + is empty (the call itself rides ``tool_call_chunks``), and when it answers it streams plain + text. Forwarding only non-empty text therefore streams the answer while keeping tool-call + payloads off the wire. + + This is a best-effort live view: a model may emit a short text preamble before a tool call, + which streams and is then superseded by the actual answer. The worker persists + ``result["output"]`` and sends it in the ``chat.end`` event, which the client treats as the + source of truth, so the stored/displayed answer is always correct. + """ + + def __init__(self, user_id: int, session_id: int, tool_names: set[str] | None = None): + self._group = chat_group_for_user(user_id) + self._session_id = session_id + # Only these tool names produce a chat.status; None means "emit any" (unit tests). + # Filtering by the real registry suppresses LangChain's internal "_Exception" pseudo-tool + # (and the raw error text) that handle_parsing_errors surfaces as agent actions. + self._tool_names = tool_names + self._channel_layer = get_channel_layer() + + def _emit(self, event: ChatEvent) -> None: + async_to_sync(self._channel_layer.group_send)(self._group, event.as_channel_message()) + + def on_llm_new_token(self, token: str, **kwargs) -> None: + # Tool-call deltas surface here with empty text; only real answer text is forwarded. + if token: + self._emit(TokenEvent(self._session_id, token)) + + def on_agent_action(self, action, **kwargs) -> None: + # Fired when the agent picks a tool; action.tool is the clean tool name. Skip anything + # that isn't a registered tool (e.g. the "_Exception" parse-error pseudo-tool, whose + # "tool" is the raw malformed model output) so internals don't leak as chat.status. + tool_name = getattr(action, "tool", "") or "" + if not tool_name or (self._tool_names is not None and tool_name not in self._tool_names): + return + self._emit(StatusEvent(self._session_id, tool_name)) + + def on_tool_end(self, output, **kwargs) -> None: + # analyze_observable returns an envelope carrying a one-time pending_id; surface it as a + # structured action so the frontend can render a Confirm button. Other tools have no + # pending_id, so this is a no-op for them. + text = getattr(output, "content", output) + try: + data = json.loads(text) + except (TypeError, ValueError): + logger.warning("on_tool_end: cannot parse tool output as JSON (%s)", type(output).__name__) + return + if isinstance(data, dict) and data.get("pending_id"): + self._emit(ActionRequiredEvent(self._session_id, data["pending_id"], data.get("plan") or {})) + else: + logger.debug("on_tool_end: no pending_id in tool output") diff --git a/api_app/chatbot_manager/agent/system_prompt.txt b/api_app/chatbot_manager/agent/system_prompt.txt new file mode 100644 index 0000000000..32d200b3c6 --- /dev/null +++ b/api_app/chatbot_manager/agent/system_prompt.txt @@ -0,0 +1,25 @@ +[Role] +You are IntelOwl AI, a concise assistant for the IntelOwl threat intelligence platform. +Answer with short, data-driven summaries. Always cite the tools you used at the end. + +[Tools — when to use each] +- search_jobs: find jobs by observable value, tag, or md5. Use for "show me jobs", "find jobs for domain X". +- get_job_details: full detail of ONE job by id. Use after search_jobs, or when the user asks "details of job #N". +- summarize_job: high-level summary of ONE job (status, analyzers, findings). Use for "summarize job #N". +- list_investigations: browse investigations with optional status/name filters. Use for "show my investigations", "what investigations are open?". +- get_investigation_tree: full tree of ONE investigation by id with all its jobs. Use after list_investigations. +- summarize_investigation: high-level summary of ONE investigation (status, job breakdown). Use for "summarize investigation #N". +- get_data_model: raw data model of ONE job. Rarely needed — only when the user explicitly asks for raw data or the data model. +- list_analyzers: list available analyzers for an observable type (domain, ip, url, hash, generic). Use for "what analyzers can I use for a domain?". +- recommend_playbook: recommend a playbook to analyze an observable. Use for "what playbook for 1.1.1.1?". +- analyze_observable: PREVIEW a real analysis (it never launches one itself). Call it to validate the request and return a plan; then tell the user to approve it with the Confirm button in the chat panel. Never claim you started an analysis. + +[Rules] +- Only access data belonging to the current user. Never reveal other users' data. +- Call the right tool instead of guessing. Say so when a tool returns no data. +- When a tool needs a value from a previous tool's result (an id, an md5), pass that actual value — never a placeholder like . +- analyze_observable only previews; never tell the user an analysis has started — they approve it themselves via the Confirm button. + +[Response style] +- One paragraph unless the user asks for a list or breakdown. +- Mention which tools you called at the end: "(used: search_jobs, summarize_job)". diff --git a/api_app/chatbot_manager/agent/tools/__init__.py b/api_app/chatbot_manager/agent/tools/__init__.py new file mode 100644 index 0000000000..ec9f3b8248 --- /dev/null +++ b/api_app/chatbot_manager/agent/tools/__init__.py @@ -0,0 +1,55 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from ._common import on_invalid_tool_args +from .analyze_observable import make_analyze_observable_tool +from .get_data_model import make_get_data_model_tool +from .get_investigation_tree import make_get_investigation_tree_tool +from .get_job_details import make_get_job_details_tool +from .list_analyzers import make_list_analyzers_tool +from .list_investigations import make_list_investigations_tool +from .recommend_playbook import make_recommend_playbook_tool +from .search_jobs import make_search_jobs_tool +from .summarize_investigation import make_summarize_investigation_tool +from .summarize_job import make_summarize_job_tool + + +def build_tools(user) -> list: + """Build the agent's tools bound to a single user. + + Each tool is produced by a factory that closes over `user`, so every ORM query the + agent can run is hard-scoped to that user's data: multi-tenancy is enforced at build + time and cannot be widened by anything the LLM emits. Job, investigation, and playbook + tools scope on `visible_for_user` (owner + same-org AMBER/RED + globally-visible + CLEAR/GREEN), matching the REST viewsets / UI. Analyzer configs are global plugin + definitions, so `list_analyzers` does not scope by owner -- it lists the enabled + analyzers and exposes per-user readiness through a `runnable` flag instead. The single + action tool `analyze_observable` can launch a real analysis; it is confirm-gated (a preview + unless `confirm=True`) and creates jobs owned by `user`. Every tool returns a string + (LangChain feeds it back to the model as the tool-call observation); see each tool + for its shape. + + Every tool also gets `handle_validation_error` set so that a schema-invalid argument the model + emits (e.g. the literal placeholder ``) becomes a recoverable observation instead of an + exception that kills the turn (see `on_invalid_tool_args`). + """ + tools = [ + make_search_jobs_tool(user), + make_get_job_details_tool(user), + make_summarize_job_tool(user), + make_list_investigations_tool(user), + make_get_investigation_tree_tool(user), + make_summarize_investigation_tool(user), + make_get_data_model_tool(user), + make_list_analyzers_tool(user), + make_recommend_playbook_tool(user), + make_analyze_observable_tool(user), + ] + # A schema-invalid tool argument (e.g. the model passing the literal "" instead of a + # real id) raises a pydantic ValidationError inside BaseTool.run, before the tool body -- and is + # NOT covered by the executor's handle_parsing_errors (model output only). Returning it as an + # observation makes a bad argument recoverable (the agent retries with the real value) instead + # of killing the turn. Set centrally so every tool shares the behavior. + for chatbot_tool in tools: + chatbot_tool.handle_validation_error = on_invalid_tool_args + return tools diff --git a/api_app/chatbot_manager/agent/tools/_common.py b/api_app/chatbot_manager/agent/tools/_common.py new file mode 100644 index 0000000000..1143c51a57 --- /dev/null +++ b/api_app/chatbot_manager/agent/tools/_common.py @@ -0,0 +1,44 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +"""Helpers shared by the chatbot agent tools (this module is not itself a tool).""" + +# Hard cap on the number of results any single tool returns to the LLM, so one call can't pull +# an unbounded list into the prompt. Shared here so every tool advertises the same ceiling. +MAX_RESULTS = 50 + + +def clamp_limit(limit, errors: list) -> int: + """Clamp an LLM-supplied result limit into ``[1, MAX_RESULTS]``. + + Tool arguments are untrusted, so the raw ``limit`` is coerced to ``int`` and bounded. When + the requested value is above the cap, a note is appended to ``errors`` so a truncated list is + never silent (the lower bound is enforced too, but a non-positive request is not worth a + warning). Returns the clamped limit. + """ + requested_limit = int(limit) + if requested_limit > MAX_RESULTS: + errors.append( + f"Requested limit {requested_limit} exceeds the maximum {MAX_RESULTS}; " + f"returning at most {MAX_RESULTS} results." + ) + return max(1, min(requested_limit, MAX_RESULTS)) + + +def on_invalid_tool_args(error: Exception) -> str: + """Observation returned when an LLM tool call fails the tool's argument schema. + + Set as ``handle_validation_error`` on every built tool (see ``build_tools``). The pydantic + ``ValidationError`` is raised in ``BaseTool._parse_input`` *before* the tool body runs, so it is + not covered by ``AgentExecutor(handle_parsing_errors=...)`` (which only feeds back the model's + unparseable *output*). Returning the error as an observation -- instead of letting it propagate + and kill the whole turn as ``UNAVAILABLE`` -- lets the agent retry with the real value; if it + keeps emitting a bad value the run force-stops at ``max_iterations`` rather than crashing. The + message surfaces the failure and steers the model to substitute the actual value from a previous + tool result, not a placeholder. + """ + return ( + f"Invalid tool arguments: {error}. " + "Use the actual value from a previous tool result (for example a numeric id), " + "not a placeholder such as ''. Then call the tool again." + ) diff --git a/api_app/chatbot_manager/agent/tools/analyze_observable.py b/api_app/chatbot_manager/agent/tools/analyze_observable.py new file mode 100644 index 0000000000..c9c48c0257 --- /dev/null +++ b/api_app/chatbot_manager/agent/tools/analyze_observable.py @@ -0,0 +1,92 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from types import SimpleNamespace + +from langchain_core.tools import tool + +from api_app.chatbot_manager.pending_action import create_pending_analysis +from api_app.chatbot_manager.serializers.analyze_observable import ( + AnalyzeObservableResultSerializer, + flatten_errors, +) +from api_app.choices import TLP +from api_app.playbooks_manager.models import PlaybookConfig +from api_app.serializers.job import ObservableAnalysisSerializer + + +def make_analyze_observable_tool(user): + # Built per-request and closed over `user`. This is the only action-capable tool, but it now + # NEVER launches: it validates and returns a `plan` plus a one-time `pending_id`. The actual + # launch happens only when the user confirms via the chat panel (POST /api/chatbot/analysis/ + # confirm), so a misbehaving model can never start an analysis on its own (guardrail M-1). + @tool("analyze_observable") + def analyze_observable( + observable_name: str, + playbook: str = "", + analyzers: str = "", + tlp: str = TLP.CLEAR.value, + ) -> str: + """Preview an IntelOwl analysis of an observable (IP, domain, URL, hash). + + This tool does NOT start anything: it validates the request and returns the `plan` that + would run plus a `pending_id`. Tell the user to approve it with the Confirm button in the + chat panel; you cannot launch the analysis yourself. + + Args: + observable_name: The observable to analyze (an IP, domain, URL or hash). + playbook: Optional playbook name (must be visible to you). Mutually exclusive with analyzers. + analyzers: Optional COMMA-SEPARATED analyzer names. Mutually exclusive with playbook. + tlp: TLP level (CLEAR, GREEN, AMBER, RED; default CLEAR). Only filters which plugins run. + + Returns: + JSON string {"errors": [...], "plan": {...} | null, "pending_id": "..." | null}. + """ + shim = SimpleNamespace(user=user) + if playbook: + # ISOLATION GUARD: ObservableAnalysisSerializer resolves playbook_requested via + # PlaybookConfig.objects.all() with no visibility filter; scope it here first so another + # org's private playbook can't leak into the plan. + if not PlaybookConfig.objects.visible_for_user(user).filter(name=playbook).exists(): + return AnalyzeObservableResultSerializer( + { + "errors": [f"Playbook '{playbook}' not found or not visible to you."], + "plan": None, + "pending_id": None, + } + ).to_json() + + data = {"observable_name": observable_name, "tlp": tlp} + if playbook: + data["playbook_requested"] = playbook + analyzers_list = [a.strip() for a in analyzers.split(",") if a.strip()] + if analyzers_list: + data["analyzers_requested"] = analyzers_list + + serializer = ObservableAnalysisSerializer(data=data, context={"request": shim}) + if not serializer.is_valid(raise_exception=False): + return AnalyzeObservableResultSerializer( + {"errors": flatten_errors(serializer.errors), "plan": None, "pending_id": None} + ).to_json() + + validated = serializer.validated_data + plan = { + "observable_name": validated["observable_name"], + "classification": validated["observable_classification"], + "tlp": validated["tlp"], + "playbook": validated["playbook_requested"].name if validated.get("playbook_requested") else None, + "analyzers": [analyzer.name for analyzer in validated["analyzers_to_execute"]], + "connectors": [connector.name for connector in validated["connectors_to_execute"]], + "skipped": list(validated.get("warnings", [])), + } + # Store the RAW inputs (re-validated at confirm time); the model cannot launch -- only a + # user POST of this pending_id to the confirm endpoint can. + pending_id = create_pending_analysis( + user.id, + {"observable_name": observable_name, "tlp": tlp, "playbook": playbook, "analyzers": analyzers}, + ) + return AnalyzeObservableResultSerializer( + {"errors": [], "plan": plan, "pending_id": pending_id} + ).to_json() + + return analyze_observable diff --git a/api_app/chatbot_manager/agent/tools/get_data_model.py b/api_app/chatbot_manager/agent/tools/get_data_model.py new file mode 100644 index 0000000000..de0b321cbb --- /dev/null +++ b/api_app/chatbot_manager/agent/tools/get_data_model.py @@ -0,0 +1,42 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from langchain_core.tools import tool + +from api_app.chatbot_manager.serializers.data_model import DataModelResultSerializer +from api_app.models import Job + + +def make_get_data_model_tool(user): + # Built per-request and closed over `user`: scoped with visible_for_user (owner + + # same-org AMBER/RED + globally-visible CLEAR/GREEN), matching the other job tools and + # the UI (multi-tenancy enforced here). LangChain feeds a tool's return value back as + # the tool-call observation, so it must be a string: we return a JSON-serialized envelope. + @tool("get_data_model") + def get_data_model(job_id: int) -> str: + """Get the aggregated data model of an IntelOwl job by its numeric ID. + + The data model is IntelOwl's normalized, analyzer-agnostic view of a job's + observable (evaluation, reliability, tags, kill-chain phase, plus type-specific + fields for domains/IPs/files). It is empty until the analysis pipeline has built + it, so an empty object is a valid result. + + Args: + job_id: The numeric ID of the job. + + Returns: + JSON string with shape {"errors": [...], "data_model": {...}}. + """ + try: + job = Job.objects.visible_for_user(user).get(pk=job_id) + except Job.DoesNotExist: + return DataModelResultSerializer( + {"errors": [f"Job with ID {job_id} not found or not accessible."], "data_model": {}} + ).to_json() + + # Mirror the public JobSerializer.get_data_model: the model serializes itself + # (polymorphic across Domain/IP/File); `data_model` is a nullable GenericForeignKey. + data_model = job.data_model.serialize() if job.data_model else {} + return DataModelResultSerializer({"errors": [], "data_model": data_model}).to_json() + + return get_data_model diff --git a/api_app/chatbot_manager/agent/tools/get_investigation_tree.py b/api_app/chatbot_manager/agent/tools/get_investigation_tree.py new file mode 100644 index 0000000000..27912b7e7d --- /dev/null +++ b/api_app/chatbot_manager/agent/tools/get_investigation_tree.py @@ -0,0 +1,123 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from dataclasses import asdict, dataclass, field +from typing import List, Optional + +from langchain_core.tools import tool + +from api_app.chatbot_manager.serializers.investigation import InvestigationTreeResultSerializer +from api_app.investigations_manager.models import Investigation + +# Bounds for the LLM-facing tree: keep the serialized payload small enough not to blow up +# the prompt regardless of how large an investigation is. +_MAX_DEPTH = 10 +_MAX_NODES = 200 + + +@dataclass +class _JobNode: + """One job in the LLM-facing tree: id, observable, status and its children.""" + + id: int + observable: Optional[str] + status: str + children: List["_JobNode"] = field(default_factory=list) + + +@dataclass +class _InvestigationTree: + """Compact job tree for an investigation. `truncated` is True when a cap was hit.""" + + id: int + name: str + status: str + jobs: List[_JobNode] = field(default_factory=list) + truncated: bool = False + + +def _node(job) -> _JobNode: + return _JobNode(id=job.pk, observable=getattr(job.analyzable, "name", None), status=job.status) + + +def _build_tree(investigation) -> _InvestigationTree: + """Assemble a compact job tree for an investigation, avoiding the treebeard N+1. + + A Job is a treebeard ``MP_Node``; walking it with ``get_children()`` recursively would + fire one query per node. Instead we fetch each root's whole subtree with a single + ``get_descendants()`` call and rebuild the nesting in Python from treebeard's + materialized ``path`` (a child's parent path is its own path minus the last step), so + there are no per-node queries. Depth and node count are capped to bound the payload. + """ + tree = _InvestigationTree( + id=investigation.pk, + name=investigation.name, + status=investigation.status, + ) + remaining = _MAX_NODES + + # `investigation.jobs` are the root jobs; their descendants live only in the tree. + for root in investigation.jobs.select_related("analyzable"): + if remaining <= 0: + tree.truncated = True + break + root_node = _node(root) + tree.jobs.append(root_node) + remaining -= 1 + # Index path -> node for every kept node, to link children to parents in O(1). + by_path = {root.path: root_node} + + # One query for the whole subtree, in path order (treebeard pre-order DFS), so a + # parent is always processed before its children. + for job in root.get_descendants().select_related("analyzable").order_by("path"): + if (job.depth - root.depth) > _MAX_DEPTH: + continue + parent = by_path.get(job.path[: -job.steplen]) + if parent is None: + # Ancestor was capped out (depth/budget); skip to avoid orphaning. + continue + if remaining <= 0: + tree.truncated = True + break + node = _node(job) + by_path[job.path] = node + parent.children.append(node) + remaining -= 1 + + return tree + + +def make_get_investigation_tree_tool(user): + # Built per-request and closed over `user`. The lookup is scoped with + # `visible_for_user` (owned + organization-shared investigations), so the LLM cannot + # reach an investigation the user can't see. Returns a string (the tool-call + # observation): a JSON-serialized envelope. + @tool("get_investigation_tree") + def get_investigation_tree(investigation_id: int) -> str: + """Get the job tree of an IntelOwl investigation by its numeric ID. + + Returns the investigation with its jobs as a nested tree (each node: id, + observable, status, children). Depth and node count are capped for large trees + (the `truncated` flag is set to true when the cap is hit). + + Args: + investigation_id: The numeric ID of the investigation. + + Returns: + JSON string with shape {"errors": [...], "investigation": {...} | null}. + """ + try: + investigation = Investigation.objects.visible_for_user(user).get(pk=investigation_id) + except Investigation.DoesNotExist: + return InvestigationTreeResultSerializer( + { + "errors": [f"Investigation with ID {investigation_id} not found or not accessible."], + "investigation": None, + } + ).to_json() + + return InvestigationTreeResultSerializer( + {"errors": [], "investigation": asdict(_build_tree(investigation))} + ).to_json() + + return get_investigation_tree diff --git a/api_app/chatbot_manager/agent/tools/get_job_details.py b/api_app/chatbot_manager/agent/tools/get_job_details.py new file mode 100644 index 0000000000..40d3519905 --- /dev/null +++ b/api_app/chatbot_manager/agent/tools/get_job_details.py @@ -0,0 +1,39 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from langchain_core.tools import tool + + +def make_get_job_details_tool(user): + # Built per-request and closed over `user`: the lookup is scoped with visible_for_user + # (owner + same-org AMBER/RED + globally-visible CLEAR/GREEN), matching the REST + # JobViewSet / UI (multi-tenancy enforced here). LangChain requires a tool to return a + # string (the tool-call observation), so we return a JSON-serialized envelope. + @tool("get_job_details") + def get_job_details(job_id: int) -> str: + """Get full details of an IntelOwl job by its numeric ID. + + Args: + job_id: The numeric ID of the job to retrieve. + + Returns: + JSON string with shape {"errors": [...], "job": {...} | null}. + """ + from api_app.chatbot_manager.serializers.job import JobDetailResultSerializer + from api_app.models import Job + + try: + job = ( + Job.objects.select_related("analyzable") + .prefetch_related("analyzerreports__config", "analyzers_to_execute", "tags") + .visible_for_user(user) + .get(pk=job_id) + ) + except Job.DoesNotExist: + return JobDetailResultSerializer( + {"errors": [f"Job with ID {job_id} not found or not accessible."], "job": None} + ).to_json() + + return JobDetailResultSerializer({"errors": [], "job": job}).to_json() + + return get_job_details diff --git a/api_app/chatbot_manager/agent/tools/list_analyzers.py b/api_app/chatbot_manager/agent/tools/list_analyzers.py new file mode 100644 index 0000000000..a2fb5df289 --- /dev/null +++ b/api_app/chatbot_manager/agent/tools/list_analyzers.py @@ -0,0 +1,72 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from langchain_core.tools import tool + +from api_app.analyzers_manager.constants import TypeChoices +from api_app.analyzers_manager.models import AnalyzerConfig +from api_app.chatbot_manager.agent.tools._common import clamp_limit +from api_app.chatbot_manager.serializers.analyzer import ListAnalyzersResultSerializer +from api_app.choices import Classification + + +def make_list_analyzers_tool(user): + # Built per-request and closed over `user`. Unlike the job/investigation tools, analyzer + # configs are GLOBAL plugin definitions (not user-owned), so the queryset is not scoped to + # the user's own rows; it lists the enabled observable analyzers and exposes per-user + # readiness through the `runnable` annotation instead. + # + # Why no `.filter(runnable=True)`: `annotate_runnable` (queryset.py) folds health-check + + # "all required parameters configured" on top of enabled/org-enabled, so a hard filter would + # silently drop every key-based analyzer (VirusTotal, ...) on a deploy without API keys -> + # near-empty lists and flaky tests. We list the enabled analyzers and surface readiness as a + # per-row flag instead, so the LLM can still say "this analyzer applies but isn't configured + # for you". `runnable` is False when the analyzer is disabled for the user's organization OR + # not fully configured/healthy. + @tool("list_analyzers") + def list_analyzers(observable_type: str = "", limit: int = 10) -> str: + """List the observable analyzers enabled on this IntelOwl instance. + + Each result carries a `runnable` flag: True means the analyzer is ready to run for you + (enabled for your organization and fully configured); False means it applies but is + currently disabled for your organization or missing required configuration (e.g. an API + key). + + Args: + observable_type: Filter to analyzers supporting this observable type. Valid values: + ip, url, domain, hash, generic. An unknown value is ignored and reported in + `errors`. + limit: Maximum number of results to return (default 10, max 50). + + Returns: + JSON string with shape {"errors": [...], "analyzers": [...]}. + """ + errors = [] + # Globally-enabled observable analyzers, annotated with per-user readiness. Org-disabled + # or unconfigured analyzers still appear, flagged `runnable=False`. + qs = AnalyzerConfig.objects.filter(type=TypeChoices.OBSERVABLE, disabled=False).annotate_runnable( + user + ) + + # The accepted observable types are an IntelOwl-core notion (every classification except + # FILE, which is the file-analysis path), so reuse the core helper instead of a local copy. + valid_types = Classification.observable_classifications() + if observable_type: + # The type string comes from the LLM; validate it against the enum so an invalid + # value surfaces a message instead of silently returning 0 results. + normalized = observable_type.strip().lower() + if normalized in set(valid_types): + # `observable_supported` is a ChoiceArrayField: `__contains` matches rows whose + # array includes the requested type. + qs = qs.filter(observable_supported__contains=[normalized]) + else: + valid = ", ".join(valid_types) + errors.append(f"Unknown observable_type '{observable_type}'; valid values are: {valid}.") + + # Treat the LLM-supplied limit as untrusted: clamp it and surface any capping in `errors`. + limit = clamp_limit(limit, errors) + qs = qs.order_by("name")[:limit] + + return ListAnalyzersResultSerializer({"errors": errors, "analyzers": qs}).to_json() + + return list_analyzers diff --git a/api_app/chatbot_manager/agent/tools/list_investigations.py b/api_app/chatbot_manager/agent/tools/list_investigations.py new file mode 100644 index 0000000000..7f099dd6e9 --- /dev/null +++ b/api_app/chatbot_manager/agent/tools/list_investigations.py @@ -0,0 +1,52 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from langchain_core.tools import tool + +from api_app.chatbot_manager.agent.tools._common import clamp_limit +from api_app.chatbot_manager.serializers.investigation import InvestigationsResultSerializer +from api_app.investigations_manager.choices import InvestigationStatusChoices +from api_app.investigations_manager.models import Investigation + + +def make_list_investigations_tool(user): + # Built per-request and closed over `user`: the queryset is scoped with + # `visible_for_user`, which returns the investigations the user owns AND those shared + # with their organization (`for_organization=True`). This is wider than the job tools' + # `user=user` scope on purpose -- investigations have an explicit org-sharing flag -- + # and the LLM can never widen it. LangChain feeds a tool's return value back as the + # tool-call observation, so it must be a string: we return a JSON-serialized envelope. + @tool("list_investigations") + def list_investigations(query: str = "", status: str = "", limit: int = 10) -> str: + """List IntelOwl investigations visible to you (owned or shared with your org). + + Args: + query: Filter by investigation name (case-insensitive partial match). + status: Filter by investigation status. Valid values: created, running, + concluded. An unknown value is ignored and reported in `errors`. + limit: Maximum number of results to return (default 10, max 50). + + Returns: + JSON string with shape {"errors": [...], "investigations": [...]}. + """ + errors = [] + qs = Investigation.objects.visible_for_user(user) + + if query: + qs = qs.filter(name__icontains=query) + if status: + # The status string comes from the LLM; validate it against the enum so an + # invalid value surfaces a message instead of silently returning 0 results. + normalized = status.strip().lower() + if normalized in set(InvestigationStatusChoices.values): + qs = qs.filter(status=normalized) + else: + valid = ", ".join(InvestigationStatusChoices.values) + errors.append(f"Unknown status '{status}'; valid values are: {valid}.") + + limit = clamp_limit(limit, errors) + qs = qs.order_by("-start_time")[:limit] + + return InvestigationsResultSerializer({"errors": errors, "investigations": qs}).to_json() + + return list_investigations diff --git a/api_app/chatbot_manager/agent/tools/recommend_playbook.py b/api_app/chatbot_manager/agent/tools/recommend_playbook.py new file mode 100644 index 0000000000..dd17bc9850 --- /dev/null +++ b/api_app/chatbot_manager/agent/tools/recommend_playbook.py @@ -0,0 +1,70 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from langchain_core.tools import tool + +from api_app.chatbot_manager.agent.tools._common import clamp_limit +from api_app.chatbot_manager.serializers.playbook import RecommendPlaybookResultSerializer +from api_app.choices import Classification +from api_app.playbooks_manager.models import PlaybookConfig + + +def make_recommend_playbook_tool(user): + # Built per-request and closed over `user`. Playbooks are owned / org-shared objects (unlike + # the global analyzer configs), so the queryset is scoped with `visible_for_user`, which + # returns the playbooks the user owns, the ones shared with their organization and the global + # ones (owner is null). This is a real visibility boundary -- another org's private playbook + # must not leak -- and the LLM can never widen it. + @tool("recommend_playbook") + def recommend_playbook(observable_name: str = "", classification: str = "", limit: int = 10) -> str: + """Suggest IntelOwl playbooks that can analyze a given observable. + + Returns the directly-launchable playbooks (those that can start an analysis on their own) + applicable to the observable's classification and visible to you. + + Args: + observable_name: The observable (IP, domain, URL, hash, ...). When `classification` + is not given, the classification is derived from this value. + classification: The observable classification to match playbooks against. Valid + values: ip, url, domain, hash, generic, file. If omitted it is derived from + `observable_name`; an unknown explicit value is reported in `errors`. + limit: Maximum number of results to return (default 10, max 50). + + Returns: + JSON string with shape {"errors": [...], "playbooks": [...]}. + """ + errors = [] + + if not observable_name.strip() and not classification.strip(): + errors.append("Provide either an observable_name or a classification.") + return RecommendPlaybookResultSerializer({"errors": errors, "playbooks": []}).to_json() + + if classification: + # Explicit classification comes from the LLM; validate it against the enum so an + # invalid value surfaces a message instead of silently returning 0 results. + classification = classification.strip().lower() + if classification not in set(Classification.values): + valid = ", ".join(Classification.values) + errors.append(f"Unknown classification '{classification}'; valid values are: {valid}.") + return RecommendPlaybookResultSerializer({"errors": errors, "playbooks": []}).to_json() + else: + # Reuse the same classification logic the real analysis pipeline uses, so the + # recommendation matches what an actual scan would pick. + classification = Classification.calculate_observable(observable_name) + + # Treat the LLM-supplied limit as untrusted: clamp it (so a broadly-supported + # classification can't flood the prompt) and surface any capping in `errors`. + limit = clamp_limit(limit, errors) + # `type` is a ChoiceArrayField of classifications: `__contains` matches playbooks declaring + # support for this one. `starting=True` = launchable on its own (not pivot-only). + # `prefetch_related` avoids the per-row N+1 from the analyzers/connectors SlugRelatedFields. + qs = ( + PlaybookConfig.objects.filter(type__contains=[classification], starting=True, disabled=False) + .visible_for_user(user) + .prefetch_related("analyzers", "connectors") + .order_by("name")[:limit] + ) + + return RecommendPlaybookResultSerializer({"errors": errors, "playbooks": qs}).to_json() + + return recommend_playbook diff --git a/api_app/chatbot_manager/agent/tools/search_jobs.py b/api_app/chatbot_manager/agent/tools/search_jobs.py new file mode 100644 index 0000000000..c26838e25e --- /dev/null +++ b/api_app/chatbot_manager/agent/tools/search_jobs.py @@ -0,0 +1,60 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from django.db import models +from langchain_core.tools import tool + +from api_app.chatbot_manager.agent.tools._common import clamp_limit +from api_app.choices import Status + + +def make_search_jobs_tool(user): + # Built per-request and closed over `user`: scoped with visible_for_user (owner + + # same-org AMBER/RED + globally-visible CLEAR/GREEN), matching the REST JobViewSet / UI. + # The LLM can never widen it. LangChain feeds a tool's return value back to the model + # as the tool-call observation, so it must be a string; we return a JSON-serialized + # envelope. + @tool("search_jobs") + def search_jobs(query: str = "", status: str = "", limit: int = 10) -> str: + """Search IntelOwl jobs by observable name, MD5, or status. + + Args: + query: Observable name or MD5 hash to search for (partial match supported). + status: Filter by job status (see api_app.choices.Status for valid values). + An unknown value is ignored and reported in `errors`. + limit: Maximum number of results to return (default 10, max 50). + + Returns: + JSON string with shape {"errors": [...], "jobs": [...]}. + """ + from api_app.chatbot_manager.serializers.job import SearchJobsResultSerializer + from api_app.models import Job + + errors = [] + limit = clamp_limit(limit, errors) + qs = ( + Job.objects.select_related("analyzable") + .prefetch_related("analyzers_to_execute") + .visible_for_user(user) + ) + + if query: + qs = qs.filter( + models.Q(analyzable__name__icontains=query) | models.Q(analyzable__md5__iexact=query) + ) + if status: + # The status string comes from the LLM; validate it against the enum so an + # invalid value surfaces a message instead of silently returning 0 results + # (mirrors list_investigations). + normalized = status.strip().lower() + if normalized in set(Status.values): + qs = qs.filter(status=normalized) + else: + valid = ", ".join(Status.values) + errors.append(f"Unknown status '{status}'; valid values are: {valid}.") + + qs = qs.order_by("-received_request_time")[:limit] + + return SearchJobsResultSerializer({"errors": errors, "jobs": qs}).to_json() + + return search_jobs diff --git a/api_app/chatbot_manager/agent/tools/summarize_investigation.py b/api_app/chatbot_manager/agent/tools/summarize_investigation.py new file mode 100644 index 0000000000..c544ec699d --- /dev/null +++ b/api_app/chatbot_manager/agent/tools/summarize_investigation.py @@ -0,0 +1,69 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from collections import Counter + +from langchain_core.tools import tool + +from api_app.chatbot_manager.serializers.investigation import SummarizeInvestigationResultSerializer +from api_app.investigations_manager.models import Investigation + + +def make_summarize_investigation_tool(user): + # Built per-request and closed over `user`. Scoped with `visible_for_user` (owned + + # organization-shared investigations). The payload is human-readable prose wrapped in + # the same envelope as the other tools (LangChain needs a string Observation). + @tool("summarize_investigation") + def summarize_investigation(investigation_id: int) -> str: + """Return a concise human-readable summary of an IntelOwl investigation. + + Includes the status, total jobs, a per-status breakdown of the jobs, TLP, tags and + the start/end times. + + Args: + investigation_id: The numeric ID of the investigation to summarize. + + Returns: + JSON string with shape {"errors": [...], "summary": "..." | null}. + """ + try: + investigation = Investigation.objects.visible_for_user(user).get(pk=investigation_id) + except Investigation.DoesNotExist: + return SummarizeInvestigationResultSerializer( + { + "errors": [f"Investigation with ID {investigation_id} not found or not accessible."], + "summary": None, + } + ).to_json() + + # Count jobs by status across the whole tree. `investigation.jobs` are the roots; + # the rest live in the treebeard tree, so we walk each subtree once (one query per + # root, status column only) rather than per node. + status_counts = Counter() + for root in investigation.jobs.all(): + status_counts[root.status] += 1 + for child_status in root.get_descendants().values_list("status", flat=True): + status_counts[child_status] += 1 + + # The total equals Investigation.total_jobs but is derived from the counts we + # already gathered, so we don't re-query the tree. + total_jobs = sum(status_counts.values()) + tags = [label for label in investigation.tags if label] + + lines = [ + f"Investigation #{investigation.pk}: {investigation.name}", + f" Status : {investigation.status}", + f" Total jobs : {total_jobs}", + f" TLP : {investigation.tlp}", + ] + if status_counts: + breakdown = ", ".join(f"{status}: {count}" for status, count in sorted(status_counts.items())) + lines.append(f" Job status : {breakdown}") + if tags: + lines.append(f" Tags : {', '.join(tags)}") + lines.append(f" Started : {investigation.start_time}") + lines.append(f" Ended : {investigation.end_time or 'N/A'}") + + return SummarizeInvestigationResultSerializer({"errors": [], "summary": "\n".join(lines)}).to_json() + + return summarize_investigation diff --git a/api_app/chatbot_manager/agent/tools/summarize_job.py b/api_app/chatbot_manager/agent/tools/summarize_job.py new file mode 100644 index 0000000000..444a9bd762 --- /dev/null +++ b/api_app/chatbot_manager/agent/tools/summarize_job.py @@ -0,0 +1,63 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from langchain_core.tools import tool + +from api_app.choices import ReportStatus + + +def make_summarize_job_tool(user): + # Built per-request and closed over `user`: the lookup is scoped with visible_for_user + # (owner + same-org AMBER/RED + globally-visible CLEAR/GREEN), matching the REST + # JobViewSet / UI (multi-tenancy enforced here). The payload is human-readable prose + # (meant to be relayed to the user) wrapped in the same envelope as the other tools. + @tool("summarize_job") + def summarize_job(job_id: int) -> str: + """Return a concise human-readable summary of an IntelOwl job. + + Args: + job_id: The numeric ID of the job to summarize. + + Returns: + JSON string with shape {"errors": [...], "summary": "..." | null}. + """ + from api_app.chatbot_manager.serializers.job import SummarizeJobResultSerializer + from api_app.models import Job + + try: + job = ( + Job.objects.select_related("analyzable") + .prefetch_related("analyzerreports__config", "analyzers_to_execute") + .visible_for_user(user) + .get(pk=job_id) + ) + except Job.DoesNotExist: + return SummarizeJobResultSerializer( + {"errors": [f"Job with ID {job_id} not found or not accessible."], "summary": None} + ).to_json() + + analyzers = list(job.analyzers_to_execute.values_list("name", flat=True)) + # `analyzerreports.*.status` uses ReportStatus (uppercase), distinct from the + # job-level Status enum: a report that did not succeed is considered failed here. + failed_reports = [ + r.config.name for r in job.analyzerreports.all() if r.status != ReportStatus.SUCCESS.value + ] + + lines = [ + f"Job #{job.pk}", + f" Observable : {job.analyzable.name} ({job.analyzable.classification})", + f" MD5 : {job.analyzable.md5}", + f" Status : {job.status}", + f" TLP : {job.tlp}", + f" Received : {job.received_request_time}", + f" Finished : {job.finished_analysis_time or 'N/A'}", + f" Analyzers : {', '.join(analyzers) or 'none'}", + ] + if job.errors: + lines.append(f" Errors : {'; '.join(job.errors[:3])}") + if failed_reports: + lines.append(f" Failed : {', '.join(failed_reports)}") + + return SummarizeJobResultSerializer({"errors": [], "summary": "\n".join(lines)}).to_json() + + return summarize_job diff --git a/api_app/chatbot_manager/apps.py b/api_app/chatbot_manager/apps.py new file mode 100644 index 0000000000..5fb305cc8a --- /dev/null +++ b/api_app/chatbot_manager/apps.py @@ -0,0 +1,8 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from django.apps import AppConfig + + +class ChatbotManagerConfig(AppConfig): + name = "api_app.chatbot_manager" diff --git a/api_app/chatbot_manager/consumers.py b/api_app/chatbot_manager/consumers.py new file mode 100644 index 0000000000..f56294002d --- /dev/null +++ b/api_app/chatbot_manager/consumers.py @@ -0,0 +1,123 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +"""WebSocket consumer for the chatbot. + +Mirrors JobConsumer (api_app/websocket.py): a synchronous JsonWebsocketConsumer that joins a +channel group on connect and relays group messages to the browser. Unlike JobConsumer it is +bidirectional — the client sends a chat message over the socket and the consumer enqueues the +Celery turn — and it groups by user (chat-) rather than by resource, because the fixed +URL ws/chat/?context_url=... carries no session id and a session belongs to exactly one user. +""" + +import logging +from urllib.parse import parse_qs + +from asgiref.sync import async_to_sync +from channels.generic.websocket import JsonWebsocketConsumer + +from api_app.chatbot_manager.events import ( + AckEvent, + ChatErrorDetail, + ErrorEvent, + chat_group_for_user, +) +from api_app.chatbot_manager.models import ChatSession +from api_app.chatbot_manager.rate_limit import _build_rate_limiter +from api_app.chatbot_manager.serializers.chat import MessageRequestSerializer +from api_app.chatbot_manager.tasks import process_chat_message + +logger = logging.getLogger(__name__) + + +class ChatConsumer(JsonWebsocketConsumer): + """Streams chat turns to the user and accepts inbound messages to start them. + + Auth is handled upstream by WSAuthMiddleware (anonymous users are rejected before reaching + here). The group key is the server-authenticated scope["user"].id, so a client can only ever + receive its own stream. + """ + + def connect(self) -> None: + user = self.scope["user"] + # context_url (the page the user is on) is captured and forwarded to the chat turn so the + # agent can resolve "this job/investigation" references (see derive_page_context in tasks.py). + self.context_url = self._parse_context_url() + self.group_name = chat_group_for_user(user.id) + self.accept() + async_to_sync(self.channel_layer.group_add)(self.group_name, self.channel_name) + logger.debug(f"user {user} connected to chat group {self.group_name}") + + def disconnect(self, close_code) -> None: + group_name = getattr(self, "group_name", None) + if group_name: + async_to_sync(self.channel_layer.group_discard)(group_name, self.channel_name) + logger.debug(f"chat ws disconnected (group={group_name}, code={close_code})") + + def receive_json(self, content, **kwargs) -> None: + """Validate one inbound message, resolve its session, and enqueue the agent turn. + + The frame is untrusted: it is validated/length-capped by + MessageRequestSerializer, and any supplied session_id is resolved scoped + to the connected user, so a client cannot drive another user's session. + Rate limiting is enforced before session resolution so a rate-limited + client cannot create sessions by sending messages that are then dropped. + """ + user = self.scope["user"] + serializer = MessageRequestSerializer(data=content) + if not serializer.is_valid(): + self.send_json(ErrorEvent(None, ChatErrorDetail.INVALID_MESSAGE.value).to_client()) + return + + limiter = _build_rate_limiter() + allowed, retry_after = limiter.allow(str(user.id)) + if not allowed: + self.send_json( + ErrorEvent(None, ChatErrorDetail.RATE_LIMITED.value, retry_after=retry_after).to_client() + ) + return + + session_id = serializer.validated_data.get("session_id") + message = serializer.validated_data["message"] + try: + session = self._resolve_session(user, session_id) + except ChatSession.DoesNotExist: + self.send_json(ErrorEvent(session_id, ChatErrorDetail.SESSION_NOT_FOUND.value).to_client()) + return + + limiter.increment(str(user.id)) + + # Ack the (possibly newly created) session id before the asynchronous stream begins. + self.send_json(AckEvent(session.id).to_client()) + process_chat_message.delay(session.id, message, user.id, self.context_url) + + @staticmethod + def _resolve_session(user, session_id) -> ChatSession: + """Return the user's existing session, or create a fresh one when none is given.""" + if session_id is None: + return ChatSession.objects.create(user=user) + return ChatSession.objects.get(pk=session_id, user=user) + + def _parse_context_url(self) -> str: + query_string = self.scope.get("query_string", b"").decode() + return parse_qs(query_string).get("context_url", [""])[0] + + # Group handlers: Channels maps an event "type" (e.g. "chat.token") to the same-named method + # with dots turned into underscores. Each one relays the payload the producer already built. + def chat_start(self, event) -> None: + self.send_json(event["payload"]) + + def chat_status(self, event) -> None: + self.send_json(event["payload"]) + + def chat_token(self, event) -> None: + self.send_json(event["payload"]) + + def chat_end(self, event) -> None: + self.send_json(event["payload"]) + + def chat_error(self, event) -> None: + self.send_json(event["payload"]) + + def chat_action_required(self, event) -> None: + self.send_json(event["payload"]) diff --git a/api_app/chatbot_manager/events.py b/api_app/chatbot_manager/events.py new file mode 100644 index 0000000000..bf0d667106 --- /dev/null +++ b/api_app/chatbot_manager/events.py @@ -0,0 +1,137 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +"""Wire protocol for the chat WebSocket. + +Single source of truth for everything that crosses the chat WebSocket, so the producer (Celery +worker / streaming callback) and the consumer never hand-build event dicts. One style throughout: +a ``ChatEvent`` dataclass per event kind, each able to render + +- ``to_client()`` — the JSON the browser receives (its ``type`` is a ``ChatEventType``), and +- ``as_channel_message()`` — that payload wrapped for a Channels ``group_send``, whose ``type`` + (``chat.token`` -> ``ChatConsumer.chat_token``) routes it to the matching consumer handler. + +A chat turn streams to a per-user group (``chat-``) rather than a per-session group: the +fixed URL ``ws/chat/?context_url=...`` carries no session id, the group key is the server- +authenticated user id (so no cross-tenant subscription is possible), and every payload carries +``session_id`` so a client with several open sessions/tabs can demultiplex. +""" + +from dataclasses import asdict, dataclass +from enum import StrEnum +from typing import ClassVar, Optional + +CHAT_GROUP_PREFIX = "chat-" + +# Inbound guardrail: mirrors MessageRequestSerializer(message=CharField(max_length=4096)). +MAX_INBOUND_MESSAGE_LEN = 4096 + + +class ChatEventType(StrEnum): + """The ``type`` discriminator of an outbound frame (what the browser switches on).""" + + ACK = "ack" + START = "start" + STATUS = "status" + TOKEN = "token" + END = "end" + ERROR = "error" + ACTION_REQUIRED = "action_required" + + +class ChatErrorDetail(StrEnum): + """Error identifiers for the chat WebSocket. + + Most members carry human-readable text values (sent directly to the client); + a few carry machine-oriented wire discriminators whose user-facing text is + generated server-side (e.g. RATE_LIMITED, whose detail is built by the REST + view with a dynamic ``retry_after`` count). + """ + + SESSION_NOT_FOUND = "Chat session not found." + INVALID_MESSAGE = "Invalid message payload." + TIMEOUT = "The assistant took too long to respond. Please try again." + UNAVAILABLE = "The assistant is currently unavailable. Please try again." + ITERATION_LIMIT = "The assistant could not complete this request. Please try rephrasing." + RATE_LIMITED = "rate_limited" # machine-readable code; REST view builds user-facing detail + + +def chat_group_for_user(user_id: int) -> str: + """Return the per-user channel group that carries this user's chat stream.""" + return f"{CHAT_GROUP_PREFIX}{user_id}" + + +@dataclass(frozen=True) +class ChatEvent: + """Base outbound chat event. + + Subclasses only declare their extra fields and set ``type``. ``to_client()`` is the frame the + browser receives; ``as_channel_message()`` wraps it for a Channels ``group_send`` and is + relayed verbatim by ``ChatConsumer`` (``chat.`` -> the same-named handler). + """ + + # Channels routing prefix; "chat." + type -> "chat.token" -> ChatConsumer.chat_token. + _CHANNEL_TYPE_PREFIX: ClassVar[str] = "chat." + type: ClassVar[ChatEventType] + + session_id: Optional[int] + + def to_client(self) -> dict: + return {"type": self.type.value, **asdict(self)} + + @property + def channel_type(self) -> str: + return f"{self._CHANNEL_TYPE_PREFIX}{self.type.value}" + + def as_channel_message(self) -> dict: + return {"type": self.channel_type, "payload": self.to_client()} + + +@dataclass(frozen=True) +class AckEvent(ChatEvent): + """Synchronous reply to an inbound frame, telling the client which session it landed on.""" + + type: ClassVar[ChatEventType] = ChatEventType.ACK + + +@dataclass(frozen=True) +class StartEvent(ChatEvent): + type: ClassVar[ChatEventType] = ChatEventType.START + + +@dataclass(frozen=True) +class StatusEvent(ChatEvent): + tool: str + type: ClassVar[ChatEventType] = ChatEventType.STATUS + + +@dataclass(frozen=True) +class TokenEvent(ChatEvent): + content: str + type: ClassVar[ChatEventType] = ChatEventType.TOKEN + + +@dataclass(frozen=True) +class EndEvent(ChatEvent): + message_id: int + content: str + type: ClassVar[ChatEventType] = ChatEventType.END + + +@dataclass(frozen=True) +class ErrorEvent(ChatEvent): + detail: str + retry_after: Optional[int] = None + type: ClassVar[ChatEventType] = ChatEventType.ERROR + + +@dataclass(frozen=True) +class ActionRequiredEvent(ChatEvent): + """A tool produced something the user must act on out-of-band (e.g. confirm an analysis). + + Carries the pending id the frontend posts to the confirm endpoint, plus the plan to display. + """ + + pending_id: str + plan: dict + type: ClassVar[ChatEventType] = ChatEventType.ACTION_REQUIRED diff --git a/api_app/chatbot_manager/health.py b/api_app/chatbot_manager/health.py new file mode 100644 index 0000000000..c9ba7620c9 --- /dev/null +++ b/api_app/chatbot_manager/health.py @@ -0,0 +1,66 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import logging +from dataclasses import dataclass + +import requests +from django.conf import settings +from django.core.cache import cache + +from intel_owl.celery import app as celery_app +from intel_owl.celery import get_queue_name + +logger = logging.getLogger(__name__) + +# Bounds for the two liveness probes, kept small so the endpoint stays snappy under a cache miss. +WORKER_INSPECT_TIMEOUT = 1 # seconds for the Celery control broadcast round-trip +OLLAMA_PING_TIMEOUT = 3 # seconds for the Ollama HTTP probe +# The result is user-agnostic, so cache it briefly: a drawer open / multiple tabs must not re-probe. +HEALTH_CACHE_KEY = "chatbot_health" +HEALTH_CACHE_TTL = 15 # seconds; recovery is reflected within at most this window +# One user-safe message; which component is down is logged, not surfaced to the user. +HEALTH_UNAVAILABLE_DETAIL = ( + "The assistant is not available. Make sure the Ollama and chatbot services are running." +) + + +@dataclass +class ChatHealth: + available: bool + detail: str + + +def chatbot_health() -> ChatHealth: + """Whether the assistant can serve a turn now (chatbot worker + Ollama), cached briefly.""" + cached = cache.get(HEALTH_CACHE_KEY) + if cached is not None: + return cached + result = _compute_health() + cache.set(HEALTH_CACHE_KEY, result, HEALTH_CACHE_TTL) + return result + + +def _compute_health() -> ChatHealth: + worker_up = _chatbot_worker_consuming() + ollama_up = _ollama_reachable() + if worker_up and ollama_up: + return ChatHealth(available=True, detail="") + logger.warning("chatbot unavailable: worker_up=%s ollama_up=%s", worker_up, ollama_up) + return ChatHealth(available=False, detail=HEALTH_UNAVAILABLE_DETAIL) + + +def _chatbot_worker_consuming() -> bool: + """True if at least one Celery worker is consuming the chatbot queue.""" + # active_queues() -> {worker: [{"name": ...}, ...]} or None when no worker replies in time. + active = celery_app.control.inspect(timeout=WORKER_INSPECT_TIMEOUT).active_queues() or {} + queue = get_queue_name(settings.CHATBOT_QUEUE) + return any(q["name"] == queue for queues in active.values() for q in queues) + + +def _ollama_reachable() -> bool: + """True if the Ollama daemon answers its version endpoint.""" + try: + return requests.get(f"{settings.OLLAMA_BASE_URL}/api/version", timeout=OLLAMA_PING_TIMEOUT).ok + except requests.RequestException: + return False diff --git a/api_app/chatbot_manager/migrations/0001_initial.py b/api_app/chatbot_manager/migrations/0001_initial.py new file mode 100644 index 0000000000..317f24923a --- /dev/null +++ b/api_app/chatbot_manager/migrations/0001_initial.py @@ -0,0 +1,47 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +# Generated by Django 4.2.27 on 2026-05-25 18:32 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='ChatSession', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(default=django.utils.timezone.now, editable=False)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chat_sessions', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-created_at'], + }, + ), + migrations.CreateModel( + name='ChatMessage', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('role', models.CharField(choices=[('user', 'User'), ('assistant', 'Assistant')], max_length=16)), + ('content', models.TextField()), + ('timestamp', models.DateTimeField(db_index=True, default=django.utils.timezone.now, editable=False)), + ('session', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='messages', to='chatbot_manager.chatsession')), + ], + options={ + 'ordering': ['timestamp'], + 'indexes': [models.Index(fields=['session', 'timestamp'], name='chatbot_man_session_00b05c_idx')], + }, + ), + ] diff --git a/api_app/chatbot_manager/migrations/__init__.py b/api_app/chatbot_manager/migrations/__init__.py new file mode 100644 index 0000000000..acb99e9651 --- /dev/null +++ b/api_app/chatbot_manager/migrations/__init__.py @@ -0,0 +1,2 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. diff --git a/api_app/chatbot_manager/models.py b/api_app/chatbot_manager/models.py new file mode 100644 index 0000000000..a3b494cdeb --- /dev/null +++ b/api_app/chatbot_manager/models.py @@ -0,0 +1,45 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from django.conf import settings +from django.db import models +from django.utils.timezone import now + + +class ChatSession(models.Model): + user = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + related_name="chat_sessions", + ) + created_at = models.DateTimeField(default=now, editable=False) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["-created_at"] + + def __str__(self) -> str: + return f"ChatSession #{self.pk} ({self.user})" + + +class ChatMessage(models.Model): + class Role(models.TextChoices): + USER = "user", "User" + ASSISTANT = "assistant", "Assistant" + + session = models.ForeignKey( + ChatSession, + on_delete=models.CASCADE, + related_name="messages", + ) + role = models.CharField(max_length=16, choices=Role.choices) + content = models.TextField() + timestamp = models.DateTimeField(default=now, editable=False, db_index=True) + + class Meta: + ordering = ["timestamp"] + indexes = [models.Index(fields=["session", "timestamp"])] + + def __str__(self) -> str: + preview = self.content[:50] + ("…" if len(self.content) > 50 else "") + return f"[{self.role}] {preview}" diff --git a/api_app/chatbot_manager/pending_action.py b/api_app/chatbot_manager/pending_action.py new file mode 100644 index 0000000000..dfbed08f71 --- /dev/null +++ b/api_app/chatbot_manager/pending_action.py @@ -0,0 +1,43 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +"""Short-lived store for pending analyze_observable confirmations (HITL guardrail). + +A preview mints a record keyed by a random id; the confirm endpoint consumes it (one-shot, +user-scoped). Backed by a dedicated Redis cache so the model can never launch an analysis on +its own -- only a real user action presenting a valid pending id can. +""" + +import uuid + +from django.conf import settings +from django.core.cache import caches + +CACHE_ALIAS = "chatbot_pending_action" +_KEY_PREFIX = "chatbot_pending_" + + +def create_pending_analysis(user_id: int, payload: dict) -> str: + """Store the previewed analysis inputs and return a fresh one-time pending id.""" + pending_id = uuid.uuid4().hex + record = {"user_id": user_id, "payload": payload} + caches[CACHE_ALIAS].set(_key(pending_id), record, timeout=settings.CHATBOT_PENDING_ACTION_TTL) + return pending_id + + +def consume_pending_analysis(user_id: int, pending_id: str) -> dict | None: + """Return and delete the pending payload iff it exists and belongs to `user_id` (one-shot).""" + cache = caches[CACHE_ALIAS] + key = _key(pending_id) + record = cache.get(key) + if not record or record.get("user_id") != user_id: + return None + # delete() reports whether the key still existed: under a concurrent double-submit only the + # caller whose delete actually removed it proceeds, keeping the launch strictly one-shot. + if not cache.delete(key): + return None + return record["payload"] + + +def _key(pending_id: str) -> str: + return f"{_KEY_PREFIX}{pending_id}" diff --git a/api_app/chatbot_manager/rate_limit.py b/api_app/chatbot_manager/rate_limit.py new file mode 100644 index 0000000000..949e6bff04 --- /dev/null +++ b/api_app/chatbot_manager/rate_limit.py @@ -0,0 +1,87 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +"""Per-user rate limiter for the chatbot. + +Fixed-window counter backed by a Django named cache (``chatbot_rate_limit``), +shared between the REST view and the WebSocket consumer so a user who sends via +both paths hits the same bucket. +""" + +import math +import time + +from django.core.cache import caches + +CACHE_ALIAS = "chatbot_rate_limit" + + +class RateLimiter: + """Count actions per key within a fixed time window. + + ``allow()`` is a read-only check (no side effect); ``increment()`` records + one action. Callers MUST pair them: after ``allow()`` passes, call + ``increment()`` before the protected action, otherwise every concurrent + request at the boundary slips through. + """ + + def __init__(self, limit: int = 5, window_seconds: int = 60): + self.limit = limit + self.window_seconds = window_seconds + self._cache = caches[CACHE_ALIAS] + + def allow(self, key: str) -> tuple[bool, int]: + """Return ``(True, 0)`` when the key is under the limit, + or ``(False, retry_after_seconds)`` when it is exhausted. + ``retry_after`` is an estimate — it subtracts elapsed time within the + current window, but a client that retries at exactly ``retry_after`` + may still be rate-limited if the window hasn't ticked over yet. + """ + current = self._cache.get(self._cache_key(key), 0) + if current < self.limit: + return True, 0 + # Estimate how long until the window rolls over. Round up: truncating with int() + # can return 0 in the final second of the window (telling the client to retry + # immediately while it is still limited), and always under-estimates the wait. + elapsed = time.time() % self.window_seconds + retry_after = math.ceil(self.window_seconds - elapsed) + return False, retry_after + + def increment(self, key: str) -> int: + """Record one action for *key* and return the new count. + + Uses ``incr`` which is atomic under Redis and sets a TTL so dead + counters don't accumulate. Handles the initial ``incr`` on a missing + key (Redis returns 1, some backends raise ValueError). + """ + cache_key = self._cache_key(key) + try: + new = self._cache.incr(cache_key) + except ValueError: + # Key doesn't exist yet — seed it with expiry. + self._cache.set(cache_key, 1, timeout=self.window_seconds) + return 1 + return new + + def _cache_key(self, key: str) -> str: + """Produce a cache key that changes every ``self.window_seconds``. + + Key format: ``chatbot_rate_{key}_{window_ts}`` + Example: ``chatbot_rate_7_1742345600`` + """ + window_ts = int(time.time() / self.window_seconds) + return f"chatbot_rate_{key}_{window_ts}" + + +def _build_rate_limiter() -> RateLimiter: + """Return a RateLimiter configured from Django settings. + + Defined here (not in views.py / consumers.py) so the threshold and window + are read from a single place. + """ + from django.conf import settings + + return RateLimiter( + limit=settings.CHATBOT_RATE_LIMIT, + window_seconds=settings.CHATBOT_RATE_LIMIT_WINDOW, + ) diff --git a/api_app/chatbot_manager/serializers/__init__.py b/api_app/chatbot_manager/serializers/__init__.py new file mode 100644 index 0000000000..acb99e9651 --- /dev/null +++ b/api_app/chatbot_manager/serializers/__init__.py @@ -0,0 +1,2 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. diff --git a/api_app/chatbot_manager/serializers/analyze_observable.py b/api_app/chatbot_manager/serializers/analyze_observable.py new file mode 100644 index 0000000000..343e4fbfac --- /dev/null +++ b/api_app/chatbot_manager/serializers/analyze_observable.py @@ -0,0 +1,60 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from typing import List + +from rest_framework import serializers + +from .base import ToolResultSerializer +from .job import JobToolSerializer + + +def flatten_errors(errors) -> List[str]: + """Flatten DRF's ``{field: [msg, ...]}`` error dict into a flat ``list[str]``. + + Shared by the analyze_observable tool (preview validation) and the confirm endpoint + (launch validation): the reused ObservableAnalysisSerializer reports failures as nested + ValidationError dicts; callers only need a flat list of messages. + """ + if isinstance(errors, dict): + flat = [] + for field_name, messages in errors.items(): + if isinstance(messages, (list, tuple)): + flat.extend(f"{field_name}: {message}" for message in messages) + else: + flat.append(f"{field_name}: {messages}") + return flat + if isinstance(errors, (list, tuple)): + return [str(message) for message in errors] + return [str(errors)] + + +class AnalysisPlanSerializer(serializers.Serializer): + """What a confirmed analyze_observable call would launch (preview, no side effects).""" + + observable_name = serializers.CharField() + classification = serializers.CharField() + tlp = serializers.CharField() + playbook = serializers.CharField(allow_null=True) + analyzers = serializers.ListField(child=serializers.CharField()) + connectors = serializers.ListField(child=serializers.CharField()) + skipped = serializers.ListField(child=serializers.CharField()) + + +class AnalyzeObservableResultSerializer(ToolResultSerializer): + """Envelope for the analyze_observable tool, which now ONLY previews. + + `plan` is what a confirmed run would do; `pending_id` is the one-time token the user's + Confirm button posts to the launch endpoint. The tool never launches an analysis itself. + """ + + plan = AnalysisPlanSerializer(allow_null=True) + pending_id = serializers.CharField(allow_null=True) + + +class ConfirmAnalysisResultSerializer(serializers.Serializer): + """Response of the analyze-confirm endpoint: the launched (or dedup-reused) job.""" + + errors = serializers.ListField(child=serializers.CharField()) + reused = serializers.BooleanField() + job = JobToolSerializer(allow_null=True) diff --git a/api_app/chatbot_manager/serializers/analyzer.py b/api_app/chatbot_manager/serializers/analyzer.py new file mode 100644 index 0000000000..438b17cbb2 --- /dev/null +++ b/api_app/chatbot_manager/serializers/analyzer.py @@ -0,0 +1,30 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from rest_framework import serializers + +from api_app.analyzers_manager.models import AnalyzerConfig + +from .base import ToolResultSerializer + + +class AnalyzerConfigToolSerializer(serializers.ModelSerializer): + """Compact, LLM-facing AnalyzerConfig view for list_analyzers (no owner/PII fields). + + `runnable` is read from a per-user queryset annotation (`annotate_runnable`): True means + ready to run for the requesting user, False means org-disabled or not fully configured. It + is a readiness flag, not a visibility filter -- the analyzer is listed either way. + """ + + # `observable_supported` is a ChoiceArrayField; DRF's ModelSerializer does not auto-map it, + # so declare it explicitly (same approach as the heavy PlaybookConfigSerializer's `type`). + observable_supported = serializers.ListField(child=serializers.CharField(), read_only=True) + runnable = serializers.BooleanField(read_only=True) + + class Meta: + model = AnalyzerConfig + fields = ["name", "description", "type", "observable_supported", "maximum_tlp", "runnable"] + + +class ListAnalyzersResultSerializer(ToolResultSerializer): + analyzers = AnalyzerConfigToolSerializer(many=True) diff --git a/api_app/chatbot_manager/serializers/base.py b/api_app/chatbot_manager/serializers/base.py new file mode 100644 index 0000000000..21e9e7e003 --- /dev/null +++ b/api_app/chatbot_manager/serializers/base.py @@ -0,0 +1,19 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import json + +from rest_framework import serializers + + +class ToolResultSerializer(serializers.Serializer): + """Base envelope for agent-tool output: an always-present `errors` list plus a payload. + + A LangChain tool must return a string (it is fed back to the model as the tool-call + observation), so `to_json()` renders the serialized envelope to a JSON string. + """ + + errors = serializers.ListField(child=serializers.CharField(), default=list) + + def to_json(self) -> str: + return json.dumps(self.data, indent=2) diff --git a/api_app/chatbot_manager/serializers/chat.py b/api_app/chatbot_manager/serializers/chat.py new file mode 100644 index 0000000000..0e8f5763cd --- /dev/null +++ b/api_app/chatbot_manager/serializers/chat.py @@ -0,0 +1,34 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from rest_framework import serializers + +from ..models import ChatMessage, ChatSession + + +class ChatSessionSerializer(serializers.ModelSerializer): + title = serializers.CharField(read_only=True, default=None) + + class Meta: + model = ChatSession + fields = ["id", "created_at", "updated_at", "title"] + read_only_fields = ["id", "created_at", "updated_at", "title"] + + +class ChatMessageSerializer(serializers.ModelSerializer): + class Meta: + model = ChatMessage + fields = ["id", "role", "content", "timestamp"] + read_only_fields = ["id", "timestamp"] + + +class MessageRequestSerializer(serializers.Serializer): + session_id = serializers.IntegerField(required=False, allow_null=True) + message = serializers.CharField(max_length=4096) + + +class MessageResponseSerializer(serializers.Serializer): + # Serializes the persisted assistant ChatMessage directly (no hand-built dict). + session_id = serializers.IntegerField() + response = serializers.CharField(source="content") + message_id = serializers.IntegerField(source="id") diff --git a/api_app/chatbot_manager/serializers/data_model.py b/api_app/chatbot_manager/serializers/data_model.py new file mode 100644 index 0000000000..fac8b98c18 --- /dev/null +++ b/api_app/chatbot_manager/serializers/data_model.py @@ -0,0 +1,13 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from rest_framework import serializers + +from .base import ToolResultSerializer + + +class DataModelResultSerializer(ToolResultSerializer): + # `data_model` is already produced by the model's own DRF serializer + # (`BaseDataModel.serialize()`, polymorphic across Domain/IP/File), so the envelope + # just carries that dict (empty `{}` when the job has no data model). + data_model = serializers.DictField(read_only=True) diff --git a/api_app/chatbot_manager/serializers/health.py b/api_app/chatbot_manager/serializers/health.py new file mode 100644 index 0000000000..c1e5d018cb --- /dev/null +++ b/api_app/chatbot_manager/serializers/health.py @@ -0,0 +1,11 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from rest_framework import serializers + + +class ChatHealthSerializer(serializers.Serializer): + """Response shape for GET /api/chatbot/health (serializes a ChatHealth dataclass).""" + + available = serializers.BooleanField(read_only=True) + detail = serializers.CharField(read_only=True, allow_blank=True) diff --git a/api_app/chatbot_manager/serializers/investigation.py b/api_app/chatbot_manager/serializers/investigation.py new file mode 100644 index 0000000000..02ede54157 --- /dev/null +++ b/api_app/chatbot_manager/serializers/investigation.py @@ -0,0 +1,37 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from rest_framework import serializers + +from api_app.investigations_manager.models import Investigation + +from .base import ToolResultSerializer + + +class InvestigationToolSerializer(serializers.ModelSerializer): + """Compact, LLM-facing Investigation view for list results (no owner/PII fields). + + Deliberately exposes only DB columns. `Investigation.tlp`, `.tags` and `.total_jobs` + are @properties that each query the `jobs` relation, so including them here would fire + extra queries for every row (N+1 across a list). Those aggregate fields are surfaced + instead by `summarize_investigation` / `get_investigation_tree`, which operate on a + single investigation where the cost is bounded. + """ + + class Meta: + model = Investigation + fields = ["id", "name", "status", "start_time", "end_time"] + + +class InvestigationsResultSerializer(ToolResultSerializer): + investigations = InvestigationToolSerializer(many=True) + + +class InvestigationTreeResultSerializer(ToolResultSerializer): + # The tree is assembled in Python (in the tool) to avoid the treebeard per-node N+1, + # so the payload is already a plain nested dict; the envelope only JSON-serializes it. + investigation = serializers.DictField(read_only=True, allow_null=True) + + +class SummarizeInvestigationResultSerializer(ToolResultSerializer): + summary = serializers.CharField(allow_null=True) diff --git a/api_app/chatbot_manager/serializers/job.py b/api_app/chatbot_manager/serializers/job.py new file mode 100644 index 0000000000..65a2c2342f --- /dev/null +++ b/api_app/chatbot_manager/serializers/job.py @@ -0,0 +1,78 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from rest_framework import serializers + +from api_app.analyzers_manager.models import AnalyzerReport +from api_app.models import Job + +from .base import ToolResultSerializer + + +class AnalyzerReportToolSerializer(serializers.ModelSerializer): + """Compact analyzer-report view used inside JobDetailToolSerializer.""" + + name = serializers.CharField(source="config.name", read_only=True) + report = serializers.SerializerMethodField() + + class Meta: + model = AnalyzerReport + fields = ["name", "status", "report"] + + def get_report(self, obj: AnalyzerReport): + # `report` is a JSONField (dict / list / scalar). Dicts pass through; anything else + # is stringified and capped so a single large report can't blow up the LLM prompt. + return obj.report if isinstance(obj.report, dict) else str(obj.report)[:500] + + +class JobToolSerializer(serializers.ModelSerializer): + """Compact, LLM-facing Job view for search results (no user/PII fields).""" + + observable_name = serializers.CharField(source="analyzable.name", read_only=True) + observable_classification = serializers.CharField(source="analyzable.classification", read_only=True) + md5 = serializers.CharField(source="analyzable.md5", read_only=True) + analyzers_to_execute = serializers.SlugRelatedField(slug_field="name", many=True, read_only=True) + + class Meta: + model = Job + fields = [ + "id", + "observable_name", + "observable_classification", + "md5", + "status", + "tlp", + "received_request_time", + "finished_analysis_time", + "analyzers_to_execute", + ] + + +class JobDetailToolSerializer(JobToolSerializer): + """Full Job view for get_job_details: adds tags, analyzer reports and errors/warnings.""" + + tags = serializers.SlugRelatedField(slug_field="label", many=True, read_only=True) + # output key stays `analyzer_reports` (like the public JobSerializer); the actual + # reverse relation is `analyzerreports`. + analyzer_reports = AnalyzerReportToolSerializer(many=True, read_only=True, source="analyzerreports") + + class Meta(JobToolSerializer.Meta): + fields = JobToolSerializer.Meta.fields + [ + "process_time", + "tags", + "errors", + "warnings", + "analyzer_reports", + ] + + +class SearchJobsResultSerializer(ToolResultSerializer): + jobs = JobToolSerializer(many=True) + + +class JobDetailResultSerializer(ToolResultSerializer): + job = JobDetailToolSerializer(allow_null=True) + + +class SummarizeJobResultSerializer(ToolResultSerializer): + summary = serializers.CharField(allow_null=True) diff --git a/api_app/chatbot_manager/serializers/playbook.py b/api_app/chatbot_manager/serializers/playbook.py new file mode 100644 index 0000000000..54ca32394e --- /dev/null +++ b/api_app/chatbot_manager/serializers/playbook.py @@ -0,0 +1,26 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from rest_framework import serializers + +from api_app.playbooks_manager.models import PlaybookConfig + +from .base import ToolResultSerializer + + +class PlaybookConfigToolSerializer(serializers.ModelSerializer): + """Compact, LLM-facing PlaybookConfig view for recommend_playbook (no owner/PII fields).""" + + # `type` is a ChoiceArrayField of classifications; declare it explicitly (DRF doesn't auto-map + # it). `analyzers`/`connectors` are M2M relations rendered as their names. + type = serializers.ListField(child=serializers.CharField(), read_only=True) + analyzers = serializers.SlugRelatedField(slug_field="name", many=True, read_only=True) + connectors = serializers.SlugRelatedField(slug_field="name", many=True, read_only=True) + + class Meta: + model = PlaybookConfig + fields = ["name", "description", "type", "analyzers", "connectors"] + + +class RecommendPlaybookResultSerializer(ToolResultSerializer): + playbooks = PlaybookConfigToolSerializer(many=True) diff --git a/api_app/chatbot_manager/tasks.py b/api_app/chatbot_manager/tasks.py new file mode 100644 index 0000000000..4cf3933440 --- /dev/null +++ b/api_app/chatbot_manager/tasks.py @@ -0,0 +1,156 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import datetime +import logging + +from asgiref.sync import async_to_sync +from celery import shared_task +from celery.exceptions import SoftTimeLimitExceeded +from channels.layers import get_channel_layer +from django.conf import settings +from django.contrib.auth import get_user_model +from django.db.models import Max +from django.db.models.functions import Coalesce +from django.utils.timezone import now + +from api_app.chatbot_manager.agent.context import derive_page_context +from api_app.chatbot_manager.events import ( + ChatErrorDetail, + EndEvent, + ErrorEvent, + StartEvent, + chat_group_for_user, +) +from intel_owl.tasks import FailureLoggedTask + +logger = logging.getLogger(__name__) + + +# soft_time_limit is 300s on purpose: a single chat turn can chain several tool-calling +# rounds, each one an Ollama inference on a local model, so it can legitimately take +# tens of seconds to a couple of minutes. A *soft* limit raises SoftTimeLimitExceeded +# (catchable, lets us clean up) rather than a hard time_limit that SIGKILLs the worker, +# and it bounds a hung Ollama call so it can't occupy a worker indefinitely. +@shared_task(base=FailureLoggedTask, soft_time_limit=300) +def process_chat_message(session_id: int, user_message: str, user_id: int, context_url: str = "") -> None: + """Run one chat turn off-request and stream it to the user's WebSocket group. + + Enqueued by ChatConsumer.receive_json. The agent runs synchronously here (so token and + tool callbacks can bridge to the async channel layer via async_to_sync) and pushes + chat.start -> chat.status/chat.token* -> chat.end onto the per-user group, which the + consumer relays to the browser. On any failure a single chat.error is emitted instead and + the turn is dropped (no assistant message persisted), mirroring the sync REST path. + + Persistence is the source of truth: the streamed tokens are a live preview, while + result["output"] is what gets stored and sent in chat.end. History is snapshotted before + this turn is written so the current message is not double-counted. + """ + # The agent/LLM stack (langchain + ChatOllama) is imported lazily so the other Celery + # workers that load this module never pay for that heavy import — only the chatbot worker, + # which actually runs this task, does. + from api_app.chatbot_manager.agent.agent import AGENT_STOPPED_OUTPUT, build_agent_executor + from api_app.chatbot_manager.agent.memory import DjangoChatMessageHistory + from api_app.chatbot_manager.agent.streaming import ChatStreamingCallbackHandler + from api_app.chatbot_manager.models import ChatMessage, ChatSession + + channel_layer = get_channel_layer() + group = chat_group_for_user(user_id) + + def emit(event) -> None: + async_to_sync(channel_layer.group_send)(group, event.as_channel_message()) + + # Defense in depth: never trust the task args. The session must exist AND belong to the + # user (the consumer already enforces this, but the task must not assume that). + try: + session = ChatSession.objects.get(pk=session_id, user_id=user_id) + except ChatSession.DoesNotExist: + logger.warning(f"process_chat_message: session {session_id} not found for user {user_id}") + emit(ErrorEvent(session_id, ChatErrorDetail.SESSION_NOT_FOUND.value)) + return + + user = get_user_model().objects.get(pk=user_id) + history = DjangoChatMessageHistory(session=session) + # LangChain message objects, fed straight into the prompt's chat_history + # MessagesPlaceholder (no text pre-rendering). Evaluated here, before this turn is + # written, so the current message is not double-counted. + chat_history = history.messages + + emit(StartEvent(session_id)) + try: + executor = build_agent_executor(user=user, streaming=True) + # Scope streamed tool-status events to the agent's real tools (drops the internal + # "_Exception" pseudo-tool that handle_parsing_errors raises on malformed model output). + handler = ChatStreamingCallbackHandler( + user_id=user_id, + session_id=session_id, + tool_names={tool.name for tool in executor.tools}, + ) + result = executor.invoke( + { + "input": user_message, + "chat_history": chat_history, + "page_context": derive_page_context(context_url), + }, + config={"callbacks": [handler]}, + ) + except SoftTimeLimitExceeded: + logger.warning(f"process_chat_message: timed out for session {session_id}") + emit(ErrorEvent(session_id, ChatErrorDetail.TIMEOUT.value)) + return + except Exception as exc: # noqa: BLE001 - any agent/Ollama failure must still reach the client + logger.exception(f"process_chat_message: agent run failed for session {session_id}: {exc}") + emit(ErrorEvent(session_id, ChatErrorDetail.UNAVAILABLE.value)) + return + + response_text = result.get("output", "") + if response_text == AGENT_STOPPED_OUTPUT: + # max_iterations force-stopped a looping model: surface a real error and drop the + # turn rather than persisting LangChain's canned string as the assistant's answer. + logger.warning(f"process_chat_message: iteration cap hit for session {session_id}") + emit(ErrorEvent(session_id, ChatErrorDetail.ITERATION_LIMIT.value)) + return + + history.add_user_message(user_message) + # Create the assistant row directly (rather than add_ai_message + a latest("timestamp") + # re-query) so chat.end carries its exact id, without a lookup two overlapping turns of the + # same session could race on. + ai_message = ChatMessage.objects.create( + session=session, role=ChatMessage.Role.ASSISTANT, content=response_text + ) + emit(EndEvent(session_id, ai_message.id, response_text)) + + +# soft_time_limit=1800: daily maintenance task whose work is a single bulk DELETE that +# cascades to ChatMessage rows. 30 minutes is generous headroom for a large backlog of stale +# sessions while still bounding a runaway query; a *soft* limit raises a catchable +# SoftTimeLimitExceeded (so the logged count stays meaningful) instead of a hard time_limit +# that SIGKILLs the worker mid-delete. +@shared_task(base=FailureLoggedTask, soft_time_limit=1800) +def delete_old_chat_sessions() -> int: + """ + Periodic cleanup: delete ChatSessions whose last activity is older than + CHATBOT_MESSAGE_RETENTION_DAYS. "Last activity" is the most recent message timestamp, + falling back to created_at for sessions with no messages yet (ChatSession.updated_at is + auto_now but is NOT touched when a ChatMessage is created, so it does not reflect activity). + Deleting the session removes its messages via the FK on_delete=CASCADE. + Returns the number of deleted sessions. + """ + from api_app.chatbot_manager.models import ChatSession + + logger.info("started delete_old_chat_sessions") + + cutoff = now() - datetime.timedelta(days=settings.CHATBOT_MESSAGE_RETENTION_DAYS) + # Coalesce(..., created_at) is required: without it, sessions with no messages get + # last_activity = NULL and __lt never matches, so they would never be cleaned up. + stale = ChatSession.objects.annotate( + last_activity=Coalesce(Max("messages__timestamp"), "created_at") + ).filter(last_activity__lt=cutoff) + # .delete() on an annotated queryset is unreliable, so materialize the pks first and + # delete them in a separate, un-annotated step. + stale_pks = list(stale.values_list("pk", flat=True)) + logger.info(f"found {len(stale_pks)} stale chat sessions") + ChatSession.objects.filter(pk__in=stale_pks).delete() # CASCADE removes the messages + + logger.info("finished delete_old_chat_sessions") + return len(stale_pks) diff --git a/api_app/chatbot_manager/urls.py b/api_app/chatbot_manager/urls.py new file mode 100644 index 0000000000..f16d9b03d0 --- /dev/null +++ b/api_app/chatbot_manager/urls.py @@ -0,0 +1,20 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from django.urls import path +from rest_framework.routers import DefaultRouter + +from .views import ChatAnalysisConfirmView, ChatHealthView, ChatSessionViewSet + +# These route suffixes are mirrored by the frontend in frontend/src/constants/apiURLs.js +# (CHATBOT_SESSIONS_URI / CHATBOT_HEALTH_URI / CHATBOT_ANALYSIS_CONFIRM_URI). They can't be imported +# across the JS/Python boundary, so a frontend coupling test +# (frontend/tests/components/chat/chatbotRouteCoupling.test.js) reads this file and breaks if a route +# is renamed here without updating the constant there. Keep both in sync. +router = DefaultRouter(trailing_slash=False) +router.register(r"sessions", ChatSessionViewSet, basename="chat-sessions") + +urlpatterns = router.urls + [ + path("health", ChatHealthView.as_view(), name="chat-health"), + path("analysis/confirm", ChatAnalysisConfirmView.as_view(), name="chat-analysis-confirm"), +] diff --git a/api_app/chatbot_manager/views.py b/api_app/chatbot_manager/views.py new file mode 100644 index 0000000000..0aa3536bad --- /dev/null +++ b/api_app/chatbot_manager/views.py @@ -0,0 +1,241 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import logging + +from django.db.models import OuterRef, Subquery +from django.db.models.functions import Substr +from django.utils.timezone import now +from rest_framework import status +from rest_framework.decorators import action +from rest_framework.generics import get_object_or_404 +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response +from rest_framework.views import APIView +from rest_framework.viewsets import ModelViewSet + +from api_app.playbooks_manager.models import PlaybookConfig +from api_app.serializers.job import ObservableAnalysisSerializer + +from .agent.agent import AGENT_STOPPED_OUTPUT, build_agent_executor +from .agent.memory import DjangoChatMessageHistory +from .events import ChatErrorDetail +from .health import chatbot_health +from .models import ChatMessage, ChatSession +from .pending_action import consume_pending_analysis +from .rate_limit import _build_rate_limiter +from .serializers.analyze_observable import ConfirmAnalysisResultSerializer, flatten_errors +from .serializers.chat import ( + ChatMessageSerializer, + ChatSessionSerializer, + MessageRequestSerializer, + MessageResponseSerializer, +) +from .serializers.health import ChatHealthSerializer + +logger = logging.getLogger(__name__) + + +class ChatSessionViewSet(ModelViewSet): + serializer_class = ChatSessionSerializer + permission_classes = [IsAuthenticated] + http_method_names = ["get", "post", "delete", "head", "options"] + + # Max characters for the annotated session title; keeps list rows single-line. + SESSION_TITLE_MAX_LEN = 40 + + def get_queryset(self): + qs = ChatSession.objects.filter(user=self.request.user) + # Annotate a title only when listing — single-object GETs (detail/messages) don't need it + # and the extra Subquery would be wasted work. + if self.action == "list": + first_user_msg = ( + ChatMessage.objects.filter( + session=OuterRef("pk"), + role=ChatMessage.Role.USER, + ) + .order_by("timestamp") + .values("content")[:1] + ) + qs = qs.annotate( + _raw_title=Subquery(first_user_msg), + title=Substr("_raw_title", 1, self.SESSION_TITLE_MAX_LEN), + ) + return qs + + def perform_create(self, serializer): + serializer.save(user=self.request.user) + + @action(detail=False, methods=["post"], url_path="message") + def message(self, request): + """Run one synchronous chat turn against the agent. + + Flow: validate the request, rate-limit per user, resolve the target + session (or create a new one), load the prior turns from the DB as the + agent's conversation history, invoke the tool-calling agent with the + user's message, then persist both the user message and the assistant + reply and return the reply. Persistence goes through + DjangoChatMessageHistory so the ORM stays the single source of truth + for history. + """ + req = MessageRequestSerializer(data=request.data) + req.is_valid(raise_exception=True) + + user_message = req.validated_data["message"] + + limiter = _build_rate_limiter() + allowed, retry_after = limiter.allow(str(request.user.id)) + if not allowed: + return Response( + { + "errors": [ + { + "detail": (f"Too many messages. Please wait {retry_after} seconds."), + "code": ChatErrorDetail.RATE_LIMITED.value, + "retry_after": retry_after, + } + ] + }, + status=status.HTTP_429_TOO_MANY_REQUESTS, + ) + + session_id = req.validated_data.get("session_id") + if session_id: + session = get_object_or_404(ChatSession, pk=session_id, user=request.user) + else: + session = ChatSession.objects.create(user=request.user) + + limiter.increment(str(request.user.id)) + + history = DjangoChatMessageHistory(session=session) + + executor = build_agent_executor(user=request.user) + try: + result = executor.invoke( + {"input": user_message, "chat_history": history.messages, "page_context": ""} + ) + except Exception: # noqa: BLE001 - any agent/Ollama failure must reach the client cleanly + # Mirror the Celery path (tasks.py): a model/Ollama failure is surfaced as a clean + # 503 envelope instead of an unhandled 500. session_id is included so a client that + # created the session via this very request can keep using it. + logger.exception(f"chatbot message: agent run failed for session {session.pk}") + return Response( + {"detail": ChatErrorDetail.UNAVAILABLE.value, "session_id": session.pk}, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + ) + response_text = result.get("output", "") + if response_text == AGENT_STOPPED_OUTPUT: + logger.warning(f"chatbot message: iteration cap hit for session {session.pk}") + return Response( + {"detail": ChatErrorDetail.ITERATION_LIMIT.value, "session_id": session.pk}, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + ) + + history.add_user_message(user_message) + history.add_ai_message(response_text) + ai_msg = ChatMessage.objects.filter(session=session, role=ChatMessage.Role.ASSISTANT).latest( + "timestamp" + ) + + return Response( + MessageResponseSerializer(ai_msg).data, + status=status.HTTP_200_OK, + ) + + @action(detail=True, methods=["get"], url_path="messages") + def messages(self, request, pk=None): + """Return a chat session's messages, oldest first, paginated. + + `self.get_object()` resolves the session through `get_queryset()`, which is + already scoped to `request.user`, so another user's session yields a 404 for + free (no manual ownership check). Messages are ordered by `timestamp` ascending + so the frontend can render the conversation top-to-bottom when a chat is reopened. + + Pagination is not automatic inside a custom @action (DRF only paginates the + default `list`), so it is driven explicitly via `paginate_queryset` / + `get_paginated_response`, mirroring the project's CustomPageNumberPagination + default. The `page is None` branch is the idiomatic fallback for when pagination + is disabled. + """ + session = self.get_object() + queryset = session.messages.order_by("timestamp") + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = ChatMessageSerializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = ChatMessageSerializer(queryset, many=True) + return Response(serializer.data) + + +class ChatHealthView(APIView): + """Proactive availability probe for the chat panel (chatbot worker + Ollama).""" + + permission_classes = [IsAuthenticated] + + def get(self, request): + return Response(ChatHealthSerializer(chatbot_health()).data) + + +class ChatAnalysisConfirmView(APIView): + """Launch a previously previewed analysis. This is the ONLY path that starts an + analyze_observable run: the agent can only preview (mint a pending id); a real user action + (the Confirm button) posts that id here, so the model can never launch on its own (M-1).""" + + permission_classes = [IsAuthenticated] + + def post(self, request): + pending_id = request.data.get("pending_id") + record = consume_pending_analysis(request.user.id, pending_id) if pending_id else None + if record is None: + return Response( + { + "errors": ["This confirmation has expired or is invalid. Ask for the analysis again."], + "reused": False, + "job": None, + }, + status=status.HTTP_410_GONE, + ) + + # Re-apply the playbook visibility guard the preview tool uses: ObservableAnalysisSerializer + # resolves playbook_requested via PlaybookConfig.objects.all() (no visibility filter), so + # without this a playbook that became invisible to the user between preview and confirm could + # still be launched. + if ( + record.get("playbook") + and not PlaybookConfig.objects.visible_for_user(request.user) + .filter(name=record["playbook"]) + .exists() + ): + return Response( + { + "errors": [f"Playbook '{record['playbook']}' is no longer visible to you."], + "reused": False, + "job": None, + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + data = {"observable_name": record["observable_name"], "tlp": record["tlp"]} + if record.get("playbook"): + data["playbook_requested"] = record["playbook"] + analyzers_list = [a.strip() for a in record.get("analyzers", "").split(",") if a.strip()] + if analyzers_list: + data["analyzers_requested"] = analyzers_list + + # Re-validate the observable/analyzers at launch time; playbook visibility is re-checked above. + serializer = ObservableAnalysisSerializer(data=data, context={"request": request}) + if not serializer.is_valid(raise_exception=False): + return Response( + {"errors": flatten_errors(serializer.errors), "reused": False, "job": None}, + status=status.HTTP_400_BAD_REQUEST, + ) + + started = now() + job = serializer.save(send_task=True) + reused = job.received_request_time < started # platform dedup returned a recent job + return Response( + ConfirmAnalysisResultSerializer({"errors": [], "reused": reused, "job": job}).data, + status=status.HTTP_200_OK, + ) diff --git a/api_app/choices.py b/api_app/choices.py index 5693f450bc..ff658a82b5 100644 --- a/api_app/choices.py +++ b/api_app/choices.py @@ -156,6 +156,14 @@ def calculate_observable(cls, value: str) -> str: return classification + @classmethod + def observable_classifications(cls) -> typing.List["Classification"]: + """Classifications that apply to observables: every value except ``FILE``, which is the + file-analysis path rather than an observable type. Single source of truth for the code + that enumerates observable types (e.g. the observable-analyzer listing and the + observable-classification job aggregation).""" + return [c for c in cls if c != cls.FILE] + @classmethod def get_data_model_class(cls, classification: str) -> typing.Type: from api_app.data_model_manager.models import ( diff --git a/api_app/connectors_manager/classes.py b/api_app/connectors_manager/classes.py index 67974cc941..1dd46d98ea 100644 --- a/api_app/connectors_manager/classes.py +++ b/api_app/connectors_manager/classes.py @@ -51,7 +51,7 @@ def before_run(self): if ( self._config.run_on_failure or not self._job.analyzerreports.count() - or self._job.analyzerreports.exclude(status=ReportStatus.FAILED.value).exists() + or not self._job.analyzerreports.filter(status=ReportStatus.FAILED.value).exists() ): logger.info( f"Running connector {self.__class__.__name__} " diff --git a/api_app/connectors_manager/connectors/email_sender.py b/api_app/connectors_manager/connectors/email_sender.py index 71f03964c8..6e6bfa76a1 100644 --- a/api_app/connectors_manager/connectors/email_sender.py +++ b/api_app/connectors_manager/connectors/email_sender.py @@ -4,7 +4,6 @@ from api_app.connectors_manager.classes import Connector from intel_owl.settings import DEFAULT_FROM_EMAIL -from tests.mock_utils import if_mock_connections, patch class EmailSender(Connector): @@ -44,15 +43,3 @@ def run(self) -> dict: def update(self) -> bool: pass - - @classmethod - def _monkeypatch(cls): - patches = [ - if_mock_connections( - patch( - "django.core.mail.EmailMessage.send", - return_value="Email sent", - ) - ) - ] - return super()._monkeypatch(patches=patches) diff --git a/api_app/connectors_manager/connectors/misp.py b/api_app/connectors_manager/connectors/misp.py index eaf4214a98..38acd6ce8b 100644 --- a/api_app/connectors_manager/connectors/misp.py +++ b/api_app/connectors_manager/connectors/misp.py @@ -1,6 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. +import logging from typing import List import pymisp @@ -9,7 +10,9 @@ from api_app import helpers from api_app.choices import Classification from api_app.connectors_manager.classes import Connector -from tests.mock_utils import if_mock_connections, patch +from api_app.connectors_manager.exceptions import ConnectorRunException + +logger = logging.getLogger(__name__) INTELOWL_MISP_TYPE_MAP = { Classification.IP: "ip-src", @@ -93,19 +96,101 @@ def _link_attr_obj(self) -> pymisp.MISPAttribute: return obj + def _handle_misp_errors(self, errors): + error_str = str(errors) + + debug_info = ( + f" [debug: PyMISP version={pymisp.__version__}," + f" ssl_check={self.ssl_check}," + f" url={self._url_key_name}]" + if self.debug + else "" + ) + + if "The plain HTTP request was sent to HTTPS port" in error_str: + raise ConnectorRunException( + "MISP connection failed: You are trying to send a plain HTTP request to an HTTPS port. " + "Please change your MISP URL in the plugin configuration from 'http://' to 'https://'." + f"{debug_info}" + ) + else: + raise ConnectorRunException(f"{errors}{debug_info}") + + def health_check(self, user=None) -> bool: + if settings.STAGE_CI or settings.MOCK_CONNECTIONS: + return True + + params = self._config.parameters.annotate_configured(self._config, user).annotate_value_for_user( + self._config, user + ) + + url = None + key = None + + ssl_check = True + self_signed_certificate = False + + for param in params: + if param.name == "url_key_name": + url = param.value + elif param.name == "api_key_name": + key = param.value + elif param.name == "ssl_check": + ssl_check = param.value + elif param.name == "self_signed_certificate": + self_signed_certificate = param.value + + if not url: + logger.info("Healthcheck failed: Missing config url") + return False + if not key: + logger.info("Healthcheck failed: Missing config api key") + return False + + ssl_param = ( + f"{settings.PROJECT_LOCATION}/configuration/misp_ssl.crt" + if ssl_check and self_signed_certificate + else ssl_check + ) + + try: + misp = pymisp.PyMISP( + url=url, + key=key, + ssl=ssl_param, + debug=False, + timeout=5, + ) + + # PyMISP has a property misp_instance_version + # that makes a GET request to servers/getVersion + # using valid API key and returns the version of + # the MISP instance if the connection is successful + # Refs: https://pymisp.readthedocs.io/en/latest/modules.html?#pymisp.PyMISP.misp_instance_version + misp.misp_instance_version + return True + + except Exception as e: + logger.info(f"MISP health check failed: {e}") + return False + def run(self): ssl_param = ( f"{settings.PROJECT_LOCATION}/configuration/misp_ssl.crt" if self.ssl_check and self.self_signed_certificate else self.ssl_check ) - misp_instance = pymisp.PyMISP( - url=self._url_key_name, - key=self._api_key_name, - ssl=ssl_param, - debug=self.debug, - timeout=5, - ) + + try: + misp_instance = pymisp.PyMISP( + url=self._url_key_name, + key=self._api_key_name, + ssl=ssl_param, + debug=self.debug, + timeout=5, + ) + except Exception as e: + self._handle_misp_errors(f"MISP initialization failed: {str(e)}") # get event and attributes event = self._event_obj @@ -127,49 +212,14 @@ def run(self): ) # single request — event + all attributes sent together - misp_event = misp_instance.add_event(event, pythonify=True) - - return misp_instance.get_event(misp_event.id) - - @classmethod - def _monkeypatch(cls): - patches = [ - if_mock_connections( - patch( - "pymisp.PyMISP", - side_effect=MockPyMISP, - ) - ) - ] - return super()._monkeypatch(patches=patches) + try: + misp_event = misp_instance.add_event(event, pythonify=True) + except Exception as e: + self._handle_misp_errors(f"MISP add event failed: {str(e)}") + if isinstance(misp_event, dict): + errors = misp_event.get("errors", []) + if errors: + self._handle_misp_errors(errors) -# Mocks -class MockUpMISPElement: - """ - Mock element(event/attribute) for testing - """ - - id: int = 1 - - -class MockPyMISP: - """ - Mock PyMISP instance for testing - methods which require connection to a MISP instance - """ - - def __init__(self, *args, **kwargs) -> None: - pass - - @staticmethod - def add_event(*args, **kwargs) -> MockUpMISPElement: - return MockUpMISPElement() - - @staticmethod - def add_attribute(*args, **kwargs) -> MockUpMISPElement: - return MockUpMISPElement() - - @staticmethod - def get_event(event_id) -> dict: - return {"Event": {"id": event_id}} + return misp_instance.get_event(misp_event.id) diff --git a/api_app/connectors_manager/connectors/opencti.py b/api_app/connectors_manager/connectors/opencti.py index ed10a115e1..608f1b0e32 100644 --- a/api_app/connectors_manager/connectors/opencti.py +++ b/api_app/connectors_manager/connectors/opencti.py @@ -1,6 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. +import logging from typing import Dict import pycti @@ -11,6 +12,8 @@ from api_app.choices import Classification from api_app.connectors_manager import classes +logger = logging.getLogger(__name__) + INTELOWL_OPENCTI_TYPE_MAP = { Classification.IP: { "v4": "ipv4-addr", @@ -180,6 +183,49 @@ def _link_report_entities(self, report_id, observable_id, external_ref_id): id=report_id, stixObjectOrStixRelationshipId=observable_id ) + def health_check(self, user=None) -> bool: + if settings.STAGE_CI or settings.MOCK_CONNECTIONS: + return True + + params = self._config.parameters.annotate_configured(self._config, user).annotate_value_for_user( + self._config, user + ) + + url = None + token = None + ssl_verify = False + proxies = None + + for param in params: + if param.name == "url_key_name": + url = param.value + elif param.name == "api_key_name": + token = param.value + elif param.name == "ssl_verify": + ssl_verify = str(param.value).lower() == "true" + elif param.name == "proxies": + proxies = param.value + + if not url: + logger.info("Healthcheck failed: Missing config url") + return False + if not token: + logger.info("Healthcheck failed: Missing config api key") + return False + + try: + client = pycti.OpenCTIApiClient(url, token, ssl_verify=ssl_verify, proxies=proxies) + + # pycti has a built-in method (health_check) that + # returns boolean True/False based on validity of + # API key and reachability of the OpenCTI instance + # Ref: https://opencti-python-client.readthedocs.io/en/latest/pycti/pycti.api.opencti_api_client.html#pycti.api.opencti_api_client.OpenCTIApiClient.health_check + resp = client.health_check() + return resp + except Exception as e: + logger.info(f"OpenCTI health check failed: {e}") + return False + def run(self): # Initialize OpenCTI client for this run. self.opencti_instance = pycti.OpenCTIApiClient( @@ -226,38 +272,3 @@ def run(self): except Exception: pass raise - - @classmethod - def _monkeypatch(cls): - """Install pycti stubs when connection mocking is enabled.""" - if not getattr(settings, "MOCK_CONNECTIONS", False): - return - - def _configure(start_fn): - def inner(self, job_id, runtime_configuration, task_id, *args, **kwargs): - # Avoid real OpenCTI network calls - pycti.OpenCTIApiClient = lambda *a, **k: None - - def _fake_create(*_args, **_kwargs): - return {"id": 1} - - def _noop(*_args, **_kwargs): - return None - - # Ensure core entities always return a dict with an id in CI generic tests. - pycti.Identity.create = _fake_create - pycti.MarkingDefinition.create = _fake_create - pycti.StixCyberObservable.create = _fake_create - pycti.Label.create = _fake_create - pycti.Report.create = _fake_create - pycti.ExternalReference.create = _fake_create - - # No-op the linking methods that would otherwise dereference opencti/app_logger. - pycti.StixDomainObject.add_external_reference = _noop - pycti.Report.add_stix_object_or_stix_relationship = _noop - - return start_fn(self, job_id, runtime_configuration, task_id, *args, **kwargs) - - return inner - - return super()._monkeypatch(patches=[_configure]) diff --git a/api_app/connectors_manager/connectors/slack.py b/api_app/connectors_manager/connectors/slack.py index ce7cd13100..56b08dcc9e 100644 --- a/api_app/connectors_manager/connectors/slack.py +++ b/api_app/connectors_manager/connectors/slack.py @@ -1,11 +1,16 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import logging from typing import Dict -from unittest.mock import patch import slack_sdk +from django.conf import settings from slack_sdk.errors import SlackApiError from api_app.connectors_manager.classes import Connector -from tests.mock_utils import if_mock_connections + +logger = logging.getLogger(__name__) class Slack(Connector): @@ -33,24 +38,37 @@ def body(self) -> str: f"for <{self._job.url}/raw|{self._job.analyzable.name}>" ) + def health_check(self, user=None) -> bool: + if settings.STAGE_CI or settings.MOCK_CONNECTIONS: + return True + + params = self._config.parameters.annotate_configured(self._config, user).annotate_value_for_user( + self._config, user + ) + token = None + for param in params: + if param.name == "token": + token = param.value + break + + if not token: + logger.info("Slack health check failed: Missing token configuration.") + return False + + try: + client = slack_sdk.WebClient(token=token) + + # slack sdk has a built-in method (auth_test) to + # test the authentication and connectivity to Slack + # (auth_test returns identity information of the + # authenticated user if the token is valid) + # Ref: https://docs.slack.dev/tools/python-slack-sdk/reference/#slack_sdk.WebClient.auth_test + client.auth_test() + return True + except Exception as e: + logger.info(f"Slack health check failed: {e}") + return False + def run(self) -> dict: self.client.chat_postMessage(text=f"{self.title}\n{self.body}", channel=self._channel, mrkdwn=True) return {} - - @classmethod - def _monkeypatch(cls): - # flake8: noqa - class MockClient: - def __init__(self, *args, **kwargs): ... - - def chat_postMessage(self, *args, **kwargs): ... - - patches = [ - if_mock_connections( - patch( - "slack_sdk.WebClient", - side_effect=MockClient, - ) - ) - ] - return super()._monkeypatch(patches=patches) diff --git a/api_app/connectors_manager/connectors/yeti.py b/api_app/connectors_manager/connectors/yeti.py index 44676f787b..2cab5bf166 100644 --- a/api_app/connectors_manager/connectors/yeti.py +++ b/api_app/connectors_manager/connectors/yeti.py @@ -1,12 +1,16 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. +import ipaddress +import logging + import requests from django.conf import settings from api_app.connectors_manager import classes from api_app.connectors_manager.exceptions import ConnectorRunException -from tests.mock_utils import MockUpResponse, if_mock_connections, patch + +logger = logging.getLogger(__name__) class YETI(classes.Connector): @@ -14,6 +18,63 @@ class YETI(classes.Connector): _url_key_name: str _api_key_name: str + def health_check(self, user=None) -> bool: + params = self._config.parameters.annotate_configured(self._config, user).annotate_value_for_user( + self._config, user + ) + url = None + api_key = None + + for param in params: + if param.name == "url_key_name": + url = param.value + elif param.name == "api_key_name": + api_key = param.value + + if not url: + logger.info("Healthcheck failed: Missing config url") + return False + if not api_key: + logger.info("Healthcheck failed: Missing config api key") + return False + + if settings.STAGE_CI or settings.MOCK_CONNECTIONS: + return True + + base_url = url.rstrip("/") + auth_url = f"{base_url}/api/v2/auth/api-token" + + auth_headers = {"x-yeti-apikey": api_key, "User-Agent": "IntelOwl"} + + try: + verify_ssl = getattr(self, "verify_ssl", False) + + # Posting the API key to YETI's authentication endpoint returns an + # access token on success (YETI API v2). A valid access token confirms + # that the API key is valid and the YETI instance is reachable. + # Ref: https://yeti-platform.io/docs/api/#authentication + auth_resp = requests.post( + url=auth_url, + headers=auth_headers, + verify=verify_ssl, + timeout=10, + ) + auth_resp.raise_for_status() + access_token = auth_resp.json().get("access_token") + + if access_token: + return True + else: + logger.info(f"Healthcheck failed for {self}: No access token in response.") + return False + + except requests.RequestException as e: + logger.info(f"Healthcheck failed: YETI Auth Request failed for {self}. Error: {e}") + return False + except Exception as e: + logger.exception(f"Unexpected error in YETI health_check: {e}") + return False + def run(self): # get observable value and type if self._job.is_sample: @@ -23,12 +84,30 @@ def run(self): obs_value = self._job.analyzable.name obs_type = self._job.analyzable.classification + # convert obs_type to YETI's expected types if possible + if obs_type == "ip": + # mark whether the IP is ipv4 or ipv6, fallback to generic on error + try: + ip_obj = ipaddress.ip_address(obs_value) + if ip_obj.version == 4: + obs_type = "ipv4" + elif ip_obj.version == 6: + obs_type = "ipv6" + else: + obs_type = "generic" + except Exception: + obs_type = "generic" + elif obs_type == "domain": + obs_type = "hostname" + elif obs_type == "hash": + obs_type = "generic" + # create context context = { "source": "IntelOwl", "report": f"{settings.WEB_CLIENT_URL}/jobs/{self.job_id}", "status": "analyzed", - "date": str(self._job.finished_analysis_time), + "date": str(self._job.received_request_time), "description": f"IntelOwl's analysis report for Job: {self.job_id} | {obs_value} | {obs_type}", "analyzers executed": ", ".join( list(self._job.analyzers_to_execute.all().values_list("name", flat=True)) @@ -40,18 +119,46 @@ def run(self): # request payload payload = { - "value": obs_value, - "source": "IntelOwl", "tags": tags, - "context": context, + "observable": { + "type": obs_type, + "value": obs_value, + "context": [context], + }, } - headers = {"Accept": "application/json", "X-Api-Key": self._api_key_name} + if self._url_key_name and self._url_key_name.endswith("/"): self._url_key_name = self._url_key_name[:-1] - url = f"{self._url_key_name}/api/v2/observables/" + + # auth + auth_url = f"{self._url_key_name}/api/v2/auth/api-token" + auth_headers = {"x-yeti-apikey": self._api_key_name, "User-Agent": "IntelOwl"} + + try: + auth_resp = requests.post( + url=auth_url, + headers=auth_headers, + verify=self.verify_ssl, + timeout=60, + ) + auth_resp.raise_for_status() + access_token = auth_resp.json().get("access_token") + + if not access_token: + raise ConnectorRunException("Failed to obtain access token from YETI.") + except requests.RequestException as e: + raise ConnectorRunException(f"YETI Auth Request failed: {e}") # create observable with `obs_value` if it doesn't exists # new context, tags, source are appended with existing ones + + url = f"{self._url_key_name}/api/v2/observables/extended" + headers = { + "Accept": "application/json", + "User-Agent": "IntelOwl", + "Authorization": f"Bearer {access_token}", + } + try: resp = requests.post( url=url, @@ -65,15 +172,3 @@ def run(self): raise ConnectorRunException(e) return resp.json() - - @classmethod - def _monkeypatch(cls): - patches = [ - if_mock_connections( - patch( - "requests.post", - return_value=MockUpResponse({}, 200), - ) - ) - ] - return super()._monkeypatch(patches=patches) diff --git a/api_app/mixins.py b/api_app/mixins.py index 8b072552b8..d291bb07ae 100644 --- a/api_app/mixins.py +++ b/api_app/mixins.py @@ -819,3 +819,75 @@ def _download_rules( logger.info( f"Rules with version: {latest_version} have been successfully downloaded at {rule_set_directory}" ) + + +class IPQualityScoreMixin: + base_url: str = "https://www.ipqualityscore.com/api/json" # Ensure correct API base + _ipqs_api_key: str + polling_interval: int + max_retries: int + scan_endpoint: str = "/malware/scan/" + lookup_endpoint: str = "/malware/lookup/" + postback_endpoint: str = "/postback/" + + def _make_request( + self, + endpoint: str, + method: str, + _api_key: str = None, + data: Dict = None, + params: Dict = None, + files: Dict = None, + ) -> Dict: + """ + A streamlined request handler with proper timeout management. + """ + url = f"{self.base_url}{endpoint}" + headers = {"IPQS-KEY": _api_key} + + try: + if method.upper() == "POST": + response = requests.post( + url, + headers=headers, + data=data, + files=files, + timeout=60, + ) + else: + response = requests.get(url, headers=headers, json=params, timeout=60) + + response.raise_for_status() + result = response.json() + + # IPQS often returns 200 OK even if the API logic failed + if not result.get("success", True): + raise AnalyzerRunException(f"IPQS API Error: {result.get('message', 'Unknown Error')}") + + return result + + except requests.exceptions.Timeout: + raise AnalyzerRunException("Request timed out after 60s. File might be too large for sync scan.") + except requests.exceptions.JSONDecodeError: + raise AnalyzerRunException(f"Failed to decode JSON. Raw response: {response.text}") + + def _poll_for_report(self, endpoint: str, _api_key: str, request_id: str) -> Dict: + """ + Standardized polling logic for asynchronous retrieval. + """ + if not request_id: + raise AnalyzerRunException("Cannot poll without a valid request_id.") + + params = {"request_id": request_id} + + for attempt in range(self.max_retries): + logger.info(f"Polling attempt {attempt + 1}/{self.max_retries} for ID: {request_id}") + result = self._make_request(endpoint, "GET", _api_key=_api_key, params=params) + + # Check if processing is finished + if result.get("status") != "pending": + break + + logger.info(f"Report pending. Retrying in {self.polling_interval}s...") + time.sleep(self.polling_interval) + return result diff --git a/api_app/playbooks_manager/migrations/0067_add_cve_exploitability_to_free_to_use.py b/api_app/playbooks_manager/migrations/0067_add_cve_exploitability_to_free_to_use.py new file mode 100644 index 0000000000..1da1e4cd19 --- /dev/null +++ b/api_app/playbooks_manager/migrations/0067_add_cve_exploitability_to_free_to_use.py @@ -0,0 +1,41 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + + +from django.db import migrations + +ANALYZER_NAME = "CVE_Exploitability" + + +def migrate(apps, schema_editor): + playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig") + AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig") + + pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS") + pc.analyzers.add(AnalyzerConfig.objects.get(name=ANALYZER_NAME).id) + pc.full_clean() + pc.save() + + +def reverse_migrate(apps, schema_editor): + playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig") + AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig") + + pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS") + pc.analyzers.remove(AnalyzerConfig.objects.get(name=ANALYZER_NAME).id) + pc.full_clean() + pc.save() + + +class Migration(migrations.Migration): + dependencies = [ + ( + "playbooks_manager", + "0066_link_crawl_visualizer_to_playbook", + ), + ("analyzers_manager", "0193_analyzer_config_cve_exploitability"), + ] + + operations = [ + migrations.RunPython(migrate, reverse_migrate), + ] diff --git a/api_app/playbooks_manager/migrations/0068_add_rdap_to_free_to_use.py b/api_app/playbooks_manager/migrations/0068_add_rdap_to_free_to_use.py new file mode 100644 index 0000000000..e45de443f8 --- /dev/null +++ b/api_app/playbooks_manager/migrations/0068_add_rdap_to_free_to_use.py @@ -0,0 +1,34 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + + +from django.db import migrations + + +def migrate(apps, schema_editor): + playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig") + AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig") + pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS") + pc.analyzers.add(AnalyzerConfig.objects.get(name="Rdap").id) + pc.full_clean() + pc.save() + + +def reverse_migrate(apps, schema_editor): + playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig") + AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig") + pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS") + pc.analyzers.remove(AnalyzerConfig.objects.get(name="Rdap").id) + pc.full_clean() + pc.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("playbooks_manager", "0067_add_cve_exploitability_to_free_to_use"), + ("analyzers_manager", "0194_analyzer_config_rdap"), + ] + + operations = [ + migrations.RunPython(migrate, reverse_migrate), + ] diff --git a/api_app/urls.py b/api_app/urls.py index cf7153ae7a..5b8fe3f697 100644 --- a/api_app/urls.py +++ b/api_app/urls.py @@ -56,6 +56,7 @@ path("", include("api_app.pivots_manager.urls")), path("", include("api_app.playbooks_manager.urls")), path("", include("api_app.investigations_manager.urls")), + path("chatbot/", include("api_app.chatbot_manager.urls")), path("data_model/", include("api_app.data_model_manager.urls")), path("user_event/", include("api_app.user_events_manager.urls")), path("", include("api_app.analyzables_manager.urls")), diff --git a/api_app/views.py b/api_app/views.py index 84ca17fc3b..1012cbd269 100644 --- a/api_app/views.py +++ b/api_app/views.py @@ -626,13 +626,7 @@ def aggregate_observable_classification(self, request): """ annotations = { oc.lower(): Count("analyzable__classification", filter=Q(analyzable__classification=oc)) - for oc in [ - Classification.DOMAIN, - Classification.IP, - Classification.HASH, - Classification.URL, - Classification.GENERIC, - ] + for oc in Classification.observable_classifications() } return self.__aggregation_response_static(annotations, users=self.get_org_members(request)) diff --git a/docker/.env b/docker/.env index 60b3677e92..081aa2400e 100644 --- a/docker/.env +++ b/docker/.env @@ -1,6 +1,6 @@ ### DO NOT CHANGE THIS VALUE !! ### It should be updated only when you pull latest changes off from the 'master' branch of IntelOwl. # this variable must start with "REACT_APP_" to be used in the frontend too -REACT_APP_INTELOWL_VERSION=v6.6.1 +REACT_APP_INTELOWL_VERSION=v6.7.0 # if you want to use a nfs volume for shared files # NFS_ADDRESS= diff --git a/docker/Dockerfile b/docker/Dockerfile index d567c4c844..2e46868ee3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -66,7 +66,7 @@ RUN touch ${LOG_PATH}/django/api_app.log ${LOG_PATH}/django/api_app_errors.log \ && touch ${LOG_PATH}/asgi/daphne.log \ && mkdir -p -m 755 ${PYTHONPATH}/.cache \ # download github stuff - && ${PYTHONPATH}/api_app/analyzers_manager/repo_downloader.sh + && ${PYTHONPATH}/api_app/analyzers_manager/repo_downloader.sh || true COPY . $PYTHONPATH diff --git a/docker/Dockerfile_nginx b/docker/Dockerfile_nginx index 684075419d..757be402c7 100644 --- a/docker/Dockerfile_nginx +++ b/docker/Dockerfile_nginx @@ -1,4 +1,4 @@ -FROM library/nginx:1.29.2-alpine +FROM library/nginx:1.31.2-alpine ENV NGINX_LOG_DIR=/var/log/nginx # this is to avoid having these logs redirected to stdout/stderr diff --git a/docker/ci.ollama.override.yml b/docker/ci.ollama.override.yml new file mode 100644 index 0000000000..5a720aef23 --- /dev/null +++ b/docker/ci.ollama.override.yml @@ -0,0 +1,9 @@ +services: + # In CI the chatbot worker must run the freshly built :ci image (like every other service in + # ci.override.yml), not the pulled release image pinned in ollama.override.yml. Without this the + # worker would run stale released code and, before a release ships the chatbot, would crash on + # the missing celery_chatbot.sh entrypoint. No source mount: CI bakes the source into the image. + celery_worker_chatbot: + image: intelowlproject/intelowl:ci + env_file: + - env_file_app_ci diff --git a/docker/entrypoints/celery_chatbot.sh b/docker/entrypoints/celery_chatbot.sh new file mode 100755 index 0000000000..6213306709 --- /dev/null +++ b/docker/entrypoints/celery_chatbot.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +until cd /opt/deploy/intel_owl +do + echo "Waiting for server volume..." +done + +if [ "$AWS_SQS" = "True" ] +then + queues="chatbot.fifo,config.fifo" +else + queues="chatbot,broadcast,config" +fi + +# Concurrency is intentionally low (-c 2): a chat turn is a long, LLM-bound ReAct run +# (soft_time_limit=300) that ends up serialized on the single Ollama backend, so a large pool +# would only spawn idle workers all blocked on the model. --time-limit=600 is a hard backstop +# comfortably above the task's 300s soft limit; it deliberately overrides the global +# task_time_limit=1800 (intel_owl/celery.py) for this LLM worker so a hung turn is reclaimed +# sooner. +ARGUMENTS="-A intel_owl.celery worker -n worker_chatbot --uid www-data --gid www-data --time-limit=600 --pidfile= -c 2 -Ofair -Q ${queues} -E --without-gossip" +if [[ $DEBUG == "True" ]] && [[ $DJANGO_TEST_SERVER == "True" ]]; +then + echo "Running celery with autoreload" + python3 manage.py celery_reload -c "$ARGUMENTS" +else + # shellcheck disable=SC2086 + /usr/local/bin/celery $ARGUMENTS +fi diff --git a/docker/entrypoints/ollama.sh b/docker/entrypoints/ollama.sh new file mode 100755 index 0000000000..112458164a --- /dev/null +++ b/docker/entrypoints/ollama.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# Start the Ollama server and ensure the default chat model is pulled. +# The model is configured via the OLLAMA_MODEL env var (set in env_file_app). + +set -e + +# Must stay in sync with the OLLAMA_MODEL defaults in intel_owl/settings/chatbot.py and +# docker/env_file_app_template; must be a model that supports Ollama tool calling. +DEFAULT_MODEL="qwen2.5:3b" +MODEL="${OLLAMA_MODEL:-$DEFAULT_MODEL}" + +echo "[ollama] Starting server..." +ollama serve & +SERVER_PID=$! + +echo "[ollama] Waiting for server to accept connections..." +until ollama list >/dev/null 2>&1; do + sleep 1 +done + +echo "[ollama] Server ready. Checking for model '${MODEL}'..." +if ollama list | awk 'NR>1 {print $1}' | grep -Fxq "${MODEL}"; then + echo "[ollama] Model '${MODEL}' already present, skipping pull." +else + echo "[ollama] Pulling model '${MODEL}'..." + PULL_OK=false + for attempt in 1 2 3; do + if ollama pull "${MODEL}"; then + PULL_OK=true + break + fi + echo "[ollama] Pull attempt ${attempt}/3 failed; retrying in 10s..." + sleep 10 + done + if [ "${PULL_OK}" != "true" ]; then + echo "[ollama] WARN: pull of '${MODEL}' failed after 3 attempts. Server will continue running without it." + fi +fi + +echo "[ollama] Ready. PID=${SERVER_PID}" +wait "${SERVER_PID}" diff --git a/docker/env_file_app_template b/docker/env_file_app_template index 650e96e3bc..8383cb0066 100644 --- a/docker/env_file_app_template +++ b/docker/env_file_app_template @@ -84,3 +84,12 @@ WEBSOCKETS_URL=redis://redis:6379/0 FLOWER_USER=flower FLOWER_PWD=flower + +# LLM Chatbot +OLLAMA_BASE_URL=http://ollama:11434 +# must be a model that supports Ollama tool calling +OLLAMA_MODEL=qwen2.5:3b +# chat sessions whose last activity is older than this are flushed periodically. Default: 90 days +CHATBOT_MESSAGE_RETENTION_DAYS=90 +CHATBOT_RATE_LIMIT=5 +CHATBOT_RATE_LIMIT_WINDOW=60 diff --git a/docker/ollama.override.yml b/docker/ollama.override.yml new file mode 100644 index 0000000000..80e96a3d1d --- /dev/null +++ b/docker/ollama.override.yml @@ -0,0 +1,55 @@ +services: + ollama: + # 0.8.0+ is required to stream responses while tools are bound (older servers return the + # whole message in one chunk, breaking token-by-token chat streaming); 0.30.7 is the + # latest stable at the time of writing. + image: ollama/ollama:0.30.7 + container_name: intelowl_ollama + hostname: ollama + restart: unless-stopped + volumes: + - ollama_data:/root/.ollama + - ./entrypoints/ollama.sh:/usr/local/bin/intelowl-ollama-entrypoint.sh:ro + expose: + - "11434" + env_file: + - env_file_app + entrypoint: + - /bin/sh + - /usr/local/bin/intelowl-ollama-entrypoint.sh + healthcheck: + test: ["CMD-SHELL", "ollama list >/dev/null 2>&1"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 180s + + # Dedicated worker that drains the chatbot queue (process_chat_message). It lives with the + # ollama stack so the same --ollama flag that starts the LLM also starts the worker that uses + # it; without this service the chatbot queue would have no consumer and chat turns would hang. + celery_worker_chatbot: + image: intelowlproject/intelowl:${REACT_APP_INTELOWL_VERSION} + container_name: intelowl_celery_worker_chatbot + hostname: celery_worker_chatbot + restart: unless-stopped + stop_grace_period: 3m + volumes: + - ../configuration:/opt/deploy/intel_owl/configuration + - generic_logs:/var/log/intel_owl + - shared_files:/opt/deploy/files_required + entrypoint: + - ./docker/entrypoints/celery_chatbot.sh + env_file: + - env_file_app + depends_on: + redis: + condition: service_started + uwsgi: + condition: service_healthy + ollama: + condition: service_healthy + healthcheck: + disable: true + +volumes: + ollama_data: diff --git a/docker/test.ollama.override.yml b/docker/test.ollama.override.yml new file mode 100644 index 0000000000..b2e18598db --- /dev/null +++ b/docker/test.ollama.override.yml @@ -0,0 +1,7 @@ +services: + celery_worker_chatbot: + image: intelowlproject/intelowl:test + volumes: + - ../:/opt/deploy/intel_owl + environment: + - DEBUG=True diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 34d3eefb73..57fff84470 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "intelowl", - "version": "6.4.0", + "version": "6.6.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "intelowl", - "version": "6.4.0", + "version": "6.6.1", "dependencies": { "@certego/certego-ui": "^0.1.19", "@dagrejs/dagre": "^1.1.4", @@ -19,6 +19,7 @@ "date-fns-tz": "^3.2.0", "flag-icons": "^7.2.3", "formik": "^2.4.6", + "http-proxy-middleware": "^2.0.6", "js-cookie": "^3.0.5", "md5": "^2.3.0", "prop-types": "^15.8.1", @@ -37,11 +38,13 @@ "reactflow": "^11.11.4", "reactstrap": "^9.2.3", "recharts": "^2.13.0", + "remark-gfm": "^3.0.1", "zustand": "^4.5.4" }, "devDependencies": { "@babel/preset-env": "^7.25.8", "@babel/preset-react": "^7.25.7", + "@testing-library/dom": "^10.4.1", "@testing-library/jest-dom": "^6.6.3", "@testing-library/react": "^12.1.5", "@testing-library/user-event": "^14.5.2", @@ -55,7 +58,6 @@ "eslint-plugin-jsx-a11y": "^6.10.0", "eslint-plugin-react": "^7.37.1", "eslint-plugin-react-hooks": "^4.6.2", - "http-proxy-middleware": "^2.0.6", "jest": "^29.7.0", "jest-environment-jsdom": "^29.7.0", "prettier": "^3.3.3", @@ -4295,92 +4297,33 @@ } }, "node_modules/@testing-library/dom": { - "version": "9.3.1", - "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-9.3.1.tgz", - "integrity": "sha512-0DGPd9AR3+iDTjGoMpxIkAsUihHZ3Ai6CneU6bRRrffXMgzCdlNk43jTrD2/5LT6CBb3MWTP8v510JzYtahD2w==", + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz", + "integrity": "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==", "dev": true, + "license": "MIT", "dependencies": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", "@types/aria-query": "^5.0.1", - "aria-query": "5.1.3", - "chalk": "^4.1.0", + "aria-query": "5.3.0", "dom-accessibility-api": "^0.5.9", "lz-string": "^1.5.0", + "picocolors": "1.1.1", "pretty-format": "^27.0.2" }, "engines": { - "node": ">=14" - } - }, - "node_modules/@testing-library/dom/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@testing-library/dom/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" + "node": ">=18" } }, - "node_modules/@testing-library/dom/node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/@testing-library/dom/node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true - }, - "node_modules/@testing-library/dom/node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/@testing-library/dom/node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "node_modules/@testing-library/dom/node_modules/aria-query": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", + "integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==", "dev": true, + "license": "Apache-2.0", "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" + "dequal": "^2.0.3" } }, "node_modules/@testing-library/jest-dom": { @@ -7030,6 +6973,15 @@ "node": ">=4" } }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -14938,6 +14890,15 @@ "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==" }, + "node_modules/longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -15046,6 +15007,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/markdown-table": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", + "integrity": "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/match-sorter": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/match-sorter/-/match-sorter-6.3.1.tgz", @@ -15089,6 +15059,32 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/mdast-util-find-and-replace": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-2.2.2.tgz", + "integrity": "sha512-MTtdFRz/eMDHXzeK6W3dO7mXUlF82Gom4y0oOgvHhh/HXZAGvIQDUvQ0SuUx+j2tv44b8xTHOm8K/9OoRFnXKw==", + "dependencies": { + "@types/mdast": "^3.0.0", + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^5.0.0", + "unist-util-visit-parents": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-find-and-replace/node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/mdast-util-from-markdown": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-1.3.1.tgz", @@ -15112,6 +15108,107 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/mdast-util-gfm": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-2.0.2.tgz", + "integrity": "sha512-qvZ608nBppZ4icQlhQQIAdc6S3Ffj9RGmzwUKUWuEICFnd1LVkN3EktF7ZHAgfcEdvZB5owU9tQgt99e2TlLjg==", + "dependencies": { + "mdast-util-from-markdown": "^1.0.0", + "mdast-util-gfm-autolink-literal": "^1.0.0", + "mdast-util-gfm-footnote": "^1.0.0", + "mdast-util-gfm-strikethrough": "^1.0.0", + "mdast-util-gfm-table": "^1.0.0", + "mdast-util-gfm-task-list-item": "^1.0.0", + "mdast-util-to-markdown": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-1.0.3.tgz", + "integrity": "sha512-My8KJ57FYEy2W2LyNom4n3E7hKTuQk/0SES0u16tjA9Z3oFkF4RrC/hPAPgjlSpezsOvI8ObcXcElo92wn5IGA==", + "dependencies": { + "@types/mdast": "^3.0.0", + "ccount": "^2.0.0", + "mdast-util-find-and-replace": "^2.0.0", + "micromark-util-character": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-footnote": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-1.0.2.tgz", + "integrity": "sha512-56D19KOGbE00uKVj3sgIykpwKL179QsVFwx/DCW0u/0+URsryacI4MAdNJl0dh+u2PSsD9FtxPFbHCzJ78qJFQ==", + "dependencies": { + "@types/mdast": "^3.0.0", + "mdast-util-to-markdown": "^1.3.0", + "micromark-util-normalize-identifier": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-strikethrough": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-1.0.3.tgz", + "integrity": "sha512-DAPhYzTYrRcXdMjUtUjKvW9z/FNAMTdU0ORyMcbmkwYNbKocDpdk+PX1L1dQgOID/+vVs1uBQ7ElrBQfZ0cuiQ==", + "dependencies": { + "@types/mdast": "^3.0.0", + "mdast-util-to-markdown": "^1.3.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-table": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-1.0.7.tgz", + "integrity": "sha512-jjcpmNnQvrmN5Vx7y7lEc2iIOEytYv7rTvu+MeyAsSHTASGCCRA79Igg2uKssgOs1i1po8s3plW0sTu1wkkLGg==", + "dependencies": { + "@types/mdast": "^3.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^1.0.0", + "mdast-util-to-markdown": "^1.3.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-task-list-item": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-1.0.2.tgz", + "integrity": "sha512-PFTA1gzfp1B1UaiJVyhJZA1rm0+Tzn690frc/L8vNX1Jop4STZgOE6bxUhnzdVSB+vm2GU1tIsuQcA9bxTQpMQ==", + "dependencies": { + "@types/mdast": "^3.0.0", + "mdast-util-to-markdown": "^1.3.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-phrasing": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-3.0.1.tgz", + "integrity": "sha512-WmI1gTXUBJo4/ZmSk79Wcb2HcjPJBzM1nlI/OUWA8yk2X9ik3ffNbBGsU+09BFmXaL1IBb9fiuvq6/KMiNycSg==", + "dependencies": { + "@types/mdast": "^3.0.0", + "unist-util-is": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/mdast-util-to-hast": { "version": "12.3.0", "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-12.3.0.tgz", @@ -15131,6 +15228,25 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/mdast-util-to-markdown": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-1.5.0.tgz", + "integrity": "sha512-bbv7TPv/WC49thZPg3jXuqzuvI45IL2EVAr/KxF0BSdHsU0ceFHOmwQn6evxAh1GaoK/6GQ1wp4R4oW2+LFL/A==", + "dependencies": { + "@types/mdast": "^3.0.0", + "@types/unist": "^2.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^3.0.0", + "mdast-util-to-string": "^3.0.0", + "micromark-util-decode-string": "^1.0.0", + "unist-util-visit": "^4.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/mdast-util-to-string": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-3.2.0.tgz", @@ -15303,6 +15419,120 @@ "uvu": "^0.5.0" } }, + "node_modules/micromark-extension-gfm": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-2.0.3.tgz", + "integrity": "sha512-vb9OoHqrhCmbRidQv/2+Bc6pkP0FrtlhurxZofvOEy5o8RtuuvTq+RQ1Vw5ZDNrVraQZu3HixESqbG+0iKk/MQ==", + "dependencies": { + "micromark-extension-gfm-autolink-literal": "^1.0.0", + "micromark-extension-gfm-footnote": "^1.0.0", + "micromark-extension-gfm-strikethrough": "^1.0.0", + "micromark-extension-gfm-table": "^1.0.0", + "micromark-extension-gfm-tagfilter": "^1.0.0", + "micromark-extension-gfm-task-list-item": "^1.0.0", + "micromark-util-combine-extensions": "^1.0.0", + "micromark-util-types": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-1.0.5.tgz", + "integrity": "sha512-z3wJSLrDf8kRDOh2qBtoTRD53vJ+CWIyo7uyZuxf/JAbNJjiHsOpG1y5wxk8drtv3ETAHutCu6N3thkOOgueWg==", + "dependencies": { + "micromark-util-character": "^1.0.0", + "micromark-util-sanitize-uri": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-1.1.2.tgz", + "integrity": "sha512-Yxn7z7SxgyGWRNa4wzf8AhYYWNrwl5q1Z8ii+CSTTIqVkmGZF1CElX2JI8g5yGoM3GAman9/PVCUFUSJ0kB/8Q==", + "dependencies": { + "micromark-core-commonmark": "^1.0.0", + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-normalize-identifier": "^1.0.0", + "micromark-util-sanitize-uri": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-strikethrough": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-1.0.7.tgz", + "integrity": "sha512-sX0FawVE1o3abGk3vRjOH50L5TTLr3b5XMqnP9YDRb34M0v5OoZhG+OHFz1OffZ9dlwgpTBKaT4XW/AsUVnSDw==", + "dependencies": { + "micromark-util-chunked": "^1.0.0", + "micromark-util-classify-character": "^1.0.0", + "micromark-util-resolve-all": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-1.0.7.tgz", + "integrity": "sha512-3ZORTHtcSnMQEKtAOsBQ9/oHp9096pI/UvdPtN7ehKvrmZZ2+bbWhi0ln+I9drmwXMt5boocn6OlwQzNXeVeqw==", + "dependencies": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-1.0.2.tgz", + "integrity": "sha512-5XWB9GbAUSHTn8VPU8/1DBXMuKYT5uOgEjJb8gN3mW0PNW5OPHpSdojoqf+iq1xo7vWzw/P8bAHY0n6ijpXF7g==", + "dependencies": { + "micromark-util-types": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-1.0.5.tgz", + "integrity": "sha512-RMFXl2uQ0pNQy6Lun2YBYT9g9INXtWJULgbt01D/x8/6yJ2qpKyzdZD3pi6UIkzF++Da49xAelVKUeUMqd5eIQ==", + "dependencies": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/micromark-factory-destination": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-1.1.0.tgz", @@ -16498,9 +16728,10 @@ "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, "node_modules/picocolors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", - "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==" + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" }, "node_modules/picomatch": { "version": "2.3.1", @@ -20788,6 +21019,21 @@ "node": ">= 0.10" } }, + "node_modules/remark-gfm": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-3.0.1.tgz", + "integrity": "sha512-lEFDoi2PICJyNrACFOfDD3JlLkuSbOa5Wd8EPt06HUdptv8Gn0bxYTdbU/XXQ3swAPkEaGxxPN9cbnMHvVu1Ig==", + "dependencies": { + "@types/mdast": "^3.0.0", + "mdast-util-gfm": "^2.0.0", + "micromark-extension-gfm": "^2.0.0", + "unified": "^10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/remark-parse": { "version": "10.0.2", "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-10.0.2.tgz", @@ -24563,6 +24809,15 @@ "optional": true } } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } } }, "dependencies": { @@ -27332,68 +27587,28 @@ } }, "@testing-library/dom": { - "version": "9.3.1", - "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-9.3.1.tgz", - "integrity": "sha512-0DGPd9AR3+iDTjGoMpxIkAsUihHZ3Ai6CneU6bRRrffXMgzCdlNk43jTrD2/5LT6CBb3MWTP8v510JzYtahD2w==", + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz", + "integrity": "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==", "dev": true, "requires": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", "@types/aria-query": "^5.0.1", - "aria-query": "5.1.3", - "chalk": "^4.1.0", + "aria-query": "5.3.0", "dom-accessibility-api": "^0.5.9", "lz-string": "^1.5.0", + "picocolors": "1.1.1", "pretty-format": "^27.0.2" }, "dependencies": { - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "aria-query": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", + "integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==", "dev": true, "requires": { - "color-convert": "^2.0.1" - } - }, - "chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "requires": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - } - }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true - }, - "has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true - }, - "supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "requires": { - "has-flag": "^4.0.0" + "dequal": "^2.0.3" } } } @@ -29498,6 +29713,11 @@ "resolved": "https://registry.npmjs.org/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.4.0.tgz", "integrity": "sha512-roIFONhcxog0JSSWbvVAh3OocukmSgpqOH6YpMkCvav/ySIV3JKg4Dc8vYtQjYi/UxpNE36r/9v+VqTQqgkYmw==" }, + "ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==" + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -35287,6 +35507,11 @@ "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==" }, + "longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==" + }, "loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -35370,6 +35595,11 @@ "integrity": "sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==", "dev": true }, + "markdown-table": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", + "integrity": "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==" + }, "match-sorter": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/match-sorter/-/match-sorter-6.3.1.tgz", @@ -35405,6 +35635,24 @@ "unist-util-visit": "^4.0.0" } }, + "mdast-util-find-and-replace": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-2.2.2.tgz", + "integrity": "sha512-MTtdFRz/eMDHXzeK6W3dO7mXUlF82Gom4y0oOgvHhh/HXZAGvIQDUvQ0SuUx+j2tv44b8xTHOm8K/9OoRFnXKw==", + "requires": { + "@types/mdast": "^3.0.0", + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^5.0.0", + "unist-util-visit-parents": "^5.0.0" + }, + "dependencies": { + "escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==" + } + } + }, "mdast-util-from-markdown": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-1.3.1.tgz", @@ -35424,6 +35672,79 @@ "uvu": "^0.5.0" } }, + "mdast-util-gfm": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-2.0.2.tgz", + "integrity": "sha512-qvZ608nBppZ4icQlhQQIAdc6S3Ffj9RGmzwUKUWuEICFnd1LVkN3EktF7ZHAgfcEdvZB5owU9tQgt99e2TlLjg==", + "requires": { + "mdast-util-from-markdown": "^1.0.0", + "mdast-util-gfm-autolink-literal": "^1.0.0", + "mdast-util-gfm-footnote": "^1.0.0", + "mdast-util-gfm-strikethrough": "^1.0.0", + "mdast-util-gfm-table": "^1.0.0", + "mdast-util-gfm-task-list-item": "^1.0.0", + "mdast-util-to-markdown": "^1.0.0" + } + }, + "mdast-util-gfm-autolink-literal": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-1.0.3.tgz", + "integrity": "sha512-My8KJ57FYEy2W2LyNom4n3E7hKTuQk/0SES0u16tjA9Z3oFkF4RrC/hPAPgjlSpezsOvI8ObcXcElo92wn5IGA==", + "requires": { + "@types/mdast": "^3.0.0", + "ccount": "^2.0.0", + "mdast-util-find-and-replace": "^2.0.0", + "micromark-util-character": "^1.0.0" + } + }, + "mdast-util-gfm-footnote": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-1.0.2.tgz", + "integrity": "sha512-56D19KOGbE00uKVj3sgIykpwKL179QsVFwx/DCW0u/0+URsryacI4MAdNJl0dh+u2PSsD9FtxPFbHCzJ78qJFQ==", + "requires": { + "@types/mdast": "^3.0.0", + "mdast-util-to-markdown": "^1.3.0", + "micromark-util-normalize-identifier": "^1.0.0" + } + }, + "mdast-util-gfm-strikethrough": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-1.0.3.tgz", + "integrity": "sha512-DAPhYzTYrRcXdMjUtUjKvW9z/FNAMTdU0ORyMcbmkwYNbKocDpdk+PX1L1dQgOID/+vVs1uBQ7ElrBQfZ0cuiQ==", + "requires": { + "@types/mdast": "^3.0.0", + "mdast-util-to-markdown": "^1.3.0" + } + }, + "mdast-util-gfm-table": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-1.0.7.tgz", + "integrity": "sha512-jjcpmNnQvrmN5Vx7y7lEc2iIOEytYv7rTvu+MeyAsSHTASGCCRA79Igg2uKssgOs1i1po8s3plW0sTu1wkkLGg==", + "requires": { + "@types/mdast": "^3.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^1.0.0", + "mdast-util-to-markdown": "^1.3.0" + } + }, + "mdast-util-gfm-task-list-item": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-1.0.2.tgz", + "integrity": "sha512-PFTA1gzfp1B1UaiJVyhJZA1rm0+Tzn690frc/L8vNX1Jop4STZgOE6bxUhnzdVSB+vm2GU1tIsuQcA9bxTQpMQ==", + "requires": { + "@types/mdast": "^3.0.0", + "mdast-util-to-markdown": "^1.3.0" + } + }, + "mdast-util-phrasing": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-3.0.1.tgz", + "integrity": "sha512-WmI1gTXUBJo4/ZmSk79Wcb2HcjPJBzM1nlI/OUWA8yk2X9ik3ffNbBGsU+09BFmXaL1IBb9fiuvq6/KMiNycSg==", + "requires": { + "@types/mdast": "^3.0.0", + "unist-util-is": "^5.0.0" + } + }, "mdast-util-to-hast": { "version": "12.3.0", "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-12.3.0.tgz", @@ -35439,6 +35760,21 @@ "unist-util-visit": "^4.0.0" } }, + "mdast-util-to-markdown": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-1.5.0.tgz", + "integrity": "sha512-bbv7TPv/WC49thZPg3jXuqzuvI45IL2EVAr/KxF0BSdHsU0ceFHOmwQn6evxAh1GaoK/6GQ1wp4R4oW2+LFL/A==", + "requires": { + "@types/mdast": "^3.0.0", + "@types/unist": "^2.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^3.0.0", + "mdast-util-to-string": "^3.0.0", + "micromark-util-decode-string": "^1.0.0", + "unist-util-visit": "^4.0.0", + "zwitch": "^2.0.0" + } + }, "mdast-util-to-string": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-3.2.0.tgz", @@ -35565,6 +35901,92 @@ "uvu": "^0.5.0" } }, + "micromark-extension-gfm": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-2.0.3.tgz", + "integrity": "sha512-vb9OoHqrhCmbRidQv/2+Bc6pkP0FrtlhurxZofvOEy5o8RtuuvTq+RQ1Vw5ZDNrVraQZu3HixESqbG+0iKk/MQ==", + "requires": { + "micromark-extension-gfm-autolink-literal": "^1.0.0", + "micromark-extension-gfm-footnote": "^1.0.0", + "micromark-extension-gfm-strikethrough": "^1.0.0", + "micromark-extension-gfm-table": "^1.0.0", + "micromark-extension-gfm-tagfilter": "^1.0.0", + "micromark-extension-gfm-task-list-item": "^1.0.0", + "micromark-util-combine-extensions": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "micromark-extension-gfm-autolink-literal": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-1.0.5.tgz", + "integrity": "sha512-z3wJSLrDf8kRDOh2qBtoTRD53vJ+CWIyo7uyZuxf/JAbNJjiHsOpG1y5wxk8drtv3ETAHutCu6N3thkOOgueWg==", + "requires": { + "micromark-util-character": "^1.0.0", + "micromark-util-sanitize-uri": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "micromark-extension-gfm-footnote": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-1.1.2.tgz", + "integrity": "sha512-Yxn7z7SxgyGWRNa4wzf8AhYYWNrwl5q1Z8ii+CSTTIqVkmGZF1CElX2JI8g5yGoM3GAman9/PVCUFUSJ0kB/8Q==", + "requires": { + "micromark-core-commonmark": "^1.0.0", + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-normalize-identifier": "^1.0.0", + "micromark-util-sanitize-uri": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "micromark-extension-gfm-strikethrough": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-1.0.7.tgz", + "integrity": "sha512-sX0FawVE1o3abGk3vRjOH50L5TTLr3b5XMqnP9YDRb34M0v5OoZhG+OHFz1OffZ9dlwgpTBKaT4XW/AsUVnSDw==", + "requires": { + "micromark-util-chunked": "^1.0.0", + "micromark-util-classify-character": "^1.0.0", + "micromark-util-resolve-all": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "micromark-extension-gfm-table": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-1.0.7.tgz", + "integrity": "sha512-3ZORTHtcSnMQEKtAOsBQ9/oHp9096pI/UvdPtN7ehKvrmZZ2+bbWhi0ln+I9drmwXMt5boocn6OlwQzNXeVeqw==", + "requires": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "micromark-extension-gfm-tagfilter": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-1.0.2.tgz", + "integrity": "sha512-5XWB9GbAUSHTn8VPU8/1DBXMuKYT5uOgEjJb8gN3mW0PNW5OPHpSdojoqf+iq1xo7vWzw/P8bAHY0n6ijpXF7g==", + "requires": { + "micromark-util-types": "^1.0.0" + } + }, + "micromark-extension-gfm-task-list-item": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-1.0.5.tgz", + "integrity": "sha512-RMFXl2uQ0pNQy6Lun2YBYT9g9INXtWJULgbt01D/x8/6yJ2qpKyzdZD3pi6UIkzF++Da49xAelVKUeUMqd5eIQ==", + "requires": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, "micromark-factory-destination": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-1.1.0.tgz", @@ -36337,9 +36759,9 @@ "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, "picocolors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", - "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==" + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" }, "picomatch": { "version": "2.3.1", @@ -39330,6 +39752,17 @@ "resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz", "integrity": "sha512-G08Dxvm4iDN3MLM0EsP62EDV9IuhXPR6blNz6Utcp7zyV3tr4HVNINt6MpaRWbxoOHT3Q7YN2P+jaHX8vUbgog==" }, + "remark-gfm": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-3.0.1.tgz", + "integrity": "sha512-lEFDoi2PICJyNrACFOfDD3JlLkuSbOa5Wd8EPt06HUdptv8Gn0bxYTdbU/XXQ3swAPkEaGxxPN9cbnMHvVu1Ig==", + "requires": { + "@types/mdast": "^3.0.0", + "mdast-util-gfm": "^2.0.0", + "micromark-extension-gfm": "^2.0.0", + "unified": "^10.0.0" + } + }, "remark-parse": { "version": "10.0.2", "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-10.0.2.tgz", @@ -42127,6 +42560,11 @@ "requires": { "use-sync-external-store": "1.2.2" } + }, + "zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==" } } } diff --git a/frontend/package.json b/frontend/package.json index 19b2c65d74..7e84163688 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "intelowl", - "version": "6.6.1", + "version": "6.7.0", "private": true, "dependencies": { "@certego/certego-ui": "^0.1.19", @@ -33,6 +33,7 @@ "reactflow": "^11.11.4", "reactstrap": "^9.2.3", "recharts": "^2.13.0", + "remark-gfm": "^3.0.1", "zustand": "^4.5.4" }, "scripts": { diff --git a/frontend/src/components/chat/ChatActionConfirm.jsx b/frontend/src/components/chat/ChatActionConfirm.jsx new file mode 100644 index 0000000000..b3341d065e --- /dev/null +++ b/frontend/src/components/chat/ChatActionConfirm.jsx @@ -0,0 +1,79 @@ +import React from "react"; +import { Card, CardBody, Button } from "reactstrap"; + +import { useChatStore } from "../../stores/useChatStore"; + +/** + * Confirmation card for an analyze_observable preview. The agent can only propose an analysis + * (it returns a plan + pending_id); the actual launch happens here, on an explicit user click, + * via the backend confirm endpoint — so the model can never start an analysis on its own. + * Renders nothing unless the store holds a pending action. + */ +export function ChatActionConfirm() { + const pendingAction = useChatStore((state) => state.pendingAction); + const confirmPendingAction = useChatStore( + (state) => state.confirmPendingAction, + ); + const cancelPendingAction = useChatStore( + (state) => state.cancelPendingAction, + ); + const isStreaming = useChatStore((state) => state.isStreaming); + const [submitting, setSubmitting] = React.useState(false); + + if (!pendingAction) return null; + const { plan } = pendingAction; + // The backend `action_required` plan always carries these lists (AnalysisPlanSerializer), but + // default to [] so a malformed/partial frame degrades to an empty row instead of crashing render. + const analyzers = plan.analyzers ?? []; + const connectors = plan.connectors ?? []; + const skipped = plan.skipped ?? []; + + const onConfirm = async () => { + setSubmitting(true); + try { + await confirmPendingAction(); + } finally { + setSubmitting(false); + } + }; + + return ( + + +
Start this analysis?
+
+ Observable: {plan.observable_name} ({plan.classification}) +
+
TLP: {plan.tlp}
+ {plan.playbook &&
Playbook: {plan.playbook}
} +
Analyzers: {analyzers.join(", ") || "default"}
+ {connectors.length > 0 && ( +
Connectors: {connectors.join(", ")}
+ )} + {skipped.length > 0 && ( +
Skipped: {skipped.join(", ")}
+ )} +
+ {/* Confirm is also gated on isStreaming (can't launch mid-turn); Cancel deliberately is + NOT — dismissing a stale card must always work, even while a reply streams. */} + + +
+ + + ); +} diff --git a/frontend/src/components/chat/ChatComposer.jsx b/frontend/src/components/chat/ChatComposer.jsx new file mode 100644 index 0000000000..3abdb9ac15 --- /dev/null +++ b/frontend/src/components/chat/ChatComposer.jsx @@ -0,0 +1,64 @@ +import React from "react"; +import PropTypes from "prop-types"; +import { Button, Input } from "reactstrap"; +import { IoSend } from "react-icons/io5"; + +import { useChatStore, ConnectionState } from "../../stores/useChatStore"; + +// Mirror MessageRequestSerializer(message=CharField(max_length=4096)) so the UI caps input before +// the server rejects it with INVALID_MESSAGE. +const MAX_MESSAGE_LEN = 4096; + +/** + * Message input. Enter sends, Shift+Enter inserts a newline. Disabled while a turn is streaming or + * the socket is not connected, so the user can't fire a second turn before the first finishes. + */ +export function ChatComposer({ onSend }) { + const [text, setText] = React.useState(""); + const isStreaming = useChatStore((state) => state.isStreaming); + const connectionState = useChatStore((state) => state.connectionState); + const disabled = isStreaming || connectionState !== ConnectionState.CONNECTED; + + const submit = () => { + const trimmed = text.trim(); + if (!trimmed || disabled) return; + onSend(trimmed); + setText(""); + }; + + const handleKeyDown = (event) => { + if (event.key === "Enter" && !event.shiftKey) { + event.preventDefault(); + submit(); + } + }; + + return ( +
+ setText(event.target.value)} + onKeyDown={handleKeyDown} + disabled={disabled} + /> + +
+ ); +} + +ChatComposer.propTypes = { + onSend: PropTypes.func.isRequired, +}; diff --git a/frontend/src/components/chat/ChatMessage.jsx b/frontend/src/components/chat/ChatMessage.jsx new file mode 100644 index 0000000000..342d822201 --- /dev/null +++ b/frontend/src/components/chat/ChatMessage.jsx @@ -0,0 +1,40 @@ +import React from "react"; +import PropTypes from "prop-types"; + +import { chatMarkdownToHtml } from "./chatMarkdown"; +import { MessageRole } from "../../stores/useChatStore"; + +/** + * A single chat bubble. User messages render as plain (newline-preserving) text and align right; + * assistant messages render markdown (GFM) and align left. + */ +export function ChatMessage({ role, content }) { + const isUser = role === MessageRole.USER; + return ( +
+
+ {isUser ? ( + {content} + ) : ( + chatMarkdownToHtml(content) + )} +
+
+ ); +} + +ChatMessage.propTypes = { + role: PropTypes.string.isRequired, + content: PropTypes.string.isRequired, +}; diff --git a/frontend/src/components/chat/ChatMessageList.jsx b/frontend/src/components/chat/ChatMessageList.jsx new file mode 100644 index 0000000000..a6d5b4366c --- /dev/null +++ b/frontend/src/components/chat/ChatMessageList.jsx @@ -0,0 +1,51 @@ +import React from "react"; +import { Spinner } from "reactstrap"; + +import { ChatMessage } from "./ChatMessage"; +import { useChatStore, MessageRole } from "../../stores/useChatStore"; + +/** + * Scrollable conversation view: committed messages, plus the in-progress assistant turn (a + * "Running …/Thinking…" indicator before any text arrives, then the live streaming bubble). + * Autoscrolls to the bottom as new content lands. + */ +export function ChatMessageList() { + const messages = useChatStore((state) => state.messages); + const streamingText = useChatStore((state) => state.streamingText); + const currentTool = useChatStore((state) => state.currentTool); + const isStreaming = useChatStore((state) => state.isStreaming); + const historyLoading = useChatStore((state) => state.historyLoading); + const bottomRef = React.useRef(null); + + React.useEffect(() => { + bottomRef.current?.scrollIntoView({ behavior: "smooth" }); + }, [messages, streamingText, currentTool]); + + return ( +
+ {historyLoading && ( +
+ +
+ )} + {messages.map((message, index) => ( + + ))} + {isStreaming && !streamingText && ( +
+ + {currentTool ? `Running ${currentTool}…` : "Thinking…"} +
+ )} + {isStreaming && streamingText && ( + + )} +
+
+ ); +} diff --git a/frontend/src/components/chat/ChatPanel.jsx b/frontend/src/components/chat/ChatPanel.jsx new file mode 100644 index 0000000000..6ecf18bace --- /dev/null +++ b/frontend/src/components/chat/ChatPanel.jsx @@ -0,0 +1,148 @@ +import React from "react"; +import { + Offcanvas, + OffcanvasHeader, + OffcanvasBody, + Badge, + Alert, + Button, + UncontrolledTooltip, +} from "reactstrap"; +import { IoListOutline, IoArrowBack } from "react-icons/io5"; +import { MdInfoOutline } from "react-icons/md"; + +import { useChatStore, ConnectionState } from "../../stores/useChatStore"; +import { useChatWebSocket } from "./useChatWebSocket"; +import { ChatMessageList } from "./ChatMessageList"; +import { ChatComposer } from "./ChatComposer"; +import { ChatSessionList } from "./ChatSessionList"; +import { QuickActions } from "./QuickActions"; +import { ChatActionConfirm } from "./ChatActionConfirm"; + +// Connection-state badge shown in the drawer header. +const CONNECTION_BADGE = { + [ConnectionState.IDLE]: { color: "secondary", label: "Idle" }, + [ConnectionState.CONNECTING]: { color: "warning", label: "Connecting" }, + [ConnectionState.CONNECTED]: { color: "success", label: "Connected" }, + [ConnectionState.RECONNECTING]: { color: "warning", label: "Reconnecting" }, + [ConnectionState.CLOSED]: { color: "danger", label: "Disconnected" }, +}; + +// Shown instead of the green "Connected" badge when the socket is up but the chatbot worker isn't +// serving turns (see useChatStore.assistantUnavailable). +const UNAVAILABLE_BADGE = { color: "warning", label: "Unavailable" }; + +// One-line description behind the header info icon, mirroring the MdInfoOutline + tooltip pattern +// used elsewhere in IntelOwl (e.g. TLPSelectInput). +const ASSISTANT_INFO_TEXT = + "Privacy-preserving assistant: ask about your IntelOwl data in natural language. " + + "All inference runs locally via Ollama — nothing is sent to external services."; + +/** + * The chat drawer. Mounted once, globally (AppMain Layout), so it overlays every authenticated + * page and survives open/close; the WebSocket lives in useChatWebSocket, which lazily connects the + * first time the drawer opens. `backdrop={false}` keeps the rest of the app usable while open. + */ +export function ChatPanel() { + const isOpen = useChatStore((state) => state.isOpen); + const close = useChatStore((state) => state.close); + const connectionState = useChatStore((state) => state.connectionState); + const isStreaming = useChatStore((state) => state.isStreaming); + const assistantUnavailable = useChatStore( + (state) => state.assistantUnavailable, + ); + const error = useChatStore((state) => state.error); + + // Disable the composer and quick actions while a turn is streaming or the socket is not + // connected. ChatComposer computes this internally; QuickActions receives it as a prop. + const inputDisabled = + isStreaming || connectionState !== ConnectionState.CONNECTED; + const checkHealth = useChatStore((state) => state.checkHealth); + const { sendMessage } = useChatWebSocket(); + + // Master-detail mode: "conversation" (messages + composer) or "sessions" (the session list). + // Local state — purely presentational and not shared with the distant header, unlike `isOpen`. + const [view, setView] = React.useState("conversation"); + // Always reopen the drawer on the conversation view. + React.useEffect(() => { + if (!isOpen) setView("conversation"); + }, [isOpen]); + + // Proactively probe availability each time the drawer opens, so a down chatbot worker / Ollama is + // obvious immediately instead of after the post-ack watchdog. + React.useEffect(() => { + if (isOpen) checkHealth(); + }, [isOpen, checkHealth]); + + // The badge reflects assistant availability, not just transport: a connected socket whose worker + // isn't serving turns (assistantUnavailable) must not keep reading green "Connected". + const badge = + assistantUnavailable && connectionState === ConnectionState.CONNECTED + ? UNAVAILABLE_BADGE + : (CONNECTION_BADGE[connectionState] ?? + CONNECTION_BADGE[ConnectionState.IDLE]); + + return ( + + + {view === "conversation" ? ( + + ) : ( + + )} + Assistant + + + {ASSISTANT_INFO_TEXT} + + + {badge.label} + + + + {error && ( + + {error} + + )} + {view === "sessions" ? ( + setView("conversation")} /> + ) : ( + <> + + + + + + )} + + + ); +} diff --git a/frontend/src/components/chat/ChatSessionList.jsx b/frontend/src/components/chat/ChatSessionList.jsx new file mode 100644 index 0000000000..859995a9a4 --- /dev/null +++ b/frontend/src/components/chat/ChatSessionList.jsx @@ -0,0 +1,118 @@ +import React from "react"; +import PropTypes from "prop-types"; +import { Button, ListGroup, ListGroupItem, Spinner, Alert } from "reactstrap"; +import { IoAdd, IoTrashOutline } from "react-icons/io5"; +import { format } from "date-fns-tz"; + +import { useChatStore, deriveSessionLabel } from "../../stores/useChatStore"; +import { areYouSureConfirmDialog } from "../common/areYouSureConfirmDialog"; + +/** + * The sessions view of the chat drawer (master-detail counterpart of the conversation view). Lists + * the user's conversations with a backend-provided title + creation time, lets them start a new + * chat, switch to an existing one, or delete one. `onSessionChosen` flips the parent panel back to + * the conversation view after a switch/new. + * + * Switching/new are allowed while a turn streams: the store bumps navEpoch to abandon the in-flight + * turn (see useChatStore). Delete is still blocked for the active-streaming session (the server is + * persisting that session's messages). The WebSocket hook is untouched—it reads the bound sessionId + * live. + */ +export function ChatSessionList({ onSessionChosen }) { + const sessions = useChatStore((state) => state.sessions); + const loading = useChatStore((state) => state.sessionsLoading); + const error = useChatStore((state) => state.sessionsError); + const sessionId = useChatStore((state) => state.sessionId); + const fetchSessions = useChatStore((state) => state.fetchSessions); + const switchSession = useChatStore((state) => state.switchSession); + const newChat = useChatStore((state) => state.newChat); + const deleteSession = useChatStore((state) => state.deleteSession); + + // Refetch every time the list is shown so deletes/new sessions from other tabs are reflected. + React.useEffect(() => { + fetchSessions(); + }, [fetchSessions]); + + const handleSelect = (id) => { + switchSession(id); + onSessionChosen(); + }; + + const handleNew = () => { + newChat(); + onSessionChosen(); + }; + + const handleDelete = async (id) => { + const sure = await areYouSureConfirmDialog("delete this conversation"); + if (sure) deleteSession(id); + }; + + return ( +
+ + {error && ( + + {error} + + )} + {loading && ( +
+ +
+ )} + {!loading && !error && sessions.length === 0 && ( +
No conversations yet.
+ )} + + {sessions.map((session) => ( + handleSelect(session.id)} + className="d-flex justify-content-between align-items-center" + > +
+
{deriveSessionLabel(session)}
+ + {format(new Date(session.created_at), "yyyy-MM-dd HH:mm")} + +
+ {/* A
+ ); +} + +ChatSessionList.propTypes = { + onSessionChosen: PropTypes.func.isRequired, +}; diff --git a/frontend/src/components/chat/QuickActions.jsx b/frontend/src/components/chat/QuickActions.jsx new file mode 100644 index 0000000000..9106de97e3 --- /dev/null +++ b/frontend/src/components/chat/QuickActions.jsx @@ -0,0 +1,103 @@ +import React from "react"; +import PropTypes from "prop-types"; +import { Button } from "reactstrap"; + +// URL patterns mirror derive_page_context in api_app/chatbot_manager/agent/context.py. +// When the frontend Routes.jsx paths change (/jobs/:id or /investigation/:id), these +// regexes must be updated in lockstep with context.py. Both sides have coupling tests: +// - backend: tests/api_app/chatbot_manager/test_context.py +// - frontend: tests/components/chat/QuickActions.test.jsx +export const JOB_DETAIL_RE = /^\/jobs\/(\d+)(?:\/|$)/; +export const INVESTIGATION_DETAIL_RE = /^\/investigation\/(\d+)(?:\/|$)/; + +const JOB_ACTIONS = [ + { label: "Summarize this job", message: "Summarize job #{id}" }, + { label: "Which plugins ran?", message: "Which plugins ran on job #{id}?" }, + { label: "Show job details", message: "Show me the details of job #{id}" }, + { label: "Evaluate results", message: "Evaluate the results of job #{id}" }, +]; + +const INVESTIGATION_ACTIONS = [ + { + label: "Summarize this investigation", + message: "Summarize investigation #{id}", + }, + { + label: "Show investigation tree", + message: "Show the tree for investigation #{id}", + }, + { + label: "Analyze this investigation", + message: "What can you tell me about investigation #{id}?", + }, +]; + +const GENERIC_ACTIONS = [ + { label: "Show my recent jobs", message: "Show my recent jobs" }, + { + label: "List my investigations", + message: "List my investigations", + }, +]; + +/** + * Derive entity info from the current page URL, mirroring derive_page_context in the + * backend. Returns { type, id } for a recognised entity detail page, or null for any + * other page (dashboard, plugins, history, etc.). + */ +function deriveEntity() { + const { pathname } = window.location; + const jobMatch = pathname.match(JOB_DETAIL_RE); + if (jobMatch) return { type: "job", id: jobMatch[1] }; + const invMatch = pathname.match(INVESTIGATION_DETAIL_RE); + if (invMatch) return { type: "investigation", id: invMatch[1] }; + return null; +} + +/** + * Context-aware quick-action chips. On job and investigation detail pages the user + * gets entity-specific suggestions; everywhere else they see generic "show me" actions. + * Each chip auto-sends its message on click (populate + send, no second step). + */ +export function QuickActions({ onSend, disabled }) { + const entity = deriveEntity(); + let actions; + + if (!entity) { + actions = GENERIC_ACTIONS; + } else if (entity.type === "job") { + actions = JOB_ACTIONS; + } else { + actions = INVESTIGATION_ACTIONS; + } + + const handleClick = (message) => { + const resolved = entity ? message.replace("{id}", entity.id) : message; + onSend(resolved); + }; + + return ( +
+ {actions.map((action) => ( + + ))} +
+ ); +} + +QuickActions.propTypes = { + onSend: PropTypes.func.isRequired, + disabled: PropTypes.bool, +}; + +QuickActions.defaultProps = { + disabled: false, +}; diff --git a/frontend/src/components/chat/chatApi.js b/frontend/src/components/chat/chatApi.js new file mode 100644 index 0000000000..75b2b85cb9 --- /dev/null +++ b/frontend/src/components/chat/chatApi.js @@ -0,0 +1,15 @@ +import axios from "axios"; + +import { CHATBOT_ANALYSIS_CONFIRM_URI } from "../../constants/apiURLs"; + +/** + * Launch a previously previewed analysis. The chatbot agent can only *preview* (it mints a + * pending_id); this confirms it as an explicit user action — the model can never launch itself. + * Resolves to { errors, reused, job }; throws on a non-2xx response (caller surfaces the error). + */ +export async function confirmAnalysis(pendingId) { + const response = await axios.post(CHATBOT_ANALYSIS_CONFIRM_URI, { + pending_id: pendingId, + }); + return response.data; +} diff --git a/frontend/src/components/chat/chatMarkdown.jsx b/frontend/src/components/chat/chatMarkdown.jsx new file mode 100644 index 0000000000..daf3ad396d --- /dev/null +++ b/frontend/src/components/chat/chatMarkdown.jsx @@ -0,0 +1,46 @@ +import React from "react"; +import ReactMarkdown from "react-markdown"; +import remarkGfm from "remark-gfm"; + +/** + * Markdown renderer for assistant chat messages. + * + * Kept separate from the shared `markdownToHtml` on purpose: the chat enables GFM (tables / + * strikethrough / task lists) because the agent renders job and analyzer summaries as markdown + * tables, which the shared renderer (no GFM) would show as broken pipe-text. Isolating it here + * avoids changing how the rest of IntelOwl renders markdown. + * + * No `rehype-raw`: react-markdown v8 does not render raw HTML by default, so untrusted LLM output + * stays XSS-safe without an extra sanitizer. + * @param {string} text + */ +export function chatMarkdownToHtml(text) { + return ( + , + // eslint-disable-next-line id-length + a: ({ node: _, ...props }) => ( + // eslint-disable-next-line jsx-a11y/anchor-has-content + + ), + table: ({ node: _, ...props }) => ( + + ), + // drop react-markdown's `inline` flag so it is not spread onto the DOM node + code: ({ node: _, inline: _inline, ...props }) => ( + + ), + }} + /> + ); +} diff --git a/frontend/src/components/chat/useChatWebSocket.js b/frontend/src/components/chat/useChatWebSocket.js new file mode 100644 index 0000000000..99070e05b8 --- /dev/null +++ b/frontend/src/components/chat/useChatWebSocket.js @@ -0,0 +1,251 @@ +import React from "react"; + +import { WEBSOCKET_CHAT_URI } from "../../constants/apiURLs"; +import { useChatStore, ConnectionState } from "../../stores/useChatStore"; + +// Inbound frame types — mirror ChatEventType in api_app/chatbot_manager/events.py. +const ChatEventType = Object.freeze({ + ACK: "ack", + START: "start", + STATUS: "status", + TOKEN: "token", + END: "end", + ERROR: "error", + ACTION_REQUIRED: "action_required", +}); + +// Normal WebSocket close code: a clean close must NOT trigger a reconnect. +const NORMAL_CLOSURE = 1000; + +// Capped exponential backoff for the reconnect (chat is long-lived, unlike the Job socket which +// closes on the final status). Give up after a few attempts and surface a CLOSED badge. +const RECONNECT_BASE_MS = 1000; +const RECONNECT_CAP_MS = 30000; +const MAX_RECONNECT_ATTEMPTS = 5; + +// Post-ack watchdog: the consumer sends `ack` synchronously *before* enqueuing the Celery turn +// (process_chat_message.delay), so the ack arrives even when the chatbot worker is down — then no +// `start`/`end`/`error` ever follows and the composer would stay locked forever. If no `start` +// lands within this window after an `ack`, give up and unlock the composer. +const POST_ACK_TIMEOUT_MS = 20000; + +// Max-turn watchdog: bounds `start` -> no `end`. Set just above the server soft_time_limit (300s) +// so the server's own timeout normally wins; this only fires when the socket drops mid-turn (the +// group `end` is sent into the dead window and is not buffered) or the worker is hard-killed +// (SIGKILL never reaches the soft limit, so no `chat.error` is emitted). +const MAX_TURN_TIMEOUT_MS = 310000; + +// Client-side error texts (mirror the user-safe strings in ChatErrorDetail). +const WORKER_UNAVAILABLE_TEXT = + "The assistant is currently unavailable. Please try again."; +const TURN_TIMEOUT_TEXT = + "The assistant took too long to respond. Please try again."; + +function buildChatWebSocketUrl() { + const scheme = window.location.protocol === "https:" ? "wss" : "ws"; + // Trailing slash matters: the backend route is `ws/chat/`. context_url carries the page the + // user is on (captured server-side for later context injection). + const contextUrl = encodeURIComponent(window.location.href); + return `${scheme}://${window.location.host}/${WEBSOCKET_CHAT_URI}/?context_url=${contextUrl}`; +} + +/** + * Owns the chat WebSocket and bridges it to useChatStore. + * + * The socket, the reconnect timer and the two per-turn watchdog timers all live in refs (never in + * the store) so re-renders don't recreate them — the same pattern JobResult uses for the job + * socket. The hook lazily connects the first time the drawer opens and then keeps the socket alive + * across open/close; it only tears down on unmount. Returns `sendMessage` for the composer. + */ +export function useChatWebSocket() { + const isOpen = useChatStore((state) => state.isOpen); + + const websocket = React.useRef(null); + const reconnectTimer = React.useRef(null); + const reconnectAttempts = React.useRef(0); + const postAckTimer = React.useRef(null); + const maxTurnTimer = React.useRef(null); + // Set on unmount so the onclose handler does not schedule a reconnect during teardown. + const isUnmounting = React.useRef(false); + + const clearTurnTimers = React.useCallback(() => { + if (postAckTimer.current) { + clearTimeout(postAckTimer.current); + postAckTimer.current = null; + } + if (maxTurnTimer.current) { + clearTimeout(maxTurnTimer.current); + maxTurnTimer.current = null; + } + }, []); + + const dispatchEvent = React.useCallback( + (event) => { + const store = useChatStore.getState(); + const activeSession = store.sessionId; + switch (event.type) { + case ChatEventType.ACK: + // `ack` reaches only this socket; bind the (possibly new) session id, then arm the + // post-ack watchdog until the agent proves it is alive by emitting `start`. + store.applyAck(event); + if (postAckTimer.current) clearTimeout(postAckTimer.current); + { + const epoch = store.navEpoch; + postAckTimer.current = setTimeout(() => { + // Check the epoch BEFORE clearing the ref: if the user navigated away this turn is + // abandoned and the ref may already hold a newer turn's timer — nulling it here would + // orphan that timer past clearTurnTimers and fire a stale error on the new turn. + if (useChatStore.getState().navEpoch !== epoch) return; + postAckTimer.current = null; + // worker-down signal: flip the availability badge, not just the error banner + useChatStore.getState().applyUnavailable(WORKER_UNAVAILABLE_TEXT); + }, POST_ACK_TIMEOUT_MS); + } + break; + case ChatEventType.START: + // Strict demux: `start`/`status`/`token`/`end` fan out to every tab of this user, so a + // tab must drop frames that belong to another session. A fresh tab has sessionId === null + // and therefore matches nothing. + if (event.session_id !== activeSession) return; + // The agent answered: cancel the post-ack watchdog and arm the max-turn one. + if (postAckTimer.current) { + clearTimeout(postAckTimer.current); + postAckTimer.current = null; + } + { + const epoch = store.navEpoch; + maxTurnTimer.current = setTimeout(() => { + // See the post-ack watchdog: epoch-check first so a stale callback can't null a ref + // that a newer overlapping turn has since taken over. + if (useChatStore.getState().navEpoch !== epoch) return; + maxTurnTimer.current = null; + useChatStore.getState().applyError(TURN_TIMEOUT_TEXT); + }, MAX_TURN_TIMEOUT_MS); + } + store.applyStart(); + break; + case ChatEventType.STATUS: + if (event.session_id !== activeSession) return; + store.applyStatus(event); + break; + case ChatEventType.TOKEN: + if (event.session_id !== activeSession) return; + store.applyToken(event); + break; + case ChatEventType.END: + if (event.session_id !== activeSession) return; + clearTurnTimers(); + store.applyEnd(event); + break; + case ChatEventType.ACTION_REQUIRED: + // A tool previewed an analysis the user must confirm. Demuxed like the streaming frames so + // another tab's preview never raises this tab's card; it does NOT end the turn (prose still + // streams), so the turn watchdogs are left untouched. + if (event.session_id !== activeSession) return; + store.applyActionRequired(event); + break; + case ChatEventType.ERROR: + // Looser guard than the streaming frames: the consumer's *direct* errors arrive only on + // this socket with session_id === null (INVALID_MESSAGE), so the null branch must pass; + // the agent errors arrive via the group with the active session id. A different non-null + // session id belongs to another tab and is ignored. + if (event.session_id != null && event.session_id !== activeSession) + return; + clearTurnTimers(); + store.applyError(event.detail); + break; + default: + break; + } + }, + [clearTurnTimers], + ); + + const connect = React.useCallback(() => { + if (websocket.current) return; + const { setConnectionState } = useChatStore.getState(); + setConnectionState(ConnectionState.CONNECTING); + const url = buildChatWebSocketUrl(); + console.debug(`connect to chat websocket: ${url}`); + const socket = new WebSocket(url); + websocket.current = socket; + + socket.onopen = () => { + reconnectAttempts.current = 0; + useChatStore.getState().setConnectionState(ConnectionState.CONNECTED); + }; + socket.onmessage = (wsData) => { + let event; + try { + event = JSON.parse(wsData.data); + } catch (parseError) { + console.error("chat ws: unparsable frame", parseError); + return; + } + dispatchEvent(event); + }; + socket.onerror = (wsError) => { + console.error("chat ws error", wsError); + }; + socket.onclose = (closeEvent) => { + websocket.current = null; + // Terminal closes (teardown, clean close, reconnect budget exhausted) end the turn for good: + // drop the watchdogs so a stale timer can't fire an error minutes later onto a dead socket. + if (isUnmounting.current || closeEvent.code === NORMAL_CLOSURE) { + clearTurnTimers(); + useChatStore.getState().setConnectionState(ConnectionState.CLOSED); + return; + } + if (reconnectAttempts.current >= MAX_RECONNECT_ATTEMPTS) { + clearTurnTimers(); + useChatStore.getState().setConnectionState(ConnectionState.CLOSED); + return; + } + // Abnormal mid-turn drop: keep the turn watchdogs armed across the reconnect. The in-flight + // turn's `start`/`end` lands in the dead window and is never replayed to the new socket (group + // frames aren't buffered), so the watchdog is the only thing that unlocks the composer here — + // clearing it on close would strand `isStreaming` forever. + const delay = Math.min( + RECONNECT_BASE_MS * 2 ** reconnectAttempts.current, + RECONNECT_CAP_MS, + ); + reconnectAttempts.current += 1; + useChatStore.getState().setConnectionState(ConnectionState.RECONNECTING); + reconnectTimer.current = setTimeout(connect, delay); + }; + }, [dispatchEvent, clearTurnTimers]); + + // Lazy connect on first open; keep the socket alive afterwards (no idle socket for users who + // never open the chat). Teardown happens only on unmount. + React.useEffect(() => { + if (isOpen && !websocket.current) { + connect(); + } + }, [isOpen, connect]); + + React.useEffect( + () => () => { + isUnmounting.current = true; + if (reconnectTimer.current) clearTimeout(reconnectTimer.current); + clearTurnTimers(); + if (websocket.current) { + websocket.current.close(NORMAL_CLOSURE); + websocket.current = null; + } + }, + [clearTurnTimers], + ); + + const sendMessage = React.useCallback((text) => { + const trimmed = text.trim(); + const socket = websocket.current; + if (!trimmed || !socket || socket.readyState !== WebSocket.OPEN) return; + const store = useChatStore.getState(); + store.enqueueUserMessage(trimmed); + socket.send( + JSON.stringify({ message: trimmed, session_id: store.sessionId }), + ); + }, []); + + return { sendMessage }; +} diff --git a/frontend/src/constants/apiURLs.js b/frontend/src/constants/apiURLs.js index 61d1331fce..48bed3fa97 100644 --- a/frontend/src/constants/apiURLs.js +++ b/frontend/src/constants/apiURLs.js @@ -51,9 +51,20 @@ export const NOTIFICATION_BASE_URI = `${API_BASE_URI}/notification`; export const AUTH_BASE_URI = `${API_BASE_URI}/auth`; export const APIACCESS_BASE_URI = `${AUTH_BASE_URI}/apiaccess`; +// chatbot (router uses trailing_slash=False, so these paths carry no trailing slash). +// These path suffixes mirror the canonical Django routes in +// api_app/chatbot_manager/urls.py; they can't be imported across the JS/Python boundary, so a +// coupling test (tests/components/chat/chatbotRouteCoupling.test.js) reads that file and fails +// loudly if a route is renamed on either side. Keep both in sync. +export const CHATBOT_BASE_URI = `${API_BASE_URI}/chatbot`; +export const CHATBOT_SESSIONS_URI = `${CHATBOT_BASE_URI}/sessions`; +export const CHATBOT_HEALTH_URI = `${CHATBOT_BASE_URI}/health`; +export const CHATBOT_ANALYSIS_CONFIRM_URI = `${CHATBOT_BASE_URI}/analysis/confirm`; + // WEBSOCKETS const WEBSOCKET_BASE_URI = "ws"; export const WEBSOCKET_JOBS_URI = `${WEBSOCKET_BASE_URI}/jobs`; +export const WEBSOCKET_CHAT_URI = `${WEBSOCKET_BASE_URI}/chat`; // user event export const USER_EVENT_BASE_URI = `${API_BASE_URI}/user_event`; diff --git a/frontend/src/layouts/AppHeader.jsx b/frontend/src/layouts/AppHeader.jsx index ab59732b81..acf111bf38 100644 --- a/frontend/src/layouts/AppHeader.jsx +++ b/frontend/src/layouts/AppHeader.jsx @@ -20,7 +20,7 @@ import { RiTwitterXFill, } from "react-icons/ri"; import { FaGithub, FaGoogle, FaLinkedin } from "react-icons/fa"; -import { IoSearch } from "react-icons/io5"; +import { IoSearch, IoChatbubbleEllipsesOutline } from "react-icons/io5"; import { TbReport, TbReportSearch, TbDatabaseSearch } from "react-icons/tb"; // lib @@ -40,6 +40,7 @@ import NotificationPopoverButton from "../components/jobs/notification/Notificat import { useAuthStore } from "../stores/useAuthStore"; import { useGuideContext } from "../contexts/GuideContext"; import { useOrganizationStore } from "../stores/useOrganizationStore"; +import { useChatStore } from "../stores/useChatStore"; const guestLinks = ( <> @@ -120,8 +121,22 @@ function AuthLinks() { function RightLinks({ handleClickStart, isAuthenticated }) { const location = useLocation(); const isRootPath = location.pathname === "/"; + const toggleChat = useChatStore((state) => state.toggle); return ( <> + {isAuthenticated && ( + + + + )} {isRootPath && isAuthenticated && ( + ); +} + +const lastSocket = () => + FakeWebSocket.instances[FakeWebSocket.instances.length - 1]; + +const initialState = { + isOpen: false, + sessionId: null, + messages: [], + streamingText: "", + currentTool: null, + isStreaming: false, + connectionState: ConnectionState.IDLE, + error: null, + navEpoch: 0, + assistantUnavailable: false, + pendingAction: null, +}; + +describe("useChatWebSocket", () => { + beforeEach(() => { + jest.useFakeTimers(); + FakeWebSocket.instances = []; + global.WebSocket = FakeWebSocket; + useChatStore.setState(initialState); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + // open the drawer so the hook lazily connects, then complete the handshake + function connectAndOpen() { + render(); + act(() => { + useChatStore.getState().open(); + }); + act(() => { + lastSocket().mockOpen(); + }); + } + + test("connects lazily on open and reaches CONNECTED", () => { + render(); + expect(FakeWebSocket.instances).toHaveLength(0); // not connected while closed + act(() => { + useChatStore.getState().open(); + }); + expect(FakeWebSocket.instances).toHaveLength(1); + expect(lastSocket().url).toContain("/ws/chat/?context_url="); + act(() => { + lastSocket().mockOpen(); + }); + expect(useChatStore.getState().connectionState).toBe( + ConnectionState.CONNECTED, + ); + }); + + test("sendMessage emits {message, session_id} and binds the session on ack", () => { + connectAndOpen(); + act(() => { + fireEvent.click(screen.getByText("send")); + }); + expect(lastSocket().sent).toEqual([{ message: "hello", session_id: null }]); + expect(useChatStore.getState().messages).toHaveLength(1); + + act(() => { + lastSocket().mockMessage({ type: "ack", session_id: 5 }); + }); + expect(useChatStore.getState().sessionId).toBe(5); + }); + + test("streams start -> token -> end for the active session", () => { + connectAndOpen(); + act(() => { + lastSocket().mockMessage({ type: "ack", session_id: 5 }); + lastSocket().mockMessage({ type: "start", session_id: 5 }); + lastSocket().mockMessage({ + type: "status", + session_id: 5, + tool: "search_jobs", + }); + lastSocket().mockMessage({ + type: "token", + session_id: 5, + content: "Hi ", + }); + lastSocket().mockMessage({ + type: "token", + session_id: 5, + content: "you", + }); + }); + expect(useChatStore.getState().streamingText).toBe("Hi you"); + expect(useChatStore.getState().currentTool).toBe("search_jobs"); + + act(() => { + lastSocket().mockMessage({ + type: "end", + session_id: 5, + message_id: 9, + content: "Hi you!", + }); + }); + const state = useChatStore.getState(); + expect(state.messages.at(-1)).toEqual({ + id: 9, + role: "assistant", + content: "Hi you!", + }); + expect(state.isStreaming).toBe(false); + }); + + test("ignores streaming frames for another session (multi-tab demux)", () => { + connectAndOpen(); + act(() => { + lastSocket().mockMessage({ type: "ack", session_id: 5 }); + lastSocket().mockMessage({ type: "start", session_id: 5 }); + lastSocket().mockMessage({ + type: "token", + session_id: 999, + content: "X", + }); + }); + expect(useChatStore.getState().streamingText).toBe(""); + }); + + test("dispatches action_required to the store, demuxed by session", () => { + connectAndOpen(); + act(() => { + useChatStore.setState({ sessionId: 5 }); + }); + // a frame for another tab's session must not raise this tab's confirmation card + act(() => { + lastSocket().mockMessage({ + type: "action_required", + session_id: 999, + pending_id: "other", + plan: { observable_name: "y" }, + }); + }); + expect(useChatStore.getState().pendingAction).toBeNull(); + // the active session's frame is stored as the pending action + act(() => { + lastSocket().mockMessage({ + type: "action_required", + session_id: 5, + pending_id: "abc", + plan: { observable_name: "x" }, + }); + }); + expect(useChatStore.getState().pendingAction.pendingId).toBe("abc"); + }); + + test("surfaces a null-session error but ignores another session's error", () => { + connectAndOpen(); + act(() => { + useChatStore.setState({ sessionId: 5 }); + lastSocket().mockMessage({ + type: "error", + session_id: 999, + detail: "other tab", + }); + }); + expect(useChatStore.getState().error).toBeNull(); + + act(() => { + lastSocket().mockMessage({ + type: "error", + session_id: null, + detail: "Invalid message payload.", + }); + }); + expect(useChatStore.getState().error).toBe("Invalid message payload."); + }); + + test("post-ack watchdog fires when no start follows the ack", () => { + connectAndOpen(); + act(() => { + lastSocket().mockMessage({ type: "ack", session_id: 5 }); + }); + act(() => { + jest.advanceTimersByTime(20000); + }); + const state = useChatStore.getState(); + expect(state.error).toMatch(/unavailable/i); + expect(state.isStreaming).toBe(false); + // the worker did not pick up the turn -> the badge must reflect it (not just the error banner) + expect(state.assistantUnavailable).toBe(true); + }); + + test("post-ack watchdog is cleared once start arrives", () => { + connectAndOpen(); + act(() => { + lastSocket().mockMessage({ type: "ack", session_id: 5 }); + lastSocket().mockMessage({ type: "start", session_id: 5 }); + }); + act(() => { + jest.advanceTimersByTime(20000); + }); + expect(useChatStore.getState().error).toBeNull(); + }); + + test("max-turn watchdog fires when no end follows start", () => { + connectAndOpen(); + act(() => { + lastSocket().mockMessage({ type: "ack", session_id: 5 }); + lastSocket().mockMessage({ type: "start", session_id: 5 }); + }); + act(() => { + jest.advanceTimersByTime(310000); + }); + const state = useChatStore.getState(); + expect(state.error).toMatch(/too long/i); + expect(state.isStreaming).toBe(false); + // a mid-turn timeout is not a worker-down signal, so it must NOT flip the availability badge + expect(state.assistantUnavailable).toBe(false); + }); + + test("reconnects with backoff on an unexpected close", () => { + connectAndOpen(); + expect(FakeWebSocket.instances).toHaveLength(1); + act(() => { + lastSocket().mockServerClose(1006); // abnormal closure + }); + expect(useChatStore.getState().connectionState).toBe( + ConnectionState.RECONNECTING, + ); + act(() => { + jest.advanceTimersByTime(1000); // first backoff step + }); + expect(FakeWebSocket.instances).toHaveLength(2); + }); + + test("keeps the max-turn watchdog armed across a mid-turn reconnect", () => { + connectAndOpen(); + act(() => { + lastSocket().mockMessage({ type: "ack", session_id: 5 }); + lastSocket().mockMessage({ type: "start", session_id: 5 }); + }); + expect(useChatStore.getState().isStreaming).toBe(true); + + // socket drops mid-turn, then reconnects; the in-flight turn's `end` is lost in the dead window + act(() => { + lastSocket().mockServerClose(1006); + }); + act(() => { + jest.advanceTimersByTime(1000); // reconnect backoff + }); + expect(FakeWebSocket.instances).toHaveLength(2); + act(() => { + lastSocket().mockOpen(); + }); + + // the watchdog survived the drop and still unlocks the composer instead of hanging on isStreaming + act(() => { + jest.advanceTimersByTime(310000); + }); + const state = useChatStore.getState(); + expect(state.error).toMatch(/too long/i); + expect(state.isStreaming).toBe(false); + }); + + test("does not reconnect on a clean close", () => { + connectAndOpen(); + act(() => { + lastSocket().mockServerClose(1000); // normal closure + }); + expect(useChatStore.getState().connectionState).toBe( + ConnectionState.CLOSED, + ); + act(() => { + jest.advanceTimersByTime(60000); + }); + expect(FakeWebSocket.instances).toHaveLength(1); + }); + + test("newChat unbinds the session so a previous session's late frame is ignored", () => { + connectAndOpen(); + act(() => { + useChatStore.setState({ + sessionId: 5, + messages: [{ role: "user", content: "x" }], + }); + }); + act(() => { + useChatStore.getState().newChat(); + }); + expect(useChatStore.getState().sessionId).toBeNull(); + // a late `end` from the abandoned session must not append to the fresh conversation + act(() => { + lastSocket().mockMessage({ + type: "end", + session_id: 5, + message_id: 1, + content: "late", + }); + }); + expect(useChatStore.getState().messages).toEqual([]); + }); + + test("the post-ack watchdog does not fire after the user navigates away mid-turn", () => { + connectAndOpen(); + act(() => { + lastSocket().mockMessage({ type: "ack", session_id: 5 }); + }); + // user opens a fresh chat before `start` arrives: the abandoned turn's post-ack watchdog + // (armed under the old navEpoch) must not raise an error onto the new conversation + act(() => { + useChatStore.getState().newChat(); + }); + act(() => { + jest.advanceTimersByTime(20000); + }); + expect(useChatStore.getState().error).toBeNull(); + }); + + test("the max-turn watchdog does not fire after the user navigates away mid-turn", () => { + connectAndOpen(); + act(() => { + lastSocket().mockMessage({ type: "ack", session_id: 5 }); + lastSocket().mockMessage({ type: "start", session_id: 5 }); + }); + act(() => { + useChatStore.getState().newChat(); + }); + act(() => { + jest.advanceTimersByTime(310000); + }); + expect(useChatStore.getState().error).toBeNull(); + }); + + test("a stale turn's watchdog does not clobber a newer overlapping turn's timer", () => { + connectAndOpen(); + // turn A starts and arms its max-turn watchdog (epoch 0) + act(() => { + lastSocket().mockMessage({ type: "ack", session_id: 5 }); + lastSocket().mockMessage({ type: "start", session_id: 5 }); + }); + act(() => { + jest.advanceTimersByTime(100000); // A still in flight + }); + // user abandons A mid-turn and starts turn B (epoch bumped by newChat) + act(() => { + useChatStore.getState().newChat(); + fireEvent.click(screen.getByText("send")); + }); + act(() => { + lastSocket().mockMessage({ type: "ack", session_id: 7 }); + lastSocket().mockMessage({ type: "start", session_id: 7 }); + }); + // A's max-turn deadline elapses while B is still streaming: A's stale callback must NOT null + // the ref that now holds B's timer (else B's timer is orphaned and clearTurnTimers can't cancel it) + act(() => { + jest.advanceTimersByTime(210000); // t = 310000 -> A's timer fires + }); + // B completes cleanly + act(() => { + lastSocket().mockMessage({ + type: "end", + session_id: 7, + message_id: 9, + content: "done", + }); + }); + // B's original deadline passes; a cancelled timer must not resurrect a timeout error on B + act(() => { + jest.advanceTimersByTime(100000); // t = 410000 -> B's (cancelled) timer would have fired + }); + expect(useChatStore.getState().error).toBeNull(); + }); + + test("after newChat, sending starts a fresh session bound by the next ack", () => { + connectAndOpen(); + act(() => { + useChatStore.setState({ sessionId: 5 }); + useChatStore.getState().newChat(); + }); + act(() => { + fireEvent.click(screen.getByText("send")); + }); + // a fresh chat sends session_id: null; the server creates the session and returns it in `ack` + expect(lastSocket().sent).toEqual([{ message: "hello", session_id: null }]); + act(() => { + lastSocket().mockMessage({ type: "ack", session_id: 7 }); + lastSocket().mockMessage({ type: "start", session_id: 7 }); + lastSocket().mockMessage({ type: "token", session_id: 7, content: "hi" }); + lastSocket().mockMessage({ + type: "end", + session_id: 7, + message_id: 9, + content: "hi!", + }); + }); + const state = useChatStore.getState(); + expect(state.sessionId).toBe(7); + expect(state.messages.at(-1)).toEqual({ + id: 9, + role: "assistant", + content: "hi!", + }); + }); +}); diff --git a/frontend/tests/stores/useChatStore.test.jsx b/frontend/tests/stores/useChatStore.test.jsx new file mode 100644 index 0000000000..aa580252f7 --- /dev/null +++ b/frontend/tests/stores/useChatStore.test.jsx @@ -0,0 +1,434 @@ +import axios from "axios"; + +import { + useChatStore, + ConnectionState, + MessageRole, + deriveSessionLabel, +} from "../../src/stores/useChatStore"; +import { CHATBOT_SESSIONS_URI } from "../../src/constants/apiURLs"; +import { confirmAnalysis } from "../../src/components/chat/chatApi"; + +jest.mock("axios"); +jest.mock("../../src/components/chat/chatApi"); + +const initialState = { + isOpen: false, + sessionId: null, + messages: [], + streamingText: "", + currentTool: null, + isStreaming: false, + connectionState: ConnectionState.IDLE, + error: null, + sessions: [], + sessionsLoading: false, + sessionsError: null, + historyLoading: false, + navEpoch: 0, + assistantUnavailable: false, + pendingAction: null, +}; + +describe("useChatStore reducers", () => { + beforeEach(() => { + useChatStore.setState(initialState); + }); + + test("toggle flips isOpen", () => { + useChatStore.getState().toggle(); + expect(useChatStore.getState().isOpen).toBe(true); + useChatStore.getState().toggle(); + expect(useChatStore.getState().isOpen).toBe(false); + }); + + test("applyAck binds the session id", () => { + useChatStore.getState().applyAck({ session_id: 42 }); + expect(useChatStore.getState().sessionId).toBe(42); + }); + + test("enqueueUserMessage appends the user message and locks the turn", () => { + useChatStore.setState({ error: "stale" }); + useChatStore.getState().enqueueUserMessage("hello"); + const state = useChatStore.getState(); + expect(state.messages).toEqual([ + { role: MessageRole.USER, content: "hello" }, + ]); + expect(state.isStreaming).toBe(true); + expect(state.error).toBeNull(); + }); + + test("applyStart resets the streaming turn", () => { + useChatStore.setState({ + streamingText: "leftover", + currentTool: "search_jobs", + }); + useChatStore.getState().applyStart(); + const state = useChatStore.getState(); + expect(state.streamingText).toBe(""); + expect(state.currentTool).toBeNull(); + expect(state.isStreaming).toBe(true); + }); + + test("applyStatus sets the current tool", () => { + useChatStore.getState().applyStatus({ tool: "get_job_details" }); + expect(useChatStore.getState().currentTool).toBe("get_job_details"); + }); + + test("applyToken appends streaming text incrementally", () => { + useChatStore.getState().applyToken({ content: "Hel" }); + useChatStore.getState().applyToken({ content: "lo" }); + expect(useChatStore.getState().streamingText).toBe("Hello"); + }); + + test("applyEnd commits the assistant message and clears the turn", () => { + useChatStore.setState({ + streamingText: "Hel", + currentTool: "search_jobs", + isStreaming: true, + }); + useChatStore.getState().applyEnd({ message_id: 7, content: "Hello there" }); + const state = useChatStore.getState(); + expect(state.messages).toEqual([ + { id: 7, role: MessageRole.ASSISTANT, content: "Hello there" }, + ]); + expect(state.streamingText).toBe(""); + expect(state.currentTool).toBeNull(); + expect(state.isStreaming).toBe(false); + }); + + test("applyError surfaces the detail and ends the turn", () => { + useChatStore.setState({ streamingText: "partial", isStreaming: true }); + useChatStore.getState().applyError("boom"); + const state = useChatStore.getState(); + expect(state.error).toBe("boom"); + expect(state.isStreaming).toBe(false); + expect(state.streamingText).toBe(""); + }); + + test("applyUnavailable flags the assistant unavailable and ends the turn", () => { + useChatStore.setState({ streamingText: "partial", isStreaming: true }); + useChatStore + .getState() + .applyUnavailable( + "The assistant is currently unavailable. Please try again.", + ); + const state = useChatStore.getState(); + expect(state.assistantUnavailable).toBe(true); + expect(state.error).toMatch(/unavailable/i); + expect(state.isStreaming).toBe(false); + expect(state.streamingText).toBe(""); + }); + + test("applyStart clears a prior assistantUnavailable flag", () => { + useChatStore.setState({ assistantUnavailable: true }); + useChatStore.getState().applyStart(); + expect(useChatStore.getState().assistantUnavailable).toBe(false); + }); + + test("setConnectionState updates the badge state", () => { + useChatStore.getState().setConnectionState(ConnectionState.CONNECTED); + expect(useChatStore.getState().connectionState).toBe( + ConnectionState.CONNECTED, + ); + }); +}); + +describe("deriveSessionLabel", () => { + // jest runs with TZ=UTC, so the date fallback is deterministic. + const session = { id: 1, created_at: "2026-06-11T14:02:00Z" }; + + test("uses the backend-provided title (trimmed) when present", () => { + expect(deriveSessionLabel({ ...session, title: " hello there " })).toBe( + "hello there", + ); + }); + + test("falls back to the formatted created_at when there is no title", () => { + expect(deriveSessionLabel({ ...session, title: null })).toBe( + "2026-06-11 14:02", + ); + expect(deriveSessionLabel(session)).toBe("2026-06-11 14:02"); + }); +}); + +describe("useChatStore session management", () => { + beforeEach(() => { + useChatStore.setState(initialState); + jest.clearAllMocks(); + }); + + test("fetchSessions loads a single page", async () => { + axios.get.mockResolvedValueOnce({ + data: { total_pages: 1, results: [{ id: 1 }, { id: 2 }] }, + }); + await useChatStore.getState().fetchSessions(); + const state = useChatStore.getState(); + expect(state.sessions).toEqual([{ id: 1 }, { id: 2 }]); + expect(state.sessionsLoading).toBe(false); + expect(state.sessionsError).toBeNull(); + }); + + test("fetchSessions concatenates every page", async () => { + axios.get + .mockResolvedValueOnce({ data: { total_pages: 2, results: [{ id: 1 }] } }) + .mockResolvedValueOnce({ + data: { total_pages: 2, results: [{ id: 2 }] }, + }); + await useChatStore.getState().fetchSessions(); + expect(useChatStore.getState().sessions).toEqual([{ id: 1 }, { id: 2 }]); + expect(axios.get).toHaveBeenCalledTimes(2); + }); + + test("fetchSessions surfaces an error and clears loading", async () => { + axios.get.mockRejectedValueOnce(new Error("boom")); + await useChatStore.getState().fetchSessions(); + const state = useChatStore.getState(); + expect(state.sessionsError).toMatch(/conversations/i); + expect(state.sessionsLoading).toBe(false); + }); + + test("switchSession loads history oldest-first and binds the session", async () => { + axios.get.mockResolvedValueOnce({ + data: { + total_pages: 1, + results: [ + { id: 10, role: "user", content: "hi", timestamp: "t1" }, + { id: 11, role: "assistant", content: "yo", timestamp: "t2" }, + ], + }, + }); + useChatStore.setState({ streamingText: "stale", currentTool: "x" }); + await useChatStore.getState().switchSession(3); + const state = useChatStore.getState(); + expect(state.sessionId).toBe(3); + expect(state.messages).toEqual([ + { id: 10, role: "user", content: "hi" }, + { id: 11, role: "assistant", content: "yo" }, + ]); + expect(state.streamingText).toBe(""); + expect(state.currentTool).toBeNull(); + expect(state.historyLoading).toBe(false); + }); + + test("switchSession is a no-op for the already-active session", async () => { + useChatStore.setState({ sessionId: 3 }); + await useChatStore.getState().switchSession(3); + expect(axios.get).not.toHaveBeenCalled(); + }); + + test("switchSession proceeds while streaming and bumps navEpoch to abandon the turn", async () => { + axios.get.mockResolvedValueOnce({ + data: { + total_pages: 1, + results: [{ id: 20, role: "user", content: "hey" }], + }, + }); + useChatStore.setState({ isStreaming: true, sessionId: 1, navEpoch: 4 }); + await useChatStore.getState().switchSession(2); + const state = useChatStore.getState(); + expect(state.sessionId).toBe(2); + expect(state.messages).toEqual([{ id: 20, role: "user", content: "hey" }]); + // the in-flight turn is abandoned: its state is reset and the epoch bumped so the WS hook's + // watchdog (armed under epoch 4) no longer applies to the now-active session. + expect(state.isStreaming).toBe(false); + expect(state.navEpoch).toBe(5); + }); + + test("switchSession drops a stale result when the user switched again mid-flight", async () => { + let resolve; + axios.get.mockReturnValueOnce( + new Promise((res) => { + resolve = res; + }), + ); + const pending = useChatStore.getState().switchSession(1); + // user switched to another session before the history request resolved + useChatStore.setState({ sessionId: 2 }); + resolve({ + data: { + total_pages: 1, + results: [{ id: 1, role: "user", content: "old" }], + }, + }); + await pending; + expect(useChatStore.getState().messages).toEqual([]); + }); + + test("newChat resets to a fresh, unbound conversation and bumps navEpoch", () => { + useChatStore.setState({ + sessionId: 7, + messages: [{ role: "user", content: "x" }], + streamingText: "partial", + error: "stale", + navEpoch: 2, + }); + useChatStore.getState().newChat(); + const state = useChatStore.getState(); + expect(state.sessionId).toBeNull(); + expect(state.messages).toEqual([]); + expect(state.streamingText).toBe(""); + expect(state.isStreaming).toBe(false); + expect(state.error).toBeNull(); + expect(state.navEpoch).toBe(3); + }); + + test("newChat resets even while a turn is streaming", () => { + useChatStore.setState({ isStreaming: true, sessionId: 7, navEpoch: 5 }); + useChatStore.getState().newChat(); + const state = useChatStore.getState(); + expect(state.sessionId).toBeNull(); + expect(state.isStreaming).toBe(false); + expect(state.navEpoch).toBe(6); + }); + + test("deleteSession removes the active session and falls back to a fresh chat", async () => { + axios.delete.mockResolvedValueOnce({}); + useChatStore.setState({ + sessions: [{ id: 1 }, { id: 2 }], + sessionId: 1, + messages: [{ role: "user", content: "x" }], + }); + await useChatStore.getState().deleteSession(1); + const state = useChatStore.getState(); + expect(axios.delete).toHaveBeenCalledWith(`${CHATBOT_SESSIONS_URI}/1`); + expect(state.sessions).toEqual([{ id: 2 }]); + expect(state.sessionId).toBeNull(); + expect(state.messages).toEqual([]); + }); + + test("deleteSession leaves the active session untouched when deleting another", async () => { + axios.delete.mockResolvedValueOnce({}); + useChatStore.setState({ + sessions: [{ id: 1 }, { id: 2 }], + sessionId: 2, + }); + await useChatStore.getState().deleteSession(1); + const state = useChatStore.getState(); + expect(state.sessions).toEqual([{ id: 2 }]); + expect(state.sessionId).toBe(2); + }); + + test("deleteSession surfaces an error and keeps the list intact", async () => { + axios.delete.mockRejectedValueOnce(new Error("boom")); + useChatStore.setState({ sessions: [{ id: 1 }] }); + await useChatStore.getState().deleteSession(1); + const state = useChatStore.getState(); + expect(state.sessionsError).toMatch(/delete/i); + expect(state.sessions).toEqual([{ id: 1 }]); + }); +}); + +describe("useChatStore checkHealth", () => { + beforeEach(() => { + useChatStore.setState(initialState); + jest.clearAllMocks(); + }); + + test("flags unavailable and shows the detail banner", async () => { + axios.get.mockResolvedValueOnce({ + data: { available: false, detail: "down" }, + }); + await useChatStore.getState().checkHealth(); + const state = useChatStore.getState(); + expect(state.assistantUnavailable).toBe(true); + expect(state.error).toBe("down"); + }); + + test("clears the unavailable banner when recovering", async () => { + useChatStore.setState({ assistantUnavailable: true, error: "down" }); + axios.get.mockResolvedValueOnce({ + data: { available: true, detail: "" }, + }); + await useChatStore.getState().checkHealth(); + const state = useChatStore.getState(); + expect(state.assistantUnavailable).toBe(false); + expect(state.error).toBeNull(); + }); + + test("preserves an unrelated turn error when already available", async () => { + useChatStore.setState({ + assistantUnavailable: false, + error: "turn failed", + }); + axios.get.mockResolvedValueOnce({ + data: { available: true, detail: "" }, + }); + await useChatStore.getState().checkHealth(); + expect(useChatStore.getState().error).toBe("turn failed"); + }); + + test("does nothing when the health request fails", async () => { + useChatStore.setState({ assistantUnavailable: false, error: null }); + axios.get.mockRejectedValueOnce(new Error("boom")); + await useChatStore.getState().checkHealth(); + const state = useChatStore.getState(); + expect(state.assistantUnavailable).toBe(false); + expect(state.error).toBeNull(); + }); +}); + +describe("pending action (analyze confirm)", () => { + beforeEach(() => { + useChatStore.setState({ pendingAction: null, messages: [], error: null }); + confirmAnalysis.mockReset(); + }); + + test("applyActionRequired stores the pending plan", () => { + useChatStore.getState().applyActionRequired({ + pending_id: "abc", + plan: { observable_name: "x" }, + }); + expect(useChatStore.getState().pendingAction).toEqual({ + pendingId: "abc", + plan: { observable_name: "x" }, + }); + }); + + test("confirmPendingAction posts, clears, and appends a result message", async () => { + useChatStore.setState({ pendingAction: { pendingId: "abc", plan: {} } }); + confirmAnalysis.mockResolvedValue({ + errors: [], + reused: false, + job: { id: 7, status: "running" }, + }); + await useChatStore.getState().confirmPendingAction(); + expect(confirmAnalysis).toHaveBeenCalledWith("abc"); + expect(useChatStore.getState().pendingAction).toBeNull(); + const last = useChatStore.getState().messages.at(-1); + expect(last.content).toContain("#7"); + }); + + test("confirmPendingAction clears the card and surfaces errors when no job is returned", async () => { + useChatStore.setState({ pendingAction: { pendingId: "abc", plan: {} } }); + confirmAnalysis.mockResolvedValue({ + errors: ["could not launch"], + reused: false, + job: null, + }); + await useChatStore.getState().confirmPendingAction(); + expect(useChatStore.getState().pendingAction).toBeNull(); + expect(useChatStore.getState().error).toBe("could not launch"); + expect(useChatStore.getState().messages).toEqual([]); + }); + + test("confirmPendingAction surfaces an error and clears the card on failure", async () => { + useChatStore.setState({ pendingAction: { pendingId: "abc", plan: {} } }); + confirmAnalysis.mockRejectedValue(new Error("gone")); + await useChatStore.getState().confirmPendingAction(); + expect(useChatStore.getState().pendingAction).toBeNull(); + expect(useChatStore.getState().error).toBeTruthy(); + }); + + test("cancelPendingAction clears it", () => { + useChatStore.setState({ pendingAction: { pendingId: "abc", plan: {} } }); + useChatStore.getState().cancelPendingAction(); + expect(useChatStore.getState().pendingAction).toBeNull(); + }); + + test("sending a new message supersedes a pending action", () => { + useChatStore.setState({ pendingAction: { pendingId: "abc", plan: {} } }); + useChatStore.getState().enqueueUserMessage("hi"); + expect(useChatStore.getState().pendingAction).toBeNull(); + }); +}); diff --git a/integrations/malware_tools_analyzers/Dockerfile b/integrations/malware_tools_analyzers/Dockerfile index 6948d5180b..25be563497 100644 --- a/integrations/malware_tools_analyzers/Dockerfile +++ b/integrations/malware_tools_analyzers/Dockerfile @@ -32,7 +32,7 @@ WORKDIR ${PROJECT_PATH}/goresym RUN if [[ $TARGETARCH == "amd64" ]]; \ then export GORESYM_ARCH="linux"; \ else export GORESYM_ARCH="mac"; fi \ - && wget -q "https://github.com/mandiant/GoReSym/releases/download/v3.0.2/GoReSym-$GORESYM_ARCH.zip" \ + && wget -q "https://github.com/mandiant/GoReSym/releases/download/v3.3/GoReSym-$GORESYM_ARCH.zip" \ && unzip "GoReSym-$GORESYM_ARCH.zip" \ && chmod +x GoReSym \ && ln -s ${PROJECT_PATH}/goresym/GoReSym /usr/local/bin/goresym diff --git a/integrations/malware_tools_analyzers/requirements/droidlysis-requirements.txt b/integrations/malware_tools_analyzers/requirements/droidlysis-requirements.txt index b2b6b09381..fa4184ca79 100644 --- a/integrations/malware_tools_analyzers/requirements/droidlysis-requirements.txt +++ b/integrations/malware_tools_analyzers/requirements/droidlysis-requirements.txt @@ -1,3 +1,3 @@ # they do not make releases # if you update this, you should take into considerations all the other dependencies in the Dockerfile too -git+https://github.com/cryptax/droidlysis@c1645a5 \ No newline at end of file +git+https://github.com/cryptax/droidlysis@ce37151 \ No newline at end of file diff --git a/intel_owl/asgi.py b/intel_owl/asgi.py index 8c9e8cfb99..32104ea052 100644 --- a/intel_owl/asgi.py +++ b/intel_owl/asgi.py @@ -13,6 +13,7 @@ get_asgi_application() # pylint: disable=wrong-import-position +from api_app.chatbot_manager.consumers import ChatConsumer # noqa: E402 from api_app.websocket import JobConsumer # noqa: E402 from intel_owl.middleware import WSAuthMiddleware # noqa: E402 @@ -25,6 +26,7 @@ URLRouter( [ path("ws/jobs/", JobConsumer.as_asgi()), + path("ws/chat/", ChatConsumer.as_asgi()), ] ) ) diff --git a/intel_owl/celery.py b/intel_owl/celery.py index 6e290f77d9..0b61088b20 100644 --- a/intel_owl/celery.py +++ b/intel_owl/celery.py @@ -89,6 +89,13 @@ def get_queue_name(queue: str) -> str: app.conf.update( task_default_queue=get_queue_name(settings.DEFAULT_QUEUE), task_queues=task_queues, + # Pin the chat turn to the dedicated chatbot queue (drained by celery_worker_chatbot) so + # its long, LLM-bound runs never tie up the default workers, regardless of the call site. + task_routes={ + "api_app.chatbot_manager.tasks.process_chat_message": { + "queue": get_queue_name(settings.CHATBOT_QUEUE), + }, + }, task_time_limit=1800, broker_url=settings.BROKER_URL, result_backend=settings.RESULT_BACKEND, @@ -180,6 +187,15 @@ def get_queue_name(queue: str) -> str: "MessageGroupId": str(uuid.uuid4()), }, }, + "delete_old_chat_sessions": { + "task": "api_app.chatbot_manager.tasks.delete_old_chat_sessions", + # daily at 04:00, clear of the 02-03 cleanup cluster above + "schedule": crontab(minute="0", hour="4"), + "options": { + "queue": get_queue_name(settings.DEFAULT_QUEUE), + "MessageGroupId": str(uuid.uuid4()), + }, + }, } app.autodiscover_tasks() diff --git a/intel_owl/settings/__init__.py b/intel_owl/settings/__init__.py index acdc5319b0..ba31ea1db1 100644 --- a/intel_owl/settings/__init__.py +++ b/intel_owl/settings/__init__.py @@ -44,6 +44,7 @@ "api_app.engines_manager", "api_app.analyzables_manager", "api_app.user_events_manager", + "api_app.chatbot_manager", # auth "rest_email_auth", # performance debugging @@ -72,6 +73,7 @@ from .auth import * # lgtm [py/polluting-import] from .aws import * # lgtm [py/polluting-import] from .cache import * # lgtm [py/polluting-import] +from .chatbot import * # lgtm [py/polluting-import] # inject from other modules from .celery import * # lgtm [py/polluting-import] diff --git a/intel_owl/settings/celery.py b/intel_owl/settings/celery.py index 0495a39ea7..e3d6dfb498 100644 --- a/intel_owl/settings/celery.py +++ b/intel_owl/settings/celery.py @@ -18,7 +18,9 @@ BROADCAST_QUEUE = "broadcast" CONFIG_QUEUE = "config" +CHATBOT_QUEUE = "chatbot" + CELERY_QUEUES = get_secret("CELERY_QUEUES", DEFAULT_QUEUE).split(",") -for queue in [DEFAULT_QUEUE, CONFIG_QUEUE]: +for queue in [DEFAULT_QUEUE, CONFIG_QUEUE, CHATBOT_QUEUE]: if queue not in CELERY_QUEUES: CELERY_QUEUES.append(queue) diff --git a/intel_owl/settings/chatbot.py b/intel_owl/settings/chatbot.py new file mode 100644 index 0000000000..da2d904825 --- /dev/null +++ b/intel_owl/settings/chatbot.py @@ -0,0 +1,39 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from intel_owl import secrets + +from .cache import CACHES + +OLLAMA_BASE_URL = secrets.get_secret("OLLAMA_BASE_URL", "http://ollama:11434") +# qwen2.5:3b is the default on purpose: the agent relies on native tool calling, and this is +# the smallest model that proved able to pick the right tool and answer from its output with +# usable latency on a CPU-only deploy (mistral 7B took ~2.5 minutes per agent round). Stronger +# hardware can override it with any tool-capable Ollama model via the OLLAMA_MODEL secret. +OLLAMA_MODEL = secrets.get_secret("OLLAMA_MODEL", "qwen2.5:3b") +CHATBOT_QUEUE = secrets.get_secret("CHATBOT_QUEUE", "chatbot") +CHATBOT_MESSAGE_RETENTION_DAYS = int(secrets.get_secret("CHATBOT_MESSAGE_RETENTION_DAYS", 90)) + +# Per-user rate limiting (messages / minute). Shared between REST and WebSocket; +# both paths call RateLimiter with the same backing cache, so a user who sends +# via REST and WS hits the same bucket. +CHATBOT_RATE_LIMIT = int(secrets.get_secret("CHATBOT_RATE_LIMIT", 5)) +CHATBOT_RATE_LIMIT_WINDOW = int(secrets.get_secret("CHATBOT_RATE_LIMIT_WINDOW", 60)) + +# Separate Redis database (2) so rate-limit keys never collide with Channels (0) +# or Celery (1). django.core.cache.backends.redis.RedisCache is built into +# Django 4.x when the redis-py extra is installed (already a transitive dep of +# channels-redis). +CACHES["chatbot_rate_limit"] = { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": "redis://redis:6379/2", +} + +# Pending analyze_observable confirmations (human-in-the-loop guardrail): a preview mints a +# short-lived record; the confirm endpoint consumes it. Same Redis db as the rate limiter (2); +# keys are namespaced by prefix so they never collide. +CHATBOT_PENDING_ACTION_TTL = int(secrets.get_secret("CHATBOT_PENDING_ACTION_TTL", 600)) +CACHES["chatbot_pending_action"] = { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": "redis://redis:6379/2", +} diff --git a/requirements/project-requirements.txt b/requirements/project-requirements.txt index a2eeaaada9..73d2f22a32 100644 --- a/requirements/project-requirements.txt +++ b/requirements/project-requirements.txt @@ -44,7 +44,7 @@ checkdmarc==5.13.1 dnspython==2.8.0 dnstwist[full]==20250130 google>=3.0.0 -google-cloud-webrisk==1.20.0 +google-cloud-webrisk==1.22.0 intezer-sdk==1.24.0 lief==0.15.1 maxminddb==2.6.0 @@ -57,7 +57,7 @@ pdfid==1.1.0 pefile==2024.8.26 Pillow==11.0.0 pydeep==0.4 -pyelftools==0.31 +pyelftools==0.33 PyExifTool==0.5.0 pyhashlookup==1.2.0 pyimpfuzzy==0.5 @@ -111,6 +111,10 @@ defusedxml==0.7.1 # required by MalwareBazaar Ingestor -> https://bazaar.abuse.ch/api/#download (read the warning) pyzipper==0.3.6 +# LangChain / Ollama +langchain==0.3.30 +langchain-ollama==0.3.3 + # others dateparser==1.2.0 DeepDiff==8.6.1 diff --git a/start b/start index 6af040f2b5..ea98ed409e 100755 --- a/start +++ b/start @@ -9,7 +9,7 @@ declare -A env_arguments=(["prod"]=1 ["test"]=1 ["ci"]=1) declare -A test_mode=(["test"]=1 ["ci"]=1) declare -A cmd_arguments=(["build"]=1 ["up"]=1 ["start"]=1 ["restart"]=1 ["down"]=1 ["stop"]=1 ["kill"]=1 ["logs"]=1 ["ps"]=1) -declare -A path_mapping=(["default"]="docker/default.yml" ["postgres"]="docker/postgres.override.yml" ["rabbitmq"]="docker/rabbitmq.override.yml" ["test"]="docker/test.override.yml" ["ci"]="docker/ci.override.yml" ["custom"]="docker/custom.override.yml" ["traefik"]="docker/traefik.yml" ["traefik_prod"]="docker/traefik_prod.yml" ["traefik_local"]="docker/traefik_local.yml" ["multi_queue"]="docker/multi-queue.override.yml" ["test_multi_queue"]="docker/test.multi-queue.override.yml" ["flower"]="docker/flower.override.yml" ["test_flower"]="docker/test.flower.override.yml" ["elastic"]="docker/elasticsearch.override.yml" ["https"]="docker/https.override.yml" ["nfs"]="docker/nfs.override.yml" ["redis"]="docker/redis.override.yml" ["nginx_default"]="docker/nginx.override.yml") +declare -A path_mapping=(["default"]="docker/default.yml" ["postgres"]="docker/postgres.override.yml" ["rabbitmq"]="docker/rabbitmq.override.yml" ["test"]="docker/test.override.yml" ["ci"]="docker/ci.override.yml" ["custom"]="docker/custom.override.yml" ["traefik"]="docker/traefik.yml" ["traefik_prod"]="docker/traefik_prod.yml" ["traefik_local"]="docker/traefik_local.yml" ["multi_queue"]="docker/multi-queue.override.yml" ["test_multi_queue"]="docker/test.multi-queue.override.yml" ["flower"]="docker/flower.override.yml" ["test_flower"]="docker/test.flower.override.yml" ["elastic"]="docker/elasticsearch.override.yml" ["https"]="docker/https.override.yml" ["nfs"]="docker/nfs.override.yml" ["redis"]="docker/redis.override.yml" ["nginx_default"]="docker/nginx.override.yml" ["ollama"]="docker/ollama.override.yml" ["test_ollama"]="docker/test.ollama.override.yml" ["ci_ollama"]="docker/ci.ollama.override.yml") print_synopsis () { echo "SYNOPSIS" echo -e " start [OPTIONS]" @@ -47,6 +47,8 @@ print_help () { echo " --use-external-redis Do NOT use redis.override.yml compose file." echo " --rabbitmq Uses the rabbitmq.override.yml compose file." echo " --flower Uses the flower.override.yml compose file." + echo " --ollama Uses the ollama.override.yml compose file" + echo " to enable the local LLM chatbot service." echo " --custom Uses custom.override.yml to leverage your" echo " customized configuration." echo " --debug-build See more verbose output from the build." @@ -215,6 +217,10 @@ while [[ $# -gt 0 ]]; do params["flower"]=true shift 1 ;; + --ollama) + params["ollama"]=true + shift 1 + ;; --custom) params["custom"]=true shift 1 @@ -322,12 +328,18 @@ done # add all the test files if [[ $env_argument == "test" ]]; then - test_values=("multi_queue" "flower") + test_values=("multi_queue" "flower" "ollama") for value in "${test_values[@]}"; do if [ "${params["$value"]}" ]; then compose_files+=("${path_mapping["test_$value"]}") fi done +elif [[ $env_argument == "ci" ]]; then + # In ci mode --ollama pulls in ollama.override.yml (via the params loop above), whose chatbot + # worker is pinned to the pulled release image; repoint it to the built :ci image. + if [ "${params["ollama"]}" ]; then + compose_files+=("${path_mapping["ci_ollama"]}") + fi fi # add and parse analyzers diff --git a/tests/api_app/analyzers_manager/observable_analyzers/test_rdap.py b/tests/api_app/analyzers_manager/observable_analyzers/test_rdap.py new file mode 100644 index 0000000000..e24dc33ed6 --- /dev/null +++ b/tests/api_app/analyzers_manager/observable_analyzers/test_rdap.py @@ -0,0 +1,76 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from unittest.mock import MagicMock, patch + +from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.analyzers_manager.observable_analyzers.rdap import Rdap +from api_app.choices import Classification +from tests import CustomTestCase + + +class RdapTestCase(CustomTestCase): + """Unit tests for the RDAP analyzer (mocked HTTP, no live calls).""" + + @staticmethod + def _analyzer(observable_name, classification): + analyzer = Rdap(config={}) + analyzer.observable_name = observable_name + analyzer.observable_classification = classification + return analyzer + + @staticmethod + def _ok(json_data): + response = MagicMock(status_code=200) + response.raise_for_status.return_value = None + response.json.return_value = json_data + return response + + @patch("api_app.analyzers_manager.observable_analyzers.rdap.requests.get") + def test_domain_found(self, mock_get): + mock_get.return_value = self._ok({"objectClassName": "domain", "ldhName": "example.com"}) + result = self._analyzer("example.com", Classification.DOMAIN).run() + self.assertTrue(result["found"]) + self.assertEqual(result["ldhName"], "example.com") + self.assertIn("rdap.org/domain/example.com", mock_get.call_args.args[0]) + + @patch("api_app.analyzers_manager.observable_analyzers.rdap.requests.get") + def test_ip_uses_ip_endpoint(self, mock_get): + mock_get.return_value = self._ok({"objectClassName": "ip network"}) + self._analyzer("1.1.1.1", Classification.IP).run() + self.assertIn("rdap.org/ip/1.1.1.1", mock_get.call_args.args[0]) + + @patch("api_app.analyzers_manager.observable_analyzers.rdap.requests.get") + def test_url_resolves_to_host_domain(self, mock_get): + mock_get.return_value = self._ok({"objectClassName": "domain"}) + self._analyzer("https://sub.example.com/path?q=1", Classification.URL).run() + self.assertIn("rdap.org/domain/sub.example.com", mock_get.call_args.args[0]) + + @patch("api_app.analyzers_manager.observable_analyzers.rdap.requests.get") + def test_url_with_ip_host_uses_ip_endpoint(self, mock_get): + mock_get.return_value = self._ok({"objectClassName": "ip network"}) + self._analyzer("http://1.1.1.1/path", Classification.URL).run() + self.assertIn("rdap.org/ip/1.1.1.1", mock_get.call_args.args[0]) + + @patch("api_app.analyzers_manager.observable_analyzers.rdap.requests.get") + def test_non_json_response_raises(self, mock_get): + response = MagicMock(status_code=200) + response.raise_for_status.return_value = None + response.json.side_effect = ValueError("Expecting value") + mock_get.return_value = response + with self.assertRaises(AnalyzerRunException): + self._analyzer("example.com", Classification.DOMAIN).run() + + @patch("api_app.analyzers_manager.observable_analyzers.rdap.requests.get") + def test_not_found_returns_clean_negative(self, mock_get): + mock_get.return_value = MagicMock(status_code=404) + result = self._analyzer("does-not-exist.invalid", Classification.DOMAIN).run() + self.assertEqual(result, {"found": False}) + + def test_unsupported_classification_raises(self): + with self.assertRaises(AnalyzerRunException): + self._analyzer("deadbeefdeadbeef", Classification.HASH).run() + + def test_url_without_hostname_raises(self): + with self.assertRaises(AnalyzerRunException): + self._analyzer("not-a-url", Classification.URL).run() diff --git a/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_ipqsfile.py b/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_ipqsfile.py new file mode 100644 index 0000000000..b54793c55a --- /dev/null +++ b/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_ipqsfile.py @@ -0,0 +1,48 @@ +from unittest.mock import patch + +from api_app.analyzers_manager.file_analyzers.ipqsfile import IPQSFileScan + +from .base_test_class import BaseFileAnalyzerTest + + +class TestIPQSFileScan(BaseFileAnalyzerTest): + analyzer_class = IPQSFileScan + + def get_extra_config(self): + return { + "_ipqs_api_key": "dummy_key", + "polling_interval": 0, + "max_retries": 1, + } + + def get_mocked_response(self): + lookup_response = { + "file_name": "test file.txt", + "success": True, + "message": "Success", + "file_hash": "abc123", + "type": "scan", + "detected": False, + "detected_scans": 0, + "total_scans": 0, + "status": "pending", + "result": [""], + "file_size": 10, + "file_type": "text/plain", + "request_id": "req1", + } + scan_response = {"success": True, "message": "Success", "request_id": "req1"} + final_response = {**lookup_response, "status": "finished"} + + return [ + patch.object( + IPQSFileScan, + "_make_request", + side_effect=[lookup_response, scan_response], + ), + patch.object( + IPQSFileScan, + "_poll_for_report", + return_value=final_response, + ), + ] diff --git a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_bgp_ranking.py b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_bgp_ranking.py index e9d59e71a1..b921207acb 100644 --- a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_bgp_ranking.py +++ b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_bgp_ranking.py @@ -1,5 +1,8 @@ +from types import SimpleNamespace from unittest.mock import patch +from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.analyzers_manager.observable_analyzers import bgp_ranking as bgp_ranking_module from api_app.analyzers_manager.observable_analyzers.bgp_ranking import BGPRanking from tests.api_app.analyzers_manager.unit_tests.observable_analyzers.base_test_class import ( BaseAnalyzerTest, @@ -10,6 +13,26 @@ class BGPRankingTestCase(BaseAnalyzerTest): analyzer_class = BGPRanking + def test_run_raises_clean_exception_on_empty_asn_history(self): + # An IP with no ASN history returns an empty "response" object. The + # analyzer must raise a clean AnalyzerRunException instead of crashing + # with "KeyError: 'popitem(): dictionary is empty'". The analyzer is + # built with a stand-in config so the test runs deterministically and is + # not skipped when no AnalyzerConfig is loaded in the DB. + analyzer = BGPRanking(SimpleNamespace(name="BGPRanking")) + analyzer.observable_name = "8.8.8.8" + analyzer.url = "https://bgp-ranking.circl.lu" + analyzer.timeout = 30 + with ( + patch.object( + bgp_ranking_module.requests, + "get", + return_value=MockUpResponse({"meta": {"ip": "8.8.8.8"}, "response": {}}, 200), + ), + self.assertRaises(AnalyzerRunException), + ): + analyzer.run() + @staticmethod def get_mocked_response(): return [ diff --git a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_cve_exploitability.py b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_cve_exploitability.py new file mode 100644 index 0000000000..bdfcbff45c --- /dev/null +++ b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_cve_exploitability.py @@ -0,0 +1,121 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from unittest.mock import patch + +from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.analyzers_manager.models import AnalyzerConfig +from api_app.analyzers_manager.observable_analyzers.cve_exploitability import ( + CVEExploitability, +) +from tests.api_app.analyzers_manager.unit_tests.observable_analyzers.base_test_class import ( + BaseAnalyzerTest, +) +from tests.mock_utils import MockUpResponse + +# Trimmed CISA KEV catalog containing Log4Shell (CVE-2021-44228). +KEV_CATALOG = { + "title": "CISA Catalog of Known Exploited Vulnerabilities", + "catalogVersion": "2024.11.01", + "count": 1, + "vulnerabilities": [ + { + "cveID": "CVE-2021-44228", + "vendorProject": "Apache", + "product": "Log4j2", + "vulnerabilityName": "Apache Log4j2 Remote Code Execution Vulnerability", + "dateAdded": "2021-12-10", + "shortDescription": "Apache Log4j2 contains a vulnerability where JNDI " + "features do not protect against attacker-controlled LDAP.", + "requiredAction": "Apply updates per vendor instructions.", + "dueDate": "2021-12-24", + "knownRansomwareCampaignUse": "Known", + "notes": "https://nvd.nist.gov/vuln/detail/CVE-2021-44228", + } + ], +} + +# FIRST.org EPSS API response for Log4Shell. +EPSS_RESPONSE = { + "status": "OK", + "status-code": 200, + "version": "1.0", + "total": 1, + "data": [ + { + "cve": "CVE-2021-44228", + "epss": "0.944260000", + "percentile": "0.999990000", + "date": "2024-11-01", + } + ], +} + +# EPSS API response shape when a CVE has no published score. +EPSS_EMPTY_RESPONSE = { + "status": "OK", + "status-code": 200, + "version": "1.0", + "total": 0, + "data": [], +} + + +def _dispatch(kev_payload, epss_payload): + """Return a requests.get side_effect that routes on the called URL.""" + + def side_effect(url, *args, **kwargs): + if "known_exploited_vulnerabilities" in url: + return MockUpResponse(kev_payload, 200) + return MockUpResponse(epss_payload, 200) + + return side_effect + + +class CVEExploitabilityTestCase(BaseAnalyzerTest): + analyzer_class = CVEExploitability + config = AnalyzerConfig.objects.get(python_module=analyzer_class.python_module) + + @staticmethod + def get_mocked_response(): + return patch( + "requests.get", + side_effect=_dispatch(KEV_CATALOG, EPSS_RESPONSE), + ) + + def test_cve_in_kev_with_epss(self): + """A KEV-listed CVE returns in_kev=True plus EPSS score and percentile.""" + with self.get_mocked_response(): + analyzer = self.analyzer_class(self.config) + analyzer.observable_name = "cve-2021-44228" + result = analyzer.run() + + self.assertEqual(result["cve"], "CVE-2021-44228") + self.assertTrue(result["in_kev"]) + self.assertEqual(result["kev_date"], "2021-12-10") + self.assertEqual(result["ransomware_use"], "Known") + self.assertAlmostEqual(result["epss_score"], 0.94426) + self.assertAlmostEqual(result["epss_percentile"], 0.99999) + + def test_cve_not_in_kev(self): + """A CVE absent from the KEV catalog and from EPSS returns null signals.""" + with patch( + "requests.get", + side_effect=_dispatch({"vulnerabilities": []}, EPSS_EMPTY_RESPONSE), + ): + analyzer = self.analyzer_class(self.config) + analyzer.observable_name = "CVE-2000-9999" + result = analyzer.run() + + self.assertFalse(result["in_kev"]) + self.assertIsNone(result["kev_date"]) + self.assertIsNone(result["ransomware_use"]) + self.assertIsNone(result["epss_score"]) + self.assertIsNone(result["epss_percentile"]) + + def test_invalid_cve_format(self): + """An observable that is not a CVE id raises before any network call.""" + analyzer = self.analyzer_class(self.config) + analyzer.observable_name = "2021-44228" + with self.assertRaises(AnalyzerRunException): + analyzer.run() diff --git a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_ipqs.py b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_ipqs.py index 27d5938560..4e5f366831 100644 --- a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_ipqs.py +++ b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_ipqs.py @@ -1,14 +1,18 @@ +"""Unit tests for the IPQualityScore observable analyzer.""" + from unittest.mock import patch -from api_app.analyzers_manager.observable_analyzers.ipqs import IPQualityScore -from tests.api_app.analyzers_manager.unit_tests.observable_analyzers.base_test_class import ( - BaseAnalyzerTest, +from api_app.analyzers_manager.observable_analyzers import ipqs +from tests.api_app.analyzers_manager.unit_tests.observable_analyzers import ( + base_test_class, ) from tests.mock_utils import MockUpResponse -class IPQualityScoreTestCase(BaseAnalyzerTest): - analyzer_class = IPQualityScore +class IPQualityScoreTestCase(base_test_class.BaseAnalyzerTest): + """Tests for the `IPQualityScore` observable analyzer.""" + + analyzer_class = ipqs.IPQualityScore @staticmethod def get_mocked_response(): @@ -19,8 +23,13 @@ def get_mocked_response(): "domain": "test.com", "risk_score": 0, } - return patch("requests.get", return_value=MockUpResponse(mock_response, 200)) + return patch( + "requests.get", + return_value=MockUpResponse(mock_response, 200), + ) @classmethod def get_extra_config(cls) -> dict: - return {"_ipqs_api_key": "dummy_key"} + return { + "_ipqs_api_key": "dummy_key", + } diff --git a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_ipqsurl.py b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_ipqsurl.py new file mode 100644 index 0000000000..3736953940 --- /dev/null +++ b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_ipqsurl.py @@ -0,0 +1,72 @@ +"""Unit tests for the IPQS URL observable analyzer.""" + +from unittest.mock import patch + +from api_app.analyzers_manager.observable_analyzers.ipqsurl import IPQSUrlScan +from tests.api_app.analyzers_manager.unit_tests.observable_analyzers import ( + base_test_class, +) +from tests.mock_utils import MockUpResponse + + +class IPQSUrlScanTestCase(base_test_class.BaseAnalyzerTest): + """Tests for the `IPQSUrlScan` observable analyzer.""" + + analyzer_class = IPQSUrlScan + + @staticmethod + def get_mocked_response(): + # Response for lookup endpoint (non-cached case) + lookup_response = { + "file_name": "www.google.com", + "success": True, + "message": "Success", + "status": "not_cached", + "request_id": "dxwrE9RhS3", + } + # Response for scan endpoint + scan_response = { + "file_name": "www.google.com", + "success": True, + "message": "Success", + "request_id": "dxwrE9RhS3", + } + # Final response from poll_for_report + final_response = { + "file_name": "www.google.com", + "success": True, + "message": "Success", + "file_hash": ("86cc9b097d5ea4ec64a086634ef0f57b864770ccd1129d"), + "type": "scan", + "detected": False, + "detected_scans": 0, + "total_scans": 0, + "status": "finished", + "result": [""], + "file_size": 272728, + "file_type": "text/html", + "sha1": "379066b095304b84a0cc53888cda558ef483a4dd", + "md5": "4eb7c45715293e6effd84f4894cff654", + "request_id": "dxwrE9RhS3", + } + return [ + patch( + "requests.post", + side_effect=[ + MockUpResponse(lookup_response, 200), + MockUpResponse(scan_response, 200), + ], + ), + patch( + "requests.get", + return_value=MockUpResponse(final_response, 200), + ), + ] + + @classmethod + def get_extra_config(cls) -> dict: + return { + "_ipqs_api_key": "dummy_key", + "polling_interval": 0, + "max_retries": 1, + } diff --git a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_netlas.py b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_netlas.py index ca9b3ef2ae..37bab9cae9 100644 --- a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_netlas.py +++ b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_netlas.py @@ -1,5 +1,8 @@ +from types import SimpleNamespace from unittest.mock import patch +from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.analyzers_manager.observable_analyzers import netlas as netlas_module from api_app.analyzers_manager.observable_analyzers.netlas import Netlas from tests.api_app.analyzers_manager.unit_tests.observable_analyzers.base_test_class import ( BaseAnalyzerTest, @@ -10,6 +13,26 @@ class NetlasTestCase(BaseAnalyzerTest): analyzer_class = Netlas + def test_run_raises_clean_exception_on_empty_items(self): + # An IP with no whois record returns an empty "items" list. The analyzer + # must raise a clean AnalyzerRunException instead of crashing with + # "IndexError: list index out of range". The analyzer is built with a + # stand-in config so the test runs deterministically and is not skipped + # when no AnalyzerConfig is loaded in the DB. + analyzer = Netlas(SimpleNamespace(name="Netlas")) + analyzer.observable_name = "8.8.8.8" + analyzer.parameters = {"q": "ip:8.8.8.8"} + analyzer.headers = {"X-API-Key": "mock-api-key"} + with ( + patch.object( + netlas_module.requests, + "get", + return_value=MockUpResponse({"items": [], "took": 1}, 200), + ), + self.assertRaises(AnalyzerRunException), + ): + analyzer.run() + @staticmethod def get_mocked_response(): mock_response = { diff --git a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_yeti.py b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_yeti.py index a986820c88..3cadd16567 100644 --- a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_yeti.py +++ b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_yeti.py @@ -1,26 +1,59 @@ -from unittest.mock import patch +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. +from unittest.mock import MagicMock, patch + +from requests.exceptions import HTTPError + +from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.analyzers_manager.observable_analyzers.yeti import YETI from tests.api_app.analyzers_manager.unit_tests.observable_analyzers.base_test_class import ( BaseAnalyzerTest, ) -from tests.mock_utils import MockUpResponse -class YETITestCase(BaseAnalyzerTest): - analyzer_class = YETI +class MockResponse: + # Mock replacement for requests.Response objects to support raise_for_status - @staticmethod - def get_mocked_response(): - mock_response = [ + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + def raise_for_status(self): + if self.status_code >= 400: + raise HTTPError(f"HTTP Error: {self.status_code}") + + +def mock_yeti_api_flow(url, *args, **kwargs): + # Dynamic mock router to handle different YETI API endpoints + + if "api-token" in url: + return MockResponse({"access_token": "mocked_jwt_token_123", "token_type": "bearer"}, 200) + + if "observables/search" in url: + return MockResponse( { - "id": "123", - "value": "example.com", - "type": "domain", - "source": "malwaredb", - } - ] - return patch("requests.post", return_value=MockUpResponse(mock_response, 200)) + "total": 1, + "observables": [ + { + "id": "75297", + "type": "ipv4", + "value": "99.99.9.9", + "context": [{"source": "IntelOwl"}], + } + ], + }, + 200, + ) + + return MockResponse({"detail": "Not Found"}, 404) + + +class YETITestCase(BaseAnalyzerTest): + analyzer_class = YETI @classmethod def get_extra_config(cls) -> dict: @@ -31,3 +64,36 @@ def get_extra_config(cls) -> dict: "results_count": 10, "regex": False, } + + @staticmethod + def get_mocked_response(): + return patch( + "api_app.analyzers_manager.observable_analyzers.yeti.requests.post", + side_effect=mock_yeti_api_flow, + ) + + def _setup_yeti_analyzer(self): + """Helper to quickly spin up the analyzer for custom error tests.""" + return self._setup_analyzer( + MagicMock(), + "ipv4", + "99.99.9.9", + ) + + def test_yeti_auth_failure_raises_exception(self): + analyzer = self._setup_yeti_analyzer() + + with patch("api_app.analyzers_manager.observable_analyzers.yeti.requests.post") as mock_post: + mock_post.return_value = MockResponse({"detail": "Unauthorized"}, 401) + + with self.assertRaisesRegex(AnalyzerRunException, "YETI Auth Request failed"): + analyzer.run() + + def test_yeti_missing_access_token_raises_exception(self): + analyzer = self._setup_yeti_analyzer() + + with patch("api_app.analyzers_manager.observable_analyzers.yeti.requests.post") as mock_post: + mock_post.return_value = MockResponse({}, 200) + + with self.assertRaisesRegex(AnalyzerRunException, "Failed to obtain access token from YETI"): + analyzer.run() diff --git a/tests/api_app/chatbot_manager/__init__.py b/tests/api_app/chatbot_manager/__init__.py new file mode 100644 index 0000000000..acb99e9651 --- /dev/null +++ b/tests/api_app/chatbot_manager/__init__.py @@ -0,0 +1,2 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. diff --git a/tests/api_app/chatbot_manager/test_agent.py b/tests/api_app/chatbot_manager/test_agent.py new file mode 100644 index 0000000000..0057a33495 --- /dev/null +++ b/tests/api_app/chatbot_manager/test_agent.py @@ -0,0 +1,148 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from unittest.mock import MagicMock, patch + +from django.conf import settings +from django.test import TestCase +from langchain_core.messages import AIMessage +from langchain_core.runnables import RunnableLambda + +from api_app.chatbot_manager.agent.agent import ( + _MAX_AGENT_ITERATIONS, + _NUM_CTX, + _SYSTEM_PROMPT, + AGENT_STOPPED_OUTPUT, + build_agent_executor, +) +from api_app.chatbot_manager.agent.tools._common import on_invalid_tool_args +from certego_saas.apps.user.models import User + +# The full tool registry the agent must expose (one factory per module in agent/tools/). +EXPECTED_TOOL_NAMES = { + "search_jobs", + "get_job_details", + "summarize_job", + "list_investigations", + "get_investigation_tree", + "summarize_investigation", + "get_data_model", + "list_analyzers", + "recommend_playbook", + "analyze_observable", +} + + +class BuildAgentExecutorTestCase(TestCase): + """The executor wires the full user-scoped tool registry and bounds the agent loop.""" + + def setUp(self): + self.user, _ = User.objects.get_or_create(username="chatbot_agent_user") + + def _build(self, **kwargs): + # ChatOllama is mocked so no Ollama/network is ever touched; create_tool_calling_agent + # only needs the mock's bind_tools() result to be composable into its runnable chain. + with patch("api_app.chatbot_manager.agent.agent.ChatOllama") as mock_llm_cls: + executor = build_agent_executor(user=self.user, **kwargs) + return executor, mock_llm_cls + + def test_executor_has_all_tools_and_a_bounded_loop(self): + executor, _ = self._build() + + self.assertEqual({tool.name for tool in executor.tools}, EXPECTED_TOOL_NAMES) + # max_iterations is the only bound that force-stops a looping model + self.assertEqual(executor.max_iterations, _MAX_AGENT_ITERATIONS) + self.assertTrue(executor.handle_parsing_errors) + + def test_streaming_flag_reaches_the_llm(self): + for streaming in (False, True): + with self.subTest(streaming=streaming): + _, mock_llm_cls = self._build(streaming=streaming) + llm_kwargs = mock_llm_cls.call_args.kwargs + self.assertEqual(llm_kwargs["streaming"], streaming) + self.assertEqual(llm_kwargs["model"], settings.OLLAMA_MODEL) + self.assertEqual(llm_kwargs["base_url"], settings.OLLAMA_BASE_URL) + # without an explicit context window Ollama truncates the multi-tool prompt + self.assertEqual(llm_kwargs["num_ctx"], _NUM_CTX) + + def test_forced_stop_output_matches_the_sentinel(self): + # Ties AGENT_STOPPED_OUTPUT to the real framework behavior: the executor is the real + # tool-calling pipeline, only the LLM is faked to request a tool on every round, so + # max_iterations force-stops it. A langchain bump that changes the canned stop message + # must fail here rather than silently persist it as an assistant answer. + tool_call_forever = AIMessage( + content="", + tool_calls=[{"name": "search_jobs", "args": {"query": ""}, "id": "call_1"}], + ) + llm = MagicMock() + llm.bind_tools.return_value = RunnableLambda(lambda _: tool_call_forever) + with patch("api_app.chatbot_manager.agent.agent.ChatOllama", return_value=llm): + executor = build_agent_executor(user=self.user) + + result = executor.invoke({"input": "loop forever", "chat_history": [], "page_context": ""}) + + self.assertEqual(result["output"], AGENT_STOPPED_OUTPUT) + + +class OnInvalidToolArgsTestCase(TestCase): + """The observation returned on a bad tool argument names the error and steers a retry.""" + + def test_message_surfaces_error_and_forbids_placeholders(self): + msg = on_invalid_tool_args(ValueError("job_id: not an integer")) + self.assertIsInstance(msg, str) + self.assertIn("job_id: not an integer", msg) # the underlying error reaches the model + self.assertIn("placeholder", msg.lower()) # tell it not to pass a placeholder + + +def _scripted_llm(responses): + """Fake ChatOllama whose bound runnable replays `responses`, one AIMessage per agent round.""" + replies = iter(responses) + llm = MagicMock() + # next() takes a default so an exhausted script can't raise StopIteration (DeepSource PTC-W0063): + # returning a plain-text AIMessage ends the agent loop benignly instead of crashing the test. + llm.bind_tools.return_value = RunnableLambda(lambda _: next(replies, AIMessage(content=""))) + return llm + + +class ToolArgRecoveryTestCase(TestCase): + """A schema-invalid tool argument is recoverable, never a turn-killing exception.""" + + def setUp(self): + self.user, _ = User.objects.get_or_create(username="chatbot_arg_recovery_user") + + def test_all_tools_handle_validation_errors(self): + with patch("api_app.chatbot_manager.agent.agent.ChatOllama"): + executor = build_agent_executor(user=self.user) + for tool in executor.tools: + self.assertEqual(tool.handle_validation_error, on_invalid_tool_args) + + def test_invalid_tool_arg_is_recoverable_not_fatal(self): + # round 1: model passes the literal placeholder -> ValidationError on job_id (an int). + # round 2: model answers in plain text. The turn must complete, not raise. + bad = AIMessage( + content="", tool_calls=[{"name": "summarize_job", "args": {"job_id": ""}, "id": "c1"}] + ) + answer = AIMessage(content="Here is the summary.") + llm = _scripted_llm([bad, answer]) + with patch("api_app.chatbot_manager.agent.agent.ChatOllama", return_value=llm): + executor = build_agent_executor(user=self.user) + result = executor.invoke({"input": "summarize my latest job", "chat_history": [], "page_context": ""}) + self.assertEqual(result["output"], "Here is the summary.") + + def test_persistent_invalid_arg_degrades_to_forced_stop(self): + # The model emits the bad placeholder on every round: instead of crashing it must + # force-stop at max_iterations (the sentinel the caller maps to ITERATION_LIMIT). + bad = AIMessage( + content="", tool_calls=[{"name": "summarize_job", "args": {"job_id": ""}, "id": "c1"}] + ) + llm = MagicMock() + llm.bind_tools.return_value = RunnableLambda(lambda _: bad) + with patch("api_app.chatbot_manager.agent.agent.ChatOllama", return_value=llm): + executor = build_agent_executor(user=self.user) + result = executor.invoke({"input": "summarize my latest job", "chat_history": [], "page_context": ""}) + self.assertEqual(result["output"], AGENT_STOPPED_OUTPUT) + + def test_system_prompt_warns_against_placeholder_args(self): + # guard the specific rule, not just the word: the token appears only in it + self.assertIn("", _SYSTEM_PROMPT) + self.assertIn("placeholder", _SYSTEM_PROMPT.lower()) diff --git a/tests/api_app/chatbot_manager/test_confirm_analysis.py b/tests/api_app/chatbot_manager/test_confirm_analysis.py new file mode 100644 index 0000000000..856fa212d3 --- /dev/null +++ b/tests/api_app/chatbot_manager/test_confirm_analysis.py @@ -0,0 +1,105 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from unittest.mock import patch + +from django.test import override_settings +from rest_framework import status +from rest_framework.test import APITestCase + +from api_app.analyzers_manager.models import AnalyzerConfig +from api_app.chatbot_manager.pending_action import create_pending_analysis +from api_app.models import Job +from api_app.playbooks_manager.models import PlaybookConfig +from certego_saas.apps.user.models import User + +_APPLY_ASYNC = "intel_owl.tasks.job_pipeline.apply_async" +_URL = "/api/chatbot/analysis/confirm" + + +@override_settings(CHATBOT_PENDING_ACTION_TTL=600) +class ConfirmAnalysisViewTestCase(APITestCase): + def setUp(self): + self.user, _ = User.objects.get_or_create(username="confirm_user") + self.other, _ = User.objects.get_or_create(username="confirm_other") + self.analyzer = AnalyzerConfig.objects.get(name="Tranco") # seeded, free, supports domain + # A playbook owned by another user and NOT shared -> invisible to self.user (TOCTOU guard test). + self.pb_private_other = PlaybookConfig.objects.create( + name="confirm_pb_private", + description="t", + type=["domain"], + owner=self.other, + for_organization=False, + starting=True, + ) + self.pb_private_other.analyzers.set([self.analyzer]) + self.client.force_authenticate(self.user) + self.payload = { + "observable_name": "example.com", + "tlp": "CLEAR", + "playbook": "", + "analyzers": "Tranco", + } + + def tearDown(self): + Job.objects.filter(user__in=[self.user, self.other]).delete() + PlaybookConfig.objects.filter(owner=self.other).delete() + + @patch(_APPLY_ASYNC) + def test_confirm_launches_valid_pending(self, mock_apply): + pending_id = create_pending_analysis(self.user.id, self.payload) + resp = self.client.post(_URL, {"pending_id": pending_id}, format="json") + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertEqual(resp.json()["errors"], []) + self.assertIsNotNone(resp.json()["job"]) + mock_apply.assert_called_once() + + @patch(_APPLY_ASYNC) + def test_launched_job_owned_by_requesting_user(self, mock_apply): + pending_id = create_pending_analysis(self.user.id, self.payload) + job_id = self.client.post(_URL, {"pending_id": pending_id}, format="json").json()["job"]["id"] + self.assertEqual(Job.objects.get(pk=job_id).user, self.user) + + @patch(_APPLY_ASYNC) + def test_confirm_is_one_shot(self, mock_apply): + pending_id = create_pending_analysis(self.user.id, self.payload) + self.client.post(_URL, {"pending_id": pending_id}, format="json") + resp = self.client.post(_URL, {"pending_id": pending_id}, format="json") + self.assertEqual(resp.status_code, status.HTTP_410_GONE) + mock_apply.assert_called_once() + + @patch(_APPLY_ASYNC) + def test_confirm_rejects_other_users_pending(self, mock_apply): + pending_id = create_pending_analysis(self.other.id, self.payload) + resp = self.client.post(_URL, {"pending_id": pending_id}, format="json") + self.assertEqual(resp.status_code, status.HTTP_410_GONE) + mock_apply.assert_not_called() + + @patch(_APPLY_ASYNC) + def test_confirm_unknown_id(self, mock_apply): + resp = self.client.post(_URL, {"pending_id": "deadbeef"}, format="json") + self.assertEqual(resp.status_code, status.HTTP_410_GONE) + mock_apply.assert_not_called() + + @patch(_APPLY_ASYNC) + def test_confirm_rejects_playbook_no_longer_visible(self, mock_apply): + # TOCTOU guard: a pending whose playbook is not visible to the user at confirm time is + # refused (the confirm view re-applies the same visible_for_user guard as the preview tool). + pending_id = create_pending_analysis( + self.user.id, + { + "observable_name": "example.com", + "tlp": "CLEAR", + "playbook": self.pb_private_other.name, + "analyzers": "", + }, + ) + resp = self.client.post(_URL, {"pending_id": pending_id}, format="json") + self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST) + self.assertTrue(resp.json()["errors"]) + mock_apply.assert_not_called() + + def test_confirm_requires_auth(self): + self.client.force_authenticate(None) + resp = self.client.post(_URL, {"pending_id": "x"}, format="json") + self.assertIn(resp.status_code, (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN)) diff --git a/tests/api_app/chatbot_manager/test_consumers.py b/tests/api_app/chatbot_manager/test_consumers.py new file mode 100644 index 0000000000..e6b32f340a --- /dev/null +++ b/tests/api_app/chatbot_manager/test_consumers.py @@ -0,0 +1,181 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from unittest.mock import AsyncMock, MagicMock, patch + +from asgiref.sync import async_to_sync +from django.contrib.auth.models import AnonymousUser +from django.test import SimpleTestCase, TestCase, override_settings + +from api_app.chatbot_manager import events +from api_app.chatbot_manager.consumers import ChatConsumer +from api_app.chatbot_manager.models import ChatSession +from certego_saas.apps.user.models import User +from intel_owl.middleware import WSAuthMiddleware + +CHANNEL_NAME = "test.channel" + + +def _make_consumer(user): + """A ChatConsumer wired for direct (synchronous) method calls. + + The consumer's sync methods (connect/receive_json/disconnect) are exercised directly with an + injected scope and a mocked channel layer. This deliberately avoids WebsocketCommunicator: a + sync consumer keeps a channel-layer receive listener bound to its event loop, and tearing + that down across the fresh per-test loops that async_to_sync creates hangs. The group_add / + group_discard / accept / send_json side effects are mocked, so each behaviour is asserted + deterministically. + """ + consumer = ChatConsumer() + consumer.scope = {"user": user, "query_string": b"", "url_route": {"kwargs": {}}} + consumer.channel_name = CHANNEL_NAME + consumer.channel_layer = MagicMock() + consumer.channel_layer.group_add = AsyncMock() + consumer.channel_layer.group_discard = AsyncMock() + consumer.accept = MagicMock() + consumer.send_json = MagicMock() + consumer.context_url = "" # connect() sets this from the scope; receive_json tests skip connect() + return consumer + + +class ChatConsumerTestCase(TestCase): + """ChatConsumer auth-scoped group join, session resolution, and task enqueue.""" + + def setUp(self): + self.user, _ = User.objects.get_or_create(username="chat_ws_user") + + def test_connect_joins_only_its_per_user_group(self): + consumer = _make_consumer(self.user) + consumer.connect() + + consumer.accept.assert_called_once() + expected_group = events.chat_group_for_user(self.user.id) + # the group is keyed on the authenticated user id, so a client only gets its own stream + self.assertEqual(consumer.group_name, expected_group) + consumer.channel_layer.group_add.assert_called_once_with(expected_group, CHANNEL_NAME) + + def test_disconnect_leaves_the_group(self): + consumer = _make_consumer(self.user) + consumer.connect() + consumer.disconnect(1000) + + consumer.channel_layer.group_discard.assert_called_once_with( + events.chat_group_for_user(self.user.id), CHANNEL_NAME + ) + + @patch("api_app.chatbot_manager.consumers.process_chat_message") + def test_message_creates_session_acks_and_enqueues(self, mock_task): + consumer = _make_consumer(self.user) + consumer.receive_json({"message": "hello"}) + + session = ChatSession.objects.get(user=self.user) + consumer.send_json.assert_called_once_with(events.AckEvent(session.id).to_client()) + mock_task.delay.assert_called_once_with(session.id, "hello", self.user.id, "") + + @patch("api_app.chatbot_manager.consumers.process_chat_message") + def test_existing_owned_session_is_reused(self, mock_task): + session = ChatSession.objects.create(user=self.user) + consumer = _make_consumer(self.user) + consumer.receive_json({"message": "again", "session_id": session.id}) + + # no new session is created when a valid owned id is supplied + self.assertEqual(ChatSession.objects.filter(user=self.user).count(), 1) + consumer.send_json.assert_called_once_with(events.AckEvent(session.id).to_client()) + mock_task.delay.assert_called_once_with(session.id, "again", self.user.id, "") + + @patch("api_app.chatbot_manager.consumers.process_chat_message") + def test_context_url_is_forwarded_to_the_task(self, mock_task): + consumer = _make_consumer(self.user) + consumer.context_url = "https://intelowl.test/jobs/42" + consumer.receive_json({"message": "summarize this"}) + + session = ChatSession.objects.get(user=self.user) + # the consumer forwards the raw context_url; the task derives the hint + mock_task.delay.assert_called_once_with( + session.id, "summarize this", self.user.id, "https://intelowl.test/jobs/42" + ) + + @patch("api_app.chatbot_manager.consumers.process_chat_message") + def test_oversized_message_is_rejected(self, mock_task): + consumer = _make_consumer(self.user) + consumer.receive_json({"message": "x" * (events.MAX_INBOUND_MESSAGE_LEN + 1)}) + + consumer.send_json.assert_called_once_with( + events.ErrorEvent(None, events.ChatErrorDetail.INVALID_MESSAGE.value).to_client() + ) + mock_task.delay.assert_not_called() + # an invalid frame must not create a session + self.assertFalse(ChatSession.objects.filter(user=self.user).exists()) + + @patch("api_app.chatbot_manager.consumers.process_chat_message") + def test_message_for_another_users_session_is_rejected(self, mock_task): + other, _ = User.objects.get_or_create(username="other_ws_user") + foreign_session = ChatSession.objects.create(user=other) + + consumer = _make_consumer(self.user) + consumer.receive_json({"message": "hi", "session_id": foreign_session.id}) + + consumer.send_json.assert_called_once_with( + events.ErrorEvent(foreign_session.id, events.ChatErrorDetail.SESSION_NOT_FOUND.value).to_client() + ) + mock_task.delay.assert_not_called() + + @override_settings( + CACHES={ + "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, + "chatbot_rate_limit": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, + } + ) + @patch("api_app.chatbot_manager.consumers.process_chat_message") + def test_rate_limit_emits_error_event(self, mock_task): + """Messages over the per-user limit emit RATE_LIMITED and are dropped.""" + with self.settings(CHATBOT_RATE_LIMIT=2, CHATBOT_RATE_LIMIT_WINDOW=60): + consumer = _make_consumer(self.user) + + # Two messages within the limit — both enqueued. + consumer.receive_json({"message": "first"}) + self.assertTrue(mock_task.delay.called) + mock_task.reset_mock() + + consumer.receive_json({"message": "second"}) + self.assertTrue(mock_task.delay.called) + mock_task.reset_mock() + + # Third message — rate-limited, not enqueued. + consumer.receive_json({"message": "too many"}) + mock_task.delay.assert_not_called() + sent = consumer.send_json.call_args[0][0] + self.assertEqual(sent["type"], "error") + self.assertEqual(sent["detail"], "rate_limited") + self.assertIsInstance(sent["retry_after"], int) + self.assertGreater(sent["retry_after"], 0) + + +class ChatConsumerRelayTestCase(SimpleTestCase): + """Each group handler relays the producer's prebuilt payload verbatim to the client.""" + + def test_handlers_forward_event_payload(self): + payload = events.TokenEvent(3, "hi").to_client() + for handler_name in ("chat_start", "chat_status", "chat_token", "chat_end", "chat_error"): + consumer = ChatConsumer() + consumer.send_json = MagicMock() + getattr(consumer, handler_name)({"payload": payload}) + consumer.send_json.assert_called_once_with(payload) + + +class WSAuthMiddlewareTestCase(SimpleTestCase): + """Anonymous users are closed out before the consumer is ever reached.""" + + def test_anonymous_connection_is_closed_and_app_not_called(self): + inner_app = AsyncMock() + middleware = WSAuthMiddleware(inner_app) + sent = [] + + async def send(message): + sent.append(message) + + scope = {"type": "websocket", "user": AnonymousUser()} + async_to_sync(middleware)(scope, AsyncMock(), send) + + inner_app.assert_not_called() + self.assertEqual(sent, [{"type": "websocket.close", "code": 1008}]) diff --git a/tests/api_app/chatbot_manager/test_context.py b/tests/api_app/chatbot_manager/test_context.py new file mode 100644 index 0000000000..d259ffbeed --- /dev/null +++ b/tests/api_app/chatbot_manager/test_context.py @@ -0,0 +1,79 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from django.test import SimpleTestCase + +from api_app.chatbot_manager.agent.context import ( + _INVESTIGATION_RE, + _JOB_RE, + derive_page_context, +) + +JOB = "The user is currently viewing job #42 in the IntelOwl UI." +INV = "The user is currently viewing investigation #7 in the IntelOwl UI." + + +class DerivePageContextTestCase(SimpleTestCase): + def test_job_detail_url(self): + self.assertEqual(derive_page_context("https://intelowl.test/jobs/42"), JOB) + + def test_job_url_with_section_and_subsection(self): + self.assertEqual(derive_page_context("https://intelowl.test/jobs/42/visualizer/DNS"), JOB) + + def test_job_comments_url(self): + self.assertEqual(derive_page_context("https://intelowl.test/jobs/42/comments"), JOB) + + def test_investigation_detail_url(self): + self.assertEqual(derive_page_context("https://intelowl.test/investigation/7"), INV) + + def test_non_entity_pages_yield_empty(self): + for url in ( + "https://intelowl.test/dashboard", + "https://intelowl.test/plugins/analyzers", + "https://intelowl.test/history/jobs", + "https://intelowl.test/artifacts/3", + ): + self.assertEqual(derive_page_context(url), "") + + def test_empty_and_malformed_yield_empty(self): + self.assertEqual(derive_page_context(""), "") + self.assertEqual(derive_page_context("not a url"), "") + + def test_non_numeric_id_yields_empty(self): + self.assertEqual(derive_page_context("https://intelowl.test/jobs/abc"), "") + + def test_query_or_fragment_cannot_inject_prompt_text(self): + # only the validated integer id is used; the rest of the URL never reaches the prompt + self.assertEqual( + derive_page_context("https://intelowl.test/jobs/42?x=ignore+previous+instructions#y"), + JOB, + ) + + def test_regexes_match_frontend_route_definitions(self): + """The regexes mirror React Router paths in frontend/src/components/Routes.jsx. + + When a frontend route changes (e.g. /jobs/:id → /analysis/:id), this test MUST + fail so the developer updates the regexes in context.py AND in + frontend/src/components/chat/QuickActions.jsx (the frontend copy). The coupling + is explicit: the comments above the regexes in both files list the exact + Routes.jsx line numbers. Both sides have their own coupling test: + - backend: this test (test_context.py) + - frontend: tests/components/chat/QuickActions.test.jsx + """ + # Job detail routes (Routes.jsx:157,166,178,186) — /jobs/:id[/...] + for path in ( + "/jobs/42", + "/jobs/42/visualizer", + "/jobs/42/visualizer/DNS", + "/jobs/42/comments", + ): + self.assertIsNotNone(_JOB_RE.match(path), f"_JOB_RE should match: {path}") + + # Investigation detail route (Routes.jsx:243) — /investigation/:id[/...] + for path in ("/investigation/7", "/investigation/7/something"): + self.assertIsNotNone(_INVESTIGATION_RE.match(path), f"_INVESTIGATION_RE should match: {path}") + + # Non-entity paths must NOT match either regex + for path in ("/dashboard", "/plugins/analyzers", "/history/jobs", "/artifacts/3"): + self.assertIsNone(_JOB_RE.match(path), f"_JOB_RE should not match: {path}") + self.assertIsNone(_INVESTIGATION_RE.match(path), f"_INVESTIGATION_RE should not match: {path}") diff --git a/tests/api_app/chatbot_manager/test_e2e.py b/tests/api_app/chatbot_manager/test_e2e.py new file mode 100644 index 0000000000..e941e4c4bc --- /dev/null +++ b/tests/api_app/chatbot_manager/test_e2e.py @@ -0,0 +1,249 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +"""End-to-end tests of the chatbot chat pipeline. + +These wire the *real* components together — ChatConsumer, the process_chat_message Celery task, +ChatStreamingCallbackHandler, the analyze_observable tool, the pending-action store and the +confirm endpoint — and mock only the external boundaries: the LLM (via build_agent_executor) and +the job pipeline (apply_async). Ollama and the network are never touched. + +Transport: the consumer leg does NOT use WebsocketCommunicator. A synchronous JsonWebsocketConsumer +keeps a channel-layer receive listener bound to its event loop, and a real InMemoryChannelLayer +round-trip reads an asyncio.Queue bound to the producer's (separate) async_to_sync loop — both are +the loop-binding hang documented in test_consumers.py. Instead _CapturingChannelLayer records the +plain-dict group_send payloads (loop-safe), which are then fed to the consumer's real chat. +relay handlers, exercising every real component bar the framework's queue plumbing. +""" + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from django.core.cache import caches +from django.test import TestCase, override_settings +from rest_framework import status +from rest_framework.test import APIClient + +from api_app.chatbot_manager import events +from api_app.chatbot_manager.agent.tools import build_tools +from api_app.chatbot_manager.consumers import ChatConsumer +from api_app.chatbot_manager.models import ChatMessage, ChatSession +from api_app.chatbot_manager.tasks import process_chat_message +from api_app.models import Job +from certego_saas.apps.user.models import User + +_ASSISTANT_OUTPUT = "Prepared an analysis plan for example.com. Click Confirm to run it." +_APPLY_ASYNC = "intel_owl.tasks.job_pipeline.apply_async" +_CONFIRM_URL = "/api/chatbot/analysis/confirm" + +_LOCMEM = "django.core.cache.backends.locmem.LocMemCache" +# Distinct LOCATIONs so the rate-limit and pending-action stores don't share one dict; locmem is +# process-global, so each test clears these in setUp to stop entries bleeding across tests. +LOCMEM_CACHES = { + "default": {"BACKEND": _LOCMEM, "LOCATION": "e2e-default"}, + "chatbot_rate_limit": {"BACKEND": _LOCMEM, "LOCATION": "e2e-rl"}, + "chatbot_pending_action": {"BACKEND": _LOCMEM, "LOCATION": "e2e-pa"}, +} + + +class _CapturingChannelLayer: + """Records group_send payloads instead of queueing them (see module docstring for why).""" + + def __init__(self): + self.sent = [] # ordered list of (group, channel_message) + + async def group_send(self, group, message): + self.sent.append((group, message)) + + async def group_add(self, group, channel): + pass + + async def group_discard(self, group, channel): + pass + + +def _fake_tool_calling_executor(user): + """A stand-in AgentExecutor whose .invoke() drives the REAL streaming callbacks exactly as a + tool-calling turn would: a tool action (chat.status), the real analyze_observable tool output + (chat.action_required, carrying a real one-time pending_id), a streamed answer token + (chat.token), then the final output. Keeps the handler, the tool and the pending-action store + real while Ollama is never reached.""" + real_tools = build_tools(user=user) + # dict lookup (not next()) so a missing tool fails loudly with a KeyError naming it + analyze_tool = {tool.name: tool for tool in real_tools}["analyze_observable"] + + class _FakeExecutor: + # The task derives handler.tool_names from this real registry, so the chat.status filter + # (which suppresses the _Exception pseudo-tool) is exercised against real tool names. + tools = real_tools + + @staticmethod + def invoke(payload, config=None): + handler = config["callbacks"][0] + # SimpleNamespace mirrors a langchain AgentAction's .tool attribute. + handler.on_agent_action(SimpleNamespace(tool="analyze_observable", tool_input={}, log="")) + # Real tool run: validates the observable and mints a real pending_id in the cache. + tool_output = analyze_tool.invoke({"observable_name": "example.com", "analyzers": "Tranco"}) + handler.on_tool_end(tool_output) + handler.on_llm_new_token("Prepared the analysis plan. ") + return {"output": _ASSISTANT_OUTPUT} + + return _FakeExecutor() + + +def _connected_consumer(user, channel_layer): + """A ChatConsumer wired for direct calls with the capturing layer; real connect() sets up the + per-user group name. accept/send_json are mocked so the relayed client frames can be captured.""" + consumer = ChatConsumer() + consumer.scope = {"user": user, "query_string": b"", "url_route": {"kwargs": {}}} + consumer.channel_name = "test.chat" + consumer.channel_layer = channel_layer + consumer.accept = MagicMock() + consumer.send_json = MagicMock() + consumer.connect() # sets group_name = chat-; group_add is a no-op on the capturing layer + return consumer + + +@override_settings( + CACHES=LOCMEM_CACHES, + CHATBOT_PENDING_ACTION_TTL=600, + CHATBOT_RATE_LIMIT=1000, + CHATBOT_RATE_LIMIT_WINDOW=60, +) +class ChatPipelineE2ETestCase(TestCase): + """The full WebSocket turn: consumer -> task -> agent -> streaming callback -> relayed frames.""" + + def setUp(self): + for alias in ("default", "chatbot_rate_limit", "chatbot_pending_action"): + caches[alias].clear() + self.user, _ = User.objects.get_or_create(username="e2e_chat_user") + + def tearDown(self): + Job.objects.filter(user=self.user).delete() + + def _run_turn(self, *, message="analyze example.com", session_id=None): + """Run one turn end-to-end and return (consumer, layer, ack_frames, relay_frames). + + ack_frames are what receive_json sent back synchronously (the ack). relay_frames are the + client payloads produced by replaying each captured group_send through the consumer's real + chat. handler — i.e. exactly what the browser would receive. + """ + layer = _CapturingChannelLayer() + consumer = _connected_consumer(self.user, layer) + with ( + patch( + "api_app.chatbot_manager.agent.agent.build_agent_executor", + return_value=_fake_tool_calling_executor(self.user), + ), + patch("api_app.chatbot_manager.tasks.get_channel_layer", return_value=layer), + patch("api_app.chatbot_manager.agent.streaming.get_channel_layer", return_value=layer), + patch("api_app.chatbot_manager.consumers.process_chat_message") as consumer_task, + ): + # The consumer enqueues via .delay; forward to the real task body so the hand-off + # contract (exact args) is exercised and the turn runs synchronously in-test. The task + # object is itself callable, so it is the side_effect directly (no wrapping lambda). + consumer_task.delay.side_effect = process_chat_message + content = {"message": message} + if session_id is not None: + content["session_id"] = session_id + consumer.receive_json(content) + + ack_frames = [call.args[0] for call in consumer.send_json.call_args_list] + consumer.send_json.reset_mock() + # Replay each captured group message through the framework's type->method mapping. + for _group, channel_message in layer.sent: + getattr(consumer, channel_message["type"].replace(".", "_"))(channel_message) + relay_frames = [call.args[0] for call in consumer.send_json.call_args_list] + return consumer, layer, ack_frames, relay_frames + + def test_full_turn_streams_tool_call_and_guardrail(self): + consumer, layer, ack_frames, relay_frames = self._run_turn() + session = ChatSession.objects.get(user=self.user) + + # ack first, carrying the resolved (newly created) session id + self.assertEqual(ack_frames, [events.AckEvent(session.id).to_client()]) + + # every producer send targeted this user's group (the consumer's own subscription) + self.assertTrue(layer.sent) + self.assertTrue(all(group == consumer.group_name for group, _ in layer.sent)) + + # ordered client frames for a tool-calling turn with an M-1 preview + self.assertEqual( + [frame["type"] for frame in relay_frames], + [ + events.ChatEventType.START.value, + events.ChatEventType.STATUS.value, + events.ChatEventType.ACTION_REQUIRED.value, + events.ChatEventType.TOKEN.value, + events.ChatEventType.END.value, + ], + ) + + # every frame is stamped with the session id (the client's demux key) + self.assertTrue(all(frame["session_id"] == session.id for frame in relay_frames)) + + # status names the real tool; action_required carries a pending id + a plan + self.assertEqual(relay_frames[1]["tool"], "analyze_observable") + action = relay_frames[2] + self.assertTrue(action["pending_id"]) + self.assertEqual(action["plan"]["observable_name"], "example.com") + self.assertEqual(action["plan"]["classification"], "domain") + + # end carries the persisted assistant message + its id + assistant = ChatMessage.objects.get(session=session, role=ChatMessage.Role.ASSISTANT) + self.assertEqual(relay_frames[-1]["message_id"], assistant.id) + self.assertEqual(relay_frames[-1]["content"], _ASSISTANT_OUTPUT) + + # the exchange is persisted user-then-assistant + self.assertEqual( + list( + ChatMessage.objects.filter(session=session) + .order_by("timestamp") + .values_list("role", "content") + ), + [(ChatMessage.Role.USER, "analyze example.com"), (ChatMessage.Role.ASSISTANT, _ASSISTANT_OUTPUT)], + ) + + def test_action_required_pending_id_confirms_and_launches_once(self): + # The pending_id surfaced to the browser in action_required is the only thing needed to + # launch: a real user POST of it to the confirm endpoint starts the job (M-1 — the model + # itself never launches). + _consumer, _layer, _ack, relay_frames = self._run_turn() + # exactly one action_required frame is expected; index it (not next()) so an empty/oversized + # match fails loudly here rather than with a bare StopIteration + action_frames = [ + frame for frame in relay_frames if frame["type"] == events.ChatEventType.ACTION_REQUIRED.value + ] + self.assertEqual(len(action_frames), 1) + pending_id = action_frames[0]["pending_id"] + + client = APIClient() + client.force_authenticate(self.user) + with patch(_APPLY_ASYNC) as mock_apply: + first = client.post(_CONFIRM_URL, {"pending_id": pending_id}, format="json") + self.assertEqual(first.status_code, status.HTTP_200_OK) + self.assertEqual(first.json()["errors"], []) + job_id = first.json()["job"]["id"] + # multi-tenancy: the launched job is owned by the confirming user + self.assertEqual(Job.objects.get(pk=job_id).user, self.user) + + # one-shot: the consumed pending id cannot launch a second job + second = client.post(_CONFIRM_URL, {"pending_id": pending_id}, format="json") + self.assertEqual(second.status_code, status.HTTP_410_GONE) + mock_apply.assert_called_once() + + def test_frames_are_tagged_per_session_for_multi_tab_demux(self): + # Frames fan out to every tab of a user (one per-user group); a tab demultiplexes by the + # session_id each frame carries. Two sessions of the same user must produce disjoint tags so + # a tab pinned to one session drops the other's frames. + session_a = ChatSession.objects.create(user=self.user) + session_b = ChatSession.objects.create(user=self.user) + + _ca, _la, _acka, frames_a = self._run_turn(session_id=session_a.id) + _cb, _lb, _ackb, frames_b = self._run_turn(session_id=session_b.id) + + self.assertNotEqual(session_a.id, session_b.id) + self.assertTrue(frames_a and all(frame["session_id"] == session_a.id for frame in frames_a)) + self.assertTrue(frames_b and all(frame["session_id"] == session_b.id for frame in frames_b)) + # session A's stream contains nothing a tab on session B would accept + self.assertFalse(any(frame["session_id"] == session_b.id for frame in frames_a)) diff --git a/tests/api_app/chatbot_manager/test_health.py b/tests/api_app/chatbot_manager/test_health.py new file mode 100644 index 0000000000..d8ed8f8842 --- /dev/null +++ b/tests/api_app/chatbot_manager/test_health.py @@ -0,0 +1,115 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from unittest.mock import MagicMock, patch + +import requests +from django.conf import settings +from django.core.cache import cache +from rest_framework import status +from rest_framework.test import APITestCase + +from api_app.chatbot_manager.health import ( + HEALTH_UNAVAILABLE_DETAIL, + ChatHealth, + chatbot_health, +) +from certego_saas.apps.user.models import User +from intel_owl.celery import get_queue_name + +CHATBOT_QUEUE_NAME = get_queue_name(settings.CHATBOT_QUEUE) + + +def _celery_app(active_queues_return): + """A fake celery_app whose control.inspect().active_queues() returns the given value.""" + app = MagicMock() + app.control.inspect.return_value.active_queues.return_value = active_queues_return + return app + + +class ChatbotHealthTestCase(APITestCase): + def setUp(self): + cache.clear() + + @patch("api_app.chatbot_manager.health.requests.get") + @patch( + "api_app.chatbot_manager.health.celery_app", + _celery_app({"worker1": [{"name": CHATBOT_QUEUE_NAME}]}), + ) + def test_available_when_worker_and_ollama_up(self, mock_get): + mock_get.return_value = MagicMock(ok=True) + health = chatbot_health() + self.assertTrue(health.available) + self.assertEqual(health.detail, "") + + @patch("api_app.chatbot_manager.health.requests.get") + @patch( + "api_app.chatbot_manager.health.celery_app", + _celery_app(None), # no worker replied within the timeout + ) + def test_unavailable_when_no_chatbot_worker(self, mock_get): + mock_get.return_value = MagicMock(ok=True) + health = chatbot_health() + self.assertFalse(health.available) + self.assertEqual(health.detail, HEALTH_UNAVAILABLE_DETAIL) + + @patch( + "api_app.chatbot_manager.health.requests.get", + side_effect=requests.RequestException, + ) + @patch( + "api_app.chatbot_manager.health.celery_app", + _celery_app({"worker1": [{"name": CHATBOT_QUEUE_NAME}]}), + ) + def test_unavailable_when_ollama_unreachable(self, mock_get): + health = chatbot_health() + self.assertFalse(health.available) + self.assertEqual(health.detail, HEALTH_UNAVAILABLE_DETAIL) + + @patch("api_app.chatbot_manager.health.requests.get") + @patch("api_app.chatbot_manager.health.celery_app") + def test_result_is_cached(self, mock_celery, mock_get): + mock_celery.control.inspect.return_value.active_queues.return_value = { + "worker1": [{"name": CHATBOT_QUEUE_NAME}] + } + mock_get.return_value = MagicMock(ok=True) + chatbot_health() + chatbot_health() + # second call is served from the cache -> the probes ran exactly once + mock_celery.control.inspect.assert_called_once() + mock_get.assert_called_once() + + @patch("api_app.chatbot_manager.health.requests.get") + @patch( + "api_app.chatbot_manager.health.celery_app", + _celery_app({"worker1": [{"name": "some-other-queue"}]}), + ) + def test_unavailable_when_worker_serves_a_different_queue(self, mock_get): + mock_get.return_value = MagicMock(ok=True) + self.assertFalse(chatbot_health().available) + + +class ChatHealthViewTestCase(APITestCase): + URL = "/api/chatbot/health" + + def setUp(self): + cache.clear() + self.user, _ = User.objects.get_or_create(username="chatbot_health_user") + + @patch( + "api_app.chatbot_manager.views.chatbot_health", + return_value=ChatHealth(available=False, detail="nope"), + ) + def test_get_returns_serialized_health(self, _mock): + self.client.force_authenticate(user=self.user) + response = self.client.get(self.URL) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.json(), {"available": False, "detail": "nope"}) + + def test_requires_authentication(self): + # IntelOwl's session/token auth returns 401 or 403 for an anonymous request. + response = self.client.get(self.URL) + self.assertIn( + response.status_code, + (status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN), + ) diff --git a/tests/api_app/chatbot_manager/test_pending_action.py b/tests/api_app/chatbot_manager/test_pending_action.py new file mode 100644 index 0000000000..ac5179da7a --- /dev/null +++ b/tests/api_app/chatbot_manager/test_pending_action.py @@ -0,0 +1,38 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from django.test import SimpleTestCase, override_settings + +from api_app.chatbot_manager.pending_action import ( + consume_pending_analysis, + create_pending_analysis, +) + +TEST_CACHES = { + "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, + "chatbot_pending_action": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "LOCATION": "pending-test", + }, +} +PAYLOAD = {"observable_name": "example.com", "tlp": "CLEAR", "playbook": "", "analyzers": "Tranco"} + + +@override_settings(CACHES=TEST_CACHES, CHATBOT_PENDING_ACTION_TTL=600) +class PendingActionTestCase(SimpleTestCase): + def test_create_then_consume_returns_payload(self): + pending_id = create_pending_analysis(7, PAYLOAD) + self.assertTrue(pending_id) + self.assertEqual(consume_pending_analysis(7, pending_id), PAYLOAD) + + def test_consume_is_one_shot(self): + pending_id = create_pending_analysis(7, PAYLOAD) + consume_pending_analysis(7, pending_id) + self.assertIsNone(consume_pending_analysis(7, pending_id)) + + def test_consume_rejects_other_user(self): + pending_id = create_pending_analysis(7, PAYLOAD) + self.assertIsNone(consume_pending_analysis(99, pending_id)) + + def test_consume_unknown_id_returns_none(self): + self.assertIsNone(consume_pending_analysis(7, "deadbeef")) diff --git a/tests/api_app/chatbot_manager/test_prompt.py b/tests/api_app/chatbot_manager/test_prompt.py new file mode 100644 index 0000000000..d631a03a54 --- /dev/null +++ b/tests/api_app/chatbot_manager/test_prompt.py @@ -0,0 +1,108 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +"""Structural tests for the agent system prompt file. + +These tests verify that system_prompt.txt is loadable, within token limits, +and covers all registered tools — no LLM inference is performed. +""" + +from pathlib import Path + +from django.test import TestCase + +from api_app.chatbot_manager.agent.agent import _SYSTEM_PROMPT, PROMPT +from api_app.chatbot_manager.agent.tools import build_tools +from certego_saas.apps.user.models import User + +PROMPT_FILE = Path(__file__).parent.parent.parent.parent.joinpath( + "api_app", "chatbot_manager", "agent", "system_prompt.txt" +) + +# The 10 tool names the prompt MUST mention so the model knows when to use each. +# Keeping this set in the test forces the author to update the prompt when tools +# are added or removed from build_tools(). +EXPECTED_TOOL_NAMES = frozenset( + { + "search_jobs", + "get_job_details", + "summarize_job", + "list_investigations", + "get_investigation_tree", + "summarize_investigation", + "get_data_model", + "list_analyzers", + "recommend_playbook", + "analyze_observable", + } +) + + +class SystemPromptTestCase(TestCase): + def test_prompt_file_readable(self): + """The file exists and loads into the module-level constant.""" + self.assertTrue(PROMPT_FILE.exists(), f"Missing: {PROMPT_FILE}") + self.assertIsInstance(_SYSTEM_PROMPT, str) + self.assertGreater(len(_SYSTEM_PROMPT), 100) + stripped = _SYSTEM_PROMPT.strip() + self.assertEqual( + stripped, + _SYSTEM_PROMPT, + "system prompt contains leading/trailing whitespace", + ) + + def test_prompt_under_token_limit(self): + """System prompt must stay under 500 tokens to leave room for tool schemas + and conversation history within Ollama's 8192 context window. + """ + tokens = len(_SYSTEM_PROMPT.split()) + self.assertLess(tokens, 500, f"system prompt is {tokens} tokens — exceeds 500") + + def test_prompt_includes_all_tool_names(self): + """Every registered tool appears in the [Tools] section, and the hardcoded + EXPECTED_TOOL_NAMES matches the live build_tools() registry. If someone adds + a tool without updating both this test and system_prompt.txt, this catches it. + """ + user, _ = User.objects.get_or_create(username="prompt_tool_user") + registered = frozenset(tool.name for tool in build_tools(user=user)) + self.assertEqual( + registered, + EXPECTED_TOOL_NAMES, + "Tool registry changed — update EXPECTED_TOOL_NAMES and system_prompt.txt", + ) + + for name in EXPECTED_TOOL_NAMES: + self.assertIn( + name, + _SYSTEM_PROMPT, + f"Tool '{name}' not found in system_prompt.txt — add it to the [Tools] section", + ) + + def test_prompt_is_part_of_the_chat_template(self): + """The loaded prompt content is embedded in the ChatPromptTemplate's + system message, so the agent actually sees the file content at runtime. + """ + # PROMPT.messages[0] is a SystemMessagePromptTemplate; its .prompt.template + # carries the system text (with {page_context} appended). + system_msg = PROMPT.messages[0] + template = system_msg.prompt.template + self.assertIn(_SYSTEM_PROMPT, template) + + def test_prompt_sections_are_present(self): + """Each planned section header appears so the structure is enforced.""" + for section in ("[Role]", "[Tools", "[Rules]", "[Response style]"): + self.assertIn(section, _SYSTEM_PROMPT, f"Missing section: {section}") + + def test_page_context_not_in_the_file(self): + """The file must NOT contain {page_context} — interpolation is the prompt + template's job, not the static file's. + """ + self.assertNotIn("{page_context}", _SYSTEM_PROMPT) + + def test_prompt_is_all_printable(self): + """Non-printable characters (except newline) will silently confuse the LLM.""" + self.assertNotIn("\r", _SYSTEM_PROMPT) + self.assertTrue( + all(c.isprintable() or c == "\n" for c in _SYSTEM_PROMPT), + "system prompt contains non-printable characters", + ) diff --git a/tests/api_app/chatbot_manager/test_query_counts.py b/tests/api_app/chatbot_manager/test_query_counts.py new file mode 100644 index 0000000000..5b6841b8c8 --- /dev/null +++ b/tests/api_app/chatbot_manager/test_query_counts.py @@ -0,0 +1,271 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +"""Efficiency (query-count) guards for the chatbot tools. + +Each guard asserts a tool's DB query count is invariant to the data dimension that would cause an +N+1 (result count, relation count, tree size) rather than checking a brittle magic number: it runs +the tool against a small and a large instance of that dimension and asserts the same count. This +catches an un-prefetched relation added to a serializer later. No LLM is invoked — the tools are +called directly with seeded data. + +`analyze_observable` is intentionally NOT guarded here: its cost is ObservableAnalysisSerializer +validation (platform code resolving analyzers/playbooks/connectors), not the chatbot's own queries, +so a guard there would be a brittle measure of someone else's query plan. +""" + +from unittest.mock import AsyncMock, MagicMock, patch +from uuid import uuid4 + +from django.db import connection +from django.test import TestCase, override_settings +from django.test.utils import CaptureQueriesContext + +from api_app.analyzables_manager.models import Analyzable +from api_app.analyzers_manager.models import AnalyzerConfig, AnalyzerReport +from api_app.chatbot_manager.agent.tools import build_tools +from api_app.chatbot_manager.models import ChatMessage, ChatSession +from api_app.chatbot_manager.tasks import process_chat_message +from api_app.choices import TLP, Classification +from api_app.investigations_manager.models import Investigation +from api_app.models import Job +from api_app.playbooks_manager.models import PlaybookConfig +from certego_saas.apps.organization.membership import Membership +from certego_saas.apps.organization.organization import Organization +from certego_saas.apps.user.models import User + +INMEMORY_CHANNEL_LAYER = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"}} + +# Generous upper bound for the single-object/single-call guards, where the N+1 dimension is the +# seeded global set and is impractical to vary in a test. It is deliberately well above the real +# (handful) count: an N+1 in per-row serialization would be 100+ queries, far above this budget. +_BOUNDED_QUERY_BUDGET = 10 + + +def _count_queries(operation): + """Return the number of DB queries `operation()` runs. + + CaptureQueriesContext forces a debug cursor regardless of settings.DEBUG, so this works in the + normal test environment. Callers warm up once before the first measurement so one-time caches + (ContentType, organization membership) don't skew the count. + """ + with CaptureQueriesContext(connection) as ctx: + operation() + return len(ctx.captured_queries) + + +class ToolQueryCountTestCase(TestCase): + """N+1 guards: a tool's query count must not grow with its result/relation size.""" + + def setUp(self): + self.user, _ = User.objects.get_or_create(username="perf_tool_user") + # Membership exists so visible_for_user's org branch is exercised (matches the sibling tests). + self.org, _ = Organization.objects.get_or_create(name="perf tool org") + Membership.objects.get_or_create(user=self.user, organization=self.org, is_owner=True) + self.analyzable, _ = Analyzable.objects.get_or_create( + name="perf.example.com", classification=Classification.DOMAIN + ) + tools_by_name = {tool.name: tool for tool in build_tools(user=self.user)} + self.search_jobs = tools_by_name["search_jobs"] + self.get_job_details = tools_by_name["get_job_details"] + self.summarize_job = tools_by_name["summarize_job"] + self.list_investigations = tools_by_name["list_investigations"] + self.get_investigation_tree = tools_by_name["get_investigation_tree"] + self.summarize_investigation = tools_by_name["summarize_investigation"] + self.recommend_playbook = tools_by_name["recommend_playbook"] + self.list_analyzers = tools_by_name["list_analyzers"] + self.get_data_model = tools_by_name["get_data_model"] + + def tearDown(self): + Job.objects.filter(user=self.user).delete() + Investigation.objects.filter(owner=self.user).delete() + PlaybookConfig.objects.filter(owner=self.user).delete() + Membership.objects.filter(organization=self.org).delete() + self.org.delete() + + def _make_job(self, analyzable=None): + # Owner-visible job (TLP RED is irrelevant for the owner) reused as the unit of the + # job-count and investigation-job dimensions. + return Job.objects.create( + user=self.user, + analyzable=analyzable or self.analyzable, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS, + tlp=TLP.RED.value, + ) + + def _add_reports(self, job, configs): + # One AnalyzerReport per (job, config). Distinct configs are required (the pair is unique), + # so callers pass a slice of the seeded AnalyzerConfig set. + for config in configs: + AnalyzerReport.objects.create( + report={}, + job=job, + config=config, + status=AnalyzerReport.STATUSES.SUCCESS.value, + task_id=str(uuid4()), + parameters={}, + ) + + def _make_playbook(self, name): + # A user-owned starting playbook supporting DOMAIN, so recommend_playbook("domain") returns it. + return PlaybookConfig.objects.create( + name=name, + description="perf", + type=[Classification.DOMAIN.value], + owner=self.user, + for_organization=False, + starting=True, + ) + + def test_search_jobs_query_count_is_constant(self): + self._make_job() + self.search_jobs.invoke({"limit": 50}) # warm up one-time caches + small = _count_queries(lambda: self.search_jobs.invoke({"limit": 50})) + for _ in range(5): + self._make_job() + large = _count_queries(lambda: self.search_jobs.invoke({"limit": 50})) + # select_related("analyzable") + prefetch_related(...) keep this constant; an N+1 in the + # per-row serializer would make `large` exceed `small`. + self.assertEqual(small, large) + + def test_get_investigation_tree_query_count_is_constant_in_depth(self): + inv = Investigation.objects.create( + owner=self.user, name="perf tree", status=Investigation.STATUSES.CREATED.value + ) + root = self._make_job() + inv.jobs.add(root) + # The baseline root must already be non-leaf: treebeard's get_descendants() short-circuits to + # an empty queryset (0 queries) for a leaf node, so a leaf baseline would measure the one-time + # leaf->non-leaf transition (a constant +1) instead of the depth/descendant-count invariant + # this guard targets. With one child present, the single subtree query is in both + # measurements and only the descendant count differs between them. + node = root.add_child( + user=self.user, analyzable=self.analyzable, status=Job.STATUSES.REPORTED_WITHOUT_FAILS + ) + self.get_investigation_tree.invoke({"investigation_id": inv.pk}) # warm up + small = _count_queries(lambda: self.get_investigation_tree.invoke({"investigation_id": inv.pk})) + # Extend the descendant chain deeper (8 descendants total, 9 nodes < _MAX_NODES; chain depth + # 8 < _MAX_DEPTH=10). treebeard fetches each root's whole subtree in ONE get_descendants(), + # rebuilding nesting from the materialized `path`, so the count must not grow with depth. + for _ in range(7): + node = node.add_child( + user=self.user, + analyzable=self.analyzable, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS, + ) + large = _count_queries(lambda: self.get_investigation_tree.invoke({"investigation_id": inv.pk})) + self.assertEqual(small, large) + + def test_get_job_details_query_count_is_constant_in_reports(self): + job = self._make_job() + configs = list(AnalyzerConfig.objects.all()[:6]) # seeded DB has hundreds + self._add_reports(job, configs[:1]) + self.get_job_details.invoke({"job_id": job.pk}) # warm up + small = _count_queries(lambda: self.get_job_details.invoke({"job_id": job.pk})) + self._add_reports(job, configs[1:6]) # 5 more reports, distinct configs + large = _count_queries(lambda: self.get_job_details.invoke({"job_id": job.pk})) + # analyzerreports are prefetched + nested-serialized; an N+1 would scale with report count. + self.assertEqual(small, large) + + def test_summarize_job_query_count_is_constant_in_reports(self): + job = self._make_job() + configs = list(AnalyzerConfig.objects.all()[:6]) + self._add_reports(job, configs[:1]) + self.summarize_job.invoke({"job_id": job.pk}) # warm up + small = _count_queries(lambda: self.summarize_job.invoke({"job_id": job.pk})) + self._add_reports(job, configs[1:6]) + large = _count_queries(lambda: self.summarize_job.invoke({"job_id": job.pk})) + self.assertEqual(small, large) + + def test_list_investigations_query_count_is_constant(self): + Investigation.objects.create( + owner=self.user, name="perf inv 0", status=Investigation.STATUSES.CREATED.value + ) + self.list_investigations.invoke({"limit": 50}) # warm up + small = _count_queries(lambda: self.list_investigations.invoke({"limit": 50})) + for i in range(5): + Investigation.objects.create( + owner=self.user, + name=f"perf inv {i + 1}", + status=Investigation.STATUSES.CREATED.value, + ) + large = _count_queries(lambda: self.list_investigations.invoke({"limit": 50})) + # The list serializer reads DB columns only (jobs-hitting properties are excluded), so the + # count must stay constant as the number of investigations grows. + self.assertEqual(small, large) + + def test_summarize_investigation_query_count_is_constant_in_jobs(self): + inv = Investigation.objects.create( + owner=self.user, name="perf summ", status=Investigation.STATUSES.CREATED.value + ) + inv.jobs.add(self._make_job()) + self.summarize_investigation.invoke({"investigation_id": inv.pk}) # warm up + small = _count_queries(lambda: self.summarize_investigation.invoke({"investigation_id": inv.pk})) + for _ in range(5): + inv.jobs.add(self._make_job()) + large = _count_queries(lambda: self.summarize_investigation.invoke({"investigation_id": inv.pk})) + # The status breakdown is a single aggregate, so adding jobs must not add queries. + self.assertEqual(small, large) + + def test_recommend_playbook_query_count_is_constant(self): + self._make_playbook("perf_pb_0") + # limit=50 (not the default 10) so neither run is capped: if both were capped at the same + # size the invariance would hold even WITH an N+1. With 1 vs 6 visible playbooks uncapped, + # an N+1 in the analyzers/connectors serialization would make `large` exceed `small`. + self.recommend_playbook.invoke({"classification": "domain", "limit": 50}) # warm up + small = _count_queries( + lambda: self.recommend_playbook.invoke({"classification": "domain", "limit": 50}) + ) + for i in range(5): + self._make_playbook(f"perf_pb_{i + 1}") + large = _count_queries( + lambda: self.recommend_playbook.invoke({"classification": "domain", "limit": 50}) + ) + self.assertEqual(small, large) + + def test_list_analyzers_query_count_is_bounded(self): + # The analyzer dimension is the seeded GLOBAL AnalyzerConfig set (100+), impractical to + # vary in a test. A bounded assertion is robust here precisely because the set is large: + # an N+1 in per-row serialization would be 100+ queries, far above this bound. + self.list_analyzers.invoke({"observable_type": "domain"}) # warm up + count = _count_queries(lambda: self.list_analyzers.invoke({"observable_type": "domain"})) + self.assertLess(count, _BOUNDED_QUERY_BUDGET) + + def test_get_data_model_query_count_is_bounded(self): + # Single object (one job's data model), no list dimension: a small fixed budget catches a + # newly-added per-call query. A job with no data model returns {} and still exercises the + # visible_for_user lookup + the nullable GenericForeignKey access. + job = self._make_job() + self.get_data_model.invoke({"job_id": job.pk}) # warm up + count = _count_queries(lambda: self.get_data_model.invoke({"job_id": job.pk})) + self.assertLess(count, _BOUNDED_QUERY_BUDGET) + + +@override_settings(CHANNEL_LAYERS=INMEMORY_CHANNEL_LAYER) +class ChatTaskQueryCountTestCase(TestCase): + """The WS turn loads prior history in a single query, so a turn's query count must not grow + with conversation length. The agent executor is mocked (no LLM, no real tool calls).""" + + def setUp(self): + self.user, _ = User.objects.get_or_create(username="perf_task_user") + self.session = ChatSession.objects.create(user=self.user) + + @patch("api_app.chatbot_manager.tasks.get_channel_layer") + @patch("api_app.chatbot_manager.agent.agent.build_agent_executor") + def test_process_chat_message_query_count_is_constant_in_history(self, mock_build, mock_get_layer): + layer = MagicMock() + layer.group_send = AsyncMock() + mock_get_layer.return_value = layer + executor = MagicMock() + executor.invoke.return_value = {"output": "ok"} + executor.tools = [] # handler.tool_names = set(); no real tools needed + mock_build.return_value = executor + + # Each call persists one user + one assistant message, so history grows naturally. + process_chat_message(self.session.id, "hi", self.user.id) # warm up + small = _count_queries(lambda: process_chat_message(self.session.id, "hi", self.user.id)) + for i in range(20): + ChatMessage.objects.create(session=self.session, role=ChatMessage.Role.USER, content=f"m{i}") + large = _count_queries(lambda: process_chat_message(self.session.id, "hi", self.user.id)) + # history.messages loads all prior turns in one query; an N+1 would scale with history size. + self.assertEqual(small, large) diff --git a/tests/api_app/chatbot_manager/test_rate_limit.py b/tests/api_app/chatbot_manager/test_rate_limit.py new file mode 100644 index 0000000000..0d44a297b1 --- /dev/null +++ b/tests/api_app/chatbot_manager/test_rate_limit.py @@ -0,0 +1,182 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +"""Unit tests for the chatbot RateLimiter. + +All tests use LocMemCache (no Redis dependency) so they work in CI and +offline. time.time is patched for window-boundary assertions. +""" + +from concurrent.futures import ThreadPoolExecutor +from unittest.mock import patch + +from django.test import SimpleTestCase, override_settings + +from api_app.chatbot_manager.rate_limit import CACHE_ALIAS, RateLimiter + +LOCMEM = {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"} +TEST_CACHES = {"default": LOCMEM, CACHE_ALIAS: LOCMEM} + +_user_b = "7" +_limit = 5 + + +def _make_limiter(limit=_limit, window=60): + return RateLimiter(limit=limit, window_seconds=window) + + +def _fresh_key(): + """Return a per-test key so LocMemCache state from one test never leaks into + another. SimpleTestCase does not flush the cache between tests, and all + tests share the same process-scoped LocMemCache.""" + import uuid + + return str(uuid.uuid4()) + + +class RateLimiterAllowTests(SimpleTestCase): + """allow() is a pure read — no counter mutation.""" + + @override_settings(CACHES=TEST_CACHES) + def test_allow_returns_true_when_under_limit(self): + user = _fresh_key() + limiter = _make_limiter() + for _ in range(4): + limiter.increment(user) + allowed, retry_after = limiter.allow(user) + self.assertTrue(allowed) + self.assertEqual(retry_after, 0) + + @override_settings(CACHES=TEST_CACHES) + def test_allow_returns_true_at_edge_of_limit(self): + user = _fresh_key() + limiter = _make_limiter() + for _ in range(4): + limiter.increment(user) + # 4 < 5, so the 5th action should be allowed + allowed, _ = limiter.allow(user) + self.assertTrue(allowed) + + @override_settings(CACHES=TEST_CACHES) + def test_allow_returns_false_at_limit(self): + user = _fresh_key() + limiter = _make_limiter() + for _ in range(_limit): + limiter.increment(user) + allowed, retry_after = limiter.allow(user) + self.assertFalse(allowed) + self.assertGreater(retry_after, 0, "retry_after should be positive") + + @override_settings(CACHES=TEST_CACHES) + def test_retry_after_positive_at_end_of_window(self): + """In the final fraction of a window retry_after must still be >= 1: int() + truncation would return 0 here (regression guard for the flaky failure).""" + limiter = _make_limiter() + user = _fresh_key() + # 1742345580 is exactly on a 60s boundary; +59.7 puts us 0.3s before the next + # rollover, so window_seconds - elapsed = 0.3 (int() -> 0, ceil() -> 1). + end_of_window = 1742345580.0 + 59.7 + with patch("api_app.chatbot_manager.rate_limit.time.time", return_value=end_of_window): + for _ in range(_limit): + limiter.increment(user) + allowed, retry_after = limiter.allow(user) + self.assertFalse(allowed) + self.assertGreaterEqual(retry_after, 1) + + @override_settings(CACHES=TEST_CACHES) + def test_retry_after_decreases_as_time_passes(self): + """retry_after is recalculated from the current time, so it shrinks as the + window progresses.""" + limiter = _make_limiter() + # Freeze time at the very start of a window. + window_start = 1742345600.0 # arbitrary, exactly on a 60s boundary + user = _fresh_key() + with patch("api_app.chatbot_manager.rate_limit.time.time", return_value=window_start): + for _ in range(_limit): + limiter.increment(user) + _, first = limiter.allow(user) + # 10 seconds later, retry_after should be ~10s smaller. + with patch( + "api_app.chatbot_manager.rate_limit.time.time", + return_value=window_start + 10, + ): + _, later = limiter.allow(user) + self.assertEqual(later, first - 10) + self.assertGreater(later, 0) + + +class RateLimiterIncrementTests(SimpleTestCase): + """increment() mutates the counter atomically.""" + + @override_settings(CACHES=TEST_CACHES) + def test_counter_resets_after_window(self): + limiter = _make_limiter() + window_start = 1742345600.0 + user = _fresh_key() + # Fill the current window. + with patch("api_app.chatbot_manager.rate_limit.time.time", return_value=window_start): + for _ in range(_limit): + limiter.increment(user) + allowed, _ = limiter.allow(user) + self.assertFalse(allowed) + # Advance to the next window. + with patch( + "api_app.chatbot_manager.rate_limit.time.time", + return_value=window_start + 61, + ): + allowed, _ = limiter.allow(user) + self.assertTrue(allowed, "counter should reset in the next window") + limiter.increment(user) + current = limiter._cache.get(limiter._cache_key(user), 0) + self.assertEqual(current, 1) + + @override_settings(CACHES=TEST_CACHES) + def test_different_users_have_independent_counters(self): + limiter = _make_limiter() + user_a = _fresh_key() + # Saturate user A. + for _ in range(_limit): + limiter.increment(user_a) + # User B should still be free. + allowed, _ = limiter.allow(_user_b) + self.assertTrue(allowed) + + @override_settings(CACHES=TEST_CACHES) + def test_increment_is_thread_safety_smoke(self): + """10 concurrent increments — at most *limit* should succeed.""" + limiter = _make_limiter() + user = _fresh_key() + results = [] + + def attempt(): + if limiter.allow(user)[0]: + limiter.increment(user) + results.append("accepted") + else: + results.append("rejected") + + with ThreadPoolExecutor(max_workers=10) as pool: + list(pool.map(lambda _: attempt(), range(10))) + + accepted = sum(1 for r in results if r == "accepted") + rejected = sum(1 for r in results if r == "rejected") + self.assertLessEqual(accepted, _limit) + self.assertEqual(accepted + rejected, 10) + + +class RateLimiterCacheKeyTests(SimpleTestCase): + """The cache key changes every window_seconds.""" + + @override_settings(CACHES=TEST_CACHES) + def test_cache_keys_differ_across_windows(self): + limiter = _make_limiter(window=60) + # 1000 and 1010 are both in window 16 (960–1019). + with patch("api_app.chatbot_manager.rate_limit.time.time", return_value=1000.0): + k1 = limiter._cache_key("u1") + with patch("api_app.chatbot_manager.rate_limit.time.time", return_value=1010.0): + k2 = limiter._cache_key("u1") + # 1020 falls into window 17. + with patch("api_app.chatbot_manager.rate_limit.time.time", return_value=1020.0): + k3 = limiter._cache_key("u1") + self.assertEqual(k1, k2, "same window → same key") + self.assertNotEqual(k2, k3, "next window → new key") diff --git a/tests/api_app/chatbot_manager/test_streaming.py b/tests/api_app/chatbot_manager/test_streaming.py new file mode 100644 index 0000000000..4507ea7754 --- /dev/null +++ b/tests/api_app/chatbot_manager/test_streaming.py @@ -0,0 +1,111 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from unittest.mock import AsyncMock, MagicMock, patch + +from django.test import SimpleTestCase + +from api_app.chatbot_manager import events +from api_app.chatbot_manager.agent.streaming import ChatStreamingCallbackHandler + +USER_ID = 7 +SESSION_ID = 3 + + +class ChatStreamingCallbackHandlerTestCase(SimpleTestCase): + """The callback streams answer text and a status per tool action.""" + + def _make_handler(self, tool_names=None): + handler = ChatStreamingCallbackHandler(user_id=USER_ID, session_id=SESSION_ID, tool_names=tool_names) + layer = MagicMock() + layer.group_send = AsyncMock() + handler._channel_layer = layer + return handler, layer + + @staticmethod + def _messages(layer): + # each group_send call is (group, channel_message) + return [call.args for call in layer.group_send.call_args_list] + + def test_text_tokens_stream_in_order(self): + handler, layer = self._make_handler() + for token in ["Hello", " world"]: + handler.on_llm_new_token(token) + + group = events.chat_group_for_user(USER_ID) + self.assertEqual( + self._messages(layer), + [ + (group, events.TokenEvent(SESSION_ID, "Hello").as_channel_message()), + (group, events.TokenEvent(SESSION_ID, " world").as_channel_message()), + ], + ) + + def test_empty_tokens_are_not_streamed(self): + # A tool-call delta reaches on_llm_new_token with empty text (the call itself rides + # tool_call_chunks); nothing may hit the wire for it. + handler, layer = self._make_handler() + for token in ["", "", ""]: + handler.on_llm_new_token(token) + + layer.group_send.assert_not_called() + + def test_mixed_stream_forwards_only_text(self): + # A turn that starts with tool-call deltas and ends with the streamed answer. + handler, layer = self._make_handler() + for token in ["", "", "The", " answer"]: + handler.on_llm_new_token(token) + + contents = [m[1]["payload"]["content"] for m in self._messages(layer)] + self.assertEqual(contents, ["The", " answer"]) + + def test_agent_action_emits_status_event_with_tool_name(self): + handler, layer = self._make_handler() + action = MagicMock() + action.tool = "search_jobs" + + handler.on_agent_action(action) + + self.assertEqual( + self._messages(layer)[0], + ( + events.chat_group_for_user(USER_ID), + events.StatusEvent(SESSION_ID, "search_jobs").as_channel_message(), + ), + ) + + def test_agent_action_for_unregistered_tool_is_suppressed(self): + # With a real tool registry, LangChain's "_Exception" parse-error pseudo-tool (and any + # raw malformed model output as a "tool") must not leak to the client as chat.status. + handler, layer = self._make_handler(tool_names={"search_jobs"}) + + exception_action = MagicMock() + exception_action.tool = "_Exception" + handler.on_agent_action(exception_action) + layer.group_send.assert_not_called() + + real_action = MagicMock() + real_action.tool = "search_jobs" + handler.on_agent_action(real_action) + layer.group_send.assert_called_once() + + def test_tool_output_with_pending_id_emits_action_required(self): + import json as _json + + handler = ChatStreamingCallbackHandler(user_id=1, session_id=42) + with patch.object(handler, "_emit") as mock_emit: + handler.on_tool_end( + _json.dumps({"errors": [], "plan": {"observable_name": "x"}, "pending_id": "abc"}), + run_id="fake-run-id", + ) + mock_emit.assert_called_once() + event = mock_emit.call_args.args[0] + self.assertEqual(event.pending_id, "abc") + + def test_plain_tool_output_emits_nothing(self): + import json as _json + + handler = ChatStreamingCallbackHandler(user_id=1, session_id=42) + with patch.object(handler, "_emit") as mock_emit: + handler.on_tool_end(_json.dumps({"errors": [], "jobs": []}), run_id="fake-run-id") + mock_emit.assert_not_called() diff --git a/tests/api_app/chatbot_manager/test_tasks.py b/tests/api_app/chatbot_manager/test_tasks.py new file mode 100644 index 0000000000..0470916715 --- /dev/null +++ b/tests/api_app/chatbot_manager/test_tasks.py @@ -0,0 +1,244 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import datetime +from unittest.mock import AsyncMock, MagicMock, patch + +from django.test import TestCase, override_settings +from django.utils.timezone import now +from langchain_core.messages import AIMessage, HumanMessage + +from api_app.chatbot_manager import events +from api_app.chatbot_manager.agent.agent import AGENT_STOPPED_OUTPUT +from api_app.chatbot_manager.models import ChatMessage, ChatSession +from api_app.chatbot_manager.tasks import delete_old_chat_sessions, process_chat_message +from certego_saas.apps.user.models import User + +INMEMORY_CHANNEL_LAYER = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"}} + + +@override_settings(CHATBOT_MESSAGE_RETENTION_DAYS=30) +class DeleteOldChatSessionsTestCase(TestCase): + def setUp(self): + self.user, _ = User.objects.get_or_create(username="chatbot_retention_user") + + def _session_with_last_message(self, days_old): + session = ChatSession.objects.create(user=self.user) + ChatMessage.objects.create( + session=session, + role=ChatMessage.Role.USER, + content="hi", + timestamp=now() - datetime.timedelta(days=days_old), + ) + return session + + def test_deletes_stale_session_and_its_messages(self): + session = self._session_with_last_message(days_old=40) + message_pks = list(session.messages.values_list("pk", flat=True)) + + self.assertEqual(delete_old_chat_sessions(), 1) + self.assertFalse(ChatSession.objects.filter(pk=session.pk).exists()) + # CASCADE: the session's messages must be gone too + self.assertFalse(ChatMessage.objects.filter(pk__in=message_pks).exists()) + + def test_keeps_recent_session(self): + session = self._session_with_last_message(days_old=5) + + self.assertEqual(delete_old_chat_sessions(), 0) + self.assertTrue(ChatSession.objects.filter(pk=session.pk).exists()) + + def test_boundary_around_cutoff(self): + # retention = 30 days: __lt cutoff means strictly older than 30 days is stale. + stale = self._session_with_last_message(days_old=31) + fresh = self._session_with_last_message(days_old=29) + + self.assertEqual(delete_old_chat_sessions(), 1) + self.assertFalse(ChatSession.objects.filter(pk=stale.pk).exists()) + self.assertTrue(ChatSession.objects.filter(pk=fresh.pk).exists()) + + def test_empty_session_uses_created_at(self): + # No messages -> last_activity falls back to created_at (the Coalesce branch). + old_empty = ChatSession.objects.create(user=self.user, created_at=now() - datetime.timedelta(days=40)) + recent_empty = ChatSession.objects.create( + user=self.user, created_at=now() - datetime.timedelta(days=5) + ) + + self.assertEqual(delete_old_chat_sessions(), 1) + self.assertFalse(ChatSession.objects.filter(pk=old_empty.pk).exists()) + self.assertTrue(ChatSession.objects.filter(pk=recent_empty.pk).exists()) + + def test_old_session_with_recent_message_is_kept(self): + # Long-running session: created long ago but still active. last_activity must come + # from the most recent message, NOT created_at, so it must survive. + session = ChatSession.objects.create(user=self.user, created_at=now() - datetime.timedelta(days=40)) + ChatMessage.objects.create( + session=session, + role=ChatMessage.Role.USER, + content="still here", + timestamp=now() - datetime.timedelta(days=2), + ) + + self.assertEqual(delete_old_chat_sessions(), 0) + self.assertTrue(ChatSession.objects.filter(pk=session.pk).exists()) + + +@override_settings(CHANNEL_LAYERS=INMEMORY_CHANNEL_LAYER) +class ProcessChatMessageTestCase(TestCase): + """The Celery turn persists the exchange, streams start/end, and fails closed. + + The agent executor is mocked, so the LLM/Ollama is never touched; the token/status events + come from the agent's own callbacks (covered in test_streaming) and are not exercised here. + """ + + def setUp(self): + self.user, _ = User.objects.get_or_create(username="chatbot_task_user") + self.session = ChatSession.objects.create(user=self.user) + + @staticmethod + def _patched_layer(mock_get_layer): + layer = MagicMock() + layer.group_send = AsyncMock() + mock_get_layer.return_value = layer + return layer + + @staticmethod + def _event_types(layer): + # client-facing payload type of each group_send (start/status/token/end/error) + return [call.args[1]["payload"]["type"] for call in layer.group_send.call_args_list] + + @patch("api_app.chatbot_manager.tasks.get_channel_layer") + @patch("api_app.chatbot_manager.agent.agent.build_agent_executor") + def test_persists_turn_and_streams_start_end(self, mock_build, mock_get_layer): + layer = self._patched_layer(mock_get_layer) + executor = MagicMock() + executor.invoke.return_value = {"output": "Hi there"} + mock_build.return_value = executor + + process_chat_message(self.session.id, "hello", self.user.id) + + messages = list( + ChatMessage.objects.filter(session=self.session) + .order_by("timestamp") + .values_list("role", "content") + ) + self.assertEqual( + messages, + [(ChatMessage.Role.USER, "hello"), (ChatMessage.Role.ASSISTANT, "Hi there")], + ) + self.assertEqual( + self._event_types(layer), + [events.ChatEventType.START.value, events.ChatEventType.END.value], + ) + + end_payload = layer.group_send.call_args_list[-1].args[1]["payload"] + assistant = ChatMessage.objects.get(session=self.session, role=ChatMessage.Role.ASSISTANT) + self.assertEqual(end_payload["message_id"], assistant.id) + self.assertEqual(end_payload["content"], "Hi there") + + # streaming requested on the model + callbacks attached at run level (not on the LLM) + self.assertTrue(mock_build.call_args.kwargs["streaming"]) + self.assertIn("callbacks", executor.invoke.call_args.kwargs["config"]) + + @patch("api_app.chatbot_manager.tasks.get_channel_layer") + @patch("api_app.chatbot_manager.agent.agent.build_agent_executor") + def test_prior_turns_reach_the_agent_as_messages(self, mock_build, mock_get_layer): + self._patched_layer(mock_get_layer) + ChatMessage.objects.create(session=self.session, role=ChatMessage.Role.USER, content="prev q") + ChatMessage.objects.create(session=self.session, role=ChatMessage.Role.ASSISTANT, content="prev a") + executor = MagicMock() + executor.invoke.return_value = {"output": "ok"} + mock_build.return_value = executor + + process_chat_message(self.session.id, "hello", self.user.id) + + # history feeds the prompt's chat_history MessagesPlaceholder directly: LangChain + # message objects (not pre-rendered text), snapshotted before this turn is persisted + # so the current message is not part of it. + invoke_input = executor.invoke.call_args.args[0] + self.assertEqual(invoke_input["input"], "hello") + self.assertEqual( + [(type(m), m.content) for m in invoke_input["chat_history"]], + [(HumanMessage, "prev q"), (AIMessage, "prev a")], + ) + + @patch("api_app.chatbot_manager.tasks.get_channel_layer") + @patch("api_app.chatbot_manager.agent.agent.build_agent_executor") + def test_agent_failure_streams_error_and_drops_the_turn(self, mock_build, mock_get_layer): + layer = self._patched_layer(mock_get_layer) + executor = MagicMock() + executor.invoke.side_effect = ConnectionError("ollama down") + mock_build.return_value = executor + + process_chat_message(self.session.id, "hello", self.user.id) + + # failed turn is dropped: neither the user nor the assistant message is stored + self.assertFalse(ChatMessage.objects.filter(session=self.session).exists()) + self.assertEqual( + self._event_types(layer), + [events.ChatEventType.START.value, events.ChatEventType.ERROR.value], + ) + error_payload = layer.group_send.call_args_list[-1].args[1]["payload"] + self.assertEqual(error_payload["detail"], events.ChatErrorDetail.UNAVAILABLE.value) + + @patch("api_app.chatbot_manager.tasks.get_channel_layer") + @patch("api_app.chatbot_manager.agent.agent.build_agent_executor") + def test_iteration_cap_streams_error_and_drops_the_turn(self, mock_build, mock_get_layer): + layer = self._patched_layer(mock_get_layer) + executor = MagicMock() + # what AgentExecutor returns when max_iterations force-stops the run + executor.invoke.return_value = {"output": AGENT_STOPPED_OUTPUT} + mock_build.return_value = executor + + process_chat_message(self.session.id, "hello", self.user.id) + + # the canned framework string must never be persisted as an assistant message + self.assertFalse(ChatMessage.objects.filter(session=self.session).exists()) + self.assertEqual( + self._event_types(layer), + [events.ChatEventType.START.value, events.ChatEventType.ERROR.value], + ) + error_payload = layer.group_send.call_args_list[-1].args[1]["payload"] + self.assertEqual(error_payload["detail"], events.ChatErrorDetail.ITERATION_LIMIT.value) + + @patch("api_app.chatbot_manager.tasks.get_channel_layer") + @patch("api_app.chatbot_manager.agent.agent.build_agent_executor") + def test_context_url_is_injected_as_page_context(self, mock_build, mock_get_layer): + self._patched_layer(mock_get_layer) + executor = MagicMock() + executor.invoke.return_value = {"output": "ok"} + mock_build.return_value = executor + + process_chat_message(self.session.id, "summarize this", self.user.id, "https://intelowl.test/jobs/42") + + invoke_input = executor.invoke.call_args.args[0] + self.assertEqual( + invoke_input["page_context"], + "The user is currently viewing job #42 in the IntelOwl UI.", + ) + + @patch("api_app.chatbot_manager.tasks.get_channel_layer") + @patch("api_app.chatbot_manager.agent.agent.build_agent_executor") + def test_missing_context_url_yields_empty_page_context(self, mock_build, mock_get_layer): + self._patched_layer(mock_get_layer) + executor = MagicMock() + executor.invoke.return_value = {"output": "ok"} + mock_build.return_value = executor + + process_chat_message(self.session.id, "hello", self.user.id) # no context_url + + self.assertEqual(executor.invoke.call_args.args[0]["page_context"], "") + + @patch("api_app.chatbot_manager.tasks.get_channel_layer") + @patch("api_app.chatbot_manager.agent.agent.build_agent_executor") + def test_session_not_owned_by_user_is_rejected(self, mock_build, mock_get_layer): + layer = self._patched_layer(mock_get_layer) + other = User.objects.create(username="chatbot_task_other") + foreign_session = ChatSession.objects.create(user=other) + + process_chat_message(foreign_session.id, "hello", self.user.id) + + mock_build.assert_not_called() + self.assertFalse(ChatMessage.objects.filter(session=foreign_session).exists()) + self.assertEqual(self._event_types(layer), [events.ChatEventType.ERROR.value]) + error_payload = layer.group_send.call_args_list[0].args[1]["payload"] + self.assertEqual(error_payload["detail"], events.ChatErrorDetail.SESSION_NOT_FOUND.value) diff --git a/tests/api_app/chatbot_manager/test_views.py b/tests/api_app/chatbot_manager/test_views.py new file mode 100644 index 0000000000..96c8f4b7f8 --- /dev/null +++ b/tests/api_app/chatbot_manager/test_views.py @@ -0,0 +1,307 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from datetime import timedelta +from unittest.mock import MagicMock, patch + +from django.test import override_settings +from django.utils.timezone import now +from langchain_core.messages import AIMessage, HumanMessage +from rest_framework import status +from rest_framework.test import APITestCase + +from api_app.chatbot_manager.agent.agent import AGENT_STOPPED_OUTPUT +from api_app.chatbot_manager.events import ChatErrorDetail +from api_app.chatbot_manager.models import ChatMessage, ChatSession +from certego_saas.apps.user.models import User + +MOCK_AGENT_OUTPUT = {"output": "Here are your recent jobs."} + + +class ChatSessionViewSetTestCase(APITestCase): + URL = "/api/chatbot/sessions" + MESSAGE_URL = "/api/chatbot/sessions/message" + + def setUp(self): + self.user, _ = User.objects.get_or_create(username="chatbot_view_user") + self.client.force_authenticate(user=self.user) + + @patch( + "api_app.chatbot_manager.views.build_agent_executor", + return_value=MagicMock(invoke=MagicMock(return_value=MOCK_AGENT_OUTPUT)), + ) + def test_message_creates_session_when_none_provided(self, mock_executor): + response = self.client.post( + self.MESSAGE_URL, + data={"message": "Show me recent jobs"}, + format="json", + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + data = response.json() + self.assertIn("session_id", data) + self.assertIn("response", data) + self.assertIn("message_id", data) + self.assertEqual(data["response"], MOCK_AGENT_OUTPUT["output"]) + self.assertTrue(ChatSession.objects.filter(pk=data["session_id"]).exists()) + + @patch( + "api_app.chatbot_manager.views.build_agent_executor", + return_value=MagicMock(invoke=MagicMock(return_value=MOCK_AGENT_OUTPUT)), + ) + def test_message_reuses_existing_session(self, mock_executor): + session = ChatSession.objects.create(user=self.user) + response = self.client.post( + self.MESSAGE_URL, + data={"message": "Hello", "session_id": session.pk}, + format="json", + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.json()["session_id"], session.pk) + + @patch( + "api_app.chatbot_manager.views.build_agent_executor", + return_value=MagicMock(invoke=MagicMock(return_value=MOCK_AGENT_OUTPUT)), + ) + def test_message_saves_user_and_assistant_messages(self, mock_executor): + response = self.client.post( + self.MESSAGE_URL, + data={"message": "Hello"}, + format="json", + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + session_id = response.json()["session_id"] + msgs = list(ChatMessage.objects.filter(session_id=session_id).order_by("timestamp")) + self.assertEqual(len(msgs), 2) + self.assertEqual(msgs[0].role, ChatMessage.Role.USER) + self.assertEqual(msgs[0].content, "Hello") + self.assertEqual(msgs[1].role, ChatMessage.Role.ASSISTANT) + self.assertEqual(msgs[1].content, MOCK_AGENT_OUTPUT["output"]) + + @patch("api_app.chatbot_manager.views.build_agent_executor") + def test_message_passes_prior_turns_as_messages(self, mock_build): + executor = MagicMock() + executor.invoke.return_value = MOCK_AGENT_OUTPUT + mock_build.return_value = executor + session = ChatSession.objects.create(user=self.user) + ChatMessage.objects.create(session=session, role=ChatMessage.Role.USER, content="prev q") + ChatMessage.objects.create(session=session, role=ChatMessage.Role.ASSISTANT, content="prev a") + + response = self.client.post( + self.MESSAGE_URL, + data={"message": "Hello", "session_id": session.pk}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + # history reaches the agent as LangChain message objects (the prompt's chat_history + # MessagesPlaceholder), read before this turn is persisted. + invoke_input = executor.invoke.call_args.args[0] + self.assertEqual(invoke_input["input"], "Hello") + self.assertEqual( + [(type(m), m.content) for m in invoke_input["chat_history"]], + [(HumanMessage, "prev q"), (AIMessage, "prev a")], + ) + + @patch("api_app.chatbot_manager.views.build_agent_executor") + def test_message_iteration_cap_returns_error_and_drops_the_turn(self, mock_build): + executor = MagicMock() + # what AgentExecutor returns when max_iterations force-stops the run + executor.invoke.return_value = {"output": AGENT_STOPPED_OUTPUT} + mock_build.return_value = executor + session = ChatSession.objects.create(user=self.user) + + response = self.client.post( + self.MESSAGE_URL, + data={"message": "Hello", "session_id": session.pk}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_503_SERVICE_UNAVAILABLE) + content = response.json() + self.assertEqual(content["detail"], ChatErrorDetail.ITERATION_LIMIT.value) + # the session id rides the error so a session created by this request stays usable + self.assertEqual(content["session_id"], session.pk) + # the canned framework string must never be persisted as an assistant message + self.assertFalse(ChatMessage.objects.filter(session=session).exists()) + + def test_message_returns_404_for_other_users_session(self): + other_user, _ = User.objects.get_or_create(username="chatbot_other_view_user") + other_session = ChatSession.objects.create(user=other_user) + response = self.client.post( + self.MESSAGE_URL, + data={"message": "Hello", "session_id": other_session.pk}, + format="json", + ) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_message_requires_authentication(self): + self.client.force_authenticate(user=None) + response = self.client.post( + self.MESSAGE_URL, + data={"message": "Hello"}, + format="json", + ) + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + def test_list_annotates_title_from_first_user_message(self): + session = ChatSession.objects.create(user=self.user) + ChatMessage.objects.create(session=session, role=ChatMessage.Role.USER, content="Show me recent jobs") + ChatMessage.objects.create(session=session, role=ChatMessage.Role.ASSISTANT, content="Here they are.") + response = self.client.get(self.URL) + self.assertEqual(response.status_code, status.HTTP_200_OK) + results = response.json()["results"] + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["title"], "Show me recent jobs") + + def test_list_truncates_title_to_40_chars(self): + session = ChatSession.objects.create(user=self.user) + long_msg = "a" * 60 + ChatMessage.objects.create(session=session, role=ChatMessage.Role.USER, content=long_msg) + response = self.client.get(self.URL) + self.assertEqual(response.status_code, status.HTTP_200_OK) + title = response.json()["results"][0]["title"] + self.assertEqual(len(title), 40) + self.assertEqual(title, long_msg[:40]) + + def test_list_title_is_null_for_empty_session(self): + ChatSession.objects.create(user=self.user) + response = self.client.get(self.URL) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIsNone(response.json()["results"][0]["title"]) + + @override_settings( + CACHES={ + "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, + "chatbot_rate_limit": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, + } + ) + @patch( + "api_app.chatbot_manager.views.build_agent_executor", + return_value=MagicMock(invoke=MagicMock(return_value=MOCK_AGENT_OUTPUT)), + ) + def test_rate_limit_returns_429_envelope(self, mock_executor): + """6th message in the same window returns 429 with IntelOwl error envelope.""" + limit = 5 + with self.settings(CHATBOT_RATE_LIMIT=limit, CHATBOT_RATE_LIMIT_WINDOW=60): + # Send 5 messages first — the mock agent returns success, so the first 5 + # pass through to the executor (200, not 429). + for i in range(limit): + response = self.client.post( + self.MESSAGE_URL, + data={"message": f"msg {i}"}, + format="json", + ) + self.assertNotEqual( + response.status_code, + status.HTTP_429_TOO_MANY_REQUESTS, + f"request {i} should not be rate-limited", + ) + + # 6th message — rate-limited. + response = self.client.post( + self.MESSAGE_URL, + data={"message": "one too many"}, + format="json", + ) + self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS) + data = response.json() + self.assertIn("errors", data) + self.assertEqual(len(data["errors"]), 1) + error = data["errors"][0] + self.assertIn("Too many messages", error["detail"]) + self.assertEqual(error["code"], "rate_limited") + self.assertGreater(error["retry_after"], 0) + + @patch( + "api_app.chatbot_manager.views.build_agent_executor", + return_value=MagicMock(invoke=MagicMock(side_effect=RuntimeError("ollama down"))), + ) + def test_message_returns_503_when_agent_unavailable(self, mock_executor): + response = self.client.post( + self.MESSAGE_URL, + data={"message": "Hello"}, + format="json", + ) + self.assertEqual(response.status_code, status.HTTP_503_SERVICE_UNAVAILABLE) + self.assertEqual(response.json()["detail"], ChatErrorDetail.UNAVAILABLE.value) + + def tearDown(self): + ChatSession.objects.filter(user=self.user).delete() + + +class ChatSessionMessagesActionTestCase(APITestCase): + """Tests for GET /api/chatbot/sessions/{id}/messages.""" + + def setUp(self): + self.user, _ = User.objects.get_or_create(username="chatbot_messages_user") + self.client.force_authenticate(user=self.user) + self.session = ChatSession.objects.create(user=self.user) + + @staticmethod + def _url(pk): + return f"/api/chatbot/sessions/{pk}/messages" + + def test_returns_messages_ordered_by_timestamp(self): + # Persist in reverse chronological order to prove the endpoint sorts by + # timestamp ascending rather than echoing insertion order. + base = now() + ChatMessage.objects.create( + session=self.session, + role=ChatMessage.Role.ASSISTANT, + content="second", + timestamp=base + timedelta(seconds=2), + ) + ChatMessage.objects.create( + session=self.session, + role=ChatMessage.Role.USER, + content="first", + timestamp=base + timedelta(seconds=1), + ) + response = self.client.get(self._url(self.session.pk)) + self.assertEqual(response.status_code, status.HTTP_200_OK) + results = response.json()["results"] + self.assertEqual([m["content"] for m in results], ["first", "second"]) + self.assertEqual([m["role"] for m in results], ["user", "assistant"]) + + def test_paginated_shape_and_page_size(self): + base = now() + for i in range(12): + ChatMessage.objects.create( + session=self.session, + role=ChatMessage.Role.USER, + content=f"m{i}", + timestamp=base + timedelta(seconds=i), + ) + response = self.client.get(self._url(self.session.pk)) + content = response.json() + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("count", content) + self.assertIn("total_pages", content) + self.assertIn("results", content) + self.assertEqual(content["count"], 12) + self.assertEqual(content["total_pages"], 2) + self.assertEqual(len(content["results"]), 10) # PAGE_SIZE default + + page2 = self.client.get(self._url(self.session.pk), {"page": 2}).json() + self.assertEqual(len(page2["results"]), 2) + + def test_empty_session_returns_empty_results(self): + response = self.client.get(self._url(self.session.pk)) + content = response.json() + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(content["count"], 0) + self.assertEqual(content["results"], []) + + def test_returns_404_for_other_users_session(self): + other_user, _ = User.objects.get_or_create(username="chatbot_messages_other_user") + other_session = ChatSession.objects.create(user=other_user) + response = self.client.get(self._url(other_session.pk)) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_requires_authentication(self): + self.client.force_authenticate(user=None) + response = self.client.get(self._url(self.session.pk)) + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + def tearDown(self): + ChatSession.objects.filter(user__username__startswith="chatbot_messages_").delete() diff --git a/tests/api_app/chatbot_manager/tools/__init__.py b/tests/api_app/chatbot_manager/tools/__init__.py new file mode 100644 index 0000000000..acb99e9651 --- /dev/null +++ b/tests/api_app/chatbot_manager/tools/__init__.py @@ -0,0 +1,2 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. diff --git a/tests/api_app/chatbot_manager/tools/test_analyze_observable.py b/tests/api_app/chatbot_manager/tools/test_analyze_observable.py new file mode 100644 index 0000000000..bbdc463d6b --- /dev/null +++ b/tests/api_app/chatbot_manager/tools/test_analyze_observable.py @@ -0,0 +1,127 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import json +from unittest.mock import patch + +from django.test import TestCase, override_settings + +from api_app.analyzers_manager.models import AnalyzerConfig +from api_app.chatbot_manager.agent.tools import build_tools +from api_app.models import Job +from api_app.playbooks_manager.models import PlaybookConfig +from certego_saas.apps.organization.membership import Membership +from certego_saas.apps.organization.organization import Organization +from certego_saas.apps.user.models import User + +# The trigger lives at `intel_owl.tasks.job_pipeline.apply_async`; patching it lets us assert +# whether an analysis would have been launched without running Celery (the create path imports +# `job_pipeline` from this module, so the patched attribute is the one it calls). +_APPLY_ASYNC = "intel_owl.tasks.job_pipeline.apply_async" + + +@override_settings(CHATBOT_PENDING_ACTION_TTL=600) +class AnalyzeObservableToolTestCase(TestCase): + def setUp(self): + self.user, _ = User.objects.get_or_create(username="analyze_obs_user") + # Another member of the same org: used to build a playbook that is private to them (i.e. + # NOT visible to self.user) for the isolation test. + self.org_member, _ = User.objects.get_or_create(username="analyze_obs_org_member") + self.org, _ = Organization.objects.get_or_create(name="analyze obs organization") + Membership.objects.get_or_create(user=self.user, organization=self.org, is_owner=True) + Membership.objects.get_or_create(user=self.org_member, organization=self.org, is_owner=False) + + # A real seeded, free (no API key) observable analyzer that supports `domain` and is runnable + # at TLP CLEAR (max_tlp AMBER). Constructing one would need a PythonModule FK. + self.analyzer = AnalyzerConfig.objects.get(name="Tranco") + + # Owned by self.user -> visible: passes the tool's visibility guard. + self.pb_owned = PlaybookConfig.objects.create( + name="analyze_obs_pb_owned", description="t", type=["domain"], owner=self.user, starting=True + ) + self.pb_owned.analyzers.set([self.analyzer]) + # Owned by another member, NOT org-shared -> invisible to self.user. + self.pb_private_other = PlaybookConfig.objects.create( + name="analyze_obs_pb_private", + description="t", + type=["domain"], + owner=self.org_member, + for_organization=False, + starting=True, + ) + self.pb_private_other.analyzers.set([self.analyzer]) + + self.analyze_observable = {t.name: t for t in build_tools(user=self.user)}["analyze_observable"] + + def tearDown(self): + Job.objects.filter(user__in=[self.user, self.org_member]).delete() + PlaybookConfig.objects.filter(owner__in=[self.user, self.org_member]).delete() + Membership.objects.filter(organization=self.org).delete() + self.org.delete() + + @patch(_APPLY_ASYNC) + def test_preview_returns_plan_and_pending_id_without_launching(self, mock_apply): + data = json.loads( + self.analyze_observable.invoke({"observable_name": "example.com", "analyzers": "Tranco"}) + ) + self.assertEqual(data["errors"], []) + self.assertIsNotNone(data["plan"]) + self.assertEqual(data["plan"]["classification"], "domain") + self.assertIn("Tranco", data["plan"]["analyzers"]) + self.assertTrue(data["pending_id"]) + mock_apply.assert_not_called() # the tool NEVER launches + + @patch(_APPLY_ASYNC) + def test_preview_mints_a_consumable_pending_record(self, mock_apply): + from api_app.chatbot_manager.pending_action import consume_pending_analysis + + data = json.loads( + self.analyze_observable.invoke({"observable_name": "example.com", "analyzers": "Tranco"}) + ) + payload = consume_pending_analysis(self.user.id, data["pending_id"]) + self.assertEqual(payload["observable_name"], "example.com") + self.assertEqual(payload["analyzers"], "Tranco") + mock_apply.assert_not_called() + + @patch(_APPLY_ASYNC) + def test_private_ip_refused_at_preview(self, mock_apply): + data = json.loads( + self.analyze_observable.invoke({"observable_name": "10.0.0.1", "analyzers": "Classic_DNS"}) + ) + self.assertTrue(data["errors"]) + self.assertIsNone(data["plan"]) + self.assertIsNone(data["pending_id"]) + mock_apply.assert_not_called() + + @patch(_APPLY_ASYNC) + def test_unknown_analyzer_refused_at_preview(self, mock_apply): + data = json.loads( + self.analyze_observable.invoke( + {"observable_name": "example.com", "analyzers": "NotARealAnalyzer"} + ) + ) + self.assertTrue(data["errors"]) + self.assertIsNone(data["pending_id"]) + mock_apply.assert_not_called() + + @patch(_APPLY_ASYNC) + def test_playbook_not_visible_refused(self, mock_apply): + data = json.loads( + self.analyze_observable.invoke( + {"observable_name": "example.com", "playbook": self.pb_private_other.name} + ) + ) + self.assertTrue(any("not found or not visible" in e for e in data["errors"])) + self.assertIsNone(data["plan"]) + self.assertIsNone(data["pending_id"]) + mock_apply.assert_not_called() + + @patch(_APPLY_ASYNC) + def test_visible_playbook_preview(self, mock_apply): + data = json.loads( + self.analyze_observable.invoke({"observable_name": "example.com", "playbook": self.pb_owned.name}) + ) + self.assertEqual(data["errors"], []) + self.assertEqual(data["plan"]["playbook"], self.pb_owned.name) + self.assertTrue(data["pending_id"]) + mock_apply.assert_not_called() diff --git a/tests/api_app/chatbot_manager/tools/test_common.py b/tests/api_app/chatbot_manager/tools/test_common.py new file mode 100644 index 0000000000..e2135dbf9c --- /dev/null +++ b/tests/api_app/chatbot_manager/tools/test_common.py @@ -0,0 +1,27 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from django.test import TestCase + +from api_app.chatbot_manager.agent.tools._common import MAX_RESULTS, clamp_limit + + +class ClampLimitHelperTestCase(TestCase): + """Unit tests for the shared `clamp_limit` helper used by the analyzer/playbook tools.""" + + def test_within_range_passes_through(self): + errors = [] + self.assertEqual(clamp_limit(10, errors), 10) + self.assertEqual(errors, []) + + def test_over_cap_clamps_and_warns(self): + errors = [] + self.assertEqual(clamp_limit(999, errors), MAX_RESULTS) + self.assertTrue(any("exceeds the maximum" in e for e in errors)) + + def test_below_one_clamps_up_silently(self): + # A non-positive limit is bounded up to 1 without a warning (only over-cap is worth one). + errors = [] + self.assertEqual(clamp_limit(0, errors), 1) + self.assertEqual(clamp_limit(-5, errors), 1) + self.assertEqual(errors, []) diff --git a/tests/api_app/chatbot_manager/tools/test_data_model.py b/tests/api_app/chatbot_manager/tools/test_data_model.py new file mode 100644 index 0000000000..61952d593d --- /dev/null +++ b/tests/api_app/chatbot_manager/tools/test_data_model.py @@ -0,0 +1,90 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import json + +from django.test import TestCase + +from api_app.analyzables_manager.models import Analyzable +from api_app.chatbot_manager.agent.tools import build_tools +from api_app.choices import TLP, Classification +from api_app.data_model_manager.models import DomainDataModel +from api_app.models import Job +from certego_saas.apps.organization.membership import Membership +from certego_saas.apps.organization.organization import Organization +from certego_saas.apps.user.models import User + + +class DataModelToolTestCase(TestCase): + def setUp(self): + self.user, _ = User.objects.get_or_create(username="dm_tool_user") + self.other_user, _ = User.objects.get_or_create(username="dm_tool_other_user") + self.org_member, _ = User.objects.get_or_create(username="dm_tool_org_member") + self.org, _ = Organization.objects.get_or_create(name="dm tool org") + Membership.objects.get_or_create(user=self.user, organization=self.org, is_owner=True) + Membership.objects.get_or_create(user=self.org_member, organization=self.org, is_owner=False) + self.analyzable, _ = Analyzable.objects.get_or_create( + name="dm.example.com", + classification=Classification.DOMAIN, + ) + # Job with an aggregated data model attached (as the analysis pipeline would set it). + self.job_with_dm = Job.objects.create( + user=self.user, + analyzable=self.analyzable, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS, + ) + self.data_model = DomainDataModel.objects.create() + self.job_with_dm.data_model = self.data_model + self.job_with_dm.save() + # Job without a data model. + self.job_without_dm = Job.objects.create( + user=self.user, + analyzable=self.analyzable, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS, + ) + # Another user's job, RED so it stays inaccessible under visible_for_user. + self.other_job = Job.objects.create( + user=self.other_user, + analyzable=self.analyzable, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS, + tlp=TLP.RED.value, + ) + # Org-mate's RED job: reachable via organization membership (no data model attached; + # the point is that it resolves instead of returning "not accessible"). + self.org_job = Job.objects.create( + user=self.org_member, + analyzable=self.analyzable, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS, + tlp=TLP.RED.value, + ) + self.get_data_model = {t.name: t for t in build_tools(user=self.user)}["get_data_model"] + + def tearDown(self): + Job.objects.filter(user__in=[self.user, self.other_user, self.org_member]).delete() + Membership.objects.filter(organization=self.org).delete() + self.org.delete() + self.data_model.delete() + + def test_get_data_model_returns_serialized(self): + data = json.loads(self.get_data_model.invoke({"job_id": self.job_with_dm.pk})) + self.assertEqual(data["errors"], []) + self.assertTrue(data["data_model"]) + self.assertIn("reliability", data["data_model"]) + + def test_get_data_model_empty_when_absent(self): + data = json.loads(self.get_data_model.invoke({"job_id": self.job_without_dm.pk})) + self.assertEqual(data["errors"], []) + self.assertEqual(data["data_model"], {}) + + def test_get_data_model_forbidden_other_user(self): + data = json.loads(self.get_data_model.invoke({"job_id": self.other_job.pk})) + self.assertEqual(data["data_model"], {}) + self.assertTrue(data["errors"]) + self.assertIn("not found or not accessible", data["errors"][0]) + + def test_get_data_model_org_shared_visible(self): + # An org-mate's RED job is reachable via organization membership (UI parity); + # the data model is empty because none is attached, but it is NOT "not accessible". + data = json.loads(self.get_data_model.invoke({"job_id": self.org_job.pk})) + self.assertEqual(data["errors"], []) + self.assertEqual(data["data_model"], {}) diff --git a/tests/api_app/chatbot_manager/tools/test_investigations.py b/tests/api_app/chatbot_manager/tools/test_investigations.py new file mode 100644 index 0000000000..dc0a33063c --- /dev/null +++ b/tests/api_app/chatbot_manager/tools/test_investigations.py @@ -0,0 +1,160 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import json + +from django.test import TestCase + +from api_app.analyzables_manager.models import Analyzable +from api_app.chatbot_manager.agent.tools import build_tools +from api_app.choices import Classification +from api_app.investigations_manager.models import Investigation +from api_app.models import Job +from certego_saas.apps.organization.membership import Membership +from certego_saas.apps.organization.organization import Organization +from certego_saas.apps.user.models import User + + +class InvestigationToolsTestCase(TestCase): + def setUp(self): + self.user, _ = User.objects.get_or_create(username="inv_tool_user") + # Same organization as self.user: used to test that org-shared investigations are + # visible (visible_for_user) while another member's private ones are not. + self.org_member, _ = User.objects.get_or_create(username="inv_tool_org_member") + self.org, _ = Organization.objects.get_or_create(name="inv tool organization") + Membership.objects.get_or_create(user=self.user, organization=self.org, is_owner=True) + Membership.objects.get_or_create(user=self.org_member, organization=self.org, is_owner=False) + + self.analyzable, _ = Analyzable.objects.get_or_create( + name="inv.example.com", + classification=Classification.DOMAIN, + ) + + self.inv_created = Investigation.objects.create( + owner=self.user, + name="alpha investigation", + status=Investigation.STATUSES.CREATED.value, + ) + self.inv_running = Investigation.objects.create( + owner=self.user, + name="beta investigation", + status=Investigation.STATUSES.RUNNING.value, + ) + # Owned by another member of the same org and shared at org level -> visible. + self.inv_org_shared = Investigation.objects.create( + owner=self.org_member, + name="gamma investigation", + status=Investigation.STATUSES.CONCLUDED.value, + for_organization=True, + ) + # Owned by another member but NOT shared -> must stay invisible to self.user. + self.inv_private = Investigation.objects.create( + owner=self.org_member, + name="delta investigation", + status=Investigation.STATUSES.CREATED.value, + for_organization=False, + ) + + # A small job tree on inv_created: root1 -> child1 (different status) + root2, + # so the tree has nesting and the status breakdown spans >= 2 statuses. + self.root1 = Job.objects.create( + user=self.user, + analyzable=self.analyzable, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS, + ) + self.child1 = self.root1.add_child( + user=self.user, + analyzable=self.analyzable, + status=Job.STATUSES.FAILED, + ) + self.root2 = Job.objects.create( + user=self.user, + analyzable=self.analyzable, + status=Job.STATUSES.RUNNING, + ) + self.inv_created.jobs.add(self.root1, self.root2) + + tools_by_name = {t.name: t for t in build_tools(user=self.user)} + self.list_investigations = tools_by_name["list_investigations"] + self.get_investigation_tree = tools_by_name["get_investigation_tree"] + self.summarize_investigation = tools_by_name["summarize_investigation"] + + def tearDown(self): + Job.objects.filter(user__in=[self.user, self.org_member]).delete() + Investigation.objects.filter(owner__in=[self.user, self.org_member]).delete() + Membership.objects.filter(organization=self.org).delete() + self.org.delete() + + def test_list_investigations_status_filter(self): + data = json.loads(self.list_investigations.invoke({"status": "running"})) + self.assertEqual(data["errors"], []) + ids = [i["id"] for i in data["investigations"]] + self.assertIn(self.inv_running.pk, ids) + self.assertNotIn(self.inv_created.pk, ids) + + def test_list_investigations_name_filter(self): + data = json.loads(self.list_investigations.invoke({"query": "beta"})) + ids = [i["id"] for i in data["investigations"]] + self.assertEqual(ids, [self.inv_running.pk]) + + def test_list_investigations_limit(self): + data = json.loads(self.list_investigations.invoke({"limit": 1})) + self.assertEqual(len(data["investigations"]), 1) + + def test_list_investigations_isolation_and_org_shared(self): + data = json.loads(self.list_investigations.invoke({})) + ids = [i["id"] for i in data["investigations"]] + # owned + org-shared are visible + self.assertIn(self.inv_created.pk, ids) + self.assertIn(self.inv_running.pk, ids) + self.assertIn(self.inv_org_shared.pk, ids) + # another member's private investigation is NOT visible + self.assertNotIn(self.inv_private.pk, ids) + + def test_list_investigations_invalid_status(self): + data = json.loads(self.list_investigations.invoke({"status": "bogus"})) + # invalid status is reported and the filter is ignored (results still returned) + self.assertTrue(data["errors"]) + self.assertIn("Unknown status", data["errors"][0]) + ids = [i["id"] for i in data["investigations"]] + self.assertIn(self.inv_created.pk, ids) + + def test_list_investigations_limit_over_cap_reports_error(self): + result = self.list_investigations.invoke({"limit": 100}) + data = json.loads(result) + self.assertTrue(any("maximum 50" in e for e in data["errors"])) + + def test_get_investigation_tree_structure(self): + data = json.loads(self.get_investigation_tree.invoke({"investigation_id": self.inv_created.pk})) + self.assertEqual(data["errors"], []) + tree = data["investigation"] + self.assertEqual(tree["id"], self.inv_created.pk) + nodes = {n["id"]: n for n in tree["jobs"]} + self.assertIn(self.root1.pk, nodes) + self.assertIn(self.root2.pk, nodes) + self.assertEqual(nodes[self.root1.pk]["observable"], "inv.example.com") + child_ids = [c["id"] for c in nodes[self.root1.pk]["children"]] + self.assertEqual(child_ids, [self.child1.pk]) + self.assertEqual(nodes[self.root2.pk]["children"], []) + + def test_get_investigation_tree_not_visible(self): + data = json.loads(self.get_investigation_tree.invoke({"investigation_id": self.inv_private.pk})) + self.assertIsNone(data["investigation"]) + self.assertTrue(data["errors"]) + self.assertIn("not found or not accessible", data["errors"][0]) + + def test_summarize_investigation_breakdown(self): + summary = json.loads(self.summarize_investigation.invoke({"investigation_id": self.inv_created.pk}))[ + "summary" + ] + self.assertIn(f"Investigation #{self.inv_created.pk}", summary) + self.assertIn("Total jobs : 3", summary) + # breakdown must span the distinct job statuses, not just one branch + self.assertIn(Job.STATUSES.REPORTED_WITHOUT_FAILS.value, summary) + self.assertIn(Job.STATUSES.FAILED.value, summary) + self.assertIn(Job.STATUSES.RUNNING.value, summary) + + def test_summarize_investigation_not_visible(self): + data = json.loads(self.summarize_investigation.invoke({"investigation_id": self.inv_private.pk})) + self.assertIsNone(data["summary"]) + self.assertTrue(data["errors"]) diff --git a/tests/api_app/chatbot_manager/tools/test_list_analyzers.py b/tests/api_app/chatbot_manager/tools/test_list_analyzers.py new file mode 100644 index 0000000000..1211770878 --- /dev/null +++ b/tests/api_app/chatbot_manager/tools/test_list_analyzers.py @@ -0,0 +1,97 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import json + +from django.test import TestCase + +from api_app.analyzers_manager.constants import TypeChoices +from api_app.analyzers_manager.models import AnalyzerConfig +from api_app.chatbot_manager.agent.tools import build_tools +from api_app.chatbot_manager.agent.tools._common import MAX_RESULTS +from api_app.models import OrganizationPluginConfiguration +from certego_saas.apps.organization.membership import Membership +from certego_saas.apps.organization.organization import Organization +from certego_saas.apps.user.models import User + + +class ListAnalyzersToolTestCase(TestCase): + def setUp(self): + self.org_user, _ = User.objects.get_or_create(username="list_analyzers_org_user") + self.outsider, _ = User.objects.get_or_create(username="list_analyzers_outsider") + self.org, _ = Organization.objects.get_or_create(name="list analyzers organization") + Membership.objects.get_or_create(user=self.org_user, organization=self.org, is_owner=True) + + # A real seeded observable analyzer (constructing one would need a PythonModule FK). + self.analyzer = ( + AnalyzerConfig.objects.filter( + type=TypeChoices.OBSERVABLE, + observable_supported__contains=["domain"], + disabled=False, + ) + .order_by("name") + .first() + ) + self.assertIsNotNone(self.analyzer, "expected a seeded observable analyzer supporting 'domain'") + + # Disable it for self.org: the per-user `runnable` flag must flip to False, but the + # analyzer must still be listed (analyzer configs are global, non-sensitive). + self.org_disable = OrganizationPluginConfiguration.objects.create( + config=self.analyzer, organization=self.org, disabled=True + ) + + self.list_analyzers = {t.name: t for t in build_tools(user=self.org_user)}["list_analyzers"] + + def tearDown(self): + self.org_disable.delete() + Membership.objects.filter(organization=self.org).delete() + self.org.delete() + + def test_list_analyzers_filters_by_observable_type(self): + data = json.loads(self.list_analyzers.invoke({"observable_type": "domain"})) + self.assertEqual(data["errors"], []) + self.assertTrue(data["analyzers"]) + # every returned analyzer supports the requested observable type + for analyzer in data["analyzers"]: + self.assertIn("domain", analyzer["observable_supported"]) + names = [a["name"] for a in data["analyzers"]] + self.assertIn(self.analyzer.name, names) + + def test_list_analyzers_unknown_observable_type(self): + data = json.loads(self.list_analyzers.invoke({"observable_type": "bogus"})) + # invalid type is reported and the filter is ignored (results still returned) + self.assertTrue(data["errors"]) + self.assertIn("Unknown observable_type", data["errors"][0]) + self.assertTrue(data["analyzers"]) + + def test_list_analyzers_limit(self): + data = json.loads(self.list_analyzers.invoke({"limit": 1})) + self.assertEqual(len(data["analyzers"]), 1) + + def test_list_analyzers_limit_clamps_to_max(self): + # An over-cap limit from the LLM is clamped to MAX_RESULTS (untrusted arg): the seeded + # observable analyzers exceed the cap, so the result is capped, never the requested 999. + # The clamp is surfaced in `errors` so a truncated list isn't silent. + data = json.loads(self.list_analyzers.invoke({"limit": 999})) + self.assertLessEqual(len(data["analyzers"]), MAX_RESULTS) + self.assertTrue(any("exceeds the maximum" in e for e in data["errors"])) + + def test_list_analyzers_org_disabled_runnable_flag(self): + # Deterministic, one-directional isolation check: an analyzer disabled for the user's + # organization comes back with runnable=False AND is still present in the list (it is + # not hidden -- list_analyzers lists, it does not filter on runnable). We do NOT assert + # the contrasting runnable=True for an outsider, because `runnable` also folds in + # configured+healthy, which a key-based analyzer lacks on a test deploy without keys. + rows = { + a["name"]: a + for a in json.loads(self.list_analyzers.invoke({"observable_type": "domain"}))["analyzers"] + } + self.assertIn(self.analyzer.name, rows) + self.assertFalse(rows[self.analyzer.name]["runnable"]) + + # The same analyzer is still listed for an unaffiliated user (global, non-sensitive). + outsider_tool = {t.name: t for t in build_tools(user=self.outsider)}["list_analyzers"] + outsider_names = [ + a["name"] for a in json.loads(outsider_tool.invoke({"observable_type": "domain"}))["analyzers"] + ] + self.assertIn(self.analyzer.name, outsider_names) diff --git a/tests/api_app/chatbot_manager/tools/test_recommend_playbook.py b/tests/api_app/chatbot_manager/tools/test_recommend_playbook.py new file mode 100644 index 0000000000..e35cf16fb5 --- /dev/null +++ b/tests/api_app/chatbot_manager/tools/test_recommend_playbook.py @@ -0,0 +1,135 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import json + +from django.test import TestCase + +from api_app.chatbot_manager.agent.tools import build_tools +from api_app.choices import ScanMode +from api_app.playbooks_manager.models import PlaybookConfig +from certego_saas.apps.organization.membership import Membership +from certego_saas.apps.organization.organization import Organization +from certego_saas.apps.user.models import User + + +class RecommendPlaybookToolTestCase(TestCase): + def setUp(self): + self.user, _ = User.objects.get_or_create(username="recommend_pb_user") + # Another member of the same org: used to test org-shared vs private visibility. + self.org_member, _ = User.objects.get_or_create(username="recommend_pb_org_member") + self.org, _ = Organization.objects.get_or_create(name="recommend pb organization") + Membership.objects.get_or_create(user=self.user, organization=self.org, is_owner=True) + Membership.objects.get_or_create(user=self.org_member, organization=self.org, is_owner=False) + + self.pb_owned = PlaybookConfig.objects.create( + name="recommend_pb_owned", description="t", type=["ip"], owner=self.user, starting=True + ) + # Owned by another member of the same org and shared at org level -> visible. + self.pb_org_shared = PlaybookConfig.objects.create( + name="recommend_pb_org_shared", + description="t", + type=["ip"], + owner=self.org_member, + for_organization=True, + starting=True, + ) + # Owned by another member but NOT shared -> must stay invisible to self.user. + self.pb_private_other = PlaybookConfig.objects.create( + name="recommend_pb_private_other", + description="t", + type=["ip"], + owner=self.org_member, + for_organization=False, + starting=True, + ) + # Not directly launchable -> must not be recommended. A non-starting playbook must force + # new analysis with no check time (model clean()), which create() enforces. + self.pb_not_starting = PlaybookConfig.objects.create( + name="recommend_pb_not_starting", + description="t", + type=["ip"], + owner=self.user, + starting=False, + scan_mode=ScanMode.FORCE_NEW_ANALYSIS.value, + scan_check_time=None, + ) + # Disabled -> must not be recommended. + self.pb_disabled = PlaybookConfig.objects.create( + name="recommend_pb_disabled", + description="t", + type=["ip"], + owner=self.user, + starting=True, + disabled=True, + ) + # Different classification -> must not match an ip observable. + self.pb_domain = PlaybookConfig.objects.create( + name="recommend_pb_domain", description="t", type=["domain"], owner=self.user, starting=True + ) + + self.recommend_playbook = {t.name: t for t in build_tools(user=self.user)}["recommend_playbook"] + + def tearDown(self): + PlaybookConfig.objects.filter(owner__in=[self.user, self.org_member]).delete() + Membership.objects.filter(organization=self.org).delete() + self.org.delete() + + def test_recommend_playbook_derives_classification_from_observable(self): + data = json.loads(self.recommend_playbook.invoke({"observable_name": "8.8.8.8"})) + self.assertEqual(data["errors"], []) + names = [p["name"] for p in data["playbooks"]] + # 8.8.8.8 is classified as ip -> the ip playbook matches, the domain one does not + self.assertIn(self.pb_owned.name, names) + self.assertNotIn(self.pb_domain.name, names) + + def test_recommend_playbook_explicit_classification(self): + data = json.loads(self.recommend_playbook.invoke({"classification": "ip"})) + self.assertEqual(data["errors"], []) + names = [p["name"] for p in data["playbooks"]] + self.assertIn(self.pb_owned.name, names) + + def test_recommend_playbook_invalid_classification(self): + data = json.loads(self.recommend_playbook.invoke({"classification": "bogus"})) + self.assertTrue(data["errors"]) + self.assertIn("Unknown classification", data["errors"][0]) + self.assertEqual(data["playbooks"], []) + + def test_recommend_playbook_requires_input(self): + data = json.loads(self.recommend_playbook.invoke({})) + self.assertTrue(data["errors"]) + self.assertEqual(data["playbooks"], []) + + def test_recommend_playbook_only_starting_and_enabled(self): + names = [ + p["name"] + for p in json.loads(self.recommend_playbook.invoke({"classification": "ip"}))["playbooks"] + ] + self.assertIn(self.pb_owned.name, names) + self.assertNotIn(self.pb_not_starting.name, names) + self.assertNotIn(self.pb_disabled.name, names) + + def test_recommend_playbook_visibility_isolation(self): + names = [ + p["name"] + for p in json.loads(self.recommend_playbook.invoke({"classification": "ip"}))["playbooks"] + ] + # owned + org-shared are visible + self.assertIn(self.pb_owned.name, names) + self.assertIn(self.pb_org_shared.name, names) + # another member's private playbook is NOT visible + self.assertNotIn(self.pb_private_other.name, names) + + def test_recommend_playbook_limit(self): + # Two visible, starting, enabled ip playbooks match (owned + org-shared); the cap must + # trim the result to the requested size. + all_matches = json.loads(self.recommend_playbook.invoke({"classification": "ip"}))["playbooks"] + self.assertGreaterEqual(len(all_matches), 2) + capped = json.loads(self.recommend_playbook.invoke({"classification": "ip", "limit": 1})) + self.assertEqual(len(capped["playbooks"]), 1) + + def test_recommend_playbook_limit_clamp_warns(self): + # An over-cap limit is clamped to MAX_RESULTS and the clamp is surfaced in `errors`, + # regardless of how many playbooks actually match (999 > 50 always warns). + data = json.loads(self.recommend_playbook.invoke({"classification": "ip", "limit": 999})) + self.assertTrue(any("exceeds the maximum" in e for e in data["errors"])) diff --git a/tests/api_app/chatbot_manager/tools/test_search_jobs.py b/tests/api_app/chatbot_manager/tools/test_search_jobs.py new file mode 100644 index 0000000000..e7542f068c --- /dev/null +++ b/tests/api_app/chatbot_manager/tools/test_search_jobs.py @@ -0,0 +1,187 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import json +from uuid import uuid4 + +from django.test import TestCase + +from api_app.analyzables_manager.models import Analyzable +from api_app.analyzers_manager.models import AnalyzerConfig, AnalyzerReport +from api_app.chatbot_manager.agent.tools import build_tools +from api_app.choices import TLP, Classification +from api_app.models import Job +from certego_saas.apps.organization.membership import Membership +from certego_saas.apps.organization.organization import Organization +from certego_saas.apps.user.models import User + + +class SearchJobsToolTestCase(TestCase): + def setUp(self): + self.user, _ = User.objects.get_or_create(username="chatbot_tool_user") + self.other_user, _ = User.objects.get_or_create(username="chatbot_other_user") + # org-mate of self.user, to exercise visible_for_user org sharing. + self.org_member, _ = User.objects.get_or_create(username="chatbot_org_member") + self.org, _ = Organization.objects.get_or_create(name="chatbot tool org") + Membership.objects.get_or_create(user=self.user, organization=self.org, is_owner=True) + Membership.objects.get_or_create(user=self.org_member, organization=self.org, is_owner=False) + + self.analyzable, _ = Analyzable.objects.get_or_create( + name="malware.example.com", + classification=Classification.DOMAIN, + ) + self.shared_analyzable, _ = Analyzable.objects.get_or_create( + name="shared.example.com", + classification=Classification.DOMAIN, + ) + self.public_analyzable, _ = Analyzable.objects.get_or_create( + name="public.example.com", + classification=Classification.DOMAIN, + ) + # TLP RED on the owner/other jobs so the cross-user isolation assertions stay + # meaningful under visible_for_user (CLEAR/GREEN jobs are globally visible by design). + self.job = Job.objects.create( + user=self.user, + analyzable=self.analyzable, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS, + tlp=TLP.RED.value, + ) + self.other_job = Job.objects.create( + user=self.other_user, + analyzable=self.analyzable, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS, + tlp=TLP.RED.value, + ) + # Org-mate's RED job (distinct observable + non-RWF status so it doesn't perturb the + # name/status filters above); visible to self.user only via organization membership. + self.org_job = Job.objects.create( + user=self.org_member, + analyzable=self.shared_analyzable, + status=Job.STATUSES.RUNNING, + tlp=TLP.RED.value, + ) + # Unrelated user's GREEN job: globally visible under UI parity. + self.green_job = Job.objects.create( + user=self.other_user, + analyzable=self.public_analyzable, + status=Job.STATUSES.RUNNING, + tlp=TLP.GREEN.value, + ) + tools = build_tools(user=self.user) + tools_by_name = {t.name: t for t in tools} + self.search_jobs = tools_by_name["search_jobs"] + self.get_job_details = tools_by_name["get_job_details"] + self.summarize_job = tools_by_name["summarize_job"] + + def test_search_jobs_returns_matching(self): + result = self.search_jobs.invoke({"query": "malware.example.com"}) + data = json.loads(result) + self.assertEqual(data["errors"], []) + self.assertEqual(len(data["jobs"]), 1) + self.assertEqual(data["jobs"][0]["id"], self.job.pk) + + def test_search_jobs_respects_user_isolation(self): + other_tools = build_tools(user=self.other_user) + search = {t.name: t for t in other_tools}["search_jobs"] + result = search.invoke({"query": "malware.example.com"}) + data = json.loads(result) + ids = [d["id"] for d in data["jobs"]] + self.assertIn(self.other_job.pk, ids) + self.assertNotIn(self.job.pk, ids) + + def test_search_jobs_no_results(self): + result = self.search_jobs.invoke({"query": "nonexistent999"}) + data = json.loads(result) + self.assertEqual(data["errors"], []) + self.assertEqual(data["jobs"], []) + + def test_get_job_details_returns_data(self): + result = self.get_job_details.invoke({"job_id": self.job.pk}) + data = json.loads(result) + self.assertEqual(data["errors"], []) + self.assertEqual(data["job"]["id"], self.job.pk) + self.assertIn("observable_name", data["job"]) + self.assertIn("status", data["job"]) + + def test_get_job_details_forbidden_other_user(self): + result = self.get_job_details.invoke({"job_id": self.other_job.pk}) + data = json.loads(result) + self.assertIsNone(data["job"]) + self.assertTrue(data["errors"]) + self.assertIn("not found or not accessible", data["errors"][0]) + + def test_summarize_job_formats_output(self): + result = self.summarize_job.invoke({"job_id": self.job.pk}) + data = json.loads(result) + self.assertEqual(data["errors"], []) + self.assertIn(f"Job #{self.job.pk}", data["summary"]) + self.assertIn("malware.example.com", data["summary"]) + self.assertIn("Status", data["summary"]) + + def test_summarize_job_failed_reports_use_report_status(self): + # Regression: analyzer report status uses ReportStatus (uppercase). Only the + # non-SUCCESS report must show up under "Failed". + config_ok, config_ko = list(AnalyzerConfig.objects.all()[:2]) + AnalyzerReport.objects.create( + report={}, + job=self.job, + config=config_ok, + status=AnalyzerReport.STATUSES.SUCCESS.value, + task_id=str(uuid4()), + parameters={}, + ) + AnalyzerReport.objects.create( + report={}, + job=self.job, + config=config_ko, + status=AnalyzerReport.STATUSES.FAILED.value, + task_id=str(uuid4()), + parameters={}, + ) + summary = json.loads(self.summarize_job.invoke({"job_id": self.job.pk}))["summary"] + self.assertIn(config_ko.name, summary) + self.assertNotIn(config_ok.name, summary) + + def test_search_jobs_invalid_status_reports_error(self): + result = self.search_jobs.invoke({"status": "not_a_status"}) + data = json.loads(result) + self.assertTrue(any("Unknown status" in e for e in data["errors"])) + + def test_search_jobs_valid_status_filters(self): + result = self.search_jobs.invoke({"status": "reported_without_fails"}) + data = json.loads(result) + self.assertEqual(data["errors"], []) + self.assertEqual([d["id"] for d in data["jobs"]], [self.job.pk]) + + def test_search_jobs_limit_over_cap_reports_error(self): + result = self.search_jobs.invoke({"limit": 100}) + data = json.loads(result) + self.assertTrue(any("maximum 50" in e for e in data["errors"])) + + def test_search_jobs_includes_org_shared(self): + # An org-mate's AMBER/RED job is visible via organization membership (UI parity). + result = self.search_jobs.invoke({"query": "shared.example.com"}) + ids = [d["id"] for d in json.loads(result)["jobs"]] + self.assertIn(self.org_job.pk, ids) + + def test_search_jobs_includes_clear_green_globally(self): + # An unrelated user's CLEAR/GREEN job surfaces in search results too (UI parity). + result = self.search_jobs.invoke({"query": "public.example.com"}) + ids = [d["id"] for d in json.loads(result)["jobs"]] + self.assertIn(self.green_job.pk, ids) + + def test_get_job_details_org_shared_visible(self): + data = json.loads(self.get_job_details.invoke({"job_id": self.org_job.pk})) + self.assertEqual(data["errors"], []) + self.assertEqual(data["job"]["id"], self.org_job.pk) + + def test_get_job_details_clear_green_globally_visible(self): + # CLEAR/GREEN jobs are visible to everyone in IntelOwl, matching the UI. + data = json.loads(self.get_job_details.invoke({"job_id": self.green_job.pk})) + self.assertEqual(data["errors"], []) + self.assertEqual(data["job"]["id"], self.green_job.pk) + + def tearDown(self): + Job.objects.filter(user__in=[self.user, self.other_user, self.org_member]).delete() + Membership.objects.filter(organization=self.org).delete() + self.org.delete() diff --git a/tests/api_app/connectors_manager/integration_tests/__init__.py b/tests/api_app/connectors_manager/integration_tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/api_app/connectors_manager/integration_tests/connectors/__init__.py b/tests/api_app/connectors_manager/integration_tests/connectors/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/api_app/connectors_manager/integration_tests/connectors/test_opencti.py b/tests/api_app/connectors_manager/integration_tests/connectors/test_opencti.py new file mode 100644 index 0000000000..b994100c30 --- /dev/null +++ b/tests/api_app/connectors_manager/integration_tests/connectors/test_opencti.py @@ -0,0 +1,108 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import os +from unittest import skipUnless + +from kombu import uuid + +from api_app.analyzables_manager.models import Analyzable +from api_app.choices import Classification +from api_app.connectors_manager.connectors.opencti import OpenCTI +from api_app.connectors_manager.models import ConnectorConfig, ConnectorReport +from api_app.models import Job, Parameter, PluginConfig, Tag +from tests import CustomTestCase + + +class OpenCTILiveIntegrationTestCase(CustomTestCase): + fixtures = [ + "api_app/fixtures/0001_user.json", + ] + + def _get_opencti_config(self): + return ConnectorConfig.objects.get(name="OpenCTI") + + def _setup_job(self, config): + """Creates a real Job, Analyzable, and Tags in the test DB with live credentials.""" + url_param = Parameter.objects.get(python_module=config.python_module, name="url_key_name") + token_param = Parameter.objects.get(python_module=config.python_module, name="api_key_name") + + pcs = [ + PluginConfig.objects.create( + parameter=url_param, + value=os.getenv("OPENCTI_URL"), + for_organization=False, + owner=None, + connector_config=config, + ), + PluginConfig.objects.create( + parameter=token_param, + value=os.getenv("OPENCTI_TOKEN"), + for_organization=False, + owner=None, + connector_config=config, + ), + ] + + analyzable = Analyzable.objects.create(name="8.8.8.8", classification=Classification.IP) + job = Job.objects.create( + analyzable=analyzable, + user=self.superuser, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS.value, + ) + job.connectors_to_execute.set([config]) + + # OpenCTI specifically requires tags to generate Labels, so we attach one to the job. + tag, _ = Tag.objects.get_or_create(label="integration-test", defaults={"color": "#ff0000"}) + job.tags.add(tag) + + return job, pcs + + def _cleanup(self, job, config, pcs): + """Cleans up the database so test runs don't overlap.""" + try: + ConnectorReport.objects.get(job=job, config=config).delete() + except ConnectorReport.DoesNotExist: + pass + analyzable = job.analyzable + job.delete() + analyzable.delete() + for pc in pcs: + pc.delete() + + @skipUnless( + os.getenv("OPENCTI_URL") and os.getenv("OPENCTI_TOKEN"), + "OpenCTI live test not configured", + ) + def test_opencti_live_integration(self): + """ + End-to-End Live Test: Executes the full base class start() method, + hits the real OpenCTI server, and verifies the DB state. + """ + config = self._get_opencti_config() + job, pcs = self._setup_job(config) + + try: + connector = OpenCTI(config) + try: + # We use .start() here to invoke the Django DB state lifecycle + connector.start(job.pk, {}, uuid()) + except Exception: + pass + + # Fetch the actual report from the database to ensure it saved + report = ConnectorReport.objects.get(job=job, config=config) + + self.assertIn( + report.status, + [ConnectorReport.STATUSES.SUCCESS, ConnectorReport.STATUSES.FAILED], + ) + + # If successful, ensure the raw dictionary was saved to the DB model + if report.status == ConnectorReport.STATUSES.SUCCESS: + self.assertIsInstance(report.report, dict) + self.assertIn("observable", report.report) + self.assertIn("report", report.report) + + finally: + self._cleanup(job, config, pcs) diff --git a/tests/api_app/connectors_manager/test_abuse_submitter.py b/tests/api_app/connectors_manager/test_abuse_submitter.py deleted file mode 100644 index 57011269fe..0000000000 --- a/tests/api_app/connectors_manager/test_abuse_submitter.py +++ /dev/null @@ -1,51 +0,0 @@ -from unittest.mock import MagicMock - -from api_app.connectors_manager.connectors.abuse_submitter import AbuseSubmitter -from api_app.connectors_manager.exceptions import ConnectorRunException -from tests import CustomTestCase - - -class AbuseSubmitterTestCase(CustomTestCase): - def setUp(self): - super().setUp() - # Mocking the ConnectorConfig - self.mock_config = MagicMock() - # Mocking the job hierarchy - self.mock_job = MagicMock() - - def test_abuse_submitter_missing_hierarchy(self): - """ - Verify that AbuseSubmitter raises ConnectorRunException when job hierarchy is missing. - This tests the fix for the AttributeError crash. - """ - self.mock_job.parent_job = None - connector = AbuseSubmitter(self.mock_config) - connector._job = self.mock_job - - # Verify subject raises ConnectorRunException - with self.assertRaisesRegex(ConnectorRunException, "Job hierarchy is invalid"): - _ = connector.subject - - # Verify body raises ConnectorRunException - with self.assertRaisesRegex(ConnectorRunException, "Job hierarchy is invalid"): - _ = connector.body - - def test_abuse_submitter_valid_hierarchy(self): - """ - Verify that AbuseSubmitter correctly returns subject and body when hierarchy is valid. - """ - # Create a valid hierarchy: Job -> Parent -> Grandparent (Analyzable) - mock_parent = MagicMock() - mock_grandparent = MagicMock() - mock_analyzable = MagicMock() - mock_analyzable.name = "malicious.domain" - - self.mock_job.parent_job = mock_parent - mock_parent.parent_job = mock_grandparent - mock_grandparent.analyzable = mock_analyzable - - connector = AbuseSubmitter(self.mock_config) - connector._job = self.mock_job - - self.assertEqual(connector.subject, "Takedown domain request for malicious.domain") - self.assertIn("Domain malicious.domain", connector.body) diff --git a/tests/api_app/connectors_manager/test_classes.py b/tests/api_app/connectors_manager/test_classes.py index b44db3ebac..a617c0f851 100644 --- a/tests/api_app/connectors_manager/test_classes.py +++ b/tests/api_app/connectors_manager/test_classes.py @@ -42,13 +42,16 @@ def run(self) -> dict: parameter=Parameter.objects.get(name="url_key_name", python_module=pm), connector_config=cc, ) - with patch("requests.head"): + + with patch("requests.head") as mock_head: + mock_head.return_value.status_code = 200 result = MockUpConnector(cc).health_check(self.user) - self.assertTrue(result) - cc.disabled = False - cc.save() - result = MockUpConnector(cc).health_check(self.user) - self.assertTrue(result) + self.assertTrue(result) + cc.disabled = False + cc.save() + result = MockUpConnector(cc).health_check(self.user) + self.assertTrue(result) + cc.delete() pc.delete() @@ -98,43 +101,57 @@ def run(self) -> dict: an.delete() def test_subclasses(self): - def handler(signum, frame): - raise TimeoutError("end of time") + subclasses = Connector.all_subclasses() + for subclass in subclasses: + configs = ConnectorConfig.objects.filter(python_module=subclass.python_module) + if not configs.exists(): + self.fail(f"There is a python module {subclass.python_module} without any configuration") - import signal + def test_before_run_partial_failure(self): + # run_on_failure=False + partial failure (mix of FAILED and SUCCESS) should raise ConnectorRunException + class MockUpConnector(Connector): + def run(self) -> dict: + return {} - signal.signal(signal.SIGALRM, handler) - an1 = Analyzable.objects.create( + an = Analyzable.objects.create( name="test.com", classification=Classification.DOMAIN, ) job = Job.objects.create( - analyzable=an1, - status="reported_without_fails", - user=self.superuser, + analyzable=an, + status=Job.STATUSES.CONNECTORS_RUNNING.value, ) - - subclasses = Connector.all_subclasses() - for subclass in subclasses: - print(f"\nTesting Connector {subclass.__name__}") - configs = ConnectorConfig.objects.filter(python_module=subclass.python_module) - if not configs.exists(): - self.fail(f"There is a python module {subclass.python_module} without any configuration") - for config in configs: - job.connectors_to_execute.set([config]) - timeout_seconds = config.soft_time_limit - timeout_seconds = min(timeout_seconds, 20) - print(f"\tTesting with config {config.name} for {timeout_seconds} seconds") - sub = subclass( - config, - ) - signal.alarm(timeout_seconds) - try: - sub.start(job.pk, {}, uuid()) - except Exception as e: - self.fail(f"Connector {subclass.__name__} with config {config.name} failed {e}") - finally: - signal.alarm(0) + AnalyzerReport.objects.create( + report={}, + job=job, + config=AnalyzerConfig.objects.first(), + status=AnalyzerReport.STATUSES.FAILED.value, + task_id=str(uuid()), + parameters={}, + ) + AnalyzerReport.objects.create( + report={}, + job=job, + config=AnalyzerConfig.objects.last(), + status=AnalyzerReport.STATUSES.SUCCESS.value, + task_id=str(uuid()), + parameters={}, + ) + cc = ConnectorConfig.objects.create( + name="test", + python_module=PythonModule.objects.get( + base_path=PythonModuleBasePaths.Connector.value, module="misp.MISP" + ), + description="test", + disabled=True, + maximum_tlp="CLEAR", + run_on_failure=False, + ) + with self.assertRaises(ConnectorRunException): + muc = MockUpConnector(cc) + muc.job_id = job.pk + muc.before_run() + cc.delete() job.delete() - an1.delete() + an.delete() diff --git a/tests/api_app/connectors_manager/test_misp.py b/tests/api_app/connectors_manager/test_misp.py deleted file mode 100644 index ca5329a41d..0000000000 --- a/tests/api_app/connectors_manager/test_misp.py +++ /dev/null @@ -1,145 +0,0 @@ -from unittest.mock import MagicMock, patch - -from kombu import uuid - -from api_app.analyzables_manager.models import Analyzable -from api_app.choices import Classification -from api_app.connectors_manager.connectors.misp import MISP -from api_app.connectors_manager.models import ConnectorConfig, ConnectorReport -from api_app.models import Job, Parameter, PluginConfig -from tests import CustomTestCase - - -class MISPConnectorTestCase(CustomTestCase): - fixtures = [ - "api_app/fixtures/0001_user.json", - ] - - @staticmethod - def _get_misp_config(): - return ConnectorConfig.objects.get(name="MISP") - - @staticmethod - def _create_plugin_configs(config): - pcs = [] - for name in ("url_key_name", "api_key_name"): - param = Parameter.objects.get(python_module=config.python_module, name=name) - pc = PluginConfig.objects.create( - parameter=param, - value="https://misp.test" if "url" in name else "test-api-key", - for_organization=False, - owner=None, - connector_config=config, - ) - pcs.append(pc) - return pcs - - def _setup_job(self): - config = self._get_misp_config() - pcs = self._create_plugin_configs(config) - analyzable = Analyzable.objects.create(name="8.8.8.8", classification=Classification.IP) - job = Job.objects.create( - analyzable=analyzable, - user=self.superuser, - status=Job.STATUSES.REPORTED_WITHOUT_FAILS.value, - ) - job.connectors_to_execute.set([config]) - return job, config, pcs - - @staticmethod - def _cleanup(job, config, pcs): - try: - ConnectorReport.objects.get(job=job, config=config).delete() - except ConnectorReport.DoesNotExist: - pass - analyzable = job.analyzable - job.delete() - analyzable.delete() - for pc in pcs: - pc.delete() - - @patch("api_app.connectors_manager.connectors.misp.MockPyMISP") - @patch("api_app.connectors_manager.connectors.misp.pymisp.PyMISP") - def test_bulk_add_event_called_once(self, mock_pymisp_cls, mock_misp_cls): - """ - run() must call add_event exactly once with all attributes already - attached to the event object. add_attribute on the MISP instance - must never be called (that was the old N+1 pattern). - """ - mock_instance = MagicMock() - mock_pymisp_cls.return_value = mock_instance - mock_misp_cls.return_value = mock_instance - - mock_event = MagicMock() - mock_event.id = 42 - mock_instance.add_event.return_value = mock_event - mock_instance.get_event.return_value = {"Event": {"id": 42}} - - job, config, pcs = self._setup_job() - try: - connector = MISP(config) - connector.start(job.pk, {}, uuid()) - - report = ConnectorReport.objects.get(job=job, config=config) - self.assertEqual(report.status, ConnectorReport.STATUSES.SUCCESS) - - mock_instance.add_event.assert_called_once() - mock_instance.add_attribute.assert_not_called() - - event_arg = mock_instance.add_event.call_args[0][0] - self.assertGreaterEqual(len(event_arg.attributes), 1) - finally: - self._cleanup(job, config, pcs) - - @patch("api_app.connectors_manager.connectors.misp.MockPyMISP") - @patch("api_app.connectors_manager.connectors.misp.pymisp.PyMISP") - def test_all_attributes_present_on_event(self, mock_pymisp_cls, mock_misp_cls): - """ - The event sent to MISP must contain the base attribute and the - link attribute — all in one shot. - """ - mock_instance = MagicMock() - mock_pymisp_cls.return_value = mock_instance - mock_misp_cls.return_value = mock_instance - - mock_event = MagicMock() - mock_event.id = 99 - mock_instance.add_event.return_value = mock_event - mock_instance.get_event.return_value = {"Event": {"id": 99}} - - job, config, pcs = self._setup_job() - try: - connector = MISP(config) - connector.start(job.pk, {}, uuid()) - - event_arg = mock_instance.add_event.call_args[0][0] - attr_types = [a.type for a in event_arg.attributes] - - self.assertIn("ip-src", attr_types) - self.assertIn("link", attr_types) - finally: - self._cleanup(job, config, pcs) - - @patch("api_app.connectors_manager.connectors.misp.MockPyMISP") - @patch("api_app.connectors_manager.connectors.misp.pymisp.PyMISP") - def test_add_event_failure_marks_report_failed(self, mock_pymisp_cls, mock_misp_cls): - """ - If add_event raises, the connector report status must be FAILED. - """ - mock_instance = MagicMock() - mock_pymisp_cls.return_value = mock_instance - mock_misp_cls.return_value = mock_instance - mock_instance.add_event.side_effect = Exception("MISP unreachable") - - job, config, pcs = self._setup_job() - try: - connector = MISP(config) - try: - connector.start(job.pk, {}, uuid()) - except Exception: - pass - - report = ConnectorReport.objects.get(job=job, config=config) - self.assertEqual(report.status, ConnectorReport.STATUSES.FAILED) - finally: - self._cleanup(job, config, pcs) diff --git a/tests/api_app/connectors_manager/test_opencti.py b/tests/api_app/connectors_manager/test_opencti.py deleted file mode 100644 index f4078669b2..0000000000 --- a/tests/api_app/connectors_manager/test_opencti.py +++ /dev/null @@ -1,462 +0,0 @@ -# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl -# See the file 'LICENSE' for copying permission. - -import os -from unittest import skipUnless -from unittest.mock import patch - -from kombu import uuid - -from api_app.analyzables_manager.models import Analyzable -from api_app.choices import Classification -from api_app.connectors_manager.connectors.opencti import OpenCTI -from api_app.connectors_manager.models import ConnectorConfig, ConnectorReport -from api_app.models import Job, Parameter, PluginConfig, Tag -from tests import CustomTestCase - -# Each scenario describes: where to inject the failure, which mock to break, -# and what substrings must appear in the partial-state error message. -PARTIAL_STATE_SCENARIOS = ( - { - "name": "failure_after_observable", - "fail_mock": "label", - "expected_exception": "label failure", - "expected_ids": ["observable=obs-1", "labels=[]"], - }, - { - "name": "failure_after_report", - "fail_mock": "external_ref", - "expected_exception": "external ref failure", - "expected_ids": ["observable=obs-1", "report=report-1", "label-1"], - }, - { - "name": "failure_after_external_ref", - "fail_mock": "link", - "expected_exception": "link failure", - "expected_ids": ["external_reference=ext-ref-1", "report=report-1", "observable=obs-1"], - }, -) - - -def _partial_state_errors(report): - """Return error entries that contain the partial-state contract message.""" - return [e for e in report.errors if "Created IDs:" in str(e)] - - -def _apply_happy_path_defaults(mocks): - """Wire up all pycti mocks to return valid dicts so the connector can proceed.""" - # subTest reuses patched objects, so reset them before each scenario. - for key in ("identity", "marking", "observable", "label", "report", "external_ref", "stix_domain"): - mocks[key].reset_mock() - - # reset_mock() does not clear nested child side_effect values, so clear them explicitly. - mocks["label"].return_value.create.side_effect = None - mocks["external_ref"].return_value.create.side_effect = None - mocks["stix_domain"].return_value.add_external_reference.side_effect = None - - mocks["identity"].return_value.create.return_value = {"id": "org-1"} - mocks["marking"].return_value.create.return_value = {"id": "mark-1"} - mocks["observable"].return_value.create.return_value = {"id": "obs-1"} - mocks["observable"].return_value.read.return_value = {"id": "obs-1"} - mocks["label"].return_value.create.return_value = {"id": "label-1"} - mocks["report"].return_value.create.return_value = {"id": "report-1"} - mocks["report"].return_value.read.return_value = {"id": "report-1"} - mocks["external_ref"].return_value.create.return_value = {"id": "ext-ref-1"} - mocks["stix_domain"].return_value.add_external_reference.return_value = None - mocks["report"].return_value.add_stix_object_or_stix_relationship.return_value = None - - -def _inject_failure_for_scenario(mocks, fail_mock): - """Inject exactly one failure point while keeping the rest on the happy path.""" - failure_targets = { - "label": (mocks["label"].return_value.create, "label failure"), - "external_ref": (mocks["external_ref"].return_value.create, "external ref failure"), - "link": (mocks["stix_domain"].return_value.add_external_reference, "link failure"), - } - try: - target, message = failure_targets[fail_mock] - except KeyError as exc: - raise AssertionError(f"Unsupported fail_mock scenario: {fail_mock}") from exc - target.side_effect = Exception(message) - - -class OpenCTIConnectorTestCase(CustomTestCase): - fixtures = [ - "api_app/fixtures/0001_user.json", - ] - - # -- Helpers (stateless where possible) -- - - @staticmethod - def _get_opencti_config(): - return ConnectorConfig.objects.get(name="OpenCTI") - - @staticmethod - def _create_plugin_configs(config): - """Create required PluginConfig entries for OpenCTI (url_key_name, api_key_name).""" - pcs = [] - for name in ("url_key_name", "api_key_name"): - param = Parameter.objects.get(python_module=config.python_module, name=name) - pc = PluginConfig.objects.create( - parameter=param, - value="https://opencti.test" if "url" in name else "test-token", - for_organization=False, - owner=None, - connector_config=config, - ) - pcs.append(pc) - return pcs - - def _setup_job_with_opencti(self, add_tag=True): - """Create Analyzable + Job + OpenCTI config. Returns (job, config, pcs).""" - config = self._get_opencti_config() - pcs = self._create_plugin_configs(config) - analyzable = Analyzable.objects.create(name="8.8.8.8", classification=Classification.IP) - job = Job.objects.create( - analyzable=analyzable, - user=self.superuser, - status=Job.STATUSES.REPORTED_WITHOUT_FAILS.value, - ) - job.connectors_to_execute.set([config]) - if add_tag: - tag, _ = Tag.objects.get_or_create(label="testtag", defaults={"color": "#ff0000"}) - job.tags.add(tag) - return job, config, pcs - - @staticmethod - def _cleanup_test_objects(job, config, pcs): - """Delete only objects created by this test.""" - try: - ConnectorReport.objects.get(job=job, config=config).delete() - except ConnectorReport.DoesNotExist: - pass - analyzable = job.analyzable - job.delete() - analyzable.delete() - for pc in pcs: - pc.delete() - - def _assert_no_traceback_in_errors(self, report): - for err in report.errors: - self.assertNotIn("Traceback", str(err)) - self.assertNotIn("File ", str(err)) - - def _collect_pycti_mocks( - self, - marking_mock, - identity_mock, - stix_observable_mock, - label_mock, - report_mock, - external_ref_mock, - stix_domain_mock, - _api_client_mock, - ): - """Bundle positional patch args into a dict for _apply_happy_path_defaults / _inject_failure.""" - return { - "marking": marking_mock, - "identity": identity_mock, - "observable": stix_observable_mock, - "label": label_mock, - "report": report_mock, - "external_ref": external_ref_mock, - "stix_domain": stix_domain_mock, - "api_client": _api_client_mock, - } - - # -- Tests -- - - @patch.object(OpenCTI, "_monkeypatch", classmethod(lambda cls: None)) - @patch("pycti.OpenCTIApiClient") - @patch("pycti.StixDomainObject") - @patch("pycti.ExternalReference") - @patch("pycti.Report") - @patch("pycti.Label") - @patch("pycti.StixCyberObservable") - @patch("pycti.Identity") - @patch("pycti.MarkingDefinition") - def test_partial_state_failure_scenarios( - self, - marking_mock, - identity_mock, - stix_observable_mock, - label_mock, - report_mock, - external_ref_mock, - stix_domain_mock, - _api_client_mock, - ): - """Iterate failure points via subTest; each verifies partial-state tracking.""" - mocks = self._collect_pycti_mocks( - marking_mock, - identity_mock, - stix_observable_mock, - label_mock, - report_mock, - external_ref_mock, - stix_domain_mock, - _api_client_mock, - ) - - for scenario in PARTIAL_STATE_SCENARIOS: - with self.subTest(scenario=scenario["name"]): - _apply_happy_path_defaults(mocks) - _inject_failure_for_scenario(mocks, scenario["fail_mock"]) - - job, config, pcs = self._setup_job_with_opencti(add_tag=True) - try: - connector = OpenCTI(config) - try: - connector.start(job.pk, {}, uuid()) - except Exception: - pass - - report = ConnectorReport.objects.get(job=job, config=config) - self.assertEqual(report.status, ConnectorReport.STATUSES.FAILED) - - partial_msgs = _partial_state_errors(report) - self.assertEqual(len(partial_msgs), 1) - self.assertEqual(len(report.errors), 2) - self.assertIn( - "OpenCTI partial state detected after exception:", - partial_msgs[0], - ) - - non_partial_errors = [str(e) for e in report.errors if "Created IDs:" not in str(e)] - self.assertTrue(non_partial_errors) - self.assertIn(scenario["expected_exception"], non_partial_errors[0]) - - err_text = " ".join(map(str, report.errors)) - for substr in scenario["expected_ids"]: - self.assertIn(substr, err_text) - self._assert_no_traceback_in_errors(report) - finally: - self._cleanup_test_objects(job, config, pcs) - - @patch.object(OpenCTI, "_monkeypatch", classmethod(lambda cls: None)) - @patch("pycti.OpenCTIApiClient") - @patch("pycti.Report") - @patch("pycti.StixDomainObject") - @patch("pycti.ExternalReference") - @patch("pycti.Label") - @patch("pycti.StixCyberObservable") - @patch("pycti.Identity") - @patch("pycti.MarkingDefinition") - def test_success_path_integrity( - self, - marking_mock, - identity_mock, - stix_observable_mock, - label_mock, - external_ref_mock, - stix_domain_mock, - report_mock, - _api_client_mock, - ): - mocks = self._collect_pycti_mocks( - marking_mock, - identity_mock, - stix_observable_mock, - label_mock, - report_mock, - external_ref_mock, - stix_domain_mock, - _api_client_mock, - ) - _apply_happy_path_defaults(mocks) - - job, config, pcs = self._setup_job_with_opencti(add_tag=True) - try: - connector = OpenCTI(config) - connector.start(job.pk, {}, uuid()) - - report = ConnectorReport.objects.get(job=job, config=config) - self.assertEqual(report.status, ConnectorReport.STATUSES.SUCCESS) - self.assertEqual(report.errors, []) - self.assertIsInstance(report.report, dict) - self.assertIn("observable", report.report) - self.assertIn("report", report.report) - finally: - self._cleanup_test_objects(job, config, pcs) - - @patch.object(OpenCTI, "_monkeypatch", classmethod(lambda cls: None)) - @patch("pycti.OpenCTIApiClient") - @patch("pycti.Report") - @patch("pycti.StixDomainObject") - @patch("pycti.ExternalReference") - @patch("pycti.Label") - @patch("pycti.StixCyberObservable") - @patch("pycti.Identity") - @patch("pycti.MarkingDefinition") - def test_organization_and_marking_called_only_once( - self, - marking_mock, - identity_mock, - stix_observable_mock, - label_mock, - external_ref_mock, - stix_domain_mock, - report_mock, - _api_client_mock, - ): - mocks = self._collect_pycti_mocks( - marking_mock, - identity_mock, - stix_observable_mock, - label_mock, - report_mock, - external_ref_mock, - stix_domain_mock, - _api_client_mock, - ) - _apply_happy_path_defaults(mocks) - - job, config, pcs = self._setup_job_with_opencti(add_tag=True) - try: - connector = OpenCTI(config) - connector.start(job.pk, {}, uuid()) - identity_mock.return_value.create.assert_called_once() - marking_mock.return_value.create.assert_called_once() - finally: - self._cleanup_test_objects(job, config, pcs) - - @patch.object(OpenCTI, "_monkeypatch", classmethod(lambda cls: None)) - @patch("pycti.OpenCTIApiClient") - @patch("pycti.StixDomainObject") - @patch("pycti.ExternalReference") - @patch("pycti.Report") - @patch("pycti.Label") - @patch("pycti.StixCyberObservable") - @patch("pycti.Identity") - @patch("pycti.MarkingDefinition") - def test_observable_create_returns_non_dict_handled_safely( - self, - marking_mock, - identity_mock, - stix_observable_mock, - label_mock, - report_mock, - external_ref_mock, - stix_domain_mock, - _api_client_mock, - ): - """StixCyberObservable.create returns non-dict -> observable=None, fails safely.""" - mocks = self._collect_pycti_mocks( - marking_mock, - identity_mock, - stix_observable_mock, - label_mock, - report_mock, - external_ref_mock, - stix_domain_mock, - _api_client_mock, - ) - _apply_happy_path_defaults(mocks) - stix_observable_mock.return_value.create.return_value = None - - job, config, pcs = self._setup_job_with_opencti(add_tag=True) - try: - connector = OpenCTI(config) - try: - connector.start(job.pk, {}, uuid()) - except Exception: - pass - - report = ConnectorReport.objects.get(job=job, config=config) - self.assertEqual(report.status, ConnectorReport.STATUSES.FAILED) - - partial_msgs = _partial_state_errors(report) - self.assertEqual(len(partial_msgs), 1) - self.assertEqual(len(report.errors), 2) - err_text = " ".join(map(str, report.errors)) - self.assertIn("observable=None", err_text) - self._assert_no_traceback_in_errors(report) - finally: - self._cleanup_test_objects(job, config, pcs) - - @patch.object(OpenCTI, "_monkeypatch", classmethod(lambda cls: None)) - @patch("pycti.OpenCTIApiClient") - @patch("pycti.Label") - @patch("pycti.StixCyberObservable") - @patch("pycti.Identity") - @patch("pycti.MarkingDefinition") - def test_label_create_returns_non_dict_raises_value_error( - self, - marking_mock, - identity_mock, - stix_observable_mock, - label_mock, - _api_client_mock, - ): - """Label.create returns non-dict -> ValueError with observable + labels=[] in partial state.""" - identity_mock.return_value.create.return_value = {"id": "org-1"} - marking_mock.return_value.create.return_value = {"id": "mark-1"} - stix_observable_mock.return_value.create.return_value = {"id": "obs-1"} - label_mock.return_value.create.return_value = None - - job, config, pcs = self._setup_job_with_opencti(add_tag=True) - try: - connector = OpenCTI(config) - try: - connector.start(job.pk, {}, uuid()) - except Exception: - pass - - report = ConnectorReport.objects.get(job=job, config=config) - self.assertEqual(report.status, ConnectorReport.STATUSES.FAILED) - - err_text = " ".join(map(str, report.errors)) - self.assertIn("Invalid response from OpenCTI Label.create", err_text) - partial_msgs = _partial_state_errors(report) - self.assertEqual(len(partial_msgs), 1) - self.assertIn("observable=obs-1", err_text) - self.assertIn("labels=[]", err_text) - self.assertEqual(len(report.errors), 2) - self._assert_no_traceback_in_errors(report) - finally: - self._cleanup_test_objects(job, config, pcs) - - @skipUnless( - os.getenv("OPENCTI_URL") and os.getenv("OPENCTI_TOKEN"), - "OpenCTI live test not configured", - ) - def test_opencti_live_integration(self): - config = self._get_opencti_config() - url_param = Parameter.objects.get(python_module=config.python_module, name="url_key_name") - token_param = Parameter.objects.get(python_module=config.python_module, name="api_key_name") - pcs = [ - PluginConfig.objects.create( - parameter=url_param, - value=os.getenv("OPENCTI_URL"), - for_organization=False, - owner=None, - connector_config=config, - ), - PluginConfig.objects.create( - parameter=token_param, - value=os.getenv("OPENCTI_TOKEN"), - for_organization=False, - owner=None, - connector_config=config, - ), - ] - analyzable = Analyzable.objects.create(name="8.8.8.8", classification=Classification.IP) - job = Job.objects.create( - analyzable=analyzable, - user=self.superuser, - status=Job.STATUSES.REPORTED_WITHOUT_FAILS.value, - ) - job.connectors_to_execute.set([config]) - try: - connector = OpenCTI(config) - try: - connector.start(job.pk, {}, uuid()) - except Exception: - pass - report = ConnectorReport.objects.get(job=job, config=config) - self.assertIn( - report.status, - [ConnectorReport.STATUSES.SUCCESS, ConnectorReport.STATUSES.FAILED], - ) - finally: - self._cleanup_test_objects(job, config, pcs) diff --git a/tests/api_app/connectors_manager/test_views.py b/tests/api_app/connectors_manager/test_views.py index 9baad9abae..35a0692593 100644 --- a/tests/api_app/connectors_manager/test_views.py +++ b/tests/api_app/connectors_manager/test_views.py @@ -1,6 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. from typing import Type +from unittest.mock import patch from api_app.analyzables_manager.models import Analyzable from api_app.choices import Classification @@ -34,12 +35,15 @@ def test_health_check(self): owner=None, connector_config=connector, ) - response = self.client.get(f"{self.URL}/{connector.name}/health_check") - self.assertEqual(response.status_code, 200) - self.client.force_authenticate(self.superuser) - response = self.client.get(f"{self.URL}/{connector.name}/health_check") - self.assertEqual(response.status_code, 200) + with patch("api_app.connectors_manager.connectors.yeti.YETI.health_check", return_value=True): + response = self.client.get(f"{self.URL}/{connector.name}/health_check") + self.assertEqual(response.status_code, 200) + + self.client.force_authenticate(self.superuser) + response = self.client.get(f"{self.URL}/{connector.name}/health_check") + self.assertEqual(response.status_code, 200) + result = response.json() self.assertIn("status", result) self.assertTrue(result["status"]) diff --git a/tests/api_app/connectors_manager/unit_tests/__init__.py b/tests/api_app/connectors_manager/unit_tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/api_app/connectors_manager/unit_tests/base_test_class.py b/tests/api_app/connectors_manager/unit_tests/base_test_class.py new file mode 100644 index 0000000000..c27a038ac3 --- /dev/null +++ b/tests/api_app/connectors_manager/unit_tests/base_test_class.py @@ -0,0 +1,138 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +import logging +from contextlib import ExitStack +from types import SimpleNamespace +from unittest import TestCase +from unittest.mock import MagicMock + +from api_app.connectors_manager.exceptions import ConnectorRunException + +logger = logging.getLogger(__name__) + + +class BaseConnectorTest(TestCase): + connector_class: type = None + suppress_connector_logs = True + + def setUp(self): + super().setUp() + logger.info(f"Setting up test environment for {self.__class__.__name__}") + + if self.suppress_connector_logs and self.connector_class: + connector_module = self.connector_class.__module__ + logging.getLogger(connector_module).setLevel(logging.CRITICAL) + logging.getLogger("api_app.connectors_manager").setLevel(logging.WARNING) + + def tearDown(self): + super().tearDown() + logger.info(f"Tearing down test environment for {self.__class__.__name__}") + + if self.suppress_connector_logs and self.connector_class: + connector_module = self.connector_class.__module__ + logging.getLogger(connector_module).setLevel(logging.NOTSET) + logging.getLogger("api_app.connectors_manager").setLevel(logging.NOTSET) + + @classmethod + def get_extra_config(cls) -> dict: + """ + Subclasses can override this to provide additional runtime configuration + specific to their connector (e.g., API keys, URLs, retry counts, etc.). + """ + return {} + + @classmethod + def get_mocked_response(cls): + """ + Subclasses override this to define expected mocked output. + """ + raise NotImplementedError("Subclasses must implement get_mocked_response()") + + @classmethod + def _apply_patches(cls, patches): + if patches is None: + return ExitStack() + + if hasattr(patches, "__enter__") and hasattr(patches, "__exit__"): + return patches + + if isinstance(patches, (list, tuple)): + stack = ExitStack() + for patch_obj in patches: + stack.enter_context(patch_obj) + return stack + + return patches + + # different connectors may require different job setups so + # we create a mock job here that can be customized as needed + # pylint: disable=no-self-use + def _create_mock_job(self, observable_name, observable_type): + mock_tlp_enum = SimpleNamespace() + mock_tlp_enum.CLEAR = SimpleNamespace(value="clear") + mock_tlp_enum.GREEN = SimpleNamespace(value="green") + mock_tlp_enum.AMBER = SimpleNamespace(value="amber") + mock_tlp_enum.RED = SimpleNamespace(value="red") + + mock_analyzable = SimpleNamespace() + mock_analyzable.name = observable_name + mock_analyzable.classification = observable_type + + mock_job = SimpleNamespace() + mock_job.analyzable = mock_analyzable + mock_job.pk = 51 + + mock_job.tlp = "clear" + mock_job.TLP = mock_tlp_enum + mock_job.user = "" + mock_job.is_sample = False + + return mock_job + + def _setup_connector(self): + logger.info(f"Setting up connector {self.connector_class.__name__} for testing") + mock_config = MagicMock() + + # we have already handled connector_class being None in + # the test method, so we can safely assume it's set here + # pylint: disable=not-callable + connector = self.connector_class(mock_config) + connector._job = self._create_mock_job("1.1.1.1", "ip") + + for key, value in self.get_extra_config().items(): + setattr(connector, key, value) + + return connector + + def test_connector_run_execution(self): + if self.connector_class is None: + self.skipTest(f"{self.__class__.__name__} does not specify a connector_class") + + logger.info(f"Starting generic connector test for {self.connector_class.__name__}") + + connector = self._setup_connector() + patches = self.get_mocked_response() + with self._apply_patches(patches): + try: + response = connector.run() + self.assertIsInstance( + response, + (dict, list), + f"Connector response should be a dictionary or a list, got {type(response)}", + ) + self.assertTrue(response, "Connector response should not be empty") + logger.info(f"Connector run successful for {self.connector_class.__name__}") + + except ConnectorRunException as e: + logger.error(f"ConnectorRunException for {self.connector_class.__name__}: {e}") + self.fail( + f"{self.__class__.__name__}: ConnectorRunException for {self.connector_class.__name__}: {e}" + ) + + except Exception as e: + logger.exception(f"Unexpected exception for {self.connector_class.__name__}") + self.fail( + f"{self.__class__.__name__}: Unexpected exception " + f"for {self.connector_class.__name__}: {type(e).__name__}: {e}" + ) diff --git a/tests/api_app/connectors_manager/unit_tests/connectors/__init__.py b/tests/api_app/connectors_manager/unit_tests/connectors/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/api_app/connectors_manager/unit_tests/connectors/test_abuse_submitter.py b/tests/api_app/connectors_manager/unit_tests/connectors/test_abuse_submitter.py new file mode 100644 index 0000000000..f71910f3ab --- /dev/null +++ b/tests/api_app/connectors_manager/unit_tests/connectors/test_abuse_submitter.py @@ -0,0 +1,68 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from api_app.connectors_manager.connectors.abuse_submitter import AbuseSubmitter +from api_app.connectors_manager.exceptions import ConnectorRunException +from tests.api_app.connectors_manager.unit_tests.base_test_class import BaseConnectorTest + + +class AbuseSubmitterTestCase(BaseConnectorTest): + connector_class = AbuseSubmitter + + @classmethod + def get_mocked_response(cls): + return [patch("django.core.mail.EmailMessage.send", return_value="Email sent")] + + def _create_mock_job(self, observable_name, observable_type): + """ + Creates a mock job with the necessary hierarchy for the AbuseSubmitter connector. + The hierarchy is: Job -> Parent Job -> Grandparent Job (Analyzable) + """ + mock_job = super()._create_mock_job(observable_name, observable_type) + + mock_grandparent = SimpleNamespace() + mock_grandparent.analyzable = mock_job.analyzable + + mock_parent = SimpleNamespace() + mock_parent.parent_job = mock_grandparent + + mock_job.parent_job = mock_parent + + return mock_job + + def test_abuse_submitter_missing_hierarchy(self): + """ + Verify that AbuseSubmitter raises ConnectorRunException when job hierarchy is missing. + This tests the fix for the AttributeError crash. + """ + + mock_config = MagicMock() + connector = AbuseSubmitter(mock_config) + + bad_job = self._create_mock_job("malicious.domain", "domain") + bad_job.parent_job = None + connector._job = bad_job + + # Verify subject raises ConnectorRunException + with self.assertRaisesRegex(ConnectorRunException, "Job hierarchy is invalid"): + _ = connector.subject + + # Verify body raises ConnectorRunException + with self.assertRaisesRegex(ConnectorRunException, "Job hierarchy is invalid"): + _ = connector.body + + def test_abuse_submitter_valid_hierarchy(self): + """ + Verify that AbuseSubmitter correctly returns subject and body when hierarchy is valid. + """ + + # Create dummy config and instantiate + mock_config = MagicMock() + connector = self.connector_class(mock_config) + connector._job = self._create_mock_job("malicious.domain", "domain") + + self.assertEqual(connector.subject, "Takedown domain request for malicious.domain") + self.assertIn("Domain malicious.domain", connector.body) diff --git a/tests/api_app/connectors_manager/unit_tests/connectors/test_misp.py b/tests/api_app/connectors_manager/unit_tests/connectors/test_misp.py new file mode 100644 index 0000000000..9eb6cab69b --- /dev/null +++ b/tests/api_app/connectors_manager/unit_tests/connectors/test_misp.py @@ -0,0 +1,212 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from unittest.mock import MagicMock, patch + +from django.test import override_settings + +from api_app.connectors_manager.connectors.misp import MISP +from api_app.connectors_manager.exceptions import ConnectorRunException +from tests.api_app.connectors_manager.unit_tests.base_test_class import BaseConnectorTest + + +class MockPyMISPClient: + def __init__(self, *args, **kwargs): + pass + + def add_event(self, event, *args, **kwargs): + mock_event = MagicMock() + mock_event.id = "51" + mock_event.attributes = event.attributes + return mock_event + + def get_event(self, event_id, *args, **kwargs): + if str(event_id) == "51": + return { + "Event": { + "id": "51", + "info": "Intelowl Job-51: 1.2.3.4", + "Attribute": [ + { + "id": "5", + "type": "ip-src", + "value": "1.2.3.4", + "event_id": "3", + "comment": "Analyzers Executed: FireHol_IPList", + }, + { + "id": "6", + "type": "link", + "value": "https://misp.test/jobs/51", + "comment": "View Analysis on IntelOwl", + "event_id": "3", + }, + ], + } + } + return {"error": "Not Found"} + + +class MISPConnectorTestCase(BaseConnectorTest): + connector_class = MISP + + @classmethod + def get_extra_config(cls) -> dict: + return { + "_url_key_name": "https://misp.test", + "_api_key_name": "test-api-key", + "tlp": "CLEAR", + "ssl_check": False, + "self_signed_certificate": "", + "debug": False, + "_job_id": 51, + } + + def _setup_connector(self): + connector = super()._setup_connector() + + mock_tag = MagicMock() + mock_tag.label = "source:intelowl" + connector._job.tags = MagicMock() + connector._job.tags.all.return_value = [mock_tag] + + connector._job.analyzers_to_execute = MagicMock() + connector._job.analyzers_to_execute.all.return_value.values_list.return_value = ["FireHol_IPList"] + + return connector + + @classmethod + def get_mocked_response(cls): + return [ + patch( + "api_app.connectors_manager.connectors.misp.pymisp.PyMISP", + side_effect=MockPyMISPClient, + ) + ] + + def test_bulk_add_event_called_once(self): + # verify that add_event is called once and that the event contains all attributes + connector = self._setup_connector() + + with patch("api_app.connectors_manager.connectors.misp.pymisp.PyMISP") as mock_misp_cls: + mock_instance = MagicMock() + mock_misp_cls.return_value = mock_instance + + mock_event = MagicMock() + mock_event.id = "51" + mock_instance.add_event.return_value = mock_event + + connector.run() + + mock_instance.add_event.assert_called_once() + mock_instance.add_attribute.assert_not_called() + + def test_all_attributes_present_on_event(self): + # verify that the event sent to MISP contains the base attribute and the link attribute + connector = self._setup_connector() + + with patch("api_app.connectors_manager.connectors.misp.pymisp.PyMISP") as mock_misp_cls: + mock_instance = MagicMock() + mock_misp_cls.return_value = mock_instance + + mock_event = MagicMock() + mock_event.id = "51" + mock_instance.add_event.return_value = MagicMock(id="51") + + connector.run() + + event_arg = mock_instance.add_event.call_args[0][0] + attr_types = [a.type for a in event_arg.attributes] + + self.assertIn("ip-src", attr_types) + self.assertIn("link", attr_types) + + def test_add_event_failure_raises_exception(self): + connector = self._setup_connector() + + with patch("api_app.connectors_manager.connectors.misp.pymisp.PyMISP") as mock_misp_cls: + mock_instance = MagicMock() + mock_misp_cls.return_value = mock_instance + + mock_instance.add_event.side_effect = Exception("MISP unreachable") + + with self.assertRaisesRegex(ConnectorRunException, "MISP unreachable"): + connector.run() + + def test_misp_initialisation_http_failure_raises_exception(self): + connector = self._setup_connector() + + with patch("api_app.connectors_manager.connectors.misp.pymisp.PyMISP") as mock_misp_cls: + mock_misp_cls.side_effect = Exception( + "The plain HTTP request was sent to HTTPS port" + ) + + with self.assertRaises(ConnectorRunException) as context: + connector.run() + + self.assertIn("plain HTTP request to an HTTPS port", str(context.exception)) + + @override_settings(STAGE_CI=False, MOCK_CONNECTIONS=False) + def test_misp_health_check_success(self): + connector = self._setup_connector() + + mock_url_param = MagicMock() + mock_url_param.name = "url_key_name" + mock_url_param.value = "http://misp.test/" + + mock_api_param = MagicMock() + mock_api_param.name = "api_key_name" + mock_api_param.value = "dummy_api_key" + + mock_ssl_param = MagicMock() + mock_ssl_param.name = "ssl_check" + mock_ssl_param.value = "false" + + mock_cert_param = MagicMock() + mock_cert_param.name = "self_signed_certificate" + mock_cert_param.value = "" + + connector._config = MagicMock() + connector._config.parameters.annotate_configured.return_value.annotate_value_for_user.return_value = [ + mock_url_param, + mock_api_param, + mock_ssl_param, + mock_cert_param, + ] + + with patch("api_app.connectors_manager.connectors.misp.pymisp.PyMISP") as mock_client_cls: + mock_instance = mock_client_cls.return_value + mock_instance.health_check.return_value = True + + self.assertTrue(connector.health_check()) + + @override_settings(STAGE_CI=False, MOCK_CONNECTIONS=False) + def test_misp_health_check_failures(self): + connector = self._setup_connector() + + mock_url_param = MagicMock() + mock_url_param.name = "url_key_name" + mock_url_param.value = "http://misp.test/" + + mock_api_param = MagicMock() + mock_api_param.name = "api_key_name" + mock_api_param.value = "dummy_api_key" + + connector._config = MagicMock() + connector._config.parameters.annotate_configured.return_value.annotate_value_for_user.return_value = [ + mock_url_param, + mock_api_param, + ] + + with ( + self.subTest("MISP Connection Exception"), + patch( + "api_app.connectors_manager.connectors.misp.pymisp.PyMISP", + side_effect=Exception("Connection refused"), + ), + ): + self.assertFalse(connector.health_check()) + + with self.subTest("Missing Configuration"): + connector._config.parameters.annotate_configured.return_value.annotate_value_for_user.return_value = [] + self.assertFalse(connector.health_check()) diff --git a/tests/api_app/connectors_manager/unit_tests/connectors/test_opencti.py b/tests/api_app/connectors_manager/unit_tests/connectors/test_opencti.py new file mode 100644 index 0000000000..c409c9e269 --- /dev/null +++ b/tests/api_app/connectors_manager/unit_tests/connectors/test_opencti.py @@ -0,0 +1,351 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from contextlib import ExitStack +from unittest.mock import MagicMock, patch + +from django.test import override_settings + +from api_app.connectors_manager.connectors.opencti import OpenCTI +from tests.api_app.connectors_manager.unit_tests.base_test_class import BaseConnectorTest + +PARTIAL_STATE_SCENARIOS = ( + { + "name": "failure_after_observable", + "fail_mock": "label", + "expected_exception": "label failure", + "expected_ids": ["observable=obs-1", "labels=[]"], + }, + { + "name": "failure_after_report", + "fail_mock": "external_ref", + "expected_exception": "external ref failure", + "expected_ids": ["observable=obs-1", "report=report-1", "label-1"], + }, + { + "name": "failure_after_external_ref", + "fail_mock": "link", + "expected_exception": "link failure", + "expected_ids": ["external_reference=ext-ref-1", "report=report-1", "observable=obs-1"], + }, +) + + +class OpenCTIConnectorTestCase(BaseConnectorTest): + connector_class = OpenCTI + + @classmethod + def get_extra_config(cls) -> dict: + """Provide static plugin configurations avoiding Django database models.""" + return { + "_url_key_name": "https://opencti.test", + "_api_key_name": "test-token", + "ssl_verify": False, + "proxies": "", + "tlp": {"type": "clear", "color": "white", "x_opencti_order": 1}, + "job_id": 51, + } + + @classmethod + def get_mocked_response(cls): + """Define the default happy-path configuration for the generic base execution test.""" + mock_id = MagicMock() + mock_id.return_value.create.return_value = {"id": "org-1"} + + mock_mark = MagicMock() + mock_mark.return_value.create.return_value = {"id": "mark-1"} + + mock_obs = MagicMock() + mock_obs.return_value.create.return_value = {"id": "obs-1"} + + mock_lbl = MagicMock() + mock_lbl.return_value.create.return_value = {"id": "label-1"} + + mock_rep = MagicMock() + mock_rep.return_value.create.return_value = {"id": "report-1"} + + mock_ref = MagicMock() + mock_ref.return_value.create.return_value = {"id": "ext-ref-1"} + + return [ + patch("api_app.connectors_manager.connectors.opencti.pycti.OpenCTIApiClient"), + patch("api_app.connectors_manager.connectors.opencti.pycti.Identity", new=mock_id), + patch("api_app.connectors_manager.connectors.opencti.pycti.MarkingDefinition", new=mock_mark), + patch("api_app.connectors_manager.connectors.opencti.pycti.StixCyberObservable", new=mock_obs), + patch("api_app.connectors_manager.connectors.opencti.pycti.Label", new=mock_lbl), + patch("api_app.connectors_manager.connectors.opencti.pycti.Report", new=mock_rep), + patch("api_app.connectors_manager.connectors.opencti.pycti.ExternalReference", new=mock_ref), + patch("api_app.connectors_manager.connectors.opencti.pycti.StixDomainObject"), + ] + + def _setup_connector(self): + """Inject customized mock managers to simulate the internal Django API objects.""" + connector = super()._setup_connector() + + # Isolate report logging arrays directly onto the runtime instance memory + connector.report = MagicMock() + connector.report.errors = [] + + # Mock the related models sets loop used during label generation + mock_tag = MagicMock() + mock_tag.label = "testtag" + mock_tag.color = "#ff0000" + connector._job.tags = MagicMock() + connector._job.tags.all.return_value = [mock_tag] + + # Mock query string values inside report descriptors + mock_analyzers = MagicMock() + mock_analyzers.all.return_value.values_list.return_value = ["VirusTotal"] + connector._job.analyzers_to_execute = mock_analyzers + + # Avoid real datetime formatting mutations + mock_date = MagicMock() + mock_date.strftime.return_value = "2026-06-16T12:00:00Z" + connector._job.received_request_time = mock_date + + return connector + + def test_partial_state_failure_scenarios(self): + """Iterate localized failure targets via subtests while verifying partial state logs.""" + for scenario in PARTIAL_STATE_SCENARIOS: + with self.subTest(scenario=scenario["name"]): + connector = self._setup_connector() + + mock_id = MagicMock() + mock_id.return_value.create.return_value = {"id": "org-1"} + mock_mark = MagicMock() + mock_mark.return_value.create.return_value = {"id": "mark-1"} + mock_obs = MagicMock() + mock_obs.return_value.create.return_value = {"id": "obs-1"} + mock_lbl = MagicMock() + mock_lbl.return_value.create.return_value = {"id": "label-1"} + mock_rep = MagicMock() + mock_rep.return_value.create.return_value = {"id": "report-1"} + mock_ref = MagicMock() + mock_ref.return_value.create.return_value = {"id": "ext-ref-1"} + mock_dom = MagicMock() + + # Sabotage the execution flow depending on specific target + if scenario["fail_mock"] == "label": + mock_lbl.return_value.create.side_effect = Exception("label failure") + elif scenario["fail_mock"] == "external_ref": + mock_ref.return_value.create.side_effect = Exception("external ref failure") + elif scenario["fail_mock"] == "link": + mock_dom.return_value.add_external_reference.side_effect = Exception("link failure") + + patches = [ + patch("api_app.connectors_manager.connectors.opencti.pycti.OpenCTIApiClient"), + patch("api_app.connectors_manager.connectors.opencti.pycti.Identity", new=mock_id), + patch( + "api_app.connectors_manager.connectors.opencti.pycti.MarkingDefinition", new=mock_mark + ), + patch( + "api_app.connectors_manager.connectors.opencti.pycti.StixCyberObservable", + new=mock_obs, + ), + patch("api_app.connectors_manager.connectors.opencti.pycti.Label", new=mock_lbl), + patch("api_app.connectors_manager.connectors.opencti.pycti.Report", new=mock_rep), + patch( + "api_app.connectors_manager.connectors.opencti.pycti.ExternalReference", new=mock_ref + ), + patch( + "api_app.connectors_manager.connectors.opencti.pycti.StixDomainObject", new=mock_dom + ), + ] + + with ExitStack() as stack: + for p in patches: + stack.enter_context(p) + + with self.assertRaises(Exception): + connector.run() + + partial_msgs = [e for e in connector.report.errors if "Created IDs:" in str(e)] + self.assertEqual(len(partial_msgs), 1) + + err_text = " ".join(map(str, connector.report.errors)) + for substr in scenario["expected_ids"]: + self.assertIn(substr, err_text) + + def test_organization_and_marking_called_only_once(self): + """Properties caching validation checking that static initialization calls occur once.""" + connector = self._setup_connector() + + # Mock the specific objects we want to assert on + mock_id = MagicMock() + mock_id.return_value.create.return_value = {"id": "org-1"} + mock_mark = MagicMock() + mock_mark.return_value.create.return_value = {"id": "mark-1"} + + # Provide safe return dictionaries for the downstream objects + # so they do not crash the pipeline before the assertions are reached. + mock_obs = MagicMock() + mock_obs.return_value.create.return_value = {"id": "obs-1"} + mock_lbl = MagicMock() + mock_lbl.return_value.create.return_value = {"id": "label-1"} + mock_rep = MagicMock() + mock_rep.return_value.create.return_value = {"id": "report-1"} + mock_ref = MagicMock() + mock_ref.return_value.create.return_value = {"id": "ext-ref-1"} + + patches = [ + patch("api_app.connectors_manager.connectors.opencti.pycti.OpenCTIApiClient"), + patch("api_app.connectors_manager.connectors.opencti.pycti.Identity", new=mock_id), + patch("api_app.connectors_manager.connectors.opencti.pycti.MarkingDefinition", new=mock_mark), + patch("api_app.connectors_manager.connectors.opencti.pycti.StixCyberObservable", new=mock_obs), + patch("api_app.connectors_manager.connectors.opencti.pycti.Label", new=mock_lbl), + patch("api_app.connectors_manager.connectors.opencti.pycti.Report", new=mock_rep), + patch("api_app.connectors_manager.connectors.opencti.pycti.ExternalReference", new=mock_ref), + patch("api_app.connectors_manager.connectors.opencti.pycti.StixDomainObject"), + ] + + with ExitStack() as stack: + for p in patches: + stack.enter_context(p) + + connector.run() + + mock_id.return_value.create.assert_called_once() + mock_mark.return_value.create.assert_called_once() + + def test_observable_create_returns_non_dict_handled_safely(self): + """Silently handled non-dictionary returns must gracefully trigger an observable failure check.""" + connector = self._setup_connector() + mock_obs = MagicMock() + mock_obs.return_value.create.return_value = None + + patches = [ + patch("api_app.connectors_manager.connectors.opencti.pycti.OpenCTIApiClient"), + patch("api_app.connectors_manager.connectors.opencti.pycti.Identity"), + patch("api_app.connectors_manager.connectors.opencti.pycti.MarkingDefinition"), + patch("api_app.connectors_manager.connectors.opencti.pycti.StixCyberObservable", new=mock_obs), + patch("api_app.connectors_manager.connectors.opencti.pycti.Label"), + patch("api_app.connectors_manager.connectors.opencti.pycti.Report"), + patch("api_app.connectors_manager.connectors.opencti.pycti.ExternalReference"), + patch("api_app.connectors_manager.connectors.opencti.pycti.StixDomainObject"), + ] + + with ExitStack() as stack: + for p in patches: + stack.enter_context(p) + with self.assertRaises(ValueError): + connector.run() + + err_text = " ".join(map(str, connector.report.errors)) + self.assertIn("observable=None", err_text) + + def test_label_create_returns_non_dict_raises_value_error(self): + """Malformed label creation returns must cancel pipeline propagation immediately.""" + connector = self._setup_connector() + mock_obs = MagicMock() + mock_obs.return_value.create.return_value = {"id": "obs-1"} + mock_lbl = MagicMock() + mock_lbl.return_value.create.return_value = None + + patches = [ + patch("api_app.connectors_manager.connectors.opencti.pycti.OpenCTIApiClient"), + patch("api_app.connectors_manager.connectors.opencti.pycti.Identity"), + patch("api_app.connectors_manager.connectors.opencti.pycti.MarkingDefinition"), + patch("api_app.connectors_manager.connectors.opencti.pycti.StixCyberObservable", new=mock_obs), + patch("api_app.connectors_manager.connectors.opencti.pycti.Label", new=mock_lbl), + patch("api_app.connectors_manager.connectors.opencti.pycti.Report"), + patch("api_app.connectors_manager.connectors.opencti.pycti.ExternalReference"), + patch("api_app.connectors_manager.connectors.opencti.pycti.StixDomainObject"), + ] + + with ExitStack() as stack: + for p in patches: + stack.enter_context(p) + with self.assertRaises(ValueError) as ctx: + connector.run() + + self.assertIn("Invalid response from OpenCTI Label.create", str(ctx.exception)) + err_text = " ".join(map(str, connector.report.errors)) + self.assertIn("observable=obs-1", err_text) + self.assertIn("labels=[]", err_text) + + def test_success_path_integrity(self): + """Validates the exact shape of the returned OpenCTI dictionary contract.""" + connector = self._setup_connector() + + with self._apply_patches(self.get_mocked_response()): + result = connector.run() + + # Enforce the strict API contract + self.assertIsInstance(result, dict) + self.assertIn("observable", result) + self.assertIn("id", result["observable"]) + self.assertIn("report", result) + self.assertIn("id", result["report"]) + + @override_settings(STAGE_CI=False, MOCK_CONNECTIONS=False) + def test_opencti_health_check_success(self): + connector = self._setup_connector() + + mock_url_param = MagicMock() + mock_url_param.name = "url_key_name" + mock_url_param.value = "http://opencti.test/" + + mock_api_param = MagicMock() + mock_api_param.name = "api_key_name" + mock_api_param.value = "dummy_api_key" + + mock_ssl_param = MagicMock() + mock_ssl_param.name = "ssl_verify" + mock_ssl_param.value = "false" + + mock_proxies_param = MagicMock() + mock_proxies_param.name = "proxies" + mock_proxies_param.value = None + + connector._config = MagicMock() + connector._config.parameters.annotate_configured.return_value.annotate_value_for_user.return_value = [ + mock_url_param, + mock_api_param, + mock_ssl_param, + mock_proxies_param, + ] + + with patch("api_app.connectors_manager.connectors.opencti.pycti.OpenCTIApiClient") as mock_client_cls: + mock_instance = mock_client_cls.return_value + mock_instance.health_check.return_value = True + + self.assertTrue(connector.health_check()) + + @override_settings(STAGE_CI=False, MOCK_CONNECTIONS=False) + def test_opencti_health_check_failures(self): + connector = self._setup_connector() + + mock_url_param = MagicMock() + mock_url_param.name = "url_key_name" + mock_url_param.value = "http://opencti.test/" + + mock_api_param = MagicMock() + mock_api_param.name = "api_key_name" + mock_api_param.value = "dummy_api_key" + + connector._config = MagicMock() + connector._config.parameters.annotate_configured.return_value.annotate_value_for_user.return_value = [ + mock_url_param, + mock_api_param, + ] + + with ( + self.subTest("OpenCTI Connection Exception"), + patch("api_app.connectors_manager.connectors.opencti.pycti.OpenCTIApiClient") as mock_client_cls, + ): + mock_instance = mock_client_cls.return_value + mock_instance.health_check.side_effect = Exception("Connection refused") + self.assertFalse(connector.health_check()) + + with ( + self.subTest("OpenCTI Reports Unhealthy"), + patch("api_app.connectors_manager.connectors.opencti.pycti.OpenCTIApiClient") as mock_client_cls, + ): + mock_instance = mock_client_cls.return_value + mock_instance.health_check.return_value = False + self.assertFalse(connector.health_check()) + + with self.subTest("Missing Configuration"): + connector._config.parameters.annotate_configured.return_value.annotate_value_for_user.return_value = [] + self.assertFalse(connector.health_check()) diff --git a/tests/api_app/connectors_manager/unit_tests/connectors/test_slack.py b/tests/api_app/connectors_manager/unit_tests/connectors/test_slack.py new file mode 100644 index 0000000000..1f2e36e598 --- /dev/null +++ b/tests/api_app/connectors_manager/unit_tests/connectors/test_slack.py @@ -0,0 +1,72 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from unittest.mock import MagicMock, patch + +from django.test import override_settings + +from api_app.connectors_manager.connectors.slack import Slack +from tests.api_app.connectors_manager.unit_tests.base_test_class import BaseConnectorTest + + +class SlackTestCase(BaseConnectorTest): + connector_class = Slack + + @classmethod + def get_extra_config(cls) -> dict: + return { + "_channel": "ABCD", + "slack_username": "intelowl_bot", + "_token": "mock-token-123", + } + + @staticmethod + def get_mocked_response(): + return [patch("api_app.connectors_manager.connectors.slack.slack_sdk.WebClient")] + + def test_connector_run_execution(self): + self.skipTest("Will implement later") + + @override_settings(STAGE_CI=False, MOCK_CONNECTIONS=False) + def test_slack_health_check_success(self): + connector = self._setup_connector() + + mock_token_param = MagicMock() + mock_token_param.name = "token" + mock_token_param.value = "mock-token-123" + + connector._config = MagicMock() + connector._config.parameters.annotate_configured.return_value.annotate_value_for_user.return_value = [ + mock_token_param + ] + + with patch("api_app.connectors_manager.connectors.slack.slack_sdk.WebClient") as mock_webclient_cls: + mock_instance = mock_webclient_cls.return_value + mock_instance.auth_test.return_value = {"ok": True} + + self.assertTrue(connector.health_check()) + + @override_settings(STAGE_CI=False, MOCK_CONNECTIONS=False) + def test_slack_health_check_failures(self): + connector = self._setup_connector() + + mock_token_param = MagicMock() + mock_token_param.name = "token" + mock_token_param.value = "mock-token-123" + + connector._config = MagicMock() + connector._config.parameters.annotate_configured.return_value.annotate_value_for_user.return_value = [ + mock_token_param + ] + + with ( + self.subTest("Slack Connection/Token Exception"), + patch("api_app.connectors_manager.connectors.slack.slack_sdk.WebClient") as mock_webclient_cls, + ): + mock_instance = mock_webclient_cls.return_value + mock_instance.auth_test.side_effect = Exception("invalid_auth") + self.assertFalse(connector.health_check()) + + with self.subTest("Missing Token Configuration"): + connector._config.parameters.annotate_configured.return_value.annotate_value_for_user.return_value = [] + self.assertFalse(connector.health_check()) diff --git a/tests/api_app/connectors_manager/unit_tests/connectors/test_yeti.py b/tests/api_app/connectors_manager/unit_tests/connectors/test_yeti.py new file mode 100644 index 0000000000..1cc8e39984 --- /dev/null +++ b/tests/api_app/connectors_manager/unit_tests/connectors/test_yeti.py @@ -0,0 +1,203 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from unittest.mock import MagicMock, patch + +import requests +from django.test import override_settings +from requests.exceptions import HTTPError + +from api_app.connectors_manager.connectors.yeti import YETI +from api_app.connectors_manager.exceptions import ConnectorRunException +from tests.api_app.connectors_manager.unit_tests.base_test_class import BaseConnectorTest + + +class MockResponse: + # Mock replacement for requests.Response objects to support raise_for_status + + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + def raise_for_status(self): + if self.status_code >= 400: + raise HTTPError(f"HTTP Error: {self.status_code}") + + +def mock_yeti_api_flow(url, *args, **kwargs): + # Dynamic mock router to handle different YETI API endpoints + + if "api-token" in url: + return MockResponse({"access_token": "mocked_jwt_token_123", "token_type": "bearer"}, 200) + + if "observables/extended" in url: + return MockResponse( + { + "context": [ + { + "source": "IntelOwl", + "report": "https://intelowl.example/jobs/51", + "status": "analyzed", + "date": "2026-06-02T12:34:56Z", + "description": "IntelOwl analysis report for Job: 51 | 1.2.3.4 | ipv4", + "analyzers executed": "IPApi", + } + ], + "tags": [ + { + "name": "intelowl", + "last_seen": "2026-06-04T08:45:45.136845Z", + "expires": "2026-09-02T08:45:45.136845Z", + "fresh": True, + "id": None, + } + ], + "value": "1.2.3.4", + "last_analysis": {}, + "created": "2026-06-04T08:45:45.086020Z", + "modified": "2026-06-04T08:45:45.186824Z", + "type": "ipv4", + "id": "12345", + "acls": {}, + "root_type": "observable", + "is_valid": True, + }, + 200, + ) + + return MockResponse({"detail": "Not Found"}, 404) + + +class YETITestCase(BaseConnectorTest): + connector_class = YETI + + @classmethod + def get_extra_config(cls) -> dict: + return { + "_url_key_name": "http://yeti.local/", + "_api_key_name": "dummy_api_key", + "verify_ssl": False, + "job_id": 51, + } + + @staticmethod + def get_mocked_response(): + return [ + patch("api_app.connectors_manager.connectors.yeti.requests.post", side_effect=mock_yeti_api_flow) + ] + + def _create_mock_job(self, observable_name, observable_type): + mock_job = super()._create_mock_job(observable_name, observable_type) + + mock_tags = MagicMock() + mock_tags.all().values_list.return_value = ["intelowl"] + mock_job.tags = mock_tags + + mock_analyzers = MagicMock() + mock_analyzers.all().values_list.return_value = ["IPApi"] + mock_job.analyzers_to_execute = mock_analyzers + + mock_job.received_request_time = "2026-06-04 08:36:35.035498+00:00" + + return mock_job + + def test_yeti_run_with_file_sample(self): + connector = self._setup_connector() + + mock_job = self._create_mock_job("xyz_filename", "file") + mock_job.is_sample = True + mock_job.analyzable.md5 = "abcdef1234567890abcdef1234567890" + connector._job = mock_job + + # enforce the mock during this specific execution + with patch( + "api_app.connectors_manager.connectors.yeti.requests.post", side_effect=mock_yeti_api_flow + ): + result = connector.run() + + self.assertEqual(result.get("id"), "12345") + self.assertEqual(result.get("type"), "ipv4") + self.assertTrue(result.get("is_valid")) + self.assertEqual(len(result.get("context", [])), 1) + + def test_yeti_auth_failure_raises_exception(self): + connector = self._setup_connector() + + with patch("api_app.connectors_manager.connectors.yeti.requests.post") as mock_post: + mock_post.return_value = MockResponse({"error": "Unauthorized"}, 401) + + with self.assertRaisesRegex(ConnectorRunException, "YETI Auth Request failed"): + connector.run() + + def test_yeti_missing_access_token_raises_exception(self): + connector = self._setup_connector() + + with patch("api_app.connectors_manager.connectors.yeti.requests.post") as mock_post: + mock_post.return_value = MockResponse({}, 200) + + with self.assertRaisesRegex(ConnectorRunException, "Failed to obtain access token from YETI"): + connector.run() + + @override_settings(STAGE_CI=False, MOCK_CONNECTIONS=False) + def test_yeti_health_check_success(self): + connector = self._setup_connector() + + mock_url_param = MagicMock() + mock_url_param.name = "url_key_name" + mock_url_param.value = "http://yeti.local/" + + mock_api_param = MagicMock() + mock_api_param.name = "api_key_name" + mock_api_param.value = "dummy_api_key" + + connector._config = MagicMock() + connector._config.parameters.annotate_configured.return_value.annotate_value_for_user.return_value = [ + mock_url_param, + mock_api_param, + ] + + with patch( + "api_app.connectors_manager.connectors.yeti.requests.post", side_effect=mock_yeti_api_flow + ): + self.assertTrue(connector.health_check()) + + @override_settings(STAGE_CI=False, MOCK_CONNECTIONS=False) + def test_yeti_health_check_failures(self): + connector = self._setup_connector() + + mock_url_param = MagicMock() + mock_url_param.name = "url_key_name" + mock_url_param.value = "http://yeti.local/" + + mock_api_param = MagicMock() + mock_api_param.name = "api_key_name" + mock_api_param.value = "dummy_api_key" + + connector._config = MagicMock() + connector._config.parameters.annotate_configured.return_value.annotate_value_for_user.return_value = [ + mock_url_param, + mock_api_param, + ] + + with ( + self.subTest("Network Exception"), + patch( + "api_app.connectors_manager.connectors.yeti.requests.post", + side_effect=requests.exceptions.Timeout, + ), + ): + self.assertFalse(connector.health_check()) + + with ( + self.subTest("Authentication Failure"), + patch("api_app.connectors_manager.connectors.yeti.requests.post") as mock_post, + ): + mock_post.return_value = MockResponse({"error": "Unauthorized"}, 401) + self.assertFalse(connector.health_check()) + + with self.subTest("Missing Configuration"): + connector._config.parameters.annotate_configured.return_value.annotate_value_for_user.return_value = [] + self.assertFalse(connector.health_check())