Merge branch 'main' into feature/mula/boefje-interval

boefjes/.ci/docker-compose.yml

@@ -85,7 +85,7 @@ services:
       - .ci/.env.test
 
   ci_xtdb:
-    image: "ghcr.io/dekkers/xtdb-http-multinode:v1.0.8"
+    image: "ghcr.io/dekkers/xtdb-http-multinode:v1.1.0"
 
   ci_octopoes_api_worker:
     build:

boefjes/boefjes/app.py

@@ -174,7 +174,7 @@ def _cleanup_pending_worker_task(self, worker: mp.Process) -> None:
         try:
             task = self.scheduler_client.get_task(handling_task_id)
 
-            if task.status is TaskStatus.DISPATCHED:
+            if task.status is TaskStatus.DISPATCHED or task.status is TaskStatus.RUNNING:
                 try:
                     self.scheduler_client.patch_task(task.id, TaskStatus.FAILED)
                     logger.warning("Set status to failed in the scheduler for task[id=%s]", handling_task_id)
@@ -244,6 +244,7 @@ def _start_working(
         handling_tasks[os.getpid()] = str(p_item.id)
 
         try:
+            scheduler_client.patch_task(p_item.id, TaskStatus.RUNNING)
             handler.handle(p_item.data)
             status = TaskStatus.COMPLETED
         except Exception:  # noqa
@@ -253,8 +254,10 @@ def _start_working(
             raise
         finally:
             try:
-                scheduler_client.patch_task(p_item.id, status)  # Note: implicitly, we have p_item.id == task_id
-                logger.info("Set status to %s in the scheduler for task[id=%s]", status, p_item.data.id)
+                if scheduler_client.get_task(p_item.id).status == TaskStatus.RUNNING:
+                    # The docker runner could have handled this already
+                    scheduler_client.patch_task(p_item.id, status)  # Note that implicitly, we have p_item.id == task_id
+                    logger.info("Set status to %s in the scheduler for task[id=%s]", status, p_item.data.id)
             except HTTPError:
                 logger.exception("Could not patch scheduler task to %s", status.value)
 

boefjes/boefjes/dependencies/plugins.py

@@ -224,5 +224,6 @@ def get_plugins_filter_parameters(
     ids: list[str] | None = Query(None),
     plugin_type: Literal["boefje", "normalizer", "bit"] | None = None,
     state: bool | None = None,
+    oci_image: str | None = None,
 ) -> FilterParameters:
-    return FilterParameters(q=q, ids=ids, type=plugin_type, state=state)
+    return FilterParameters(q=q, ids=ids, type=plugin_type, state=state, oci_image=oci_image)

boefjes/boefjes/docker_boefjes_runner.py

@@ -36,7 +36,6 @@ def run(self) -> None:
         stderr_mime_types = boefjes.plugins.models._default_mime_types(self.boefje_meta.boefje)
 
         task_id = self.boefje_meta.id
-        self.scheduler_client.patch_task(task_id, TaskStatus.RUNNING)
         self.boefje_meta.started_at = datetime.now(timezone.utc)
 
         try:
@@ -63,7 +62,11 @@ def run(self) -> None:
                 # have to raise exception to prevent _start_working function from setting status to completed
                 raise RuntimeError("Boefje did not call output API endpoint")
         except ContainerError as e:
-            logger.exception("Container error")
+            logger.error(
+                "Container for task %s failed and returned exit status %d, stderr saved to bytes",
+                task_id,
+                e.exit_status,
+            )
 
             # save container log (stderr) to bytes
             self.bytes_api_client.login()
@@ -75,9 +78,9 @@ def run(self) -> None:
                 logger.error("Failed to save boefje meta to bytes, continuing anyway")
             self.bytes_api_client.save_raw(task_id, e.stderr, stderr_mime_types)
             self.scheduler_client.patch_task(task_id, TaskStatus.FAILED)
-            # have to raise exception to prevent _start_working function from setting status to completed
-            raise e
-        except (APIError, ImageNotFound) as e:
-            logger.exception("API error or image not found")
+        except ImageNotFound:
+            logger.error("Docker image %s not found", self.boefje_resource.oci_image)
+            self.scheduler_client.patch_task(task_id, TaskStatus.FAILED)
+        except APIError as e:
+            logger.error("Docker API error: %s", e)
             self.scheduler_client.patch_task(task_id, TaskStatus.FAILED)
-            raise e

boefjes/boefjes/katalogus/plugins.py

@@ -65,6 +65,10 @@ def list_plugins(
     if filter_params.state is not None:
         plugins = filter(lambda x: x.enabled is filter_params.state, plugins)
 
+    # filter plugins by oci_image
+    if filter_params.oci_image is not None:
+        plugins = filter(lambda x: x.type == "boefje" and x.oci_image == filter_params.oci_image, plugins)
+
     # filter plugins by scan level for boefje plugins
     plugins = list(filter(lambda x: x.type != "boefje" or x.scan_level >= filter_params.scan_level, plugins))
 

boefjes/boefjes/logging.json

@@ -3,8 +3,7 @@
   "disable_existing_loggers": 0,
   "formatters": {
     "default": {
-      "format": "%(asctime)s [%(process)d] [%(levelname)s] [%(module)s] %(message)s",
-      "datefmt": "[%Y-%m-%d %H:%M:%S %z]"
+      "format": "%(message)s"
     }
   },
   "handlers": {

boefjes/boefjes/migrations/versions/5be152459a7b_introduce_schema_field_to_boefje_model.py

@@ -38,7 +38,6 @@ def upgrade() -> None:
 
         for plugin in local_repo.get_all():
             schema = local_repo.schema(plugin.id)
-
             if schema:
                 try:
                     # This way we avoid the safeguard that updating static boefjes is not allowed

boefjes/boefjes/models.py

@@ -79,3 +79,4 @@ class FilterParameters(BaseModel):
     ids: list[str] | None = None
     state: bool | None = None
     scan_level: int = 0
+    oci_image: str | None = None

boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json

@@ -13,14 +13,14 @@
     "impact": "The usage possibility of JavaScript is not limited by the website. If the website contains a cross-site scripting vulnerability, then JavaScript code can be injected into the web page. This code is then executed by the browser of the victim. If a well-established Content Security Policy is active, the attacker can inject JavaScript code into the browser of the victim, but then the code will not get executed by the browser. A good configured Content Security Policy is a strong protection against cross-site scripting vulnerabilities.",
     "recommendation": "1. Set the Content-Security-Policy HTTP header in all HTTP answers. 2. Make sure that when the Content Security Policy is violated by a browser, that this violation is logged and monitored. Point the content security violation variable report-uri to a server-side log script. 3. Implement a process that periodically analyses these logs for programming errors and hack attacks."
   },
-  "KAT-NO-X-PERMITTED-CROSS-DOMAIN-POLICIES": {
+  "KAT-X-PERMITTED-CROSS-DOMAIN-POLICIES": {
     "description": "The HTTP header X-Permitted-Cross-Domain- Policies is missing in HTTP responses. This header is not officially supported by Mozilla MDN.",
     "source": "https://owasp.org/www-project-secure-headers/#div-headers",
     "risk": "recommendation",
     "impact": "When the value of this header is not set to master- only, Adobe Flash or Adobe Acrobat (and possibly other software) can also look at cross-domain configuration files hosted at the web server.",
     "recommendation": "This header is not supported by default by Mozilla. If this header is required for your environment: Set the HTTP header X-Permitted-Cross- Domain-Policies: none in all HTTP responses. Use value master-only if a Flash or Acrobat cross- domain configuration file is used that is placed in the root of the web server"
   },
-  "KAT-NO-EXPLICIT-XSS-PROTECTION": {
+  "KAT-EXPLICIT-XSS-PROTECTION": {
     "description": "The 'X-XSS-Protection' header is a deprecated header previously used to prevent against Cross-Site-Scripting attacks. Support in modern browsers could introduce XSS attacks again.",
     "source": "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection",
     "risk": "recommendation",
@@ -34,14 +34,14 @@
     "impact": "There is a change that clickjacking is possible. This is an attack technique in which the website is invisibly loaded. On top of the original website, another malicious website is loaded that contains specially placed buttons or links. When the victim clicks on those buttons or links, the mouse click and thus its corresponding action is performed on the original website (which is made invisible). If the victim is logged in, then this click can perform an unauthorized action.",
     "recommendation": "1. Set the HTTP header <c>X-Frame- Options</c> with value deny (safest) or sameorigin in every HTTP answer for older browsers. 2. Set the frame-ancestors variable in the Content-Security-Policy header for modern browsers. 3. Add JavaScript code to all pages to ensure that these web pages may not be loaded within an <iframe>. In this manner also very old browsers are protected that do not support the HTTP header X-Frame-Options."
   },
-  "KAT-NO-X-DNS-PREFETCH-CONTROL": {
+  "KAT-X-DNS-PREFETCH-CONTROL": {
     "description": "This is a non-standard header. The HTTP header X-DNS-Prefetch-Control is missing. The X-DNS-Prefetch-Control HTTP response header controls DNS prefetching, a feature by which browsers proactively perform domain name resolution on both links that the user may choose to follow as well as URLs for items referenced by the document.",
     "source": "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-DNS-Prefetch-Control",
     "risk": "recommendation",
     "impact": "This header not production ready and thus not officially supported by Mozilla MDN.",
     "recommendation": "If support is required: Set the HTTP header to: `X-DNS-Prefetch-Control: off` in all HTTP answers."
   },
-  "KAT-NO-EXCPECT-CT": {
+  "KAT-EXCPECT-CT": {
     "description": "The 'Expect-CT' header is deprecated. The Expect-CT header allowed sites to opt in to reporting and/or enforcement of Certificate Transparency requirements. This header is not supported by common browsers, as certificate transparency is now a standard functionality.",
     "source": "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expect-CT",
     "risk": "recommendation",
@@ -419,7 +419,7 @@
     "impact": "An attacker using your hosting provider may setup a virtual host for your domain and thus intercept and trick users.",
     "recommendation": "To prevent subdomain takeover, organizations should regularly monitor their DNS records to identify and remove any unused subdomains. Additionally, they should ensure that all subdomains are properly configured and point to valid services."
   },
-  "EXPOSED-PANELS": {
+  "EXPOSED-ADMIN-PANELS": {
     "description": "Exposed login panels for services can pose security risks as they can be targeted by malicious actors for brute-force attacks, phishing attempts, and other forms of unauthorized access.",
     "source": "https://resources.infosecinstitute.com/topics/application-security/dangers-web-management/",
     "risk": "recommendation",
@@ -463,6 +463,7 @@
   },
   "KAT-INVALID-RPKI": {
     "description": "A route announcement that is matched by the published Route Policy and Authorization (RPKI) is invalid",
+    "source": "https://blog.cloudflare.com/rpki/",
     "risk": "medium",
     "impact": "Without RPKI validation, your servers might be more vulnerable to unintended or malicious routing configuration errors, potentially leading to inaccessibility of your servers or interception of internet traffic directed to them.",
     "recommendation": "Make sure that the Route Origin Authorizations (ROAs) that specify which Autonomous Systems (AS) are authorized to announce your IP addresses are valid and not expired."
@@ -476,6 +477,7 @@
   },
   "KAT-DISALLOWED-DOMAIN-IN-CSP": {
     "description": "This CSP header contains domains that are not allowed, If the website contains a cross-site scripting vulnerability, then JavaScript code can be injected into the web page hosted on these domains which can host files for anyone.",
+    "source": "https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP",
     "risk": "medium",
     "impact": "Disallowed domains are domains that are for example 'world writable', this opens up the possibility for an atacker to host malicious files on a csp whitelisted domain.",
     "recommendation": "Remove the offending hostname from the CSP header."
@@ -487,9 +489,18 @@
     "impact": "Nonstandard headers may not be supported by all browsers and may not provide the security that is expected.",
     "recommendation": "Remove the nonstandard headers from the response."
   },
+  "KAT-DOMAIN-OWNERSHIP-PENDING": {
+    "description": "The domain requires Ownership verification. An email has been sent to the registered owner by the registrar. The domain is currently offline.",
+    "risk": "high",
+    "source": "https://www.icann.org/resources/pages/contact-verification-2013-05-03-en",
+    "impact": "Domain points to placeholder DNS-Servers, and is offline for regular use.",
+    "recommendation": "Verify ownership by following the emailed link."
+  },
   "KAT-SOFTWARE-VERSION-NOT-FOUND": {
     "description": "The version of the software is not found.",
     "risk": "recommendation",
-    "recommendation": "There was no version found for this software but there are known vulnerabilities for this software."
+    "source": "Check the version of the host manually.",
+    "impact": "Unknown. The server may or may not be vulnerable. OpenKAT is not able to determine the version.",
+    "recommendation": "Verify manually if the software is up to date as OpenKAT is not able to determine the software version ."
   }
 }

boefjes/boefjes/plugins/kat_nmap_ports/schema.json

@@ -6,6 +6,7 @@
       "title": "PORTS",
       "maxLength": 2048,
       "type": "string",
+      "pattern": "^((6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d)|(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d)-(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d))$|^((6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d)|(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d)-(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d))(,((6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d)|(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d)-(6553[0-5]|655[0-2]\\d|65[0-4]\\d{2}|6[0-4]\\d{3}|[1-5]\\d{0,4}|\\d)))+$",
       "description": "Specify the ports that need to be scanned (nmap format). Single ports are comma separated, port ranges can be specified using the dash symbol. For example: 22,111,137,80-100 will scan ports 22, 111, 137 and the port range 80 up to 100."
     }
   },

boefjes/boefjes/sql/session.py

@@ -37,7 +37,7 @@ def __exit__(self, exc_type: type[Exception], exc_value: str, exc_traceback: str
         error = None
 
         try:
-            logger.info("Committing session")
+            logger.debug("Committing session")
             self.session.commit()
         except DatabaseError as e:
             error = e

boefjes/images/oci_adapter.py

@@ -9,7 +9,11 @@
 
 def main():
     input_url = sys.argv[-1]
-    boefje_input = httpx.get(input_url).json()
+    try:
+        boefje_input = httpx.get(input_url).json()
+    except httpx.HTTPError as e:
+        # sys.exit will print the message on stderr and return with exit code 1
+        sys.exit(f"Failed to get input from boefje API: {e}")
 
     try:
         os.environ.update(boefje_input["boefje_meta"]["environment"])
@@ -32,7 +36,10 @@ def main():
             ],
         }
 
-    httpx.post(boefje_input["output_url"], json=out)
+    try:
+        httpx.post(boefje_input["output_url"], json=out)
+    except httpx.HTTPError as e:
+        sys.exit(f"Failed to post output to boefje API: {e}")
 
 
 if __name__ == "__main__":

boefjes/pyproject.toml

@@ -34,6 +34,7 @@ beautifulsoup4 = "4.11.1"
 censys = "2.1.8"
 # required by kat_dicom boefje
 pynetdicom = "2.0.2"
+pydicom = "2.4.4"
 # required by kat_dns, kat_dns_zone boefjes
 dnspython = "^2.6.1"
 # required by kat_fierce
@@ -77,6 +78,7 @@ maxminddb = "^2.6.2"
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.2.0"
 pytest-env = "^1.1.3"
+pytest-mock = "^3.14.0"
 
 [build-system]
 requires = ["setuptools>=45", "wheel"]

boefjes/requirements-dev.txt

@@ -1309,6 +1309,9 @@ pynetdicom==2.0.2 ; python_version >= "3.10" and python_version < "4.0" \
 pytest-env==1.1.3 ; python_version >= "3.10" and python_version < "4.0" \
     --hash=sha256:aada77e6d09fcfb04540a6e462c58533c37df35fa853da78707b17ec04d17dfc \
     --hash=sha256:fcd7dc23bb71efd3d35632bde1bbe5ee8c8dc4489d6617fb010674880d96216b
+pytest-mock==3.14.0 ; python_version >= "3.10" and python_version < "4.0" \
+    --hash=sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f \
+    --hash=sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0
 pytest==8.3.2 ; python_version >= "3.10" and python_version < "4.0" \
     --hash=sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5 \
     --hash=sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce