diff --git a/docs/source/developer_documentation/rocky.md b/docs/source/developer_documentation/rocky.md index 0f27bd19ee..798ccd11a9 100644 --- a/docs/source/developer_documentation/rocky.md +++ b/docs/source/developer_documentation/rocky.md @@ -220,16 +220,18 @@ classDiagram direction RL class OrganizationView class OctopoesView - class BoefjeMixin + class SchedulerView + class TaskListView OctopoesView <|-- OrganizationView - BoefjeMixin <|-- OctopoesView - BoefjeDetailView <|-- BoefjeMixin - OOIDetailView <|-- BoefjeMixin + SchedulerView <|-- OctopoesView + TaskListView <|-- SchedulerView + BoefjeDetailView <|-- TaskListView + OOIDetailView <|-- TaskListView OOIDetailView <|-- OOIRelatedObjectAddView OOIDetailView <|-- OOIFindingManager - ChangeClearanceLevel <|-- BoefjeMixin + ChangeClearanceLevel <|-- SchedulerView SingleOOIMixin <|-- OctopoesView SingleOOITreeMixin <|-- SingleOOIMixin @@ -300,13 +302,14 @@ direction RL class PluginSettingsDeleteView class BoefjeDetailView + class TaskListView KATalogusView <|-- OrganizationView KATalogusView <|-- FormView SinglePluginView <|-- OrganizationView SingleSettingView <|-- SinglePluginView BoefjeDetailView <|-- PluginSettingsListView - BoefjeDetailView <|-- BoefjeMixin + BoefjeDetailView <|-- TaskListView PluginEnableDisableView <|-- SinglePluginView PluginSettingsAddView <|-- FormView PluginSettingsAddView <|-- SinglePluginView diff --git a/rocky/account/mixins.py b/rocky/account/mixins.py index 951c670762..3c01410ff4 100644 --- a/rocky/account/mixins.py +++ b/rocky/account/mixins.py @@ -108,6 +108,12 @@ def get_context_data(self, **kwargs): context["perms"] = OrganizationPermWrapper(self.organization_member) return context + def indemnification_error(self): + return messages.error( + self.request, + f"Indemnification not present at organization {self.organization}.", + ) + @property def may_update_clearance_level(self) -> bool: if not self.indemnification_present: diff --git a/rocky/assets/css/components/table-state-icons.scss b/rocky/assets/css/components/table-state-icons.scss index 5bf7550bd0..fbf36439a9 100644 --- a/rocky/assets/css/components/table-state-icons.scss +++ b/rocky/assets/css/components/table-state-icons.scss @@ -47,6 +47,13 @@ table td .icon { } } + &.completed { + &::before { + content: "\ea67"; // $ti-icon-circle-check + color: var(--color-alert-positive); + } + } + &.negative { &::before { content: "\ea6a"; // $ti-icon-circle-x @@ -54,6 +61,13 @@ table td .icon { } } + &.failed { + &::before { + content: "\ea6a"; // $ti-icon-circle-x + color: var(--color-alert-negative); + } + } + &.incomplete { &::before { content: "\ea6a"; // $ti-icon-circle-x @@ -68,6 +82,13 @@ table td .icon { } } + &.dispatched { + &::before { + content: "\ed27"; // $ti-icon-circle-dashed (should become ti-icon-progress, when we have it) + color: var(--color-alert-informative); + } + } + &.queued { &::before { content: "\ea70"; // $ti-icon-clock @@ -75,6 +96,13 @@ table td .icon { } } + &.pending { + &::before { + content: "\ea70"; // $ti-icon-clock + color: var(--color-alert-warning); + } + } + &.cancelled { &::before { content: "\ea05"; // $ti-icon-alert-circle diff --git a/rocky/katalogus/client.py b/rocky/katalogus/client.py index b990cd17e5..b5e06db177 100644 --- a/rocky/katalogus/client.py +++ b/rocky/katalogus/client.py @@ -109,6 +109,7 @@ def get_plugins(self, **params): def get_plugin(self, plugin_id: str) -> Plugin: response = self.session.get(f"{self.organization_uri}/plugins/{plugin_id}") response.raise_for_status() + return parse_plugin(response.json()) def get_plugin_schema(self, plugin_id) -> dict | None: diff --git a/rocky/katalogus/templates/boefje_detail.html b/rocky/katalogus/templates/boefje_detail.html index 8b9ae11e19..a6229151d6 100644 --- a/rocky/katalogus/templates/boefje_detail.html +++ b/rocky/katalogus/templates/boefje_detail.html @@ -37,7 +37,7 @@

{{ plugin.name }}

{% if perms.tools.can_view_katalogus_settings %} - {% include "plugin_settings_list.html" with object_list=object_list plugin=plugin %} + {% include "plugin_settings_list.html" with object_list=plugin_settings plugin=plugin %} {% endif %}
@@ -73,7 +73,7 @@

{% translate "Produces" %}

- {% include "tasks/partials/boefje_task_history.html" %} + {% include "tasks/plugin_detail_task_list.html" %}

diff --git a/rocky/katalogus/templates/normalizer_detail.html b/rocky/katalogus/templates/normalizer_detail.html index 162778077d..dbb3d3cbc3 100644 --- a/rocky/katalogus/templates/normalizer_detail.html +++ b/rocky/katalogus/templates/normalizer_detail.html @@ -31,7 +31,7 @@

{{ plugin.name }}

{% if perms.tools.can_view_katalogus_settings %} - {% include "plugin_settings_list.html" with object_list=object_list plugin=plugin %} + {% include "plugin_settings_list.html" with object_list=plugin_settings plugin=plugin %} {% endif %}
@@ -74,7 +74,7 @@

{% translate "Produces" %}

{% endif %}

- {% include "tasks/partials/normalizer_task_history.html" %} + {% include "tasks/plugin_detail_task_list.html" %}

diff --git a/rocky/katalogus/templates/partials/objects_to_scan.html b/rocky/katalogus/templates/partials/objects_to_scan.html index 7dd98fa166..a296673e19 100644 --- a/rocky/katalogus/templates/partials/objects_to_scan.html +++ b/rocky/katalogus/templates/partials/objects_to_scan.html @@ -36,7 +36,7 @@

{{ form_title }}

{% endif %} {% if select_oois_form.fields.ooi.choices %} - {% include "partials/form/checkbox_group_table_form.html" with checkbox_group_table_form=select_oois_form btn_text="Start scan" plugin_enabled=plugin.enabled key="boefje_id" value=plugin.id action="scan" checkbox_group_table_filter_form=select_ooi_filter_form unique_id="" %} + {% include "partials/form/checkbox_group_table_form.html" with checkbox_group_table_form=select_oois_form btn_text="Start scan" plugin_enabled=plugin.enabled key="boefje_id" value=plugin.id action="scan_oois" checkbox_group_table_filter_form=select_ooi_filter_form unique_id="" %} {% elif has_consumable_oois %} {% blocktranslate trimmed with name=plugin.name %} diff --git a/rocky/katalogus/templates/plugin_settings_list.html b/rocky/katalogus/templates/plugin_settings_list.html index 88e407208b..3ca5ca51df 100644 --- a/rocky/katalogus/templates/plugin_settings_list.html +++ b/rocky/katalogus/templates/plugin_settings_list.html @@ -57,8 +57,6 @@

{{ plugin.type|title }}{% translate " Details" %}

{% csrf_token %} - {% include "partials/pagination.html" %} -
diff --git a/rocky/katalogus/views/change_clearance_level.py b/rocky/katalogus/views/change_clearance_level.py index f3dcbbf2ea..2f1560493b 100644 --- a/rocky/katalogus/views/change_clearance_level.py +++ b/rocky/katalogus/views/change_clearance_level.py @@ -5,10 +5,11 @@ from django.utils.translation import gettext_lazy as _ from django.views.generic import TemplateView -from katalogus.views.mixins import BoefjeMixin, SinglePluginView +from katalogus.views.mixins import SinglePluginView +from rocky.views.scheduler import SchedulerView -class ChangeClearanceLevel(OrganizationPermissionRequiredMixin, BoefjeMixin, SinglePluginView, TemplateView): +class ChangeClearanceLevel(OrganizationPermissionRequiredMixin, SchedulerView, SinglePluginView, TemplateView): template_name = "change_clearance_level.html" permission_required = "tools.can_set_clearance_level" diff --git a/rocky/katalogus/views/mixins.py b/rocky/katalogus/views/mixins.py index fe4146bdf5..9e77083724 100644 --- a/rocky/katalogus/views/mixins.py +++ b/rocky/katalogus/views/mixins.py @@ -8,21 +8,15 @@ from django.utils.translation import gettext_lazy as _ from httpx import HTTPError, HTTPStatusError from rest_framework.status import HTTP_404_NOT_FOUND -from tools.view_helpers import schedule_task -from katalogus.client import Boefje as KATalogusBoefje -from katalogus.client import KATalogusClientV1, get_katalogus -from katalogus.client import Normalizer as KATalogusNormalizer -from octopoes.models import OOI -from rocky.scheduler import Boefje, BoefjeTask, Normalizer, NormalizerTask, PrioritizedItem, RawData -from rocky.views.mixins import OctopoesView +from katalogus.client import KATalogusClientV1, Plugin, get_katalogus logger = getLogger(__name__) class SinglePluginView(OrganizationView): katalogus_client: KATalogusClientV1 - plugin: KATalogusBoefje | KATalogusNormalizer + plugin: Plugin def setup(self, request, *args, plugin_id: str, **kwargs): """ @@ -57,47 +51,3 @@ def is_required_field(self, field: str) -> bool: def is_secret_field(self, field: str) -> bool: """Check whether this field should be secret, defaults to False.""" return bool(self.plugin_schema and field in self.plugin_schema.get("secret", [])) - - -class NormalizerMixin(OctopoesView): - """ - When a user wants to run a normalizer on a given set of raw data, - this mixin provides the method to construct the normalizer task for that data and run it. - """ - - def run_normalizer(self, normalizer: KATalogusNormalizer, raw_data: RawData) -> None: - normalizer_task = NormalizerTask(normalizer=Normalizer(id=normalizer.id, version=None), raw_data=raw_data) - - task = PrioritizedItem(priority=1, data=normalizer_task) - - schedule_task(self.request, self.organization.code, task) - - -class BoefjeMixin(OctopoesView): - """ - When a user wants to scan one or multiple OOI's, - this mixin provides the methods to construct the boefjes for the OOI's and run them. - """ - - def run_boefje(self, katalogus_boefje: KATalogusBoefje, ooi: OOI | None) -> None: - boefje_task = BoefjeTask( - boefje=Boefje.model_validate(katalogus_boefje.model_dump()), - input_ooi=ooi.reference if ooi else None, - organization=self.organization.code, - ) - - task = PrioritizedItem(priority=1, data=boefje_task) - schedule_task(self.request, self.organization.code, task) - - def run_boefje_for_oois( - self, - boefje: KATalogusBoefje, - oois: list[OOI], - ) -> None: - if not oois and not boefje.consumes: - self.run_boefje(boefje, None) - - for ooi in oois: - if ooi.scan_profile and ooi.scan_profile.level < boefje.scan_level: - self.can_raise_clearance_level(ooi, boefje.scan_level) - self.run_boefje(boefje, ooi) diff --git a/rocky/katalogus/views/plugin_detail.py b/rocky/katalogus/views/plugin_detail.py index b1e524f2bd..d0ecbe526b 100644 --- a/rocky/katalogus/views/plugin_detail.py +++ b/rocky/katalogus/views/plugin_detail.py @@ -1,33 +1,22 @@ from datetime import datetime, timezone -from enum import Enum from logging import getLogger from typing import Any from account.mixins import OrganizationView from django.contrib import messages -from django.core.exceptions import BadRequest -from django.core.paginator import Page, Paginator from django.http import FileResponse from django.shortcuts import redirect from django.urls.base import reverse from django.utils.translation import gettext_lazy as _ -from django.views.generic import TemplateView from tools.forms.ooi import SelectOOIFilterForm, SelectOOIForm -from tools.view_helpers import reschedule_task -from katalogus.client import Boefje as KATalogusBoefje -from katalogus.client import get_katalogus -from katalogus.views.mixins import BoefjeMixin +from katalogus.client import Boefje, Normalizer, get_katalogus from katalogus.views.plugin_settings_list import PluginSettingsListView -from rocky import scheduler +from rocky.views.tasks import TaskListView logger = getLogger(__name__) -class PageActions(Enum): - RESCHEDULE_TASK = "reschedule_task" - - class PluginCoverImgView(OrganizationView): """Get the cover image of a plugin.""" @@ -37,72 +26,78 @@ def get(self, request, *args, **kwargs): return file -class PluginDetailView(PluginSettingsListView, TemplateView): - task_history_limit = 10 - - def get_task_history(self) -> Page: - scheduler_id = f"{self.plugin.type}-{self.organization.code}" - plugin_type = self.plugin.type - plugin_id = self.plugin.id - input_ooi = self.request.GET.get("task_history_search") - status = self.request.GET.get("task_history_status") - - if self.request.GET.get("task_history_from"): - min_created_at = datetime.strptime(self.request.GET.get("task_history_from"), "%Y-%m-%d") - else: - min_created_at = None - - if self.request.GET.get("task_history_to"): - max_created_at = datetime.strptime(self.request.GET.get("task_history_to"), "%Y-%m-%d") - else: - max_created_at = None +class PluginDetailView(TaskListView, PluginSettingsListView): + def post(self, request, *args, **kwargs): + if self.action == self.SCAN_OOIS: + selected_oois = request.POST.getlist("ooi", []) - page = int(self.request.GET.get("task_history_page", 1)) + if selected_oois and self.plugin.id: + oois = self.get_oois(selected_oois) + boefje = self.katalogus_client.get_plugin(self.plugin.id) - task_history = scheduler.client.get_lazy_task_list( - scheduler_id=scheduler_id, - task_type=plugin_type, - plugin_id=plugin_id, - input_ooi=input_ooi, - status=status, - min_created_at=min_created_at, - max_created_at=max_created_at, - ) + oois_with_clearance_level = oois["oois_with_clearance"] + oois_without_clearance_level = oois["oois_without_clearance"] - return Paginator(task_history, self.task_history_limit).page(page) + if oois_with_clearance_level: + self.run_boefje_for_oois( + boefje=boefje, + oois=oois_with_clearance_level, + ) - def post(self, request, *args, **kwargs): - action = request.POST["action"] + if oois_without_clearance_level: + if not self.organization_member.has_perm("tools.can_set_clearance_level"): + messages.error( + request, + _( + "Some selected OOIs needs an increase of clearance level to perform scans." + " You do not have the permission to change clearance level." + ), + ) + else: + request.session["selected_oois"] = oois_without_clearance_level + return redirect( + reverse( + "change_clearance_level", + kwargs={ + "plugin_type": "boefje", + "organization_code": self.organization.code, + "plugin_id": self.plugin.id, + "scan_level": self.plugin.scan_level.value, + }, + ) + ) + return super().post(request, *args, **kwargs) - if action: - self.handle_page_action(action) - return redirect(request.path) - else: - return self.get(request, *args, **kwargs) + def get_task_filters(self) -> dict[str, str | datetime | None]: + filters = super().get_task_filters() + filters["plugin_id"] = self.plugin.id # fetch only tasks for a specific plugin by id + return filters - def handle_page_action(self, action: str) -> None: - if action == PageActions.RESCHEDULE_TASK.value: - task_id = self.request.POST.get("task_id") - reschedule_task(self.request, self.organization.code, task_id) + def get_oois(self, selected_oois: list[str]) -> dict[str, Any]: + oois_with_clearance = [] + oois_without_clearance = [] + for ooi in selected_oois: + ooi_object = self.get_single_ooi(pk=ooi) + if ooi_object.scan_profile and ooi_object.scan_profile.level >= self.plugin.scan_level.value: + oois_with_clearance.append(ooi_object) + else: + oois_without_clearance.append(ooi_object.primary_key) + return { + "oois_with_clearance": oois_with_clearance, + "oois_without_clearance": oois_without_clearance, + } def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context["plugin"] = self.plugin.model_dump() - context["task_history"] = self.get_task_history() - context["task_history_form_fields"] = [ - "task_history_from", - "task_history_to", - "task_history_status", - "task_history_search", - "task_history_page", - ] - + context["plugin_settings"] = self.get_plugin_settings() return context class NormalizerDetailView(PluginDetailView): template_name = "normalizer_detail.html" + plugin: Normalizer + task_type = "normalizer" def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -126,12 +121,13 @@ def get_context_data(self, **kwargs): return context -class BoefjeDetailView(BoefjeMixin, PluginDetailView): +class BoefjeDetailView(PluginDetailView): """Detail view for a specific boefje. Shows boefje settings and consumable oois for scanning.""" template_name = "boefje_detail.html" limit_ooi_list = 9999 - plugin: KATalogusBoefje + plugin: Boefje + task_type = "boefje" def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -139,13 +135,15 @@ def get_context_data(self, **kwargs): context["select_ooi_filter_form"] = SelectOOIFilterForm if "show_all" in self.request.GET: context["select_oois_form"] = SelectOOIForm( - oois=self.get_form_consumable_oois(), organization_code=self.organization.code + oois=self.get_form_consumable_oois(), + organization_code=self.organization.code, ) else: context["select_oois_form"] = SelectOOIForm( - oois=self.get_form_filtered_consumable_oois(), organization_code=self.organization.code + oois=self.get_form_filtered_consumable_oois(), + organization_code=self.organization.code, ) - context["plugin"] = self.plugin.model_dump() + context["breadcrumbs"] = [ { "url": reverse("katalogus", kwargs={"organization_code": self.organization.code}), @@ -165,84 +163,15 @@ def get_context_data(self, **kwargs): return context - def post(self, request, *args, **kwargs): - action = request.POST["action"] - - if action == PageActions.RESCHEDULE_TASK.value: - self.handle_page_action(action) - return redirect(request.path) - - """Start scanning oois at plugin detail page.""" - if not self.indemnification_present: - return self.get(request, *args, **kwargs) - - if "boefje_id" not in request.POST: - raise BadRequest("No boefje_id provided") - - selected_oois = request.POST.getlist("ooi") - plugin_id = request.POST["boefje_id"] - if selected_oois and plugin_id: - oois = self.get_oois(selected_oois) - boefje = self.katalogus_client.get_plugin(plugin_id) - - oois_with_clearance_level = oois["oois_with_clearance"] - oois_without_clearance_level = oois["oois_without_clearance"] - - if oois_with_clearance_level: - self.run_boefje_for_oois( - boefje=boefje, - oois=oois_with_clearance_level, - ) - - if oois_without_clearance_level: - if not self.organization_member.has_perm("tools.can_set_clearance_level"): - messages.add_message( - self.request, - messages.ERROR, - _( - "Some selected OOIs needs an increase of clearance level to perform scans." - " You do not have the permission to change clearance level." - ), - ) - else: - request.session["selected_oois"] = oois_without_clearance_level - return redirect( - reverse( - "change_clearance_level", - kwargs={ - "plugin_type": "boefje", - "organization_code": self.organization.code, - "plugin_id": plugin_id, - "scan_level": self.plugin.scan_level.value, - }, - ) - ) - return redirect(reverse("task_list", kwargs={"organization_code": self.organization.code})) - - messages.add_message(self.request, messages.ERROR, _("Please select an OOI to start scan.")) - return self.get(request, *args, **kwargs) - def get_form_consumable_oois(self): """Get all available OOIS that plugin can consume.""" return self.octopoes_api_connector.list_objects( - self.plugin.consumes, valid_time=datetime.now(timezone.utc), limit=self.limit_ooi_list + self.plugin.consumes, + valid_time=datetime.now(timezone.utc), + limit=self.limit_ooi_list, ).items def get_form_filtered_consumable_oois(self): """Return a list of oois that is filtered for oois that meets clearance level.""" oois = self.get_form_consumable_oois() return [ooi for ooi in oois if ooi.scan_profile.level >= self.plugin.scan_level.value] - - def get_oois(self, selected_oois: list[str]) -> dict[str, Any]: - oois_with_clearance = [] - oois_without_clearance = [] - for ooi in selected_oois: - ooi_object = self.get_single_ooi(pk=ooi) - if ooi_object.scan_profile and ooi_object.scan_profile.level >= self.plugin.scan_level.value: - oois_with_clearance.append(ooi_object) - else: - oois_without_clearance.append(ooi_object.primary_key) - return { - "oois_with_clearance": oois_with_clearance, - "oois_without_clearance": oois_without_clearance, - } diff --git a/rocky/katalogus/views/plugin_settings_list.py b/rocky/katalogus/views/plugin_settings_list.py index 0c0bb15715..370e93a0bf 100644 --- a/rocky/katalogus/views/plugin_settings_list.py +++ b/rocky/katalogus/views/plugin_settings_list.py @@ -1,41 +1,42 @@ +from typing import Any + from django.contrib import messages from django.utils.translation import gettext_lazy as _ -from django.views.generic import ListView from httpx import HTTPError from katalogus.views.mixins import SinglePluginView +from rocky.paginator import RockyPaginator -class PluginSettingsListView(SinglePluginView, ListView): +class PluginSettingsListView(SinglePluginView): """ Shows all settings available for a specific plugin (plugin schema settings). """ - def get(self, request, *args, **kwargs): + paginator_class = RockyPaginator + paginate_by = 10 + context_object_name = "plugin_settings" + + def get_plugin_settings(self) -> list[dict[str, Any]]: + """Gets schema setting with additional info of the value of a setting.""" try: - self.object_list = self.get_queryset() + if self.plugin_schema is None: + return [] + + settings = self.katalogus_client.get_plugin_settings(plugin_id=self.plugin.id) + props = self.plugin_schema["properties"] + + return [ + { + "name": prop, + "value": settings.get(prop), + "required": self.is_required_field(prop), + "secret": self.is_secret_field(prop), + } + for prop in props + ] except HTTPError: messages.add_message( self.request, messages.ERROR, _("Failed getting settings for boefje {}").format(self.plugin.id) ) - self.object_list = [] - - return super().get(request, *args, **kwargs) - - def get_queryset(self): - """Gets schema setting with additional info of the value of a setting.""" - if self.plugin_schema is None: return [] - - settings = self.katalogus_client.get_plugin_settings(plugin_id=self.plugin.id) - props = self.plugin_schema["properties"] - - return [ - { - "name": prop, - "value": settings.get(prop), - "required": self.is_required_field(prop), - "secret": self.is_secret_field(prop), - } - for prop in props - ] diff --git a/rocky/reports/report_types/aggregate_organisation_report/report.py b/rocky/reports/report_types/aggregate_organisation_report/report.py index ea938ecf72..d929718491 100644 --- a/rocky/reports/report_types/aggregate_organisation_report/report.py +++ b/rocky/reports/report_types/aggregate_organisation_report/report.py @@ -42,7 +42,7 @@ class AggregateOrganisationReport(AggregateReport): } template_path = "aggregate_organisation_report/report.html" - def post_process_data(self, data: dict[str, Any], valid_time) -> dict[str, Any]: + def post_process_data(self, data: dict[str, Any], valid_time: datetime, organization_code: str) -> dict[str, Any]: systems: dict[str, dict[str, Any]] = {"services": {}} services = {} open_ports = {} @@ -392,7 +392,7 @@ def is_mail_compliant(result): config_oois = self.octopoes_api_connector.list_objects(types={Config}, valid_time=valid_time).items - flattened_health = flatten_health(get_rocky_health(self.octopoes_api_connector)) + flattened_health = flatten_health(get_rocky_health(organization_code, self.octopoes_api_connector)) return { "systems": systems, @@ -441,6 +441,7 @@ def aggregate_reports( input_ooi_references: list[OOI], selected_report_types: list[str], valid_time: datetime, + organization_code: str, ) -> tuple[AggregateOrganisationReport, dict[str, Any], dict[str, Any], list[str]]: by_type: dict[str, list[str]] = {} @@ -475,6 +476,6 @@ def aggregate_reports( report_data[ooi_str][report_type.id] = data aggregate_report = AggregateOrganisationReport(connector) - post_processed_data = aggregate_report.post_process_data(report_data, valid_time=valid_time) + post_processed_data = aggregate_report.post_process_data(report_data, valid_time, organization_code) return aggregate_report, post_processed_data, report_data, errors diff --git a/rocky/reports/report_types/definitions.py b/rocky/reports/report_types/definitions.py index cdf44984eb..970e6cda56 100644 --- a/rocky/reports/report_types/definitions.py +++ b/rocky/reports/report_types/definitions.py @@ -161,7 +161,7 @@ class AggregateReportSubReports(TypedDict): class AggregateReport(BaseReport): reports: AggregateReportSubReports - def post_process_data(self, data: dict[str, Any], valid_time: datetime) -> dict[str, Any]: + def post_process_data(self, data: dict[str, Any], valid_time: datetime, organization_code: str) -> dict[str, Any]: raise NotImplementedError diff --git a/rocky/reports/views/aggregate_report.py b/rocky/reports/views/aggregate_report.py index 7c9ed7530f..11d3c0963f 100644 --- a/rocky/reports/views/aggregate_report.py +++ b/rocky/reports/views/aggregate_report.py @@ -145,6 +145,7 @@ def save_report(self) -> ReportOOI: input_oois, self.selected_report_types, self.observed_at, + self.organization.code, ) # If OOI could not be found or the date is incorrect, it will be shown to the user as a message error diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index f799602fdf..56326a1a53 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-07-09 07:22+0000\n" +"POT-Creation-Date: 2024-07-11 11:23+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -724,7 +724,6 @@ msgstr "" #: reports/templates/partials/report_ooi_list.html #: reports/templates/report_overview/report_history_table.html #: reports/templates/summary/ooi_selection.html -#: rocky/templates/partials/task_history.html msgid "Object" msgstr "" @@ -817,7 +816,7 @@ msgid "Latest plugin settings" msgstr "" #: katalogus/templates/katalogus_settings.html -#: rocky/templates/partials/task_history.html +#: rocky/templates/tasks/ooi_detail_task_list.html msgid "Plugin" msgstr "" @@ -991,6 +990,7 @@ msgid "Plugins Navigation" msgstr "" #: katalogus/templates/partials/plugins_navigation.html +#: tools/forms/scheduler.py msgid "All" msgstr "" @@ -1136,10 +1136,6 @@ msgid "" "You do not have the permission to change clearance level." msgstr "" -#: katalogus/views/plugin_detail.py -msgid "Please select an OOI to start scan." -msgstr "" - #: katalogus/views/plugin_enable_disable.py msgid "{} '{}' disabled." msgstr "" @@ -1302,7 +1298,6 @@ msgstr "" #: rocky/templates/partials/elements/ooi_detail_settings.html #: rocky/templates/partials/elements/ooi_report_settings.html #: rocky/templates/partials/form/indemnification_add_form.html -#: rocky/templates/partials/task_history.html #: rocky/templates/two_factor/_wizard_actions.html msgid "Submit" msgstr "" @@ -2683,9 +2678,9 @@ msgstr "" #: rocky/templates/findings/finding_list.html #: rocky/templates/organizations/organization_crisis_room.html #: rocky/templates/partials/ooi_report_findings_block_table.html -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/boefje_task_history.html -#: rocky/templates/tasks/partials/normalizer_task_history.html +#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html +#: rocky/templates/tasks/ooi_detail_task_list.html +#: rocky/templates/tasks/plugin_detail_task_list.html msgid "Details" msgstr "" @@ -2695,9 +2690,9 @@ msgstr "" #: rocky/templates/crisis_room/crisis_room_findings_block.html #: rocky/templates/findings/finding_list.html #: rocky/templates/organizations/organization_crisis_room.html -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/boefje_task_history.html -#: rocky/templates/tasks/partials/normalizer_task_history.html +#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html +#: rocky/templates/tasks/ooi_detail_task_list.html +#: rocky/templates/tasks/plugin_detail_task_list.html msgid "Close details" msgstr "" @@ -2707,9 +2702,9 @@ msgstr "" #: rocky/templates/crisis_room/crisis_room_findings_block.html #: rocky/templates/findings/finding_list.html #: rocky/templates/organizations/organization_crisis_room.html -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/boefje_task_history.html -#: rocky/templates/tasks/partials/normalizer_task_history.html +#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html +#: rocky/templates/tasks/ooi_detail_task_list.html +#: rocky/templates/tasks/plugin_detail_task_list.html msgid "Open details" msgstr "" @@ -3187,10 +3182,9 @@ msgstr "" #: reports/report_types/tls_report/report.html #: reports/templates/partials/plugin_overview_table.html #: rocky/templates/organizations/organization_member_list.html -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/boefje_task_history.html -#: rocky/templates/tasks/partials/normalizer_task_history.html -#: rocky/templates/tasks/partials/task_filter.html +#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html +#: rocky/templates/tasks/ooi_detail_task_list.html +#: rocky/templates/tasks/plugin_detail_task_list.html msgid "Status" msgstr "" @@ -3443,7 +3437,6 @@ msgstr "" #: reports/templates/partials/report_ooi_list.html #: rocky/templates/admin/change_list.html rocky/templates/oois/ooi_list.html -#: rocky/templates/partials/task_history.html #: rocky/templates/tasks/partials/task_filter.html msgid "Filter" msgstr "" @@ -3780,7 +3773,8 @@ msgid "Subreports:" msgstr "" #: reports/templates/report_overview/subreports_table.html -#: rocky/templates/tasks/partials/boefje_task_history.html +#: rocky/templates/tasks/boefjes.html +#: rocky/templates/tasks/plugin_detail_task_list.html msgid "Input Object" msgstr "" @@ -3891,7 +3885,7 @@ msgstr "" msgid "Date" msgstr "" -#: tools/forms/base.py +#: tools/forms/base.py tools/forms/scheduler.py msgid "The selected date is in the future. Please select a different date." msgstr "" @@ -4070,6 +4064,50 @@ msgstr "" msgid "Filter by clearance type" msgstr "" +#: tools/forms/scheduler.py +msgid "From" +msgstr "" + +#: tools/forms/scheduler.py +msgid "To" +msgstr "" + +#: tools/forms/scheduler.py rocky/templates/tasks/partials/stats.html +msgid "Cancelled" +msgstr "" + +#: tools/forms/scheduler.py rocky/templates/tasks/partials/stats.html +msgid "Completed" +msgstr "" + +#: tools/forms/scheduler.py rocky/templates/tasks/partials/stats.html +msgid "Dispatched" +msgstr "" + +#: tools/forms/scheduler.py rocky/templates/tasks/partials/stats.html +msgid "Failed" +msgstr "" + +#: tools/forms/scheduler.py rocky/templates/tasks/partials/stats.html +msgid "Pending" +msgstr "" + +#: tools/forms/scheduler.py rocky/templates/tasks/partials/stats.html +msgid "Queued" +msgstr "" + +#: tools/forms/scheduler.py rocky/templates/tasks/partials/stats.html +msgid "Running" +msgstr "" + +#: tools/forms/scheduler.py +msgid "Search" +msgstr "" + +#: tools/forms/scheduler.py +msgid "Search by object name" +msgstr "" + #: tools/forms/settings.py msgid "--- Show all ----" msgstr "" @@ -4223,17 +4261,6 @@ msgstr "" msgid "Members" msgstr "" -#: tools/view_helpers.py -msgid "" -"Your task is scheduled and will soon be started in the background. Results " -"will be added to the object list when they are in. It may take some time, a " -"refresh of the page may be needed to show the results." -msgstr "" - -#: tools/view_helpers.py rocky/scheduler.py -msgid "Task not found." -msgstr "" - #: rocky/messaging.py msgid "" "You have trusted this member with a clearance level of L{}. This member " @@ -4254,19 +4281,47 @@ msgid "That page contains no results" msgstr "" #: rocky/scheduler.py -msgid "Connectivity issues with Mula." +msgid "" +"The Scheduler has an unexpected error. Check the Scheduler logs for further " +"details." +msgstr "" + +#: rocky/scheduler.py +msgid "Could not connect to Scheduler. Service is possibly down." +msgstr "" + +#: rocky/scheduler.py +msgid "Your request could not be validated." msgstr "" #: rocky/scheduler.py -msgid "Task queue is full, please try again later." +msgid "Task could not be found." msgstr "" #: rocky/scheduler.py -msgid "Task is invalid." +msgid "" +"Scheduler is receiving too many requests. Increase SCHEDULER_PQ_MAXSIZE or " +"wait for task to finish." +msgstr "" + +#: rocky/scheduler.py +msgid "Bad request. Your request could not be interpreted by the Scheduler." msgstr "" #: rocky/scheduler.py -msgid "Task already queued." +msgid "The Scheduler has received a conflict. Your task is already in queue." +msgstr "" + +#: rocky/scheduler.py +msgid "A HTTPError occurred. See Scheduler logs for more info." +msgstr "" + +#: rocky/scheduler.py +msgid "Task list: " +msgstr "" + +#: rocky/scheduler.py +msgid "Task statistics: " msgstr "" #: rocky/settings.py @@ -4755,9 +4810,8 @@ msgstr "" msgid "Crisis room" msgstr "" -#: rocky/templates/header.html rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/boefje_task_history.html -#: rocky/templates/tasks/partials/normalizer_task_history.html +#: rocky/templates/header.html rocky/templates/tasks/ooi_detail_task_list.html +#: rocky/templates/tasks/plugin_detail_task_list.html #: rocky/views/task_detail.py rocky/views/tasks.py msgid "Tasks" msgstr "" @@ -4944,9 +4998,8 @@ msgstr "" #: rocky/templates/oois/ooi_detail.html #: rocky/templates/oois/ooi_detail_origins_observations.html -#: rocky/templates/scan.html -#: rocky/templates/tasks/partials/boefje_task_history.html -#: rocky/templates/tasks/partials/normalizer_task_history.html +#: rocky/templates/scan.html rocky/templates/tasks/boefjes.html +#: rocky/templates/tasks/normalizers.html msgid "Boefje" msgstr "" @@ -5798,85 +5851,6 @@ msgstr "" msgid "Go to content" msgstr "" -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/task_filter.html -msgid "Hide filters" -msgstr "" - -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/task_filter.html -msgid "Show filters" -msgstr "" - -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/task_filter.html -msgid "From" -msgstr "" - -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/task_filter.html -msgid "To" -msgstr "" - -#: rocky/templates/partials/task_history.html -msgid "Select status" -msgstr "" - -#: rocky/templates/partials/task_history.html -msgid "Success" -msgstr "" - -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/stats.html -#: rocky/templates/tasks/partials/task_filter.html -msgid "Failed" -msgstr "" - -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/task_filter.html -msgid "Search" -msgstr "" - -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/task_filter.html -msgid "Search by object name" -msgstr "" - -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/normalizer_task_history.html -msgid "No tasks found for this object." -msgstr "" - -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/boefje_task_history.html -#: rocky/templates/tasks/partials/normalizer_task_history.html -msgid "Created date" -msgstr "" - -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/normalizer_task_history.html -msgid "Yielded objects" -msgstr "" - -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/boefje_task_detail.html -#: rocky/templates/tasks/partials/boefje_task_history.html -#: rocky/templates/tasks/partials/normalizer_task_history.html -msgid "Download meta and raw data" -msgstr "" - -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/boefje_task_history.html -#: rocky/templates/tasks/partials/normalizer_task_history.html -msgid "Reschedule" -msgstr "" - -#: rocky/templates/partials/task_history.html -#: rocky/templates/tasks/partials/boefje_task_history.html -#: rocky/templates/tasks/partials/normalizer_task_history.html -msgid "Download task data" -msgstr "" - #: rocky/templates/scan_profiles/scan_profile_detail.html msgid "Current clearance level" msgstr "" @@ -5945,6 +5919,11 @@ msgid "" "An overview of the boefje task, the input OOI and the RAW data it generated." msgstr "" +#: rocky/templates/tasks/boefje_task_detail.html +#: rocky/templates/tasks/partials/task_actions.html +msgid "Download meta and raw data" +msgstr "" + #: rocky/templates/tasks/boefje_task_detail.html msgid "Download meta data" msgstr "" @@ -5961,6 +5940,12 @@ msgstr "" msgid "List of tasks for boefjes" msgstr "" +#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html +#: rocky/templates/tasks/ooi_detail_task_list.html +#: rocky/templates/tasks/plugin_detail_task_list.html +msgid "Created date" +msgstr "" + #: rocky/templates/tasks/normalizers.html msgid "There are no tasks for normalizers" msgstr "" @@ -5969,68 +5954,62 @@ msgstr "" msgid "List of tasks for normalizers" msgstr "" -#: rocky/templates/tasks/partials/boefje_task_history.html -msgid "No scans found for this object." -msgstr "" - -#: rocky/templates/tasks/partials/normalizer_task_history.html +#: rocky/templates/tasks/normalizers.html msgid "Normalizer" msgstr "" -#: rocky/templates/tasks/partials/normalizer_task_history.html +#: rocky/templates/tasks/normalizers.html msgid "Boefje input OOI" msgstr "" -#: rocky/templates/tasks/partials/stats.html -msgid "Stats - Last 24 hours" +#: rocky/templates/tasks/ooi_detail_task_list.html +#: rocky/templates/tasks/plugin_detail_task_list.html +msgid "There are no tasks for" msgstr "" -#: rocky/templates/tasks/partials/stats.html -msgid "All times in UTC, blocks of 1 hour." +#: rocky/templates/tasks/ooi_detail_task_list.html +#: rocky/templates/tasks/plugin_detail_task_list.html +msgid "List of tasks for" msgstr "" #: rocky/templates/tasks/partials/stats.html -msgid "Timeslot" +msgid "Task statistics - Last 24 hours" msgstr "" #: rocky/templates/tasks/partials/stats.html -#: rocky/templates/tasks/partials/task_filter.html -msgid "Pending" +msgid "All times in UTC, blocks of 1 hour." msgstr "" #: rocky/templates/tasks/partials/stats.html -#: rocky/templates/tasks/partials/task_filter.html -msgid "Queued" +msgid "Timeslot" msgstr "" #: rocky/templates/tasks/partials/stats.html -#: rocky/templates/tasks/partials/task_filter.html -msgid "Dispatched" +msgid "Could not load stats, Scheduler error." msgstr "" -#: rocky/templates/tasks/partials/stats.html -msgid "Running" +#: rocky/templates/tasks/partials/tab_navigation.html +msgid "List of tasks" msgstr "" -#: rocky/templates/tasks/partials/stats.html -#: rocky/templates/tasks/partials/task_filter.html -msgid "Completed" +#: rocky/templates/tasks/partials/task_actions.html +msgid "Yielded objects" msgstr "" -#: rocky/templates/tasks/partials/stats.html -msgid "Cancelled" +#: rocky/templates/tasks/partials/task_actions.html +msgid "Reschedule" msgstr "" -#: rocky/templates/tasks/partials/stats.html -msgid "Could not load stats, Scheduler error." +#: rocky/templates/tasks/partials/task_actions.html +msgid "Download task data" msgstr "" -#: rocky/templates/tasks/partials/tab_navigation.html -msgid "List of tasks" +#: rocky/templates/tasks/partials/task_filter.html +msgid "Hide filters" msgstr "" #: rocky/templates/tasks/partials/task_filter.html -msgid "all" +msgid "Show filters" msgstr "" #: rocky/templates/two_factor/_wizard_actions.html @@ -6313,6 +6292,10 @@ msgstr "" msgid "Only Question OOIs can be answered." msgstr "" +#: rocky/views/ooi_detail.py +msgid "Question has been answered." +msgstr "" + #: rocky/views/ooi_detail_related_object.py msgid " (as " msgstr "" @@ -6552,6 +6535,10 @@ msgstr "" msgid "Recalculated {number_of_bits} bits. Duration: {duration}" msgstr "" +#: rocky/views/page_actions.py +msgid "Could not process your request, action required." +msgstr "" + #: rocky/views/scan_profile.py #, python-brace-format msgid "Can not reset scan level. Scan level of {ooi_name} not declared" @@ -6561,8 +6548,11 @@ msgstr "" msgid "Reset" msgstr "" -#: rocky/views/tasks.py -msgid "Fetching tasks failed: no connection with scheduler" +#: rocky/views/scheduler.py +msgid "" +"Your task is scheduled and will soon be started in the background. Results " +"will be added to the object list when they are in. It may take some time, a " +"refresh of the page may be needed to show the results." msgstr "" #: rocky/views/upload_csv.py diff --git a/rocky/rocky/scheduler.py b/rocky/rocky/scheduler.py index f9ac40fb63..5b80b2c7c9 100644 --- a/rocky/rocky/scheduler.py +++ b/rocky/rocky/scheduler.py @@ -4,14 +4,15 @@ import json import uuid from enum import Enum +from functools import cached_property from logging import getLogger from typing import Any import httpx from django.conf import settings from django.utils.translation import gettext_lazy as _ -from httpx import HTTPError, HTTPStatusError, RequestError, codes -from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny +from httpx import ConnectError, HTTPError, HTTPStatusError, RequestError, codes +from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny, ValidationError from rocky.health import ServiceHealth @@ -123,6 +124,8 @@ class PaginatedTasksResponse(BaseModel): class LazyTaskList: + HARD_LIMIT = 500 + def __init__( self, scheduler_client: SchedulerClient, @@ -132,7 +135,7 @@ def __init__( self.kwargs = kwargs self._count: int | None = None - @property + @cached_property def count(self) -> int: if self._count is None: self._count = self.scheduler_client.list_tasks( @@ -147,7 +150,8 @@ def __len__(self): def __getitem__(self, key) -> list[Task]: if isinstance(key, slice): offset = key.start or 0 - limit = key.stop - offset + limit = min(LazyTaskList.HARD_LIMIT, key.stop - offset or key.stop or LazyTaskList.HARD_LIMIT) + elif isinstance(key, int): offset = key limit = 1 @@ -161,83 +165,89 @@ def __getitem__(self, key) -> list[Task]: ) self._count = res.count + return res.results class SchedulerError(Exception): - message = _("Connectivity issues with Mula.") + message: str = _("The Scheduler has an unexpected error. Check the Scheduler logs for further details.") + + def __init__(self, *args: object, extra_message: str | None = None) -> None: + super().__init__(*args) + if extra_message is not None: + self.message = extra_message + self.message def __str__(self): return str(self.message) -class TooManyRequestsError(SchedulerError): - message = _("Task queue is full, please try again later.") +class SchedulerConnectError(SchedulerError): + message = _("Could not connect to Scheduler. Service is possibly down.") + + +class SchedulerValidationError(SchedulerError): + message = _("Your request could not be validated.") -class BadRequestError(SchedulerError): - message = _("Task is invalid.") +class SchedulerTaskNotFound(SchedulerError): + message = _("Task could not be found.") -class ConflictError(SchedulerError): - message = _("Task already queued.") +class SchedulerTooManyRequestError(SchedulerError): + message = _("Scheduler is receiving too many requests. Increase SCHEDULER_PQ_MAXSIZE or wait for task to finish.") -class TaskNotFoundError(SchedulerError): - message = _("Task not found.") +class SchedulerBadRequestError(SchedulerError): + message = _("Bad request. Your request could not be interpreted by the Scheduler.") + + +class SchedulerConflictError(SchedulerError): + message = _("The Scheduler has received a conflict. Your task is already in queue.") + + +class SchedulerHTTPError(SchedulerError): + message = _("A HTTPError occurred. See Scheduler logs for more info.") class SchedulerClient: - def __init__(self, base_uri: str): + def __init__(self, base_uri: str, organization_code: str): self._client = httpx.Client(base_url=base_uri) + self.organization_code = organization_code def list_tasks( self, **kwargs, ) -> PaginatedTasksResponse: - kwargs = {k: v for k, v in kwargs.items() if v is not None} # filter Nones from kwargs - res = self._client.get("/tasks", params=kwargs) - return PaginatedTasksResponse.model_validate_json(res.content) - - def get_lazy_task_list( - self, - scheduler_id: str, - task_type: str | None = None, - status: str | None = None, - min_created_at: datetime.datetime | None = None, - max_created_at: datetime.datetime | None = None, - input_ooi: str | None = None, - plugin_id: str | None = None, - boefje_name: str | None = None, - ) -> LazyTaskList: - return LazyTaskList( - self, - scheduler_id=scheduler_id, - type=task_type, - status=status, - min_created_at=min_created_at, - max_created_at=max_created_at, - input_ooi=input_ooi, - plugin_id=plugin_id, - boefje_name=boefje_name, - ) + try: + kwargs = {k: v for k, v in kwargs.items() if v is not None} # filter Nones from kwargs + res = self._client.get("/tasks", params=kwargs) + return PaginatedTasksResponse.model_validate_json(res.content) + except ValidationError: + raise SchedulerValidationError(extra_message=_("Task list: ")) + except ConnectError: + raise SchedulerConnectError(extra_message=_("Task list: ")) + + def get_task_details(self, task_id: str) -> Task: + try: + res = self._client.get(f"/tasks/{task_id}") + res.raise_for_status() + task_details = Task.model_validate_json(res.content) - def get_task_details(self, organization_code: str, task_id: str) -> Task: - res = self._client.get(f"/tasks/{task_id}") - res.raise_for_status() - task_details = Task.model_validate_json(res.content) + if task_details.type == "normalizer": + organization = task_details.p_item.data.raw_data.boefje_meta.organization + else: + organization = task_details.p_item.data.organization - if task_details.type == "normalizer": - organization = task_details.p_item.data.raw_data.boefje_meta.organization - else: - organization = task_details.p_item.data.organization + if organization != self.organization_code: + raise SchedulerTaskNotFound() - if organization != organization_code: - raise TaskNotFoundError() - return task_details + return task_details + except ConnectError: + raise SchedulerConnectError() - def push_task(self, queue_name: str, prioritized_item: PrioritizedItem) -> None: + def push_task(self, prioritized_item: PrioritizedItem) -> None: try: + queue_name = f"{prioritized_item.data.type}-{self.organization_code}" res = self._client.post( f"/queues/{queue_name}/push", content=prioritized_item.json(), @@ -247,29 +257,33 @@ def push_task(self, queue_name: str, prioritized_item: PrioritizedItem) -> None: except HTTPStatusError as http_error: code = http_error.response.status_code if code == codes.TOO_MANY_REQUESTS: - raise TooManyRequestsError() + raise SchedulerTooManyRequestError() elif code == codes.BAD_REQUEST: - raise BadRequestError() + raise SchedulerBadRequestError() elif code == codes.CONFLICT: - raise ConflictError() - else: - raise SchedulerError() + raise SchedulerConflictError() except RequestError: raise SchedulerError() def health(self) -> ServiceHealth: - health_endpoint = self._client.get("/health") - health_endpoint.raise_for_status() - return ServiceHealth.model_validate_json(health_endpoint.content) + try: + health_endpoint = self._client.get("/health") + health_endpoint.raise_for_status() + return ServiceHealth.model_validate_json(health_endpoint.content) + except HTTPError: + raise SchedulerHTTPError() + except ConnectError: + raise SchedulerConnectError() - def get_task_stats(self, organization_code: str, task_type: str) -> dict: + def get_task_stats(self, task_type: str) -> dict: try: - res = self._client.get(f"/tasks/stats/{task_type}-{organization_code}") + res = self._client.get(f"/tasks/stats/{task_type}-{self.organization_code}") res.raise_for_status() - except HTTPError: - raise SchedulerError() + except ConnectError: + raise SchedulerConnectError(extra_message=_("Task statistics: ")) task_stats = json.loads(res.content) return task_stats -client = SchedulerClient(settings.SCHEDULER_API) +def scheduler_client(organization_code: str) -> SchedulerClient: + return SchedulerClient(settings.SCHEDULER_API, organization_code) diff --git a/rocky/rocky/templates/oois/ooi_detail.html b/rocky/rocky/templates/oois/ooi_detail.html index a85076fcd0..9a55c6511b 100644 --- a/rocky/rocky/templates/oois/ooi_detail.html +++ b/rocky/rocky/templates/oois/ooi_detail.html @@ -31,7 +31,7 @@ {% include "oois/ooi_detail_origins_observations.html" %} {% include "oois/ooi_detail_origins_inference.html" %} {% include "partials/ooi_detail_related_object.html" with query=mandatory_fields ooi_past_due=ooi_past_due related=related ooi=ooi %} - {% include "partials/task_history.html" %} + {% include "tasks/ooi_detail_task_list.html" %}
diff --git a/rocky/rocky/templates/partials/task_history.html b/rocky/rocky/templates/partials/task_history.html deleted file mode 100644 index 754d8a166d..0000000000 --- a/rocky/rocky/templates/partials/task_history.html +++ /dev/null @@ -1,146 +0,0 @@ -{% load i18n %} - -
-

{% translate "Tasks" %}

-
-
-

{% translate "Filter" %}

- -
-
-
- - -
-
- - -
-
- - -
-
- - -
- {% for key, value in request.GET.items %} - {% if key not in task_history_form_fields %}{% endif %} - {% endfor %} -
- -
-
-
- {% if not task_history %} - {% translate "No tasks found for this object." %} - {% else %} - - - - - - - - - - - - {% for task in task_history %} - - - - - - - - - - - {% endfor %} - -
{% translate "Object" %}{% translate "Plugin" %}{% translate "Status" %}{% translate "Created date" %}
- {% if task.type == "boefje" %} - {{ task.p_item.data.input_ooi }} - {% elif task.type == "normalizer" %} - {{ task.p_item.data.raw_data.boefje_meta.input_ooi }} - {% else %} - {{ task.id }} - {% endif %} - - {% if task.type == "boefje" %} - {{ task.p_item.data.boefje.name }} - {% elif task.type == "normalizer" %} - {{ task.p_item.data.normalizer.id }} - {% else %} - {{ task.id }} - {% endif %} - - {% if task.status.value == "pending" or task.status.value == "queued" %} -  {{ task.status.value|capfirst }} - {% elif task.status.value == "running" or task.status.value == "dispatched" %} -  {{ task.status.value|capfirst }} - {% elif task.status.value == "cancelled" %} -  {{ task.status.value|capfirst }} - {% elif task.status.value == "failed" %} -  {{ task.status.value|capfirst }} - {% elif task.status.value == "completed" %} -  {{ task.status.value|capfirst }} - {% else %} -  {{ task.status.value|capfirst }} - {% endif %} - {{ task.created_at }} - -
- {% if task.type == "normalizer" %} -
-
-

{% translate "Yielded objects" %}

-
-
-
- {% endif %} -
-
- {% if task.status.value in "completed,failed" %} - {% translate "Download meta and raw data" %} - {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" value=task.id %} - - {% else %} - {% translate "Download task data" %} - {% endif %} -
-
-
- {% endif %} -
-{% include "partials/list_paginator.html" with page_obj=task_history page_param="task_history_page" %} diff --git a/rocky/rocky/templates/tasks/boefjes.html b/rocky/rocky/templates/tasks/boefjes.html index 60e651ecaf..7b8ee24d0e 100644 --- a/rocky/rocky/templates/tasks/boefjes.html +++ b/rocky/rocky/templates/tasks/boefjes.html @@ -13,18 +13,60 @@ {% include "tasks/partials/tab_navigation.html" with view="boefjes_tasks" %}

{% translate "Boefjes" %}

- {% include "tasks/partials/task_filter.html" %} - - {% if not object_list %} + {% if not task_list %}

{% translate "There are no tasks for boefjes" %}

+ {% include "tasks/partials/task_filter.html" %} + {% else %}

{% translate "List of tasks for boefjes" %}

+ {% include "tasks/partials/task_filter.html" %} +
- {% include "tasks/partials/boefje_task_history.html" with task_history=object_list %} + + + + + + + + + + + + {% for task in task_list %} + + + + + + + + + + + {% endfor %} + +
{% translate "Boefje" %}{% translate "Status" %}{% translate "Created date" %}{% translate "Input Object" %}
+ {{ task.p_item.data.boefje.name }} + +  {{ task.status.value|capfirst }} + {{ task.created_at }} + {{ task.p_item.data.input_ooi }} + + +
+ {% include "tasks/partials/task_actions.html" %} - - {% include "partials/list_paginator.html" %} +
+ {% include "partials/list_paginator.html" %} +
{% endif %}
diff --git a/rocky/rocky/templates/tasks/normalizers.html b/rocky/rocky/templates/tasks/normalizers.html index 10ac611965..9ead44abaf 100644 --- a/rocky/rocky/templates/tasks/normalizers.html +++ b/rocky/rocky/templates/tasks/normalizers.html @@ -14,18 +14,64 @@ {% include "tasks/partials/tab_navigation.html" with view="normalizers_tasks" %}

{% translate "Normalizers" %}

- {% include "tasks/partials/task_filter.html" %} - - {% if not object_list %} + {% if not task_list %}

{% translate "There are no tasks for normalizers" %}

+ {% include "tasks/partials/task_filter.html" %} + {% else %}

{% translate "List of tasks for normalizers" %}

+ {% include "tasks/partials/task_filter.html" %} +
- {% include "tasks/partials/normalizer_task_history.html" with task_history=object_list %} + + + + + + + + + + + + + {% for task in task_list %} + + + + + + + + + + + + {% endfor %} + +
{% translate "Normalizer" %}{% translate "Status" %}{% translate "Created date" %}{% translate "Boefje" %}{% translate "Boefje input OOI" %}
+ {{ task.p_item.data.normalizer.id }} + +  {{ task.status.value|capfirst }} + {{ task.created_at }} + {{ task.p_item.data.raw_data.boefje_meta.boefje.id }} + + {{ task.p_item.data.raw_data.boefje_meta.input_ooi }} + + +
+ {% include "tasks/partials/task_actions.html" %} - - {% include "partials/list_paginator.html" %} +
+ {% include "partials/list_paginator.html" %} +
{% endif %} diff --git a/rocky/rocky/templates/tasks/ooi_detail_task_list.html b/rocky/rocky/templates/tasks/ooi_detail_task_list.html new file mode 100644 index 0000000000..bfd85589b8 --- /dev/null +++ b/rocky/rocky/templates/tasks/ooi_detail_task_list.html @@ -0,0 +1,60 @@ +{% load i18n %} + +
+

{% translate "Tasks" %}

+ {% ooi_url 'ooi_detail' ooi.primary_key organization.code query=mandatory_fields as this_url %} + {% if not task_list %} +

{% translate "There are no tasks for" %} {{ ooi }}

+ {% include "tasks/partials/task_filter.html" with clear_filter_url=this_url %} + + {% else %} +

{% translate "List of tasks for" %} {{ ooi }}

+ {% include "tasks/partials/task_filter.html" with clear_filter_url=this_url %} + + + + + + + + + + + + {% for task in task_list %} + + + + + + + + + + {% endfor %} + +
{% translate "Plugin" %}{% translate "Status" %}{% translate "Created date" %}
+ {% if task.type == "boefje" %} + {{ task.p_item.data.boefje.name }} + {% elif task.type == "normalizer" %} + {{ task.p_item.data.normalizer.id }} + {% else %} + {{ task.id }} + {% endif %} + +  {{ task.status.value|capfirst }} + {{ task.created_at }} + +
+ {% include "tasks/partials/task_actions.html" %} + +
+ {% endif %} +
+{% include "partials/list_paginator.html" %} diff --git a/rocky/rocky/templates/tasks/partials/boefje_task_history.html b/rocky/rocky/templates/tasks/partials/boefje_task_history.html deleted file mode 100644 index 94468bb3d4..0000000000 --- a/rocky/rocky/templates/tasks/partials/boefje_task_history.html +++ /dev/null @@ -1,76 +0,0 @@ -{% load i18n %} - -
-

{% translate "Tasks" %}

- {% if not task_history %} - {% translate "No scans found for this object." %} - {% else %} - - - - - - - - - - - - {% for task in task_history %} - - - - - - {# FIXME: implement detail page according to designs #} - - - - - - {% endfor %} - -
{% translate "Boefje" %}{% translate "Status" %}{% translate "Created date" %}{% translate "Input Object" %}
- {{ task.p_item.data.boefje.name }} - - {% if task.status.value == "pending" or task.status.value == "queued" %} -  {{ task.status.value|capfirst }} - {% elif task.status.value == "running" or task.status.value == "dispatched" %} -  {{ task.status.value|capfirst }} - {% elif task.status.value == "cancelled" %} -  {{ task.status.value|capfirst }} - {% elif task.status.value == "failed" %} -  {{ task.status.value|capfirst }} - {% elif task.status.value == "completed" %} -  {{ task.status.value|capfirst }} - {% else %} -  {{ task.status.value|capfirst }} - {% endif %} - {{ task.created_at }} - {{ task.p_item.data.input_ooi }} - - -
-
-
- {% if task.status.value in "completed,failed" %} - {% translate "Download meta and raw data" %} - {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" value=task.id %} - - {% else %} - {% translate "Download task data" %} - {% endif %} -
-
-
- {% endif %} -
-{% include "partials/list_paginator.html" with page_obj=task_history page_param="task_history_page" %} diff --git a/rocky/rocky/templates/tasks/partials/normalizer_task_history.html b/rocky/rocky/templates/tasks/partials/normalizer_task_history.html deleted file mode 100644 index 77529ce57d..0000000000 --- a/rocky/rocky/templates/tasks/partials/normalizer_task_history.html +++ /dev/null @@ -1,88 +0,0 @@ -{% load i18n %} - -
-

{% translate "Tasks" %}

- {% if not task_history %} - {% translate "No tasks found for this object." %} - {% else %} - - - - - - - - - - - - - {% for task in task_history %} - - - - - - - {# FIXME: implement detail page according to designs #} - - - - - - {% endfor %} - -
{% translate "Normalizer" %}{% translate "Status" %}{% translate "Created date" %}{% translate "Boefje" %}{% translate "Boefje input OOI" %}
- {{ task.p_item.data.normalizer.id }} - - {% if task.status.value == "pending" or task.status.value == "queued" %} -  {{ task.status.value|capfirst }} - {% elif task.status.value == "running" or task.status.value == "dispatched" %} -  {{ task.status.value|capfirst }} - {% elif task.status.value == "cancelled" %} -  {{ task.status.value|capfirst }} - {% elif task.status.value == "failed" %} -  {{ task.status.value|capfirst }} - {% elif task.status.value == "completed" %} -  {{ task.status.value|capfirst }} - {% else %} -  {{ task.status.value|capfirst }} - {% endif %} - {{ task.created_at }} - {{ task.p_item.data.raw_data.boefje_meta.boefje.id }} - - {{ task.p_item.data.raw_data.boefje_meta.input_ooi }} - - -
-
-
-

{% translate "Yielded objects" %}

-
-
-
-
-
-
- {% if task.status.value in "completed,failed" %} - {% translate "Download meta and raw data" %} - {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" value=task.id %} - - {% else %} - {% translate "Download task data" %} - {% endif %} -
-
-
-
- {% endif %} -
-{% include "partials/list_paginator.html" with page_obj=task_history page_param="task_history_page" %} diff --git a/rocky/rocky/templates/tasks/partials/stats.html b/rocky/rocky/templates/tasks/partials/stats.html index 78202add53..4c06451f00 100644 --- a/rocky/rocky/templates/tasks/partials/stats.html +++ b/rocky/rocky/templates/tasks/partials/stats.html @@ -2,7 +2,7 @@
-

{% translate "Stats - Last 24 hours" %}

+

{% translate "Task statistics - Last 24 hours" %}

{% if not stats_error %}
diff --git a/rocky/rocky/templates/tasks/partials/task_actions.html b/rocky/rocky/templates/tasks/partials/task_actions.html new file mode 100644 index 0000000000..5a21db4996 --- /dev/null +++ b/rocky/rocky/templates/tasks/partials/task_actions.html @@ -0,0 +1,30 @@ +{% load i18n %} + +{% if task.type == "normalizer" %} +
+
+

{% translate "Yielded objects" %}

+
+
+
+{% endif %} +
+
+ {% if task.status.value in "completed,failed" %} + {% if task.type == "normalizer" %} + {% translate "Download meta and raw data" %} + {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" value=task.id %} + + {% elif task.type == "boefje" %} + {% translate "Download meta and raw data" %} + {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" value=task.id %} + + {% endif %} + {% else %} + {% translate "Download task data" %} + {% endif %} +
+
diff --git a/rocky/rocky/templates/tasks/partials/task_filter.html b/rocky/rocky/templates/tasks/partials/task_filter.html index db748a9026..92b34d69a0 100644 --- a/rocky/rocky/templates/tasks/partials/task_filter.html +++ b/rocky/rocky/templates/tasks/partials/task_filter.html @@ -6,62 +6,14 @@ - -
- - -
-
- - -
-
- - -
-
- - - {% for key, value in request.GET.items %} - {% if key not in scan_history_form_fields %}{% endif %} - {% endfor %} -
- - {% translate "Clear filters" %} + + {% include "partials/form/fieldset.html" with fields=task_filter_form %} + +
+ + {% translate "Clear filters" %} +
diff --git a/rocky/rocky/templates/tasks/plugin_detail_task_list.html b/rocky/rocky/templates/tasks/plugin_detail_task_list.html new file mode 100644 index 0000000000..55079e91cd --- /dev/null +++ b/rocky/rocky/templates/tasks/plugin_detail_task_list.html @@ -0,0 +1,62 @@ +{% load i18n %} + +
+

{% translate "Tasks" %}

+ {% if not task_list %} +

{% translate "There are no tasks for" %} {{ plugin.name }}

+ {% include "tasks/partials/task_filter.html" %} + + {% else %} +

{% translate "List of tasks for" %} {{ plugin.name }}

+ {% include "tasks/partials/task_filter.html" %} + +
+ + + + + + + + + + {% for task in task_list %} + + + + + + + + + + {% endfor %} + +
{% translate "Status" %}{% translate "Created date" %}{% translate "Input Object" %}
+  {{ task.status.value|capfirst }} + {{ task.created_at }} + {% if task.type == "boefje" %} + {% with ooi=task.p_item.data.input_ooi %} + {{ ooi }} + {% endwith %} + {% elif task.type == "normalizer" %} + {% with ooi=task.p_item.data.raw_data.boefje_meta.input_ooi %} + {{ ooi }} + {% endwith %} + {% endif %} + + +
+ {% include "tasks/partials/task_actions.html" %} + +
+ {% include "partials/list_paginator.html" %} + + {% endif %} +
diff --git a/rocky/rocky/urls.py b/rocky/rocky/urls.py index f5e17a8ba6..bb1157c840 100644 --- a/rocky/rocky/urls.py +++ b/rocky/rocky/urls.py @@ -39,8 +39,8 @@ from rocky.views.privacy_statement import PrivacyStatementView from rocky.views.scan_profile import ScanProfileDetailView, ScanProfileResetView from rocky.views.scans import ScanListView -from rocky.views.task_detail import BoefjeTaskDetailView, NormalizerTaskJSONView -from rocky.views.tasks import BoefjesTaskListView, DownloadTaskDetail, NormalizersTaskListView +from rocky.views.task_detail import BoefjeTaskDetailView, DownloadTaskDetail, NormalizerTaskJSONView +from rocky.views.tasks import BoefjesTaskListView, NormalizersTaskListView from rocky.views.upload_csv import UploadCSV from rocky.views.upload_raw import UploadRaw diff --git a/rocky/rocky/views/health.py b/rocky/rocky/views/health.py index ea8de1c494..1521721111 100644 --- a/rocky/rocky/views/health.py +++ b/rocky/rocky/views/health.py @@ -11,7 +11,7 @@ from rocky.bytes_client import get_bytes_client from rocky.health import ServiceHealth from rocky.keiko import keiko_client -from rocky.scheduler import client +from rocky.scheduler import SchedulerError, scheduler_client from rocky.version import __version__ logger = structlog.get_logger(__name__) @@ -20,7 +20,7 @@ class Health(OrganizationView, View): def get(self, request, *args, **kwargs) -> JsonResponse: octopoes_connector = self.octopoes_api_connector - rocky_health = get_rocky_health(octopoes_connector) + rocky_health = get_rocky_health(self.organization.code, octopoes_connector) return JsonResponse(rocky_health.model_dump()) @@ -51,10 +51,10 @@ def get_octopoes_health(octopoes_api_connector: OctopoesAPIConnector) -> Service return octopoes_health -def get_scheduler_health() -> ServiceHealth: +def get_scheduler_health(organization_code: str) -> ServiceHealth: try: - scheduler_health = client.health() - except HTTPError: + scheduler_health = scheduler_client(organization_code).health() + except SchedulerError: logger.exception("Error while retrieving Scheduler health state") scheduler_health = ServiceHealth( service="scheduler", @@ -76,11 +76,11 @@ def get_keiko_health() -> ServiceHealth: ) -def get_rocky_health(octopoes_api_connector: OctopoesAPIConnector) -> ServiceHealth: +def get_rocky_health(organization_code: str, octopoes_api_connector: OctopoesAPIConnector) -> ServiceHealth: services = [ get_octopoes_health(octopoes_api_connector), get_katalogus_health(), - get_scheduler_health(), + get_scheduler_health(organization_code), get_bytes_health(), get_keiko_health(), ] @@ -119,6 +119,8 @@ def get_context_data(self, **kwargs): "text": _("Beautified"), }, ] - rocky_health = get_rocky_health(self.octopoes_api_connector) + + rocky_health = get_rocky_health(self.organization.code, self.octopoes_api_connector) context["health_checks"] = flatten_health(rocky_health) + return context diff --git a/rocky/rocky/views/ooi_detail.py b/rocky/rocky/views/ooi_detail.py index 3052b3e163..fac902817f 100644 --- a/rocky/rocky/views/ooi_detail.py +++ b/rocky/rocky/views/ooi_detail.py @@ -1,153 +1,77 @@ import json from collections import defaultdict -from datetime import datetime, timezone -from enum import Enum +from datetime import datetime from django.contrib import messages -from django.core.paginator import Page, Paginator -from django.http import Http404 -from django.shortcuts import redirect from django.utils.translation import gettext_lazy as _ -from httpx import HTTPError from jsonschema.validators import Draft202012Validator from katalogus.client import get_katalogus from katalogus.utils import get_enabled_boefjes_for_ooi_class -from katalogus.views.mixins import BoefjeMixin -from tools.forms.base import ObservedAtForm from tools.forms.ooi import PossibleBoefjesFilterForm -from tools.models import Indemnification +from tools.forms.scheduler import OOIDetailTaskFilterForm from tools.ooi_helpers import format_display -from tools.view_helpers import reschedule_task -from octopoes.models import OOI, Reference +from octopoes.models import Reference from octopoes.models.ooi.question import Question -from rocky import scheduler from rocky.views.ooi_detail_related_object import OOIFindingManager, OOIRelatedObjectAddView from rocky.views.ooi_view import BaseOOIDetailView - - -class PageActions(Enum): - START_SCAN = "start_scan" - SUBMIT_ANSWER = "submit_answer" - RESCHEDULE_TASK = "reschedule_task" - CHANGE_CLEARANCE_LEVEL = "change_clearance_level" +from rocky.views.tasks import TaskListView class OOIDetailView( - BoefjeMixin, + BaseOOIDetailView, OOIRelatedObjectAddView, OOIFindingManager, - BaseOOIDetailView, + TaskListView, ): template_name = "oois/ooi_detail.html" - connector_form_class = ObservedAtForm - task_history_limit = 10 + task_filter_form = OOIDetailTaskFilterForm + task_type = "boefje" def post(self, request, *args, **kwargs): + if self.action == self.CHANGE_CLEARANCE_LEVEL: + self.set_clearance_level() + if self.action == self.SUBMIT_ANSWER: + self.answer_ooi_questions() + if self.action == self.START_SCAN: + self.start_boefje_scan() + return super().post(request, *args, **kwargs) + + def set_clearance_level(self) -> None: if not self.indemnification_present: - messages.add_message( - request, messages.ERROR, f"Indemnification not present at organization {self.organization}." - ) - return self.get(request, status_code=403, *args, **kwargs) - - if "action" not in self.request.POST: - return self.get(request, status_code=404, *args, **kwargs) - - self.ooi = self.get_ooi() - - action = self.request.POST.get("action") - return self.handle_page_action(action) - - def handle_page_action(self, action: str) -> bool: - try: - if action == PageActions.CHANGE_CLEARANCE_LEVEL.value: - clearance_level = int(self.request.POST.get("level")) - if not self.can_raise_clearance_level(self.ooi, clearance_level): - return redirect("account_detail", organization_code=self.organization.code) - return self.get(self.request, *self.args, **self.kwargs) - - if action == PageActions.RESCHEDULE_TASK.value: - task_id = self.request.POST.get("task_id") - reschedule_task(self.request, self.organization.code, task_id) - - if action == PageActions.START_SCAN.value: - boefje_id = self.request.POST.get("boefje_id") - ooi_id = self.request.GET.get("ooi_id") - - boefje = get_katalogus(self.organization.code).get_plugin(boefje_id) - ooi = self.get_single_ooi(pk=ooi_id) - self.run_boefje_for_oois(boefje, [ooi]) - return redirect("task_list", organization_code=self.organization.code) - - if action == PageActions.SUBMIT_ANSWER.value: - if not isinstance(self.ooi, Question): - messages.add_message(self.request, messages.ERROR, _("Only Question OOIs can be answered.")) - return self.get(self.request, status_code=500, *self.args, **self.kwargs) - - schema_answer = self.request.POST.get("schema") - parsed_schema_answer = json.loads(schema_answer) - validator = Draft202012Validator(json.loads(self.ooi.json_schema)) - - if not validator.is_valid(parsed_schema_answer): - for error in validator.iter_errors(parsed_schema_answer): - messages.add_message(self.request, messages.ERROR, error.message) - - return self.get(self.request, status_code=422, *self.args, **self.kwargs) - - raw = json.dumps({"schema": self.ooi.schema_id, "answer": parsed_schema_answer}).encode() - self.bytes_client.upload_raw(raw, {"answer"}, self.ooi.ooi) - messages.add_message(self.request, messages.SUCCESS, "Question has been answered.") - return self.get(self.request, status_code=201, *self.args, **self.kwargs) - - return self.get(self.request, status_code=404, *self.args, **self.kwargs) - except HTTPError as exception: - messages.add_message(self.request, messages.ERROR, f"{action} failed: '{exception}'") - return self.get(self.request, status_code=500, *self.args, **self.kwargs) - - def get_current_ooi(self) -> OOI | None: - # self.ooi is already the current state of the OOI - if self.observed_at.date() == datetime.utcnow().date(): - return self.ooi - try: - return self.get_ooi(pk=self.get_ooi_id(), observed_at=datetime.now(timezone.utc)) - except Http404: - return None - - def get_organization_indemnification(self): - return Indemnification.objects.filter(organization=self.organization).exists() - - def get_task_history(self) -> Page: - scheduler_id = f"boefje-{self.organization.code}" - - # FIXME: in context of ooi detail is doesn't make sense to search - # for an object name, so we search on plugin id - plugin_id = self.request.GET.get("task_history_search") or None - - page = int(self.request.GET.get("task_history_page", 1)) - - status = self.request.GET.get("task_history_status") - - if self.request.GET.get("task_history_from"): - min_created_at = datetime.strptime(self.request.GET.get("task_history_from"), "%Y-%m-%d") + return self.indemnification_error() else: - min_created_at = None - - if self.request.GET.get("task_history_to"): - max_created_at = datetime.strptime(self.request.GET.get("task_history_to"), "%Y-%m-%d") - else: - max_created_at = None - - task_history = scheduler.client.get_lazy_task_list( - scheduler_id=scheduler_id, - status=status, - min_created_at=min_created_at, - max_created_at=max_created_at, - task_type="boefje", - input_ooi=self.get_ooi_id(), - plugin_id=plugin_id, - ) - - return Paginator(task_history, self.task_history_limit).page(page) + clearance_level = int(self.request.POST.get("level")) + self.can_raise_clearance_level(self.ooi, clearance_level) # returns appropriate messages + + def answer_ooi_questions(self) -> None: + if not isinstance(self.ooi, Question): + messages.error(self.request, _("Only Question OOIs can be answered.")) + return + + schema_answer = self.request.POST.get("schema", "") + parsed_schema_answer = json.loads(schema_answer) + validator = Draft202012Validator(json.loads(self.ooi.json_schema)) + + if not validator.is_valid(parsed_schema_answer): + for error in validator.iter_errors(parsed_schema_answer): + messages.error(self.request, error.message) + return + + self.bytes_client.upload_raw(schema_answer, {"answer", f"{self.ooi.schema_id}"}, self.ooi.ooi) + messages.success(self.request, _("Question has been answered.")) + + def start_boefje_scan(self) -> None: + boefje_id = self.request.POST.get("boefje_id") + boefje = get_katalogus(self.organization.code).get_plugin(boefje_id) + ooi_id = self.request.GET.get("ooi_id") + ooi = self.get_single_ooi(pk=ooi_id) + self.run_boefje(boefje, ooi) + + def get_task_filters(self) -> dict[str, str | datetime | None]: + filters = super().get_task_filters() + filters["input_ooi"] = self.ooi.primary_key # shows only tasks for this particular ooi + return filters def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -156,7 +80,7 @@ def get_context_data(self, **kwargs): # List from katalogus boefjes = [] - if self.get_organization_indemnification(): + if self.indemnification_present: boefjes = get_enabled_boefjes_for_ooi_class(self.ooi.__class__, self.organization) if boefjes: @@ -184,6 +108,7 @@ def get_context_data(self, **kwargs): inference_origin_params.append((inference, inference_params_per_inference[inference.origin.id])) context["declarations"] = declarations + context["observations"] = observations context["inferences"] = inferences context["inference_origin_params"] = inference_origin_params @@ -192,26 +117,17 @@ def get_context_data(self, **kwargs): # TODO: generic solution to render ooi fields properly: https://github.com/minvws/nl-kat-coordination/issues/145 context["object_details"] = format_display(self.get_ooi_properties(self.ooi), ignore=["json_schema"]) context["ooi_types"] = self.get_ooi_types_input_values(self.ooi) - context["observed_at_form"] = self.get_connector_form() - context["observed_at"] = self.observed_at + context["is_question"] = isinstance(self.ooi, Question) context["ooi_past_due"] = context["observed_at"].date() < datetime.utcnow().date() context["related"] = self.get_related_objects(context["observed_at"]) - context["ooi_current"] = self.get_current_ooi() context["count_findings_per_severity"] = dict(self.count_findings_per_severity()) context["severity_summary_totals"] = sum(context["count_findings_per_severity"].values()) context["possible_boefjes_filter_form"] = filter_form - context["organization_indemnification"] = self.get_organization_indemnification() - context["task_history"] = self.get_task_history() - context["task_history_form_fields"] = [ - "task_history_from", - "task_history_to", - "task_history_status", - "task_history_search", - "task_history_page", - ] + context["organization_indemnification"] = self.indemnification_present + if self.request.GET.get("show_clearance_level_inheritance"): clearance_level_inheritance = self.get_scan_profile_inheritance(self.ooi) formatted_inheritance = [ diff --git a/rocky/rocky/views/ooi_detail_related_object.py b/rocky/rocky/views/ooi_detail_related_object.py index e34eeabc36..356c189b9c 100644 --- a/rocky/rocky/views/ooi_detail_related_object.py +++ b/rocky/rocky/views/ooi_detail_related_object.py @@ -3,7 +3,6 @@ from django.shortcuts import redirect from django.urls import reverse from django.utils.translation import gettext_lazy as _ -from django.views.generic import TemplateView from tools.ooi_helpers import format_attr_name from tools.view_helpers import existing_ooi_type, get_mandatory_fields, url_with_querystring @@ -59,7 +58,7 @@ def get_finding_details(self) -> list[tuple[Finding, FindingType]]: return [(finding, self.tree.store[str(finding.finding_type)]) for finding in self.get_findings()] -class OOIRelatedObjectAddView(OOIRelatedObjectManager, TemplateView): +class OOIRelatedObjectAddView(OOIRelatedObjectManager): template_name = "oois/ooi_detail_add_related_object.html" def get(self, request, *args, **kwargs): diff --git a/rocky/rocky/views/ooi_findings.py b/rocky/rocky/views/ooi_findings.py index 8d7b73dd2e..6b2ad33e41 100644 --- a/rocky/rocky/views/ooi_findings.py +++ b/rocky/rocky/views/ooi_findings.py @@ -1,4 +1,5 @@ from django.utils.translation import gettext_lazy as _ +from django.views.generic import TemplateView from tools.forms.base import ObservedAtForm from tools.view_helpers import Breadcrumb, get_ooi_url @@ -6,7 +7,7 @@ from rocky.views.ooi_view import BaseOOIDetailView -class OOIFindingListView(OOIFindingManager, BaseOOIDetailView): +class OOIFindingListView(OOIFindingManager, BaseOOIDetailView, TemplateView): template_name = "oois/ooi_findings.html" connector_form_class = ObservedAtForm diff --git a/rocky/rocky/views/ooi_report.py b/rocky/rocky/views/ooi_report.py index 3b2c6e9882..23488c1731 100644 --- a/rocky/rocky/views/ooi_report.py +++ b/rocky/rocky/views/ooi_report.py @@ -7,6 +7,7 @@ from django.shortcuts import redirect from django.urls import reverse from django.utils.translation import gettext_lazy as _ +from django.views.generic import TemplateView from katalogus.client import get_katalogus from tools.forms.ooi import OOIReportSettingsForm from tools.models import Organization @@ -36,7 +37,7 @@ from rocky.views.ooi_view import BaseOOIDetailView -class OOIReportView(BaseOOIDetailView): +class OOIReportView(BaseOOIDetailView, TemplateView): template_name = "oois/ooi_report.html" connector_form_class = OOIReportSettingsForm @@ -83,10 +84,16 @@ def get(self, request, *args, **kwargs): ), ) except GeneratingReportFailed: - messages.error(self.request, _("Generating report failed. See Keiko logs for more information.")) + messages.error( + self.request, + _("Generating report failed. See Keiko logs for more information."), + ) return redirect(get_ooi_url("ooi_report", ooi.primary_key, self.organization.code)) except ReportNotFoundException: - messages.error(self.request, _("Timeout reached generating report. See Keiko logs for more information.")) + messages.error( + self.request, + _("Timeout reached generating report. See Keiko logs for more information."), + ) return redirect(get_ooi_url("ooi_report", ooi.primary_key, self.organization.code)) return FileResponse( @@ -135,10 +142,16 @@ def get(self, request, *args, **kwargs): ), ) except GeneratingReportFailed: - messages.error(request, _("Generating report failed. See Keiko logs for more information.")) + messages.error( + request, + _("Generating report failed. See Keiko logs for more information."), + ) return redirect(reverse("finding_list", kwargs={"organization_code": self.organization.code})) except ReportNotFoundException: - messages.error(request, _("Timeout reached generating report. See Keiko logs for more information.")) + messages.error( + request, + _("Timeout reached generating report. See Keiko logs for more information."), + ) return redirect(reverse("finding_list", kwargs={"organization_code": self.organization.code})) return FileResponse( diff --git a/rocky/rocky/views/ooi_tree.py b/rocky/rocky/views/ooi_tree.py index 032abe8aa4..e943980581 100644 --- a/rocky/rocky/views/ooi_tree.py +++ b/rocky/rocky/views/ooi_tree.py @@ -1,4 +1,5 @@ from django.utils.translation import gettext_lazy as _ +from django.views.generic import TemplateView from tools.forms.ooi import OoiTreeSettingsForm from tools.ooi_helpers import create_object_tree_item_from_ref, filter_ooi_tree, get_ooi_types_from_tree from tools.view_helpers import Breadcrumb, get_ooi_url @@ -6,7 +7,7 @@ from rocky.views.ooi_view import BaseOOIDetailView -class OOITreeView(BaseOOIDetailView): +class OOITreeView(BaseOOIDetailView, TemplateView): template_name = "oois/ooi_tree.html" connector_form_class = OoiTreeSettingsForm diff --git a/rocky/rocky/views/ooi_view.py b/rocky/rocky/views/ooi_view.py index d48205d9d9..c5e5050fb5 100644 --- a/rocky/rocky/views/ooi_view.py +++ b/rocky/rocky/views/ooi_view.py @@ -1,12 +1,12 @@ from datetime import datetime, timezone from time import sleep -from typing import Any -from django import forms, http +from django import forms +from django.http import Http404 from django.shortcuts import redirect from django.urls import reverse from django.utils.translation import gettext_lazy as _ -from django.views.generic import ListView, TemplateView +from django.views.generic import ListView from django.views.generic.edit import FormView from pydantic import ValidationError from tools.forms.base import BaseRockyForm, ObservedAtForm @@ -101,17 +101,33 @@ def get_context_data(self, **kwargs): return context -class BaseOOIDetailView(SingleOOITreeMixin, BreadcrumbsMixin, ConnectorFormMixin, TemplateView): - def get(self, request: http.HttpRequest, *args: Any, **kwargs: Any) -> http.HttpResponse: +class BaseOOIDetailView(BreadcrumbsMixin, SingleOOITreeMixin, ConnectorFormMixin): + connector_form_class = ObservedAtForm + + def setup(self, request, *args, **kwargs): + super().setup(request, *args, **kwargs) self.ooi = self.get_ooi() - return super().get(request, *args, **kwargs) + + def get_current_ooi(self) -> OOI | None: + """ + Some OOIs have an old valid time, this will fetch the latest OOI for today. + """ + now = datetime.now(timezone.utc) + if self.observed_at.date() == now.date(): + return self.ooi + try: + return self.get_ooi(pk=self.get_ooi_id(), observed_at=now) + except Http404: + return None def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["ooi"] = self.ooi + context["ooi_current"] = self.get_current_ooi() context["mandatory_fields"] = get_mandatory_fields(self.request) context["observed_at"] = self.observed_at + context["observed_at_form"] = self.get_connector_form() return context @@ -165,8 +181,13 @@ def get_form_kwargs(self): def form_valid(self, form): # Transform into OOI try: - new_ooi = self.ooi_class.model_validate(form.cleaned_data) - create_ooi(self.octopoes_api_connector, self.bytes_client, new_ooi, datetime.now(timezone.utc)) + new_ooi = self.ooi_class.parse_obj(form.cleaned_data) + create_ooi( + self.octopoes_api_connector, + self.bytes_client, + new_ooi, + datetime.now(timezone.utc), + ) sleep(1) return redirect(self.get_ooi_success_url(new_ooi)) except ValidationError as exception: diff --git a/rocky/rocky/views/page_actions.py b/rocky/rocky/views/page_actions.py new file mode 100644 index 0000000000..2b007302aa --- /dev/null +++ b/rocky/rocky/views/page_actions.py @@ -0,0 +1,32 @@ +from enum import Enum +from typing import Any + +from django.contrib import messages +from django.http import HttpRequest, HttpResponse +from django.utils.translation import gettext_lazy as _ +from django.views.generic.edit import ProcessFormView + + +class PageActions(Enum): + START_SCAN = "start_scan" + SUBMIT_ANSWER = "submit_answer" + RESCHEDULE_TASK = "reschedule_task" + CHANGE_CLEARANCE_LEVEL = "change_clearance_level" + SCAN_OOIS = "scan_oois" + + +class PageActionsView(ProcessFormView): + def setup(self, request, *args, **kwargs): + super().setup(request, *args, **kwargs) + self.START_SCAN = PageActions.START_SCAN.value + self.SUBMIT_ANSWER = PageActions.SUBMIT_ANSWER.value + self.RESCHEDULE_TASK = PageActions.RESCHEDULE_TASK.value + self.CHANGE_CLEARANCE_LEVEL = PageActions.CHANGE_CLEARANCE_LEVEL.value + self.SCAN_OOIS = PageActions.SCAN_OOIS.value + self.action = request.POST.get("action") + + def post(self, request: HttpRequest, *args: str, **kwargs: Any) -> HttpResponse: + if not self.action or self.action is None: + messages.error(self.request, _("Could not process your request, action required.")) + + return self.get(request, *args, **kwargs) diff --git a/rocky/rocky/views/scan_profile.py b/rocky/rocky/views/scan_profile.py index 0df282c530..0be16e7e67 100644 --- a/rocky/rocky/views/scan_profile.py +++ b/rocky/rocky/views/scan_profile.py @@ -6,23 +6,19 @@ from django.utils.translation import gettext_lazy as _ from django.views.generic import FormView from tools.forms.ooi import SetClearanceLevelForm -from tools.models import Indemnification from tools.view_helpers import Breadcrumb, get_mandatory_fields, get_ooi_url from octopoes.models import EmptyScanProfile, InheritedScanProfile from rocky.views.ooi_detail import OOIDetailView -class ScanProfileDetailView(OOIDetailView, FormView): +class ScanProfileDetailView(FormView, OOIDetailView): template_name = "scan_profiles/scan_profile_detail.html" form_class = SetClearanceLevelForm def get_context_data(self, **kwargs) -> dict[str, Any]: context = super().get_context_data(**kwargs) context["mandatory_fields"] = get_mandatory_fields(self.request) - context["organization_indemnification"] = Indemnification.objects.filter( - organization=self.organization - ).exists() return context def get_initial(self): diff --git a/rocky/rocky/views/scheduler.py b/rocky/rocky/views/scheduler.py new file mode 100644 index 0000000000..3378fc361b --- /dev/null +++ b/rocky/rocky/views/scheduler.py @@ -0,0 +1,190 @@ +from datetime import datetime +from typing import Any + +from django.contrib import messages +from django.http import JsonResponse +from django.utils.translation import gettext_lazy as _ +from katalogus.client import Boefje, Normalizer +from tools.forms.scheduler import TaskFilterForm + +from octopoes.models import OOI +from rocky.scheduler import Boefje as SchedulerBoefje +from rocky.scheduler import ( + BoefjeTask, + LazyTaskList, + NormalizerTask, + PrioritizedItem, + RawData, + SchedulerError, + Task, + scheduler_client, +) +from rocky.scheduler import Normalizer as SchedulerNormalizer +from rocky.views.mixins import OctopoesView + + +def get_date_time(date: str | None) -> datetime | None: + if date: + return datetime.strptime(date, "%Y-%m-%d") + return None + + +class SchedulerView(OctopoesView): + task_type: str + task_filter_form = TaskFilterForm + + def setup(self, request, *args, **kwargs): + super().setup(request, *args, **kwargs) + self.scheduler_client = scheduler_client(self.organization.code) + + def get_task_filters(self) -> dict[str, Any]: + return { + "scheduler_id": f"{self.task_type}-{self.organization.code}", + "task_type": self.task_type, + "plugin_id": None, # plugin_id present and set at plugin detail + **self.get_form_data(), + } + + def get_form_data(self) -> dict[str, Any]: + form_data = self.get_task_filter_form().data.dict() + return {k: v for k, v in form_data.items() if v} + + def get_task_filter_form(self) -> TaskFilterForm: + return self.task_filter_form(self.request.GET) + + def get_task_list(self) -> LazyTaskList | list[Any]: + try: + return LazyTaskList(self.scheduler_client, **self.get_task_filters()) + except SchedulerError as error: + messages.error(self.request, error.message) + return [] + + def get_task_details(self, task_id: str) -> Task | None: + try: + return self.scheduler_client.get_task_details(task_id) + except SchedulerError as error: + return messages.error(self.request, error.message) + + def get_task_statistics(self) -> dict[Any, Any]: + stats = {} + try: + stats = self.scheduler_client.get_task_stats(self.task_type) + except SchedulerError as error: + messages.error(self.request, error.message) + return stats + + def get_output_oois(self, task): + try: + return self.octopoes_api_connector.list_origins( + valid_time=task.p_item.data.raw_data.boefje_meta.ended_at, + task_id=task.id, + )[0].result + except IndexError: + return [] + except SchedulerError as error: + messages.error(self.request, error.message) + return [] + + def get_json_task_details(self) -> JsonResponse | None: + try: + task = self.get_task_details(self.kwargs["task_id"]) + if task: + return JsonResponse( + { + "oois": self.get_output_oois(task), + "valid_time": task.p_item.data.raw_data.boefje_meta.ended_at.strftime("%Y-%m-%dT%H:%M:%S"), + }, + safe=False, + ) + return task + except SchedulerError as error: + return messages.error(self.request, error.message) + + def schedule_task(self, p_item: PrioritizedItem) -> None: + if not self.indemnification_present: + return self.indemnification_error() + try: + # Remove id attribute of both p_item and p_item.data, since the + # scheduler will create a new task with new id's. However, pydantic + # requires an id attribute to be present in its definition and the + # default set to None when the attribute is optional, otherwise it + # will not serialize the id if it is not present in the definition. + if hasattr(p_item, "id"): + delattr(p_item, "id") + + if hasattr(p_item.data, "id"): + delattr(p_item.data, "id") + + self.scheduler_client.push_task(p_item) + + except SchedulerError as error: + messages.error(self.request, error.message) + else: + messages.success( + self.request, + _( + "Your task is scheduled and will soon be started in the background. " + "Results will be added to the object list when they are in. " + "It may take some time, a refresh of the page may be needed to show the results." + ), + ) + + # FIXME: Tasks should be (re)created with supplied data, not by fetching prior + # task info from the scheduler. Task data should be available from the context + # from which the task is created. + def reschedule_task(self, task_id: str) -> None: + try: + task = self.scheduler_client.get_task_details(task_id) + + new_p_item = PrioritizedItem( + data=task.p_item.data, + priority=1, + ) + + self.schedule_task(new_p_item) + except SchedulerError as error: + messages.error(self.request, error.message) + + def run_normalizer(self, katalogus_normalizer: Normalizer, raw_data: RawData) -> None: + try: + normalizer_task = NormalizerTask( + normalizer=SchedulerNormalizer.model_validate(katalogus_normalizer.model_dump()), + raw_data=raw_data, + ) + + p_item = PrioritizedItem(priority=1, data=normalizer_task) + + self.schedule_task(p_item) + except SchedulerError as error: + messages.error(self.request, error.message) + + def run_boefje(self, katalogus_boefje: Boefje, ooi: OOI | None) -> None: + try: + boefje_task = BoefjeTask( + boefje=SchedulerBoefje.model_validate(katalogus_boefje.model_dump()), + input_ooi=ooi.reference if ooi else None, + organization=self.organization.code, + ) + + p_item = PrioritizedItem(priority=1, data=boefje_task) + + self.schedule_task(p_item) + + except SchedulerError as error: + messages.error(self.request, error.message) + + def run_boefje_for_oois( + self, + boefje: Boefje, + oois: list[OOI], + ) -> None: + try: + if not oois and not boefje.consumes: + self.run_boefje(boefje, None) + + for ooi in oois: + if ooi.scan_profile and ooi.scan_profile.level < boefje.scan_level: + self.can_raise_clearance_level(ooi, boefje.scan_level) + self.run_boefje(boefje, ooi) + except SchedulerError as error: + messages.error(self.request, error.message) diff --git a/rocky/rocky/views/task_detail.py b/rocky/rocky/views/task_detail.py index d46de98121..db08b03317 100644 --- a/rocky/rocky/views/task_detail.py +++ b/rocky/rocky/views/task_detail.py @@ -1,32 +1,41 @@ from django.contrib import messages -from django.http import JsonResponse +from django.http import FileResponse, HttpResponse, JsonResponse +from django.shortcuts import redirect from django.urls import reverse from django.utils.translation import gettext_lazy as _ from django.views.generic import TemplateView -from katalogus.views.mixins import BoefjeMixin, NormalizerMixin -from rocky.scheduler import TaskNotFoundError, client -from rocky.views.mixins import OctopoesView +from rocky.scheduler import SchedulerError +from rocky.views.tasks import SchedulerView -class TaskDetailView(OctopoesView, TemplateView): - def get_task(self, task_id): - try: - return client.get_task_details(self.organization.code, task_id) - except TaskNotFoundError as error: - messages.error(self.request, error.message) +class DownloadTaskDetail(SchedulerView): + def get(self, request, *args, **kwargs): + task_id = kwargs["task_id"] + filename = "task_" + task_id + ".json" + task_details = self.get_task_details(task_id) + if task_details is not None: + response = HttpResponse(FileResponse(task_details.json()), content_type="application/json") + response["Content-Disposition"] = "attachment; filename=" + filename + return response + return redirect(reverse("task_list", kwargs={"organization_code": self.organization.code})) + + +class TaskDetailView(SchedulerView, TemplateView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["task_id"] = kwargs["task_id"] - context["task"] = self.get_task(context["task_id"]) + try: + context["task"] = self.get_task_details(context["task_id"]) + except SchedulerError as error: + messages.error(self.request, error.message) return context -class BoefjeTaskDetailView(BoefjeMixin, TaskDetailView): +class BoefjeTaskDetailView(TaskDetailView): template_name = "tasks/boefje_task_detail.html" - plugin_type = "boefje" def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -48,24 +57,11 @@ def get_context_data(self, **kwargs): return context -class NormalizerTaskJSONView(NormalizerMixin, TaskDetailView): +class NormalizerTaskJSONView(TaskDetailView): plugin_type = "normalizer" - def get_output_oois(self, task): - try: - return self.octopoes_api_connector.list_origins( - valid_time=task.p_item.data.raw_data.boefje_meta.ended_at, task_id=task.id - )[0].result - except IndexError: - return [] - - def get(self, request, *args, **kwargs): - task_id = kwargs["task_id"] - task = self.get_task(task_id) - return JsonResponse( - { - "oois": self.get_output_oois(task), - "valid_time": task.p_item.data.raw_data.boefje_meta.ended_at.strftime("%Y-%m-%dT%H:%M:%S"), - }, - safe=False, - ) + def get(self, request, *args, **kwargs) -> JsonResponse | HttpResponse: + task = self.get_json_task_details() + if task is not None: + return task + return super().get(request, *args, **kwargs) diff --git a/rocky/rocky/views/tasks.py b/rocky/rocky/views/tasks.py index 897f89eec5..8099d03749 100644 --- a/rocky/rocky/views/tasks.py +++ b/rocky/rocky/views/tasks.py @@ -1,117 +1,64 @@ -from datetime import datetime -from enum import Enum +from typing import Any -from account.mixins import OrganizationView from django.contrib import messages -from django.http import FileResponse, HttpResponse -from django.shortcuts import redirect from django.urls import reverse from django.utils.translation import gettext_lazy as _ from django.views.generic.list import ListView from httpx import HTTPError -from katalogus.views.mixins import BoefjeMixin, NormalizerMixin -from tools.view_helpers import reschedule_task from rocky.paginator import RockyPaginator -from rocky.scheduler import SchedulerError, TaskNotFoundError, client +from rocky.scheduler import SchedulerError +from rocky.views.page_actions import PageActionsView +from rocky.views.scheduler import SchedulerView -class PageActions(Enum): - RESCHEDULE_TASK = "reschedule_task" - - -class DownloadTaskDetail(OrganizationView): - def get(self, request, *args, **kwargs): +class SchedulerListView(ListView): + def get_context_data(self, **kwargs: Any) -> dict[str, Any]: try: - task_id = kwargs["task_id"] - filename = "task_" + task_id + ".json" - task_details = client.get_task_details(self.organization.code, task_id) - response = HttpResponse(FileResponse(task_details.json()), content_type="application/json") - response["Content-Disposition"] = "attachment; filename=" + filename - return response - except TaskNotFoundError as error: + return super().get_context_data(**kwargs) + except SchedulerError as error: messages.error(self.request, error.message) - return redirect(reverse("task_list", kwargs={"organization_code": self.organization.code})) + return {} -class TaskListView(OrganizationView, ListView): - paginate_by = 20 +class TaskListView(SchedulerView, SchedulerListView, PageActionsView): paginator_class = RockyPaginator + paginate_by = 20 + context_object_name = "task_list" def get_queryset(self): - scheduler_id = self.plugin_type + "-" + self.organization.code - task_type = self.request.GET.get("type", self.plugin_type) - - status = self.request.GET.get("scan_history_status") if self.request.GET.get("scan_history_status") else None - - input_ooi = self.request.GET.get("scan_history_search") if self.request.GET.get("scan_history_search") else None - - if self.request.GET.get("scan_history_from"): - min_created_at = datetime.strptime(self.request.GET.get("scan_history_from"), "%Y-%m-%d") - else: - min_created_at = None - - if self.request.GET.get("scan_history_to"): - max_created_at = datetime.strptime(self.request.GET.get("scan_history_to"), "%Y-%m-%d") - else: - max_created_at = None - - try: - return client.get_lazy_task_list( - scheduler_id=scheduler_id, - task_type=task_type, - status=status, - min_created_at=min_created_at, - max_created_at=max_created_at, - input_ooi=input_ooi, - ) - - except HTTPError: - error_message = _("Fetching tasks failed: no connection with scheduler") - messages.add_message(self.request, messages.ERROR, error_message) - return [] + return self.get_task_list() def post(self, request, *args, **kwargs): - self.handle_page_action(request.POST["action"]) - - return redirect(request.path) - - def handle_page_action(self, action: str) -> None: - if action == PageActions.RESCHEDULE_TASK.value: - task_id = self.request.POST.get("task_id") - reschedule_task(self.request, self.organization.code, task_id) + try: + if self.action == self.RESCHEDULE_TASK: + task_id = self.request.POST.get("task_id", "") + self.reschedule_task(task_id) + except HTTPError as exc: + message = f"HTTP error for {exc.request.url} - {exc}" + messages.error(request, message) + except SchedulerError as error: + messages.error(request, error.message) + return super().post(request, *args, **kwargs) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - try: - context["stats"] = client.get_task_stats(self.organization.code, self.plugin_type) - except SchedulerError: - context["stats_error"] = True - else: - context["stats_error"] = False + context["task_filter_form"] = self.get_task_filter_form() + context["stats"] = self.get_task_statistics() context["breadcrumbs"] = [ - {"url": reverse("task_list", kwargs={"organization_code": self.organization.code}), "text": _("Tasks")}, + { + "url": reverse("task_list", kwargs={"organization_code": self.organization.code}), + "text": _("Tasks"), + }, ] return context -class BoefjesTaskListView(BoefjeMixin, TaskListView): +class BoefjesTaskListView(TaskListView): template_name = "tasks/boefjes.html" - plugin_type = "boefje" + task_type = "boefje" -class NormalizersTaskListView(NormalizerMixin, TaskListView): +class NormalizersTaskListView(TaskListView): template_name = "tasks/normalizers.html" - plugin_type = "normalizer" - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - - context["breadcrumbs"] = [ - { - "url": reverse("task_list", kwargs={"organization_code": self.organization.code}), - "text": _("Tasks"), - }, - ] - - return context + task_type = "normalizer" diff --git a/rocky/tests/conftest.py b/rocky/tests/conftest.py index bb40b78a97..14ee2f7b6d 100644 --- a/rocky/tests/conftest.py +++ b/rocky/tests/conftest.py @@ -39,7 +39,7 @@ from octopoes.models.transaction import TransactionRecord from octopoes.models.tree import ReferenceTree from octopoes.models.types import OOIType -from rocky.scheduler import Task +from rocky.scheduler import PaginatedTasksResponse, Task LANG_LIST = [code for code, _ in settings.LANGUAGES] @@ -159,14 +159,24 @@ def organization_b(): @pytest.fixture def superuser(django_user_model): return create_user( - django_user_model, "superuser@openkat.nl", "SuperSuper123!!", "Superuser name", "default", superuser=True + django_user_model, + "superuser@openkat.nl", + "SuperSuper123!!", + "Superuser name", + "default", + superuser=True, ) @pytest.fixture def superuser_b(django_user_model): return create_user( - django_user_model, "superuserB@openkat.nl", "SuperBSuperB123!!", "Superuser B name", "default_b", superuser=True + django_user_model, + "superuserB@openkat.nl", + "SuperBSuperB123!!", + "Superuser B name", + "default_b", + superuser=True, ) @@ -182,12 +192,24 @@ def superuser_member_b(superuser_b, organization_b): @pytest.fixture def adminuser(django_user_model): - return create_user(django_user_model, "admin@openkat.nl", "AdminAdmin123!!", "Admin name", "default_admin") + return create_user( + django_user_model, + "admin@openkat.nl", + "AdminAdmin123!!", + "Admin name", + "default_admin", + ) @pytest.fixture def adminuser_b(django_user_model): - return create_user(django_user_model, "adminB@openkat.nl", "AdminBAdminB123!!", "Admin B name", "default_admin_b") + return create_user( + django_user_model, + "adminB@openkat.nl", + "AdminBAdminB123!!", + "Admin B name", + "default_admin_b", + ) @pytest.fixture @@ -209,14 +231,22 @@ def admin_member_b(adminuser_b, organization_b): @pytest.fixture def redteamuser(django_user_model): return create_user( - django_user_model, "redteamer@openkat.nl", "RedteamRedteam123!!", "Redteam name", "default_redteam" + django_user_model, + "redteamer@openkat.nl", + "RedteamRedteam123!!", + "Redteam name", + "default_redteam", ) @pytest.fixture def redteamuser_b(django_user_model): return create_user( - django_user_model, "redteamerB@openkat.nl", "RedteamBRedteamB123!!", "Redteam B name", "default_redteam_b" + django_user_model, + "redteamerB@openkat.nl", + "RedteamBRedteamB123!!", + "Redteam B name", + "default_redteam_b", ) @@ -236,13 +266,23 @@ def redteam_member_b(redteamuser_b, organization_b): @pytest.fixture def clientuser(django_user_model): - return create_user(django_user_model, "client@openkat.nl", "ClientClient123!!", "Client name", "default_client") + return create_user( + django_user_model, + "client@openkat.nl", + "ClientClient123!!", + "Client name", + "default_client", + ) @pytest.fixture def clientuser_b(django_user_model): return create_user( - django_user_model, "clientB@openkat.nl", "ClientBClientB123!!", "Client B name", "default_client_b" + django_user_model, + "clientB@openkat.nl", + "ClientBClientB123!!", + "Client B name", + "default_client_b", ) @@ -271,7 +311,13 @@ def client_user_two_organizations(clientuser, organization, organization_b): @pytest.fixture def new_member(django_user_model, organization): - user = create_user(django_user_model, "cl1@openkat.nl", "TestTest123!!", "New user", "default_new_user") + user = create_user( + django_user_model, + "cl1@openkat.nl", + "TestTest123!!", + "New user", + "default_new_user", + ) member = create_member(user, organization) member.status = OrganizationMember.STATUSES.NEW member.save() @@ -280,7 +326,13 @@ def new_member(django_user_model, organization): @pytest.fixture def active_member(django_user_model, organization): - user = create_user(django_user_model, "cl2@openkat.nl", "TestTest123!!", "Active user", "default_active_user") + user = create_user( + django_user_model, + "cl2@openkat.nl", + "TestTest123!!", + "Active user", + "default_active_user", + ) member = create_member(user, organization) member.status = OrganizationMember.STATUSES.ACTIVE member.save() @@ -289,7 +341,13 @@ def active_member(django_user_model, organization): @pytest.fixture def blocked_member(django_user_model, organization): - user = create_user(django_user_model, "cl3@openkat.nl", "TestTest123!!", "Blocked user", "default_blocked_user") + user = create_user( + django_user_model, + "cl3@openkat.nl", + "TestTest123!!", + "Blocked user", + "default_blocked_user", + ) member = create_member(user, organization) member.status = OrganizationMember.STATUSES.ACTIVE member.blocked = True @@ -446,7 +504,9 @@ def network() -> Network: def url(network) -> URL: return URL( scan_profile=DeclaredScanProfile( - scan_profile_type="declared", reference=Reference("URL|testnetwork|http://example.com/"), level=ScanLevel.L1 + scan_profile_type="declared", + reference=Reference("URL|testnetwork|http://example.com/"), + level=ScanLevel.L1, ), primary_key="URL|testnetwork|http://example.com/", network=network.reference, @@ -1297,7 +1357,7 @@ def setup_request(request, user): @pytest.fixture def mock_scheduler(mocker): - return mocker.patch("rocky.views.ooi_detail.scheduler.client") + return mocker.patch("rocky.views.scheduler.scheduler_client")() def get_stub_path(file_name: str) -> Path: @@ -1322,8 +1382,8 @@ def mock_mixins_katalogus(mocker): @pytest.fixture -def mock_scheduler_client_task_list(mocker): - mock_scheduler_client_session = mocker.patch("rocky.scheduler.client._client") +def mock_scheduler_client_task_list(mock_scheduler): + mock_scheduler_session = mock_scheduler._client response = Response( 200, content=( @@ -1362,9 +1422,9 @@ def mock_scheduler_client_task_list(mocker): ), ) - mock_scheduler_client_session.get.return_value = response + mock_scheduler_session.get.return_value = response - return mock_scheduler_client_session + return mock_scheduler_session class MockOctopoesAPIConnector: @@ -1379,7 +1439,11 @@ def get(self, reference: Reference, valid_time: datetime | None = None) -> OOI: return self.oois[reference] def get_tree( - self, reference: Reference, valid_time: datetime, types: set = frozenset(), depth: int = 1 + self, + reference: Reference, + valid_time: datetime, + types: set = frozenset(), + depth: int = 1, ) -> ReferenceTree: return self.tree[reference] @@ -1457,6 +1521,16 @@ def listed_hostnames(network) -> list[Hostname]: ] +@pytest.fixture +def paginated_task_list(task): + return PaginatedTasksResponse( + count=1, + next="", + previous=None, + results=[task], + ) + + @pytest.fixture def reports_more_input_oois(): return [ diff --git a/rocky/tests/integration/test_bench.py b/rocky/tests/integration/test_bench.py index d8c722c33f..9c55e17263 100644 --- a/rocky/tests/integration/test_bench.py +++ b/rocky/tests/integration/test_bench.py @@ -7,7 +7,7 @@ @pytest.mark.slow -def test_aggregate_report_benchmark(octopoes_api_connector, valid_time): +def test_aggregate_report_benchmark(octopoes_api_connector, valid_time, organization): hostname_range = range(0, 20) for x in hostname_range: seed_system( @@ -26,6 +26,7 @@ def test_aggregate_report_benchmark(octopoes_api_connector, valid_time): [Hostname(name=f"{x}.com", network=Network(name="test").reference) for x in hostname_range], reports, valid_time, + organization.code, ) assert data["systems"] diff --git a/rocky/tests/integration/test_reports.py b/rocky/tests/integration/test_reports.py index d18bf7acbb..44b49a5c7a 100644 --- a/rocky/tests/integration/test_reports.py +++ b/rocky/tests/integration/test_reports.py @@ -93,14 +93,14 @@ def test_system_report(octopoes_api_connector: OctopoesAPIConnector, valid_time) } -def test_aggregate_report(octopoes_api_connector: OctopoesAPIConnector, valid_time, hostname_oois): +def test_aggregate_report(octopoes_api_connector: OctopoesAPIConnector, valid_time, hostname_oois, organization): seed_system(octopoes_api_connector, valid_time) reports: list[type[Report] | type[MultiReport]] = ( AggregateOrganisationReport.reports["required"] + AggregateOrganisationReport.reports["optional"] ) report_ids = [report_type.id for report_type in reports] - _, data, _, _ = aggregate_reports(octopoes_api_connector, hostname_oois, report_ids, valid_time) + _, data, _, _ = aggregate_reports(octopoes_api_connector, hostname_oois, report_ids, valid_time, organization.code) v4_test_hostnames = [ "Hostname|test|a.example.com", @@ -245,14 +245,19 @@ def test_multi_report( octopoes_api_connector_2: OctopoesAPIConnector, valid_time, hostname_oois, + organization, ): seed_system(octopoes_api_connector, valid_time) seed_system(octopoes_api_connector_2, valid_time) reports = AggregateOrganisationReport.reports["required"] + AggregateOrganisationReport.reports["optional"] report_ids = [report_type.id for report_type in reports] - _, data, report_data, _ = aggregate_reports(octopoes_api_connector, hostname_oois, report_ids, valid_time) - _, data_2, report_data_2, _ = aggregate_reports(octopoes_api_connector_2, hostname_oois, report_ids, valid_time) + _, data, report_data, _ = aggregate_reports( + octopoes_api_connector, hostname_oois, report_ids, valid_time, organization.code + ) + _, data_2, report_data_2, _ = aggregate_reports( + octopoes_api_connector_2, hostname_oois, report_ids, valid_time, organization.code + ) report_data_object = ReportData( organization_code=octopoes_api_connector.client, diff --git a/rocky/tests/katalogus/test_katalogus_plugin_detail.py b/rocky/tests/katalogus/test_katalogus_plugin_detail.py index 03d717aa88..1585f32f43 100644 --- a/rocky/tests/katalogus/test_katalogus_plugin_detail.py +++ b/rocky/tests/katalogus/test_katalogus_plugin_detail.py @@ -22,7 +22,9 @@ def test_plugin_detail_view( ) assertContains(response, "TestBoefje") - assertContains(response, "Completed") + assertContains(response, "Produces") + assertContains(response, "Tasks") + assertContains(response, "Object list") assertContains(response, "Consumes") assertContains(response, plugin_details.description) diff --git a/rocky/tests/objects/test_objects_detail.py b/rocky/tests/objects/test_objects_detail.py index 89e5509ab2..71d5d4d63a 100644 --- a/rocky/tests/objects/test_objects_detail.py +++ b/rocky/tests/objects/test_objects_detail.py @@ -1,7 +1,6 @@ from urllib.parse import urlencode import pytest -from django.http import HttpResponseRedirect from katalogus.client import Boefje from pytest_django.asserts import assertContains, assertNotContains from tools.enums import SCAN_LEVEL @@ -56,29 +55,31 @@ def test_ooi_detail( rf, client_member, - mock_scheduler, mock_organization_view_octopoes, - lazy_task_list_with_boefje, + mock_scheduler, + paginated_task_list, mocker, ): mocker.patch("katalogus.client.KATalogusClientV1") - request = setup_request(rf.get("ooi_detail", {"ooi_id": "Network|testnetwork"}), client_member.user) - mock_organization_view_octopoes().get_tree.return_value = ReferenceTree.model_validate(TREE_DATA) - mock_scheduler.get_lazy_task_list.return_value = lazy_task_list_with_boefje + + mock_scheduler.list_tasks.return_value = paginated_task_list + + request = setup_request(rf.get("ooi_detail", {"ooi_id": "Network|testnetwork"}), client_member.user) response = OOIDetailView.as_view()(request, organization_code=client_member.organization.code) assert response.status_code == 200 assert mock_organization_view_octopoes().get_tree.call_count == 2 assertContains(response, "Object") - assertContains(response, "Hostname|internet|mispo.es") + assertContains(response, "Network|testnetwork") assertContains(response, "Plugin") - assertContains(response, "test-boefje") + assertContains(response, "TestBoefje") assertContains( - response, f'href="/en/{client_member.organization.code}/kat-alogus/plugins/boefje/test-boefje/">TestBoefje' + response, + f'href="/en/{client_member.organization.code}/kat-alogus/plugins/boefje/test-boefje/">TestBoefje', ) assertContains(response, "Status") assertContains(response, "Completed") @@ -91,17 +92,19 @@ def test_ooi_detail( def test_question_detail( rf, client_member, - mock_scheduler, mock_organization_view_octopoes, - lazy_task_list_with_boefje, + mock_scheduler, + paginated_task_list, mocker, ): mocker.patch("katalogus.client.KATalogusClientV1") - request = setup_request(rf.get("ooi_detail", {"ooi_id": "Question|/test|Network|testnetwork"}), client_member.user) + request = setup_request( + rf.get("ooi_detail", {"ooi_id": "Question|/test|Network|testnetwork"}), + client_member.user, + ) mock_organization_view_octopoes().get_tree.return_value = ReferenceTree.model_validate(QUESTION_DATA) - mock_scheduler.get_lazy_task_list.return_value = lazy_task_list_with_boefje response = OOIDetailView.as_view()(request, organization_code=client_member.organization.code) @@ -124,7 +127,6 @@ def test_answer_question( ): mocker.patch("katalogus.client.KATalogusClientV1") mock_organization_view_octopoes().get_tree.return_value = ReferenceTree.model_validate(QUESTION_DATA) - mock_scheduler.get_lazy_task_list.return_value = lazy_task_list_with_boefje query_string = urlencode({"ooi_id": "Question|/test|Network|testnetwork"}, doseq=True) request = setup_request( @@ -139,8 +141,8 @@ def test_answer_question( ) response = OOIDetailView.as_view()(request, organization_code=client_member.organization.code) - assertContains(response, "Question has been answered.", status_code=201) - assert mock_organization_view_octopoes().get_tree.call_count == 3 + assertContains(response, "Question has been answered.", status_code=200) + assert mock_organization_view_octopoes().get_tree.call_count == 2 def test_answer_question_bad_schema( @@ -154,7 +156,6 @@ def test_answer_question_bad_schema( ): mocker.patch("katalogus.client.KATalogusClientV1") mock_organization_view_octopoes().get_tree.return_value = ReferenceTree.model_validate(QUESTION_DATA) - mock_scheduler.get_lazy_task_list.return_value = lazy_task_list_with_boefje query_string = urlencode({"ooi_id": "Question|/test|Network|testnetwork"}, doseq=True) @@ -170,24 +171,26 @@ def test_answer_question_bad_schema( ) response = OOIDetailView.as_view()(request, organization_code=client_member.organization.code) - assert response.status_code == 422 + assert response.status_code == 200 quote_enc = "'" - assertContains(response, f"314159 is not of type {quote_enc}string{quote_enc}", status_code=422) + assertContains(response, f"314159 is not of type {quote_enc}string{quote_enc}", status_code=200) def test_ooi_detail_start_scan( rf, client_member, mock_organization_view_octopoes, + mock_scheduler, + paginated_task_list, mocker, network, ): mock_katalogus = mocker.patch("katalogus.client.KATalogusClientV1") - mocker.patch("katalogus.views.mixins.schedule_task") mock_organization_view_octopoes().get_tree.return_value = ReferenceTree.model_validate(TREE_DATA) mock_organization_view_octopoes().get.return_value = network + mock_katalogus().get_plugin.return_value = Boefje( id="nmap", name="", @@ -214,18 +217,17 @@ def test_ooi_detail_start_scan( ) response = OOIDetailView.as_view()(request, organization_code=client_member.organization.code) - assert mock_organization_view_octopoes().get_tree.call_count == 1 - assert isinstance(response, HttpResponseRedirect) - assert response.status_code == 302 - assert response.url == f"/en/{client_member.organization.code}/tasks/" + assert mock_organization_view_octopoes().get_tree.call_count == 2 + + assert response.status_code == 200 def test_ooi_detail_start_scan_no_indemnification( rf, client_member, - mock_scheduler, mock_organization_view_octopoes, - lazy_task_list_with_boefje, + mock_scheduler, + paginated_task_list, mocker, network, ): @@ -233,6 +235,17 @@ def test_ooi_detail_start_scan_no_indemnification( mock_organization_view_octopoes().get_tree.return_value = ReferenceTree.model_validate(TREE_DATA) mock_organization_view_octopoes().get.return_value = network + mock_katalogus = mocker.patch("katalogus.client.KATalogusClientV1") + mock_katalogus().get_plugin.return_value = Boefje( + id="nmap", + name="", + description="", + enabled=True, + type="boefje", + scan_level=SCAN_LEVEL.L2, + consumes=[], + produces=[], + ) Indemnification.objects.get(user=client_member.user).delete() @@ -242,7 +255,7 @@ def test_ooi_detail_start_scan_no_indemnification( rf.post( f"/en/{client_member.organization.code}/objects/details/?{query_string}", data={ - "boefje_id": "nmap", + "boefje_id": "test-boefje", "action": "start_scan", }, ), @@ -251,8 +264,8 @@ def test_ooi_detail_start_scan_no_indemnification( response = OOIDetailView.as_view()(request, organization_code=client_member.organization.code) assert mock_organization_view_octopoes().get_tree.call_count == 2 - assertContains(response, "Object details", status_code=403) - assertContains(response, "Indemnification not present", status_code=403) + assertContains(response, "Object details") + assertContains(response, "Indemnification not present") def test_ooi_detail_start_scan_no_action( @@ -283,7 +296,7 @@ def test_ooi_detail_start_scan_no_action( response = OOIDetailView.as_view()(request, organization_code=client_member.organization.code) assert mock_organization_view_octopoes().get_tree.call_count == 2 - assertContains(response, "Object details", status_code=404) + assertContains(response, "Object details") @pytest.mark.parametrize("member", ["superuser_member", "admin_member", "redteam_member"]) diff --git a/rocky/tests/objects/test_objects_findings.py b/rocky/tests/objects/test_objects_findings.py index 1f60f08c5a..c0d9dcaadc 100644 --- a/rocky/tests/objects/test_objects_findings.py +++ b/rocky/tests/objects/test_objects_findings.py @@ -109,7 +109,10 @@ def test_mute_finding_button_is_not_visible_without_perms( def test_mute_finding_form_view(request, member, rf, mock_organization_view_octopoes): member = request.getfixturevalue(member) response = MuteFindingView.as_view()( - setup_request(rf.get("finding_mute", {"ooi_id": "Finding|Network|testnetwork|KAT-000"}), member.user), + setup_request( + rf.get("finding_mute", {"ooi_id": "Finding|Network|testnetwork|KAT-000"}), + member.user, + ), organization_code=member.organization.code, ) @@ -126,13 +129,12 @@ def test_mute_finding_form_view_no_perms(request, member, rf, mock_organization_ member = request.getfixturevalue(member) with pytest.raises(PermissionDenied): MuteFindingView.as_view()( - setup_request(rf.get("finding_mute", {"ooi_id": "Finding|Network|testnetwork|KAT-000"}), member.user), + setup_request( + rf.get("finding_mute", {"ooi_id": "Finding|Network|testnetwork|KAT-000"}), + member.user, + ), organization_code=member.organization.code, ) - with pytest.raises(PermissionDenied): - MuteFindingView.as_view()( - setup_request(rf.post("finding_mute"), member.user), organization_code=member.organization.code - ) def test_mute_finding_post( @@ -159,7 +161,9 @@ def test_mute_finding_post( ) # Uses same ooi_add post request to add a MuteFinding object response = OOIAddView.as_view()( - request, organization_code=redteam_member.organization.code, ooi_type="MutedFinding" + request, + organization_code=redteam_member.organization.code, + ooi_type="MutedFinding", ) # Redirects to ooi_detail @@ -253,8 +257,16 @@ def test_muted_finding_button_presence_more_findings_and_post( ) assert response.status_code == 200 - assertContains(response, '', html=True) - assertContains(response, '', html=True) + assertContains( + response, + '', + html=True, + ) + assertContains( + response, + '', + html=True, + ) assertContains(response, '') request = setup_request( diff --git a/rocky/tests/objects/test_objects_scan_profile.py b/rocky/tests/objects/test_objects_scan_profile.py index 2534971b47..1c2f6fa5aa 100644 --- a/rocky/tests/objects/test_objects_scan_profile.py +++ b/rocky/tests/objects/test_objects_scan_profile.py @@ -10,7 +10,14 @@ TREE_DATA = { "root": { "reference": "Network|testnetwork", - "children": {"urls": [{"reference": "HostnameHTTPURL|https|internet|scanme.org|443|/", "children": {}}]}, + "children": { + "urls": [ + { + "reference": "HostnameHTTPURL|https|internet|scanme.org|443|/", + "children": {}, + } + ] + }, }, "store": { "Network|testnetwork": { @@ -45,7 +52,10 @@ def test_scan_profile(rf, redteam_member, mock_scheduler, mock_organization_view mocker.patch("katalogus.utils.get_katalogus") mock_organization_view_octopoes().get_tree.return_value = ReferenceTree.model_validate(TREE_DATA) - request = setup_request(rf.get("scan_profile_detail", {"ooi_id": "Network|testnetwork"}), redteam_member.user) + request = setup_request( + rf.get("scan_profile_detail", {"ooi_id": "Network|testnetwork"}), + redteam_member.user, + ) response = ScanProfileDetailView.as_view()(request, organization_code=redteam_member.organization.code) assert response.status_code == 200 @@ -84,12 +94,19 @@ def test_scan_profile_submit_no_indemnification( # Passing query params in POST requests is not well-supported for RequestFactory it seems, hence the absolute path query_string = urlencode({"ooi_id": "Network|testnetwork"}, doseq=True) request = setup_request( - rf.post(f"/en/{redteam_member.organization.code}/objects/scan-profile/?{query_string}", data={"level": "L1"}), + rf.post( + f"/en/{redteam_member.organization.code}/objects/scan-profile/?{query_string}", + data={"level": "1", "action": "change_clearance_level"}, + ), redteam_member.user, ) response = ScanProfileDetailView.as_view()(request, organization_code=redteam_member.organization.code) - assert response.status_code == 403 + assert response.status_code == 200 + assertContains( + response, + "Indemnification not present at organization " + redteam_member.organization.name, + ) def test_scan_profile_no_permissions_acknowledged( @@ -101,7 +118,10 @@ def test_scan_profile_no_permissions_acknowledged( redteam_member.acknowledged_clearance_level = -1 redteam_member.save() - request = setup_request(rf.get("scan_profile_detail", {"ooi_id": "Network|testnetwork"}), redteam_member.user) + request = setup_request( + rf.get("scan_profile_detail", {"ooi_id": "Network|testnetwork"}), + redteam_member.user, + ) response = ScanProfileDetailView.as_view()(request, organization_code=redteam_member.organization.code) assert response.status_code == 200 @@ -119,7 +139,10 @@ def test_scan_profile_no_permissions_trusted( redteam_member.trusted_clearance_level = -1 redteam_member.save() - request = setup_request(rf.get("scan_profile_detail", {"ooi_id": "Network|testnetwork"}), redteam_member.user) + request = setup_request( + rf.get("scan_profile_detail", {"ooi_id": "Network|testnetwork"}), + redteam_member.user, + ) response = ScanProfileDetailView.as_view()(request, organization_code=redteam_member.organization.code) assert response.status_code == 200 @@ -132,7 +155,10 @@ def test_scan_profile_reset_view(rf, redteam_member, mock_scheduler, mock_organi mock_organization_view_octopoes().get_tree.return_value = ReferenceTree.model_validate(TREE_DATA) mocker.patch("katalogus.utils.get_katalogus") - request = setup_request(rf.get("scan_profile_reset", {"ooi_id": "Network|testnetwork"}), redteam_member.user) + request = setup_request( + rf.get("scan_profile_reset", {"ooi_id": "Network|testnetwork"}), + redteam_member.user, + ) response = ScanProfileResetView.as_view()(request, organization_code=redteam_member.organization.code) assert response.status_code == 200 diff --git a/rocky/tests/scheduler/__init__.py b/rocky/tests/scheduler/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/rocky/tests/scheduler/test_scheduler_errors.py b/rocky/tests/scheduler/test_scheduler_errors.py new file mode 100644 index 0000000000..b9229216aa --- /dev/null +++ b/rocky/tests/scheduler/test_scheduler_errors.py @@ -0,0 +1,39 @@ +from rocky.scheduler import SchedulerConnectError, SchedulerTooManyRequestError, SchedulerValidationError +from rocky.views.tasks import BoefjesTaskListView +from tests.conftest import setup_request + + +def test_tasks_view_connect_error(rf, client_member, mock_scheduler): + mock_scheduler.list_tasks.side_effect = SchedulerConnectError + + request = setup_request(rf.get("boefjes_task_list"), client_member.user) + response = BoefjesTaskListView.as_view()(request, organization_code=client_member.organization.code) + + assert response.status_code == 200 + + assert list(request._messages)[0].message == "Could not connect to Scheduler. Service is possibly down." + + +def test_tasks_view_validation_error(rf, client_member, mock_scheduler): + mock_scheduler.list_tasks.side_effect = SchedulerValidationError + + request = setup_request(rf.get("boefjes_task_list"), client_member.user) + response = BoefjesTaskListView.as_view()(request, organization_code=client_member.organization.code) + + assert response.status_code == 200 + + assert list(request._messages)[0].message == "Your request could not be validated." + + +def test_tasks_view_too_many_requests_error(rf, client_member, mock_scheduler): + mock_scheduler.list_tasks.side_effect = SchedulerTooManyRequestError + + request = setup_request(rf.get("boefjes_task_list"), client_member.user) + response = BoefjesTaskListView.as_view()(request, organization_code=client_member.organization.code) + + assert response.status_code == 200 + + assert ( + list(request._messages)[0].message + == "Scheduler is receiving too many requests. Increase SCHEDULER_PQ_MAXSIZE or wait for task to finish." + ) diff --git a/rocky/tests/test_boefjes_tasks.py b/rocky/tests/test_boefjes_tasks.py index 88ebe4a10f..6d62ad5748 100644 --- a/rocky/tests/test_boefjes_tasks.py +++ b/rocky/tests/test_boefjes_tasks.py @@ -1,79 +1,38 @@ -from unittest.mock import call - import pytest from django.http import Http404 -from httpx import HTTPError from pytest_django.asserts import assertContains -from rocky.scheduler import TooManyRequestsError +from rocky.scheduler import SchedulerTooManyRequestError from rocky.views.bytes_raw import BytesRawView from rocky.views.tasks import BoefjesTaskListView from tests.conftest import setup_request -def test_boefjes_tasks(rf, client_member, mocker, lazy_task_list_empty): - mock_scheduler_client = mocker.patch("rocky.views.tasks.client") - mock_scheduler_client.get_lazy_task_list.return_value = lazy_task_list_empty - - request = setup_request(rf.get("boefjes_task_list"), client_member.user) - response = BoefjesTaskListView.as_view()(request, organization_code=client_member.organization.code) - - assert response.status_code == 200 - - mock_scheduler_client.get_lazy_task_list.assert_has_calls( - [ - call( - scheduler_id="boefje-test", - task_type="boefje", - status=None, - min_created_at=None, - max_created_at=None, - input_ooi=None, - ) - ] - ) - - -def test_tasks_view_simple(rf, client_member, mocker, lazy_task_list_with_boefje): - mock_scheduler_client = mocker.patch("rocky.views.tasks.client") - mock_scheduler_client.get_lazy_task_list.return_value = lazy_task_list_with_boefje - +def test_boefjes_tasks(rf, client_member, mock_scheduler): request = setup_request(rf.get("boefjes_task_list"), client_member.user) - response = BoefjesTaskListView.as_view()(request, organization_code=client_member.organization.code) - - assertContains(response, "1b20f85f") - assertContains(response, "Hostname|internet|mispo.es") - - mock_scheduler_client.get_lazy_task_list.assert_has_calls( - [ - call( - scheduler_id="boefje-test", - task_type="boefje", - status=None, - min_created_at=None, - max_created_at=None, - input_ooi=None, - ) - ] + response = BoefjesTaskListView.as_view()( + request, + organization_code=client_member.organization.code, + scheduler_id="boefje-test", + task_type="boefje", + status=None, + min_created_at=None, + max_created_at=None, + input_ooi=None, ) + assert response.status_code == 200 -def test_tasks_view_error(rf, client_member, mocker, lazy_task_list_with_boefje): - mock_scheduler_client = mocker.patch("rocky.views.tasks.client") - mock_scheduler_client.get_lazy_task_list.return_value = lazy_task_list_with_boefje - mock_scheduler_client.get_lazy_task_list.side_effect = HTTPError("error") +def test_tasks_view_simple(rf, client_member, mock_scheduler, mock_scheduler_client_task_list): request = setup_request(rf.get("boefjes_task_list"), client_member.user) response = BoefjesTaskListView.as_view()(request, organization_code=client_member.organization.code) - assertContains(response, "error") - assertContains(response, "Fetching tasks failed") + assertContains(response, "Completed") -def test_reschedule_task(rf, client_member, mocker, task): - mock_scheduler = mocker.patch("tools.view_helpers.client") +def test_reschedule_task(rf, client_member, mock_scheduler, task): mock_scheduler.get_task_details.return_value = task - request = setup_request( rf.post( f"/en/{client_member.organization.code}/tasks/boefjes/?task_id={task.id}", @@ -83,7 +42,7 @@ def test_reschedule_task(rf, client_member, mocker, task): ) response = BoefjesTaskListView.as_view()(request, organization_code=client_member.organization.code) - assert response.status_code == 302 + assert response.status_code == 200 assert list(request._messages)[0].message == ( "Your task is scheduled and will soon be started in the background. " "Results will be added to the object list when they are in. " @@ -91,10 +50,9 @@ def test_reschedule_task(rf, client_member, mocker, task): ) -def test_reschedule_task_already_queued(rf, client_member, mocker, task): - mock_scheduler = mocker.patch("tools.view_helpers.client") +def test_reschedule_task_already_queued(rf, client_member, mock_scheduler, mocker, task): mock_scheduler.get_task_details.return_value = task - mock_scheduler.push_task.side_effect = TooManyRequestsError + mock_scheduler.push_task.side_effect = SchedulerTooManyRequestError request = setup_request( rf.post( @@ -109,13 +67,15 @@ def test_reschedule_task_already_queued(rf, client_member, mocker, task): organization_code=client_member.organization.code, ) - assert response.status_code == 302 - assert list(request._messages)[0].message == "Task queue is full, please try again later." + assert response.status_code == 200 + assert ( + list(request._messages)[0].message + == "Scheduler is receiving too many requests. Increase SCHEDULER_PQ_MAXSIZE or wait for task to finish." + ) -def test_reschedule_task_from_other_org(rf, client_member, client_member_b, mocker, task): - mock_scheduler_client = mocker.patch("rocky.views.tasks.client") - mock_scheduler_client.get_task_details.return_value = task +def test_reschedule_task_from_other_org(rf, client_member, client_member_b, mock_scheduler, task): + mock_scheduler.get_task_details.return_value = task request = setup_request( rf.post( diff --git a/rocky/tests/test_groups_and_permissions.py b/rocky/tests/test_groups_and_permissions.py index f477f5a1c5..2281aaa6a1 100644 --- a/rocky/tests/test_groups_and_permissions.py +++ b/rocky/tests/test_groups_and_permissions.py @@ -52,13 +52,13 @@ def test_plugin_settings_list_perms( mock_mixins_katalogus, plugin_details, plugin_schema, + mock_scheduler, mock_organization_view_octopoes, network, mocker, lazy_task_list_with_boefje, ): - mock_scheduler_client = mocker.patch("katalogus.views.plugin_detail.scheduler") - mock_scheduler_client.client.get_lazy_task_list.return_value = lazy_task_list_with_boefje + mock_scheduler.client.get_lazy_task_list.return_value = lazy_task_list_with_boefje mock_organization_view_octopoes().list_objects.return_value = Paginated[OOIType](count=1, items=[network]) mock_mixins_katalogus().get_plugin.return_value = plugin_details @@ -86,13 +86,13 @@ def test_plugin_settings_list_perms_2( mock_mixins_katalogus, plugin_details, plugin_schema, + mock_scheduler, mock_organization_view_octopoes, network, mocker, lazy_task_list_with_boefje, ): - mock_scheduler_client = mocker.patch("katalogus.views.plugin_detail.scheduler") - mock_scheduler_client.client.get_lazy_task_list.return_value = lazy_task_list_with_boefje + mock_scheduler.client.get_lazy_task_list.return_value = lazy_task_list_with_boefje mock_organization_view_octopoes().list_objects.return_value = Paginated[OOIType](count=1, items=[network]) mock_mixins_katalogus().get_plugin.return_value = plugin_details diff --git a/rocky/tools/forms/scheduler.py b/rocky/tools/forms/scheduler.py new file mode 100644 index 0000000000..b7682bf3ce --- /dev/null +++ b/rocky/tools/forms/scheduler.py @@ -0,0 +1,67 @@ +from datetime import datetime, timezone + +from django import forms +from django.utils.translation import gettext_lazy as _ + +from tools.forms.base import DateInput + + +class TaskFilterForm(forms.Form): + min_created_at = forms.DateField( + label=_("From"), + widget=DateInput(format="%Y-%m-%d"), + required=False, + ) + max_created_at = forms.DateField( + label=_("To"), + widget=DateInput(format="%Y-%m-%d"), + required=False, + ) + status = forms.ChoiceField( + choices=( + ("", _("All")), + ("cancelled", _("Cancelled")), + ("completed", _("Completed")), + ("dispatched", _("Dispatched")), + ("failed", _("Failed")), + ("pending", _("Pending")), + ("queued", _("Queued")), + ("running", _("Running")), + ), + required=False, + ) + input_ooi = forms.CharField( + label=_("Search"), + widget=forms.TextInput(attrs={"placeholder": _("Search by object name")}), + required=False, + ) + + def clean(self): + cleaned_data = super().clean() + + min_created_at = cleaned_data.get("min_created_at") + max_created_at = cleaned_data.get("max_created_at") + + date_message = _("The selected date is in the future. Please select a different date.") + + now = datetime.now(tz=timezone.utc) + + if min_created_at is not None and min_created_at > now.date(): + self.add_error("min_created_at", date_message) + + if max_created_at is not None and max_created_at > now.date(): + self.add_error("max_created_at", date_message) + + return cleaned_data + + +class OOIDetailTaskFilterForm(TaskFilterForm): + """ + Task filter at OOI detail to pass observed_at and ooi_id values. + """ + + observed_at = forms.CharField(widget=forms.HiddenInput(), required=False) + ooi_id = forms.CharField(widget=forms.HiddenInput(), required=False) + + # No need to search for OOI if you are already at the OOI detail page. + input_ooi = None diff --git a/rocky/tools/view_helpers.py b/rocky/tools/view_helpers.py index 741c59c868..e216d94433 100644 --- a/rocky/tools/view_helpers.py +++ b/rocky/tools/view_helpers.py @@ -3,13 +3,11 @@ from typing import TypedDict from urllib.parse import urlencode, urlparse, urlunparse -from django.contrib import messages from django.http import HttpRequest from django.urls.base import reverse, reverse_lazy from django.utils.translation import gettext_lazy as _ from octopoes.models.types import OOI_TYPES -from rocky.scheduler import PrioritizedItem, SchedulerError, client from tools.models import Organization @@ -165,56 +163,3 @@ def build_breadcrumbs(self): "text": _("Objects"), } ] - - -def schedule_task(request: HttpRequest, organization_code: str, p_item: PrioritizedItem) -> None: - try: - # Remove id attribute of both p_item and p_item.data, since the - # scheduler will create a new task with new id's. However, pydantic - # requires an id attribute to be present in its definition and the - # default set to None when the attribute is optional, otherwise it - # will not serialize the id if it is not present in the definition. - if hasattr(p_item, "id"): - delattr(p_item, "id") - - if hasattr(p_item.data, "id"): - delattr(p_item.data, "id") - - client.push_task(f"{p_item.data.type}-{organization_code}", p_item) - except SchedulerError as error: - messages.error(request, error.message) - else: - messages.success( - request, - _( - "Your task is scheduled and will soon be started in the background. " - "Results will be added to the object list when they are in. " - "It may take some time, a refresh of the page may be needed to show the results." - ), - ) - - -# FIXME: Tasks should be (re)created with supplied data, not by fetching prior -# task info from the scheduler. Task data should be available from the context -# from which the task is created. -def reschedule_task(request: HttpRequest, organization_code: str, task_id: str) -> None: - try: - task = client.get_task_details(organization_code, task_id) - except SchedulerError as error: - messages.error(request, error.message) - return - - if not task: - messages.error(request, _("Task not found.")) - return - - try: - new_p_item = PrioritizedItem( - data=task.p_item.data, - priority=1, - ) - - schedule_task(request, organization_code, new_p_item) - except SchedulerError as error: - messages.error(request, error.message) - return