Improve recalculate_bits endpoint logging

Add valid_time as optional parameter to upgrade function
Fix integration test with unwanted deletion propagation happening

Signed-off-by: Donny Peeters <donny@bitestreams.com>

boefjes/Makefile

@@ -81,6 +81,7 @@ bench:  ## Run the report benchmark.
 	$(ci-docker-compose) build
 	$(ci-docker-compose) down --remove-orphans
 	$(ci-docker-compose) run --rm migration_bench
+	$(ci-docker-compose) stop
 
 debian12:
 	docker run --rm \

boefjes/tests/conftest.py

@@ -224,7 +224,7 @@ def seed_system(
         Observation(
             method="",
             source_method="test",
-            source=network.reference,
+            source=hostnames[0].reference,
             task_id=uuid.uuid4(),
             valid_time=valid_time,
             result=oois,

boefjes/tests/integration/test_bench.py

@@ -46,7 +46,7 @@ def test_migration(octopoes_api_connector: OctopoesAPIConnector, bytes_client: B
 
         bytes_client.save_normalizer_meta(normalizer_meta)
 
-    total_processed, total_failed = upgrade()
+    total_processed, total_failed = upgrade(valid_time)
 
     assert total_processed == 0
     assert total_failed == 0

boefjes/tools/upgrade_v1_16_0.py

@@ -30,14 +30,16 @@
 logger = logging.getLogger(__name__)
 
 
-def upgrade() -> tuple[int, int]:
+def upgrade(valid_time: datetime | None = None) -> tuple[int, int]:
     """
     Perform the migration for all organisations in the database. The happy flow in this script is idempotent,
     meaning that it can be rerun until there are no, or only expected, exceptions.
     """
 
     organisations = create_organisation_storage(sessionmaker(bind=get_engine())()).get_all()
-    valid_time = datetime.now(timezone.utc)
+
+    if valid_time is None:
+        valid_time = datetime.now(timezone.utc)
 
     bytes_client = get_bytes_client()
     bytes_client.login()
@@ -60,6 +62,7 @@ def upgrade() -> tuple[int, int]:
 
     return total_processed, total_failed
 
+
 def migrate_org(bytes_client, connector, organisation_id, valid_time) -> tuple[int, int]:
     """
     For each organisation, we paginate through the origin API, find the matching normalizer meta in Bytes,

octopoes/octopoes/api/router.py

@@ -512,7 +512,12 @@ def delete_node(xtdb_session_: XTDBSession = Depends(xtdb_session)) -> None:
 
 @router.post("/bits/recalculate", tags=["Bits"])
 def recalculate_bits(octopoes: OctopoesService = Depends(octopoes_service)) -> int:
-    inference_count = octopoes.recalculate_bits()
+    try:
+        inference_count = octopoes.recalculate_bits()
+    except ObjectNotFoundException:
+        logger.exception("Failed to recalculate bits")
+        raise
+
     octopoes.commit()
 
     return inference_count