Dashboard support

NablaChem · Jul 11, 2023 · c427ab1 · c427ab1
1 parent a43188c
commit c427ab1
Show file tree

Hide file tree

Showing 6 changed files with 112 additions and 46 deletions.
diff --git a/src/Makefile b/src/Makefile
@@ -1,10 +1,10 @@
 devserver:
 	docker volume create dev-mongodb
-	docker volume create dev-minio
+	#docker volume create dev-minio
 	docker-compose -f docker-compose.yml -f docker-compose.dev.yml -p dev --env-file env.dev.yml up --build -d
 prodserver:
 	docker volume create prod-mongodb
-	docker volume create prod-minio
+	#docker volume create prod-minio
 	docker volume create prod-redis
 	docker-compose -f docker-compose.yml -f docker-compose.prod.yml -p prod --env-file env.prod.yml up --build -d
 testserver:

diff --git a/src/docker-compose.dev.yml b/src/docker-compose.dev.yml
@@ -4,15 +4,15 @@ services:
   hmq:
     volumes:
       - "./server/app/:/code/app/"
-  minio:
-    volumes:
-      - "dev-minio:/data"
+  # minio:
+  #   volumes:
+  #     - "dev-minio:/data"
   mongo:
     volumes:
      - "dev-mongodb:/data/db"
 
 volumes:
-  dev-minio:
-    external: True
+  # dev-minio:
+  #   external: True
   dev-mongodb:
     external: True
diff --git a/src/docker-compose.prod.yml b/src/docker-compose.prod.yml
@@ -7,9 +7,9 @@ services:
   mongo:
     volumes:
      - "prod-mongodb:/data/db"
-  minio:
-    volumes:
-      - "prod-minio:/data"
+  # minio:
+  #   volumes:
+  #     - "prod-minio:/data"
   redis:
     volumes:
       - "prod-redis:/data"
@@ -30,12 +30,12 @@ services:
       - "traefik.http.routers.traefiksecure.tls.certresolver=letsencrypt"
       - "traefik.http.routers.traefiksecure.tls.domains[0].main=*.${BASEURL}"
 
-  minio:
-    volumes:
-      - "prod-minio:/data"
-    labels:
-      - "traefik.http.routers.miniosecure.tls.certresolver=letsencrypt"
-      - "traefik.http.routers.minio-consolesecure.tls.certresolver=letsencrypt"
+  # minio:
+  #   volumes:
+  #     - "prod-minio:/data"
+  #   labels:
+  #     - "traefik.http.routers.miniosecure.tls.certresolver=letsencrypt"
+  #     - "traefik.http.routers.minio-consolesecure.tls.certresolver=letsencrypt"
 
   hmq:
     volumes:

diff --git a/src/docker-compose.yml b/src/docker-compose.yml
@@ -15,6 +15,7 @@ services:
 
       # API
       - "--api=true"
+      - "--log.level=DEBUG"
     ports:
       - "80:80"
       - "443:443"
@@ -64,41 +65,42 @@ services:
      - "MONGO_INITDB_ROOT_USERNAME=${MONGO_ROOT_USER}"
      - "MONGO_INITDB_ROOT_PASSWORD=${MONGO_ROOT_PASS}"
 
-  minio:
-    image: "minio/minio:RELEASE.2022-01-08T03-11-54Z"
-    restart: "unless-stopped"
-    command: "server /data --console-address :9001 --address :9000"
-    expose:
-      - 9000
-      - 9001
-    environment:
-      - "MINIO_ROOT_USER=${MINIO_ROOT_USER}"
-      - "MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASS}"
-      - "MINIO_BROWSER_REDIRECT_URL=http://s3console.${BASEURL}"
-      - "MINIO_DOMAIN=http://s3.${BASEURL}"
-    labels:
-      # general
-      - "traefik.enable=true"
-      - "traefik.docker.network=traefik"
+  # minio:
+  #   image: "minio/minio:RELEASE.2022-01-08T03-11-54Z"
+  #   restart: "unless-stopped"
+  #   command: "server /data --console-address :9001 --address :9000"
+  #   expose:
+  #     - 9000
+  #     - 9001
+  #   environment:
+  #     - "MINIO_ROOT_USER=${MINIO_ROOT_USER}"
+  #     - "MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASS}"
+  #     - "MINIO_BROWSER_REDIRECT_URL=http://s3console.${BASEURL}"
+  #     - "MINIO_DOMAIN=http://s3.${BASEURL}"
+  #   labels:
+  #     # general
+  #     - "traefik.enable=true"
+  #     - "traefik.docker.network=traefik"
 
-      # MinIO
-      - "traefik.http.routers.miniosecure.entrypoints=websecure"  
-      - "traefik.http.routers.miniosecure.service=minio"
-      - "traefik.http.routers.miniosecure.rule=Host(`s3.${BASEURL}`)"
-      - "traefik.http.services.minio.loadbalancer.server.port=9000"
+  #     # MinIO
+  #     - "traefik.http.routers.miniosecure.entrypoints=websecure"  
+  #     - "traefik.http.routers.miniosecure.service=minio"
+  #     - "traefik.http.routers.miniosecure.rule=Host(`s3.${BASEURL}`)"
+  #     - "traefik.http.services.minio.loadbalancer.server.port=9000"
 
-      # MinIO console
-      - "traefik.http.routers.minio-consolesecure.entrypoints=websecure"  
-      - "traefik.http.routers.minio-consolesecure.service=minio-console"
-      - "traefik.http.routers.minio-consolesecure.rule=Host(`s3console.${BASEURL}`)"
-      - "traefik.http.services.minio-console.loadbalancer.server.port=9001"
+  #     # MinIO console
+  #     - "traefik.http.routers.minio-consolesecure.entrypoints=websecure"  
+  #     - "traefik.http.routers.minio-consolesecure.service=minio-console"
+  #     - "traefik.http.routers.minio-consolesecure.rule=Host(`s3console.${BASEURL}`)"
+  #     - "traefik.http.services.minio-console.loadbalancer.server.port=9001"
 
-      # HTTPS
-      - "traefik.http.routers.minio-consolesecure.tls=true"
-      - "traefik.http.routers.miniosecure.tls=true"
+  #     # HTTPS
+  #     - "traefik.http.routers.minio-consolesecure.tls=true"
+  #     - "traefik.http.routers.miniosecure.tls=true"
   hmq:
     build:
       context: "server"
+    restart: "unless-stopped"
     ports:
       - "80"
     labels:
@@ -124,3 +126,29 @@ services:
       - "MONGODB_CONNSTR=mongodb://${MONGO_USER}:${MONGO_PASS}@mongo/hq"
       - "API_TOKEN=${API_TOKEN}"
 
+  dash:
+    build:
+      context: "dashboard"
+    restart: "unless-stopped"
+    ports:
+      - "80"
+    labels:
+      # genereal
+      - "traefik.enable=true"
+      - "traefik.docker.network=traefik"
+
+      # Dashboard
+      - "traefik.http.routers.dashsecure.rule=Host(`queue.${BASEURL}`)"
+      - "traefik.http.routers.dashsecure.entrypoints=websecure"
+        # - "traefik.http.routers.dashsecure.loadbalancer.server.port=8501"
+
+      # HTTP 
+      - "traefik.http.routers.dash.entrypoints=web"
+      - "traefik.http.routers.dash.rule=Host(`queue.${BASEURL}`)"
+      - "traefik.http.middlewares.upgradeHTTP.redirectscheme.scheme=https"
+      - "traefik.http.middlewares.upgradeHTTP.redirectscheme.permanent=true"
+      - "traefik.http.routers.dash.middlewares=upgradeHTTP"
+
+      # HTTPS
+      - "traefik.http.routers.dashsecure.tls=true"
+
diff --git a/src/server/app/maintenance.py b/src/server/app/maintenance.py
@@ -1,4 +1,4 @@
-from rq import Queue
+from rq import Queue, Worker
 from redis import Redis
 import time
 from . import auth
@@ -31,9 +31,35 @@ def refill_redis(njobs: int):
         redis_conn.hset("id2id", mapping=idmapping)
 
 
+def update_stats(last_update: float):
+    now = time.time()
+    unixminute_now = int(now / 60)
+    unixminute_last = int(last_update / 60)
+
+    if unixminute_now == unixminute_last:
+        return last_update
+
+    workers = Worker.all(connection=redis_conn)
+    nworkers = len(workers)
+    njobs = q.count + auth.db.tasks.count_documents({"status": "pending"})
+    nrunning = len([_ for _ in workers if _.state == "busy"])
+    auth.db.stats.insert_one(
+        {
+            "cores_available": nworkers,
+            "cores_used": nrunning,
+            "tasks_queued": njobs,
+            "tasks_running": nrunning,
+            "ts": unixminute_now,
+        }
+    )
+    return now
+
+
 def flow_control():
     # MongoDB -> redis
+    last_update = 0
     while True:
         missing = max(100 - q.count, 0)
         refill_redis(missing)
+        last_update = update_stats(last_update)
         time.sleep(1)
diff --git a/src/server/app/routers/compute.py b/src/server/app/routers/compute.py
@@ -209,3 +209,15 @@ def results_retreive(body: ResultsRetrieve):
             results[task["id"]] = entry
 
     return results
+
+
+@app.get("/usage/inspect", tags=["statistics"])
+def inspect_usage():
+    ret = {}
+    unixminute_now = int(time.time() / 60)
+    unixminute_first = unixminute_now - 60
+    for stats in auth.db.stats.find({"ts": {"$gte": unixminute_first}}):
+        refts = stats["ts"]
+        stats = {k: v for k, v in stats.items() if k != "_id" and k != "ts"}
+        ret[unixminute_now - refts] = stats
+    return ret