Add job start and end time

README.md

@@ -151,7 +151,7 @@ curl http://localhost:6800/listjobs.json
 {
   "finished":[],
   "pending":[],
-  "running":[{"id":"e9b81fccbec211eeb3b109f30f136c01","project":"example","spider":"quotes","state":"running"}],
+  "running":[{"id":"e9b81fccbec211eeb3b109f30f136c01","project":"example","spider":"quotes","state":"running", "start_time": "2012-09-12 10:14:03.594664"}],
   "status":"ok"
 }
 ```

scrapyd_k8s/launcher/docker.py

@@ -1,8 +1,13 @@
 import re
-import os
 import socket
+from datetime import datetime
+from typing import Optional
 
 import docker
+from docker.models.containers import Container
+
+from scrapyd_k8s.settings import TIME_FORMAT
+
 from ..utils import native_stringify_dict
 
 class Docker:
@@ -55,36 +60,44 @@ def schedule(self, project, version, spider, job_id, settings, args):
         )
 
     def cancel(self, project_id, job_id, signal):
-        c = self._get_container(project_id, job_id)
-        if not c:
+        container = self._get_container(project_id, job_id)
+        if not container:
             return None
 
-        prevstate = self._docker_to_scrapyd_status(c.status)
-        if c.status == 'created' or c.status == 'scheduled':
-            c.remove()
-        elif c.status == 'running':
-            c.kill(signal='SIG' + signal)
+        prevstate = self._docker_to_scrapyd_status(container.status)
+        if container.status == 'created' or container.status == 'scheduled':
+            container.remove()
+        elif container.status == 'running':
+            container.kill(signal='SIG' + signal)
         return prevstate
 
-    def _parse_job(self, c):
-        return {
-            'id': c.labels.get(self.LABEL_JOB_ID),
-            'state': self._docker_to_scrapyd_status(c.status),
-            'project': c.labels.get(self.LABEL_PROJECT),
-            'spider': c.labels.get(self.LABEL_SPIDER)
+    def _parse_job(self, container: Container):
+        state = self._docker_to_scrapyd_status(container.status)
+        job = {
+            'id': container.labels.get(self.LABEL_JOB_ID),
+            'state': state,
+            'project': container.labels.get(self.LABEL_PROJECT),
+            'spider': container.labels.get(self.LABEL_SPIDER),
         }
 
-    def _get_container(self, project_id, job_id):
-        filters = { 'label': self.LABEL_JOB_ID + '=' + job_id }
-        c = self._docker.containers.list(all=True, filters=filters)
-        if not c:
+        if state in ['running', 'finished']:
+            job['start_time'] = datetime.fromisoformat(container.attrs['State']['StartedAt']).strftime(TIME_FORMAT)
+        if state == 'finished':
+            job['end_time'] = datetime.fromisoformat(container.attrs['State']['FinishedAt']).strftime(TIME_FORMAT)
+
+        return job
+
+    def _get_container(self, project_id: str, job_id: str) -> Optional[Container]:
+        filters = {'label': self.LABEL_JOB_ID + '=' + job_id}
+        containers = self._docker.containers.list(all=True, filters=filters)
+        if not containers:
             return None
-        c = c[0]
 
-        if c.labels.get(self.LABEL_PROJECT) != project_id:
+        container = containers[0]
+        if container.labels.get(self.LABEL_PROJECT) != project_id:
             return None
 
-        return c
+        return container
 
     def _docker_to_scrapyd_status(self, status):
         return self.STATUS_MAP.get(status, status)

scrapyd_k8s/launcher/k8s.py

@@ -1,9 +1,12 @@
 import os
+from signal import Signals
+from typing import Optional
 
 import kubernetes
 import kubernetes.stream
-from signal import Signals
-from subprocess import check_output, CalledProcessError
+from kubernetes.client.models import V1Job
+
+from scrapyd_k8s.settings import TIME_FORMAT
 
 from ..utils import native_stringify_dict
 
@@ -121,15 +124,26 @@ def cancel(self, project, job_id, signal):
             )
         return prevstate
 
-    def _parse_job(self, job):
-        return {
+    def _parse_job(self, job: V1Job):
+        state = self._k8s_job_to_scrapyd_status(job)
+        _job = {
             'id': job.metadata.labels.get(self.LABEL_JOB_ID),
-            'state': self._k8s_job_to_scrapyd_status(job),
+            'state': state,
             'project': job.metadata.labels.get(self.LABEL_PROJECT),
-            'spider': job.metadata.labels.get(self.LABEL_SPIDER)
+            'spider': job.metadata.labels.get(self.LABEL_SPIDER),
         }
 
-    def _get_job(self, project, job_id):
+        if state in ['running', 'finished']:
+            _job['start_time'] = job.status.start_time.strftime(TIME_FORMAT)
+        if state == 'finished':
+            # Finished is mapped to succeeded and pending. Only when a job has succeeded will it have a 
+            # completion time.
+            completion_time = job.status.completion_time
+            _job['end_time'] = completion_time.strftime(TIME_FORMAT) if completion_time else None
+
+        return _job
+
+    def _get_job(self, project, job_id) -> Optional[V1Job]:
         label = self.LABEL_JOB_ID + '=' + job_id
         r = self._k8s_batch.list_namespaced_job(namespace=self._namespace, label_selector=label)
         if not r or not r.items:
@@ -170,13 +184,12 @@ def _k8s_job_name(self, project, job_id):
 
     def _k8s_kill(self, pod_name, signal):
         # exec needs stream, which modifies client, so use separate instance
-        k8s = kubernetes.client.CoreV1Api()
-        resp = kubernetes.stream.stream(
-            k8s.connect_get_namespaced_pod_exec,
+        kubernetes.stream.stream(
+            self._k8s.connect_get_namespaced_pod_exec,
             pod_name,
             namespace=self._namespace,
             # this is a bit blunt, bit it works and is usually available
             command=['/usr/sbin/killall5', '-' + str(signal)],
             stderr=True
         )
-        # TODO figure out how to get return value
+        # TODO figure out how to get return value

scrapyd_k8s/settings.py

@@ -0,0 +1 @@
+TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%f'

test_api.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 import os
-import pytest
-import requests
 import time
+from datetime import datetime
+
+import requests
 
+from scrapyd_k8s.settings import TIME_FORMAT
 
 BASE_URL = os.getenv('TEST_BASE_URL', 'http://localhost:6800')
 AVAIL_PROJECTS = os.getenv('TEST_AVAILABLE_PROJECTS', 'example').split(',')
@@ -159,7 +161,11 @@ def test_scenario_cancel_running_finished_ok():
     # wait until the job has stopped
     listjobs_wait(jobid, 'finished')
     jobinfo = assert_listjobs(finished=jobid)
-    assert jobinfo == { 'id': jobid, 'project': RUN_PROJECT, 'spider': RUN_SPIDER, 'state': 'finished' }
+    assert jobinfo['id'] == jobid
+    assert jobinfo['project'] == RUN_PROJECT
+    assert jobinfo['spider'] == RUN_SPIDER
+    assert jobinfo['state'] == 'finished'
+    assert datetime.strptime(jobinfo['start_time'], TIME_FORMAT)
     # then cancel it again, though nothing would happen
     response = requests.post(BASE_URL + '/cancel.json', data={ 'project': RUN_PROJECT, 'job': jobid })
     assert_response_ok(response)
@@ -178,12 +184,22 @@ def scenario_regular(schedule_args):
     # wait until the job is running
     listjobs_wait(jobid, 'running')
     jobinfo = assert_listjobs(running=jobid)
-    assert jobinfo == { 'id': jobid, 'project': RUN_PROJECT, 'spider': RUN_SPIDER, 'state': 'running' }
+    assert jobinfo['id'] == jobid
+    assert jobinfo['project'] == RUN_PROJECT
+    assert jobinfo['spider'] == RUN_SPIDER
+    assert jobinfo['state'] == 'running'
+    assert datetime.strptime(jobinfo['start_time'], TIME_FORMAT)
+    assert not jobinfo.get('end_time')
     # wait until the job has finished
     listjobs_wait(jobid, 'finished')
     # check listjobs output
     jobinfo = assert_listjobs(finished=jobid)
-    assert jobinfo == { 'id': jobid, 'project': RUN_PROJECT, 'spider': RUN_SPIDER, 'state': 'finished' }
+    assert jobinfo['id'] == jobid
+    assert jobinfo['project'] == RUN_PROJECT
+    assert jobinfo['spider'] == RUN_SPIDER
+    assert jobinfo['state'] == 'finished'
+    assert datetime.strptime(jobinfo['start_time'], TIME_FORMAT)
+    assert datetime.strptime(jobinfo['end_time'], TIME_FORMAT)
 
 def assert_response_ok(response):
     assert response.status_code == 200