From c51ab65000ae187d78290adf23c4bc178de5454e Mon Sep 17 00:00:00 2001 From: Joe Cheng Date: Tue, 22 Aug 2023 13:40:25 -0700 Subject: [PATCH 1/2] Prune autogenerated model score data every minute --- examples/model-score/scoredata.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/examples/model-score/scoredata.py b/examples/model-score/scoredata.py index 11613c4da..53edeb4c6 100644 --- a/examples/model-score/scoredata.py +++ b/examples/model-score/scoredata.py @@ -22,8 +22,8 @@ def init_db(): now = datetime.datetime.utcnow() position = now.minute * 60 + now.second + 1 - # Simulate 100 seconds of historical data - offset_secs = -np.arange(100) - 1 + # Simulate an hour of historical data + offset_secs = -np.arange(60 * 60) - 1 abs_secs = (position + offset_secs) % (60 * 60) + 1 initial_scores = accuracy_scores.loc[abs_secs] timestamps = pd.DataFrame( @@ -50,9 +50,25 @@ async def update_db(position): new_data["timestamp"] = datetime.datetime.utcnow() new_data.to_sql("accuracy_scores", con, index=False, if_exists="append") position = (position % (60 * 60)) + 1 + + # Every minute, delete old data + if position % 60 == 0: + cleanup_db(con) + await asyncio.sleep(1) +def cleanup_db(con): + try: + con.execute("PRAGMA wal_checkpoint;") + except sqlite3.OperationalError: + print("wal_checkpoint failed") + con.execute( + "DELETE FROM accuracy_scores WHERE timestamp < ?;", + (datetime.datetime.utcnow() - datetime.timedelta(hours=1),), + ) + + def begin(): position = init_db() From 09ebece81205741cbe976f2dcf8f072e94c866d2 Mon Sep 17 00:00:00 2001 From: Joe Cheng Date: Thu, 24 Aug 2023 09:41:27 -0700 Subject: [PATCH 2/2] Simplify requirements.txt --- examples/model-score/requirements.in | 4 - examples/model-score/requirements.txt | 178 +------------------------- 2 files changed, 4 insertions(+), 178 deletions(-) delete mode 100644 examples/model-score/requirements.in diff --git a/examples/model-score/requirements.in b/examples/model-score/requirements.in deleted file mode 100644 index 1a5617b60..000000000 --- a/examples/model-score/requirements.in +++ /dev/null @@ -1,4 +0,0 @@ -pandas -plotly -shiny -shinywidgets diff --git a/examples/model-score/requirements.txt b/examples/model-score/requirements.txt index 17d358c9b..1a5617b60 100644 --- a/examples/model-score/requirements.txt +++ b/examples/model-score/requirements.txt @@ -1,174 +1,4 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile -# -anyio==3.7.1 - # via - # starlette - # watchfiles -appdirs==1.4.4 - # via shiny -appnope==0.1.3 - # via - # ipykernel - # ipython -asgiref==3.7.2 - # via shiny -asttokens==2.2.1 - # via stack-data -backcall==0.2.0 - # via ipython -click==8.1.6 - # via - # shiny - # uvicorn -comm==0.1.3 - # via ipykernel -contextvars==2.4 - # via shiny -debugpy==1.6.7 - # via ipykernel -decorator==5.1.1 - # via ipython -exceptiongroup==1.1.2 - # via anyio -executing==1.2.0 - # via stack-data -h11==0.14.0 - # via uvicorn -htmltools==0.2.1 - # via shiny -idna==3.4 - # via anyio -immutables==0.19 - # via contextvars -ipykernel==6.25.0 - # via ipywidgets -ipython==8.14.0 - # via - # ipykernel - # ipywidgets -ipywidgets==8.0.7 - # via shinywidgets -jedi==0.18.2 - # via ipython -jupyter-client==8.3.0 - # via ipykernel -jupyter-core==5.3.1 - # via - # ipykernel - # jupyter-client - # shinywidgets -jupyterlab-widgets==3.0.8 - # via ipywidgets -linkify-it-py==2.0.2 - # via shiny -markdown-it-py==3.0.0 - # via - # mdit-py-plugins - # shiny -matplotlib-inline==0.1.6 - # via - # ipykernel - # ipython -mdit-py-plugins==0.4.0 - # via shiny -mdurl==0.1.2 - # via markdown-it-py -nest-asyncio==1.5.7 - # via ipykernel -numpy==1.25.1 - # via pandas -packaging==23.1 - # via - # htmltools - # ipykernel - # plotly -pandas==2.0.3 - # via -r requirements.in -parso==0.8.3 - # via jedi -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -platformdirs==3.9.1 - # via jupyter-core -plotly==5.15.0 - # via -r requirements.in -prompt-toolkit==3.0.39 - # via ipython -psutil==5.9.5 - # via ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -pygments==2.15.1 - # via ipython -python-dateutil==2.8.2 - # via - # jupyter-client - # pandas - # shinywidgets -python-multipart==0.0.6 - # via shiny -pytz==2023.3 - # via pandas -pyzmq==25.1.0 - # via - # ipykernel - # jupyter-client -shiny==0.4.0 - # via - # -r requirements.in - # shinywidgets -shinywidgets==0.2.1 - # via -r requirements.in -six==1.16.0 - # via - # asttokens - # python-dateutil -sniffio==1.3.0 - # via anyio -stack-data==0.6.2 - # via ipython -starlette==0.31.0 - # via shiny -tenacity==8.2.2 - # via plotly -tornado==6.3.2 - # via - # ipykernel - # jupyter-client -traitlets==5.9.0 - # via - # comm - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # matplotlib-inline -typing-extensions==4.7.1 - # via - # asgiref - # htmltools - # shiny - # uvicorn -tzdata==2023.3 - # via pandas -uc-micro-py==1.0.2 - # via linkify-it-py -uvicorn==0.23.1 - # via shiny -watchfiles==0.19.0 - # via shiny -wcwidth==0.2.6 - # via prompt-toolkit -websockets==11.0.3 - # via shiny -widgetsnbextension==4.0.8 - # via ipywidgets +pandas +plotly +shiny +shinywidgets