From 5d29a19318acd8b4d85fcca115915bbcfbea4c1f Mon Sep 17 00:00:00 2001 From: Delgermurun Purevkhuu Date: Tue, 2 Aug 2022 18:59:21 +0200 Subject: [PATCH 01/13] feat: use clip-as-service as an external executor --- executors/glid3/executor.py | 2 +- executors/rerank/executor.py | 13 ------------- flow.yml | 18 ++++++++++++------ requirements.txt | 1 - 4 files changed, 13 insertions(+), 21 deletions(-) delete mode 100644 executors/rerank/executor.py diff --git a/executors/glid3/executor.py b/executors/glid3/executor.py index 7664d41..cf7a788 100644 --- a/executors/glid3/executor.py +++ b/executors/glid3/executor.py @@ -45,7 +45,7 @@ async def run_glid3(self, d: Document, text: str, skip_rate: float, num_images: from dalle_flow_glid3.sample import do_run args = parser.parse_args(kw_str_list) - await do_run(args) + await do_run(args, d.embedding) kw.update({ 'generator': 'GLID3-XL', diff --git a/executors/rerank/executor.py b/executors/rerank/executor.py deleted file mode 100644 index bf57088..0000000 --- a/executors/rerank/executor.py +++ /dev/null @@ -1,13 +0,0 @@ -from clip_client import Client -from jina import Executor, requests, DocumentArray - - -class ReRank(Executor): - - def __init__(self, clip_server: str, **kwargs): - super().__init__(**kwargs) - self._client = Client(server=clip_server) - - @requests(on='/') - async def rerank(self, docs: DocumentArray, **kwargs): - return await self._client.arank(docs) diff --git a/flow.yml b/flow.yml index 89b0062..86b64cc 100644 --- a/flow.yml +++ b/flow.yml @@ -6,6 +6,11 @@ with: env: JINA_LOG_LEVEL: debug executors: + - name: clip_encoder + host: 'demo-cas.jina.ai' + port: 2096 + tls: true + external: true - name: dalle uses: executors/dalle/config.yml timeout_ready: -1 # slow download speed often leads to timeout @@ -24,13 +29,14 @@ executors: CUDA_VISIBLE_DEVICES: 0 # change this if you have multiple GPU XLA_PYTHON_CLIENT_ALLOCATOR: platform # https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html replicas: 1 # change this if you have larger VRAM - needs: [gateway] + needs: [gateway, clip_encoder] - name: rerank - uses: ReRank - uses_with: - clip_server: grpcs://demo-cas.jina.ai:2096 - py_modules: - - executors/rerank/executor.py + host: 'demo-cas.jina.ai' + port: 2096 + uses_requests: + '/': rank + tls: true + external: true needs: [dalle, diffusion] - name: upscaler uses: SwinIRUpscaler diff --git a/requirements.txt b/requirements.txt index 7708c3f..ec56e2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ # jina-related jina>=3.4.5 -clip_client>=0.4.20 docarray>=0.13.5 # dalle-mini flax From 96e2e64188235af50bd56e839b1d5c90de3e90af Mon Sep 17 00:00:00 2001 From: Delgermurun Purevkhuu Date: Wed, 3 Aug 2022 15:09:58 +0200 Subject: [PATCH 02/13] feat: use clip-as-service as an external executor --- Dockerfile | 2 +- README.md | 3 ++- client.ipynb | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e44e4ab..b3f81bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN apt-get update \ RUN if [ -n "${APT_PACKAGES}" ]; then apt-get update && apt-get install --no-install-recommends -y ${APT_PACKAGES}; fi && \ git clone --depth=1 https://github.com/JingyunLiang/SwinIR.git && \ git clone --depth=1 https://github.com/CompVis/latent-diffusion.git && \ - git clone --depth=1 https://github.com/hanxiao/glid-3-xl.git && \ + git clone --depth=1 -b pref-remove-clip-as-service https://github.com/jina-ai/glid-3-xl.git && \ pip install jax[cuda11_cudnn82]==0.3.13 -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html && \ pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113 && \ cd latent-diffusion && pip install --timeout=1000 -e . && cd - && \ diff --git a/README.md b/README.md index e770d26..9e9b287 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ The 16 candidates are sorted by [CLIP-as-service](https://github.com/jina-ai/cli ```python fav_id = 3 fav = da[fav_id] +fav.embedding = da.embedding fav.display() ``` @@ -243,7 +244,7 @@ mkdir dalle && cd dalle git clone https://github.com/jina-ai/dalle-flow.git git clone https://github.com/JingyunLiang/SwinIR.git git clone https://github.com/CompVis/latent-diffusion.git -git clone https://github.com/hanxiao/glid-3-xl.git +git clone https://github.com/jina-ai/glid-3-xl.git ``` You should have the following folder structure: diff --git a/client.ipynb b/client.ipynb index 925ef10..1662424 100644 --- a/client.ipynb +++ b/client.ipynb @@ -259,6 +259,7 @@ "fav_id = 3\n", "\n", "fav = da[fav_id]\n", + "fav.embedding = da.embedding\n", "\n", "fav.display()" ] From 950c157310990810cfb94f293096ad406d26eb4d Mon Sep 17 00:00:00 2001 From: Delgermurun Purevkhuu Date: Wed, 3 Aug 2022 15:24:16 +0200 Subject: [PATCH 03/13] feat: use clip-as-service as an external executor --- .gitignore | 6 ++++++ flow.yml | 2 ++ 2 files changed, 8 insertions(+) diff --git a/.gitignore b/.gitignore index b6e4761..7c1421b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# OS +.DS_Store + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -127,3 +130,6 @@ dmypy.json # Pyre type checker .pyre/ + +# Jina +.jina/ \ No newline at end of file diff --git a/flow.yml b/flow.yml index 86b64cc..be052c4 100644 --- a/flow.yml +++ b/flow.yml @@ -7,6 +7,7 @@ with: JINA_LOG_LEVEL: debug executors: - name: clip_encoder + uses: jinahub+docker://CLIPTorchEncoder/latest host: 'demo-cas.jina.ai' port: 2096 tls: true @@ -31,6 +32,7 @@ executors: replicas: 1 # change this if you have larger VRAM needs: [gateway, clip_encoder] - name: rerank + uses: jinahub+docker://CLIPTorchEncoder/latest host: 'demo-cas.jina.ai' port: 2096 uses_requests: From b3f8c6cb062a65e9d1527ba0d66dfc1fd157151d Mon Sep 17 00:00:00 2001 From: Delgermurun Purevkhuu Date: Thu, 4 Aug 2022 11:28:58 +0200 Subject: [PATCH 04/13] fix: flow topology --- flow.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/flow.yml b/flow.yml index be052c4..f6cb4a5 100644 --- a/flow.yml +++ b/flow.yml @@ -6,12 +6,6 @@ with: env: JINA_LOG_LEVEL: debug executors: - - name: clip_encoder - uses: jinahub+docker://CLIPTorchEncoder/latest - host: 'demo-cas.jina.ai' - port: 2096 - tls: true - external: true - name: dalle uses: executors/dalle/config.yml timeout_ready: -1 # slow download speed often leads to timeout @@ -19,6 +13,13 @@ executors: CUDA_VISIBLE_DEVICES: 0 # change this if you have multiple GPU XLA_PYTHON_CLIENT_ALLOCATOR: platform # https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html replicas: 1 # change this if you have larger VRAM + - name: clip_encoder + uses: jinahub+docker://CLIPTorchEncoder/latest + host: 'demo-cas.jina.ai' + port: 2096 + tls: true + external: true + needs: [gateway] - name: diffusion uses: GLID3Diffusion uses_with: @@ -30,7 +31,7 @@ executors: CUDA_VISIBLE_DEVICES: 0 # change this if you have multiple GPU XLA_PYTHON_CLIENT_ALLOCATOR: platform # https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html replicas: 1 # change this if you have larger VRAM - needs: [gateway, clip_encoder] + needs: [clip_encoder] - name: rerank uses: jinahub+docker://CLIPTorchEncoder/latest host: 'demo-cas.jina.ai' From 4022e0b7cc3c6ce6ab5dd6c2ebfa96fbdad7c1cc Mon Sep 17 00:00:00 2001 From: Delgermurun Purevkhuu Date: Thu, 4 Aug 2022 11:44:30 +0200 Subject: [PATCH 05/13] docs: mentioned about running CLIP internally not externally --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 9e9b287..6969ceb 100644 --- a/README.md +++ b/README.md @@ -315,6 +315,10 @@ Congrats! Now you should be able to [run the client](#client). You can modify and extend the server flow as you like, e.g. changing the model, adding persistence, or even auto-posting to Instagram/OpenSea. With Jina and DocArray, you can easily make DALL·E Flow [cloud-native and ready for production](https://github.com/jina-ai/jina). +### Run your own CLIP + +By default [`CLIPTorchEncoder`](https://hub.jina.ai/executor/gzpbl8jh) runs as an [external executor](https://docs.jina.ai/fundamentals/flow/add-executors/#external-executors). +If want to run your own CLIP, you can do that by just changing `external: true` to `external: false` in [`flow.yml`](./flow.yml). ## Support From 298a1e4290f33d0a7f66ef9de5bc716605ee23d6 Mon Sep 17 00:00:00 2001 From: Delgermurun Purevkhuu Date: Thu, 4 Aug 2022 11:47:50 +0200 Subject: [PATCH 06/13] docs: re-generate latest flow graph --- .github/flow.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/flow.svg b/.github/flow.svg index cd3bee0..5250b0b 100644 --- a/.github/flow.svg +++ b/.github/flow.svg @@ -1 +1 @@ -
store
MyStore
upscaler
SwinIRUpscaler
rerank
ReRank
diffusion
GLID3Diffusion
dalle
executors/dalle/config.yml
gateway
gateway
 \ No newline at end of file +
store
MyStore
upscaler
SwinIRUpscaler
diffusion
GLID3Diffusion
dalle
executors/dalle/config.yml
gateway
clip_encoder
rerank
gateway
 \ No newline at end of file From fe528186b786ddcd7c174934c9e10bf812b4b4bb Mon Sep 17 00:00:00 2001 From: Delgermurun Purevkhuu Date: Thu, 4 Aug 2022 14:05:26 +0200 Subject: [PATCH 07/13] fix: remove async/await --- executors/glid3/executor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/executors/glid3/executor.py b/executors/glid3/executor.py index cf7a788..7dd6c05 100644 --- a/executors/glid3/executor.py +++ b/executors/glid3/executor.py @@ -17,7 +17,7 @@ def __init__(self, glid3_path: str, steps: int = 100, **kwargs): from dalle_flow_glid3.sample import static_args assert static_args - async def run_glid3(self, d: Document, text: str, skip_rate: float, num_images: int): + def run_glid3(self, d: Document, text: str, skip_rate: float, num_images: int): request_time = time.time() with tempfile.NamedTemporaryFile( @@ -45,7 +45,7 @@ async def run_glid3(self, d: Document, text: str, skip_rate: float, num_images: from dalle_flow_glid3.sample import do_run args = parser.parse_args(kw_str_list) - await do_run(args, d.embedding) + do_run(args, d.embedding) kw.update({ 'generator': 'GLID3-XL', @@ -62,8 +62,8 @@ async def run_glid3(self, d: Document, text: str, skip_rate: float, num_images: self.logger.info(f'done with [{text}]!') @requests(on='/') - async def diffusion(self, docs: DocumentArray, parameters: Dict, **kwargs): + def diffusion(self, docs: DocumentArray, parameters: Dict, **kwargs): skip_rate = float(parameters.get('skip_rate', 0.5)) num_images = max(1, min(9, int(parameters.get('num_images', 1)))) for d in docs: - await self.run_glid3(d, d.text, skip_rate=skip_rate, num_images=num_images) + self.run_glid3(d, d.text, skip_rate=skip_rate, num_images=num_images) From 5e811fb536ae11686d50a3afc34124f6acff4b52 Mon Sep 17 00:00:00 2001 From: Delgermurun Purevkhuu Date: Thu, 4 Aug 2022 17:57:17 +0200 Subject: [PATCH 08/13] refactor: cache blank embeddings --- executors/glid3/executor.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/executors/glid3/executor.py b/executors/glid3/executor.py index 7dd6c05..91c3392 100644 --- a/executors/glid3/executor.py +++ b/executors/glid3/executor.py @@ -14,9 +14,14 @@ def __init__(self, glid3_path: str, steps: int = 100, **kwargs): os.environ['GLID_MODEL_PATH'] = glid3_path os.environ['GLID3_STEPS'] = str(steps) self.diffusion_steps = steps - from dalle_flow_glid3.sample import static_args + from dalle_flow_glid3.model import static_args + from dalle_flow_glid3.blank_encoding import generate_blank_embeddings + assert static_args + self.logger.info('Generating blank embeddings') + self.blank_bert_embedding, self.blank_clip_embedding = generate_blank_embeddings('a') + def run_glid3(self, d: Document, text: str, skip_rate: float, num_images: int): request_time = time.time() @@ -45,7 +50,7 @@ def run_glid3(self, d: Document, text: str, skip_rate: float, num_images: int): from dalle_flow_glid3.sample import do_run args = parser.parse_args(kw_str_list) - do_run(args, d.embedding) + do_run(args, d.embedding, self.blank_bert_embedding, self.blank_clip_embedding) kw.update({ 'generator': 'GLID3-XL', From 1c205d462bb2a7a8eb843f9f57a20363b6fa1529 Mon Sep 17 00:00:00 2001 From: Delgermurun Purevkhuu Date: Fri, 5 Aug 2022 14:30:32 +0200 Subject: [PATCH 09/13] fix: add cas url parameter --- executors/dalle/executor/dalle.py | 2 +- executors/glid3/executor.py | 6 +++--- flow.yml | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/executors/dalle/executor/dalle.py b/executors/dalle/executor/dalle.py index 6c5aa80..54307bb 100644 --- a/executors/dalle/executor/dalle.py +++ b/executors/dalle/executor/dalle.py @@ -8,7 +8,7 @@ class DalleGenerator(Executor): - @requests(on='/') + @requests def generate(self, docs: DocumentArray, parameters: Dict, **kwargs): # can be of course larger but to save time and reduce the queue when serving public diff --git a/executors/glid3/executor.py b/executors/glid3/executor.py index 91c3392..96cdf4a 100644 --- a/executors/glid3/executor.py +++ b/executors/glid3/executor.py @@ -9,7 +9,7 @@ class GLID3Diffusion(Executor): - def __init__(self, glid3_path: str, steps: int = 100, **kwargs): + def __init__(self, glid3_path: str, clip_as_service_url: str, steps: int = 100, **kwargs): super().__init__(**kwargs) os.environ['GLID_MODEL_PATH'] = glid3_path os.environ['GLID3_STEPS'] = str(steps) @@ -20,7 +20,7 @@ def __init__(self, glid3_path: str, steps: int = 100, **kwargs): assert static_args self.logger.info('Generating blank embeddings') - self.blank_bert_embedding, self.blank_clip_embedding = generate_blank_embeddings('a') + self.blank_bert_embedding, self.blank_clip_embedding = generate_blank_embeddings('a', clip_as_service_url) def run_glid3(self, d: Document, text: str, skip_rate: float, num_images: int): request_time = time.time() @@ -66,7 +66,7 @@ def run_glid3(self, d: Document, text: str, skip_rate: float, num_images: int): self.logger.info(f'done with [{text}]!') - @requests(on='/') + @requests def diffusion(self, docs: DocumentArray, parameters: Dict, **kwargs): skip_rate = float(parameters.get('skip_rate', 0.5)) num_images = max(1, min(9, int(parameters.get('num_images', 1)))) diff --git a/flow.yml b/flow.yml index f6cb4a5..f5d4fbe 100644 --- a/flow.yml +++ b/flow.yml @@ -24,6 +24,7 @@ executors: uses: GLID3Diffusion uses_with: glid3_path: ../glid-3-xl + clip_as_service_url: https://demo-cas.jina.ai:8443 steps: 100 py_modules: - executors/glid3/executor.py From 9eed6be369e407961bfc3695a52f81f8fdbecce9 Mon Sep 17 00:00:00 2001 From: Delgermurun Purevkhuu Date: Fri, 5 Aug 2022 15:47:04 +0200 Subject: [PATCH 10/13] feat: load clip blank encoding from file --- clip_blank_encoding.json | 770 ++++++++++++++++++++++++++++++++++++ executors/glid3/executor.py | 6 +- flow.yml | 2 +- 3 files changed, 775 insertions(+), 3 deletions(-) create mode 100644 clip_blank_encoding.json diff --git a/clip_blank_encoding.json b/clip_blank_encoding.json new file mode 100644 index 0000000..7878717 --- /dev/null +++ b/clip_blank_encoding.json @@ -0,0 +1,770 @@ +[ + 0.4267578125, + 0.09423828125, + 0.319091796875, + -0.272705078125, + -0.12139892578125, + -0.0032444000244140625, + -0.314208984375, + 0.41796875, + 0.1824951171875, + -0.464111328125, + 0.1988525390625, + -0.0894775390625, + -0.4609375, + 0.402099609375, + -0.326416015625, + 0.286376953125, + 0.1968994140625, + 0.2425537109375, + 0.005565643310546875, + -0.478759765625, + 0.1920166015625, + -0.54150390625, + 0.09613037109375, + 0.265869140625, + 0.2330322265625, + 0.2034912109375, + 0.16455078125, + -0.1453857421875, + 0.59033203125, + 0.31640625, + -0.0650634765625, + -0.1490478515625, + -0.52197265625, + 0.09747314453125, + -0.6328125, + 0.2423095703125, + 0.76904296875, + -0.34130859375, + -0.294921875, + -0.71728515625, + -0.1629638671875, + -0.2763671875, + 0.349853515625, + -0.12384033203125, + 0.31591796875, + -0.1033935546875, + -0.1754150390625, + 0.0222320556640625, + -0.276123046875, + -0.80126953125, + 0.07586669921875, + -0.1688232421875, + 0.166748046875, + 0.043365478515625, + 0.5517578125, + -0.2442626953125, + 0.1434326171875, + -0.322021484375, + 0.2464599609375, + 0.013641357421875, + -0.369384765625, + -0.0181884765625, + -0.041259765625, + -0.0828857421875, + 0.2039794921875, + 0.8564453125, + 0.1837158203125, + 0.13818359375, + -0.338134765625, + -0.6181640625, + -0.00714874267578125, + 0.389892578125, + -0.01666259765625, + -0.40869140625, + -0.20849609375, + 0.36474609375, + 0.26416015625, + 0.202880859375, + -0.26904296875, + -0.9296875, + -0.205078125, + -0.1761474609375, + 0.30126953125, + -0.094970703125, + -0.035430908203125, + 0.155029296875, + 0.0236663818359375, + -0.033538818359375, + -0.5654296875, + 0.46044921875, + -0.08294677734375, + 0.09405517578125, + -0.2406005859375, + -0.04327392578125, + 0.050079345703125, + -0.2078857421875, + 0.79541015625, + -0.27294921875, + 0.477783203125, + -0.234619140625, + 0.2376708984375, + -0.2392578125, + -0.501953125, + -0.09136962890625, + 0.251220703125, + -0.01557159423828125, + 0.16455078125, + -0.177001953125, + 0.045135498046875, + 0.01027679443359375, + -0.045440673828125, + 0.04107666015625, + -0.070556640625, + 0.325927734375, + 0.11676025390625, + 0.29052734375, + -0.8583984375, + -0.19677734375, + -0.0970458984375, + -0.0728759765625, + -0.28662109375, + -0.213134765625, + 0.01155853271484375, + 0.55859375, + -0.13916015625, + -0.1534423828125, + 0.226318359375, + 0.128173828125, + -0.330322265625, + -0.1881103515625, + -0.05975341796875, + -0.7568359375, + 0.2724609375, + -0.65966796875, + 0.1209716796875, + 0.5615234375, + 0.218017578125, + 0.51953125, + 0.1649169921875, + -0.0517578125, + 0.2022705078125, + -0.08172607421875, + -0.0079498291015625, + -0.2626953125, + 0.0416259765625, + 0.188720703125, + 0.62744140625, + -0.599609375, + -0.7734375, + -0.44580078125, + 0.08416748046875, + -0.58349609375, + 0.283447265625, + -0.4384765625, + 0.57080078125, + -0.223388671875, + 0.273193359375, + 0.35595703125, + 0.1824951171875, + 0.034271240234375, + 0.33447265625, + 0.369384765625, + 0.2498779296875, + 0.2017822265625, + -0.314208984375, + -0.61376953125, + 0.6328125, + -0.00018024444580078125, + 0.487548828125, + -0.39794921875, + 0.95849609375, + -0.1937255859375, + 0.361328125, + -0.0301513671875, + 0.183349609375, + -0.059173583984375, + 0.1392822265625, + -1.455078125, + 0.44287109375, + 0.5126953125, + 1.1103515625, + -0.0845947265625, + -0.2427978515625, + 0.52734375, + -0.00655364990234375, + -0.544921875, + -0.310791015625, + -0.040618896484375, + -0.1861572265625, + -0.27490234375, + -0.57666015625, + -0.1861572265625, + 0.31005859375, + 0.40771484375, + 0.1600341796875, + -2.431640625, + -0.138671875, + 0.490234375, + 0.0030040740966796875, + 0.12548828125, + 0.29736328125, + 0.042999267578125, + -0.1131591796875, + -0.0635986328125, + 0.2493896484375, + -0.268798828125, + -0.599609375, + -0.2376708984375, + 0.492431640625, + 0.02752685546875, + 0.4111328125, + -0.03717041015625, + 0.10552978515625, + 0.24169921875, + 0.294921875, + 0.5185546875, + 0.0855712890625, + 0.0254669189453125, + 0.138671875, + 0.13427734375, + -0.06884765625, + -0.394287109375, + 0.1759033203125, + -0.24560546875, + 0.12469482421875, + 0.367919921875, + -0.1817626953125, + -0.09893798828125, + 0.576171875, + -0.1810302734375, + 0.03875732421875, + 0.496337890625, + 0.09124755859375, + -0.437744140625, + 0.19189453125, + 0.135498046875, + -0.379150390625, + 0.28173828125, + 0.257080078125, + -0.1134033203125, + 0.350830078125, + 0.10089111328125, + 0.482177734375, + 0.6015625, + -0.07183837890625, + 0.67578125, + -0.006755828857421875, + -0.0211334228515625, + 0.4423828125, + 0.373779296875, + 0.151611328125, + 0.1829833984375, + -0.51025390625, + -0.412841796875, + 0.482666015625, + 0.6015625, + 0.023101806640625, + 0.7373046875, + -0.0750732421875, + 0.548828125, + 0.310302734375, + 0.9140625, + -0.1444091796875, + -0.2294921875, + -0.05206298828125, + 0.049163818359375, + -0.2274169921875, + -0.81298828125, + -0.1673583984375, + -0.0038909912109375, + -0.53759765625, + 0.435302734375, + 0.29931640625, + -0.62890625, + 0.031890869140625, + -0.16650390625, + -0.2264404296875, + 0.2166748046875, + -0.094482421875, + -0.0994873046875, + 0.1805419921875, + -0.69140625, + -0.42333984375, + 0.0278167724609375, + 0.127685546875, + -0.2078857421875, + 0.046356201171875, + 0.39306640625, + -2.359375, + 0.1492919921875, + -0.365234375, + -0.125732421875, + -0.76416015625, + -0.093994140625, + -0.1043701171875, + 0.2237548828125, + -0.26123046875, + 0.76904296875, + -0.248046875, + -0.1484375, + -0.341552734375, + -0.224853515625, + 0.02294921875, + -0.1273193359375, + 0.32666015625, + -0.107421875, + -0.10931396484375, + 0.031494140625, + -0.043121337890625, + -0.181640625, + -0.501953125, + -0.07122802734375, + 0.0350341796875, + -0.2139892578125, + 0.059356689453125, + -0.1104736328125, + -12.5390625, + -0.17724609375, + -0.51806640625, + 0.52099609375, + -0.275634765625, + -0.415771484375, + 0.08636474609375, + 0.6279296875, + 0.042327880859375, + 0.91357421875, + -0.7451171875, + -0.353515625, + -0.0435791015625, + -0.34814453125, + 0.1463623046875, + 0.279296875, + -0.006053924560546875, + -0.175048828125, + -0.07208251953125, + 0.28857421875, + 0.0623779296875, + -0.170654296875, + -0.57373046875, + -0.307861328125, + -0.61376953125, + 0.064453125, + 0.10040283203125, + -0.12249755859375, + 0.053375244140625, + -0.0155029296875, + 0.9970703125, + -0.0170745849609375, + 0.08184814453125, + 0.33642578125, + 0.2342529296875, + -0.2318115234375, + -0.08251953125, + -0.39013671875, + 0.048095703125, + -0.298583984375, + 1.205078125, + -0.1558837890625, + -0.28466796875, + -0.54248046875, + -0.0079498291015625, + -0.29638671875, + 0.66357421875, + 0.19921875, + 0.44189453125, + 0.34619140625, + 0.2890625, + 0.20458984375, + 0.12445068359375, + 0.263427734375, + -0.287353515625, + 0.1959228515625, + 0.08648681640625, + -0.07989501953125, + -0.23046875, + -0.048980712890625, + -0.0234527587890625, + 0.0308685302734375, + 0.50927734375, + 0.269287109375, + 0.470947265625, + -0.3515625, + -0.1947021484375, + -0.2296142578125, + 0.1258544921875, + -0.046051025390625, + 0.4296875, + 0.064453125, + 0.272216796875, + 0.072265625, + 0.256103515625, + -0.018341064453125, + -0.011199951171875, + 0.287109375, + -0.187744140625, + -0.2470703125, + 0.0248260498046875, + 0.18798828125, + 0.369140625, + 0.1859130859375, + 0.51318359375, + -0.1571044921875, + 0.24658203125, + 0.418701171875, + 0.91455078125, + 0.432861328125, + -0.140869140625, + -0.0196990966796875, + -0.244140625, + 0.342529296875, + 0.338623046875, + -0.609375, + 0.14404296875, + -1.466796875, + 0.10284423828125, + 0.5791015625, + -0.488525390625, + -0.18310546875, + -0.5390625, + -0.41357421875, + -0.62451171875, + 0.29736328125, + 0.037872314453125, + -0.39453125, + -0.12164306640625, + -0.1346435546875, + -0.375244140625, + -0.0999755859375, + 0.3896484375, + -0.165283203125, + -0.517578125, + 0.06317138671875, + 0.42041015625, + 0.1961669921875, + 0.12164306640625, + 0.56640625, + -0.078369140625, + -0.1783447265625, + -0.350341796875, + 0.08612060546875, + 11.09375, + -0.193115234375, + 0.638671875, + 0.49609375, + -0.498779296875, + 0.16259765625, + 0.32470703125, + 0.5302734375, + 0.49853515625, + -0.65966796875, + -0.94091796875, + -0.041412353515625, + 0.463623046875, + 0.11834716796875, + -0.47900390625, + 0.25244140625, + 0.46826171875, + 0.0777587890625, + -0.05560302734375, + 0.5419921875, + -0.48583984375, + 0.630859375, + 0.277099609375, + 0.313232421875, + -0.09954833984375, + 0.07550048828125, + 0.01261138916015625, + 0.65087890625, + -0.01079559326171875, + -0.077392578125, + 0.11322021484375, + -0.227783203125, + 0.1246337890625, + -0.431884765625, + 0.1763916015625, + -0.60302734375, + 0.2264404296875, + 0.296630859375, + -0.3349609375, + -0.0023040771484375, + -0.1368408203125, + -0.4296875, + 0.03179931640625, + -1.041015625, + -0.849609375, + 0.08282470703125, + 0.62939453125, + -0.326416015625, + 0.18896484375, + 0.40869140625, + 0.05841064453125, + -0.07391357421875, + -0.470458984375, + -0.14501953125, + -0.65087890625, + -0.150634765625, + 0.06378173828125, + -0.7314453125, + 0.06561279296875, + 0.76025390625, + 0.22998046875, + -0.399658203125, + 0.1884765625, + 0.29443359375, + -0.6240234375, + -0.306396484375, + 0.1612548828125, + -0.28955078125, + -0.6435546875, + -0.01000213623046875, + 0.564453125, + -0.0131988525390625, + 0.299072265625, + -0.0295562744140625, + -0.40185546875, + 0.25830078125, + 6.580352783203125e-05, + -0.3115234375, + 0.033935546875, + -0.0880126953125, + 0.223876953125, + -0.328857421875, + -0.11358642578125, + 0.06439208984375, + 0.14013671875, + 0.48095703125, + 0.556640625, + -0.271728515625, + -1.8212890625, + -0.12078857421875, + -0.1739501953125, + -0.91650390625, + -0.447265625, + 0.180419921875, + 0.6494140625, + 0.00035500526428222656, + -0.23095703125, + -0.005664825439453125, + -0.787109375, + -0.2117919921875, + 0.238037109375, + 0.053955078125, + -0.00911712646484375, + 0.09051513671875, + 0.057525634765625, + 0.199462890625, + 0.1065673828125, + -0.037811279296875, + 0.2724609375, + -0.0565185546875, + 0.4453125, + -0.1824951171875, + -0.07171630859375, + 0.1258544921875, + -0.05780029296875, + 0.3310546875, + -0.1885986328125, + 0.223876953125, + -0.11669921875, + -0.09320068359375, + 0.6796875, + 0.463134765625, + -0.1151123046875, + -0.10345458984375, + 0.9521484375, + 0.480224609375, + 0.0926513671875, + 0.2548828125, + 0.12841796875, + 0.353759765625, + 0.53076171875, + -0.4404296875, + -0.52880859375, + -0.1378173828125, + 0.029510498046875, + -0.407958984375, + 0.1527099609375, + -0.1304931640625, + 0.312744140625, + -0.016143798828125, + -0.10784912109375, + -0.0928955078125, + -0.433837890625, + 0.2420654296875, + 0.1614990234375, + -0.5810546875, + 0.0665283203125, + -0.218017578125, + -0.06744384765625, + -0.31689453125, + -0.06787109375, + -0.063720703125, + -0.420654296875, + 0.1513671875, + -0.1632080078125, + 0.75390625, + 0.317138671875, + -0.4560546875, + -0.145263671875, + 0.55615234375, + -0.6171875, + 0.277099609375, + 0.0941162109375, + 0.182861328125, + -0.2030029296875, + 0.38134765625, + 0.1688232421875, + 0.1578369140625, + 0.1190185546875, + 0.347412109375, + 0.3486328125, + -0.359375, + -0.9091796875, + 0.0548095703125, + 0.10174560546875, + -0.3486328125, + 0.264404296875, + -0.798828125, + 0.08587646484375, + 0.08038330078125, + 0.35595703125, + 0.33544921875, + -0.18505859375, + 0.09552001953125, + -0.71923828125, + 0.30908203125, + 0.2337646484375, + -0.20703125, + 0.54150390625, + -0.114501953125, + 0.013885498046875, + 0.1988525390625, + -0.1873779296875, + 0.0638427734375, + -0.62939453125, + -0.09820556640625, + 0.301025390625, + -0.6513671875, + 0.1280517578125, + -0.416015625, + 0.1966552734375, + 0.45263671875, + 0.27490234375, + -0.421630859375, + 1.4736328125, + 0.0106353759765625, + 0.07696533203125, + -0.281494140625, + 0.5302734375, + -0.54345703125, + 0.226806640625, + -0.1661376953125, + -0.211669921875, + 0.042388916015625, + 0.328369140625, + 0.693359375, + 0.2080078125, + -0.451416015625, + 0.036102294921875, + -0.2225341796875, + -0.371826171875, + -0.6337890625, + 0.072998046875, + 0.1712646484375, + 0.28955078125, + -0.045562744140625, + -0.436767578125, + 0.2841796875, + -0.88525390625, + 0.417236328125, + -0.57373046875, + -0.05206298828125, + 0.119384765625, + 0.08599853515625, + -0.10137939453125, + -0.5302734375, + 0.08843994140625, + -0.141845703125, + -0.06201171875, + -0.440673828125, + -0.25830078125, + -0.1640625, + -0.25634765625, + -0.2021484375, + 0.446533203125, + 0.1524658203125, + -0.126708984375, + 0.352783203125, + 0.0272369384765625, + 0.34375, + 0.211181640625, + 0.042327880859375, + -0.07745361328125, + -0.051666259765625, + 0.34765625, + -0.0138702392578125, + -0.1356201171875, + 0.049652099609375, + -0.0168914794921875, + -0.33837890625, + -0.05548095703125, + 0.2218017578125, + 0.07037353515625, + -0.91015625, + 0.240966796875, + 0.06134033203125, + -0.200927734375, + -0.18310546875, + 0.2489013671875, + -0.2152099609375, + -0.131591796875, + 0.1297607421875, + -0.4375, + 0.693359375, + 0.271484375, + 0.05218505859375, + 0.057647705078125, + -0.196533203125, + 0.1334228515625, + 0.304443359375, + -0.11651611328125, + 0.2276611328125, + 0.327880859375, + -0.849609375, + 0.8544921875, + 0.274658203125, + 0.29345703125, + 0.11578369140625, + -0.2430419921875, + 0.195556640625, + 0.195068359375, + 0.279541015625, + 0.33740234375, + -0.49267578125, + 0.890625, + 0.43603515625, + 0.08782958984375, + 0.1571044921875, + 0.323974609375, + 0.60498046875, + 0.57958984375, + 0.404052734375, + 0.0965576171875, + -0.01528167724609375, + 0.515625, + -0.06396484375, + 0.5869140625, + 0.123291015625, + -0.05450439453125, + -0.78369140625, + -0.040252685546875, + -0.00777435302734375, + 0.1846923828125, + 0.2239990234375, + -0.337890625, + -0.322265625, + 0.402099609375, + 0.466552734375, + 0.10711669921875, + -0.391357421875, + 0.098876953125, + -0.6328125, + 0.77685546875, + 0.260986328125, + 0.63916015625, + -0.020294189453125, + -0.267822265625, + 0.18408203125 +] \ No newline at end of file diff --git a/executors/glid3/executor.py b/executors/glid3/executor.py index 96cdf4a..9ebe1d9 100644 --- a/executors/glid3/executor.py +++ b/executors/glid3/executor.py @@ -4,12 +4,13 @@ import tempfile import time from typing import Dict +import json from jina import Executor, DocumentArray, Document, requests class GLID3Diffusion(Executor): - def __init__(self, glid3_path: str, clip_as_service_url: str, steps: int = 100, **kwargs): + def __init__(self, glid3_path: str, clip_blank_encoding_file_path: str, steps: int = 100, **kwargs): super().__init__(**kwargs) os.environ['GLID_MODEL_PATH'] = glid3_path os.environ['GLID3_STEPS'] = str(steps) @@ -20,7 +21,8 @@ def __init__(self, glid3_path: str, clip_as_service_url: str, steps: int = 100, assert static_args self.logger.info('Generating blank embeddings') - self.blank_bert_embedding, self.blank_clip_embedding = generate_blank_embeddings('a', clip_as_service_url) + with open(clip_blank_encoding_file_path, 'r') as f: + self.blank_bert_embedding, self.blank_clip_embedding = generate_blank_embeddings('a', json.load(f)) def run_glid3(self, d: Document, text: str, skip_rate: float, num_images: int): request_time = time.time() diff --git a/flow.yml b/flow.yml index f5d4fbe..1096840 100644 --- a/flow.yml +++ b/flow.yml @@ -24,7 +24,7 @@ executors: uses: GLID3Diffusion uses_with: glid3_path: ../glid-3-xl - clip_as_service_url: https://demo-cas.jina.ai:8443 + clip_blank_encoding_file_path: ./clip_blank_encoding.json steps: 100 py_modules: - executors/glid3/executor.py From f6a81aeb33c6418723fbbc0bd8cad56bdef16837 Mon Sep 17 00:00:00 2001 From: Delgermurun Purevkhuu Date: Fri, 5 Aug 2022 18:01:54 +0200 Subject: [PATCH 11/13] fix: make embedding file path hardcoded in code --- .../glid3/clip_blank_encoding.json | 0 executors/glid3/executor.py | 4 ++-- flow.yml | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) rename clip_blank_encoding.json => executors/glid3/clip_blank_encoding.json (100%) diff --git a/clip_blank_encoding.json b/executors/glid3/clip_blank_encoding.json similarity index 100% rename from clip_blank_encoding.json rename to executors/glid3/clip_blank_encoding.json diff --git a/executors/glid3/executor.py b/executors/glid3/executor.py index 9ebe1d9..bac9e34 100644 --- a/executors/glid3/executor.py +++ b/executors/glid3/executor.py @@ -10,7 +10,7 @@ class GLID3Diffusion(Executor): - def __init__(self, glid3_path: str, clip_blank_encoding_file_path: str, steps: int = 100, **kwargs): + def __init__(self, glid3_path: str, steps: int = 100, **kwargs): super().__init__(**kwargs) os.environ['GLID_MODEL_PATH'] = glid3_path os.environ['GLID3_STEPS'] = str(steps) @@ -21,7 +21,7 @@ def __init__(self, glid3_path: str, clip_blank_encoding_file_path: str, steps: i assert static_args self.logger.info('Generating blank embeddings') - with open(clip_blank_encoding_file_path, 'r') as f: + with open(os.path.join(os.path.dirname(__file__), 'clip_blank_encoding.json')) as f: self.blank_bert_embedding, self.blank_clip_embedding = generate_blank_embeddings('a', json.load(f)) def run_glid3(self, d: Document, text: str, skip_rate: float, num_images: int): diff --git a/flow.yml b/flow.yml index 1096840..f6cb4a5 100644 --- a/flow.yml +++ b/flow.yml @@ -24,7 +24,6 @@ executors: uses: GLID3Diffusion uses_with: glid3_path: ../glid-3-xl - clip_blank_encoding_file_path: ./clip_blank_encoding.json steps: 100 py_modules: - executors/glid3/executor.py From 2e789a4442cbc7cd60ff1902569affb289d00174 Mon Sep 17 00:00:00 2001 From: Delgermurun Purevkhuu Date: Mon, 8 Aug 2022 09:27:34 +0200 Subject: [PATCH 12/13] chore: dockerfile clone master branch of glid --- Dockerfile | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b3f81bc..6fb166c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN apt-get update \ RUN if [ -n "${APT_PACKAGES}" ]; then apt-get update && apt-get install --no-install-recommends -y ${APT_PACKAGES}; fi && \ git clone --depth=1 https://github.com/JingyunLiang/SwinIR.git && \ git clone --depth=1 https://github.com/CompVis/latent-diffusion.git && \ - git clone --depth=1 -b pref-remove-clip-as-service https://github.com/jina-ai/glid-3-xl.git && \ + git clone --depth=1 https://github.com/jina-ai/glid-3-xl.git && \ pip install jax[cuda11_cudnn82]==0.3.13 -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html && \ pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113 && \ cd latent-diffusion && pip install --timeout=1000 -e . && cd - && \ diff --git a/README.md b/README.md index 6969ceb..82cdd92 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ You can modify and extend the server flow as you like, e.g. changing the model, ### Run your own CLIP By default [`CLIPTorchEncoder`](https://hub.jina.ai/executor/gzpbl8jh) runs as an [external executor](https://docs.jina.ai/fundamentals/flow/add-executors/#external-executors). -If want to run your own CLIP, you can do that by just changing `external: true` to `external: false` in [`flow.yml`](./flow.yml). +If you want to run your own CLIP, you can do that by just changing `external: true` to `external: false` in [`flow.yml`](./flow.yml). ## Support From 659750fa561832cb6790df5adf33a3f670fc9a18 Mon Sep 17 00:00:00 2001 From: Delgermurun Purevkhuu Date: Mon, 8 Aug 2022 11:08:13 +0200 Subject: [PATCH 13/13] docs: added change notes --- README.md | 1 + requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 82cdd92..b8016f8 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ DALL·E Flow is in client-server architecture. ## Updates +- ⚠️ **2022/8/8** Started using CLIP-as-servive as an [external executor](https://docs.jina.ai/fundamentals/flow/add-executors/#external-executors). Now you can easily [deploy your own CLIP executor](#run-your-own-clip) if you want. There is [a small breaking change](https://github.com/jina-ai/dalle-flow/pull/74/files#diff-b335630551682c19a781afebcf4d07bf978fb1f8ac04c6bf87428ed5106870f5R103) as a result of this improvement, so [please _reopen_ the notebook in Google Colab](https://colab.research.google.com/github/jina-ai/dalle-flow/blob/main/client.ipynb). - ⚠️ **2022/7/6** Demo server migration to AWS EKS for better availability and robustness, **server URL is now changing to `grpcs://dalle-flow.dev.jina.ai`**. All connections are now with TLS encryption, [please _reopen_ the notebook in Google Colab](https://colab.research.google.com/github/jina-ai/dalle-flow/blob/main/client.ipynb). - ⚠️ **2022/6/25** Unexpected downtime between 6/25 0:00 - 12:00 CET due to out of GPU quotas. The new server now has 2 GPUs, add healthcheck in client notebook. - **2022/6/3** Reduce default number of images to 2 per pathway, 4 for diffusion. diff --git a/requirements.txt b/requirements.txt index ec56e2b..c81dbf7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # jina-related -jina>=3.4.5 -docarray>=0.13.5 +jina>=3.7.4 +docarray>=0.14.8 # dalle-mini flax git+https://github.com/openai/CLIP.git