Merge branch 'development' into dependabot/npm_and_yarn/json-schema-t…

…o-typescript-15.0.2
huridocs · Sep 20, 2024 · 8653082 · 8653082
2 parents 339e824 + e2499f8
commit 8653082
Show file tree

Hide file tree

Showing 9 changed files with 141 additions and 50 deletions.
diff --git a/app/api/files/S3Storage.ts b/app/api/files/S3Storage.ts
@@ -6,44 +6,56 @@ import {
   S3Client,
   _Object,
 } from '@aws-sdk/client-s3';
-import { NodeHttpHandler } from '@smithy/node-http-handler';
 import { config } from 'api/config';
 
+class S3TimeoutError extends Error {
+  readonly s3TimeoutError: Error;
+
+  constructor(s3TimeoutError: Error) {
+    super(s3TimeoutError.message);
+    this.s3TimeoutError = s3TimeoutError;
+  }
+}
+
+const catchS3Errors = async <T>(cb: () => Promise<T>): Promise<T> => {
+  try {
+    return await cb();
+  } catch (err) {
+    if (err.name === 'TimeoutError') {
+      throw new S3TimeoutError(err);
+    }
+    throw err;
+  }
+};
+
 export class S3Storage {
   private client: S3Client;
 
-  constructor() {
-    this.client = new S3Client({
-      requestHandler: new NodeHttpHandler({
-        socketTimeout: 30000,
-      }),
-      apiVersion: 'latest',
-      region: 'placeholder-region',
-      endpoint: config.s3.endpoint,
-      credentials: config.s3.credentials,
-      forcePathStyle: true,
-    });
+  constructor(s3Client: S3Client) {
+    this.client = s3Client;
   }
 
   static bucketName() {
     return config.s3.bucket;
   }
 
   async upload(key: string, body: Buffer) {
-    return this.client.send(
-      new PutObjectCommand({ Bucket: S3Storage.bucketName(), Key: key, Body: body })
+    return catchS3Errors(async () =>
+      this.client.send(
+        new PutObjectCommand({ Bucket: S3Storage.bucketName(), Key: key, Body: body })
+      )
     );
   }
 
   async get(key: string) {
-    const response = await this.client.send(
-      new GetObjectCommand({
-        Bucket: S3Storage.bucketName(),
-        Key: key,
-      })
+    return catchS3Errors(async () =>
+      this.client.send(
+        new GetObjectCommand({
+          Bucket: S3Storage.bucketName(),
+          Key: key,
+        })
+      )
     );
-
-    return response;
   }
 
   async list(prefix?: string) {
@@ -61,21 +73,26 @@ export class S3Storage {
       return response.NextContinuationToken;
     };
 
-    let continuationToken = await requestNext();
-    while (continuationToken) {
-      // eslint-disable-next-line no-await-in-loop
-      continuationToken = await requestNext(continuationToken);
-    }
-
-    return objects;
+    return catchS3Errors(async () => {
+      let continuationToken = await requestNext();
+      while (continuationToken) {
+        // eslint-disable-next-line no-await-in-loop
+        continuationToken = await requestNext(continuationToken);
+      }
+      return objects;
+    });
   }
 
   async delete(key: string) {
-    await this.client.send(
-      new DeleteObjectCommand({
-        Bucket: S3Storage.bucketName(),
-        Key: key,
-      })
+    return catchS3Errors(async () =>
+      this.client.send(
+        new DeleteObjectCommand({
+          Bucket: S3Storage.bucketName(),
+          Key: key,
+        })
+      )
     );
   }
 }
+
+export { S3TimeoutError };
diff --git a/app/api/files/specs/s3Storage.spec.ts b/app/api/files/specs/s3Storage.spec.ts
@@ -0,0 +1,45 @@
+import { S3Storage, S3TimeoutError } from '../S3Storage';
+
+let s3Storage: S3Storage;
+
+class S3TimeoutClient {
+  // eslint-disable-next-line class-methods-use-this
+  send() {
+    const error = new Error();
+    error.name = 'TimeoutError';
+    throw error;
+  }
+}
+
+describe('s3Storage', () => {
+  beforeAll(async () => {
+    // @ts-ignore
+    s3Storage = new S3Storage(new S3TimeoutClient());
+  });
+
+  describe('get', () => {
+    it('should throw S3TimeoutError on timeout', async () => {
+      await expect(s3Storage.get('dummy_key')).rejects.toBeInstanceOf(S3TimeoutError);
+    });
+  });
+
+  describe('upload', () => {
+    it('should throw S3TimeoutError on timeout', async () => {
+      await expect(
+        s3Storage.upload('dummy_key', Buffer.from('dummy buffer', 'utf-8'))
+      ).rejects.toBeInstanceOf(S3TimeoutError);
+    });
+  });
+
+  describe('delete', () => {
+    it('should throw S3TimeoutError on timeout', async () => {
+      await expect(s3Storage.delete('dummy_key')).rejects.toBeInstanceOf(S3TimeoutError);
+    });
+  });
+
+  describe('list', () => {
+    it('should throw S3TimeoutError on timeout', async () => {
+      await expect(s3Storage.list()).rejects.toBeInstanceOf(S3TimeoutError);
+    });
+  });
+});
diff --git a/app/api/files/storage.ts b/app/api/files/storage.ts
@@ -1,6 +1,7 @@
-import { NoSuchKey } from '@aws-sdk/client-s3';
+import { NoSuchKey, S3Client } from '@aws-sdk/client-s3';
 import { config } from 'api/config';
 import { tenants } from 'api/tenants';
+import { NodeHttpHandler } from '@smithy/node-http-handler';
 // eslint-disable-next-line node/no-restricted-import
 import { createReadStream, createWriteStream } from 'fs';
 // eslint-disable-next-line node/no-restricted-import
@@ -9,6 +10,7 @@ import path from 'path';
 import { FileType } from 'shared/types/fileType';
 import { Readable } from 'stream';
 import { pipeline } from 'stream/promises';
+import { FileNotFound } from './FileNotFound';
 import {
   activityLogPath,
   attachmentsPath,
@@ -18,14 +20,24 @@ import {
   uploadsPath,
 } from './filesystem';
 import { S3Storage } from './S3Storage';
-import { FileNotFound } from './FileNotFound';
 
 type FileTypes = NonNullable<FileType['type']> | 'activitylog' | 'segmentation';
 
 let s3Instance: S3Storage;
 const s3 = () => {
   if (config.s3.endpoint && !s3Instance) {
-    s3Instance = new S3Storage();
+    s3Instance = new S3Storage(
+      new S3Client({
+        requestHandler: new NodeHttpHandler({
+          socketTimeout: 30000,
+        }),
+        apiVersion: 'latest',
+        region: 'placeholder-region',
+        endpoint: config.s3.endpoint,
+        credentials: config.s3.credentials,
+        forcePathStyle: true,
+      })
+    );
   }
   return s3Instance;
 };

diff --git a/app/api/services/convertToPDF/specs/convertToPdfWorker.spec.ts b/app/api/services/convertToPDF/specs/convertToPdfWorker.spec.ts
@@ -22,13 +22,13 @@ describe('convertToPdfWorker', () => {
 
   const recreateRedisQueue = async () => {
     try {
-      await redisSMQ.deleteQueueAsync({ qname: 'convert-to-pdf_results' });
+      await redisSMQ.deleteQueueAsync({ qname: 'development_convert-to-pdf_results' });
     } catch (err) {
       if (err instanceof Error && err.name !== 'queueNotFound') {
         throw err;
       }
     }
-    await redisSMQ.createQueueAsync({ qname: 'convert-to-pdf_results' });
+    await redisSMQ.createQueueAsync({ qname: 'development_convert-to-pdf_results' });
   };
 
   beforeAll(async () => {
@@ -88,7 +88,7 @@ describe('convertToPdfWorker', () => {
       };
 
       await redisSMQ.sendMessageAsync({
-        qname: 'convert-to-pdf_results',
+        qname: 'development_convert-to-pdf_results',
         message: JSON.stringify(message),
       });
     });
@@ -162,7 +162,7 @@ describe('convertToPdfWorker', () => {
       jest.spyOn(handleError, 'handleError').mockImplementationOnce(() => {});
 
       await redisSMQ.sendMessageAsync({
-        qname: 'convert-to-pdf_results',
+        qname: 'development_convert-to-pdf_results',
         message: JSON.stringify(message),
       });
 

diff --git a/app/api/services/tasksmanager/TaskManager.ts b/app/api/services/tasksmanager/TaskManager.ts
@@ -52,8 +52,8 @@ export class TaskManager<
 
   constructor(service: Service<T, R>) {
     this.service = service;
-    this.taskQueue = `${service.serviceName}_tasks`;
-    this.resultsQueue = `${service.serviceName}_results`;
+    this.taskQueue = `${config.ENVIRONMENT}_${service.serviceName}_tasks`;
+    this.resultsQueue = `${config.ENVIRONMENT}_${service.serviceName}_results`;
     const redisUrl = `redis://${config.redis.host}:${config.redis.port}`;
     this.redisClient = Redis.createClient(redisUrl);
     this.redisSMQ = new RedisSMQ({ client: this.redisClient });

diff --git a/app/api/services/tasksmanager/specs/ExternalDummyService.ts b/app/api/services/tasksmanager/specs/ExternalDummyService.ts
@@ -122,16 +122,16 @@ export class ExternalDummyService {
   }
 
   async resetQueue() {
-    await this.deleteQueue(`${this.serviceName}_tasks`);
-    await this.deleteQueue(`${this.serviceName}_results`);
+    await this.deleteQueue(`development_${this.serviceName}_tasks`);
+    await this.deleteQueue(`development_${this.serviceName}_results`);
 
-    await this.createQueue(`${this.serviceName}_tasks`);
-    await this.createQueue(`${this.serviceName}_results`);
+    await this.createQueue(`development_${this.serviceName}_tasks`);
+    await this.createQueue(`development_${this.serviceName}_results`);
   }
 
   async readFirstTaskMessage() {
     const message: RedisSMQ.QueueMessage | {} = await this.rsmq.receiveMessageAsync({
-      qname: `${this.serviceName}_tasks`,
+      qname: `development_${this.serviceName}_tasks`,
     });
     const queueMessage = message as QueueMessage;
 
@@ -140,7 +140,7 @@ export class ExternalDummyService {
     }
 
     await this.rsmq.deleteMessageAsync({
-      qname: `${this.serviceName}_tasks`,
+      qname: `development_${this.serviceName}_tasks`,
       id: queueMessage.id,
     });
 
@@ -186,7 +186,7 @@ export class ExternalDummyService {
   async sendFinishedMessage(task: ResultsMessage) {
     try {
       await this.rsmq.sendMessageAsync({
-        qname: `${this.serviceName}_results`,
+        qname: `development_${this.serviceName}_results`,
         message: JSON.stringify(task),
       });
     } catch (err) {

diff --git a/app/api/utils/handleError.js b/app/api/utils/handleError.js
@@ -5,6 +5,7 @@ import { appContext } from 'api/utils/AppContext';
 import { UnauthorizedError } from 'api/authorization.v2/errors/UnauthorizedError';
 import { ValidationError } from 'api/common.v2/validation/ValidationError';
 import { FileNotFound } from 'api/files/FileNotFound';
+import { S3TimeoutError } from 'api/files/S3Storage';
 
 const ajvPrettifier = error => {
   const errorMessage = [error.message];
@@ -62,6 +63,10 @@ const prettifyError = (error, { req = {}, uncaught = false } = {}) => {
     result = { code: 500, message: error.stack, logLevel: 'error' };
   }
 
+  if (error instanceof S3TimeoutError) {
+    result = { code: 408, message: `${error.message}\n${error.stack}`, logLevel: 'debug' };
+  }
+
   if (error instanceof Ajv.ValidationError) {
     result = { code: 422, message: error.message, validations: error.errors, logLevel: 'debug' };
   }

diff --git a/app/api/utils/specs/handleError.spec.js b/app/api/utils/specs/handleError.spec.js
@@ -4,6 +4,7 @@ import { legacyLogger } from 'api/log';
 import { errors as elasticErrors } from '@elastic/elasticsearch';
 import { appContext } from 'api/utils/AppContext';
 import { handleError, prettifyError } from '../handleError';
+import { S3TimeoutError } from 'api/files/S3Storage';
 
 const contextRequestId = '1234';
 
@@ -18,6 +19,17 @@ describe('handleError', () => {
   });
 
   describe('errors by type', () => {
+    describe('and is instance of S3TimeoutError', () => {
+      it('should be a debug logLevel and a 408 http code', () => {
+        const errorInstance = new S3TimeoutError(new Error('timeout'));
+        const error = handleError(errorInstance);
+        expect(error).toMatchObject({
+          code: 408,
+          logLevel: 'debug',
+        });
+        expect(legacyLogger.debug.mock.calls[0][0]).toContain('timeout');
+      });
+    });
     describe('when error is instance of Error', () => {
       it('should return the error with 500 code without the original error and error stack', () => {
         const errorInstance = new Error('error');

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "uwazi",
-  "version": "1.185.0-rc5",
+  "version": "1.185.0-rc6",
   "description": "Uwazi is a free, open-source solution for organising, analysing and publishing your documents.",
   "keywords": [
     "react"