diff --git a/.eslintrc.json b/.eslintrc.json index ca3801f25..65f42ce94 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -43,10 +43,15 @@ }, { "files": ["*.spec.ts", "*.spec.tsx", "*.spec.js", "*.spec.jsx"], + "plugins": ["jest"], "env": { "jest": true }, - "rules": {} + "rules": { + // you should turn the original rule off *only* for test files + "@typescript-eslint/unbound-method": "off", + "jest/unbound-method": "error" + } } ] } diff --git a/libs/extensions/std/exec/src/file-util.spec.ts b/libs/execution/src/lib/util/file-util.spec.ts similarity index 97% rename from libs/extensions/std/exec/src/file-util.spec.ts rename to libs/execution/src/lib/util/file-util.spec.ts index decef2317..5d3074157 100644 --- a/libs/extensions/std/exec/src/file-util.spec.ts +++ b/libs/execution/src/lib/util/file-util.spec.ts @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0-only -import { FileExtension, MimeType } from '@jvalue/jayvee-execution'; +import { FileExtension, MimeType } from '../types'; import { inferFileExtensionFromContentTypeString, diff --git a/libs/extensions/std/exec/src/file-util.ts b/libs/execution/src/lib/util/file-util.ts similarity index 95% rename from libs/extensions/std/exec/src/file-util.ts rename to libs/execution/src/lib/util/file-util.ts index ff828156d..bab374acf 100644 --- a/libs/extensions/std/exec/src/file-util.ts +++ b/libs/execution/src/lib/util/file-util.ts @@ -2,9 +2,10 @@ // // SPDX-License-Identifier: AGPL-3.0-only -import { FileExtension, MimeType } from '@jvalue/jayvee-execution'; import * as mime from 'mime-types'; +import { FileExtension, MimeType } from '../types'; + export function inferMimeTypeFromFileExtensionString( fileExtension: string | undefined, ): MimeType | undefined { diff --git a/libs/execution/src/lib/util/index.ts b/libs/execution/src/lib/util/index.ts index 28697b9ea..9e67d5d9d 100644 --- a/libs/execution/src/lib/util/index.ts +++ b/libs/execution/src/lib/util/index.ts @@ -3,3 +3,5 @@ // SPDX-License-Identifier: AGPL-3.0-only export * from './implements-static-decorator'; +export * from './file-util'; +export * from './string-util'; diff --git a/libs/extensions/std/exec/src/string-util.ts b/libs/execution/src/lib/util/string-util.ts similarity index 100% rename from libs/extensions/std/exec/src/string-util.ts rename to libs/execution/src/lib/util/string-util.ts diff --git a/libs/extensions/std/exec/test/utils.ts b/libs/execution/test/utils/file-util.ts similarity index 91% rename from libs/extensions/std/exec/test/utils.ts rename to libs/execution/test/utils/file-util.ts index 7b7d43d5b..be8689515 100644 --- a/libs/extensions/std/exec/test/utils.ts +++ b/libs/execution/test/utils/file-util.ts @@ -10,13 +10,10 @@ import { FileExtension, MimeType, TextFile, -} from '@jvalue/jayvee-execution'; - -import { inferFileExtensionFromFileExtensionString, inferMimeTypeFromFileExtensionString, -} from '../src/file-util'; -import { splitLines } from '../src/string-util'; + splitLines, +} from '../../src'; export function createBinaryFileFromLocalFile(fileName: string): BinaryFile { const extName = path.extname(fileName); diff --git a/libs/execution/test/utils/index.ts b/libs/execution/test/utils/index.ts new file mode 100644 index 000000000..144030b35 --- /dev/null +++ b/libs/execution/test/utils/index.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +export * from './test-infrastructure-util'; +export * from './file-util'; diff --git a/libs/execution/test/utils.ts b/libs/execution/test/utils/test-infrastructure-util.ts similarity index 99% rename from libs/execution/test/utils.ts rename to libs/execution/test/utils/test-infrastructure-util.ts index 671f10873..1565bf602 100644 --- a/libs/execution/test/utils.ts +++ b/libs/execution/test/utils/test-infrastructure-util.ts @@ -19,7 +19,7 @@ import { TableColumn, blockExecutorRegistry, constraintExecutorRegistry, -} from '../src'; +} from '../../src'; export function clearBlockExecutorRegistry() { blockExecutorRegistry.clear(); diff --git a/libs/extensions/rdbms/exec/src/lib/postgres-loader-executor.spec.ts b/libs/extensions/rdbms/exec/src/lib/postgres-loader-executor.spec.ts new file mode 100644 index 000000000..2b7c9a0c6 --- /dev/null +++ b/libs/extensions/rdbms/exec/src/lib/postgres-loader-executor.spec.ts @@ -0,0 +1,174 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { + constructTable, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + PrimitiveValuetypes, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { PostgresLoaderExecutor } from './postgres-loader-executor'; + +// eslint-disable-next-line no-var +var databaseConnectMock: jest.Mock; +// eslint-disable-next-line no-var +var databaseQueryMock: jest.Mock; +// eslint-disable-next-line no-var +var databaseEndMock: jest.Mock; +jest.mock('pg', () => { + databaseConnectMock = jest.fn(); + databaseQueryMock = jest.fn(); + databaseEndMock = jest.fn(); + const mClient = { + connect: databaseConnectMock, + query: databaseQueryMock, + end: databaseEndMock, + }; + return { Client: jest.fn(() => mClient) }; +}); + +describe('Validation of PostgresLoaderExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/postgres-loader-executor/', + ); + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Table, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new PostgresLoaderExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should diagnose no error on valid loader config', async () => { + const text = readJvTestAsset('valid-postgres-loader.jv'); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: [20.2], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 1, + ); + const result = await parseAndExecuteExecutor(text, inputTable); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.NONE); + expect(databaseConnectMock).toBeCalledTimes(1); + expect(databaseQueryMock).nthCalledWith( + 1, + 'DROP TABLE IF EXISTS "Test";', + ); + expect(databaseQueryMock).nthCalledWith( + 2, + `CREATE TABLE IF NOT EXISTS "Test" ("Column1" text,"Column2" real);`, + ); + expect(databaseQueryMock).nthCalledWith( + 3, + `INSERT INTO "Test" ("Column1","Column2") VALUES ('value 1',20.2)`, + ); + expect(databaseEndMock).toBeCalledTimes(1); + } + }); + + it('should diagnose error on pg client connect error', async () => { + const text = readJvTestAsset('valid-postgres-loader.jv'); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: [20.2], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 1, + ); + databaseConnectMock.mockImplementation(() => { + throw new Error('Connection error'); + }); + const result = await parseAndExecuteExecutor(text, inputTable); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'Could not write to postgres database: Connection error', + ); + expect(databaseConnectMock).toBeCalledTimes(1); + expect(databaseQueryMock).toBeCalledTimes(0); + expect(databaseEndMock).toBeCalledTimes(1); + } + }); +}); diff --git a/libs/extensions/rdbms/exec/src/lib/sqlite-loader-executor.spec.ts b/libs/extensions/rdbms/exec/src/lib/sqlite-loader-executor.spec.ts new file mode 100644 index 000000000..26613a35c --- /dev/null +++ b/libs/extensions/rdbms/exec/src/lib/sqlite-loader-executor.spec.ts @@ -0,0 +1,200 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { + constructTable, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + PrimitiveValuetypes, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; +import * as sqlite3 from 'sqlite3'; + +import { SQLiteLoaderExecutor } from './sqlite-loader-executor'; + +type SqliteRunCallbackType = ( + result: sqlite3.RunResult, + err: Error | null, +) => void; +// eslint-disable-next-line no-var +var databaseMock: jest.Mock; +// eslint-disable-next-line no-var +var databaseRunMock: jest.Mock; +// eslint-disable-next-line no-var +var databaseCloseMock: jest.Mock; +jest.mock('sqlite3', () => { + databaseMock = jest.fn(); + databaseRunMock = jest.fn(); + databaseCloseMock = jest.fn(); + return { + Database: databaseMock, + }; +}); +function mockDatabaseDefault() { + const mockDB = { + close: databaseCloseMock, + run: databaseRunMock, + }; + databaseMock.mockImplementation(() => { + return mockDB; + }); +} + +describe('Validation of SQLiteLoaderExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/sqlite-loader-executor/', + ); + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Table, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new SQLiteLoaderExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should diagnose no error on valid loader config', async () => { + mockDatabaseDefault(); + databaseRunMock.mockImplementation( + (sql: string, callback: SqliteRunCallbackType) => { + callback( + { + lastID: 0, + changes: 0, + } as sqlite3.RunResult, + null, + ); + return this; + }, + ); + const text = readJvTestAsset('valid-sqlite-loader.jv'); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: [20.2], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 1, + ); + const result = await parseAndExecuteExecutor(text, inputTable); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.NONE); + expect(databaseRunMock).toBeCalledTimes(3); + expect(databaseRunMock).nthCalledWith( + 1, + 'DROP TABLE IF EXISTS "Test";', + expect.any(Function), + ); + expect(databaseRunMock).nthCalledWith( + 2, + `CREATE TABLE IF NOT EXISTS "Test" ("Column1" text,"Column2" real);`, + expect.any(Function), + ); + expect(databaseRunMock).nthCalledWith( + 3, + `INSERT INTO "Test" ("Column1","Column2") VALUES ('value 1',20.2)`, + expect.any(Function), + ); + expect(databaseCloseMock).toBeCalledTimes(1); + } + }); + + it('should diagnose error on sqlite database open error', async () => { + databaseMock.mockImplementation(() => { + throw new Error('File not found'); + }); + const text = readJvTestAsset('valid-sqlite-loader.jv'); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: [20.2], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 1, + ); + const result = await parseAndExecuteExecutor(text, inputTable); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'Could not write to sqlite database: File not found', + ); + expect(databaseRunMock).toBeCalledTimes(0); + expect(databaseCloseMock).toBeCalledTimes(0); + } + }); +}); diff --git a/libs/extensions/rdbms/exec/test/assets/postgres-loader-executor/valid-postgres-loader.jv b/libs/extensions/rdbms/exec/test/assets/postgres-loader-executor/valid-postgres-loader.jv new file mode 100644 index 000000000..93b769bc9 --- /dev/null +++ b/libs/extensions/rdbms/exec/test/assets/postgres-loader-executor/valid-postgres-loader.jv @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestFileExtractor { + } + + block TestBlock oftype PostgresLoader { + host: "localhost"; + port: 5432; + username: "postgres"; + password: "postgres"; + database: "TestDB"; + table: "Test"; + } + + TestExtractor -> TestBlock; +} diff --git a/libs/extensions/rdbms/exec/test/assets/sqlite-loader-executor/valid-sqlite-loader.jv b/libs/extensions/rdbms/exec/test/assets/sqlite-loader-executor/valid-sqlite-loader.jv new file mode 100644 index 000000000..3b9354b53 --- /dev/null +++ b/libs/extensions/rdbms/exec/test/assets/sqlite-loader-executor/valid-sqlite-loader.jv @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestFileExtractor { + } + + block TestBlock oftype SQLiteLoader { + table: "Test"; + file: "./test.db"; + } + + TestExtractor -> TestBlock; +} diff --git a/libs/extensions/rdbms/exec/test/mocks/postgres-loader-executor-mock.ts b/libs/extensions/rdbms/exec/test/mocks/postgres-loader-executor-mock.ts index 6aba1c095..3c7d0cd77 100644 --- a/libs/extensions/rdbms/exec/test/mocks/postgres-loader-executor-mock.ts +++ b/libs/extensions/rdbms/exec/test/mocks/postgres-loader-executor-mock.ts @@ -7,7 +7,7 @@ import * as assert from 'assert'; import { BlockExecutorMock } from '@jvalue/jayvee-execution/test'; import { Client } from 'pg'; -type MockedPgClient = jest.Mocked>; +type MockedPgClient = jest.Mocked; export class PostgresLoaderExecutorMock implements BlockExecutorMock { private _pgClient: MockedPgClient | undefined; @@ -26,7 +26,7 @@ export class PostgresLoaderExecutorMock implements BlockExecutorMock { ) => void = defaultPostgresMockRegistration, ) { // setup pg mock - this._pgClient = new Client(); + this._pgClient = new Client() as MockedPgClient; registerMocks(this._pgClient); } restore() { diff --git a/libs/extensions/rdbms/exec/test/mocks/sqlite-loader-executor-mock.ts b/libs/extensions/rdbms/exec/test/mocks/sqlite-loader-executor-mock.ts index 888cce28b..566d379a7 100644 --- a/libs/extensions/rdbms/exec/test/mocks/sqlite-loader-executor-mock.ts +++ b/libs/extensions/rdbms/exec/test/mocks/sqlite-loader-executor-mock.ts @@ -7,7 +7,7 @@ import * as assert from 'assert'; import { BlockExecutorMock } from '@jvalue/jayvee-execution/test'; import * as sqlite3 from 'sqlite3'; -type MockedSqlite3Database = jest.Mocked>; +type MockedSqlite3Database = jest.Mocked; export class SQLiteLoaderExecutorMock implements BlockExecutorMock { private _sqliteClient: MockedSqlite3Database | undefined; @@ -26,7 +26,7 @@ export class SQLiteLoaderExecutorMock implements BlockExecutorMock { ) => void = defaultSQLiteMockRegistration, ) { // setup sqlite3 mock - this._sqliteClient = new sqlite3.Database('test'); + this._sqliteClient = new sqlite3.Database('test') as MockedSqlite3Database; registerMocks(this._sqliteClient); } restore() { diff --git a/libs/extensions/rdbms/exec/test/test-extension/TestBlockTypes.jv b/libs/extensions/rdbms/exec/test/test-extension/TestBlockTypes.jv new file mode 100644 index 000000000..e050777aa --- /dev/null +++ b/libs/extensions/rdbms/exec/test/test-extension/TestBlockTypes.jv @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +builtin blocktype TestTableExtractor { + input inPort oftype None; + output outPort oftype Table; +} diff --git a/libs/extensions/std/exec/src/archive-interpreter-executor.spec.ts b/libs/extensions/std/exec/src/archive-interpreter-executor.spec.ts index 7b174ae11..ae4eb2fcd 100644 --- a/libs/extensions/std/exec/src/archive-interpreter-executor.spec.ts +++ b/libs/extensions/std/exec/src/archive-interpreter-executor.spec.ts @@ -5,7 +5,10 @@ import * as path from 'path'; import * as R from '@jvalue/jayvee-execution'; -import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + createBinaryFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; import { BlockDefinition, IOType, @@ -21,8 +24,6 @@ import { import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; import { NodeFileSystem } from 'langium/node'; -import { createBinaryFileFromLocalFile } from '../test'; - import { ArchiveInterpreterExecutor } from './archive-interpreter-executor'; describe('Validation of ArchiveInterpreterExecutor', () => { diff --git a/libs/extensions/std/exec/src/archive-interpreter-executor.ts b/libs/extensions/std/exec/src/archive-interpreter-executor.ts index ccd07d491..0fa207bed 100644 --- a/libs/extensions/std/exec/src/archive-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/archive-interpreter-executor.ts @@ -18,15 +18,12 @@ import { MimeType, err, implementsStatic, + inferFileExtensionFromFileExtensionString, + inferMimeTypeFromFileExtensionString, } from '@jvalue/jayvee-execution'; import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server'; import * as JSZip from 'jszip'; -import { - inferFileExtensionFromFileExtensionString, - inferMimeTypeFromFileExtensionString, -} from './file-util'; - @implementsStatic() export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< IOType.FILE, diff --git a/libs/extensions/std/exec/src/file-picker-executor.spec.ts b/libs/extensions/std/exec/src/file-picker-executor.spec.ts index 8fd8e3578..f89119a76 100644 --- a/libs/extensions/std/exec/src/file-picker-executor.spec.ts +++ b/libs/extensions/std/exec/src/file-picker-executor.spec.ts @@ -5,7 +5,10 @@ import * as path from 'path'; import * as R from '@jvalue/jayvee-execution'; -import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + createBinaryFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; import { BlockDefinition, IOType, @@ -21,8 +24,6 @@ import { import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; import { NodeFileSystem } from 'langium/node'; -import { createBinaryFileFromLocalFile } from '../test'; - import { FilePickerExecutor } from './file-picker-executor'; describe('Validation of FilePickerExecutor', () => { diff --git a/libs/extensions/std/exec/src/gtfs-rt-interpreter-executor.spec.ts b/libs/extensions/std/exec/src/gtfs-rt-interpreter-executor.spec.ts index 75d8ae8d9..65cdcfb86 100644 --- a/libs/extensions/std/exec/src/gtfs-rt-interpreter-executor.spec.ts +++ b/libs/extensions/std/exec/src/gtfs-rt-interpreter-executor.spec.ts @@ -5,7 +5,10 @@ import * as path from 'path'; import * as R from '@jvalue/jayvee-execution'; -import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + createBinaryFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; import { BlockDefinition, IOType, @@ -21,8 +24,6 @@ import { import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; import { NodeFileSystem } from 'langium/node'; -import { createBinaryFileFromLocalFile } from '../test'; - import { GtfsRTInterpreterExecutor } from './gtfs-rt-interpreter-executor'; describe('Validation of GtfsRTInterpreterExecutor', () => { diff --git a/libs/extensions/std/exec/src/http-extractor-executor.ts b/libs/extensions/std/exec/src/http-extractor-executor.ts index c9f052e85..61257b042 100644 --- a/libs/extensions/std/exec/src/http-extractor-executor.ts +++ b/libs/extensions/std/exec/src/http-extractor-executor.ts @@ -16,16 +16,14 @@ import { MimeType, None, implementsStatic, + inferFileExtensionFromContentTypeString, + inferFileExtensionFromFileExtensionString, + inferMimeTypeFromFileExtensionString, } from '@jvalue/jayvee-execution'; import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server'; import { http, https } from 'follow-redirects'; import { AstNode } from 'langium'; -import { - inferFileExtensionFromContentTypeString, - inferFileExtensionFromFileExtensionString, - inferMimeTypeFromFileExtensionString, -} from './file-util'; import { createBackoffStrategy, isBackoffStrategyHandle, diff --git a/libs/extensions/std/exec/src/text-file-interpreter-executor.spec.ts b/libs/extensions/std/exec/src/text-file-interpreter-executor.spec.ts index 342bceeab..0c9fd6462 100644 --- a/libs/extensions/std/exec/src/text-file-interpreter-executor.spec.ts +++ b/libs/extensions/std/exec/src/text-file-interpreter-executor.spec.ts @@ -5,7 +5,10 @@ import * as path from 'path'; import * as R from '@jvalue/jayvee-execution'; -import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + createBinaryFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; import { BlockDefinition, IOType, @@ -21,8 +24,6 @@ import { import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; import { NodeFileSystem } from 'langium/node'; -import { createBinaryFileFromLocalFile } from '../test'; - import { TextFileInterpreterExecutor } from './text-file-interpreter-executor'; describe('Validation of TextFileInterpreterExecutor', () => { diff --git a/libs/extensions/std/exec/src/text-file-interpreter-executor.ts b/libs/extensions/std/exec/src/text-file-interpreter-executor.ts index c265aae3a..e3fb113e2 100644 --- a/libs/extensions/std/exec/src/text-file-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/text-file-interpreter-executor.ts @@ -12,11 +12,10 @@ import { ExecutionContext, TextFile, implementsStatic, + splitLines, } from '@jvalue/jayvee-execution'; import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server'; -import { splitLines } from './string-util'; - @implementsStatic() export class TextFileInterpreterExecutor extends AbstractBlockExecutor< IOType.FILE, diff --git a/libs/extensions/std/exec/src/text-line-deleter-executor.spec.ts b/libs/extensions/std/exec/src/text-line-deleter-executor.spec.ts index 8ca8a191e..156151fce 100644 --- a/libs/extensions/std/exec/src/text-line-deleter-executor.spec.ts +++ b/libs/extensions/std/exec/src/text-line-deleter-executor.spec.ts @@ -5,7 +5,10 @@ import * as path from 'path'; import * as R from '@jvalue/jayvee-execution'; -import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + createTextFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; import { BlockDefinition, IOType, @@ -21,8 +24,6 @@ import { import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; import { NodeFileSystem } from 'langium/node'; -import { createTextFileFromLocalFile } from '../test'; - import { TextLineDeleterExecutor } from './text-line-deleter-executor'; describe('Validation of TextLineDeleterExecutor', () => { diff --git a/libs/extensions/std/exec/src/text-range-selector-executor.spec.ts b/libs/extensions/std/exec/src/text-range-selector-executor.spec.ts index 73e333359..b9ad72c15 100644 --- a/libs/extensions/std/exec/src/text-range-selector-executor.spec.ts +++ b/libs/extensions/std/exec/src/text-range-selector-executor.spec.ts @@ -5,7 +5,10 @@ import * as path from 'path'; import * as R from '@jvalue/jayvee-execution'; -import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + createTextFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; import { BlockDefinition, IOType, @@ -21,8 +24,6 @@ import { import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; import { NodeFileSystem } from 'langium/node'; -import { createTextFileFromLocalFile } from '../test'; - import { TextRangeSelectorExecutor } from './text-range-selector-executor'; describe('Validation of TextRangeSelectorExecutor', () => { diff --git a/libs/extensions/std/exec/test/index.ts b/libs/extensions/std/exec/test/index.ts index cfe12da6f..460aca6a3 100644 --- a/libs/extensions/std/exec/test/index.ts +++ b/libs/extensions/std/exec/test/index.ts @@ -3,4 +3,3 @@ // SPDX-License-Identifier: AGPL-3.0-only export * from './mocks'; -export * from './utils'; diff --git a/libs/extensions/tabular/exec/jest.config.ts b/libs/extensions/tabular/exec/jest.config.ts index e593333a4..a5a6a3c7d 100644 --- a/libs/extensions/tabular/exec/jest.config.ts +++ b/libs/extensions/tabular/exec/jest.config.ts @@ -5,6 +5,7 @@ export default { displayName: 'extensions-tabular-exec', preset: '../../../../jest.preset.js', + testEnvironment: './test/jsdom-environment-fix.ts', globals: { 'ts-jest': { tsconfig: '/tsconfig.spec.json', diff --git a/libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts new file mode 100644 index 000000000..c39d6a3c4 --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { createWorkbookFromLocalExcelFile } from '../../test/util'; + +import { CellRangeSelectorExecutor } from './cell-range-selector-executor'; + +describe('Validation of CellRangeSelectorExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/cell-range-selector-executor/', + ); + + async function readTestWorkbook(fileName: string): Promise { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/cell-range-selector-executor/', + fileName, + ); + return await createWorkbookFromLocalExcelFile(absoluteFileName); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Sheet, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new CellRangeSelectorExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid selector', async () => { + const text = readJvTestAsset('valid-A1:C*.jv'); + + const testWorkbook = await readTestWorkbook('test-A1:C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getHeaderRow()).toEqual(['0', 'Test', 'true']); + expect(result.right.getData()).toEqual( + expect.arrayContaining([ + ['0', 'Test', 'true'], + ['1', 'Test', 'false'], + ['15', 'Test', 'true'], + ]), + ); + } + }); + + it('should diagnose no error on empty column', async () => { + const text = readJvTestAsset('valid-A1:C*.jv'); + + const testWorkbook = await readTestWorkbook('test-B1:C2.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(2); + expect(result.right.getHeaderRow()).toEqual(['', 'Test', 'true']); + expect(result.right.getData()).toEqual([ + ['', 'Test', 'true'], + ['', 'Test', 'false'], + ]); + } + }); + + it('should diagnose error on selector out of bounds', async () => { + const text = readJvTestAsset('valid-A1:E4.jv'); + + const testWorkbook = await readTestWorkbook('test-A1:C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'The specified cell range does not fit the sheet', + ); + } + }); + + it('should diagnose error on selector on empty sheet', async () => { + const text = readJvTestAsset('valid-A1:C*.jv'); + + const testWorkbook = await readTestWorkbook('test-empty.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'The specified cell range does not fit the sheet', + ); + } + }); + + it('should diagnose error on single column selector on empty sheet', async () => { + const text = readJvTestAsset('valid-A1:A4.jv'); + + const testWorkbook = await readTestWorkbook('test-empty.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'The specified cell range does not fit the sheet', + ); + } + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts new file mode 100644 index 000000000..99ac1d493 --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { createWorkbookFromLocalExcelFile } from '../../test/util'; + +import { CellWriterExecutor } from './cell-writer-executor'; + +describe('Validation of CellWriterExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/cell-writer-executor/', + ); + + async function readTestWorkbook(fileName: string): Promise { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/cell-writer-executor/', + fileName, + ); + return await createWorkbookFromLocalExcelFile(absoluteFileName); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Sheet, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new CellWriterExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid single cell writer', async () => { + const text = readJvTestAsset('valid-single-cell-writer.jv'); + + const testWorkbook = await readTestWorkbook('test-A1:C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getHeaderRow()).toEqual(['16', 'Test', 'true']); + expect(result.right.getData()).toEqual( + expect.arrayContaining([ + ['16', 'Test', 'true'], + ['1', 'Test', 'false'], + ['15', 'Test', 'true'], + ]), + ); + } + }); + + it('should diagnose no error on valid cell range writer', async () => { + const text = readJvTestAsset('valid-cell-range-writer.jv'); + + const testWorkbook = await readTestWorkbook('test-A1:C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getHeaderRow()).toEqual(['16', 'Test2', '']); + expect(result.right.getData()).toEqual( + expect.arrayContaining([ + ['16', 'Test2', ''], + ['1', 'Test', 'false'], + ['15', 'Test', 'true'], + ]), + ); + } + }); + + it('should diagnose error on single cell writer on empty sheet', async () => { + const text = readJvTestAsset('valid-single-cell-writer.jv'); + + const testWorkbook = await readTestWorkbook('test-empty.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'Some specified cells do not exist in the sheet', + ); + } + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts new file mode 100644 index 000000000..7150e817e --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { createWorkbookFromLocalExcelFile } from '../../test/util'; + +import { ColumnDeleterExecutor } from './column-deleter-executor'; + +describe('Validation of ColumnDeleterExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/column-deleter-executor/', + ); + + async function readTestWorkbook(fileName: string): Promise { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/column-deleter-executor/', + fileName, + ); + return await createWorkbookFromLocalExcelFile(absoluteFileName); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Sheet, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new ColumnDeleterExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid single column deleter', async () => { + const text = readJvTestAsset('valid-single-column-deleter.jv'); + + const testWorkbook = await readTestWorkbook('test-A1:C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(2); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getHeaderRow()).toEqual(['Test', 'true']); + expect(result.right.getData()).toEqual( + expect.arrayContaining([ + ['Test', 'true'], + ['Test', 'false'], + ['Test', 'true'], + ]), + ); + } + }); + + it('should diagnose no error on valid multiple column deleter', async () => { + const text = readJvTestAsset('valid-multiple-column-deleter.jv'); + + const testWorkbook = await readTestWorkbook('test-A1:C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(1); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getHeaderRow()).toEqual(['Test']); + expect(result.right.getData()).toEqual( + expect.arrayContaining([['Test'], ['Test'], ['Test']]), + ); + } + }); + + it('should diagnose error on deleting non existing column', async () => { + const text = readJvTestAsset('valid-multiple-column-deleter.jv'); + + const testWorkbook = await readTestWorkbook('test-A1:B2.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'The specified column C does not exist in the sheet', + ); + } + }); + + it('should diagnose only one column deletion on duplicate columns', async () => { + const text = readJvTestAsset('valid-duplicate-column.jv'); + + const testWorkbook = await readTestWorkbook('test-A1:C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(2); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getHeaderRow()).toEqual(['Test', 'true']); + expect(result.right.getData()).toEqual( + expect.arrayContaining([ + ['Test', 'true'], + ['Test', 'false'], + ['Test', 'true'], + ]), + ); + } + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/csv-interpreter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/csv-interpreter-executor.spec.ts new file mode 100644 index 000000000..c4a8b938e --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/csv-interpreter-executor.spec.ts @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { + createTextFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { CSVInterpreterExecutor } from './csv-interpreter-executor'; + +describe('Validation of CSVInterpreterExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/csv-interpreter-executor/', + ); + + function readTestFile(fileName: string): R.TextFile { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/csv-interpreter-executor/', + fileName, + ); + return createTextFileFromLocalFile(absoluteFileName); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.TextFile, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new CSVInterpreterExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid csv file', async () => { + const text = readJvTestAsset('valid-csv-interpreter.jv'); + + const testCsv = readTestFile('valid-csv.csv'); + const result = await parseAndExecuteExecutor(text, testCsv); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(2); + expect(result.right.getNumberOfRows()).toEqual(2); + expect(result.right.getData()).toEqual([ + ['Test', 'true'], + ['Test', 'false'], + ]); + } + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts new file mode 100644 index 000000000..03bea0917 --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { createWorkbookFromLocalExcelFile } from '../../test/util'; + +import { RowDeleterExecutor } from './row-deleter-executor'; + +describe('Validation of RowDeleterExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/row-deleter-executor/', + ); + + async function readTestWorkbook(fileName: string): Promise { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/row-deleter-executor/', + fileName, + ); + return await createWorkbookFromLocalExcelFile(absoluteFileName); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Sheet, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new RowDeleterExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid single row deleter', async () => { + const text = readJvTestAsset('valid-single-row-deleter.jv'); + + const testWorkbook = await readTestWorkbook('test-A1:C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(15); + expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.getData()).not.toEqual( + expect.arrayContaining([['0', 'Test', 'true']]), + ); + } + }); + + it('should diagnose no error on valid multiple row deleter', async () => { + const text = readJvTestAsset('valid-multiple-row-deleter.jv'); + + const testWorkbook = await readTestWorkbook('test-A1:C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(14); + expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.getData()).not.toEqual( + expect.arrayContaining([ + ['0', 'Test', 'true'], + ['4', 'Test', 'true'], + ]), + ); + } + }); + + it('should diagnose error on deleting non existing row', async () => { + const text = readJvTestAsset('valid-multiple-row-deleter.jv'); + + const testWorkbook = await readTestWorkbook('test-A1:B2.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'The specified row 5 does not exist in the sheet', + ); + } + }); + + it('should diagnose only one row deletion on duplicate rows', async () => { + const text = readJvTestAsset('valid-duplicate-row.jv'); + + const testWorkbook = await readTestWorkbook('test-A1:C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(15); + expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.getData()).not.toEqual( + expect.arrayContaining([['0', 'Test', 'true']]), + ); + } + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/sheet-picker-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/sheet-picker-executor.spec.ts new file mode 100644 index 000000000..4af35e9f5 --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/sheet-picker-executor.spec.ts @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { createWorkbookFromLocalExcelFile } from '../../test/util'; + +import { SheetPickerExecutor } from './sheet-picker-executor'; + +describe('Validation of SheetPickerExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/sheet-picker-executor/', + ); + + async function readTestWorkbook(fileName: string): Promise { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/sheet-picker-executor/', + fileName, + ); + return await createWorkbookFromLocalExcelFile(absoluteFileName); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Workbook, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new SheetPickerExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid workbook', async () => { + const text = readJvTestAsset('valid-sheet-picker.jv'); + + const testWorkbook = await readTestWorkbook('test-A1:C16.xlsx'); + const result = await parseAndExecuteExecutor(text, testWorkbook); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getHeaderRow()).toEqual(['0', 'Test', 'true']); + expect(result.right.getData()).toEqual( + expect.arrayContaining([ + ['0', 'Test', 'true'], + ['1', 'Test', 'false'], + ['15', 'Test', 'true'], + ]), + ); + } + }); + + it('should diagnose error on sheet not found', async () => { + const text = readJvTestAsset('valid-custom-sheet-name.jv'); + + const testWorkbook = await readTestWorkbook('test-A1:C16.xlsx'); + const result = await parseAndExecuteExecutor(text, testWorkbook); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'Workbook does not contain a sheet named MyCustomSheet', + ); + } + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts new file mode 100644 index 000000000..feb52fac5 --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts @@ -0,0 +1,297 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { createWorkbookFromLocalExcelFile } from '../../test/util'; + +import { TableInterpreterExecutor } from './table-interpreter-executor'; + +describe('Validation of TableInterpreterExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/table-interpreter-executor/', + ); + + async function readTestWorkbook(fileName: string): Promise { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/table-interpreter-executor/', + fileName, + ); + return await createWorkbookFromLocalExcelFile(absoluteFileName); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Sheet, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new TableInterpreterExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + describe('validation of sheet with header', () => { + it('should diagnose no error on valid sheet', async () => { + const text = readJvTestAsset('valid-with-header.jv'); + + const testWorkbook = await readTestWorkbook('test-with-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getColumn('index')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([0, 1, 2, 15]) as number[], + }), + ); + expect(result.right.getColumn('name')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining(['Test']) as string[], + }), + ); + expect(result.right.getColumn('flag')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([true, false]) as boolean[], + }), + ); + } + }); + + it('should diagnose empty table on empty column parameter', async () => { + const text = readJvTestAsset('valid-empty-columns-with-header.jv'); + + const testWorkbook = await readTestWorkbook('test-with-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(0); + expect(result.right.getNumberOfRows()).toEqual(0); + } + }); + + it('should diagnose empty table on wrong header case', async () => { + const text = readJvTestAsset('valid-with-capitalized-header.jv'); + + const testWorkbook = await readTestWorkbook('test-with-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(0); + expect(result.right.getNumberOfRows()).toEqual(0); + } + }); + + it('should diagnose error on empty sheet', async () => { + const text = readJvTestAsset('valid-with-header.jv'); + + const testWorkbook = await readTestWorkbook('test-empty.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'The input sheet is empty and thus has no header', + ); + } + }); + + it('should diagnose skipping row on wrong cell valuetype', async () => { + const text = readJvTestAsset('valid-wrong-valuetype-with-header.jv'); + + const testWorkbook = await readTestWorkbook('test-with-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(0); + } + }); + }); + + describe('validation of sheet without header', () => { + it('should diagnose no error on valid sheet', async () => { + const text = readJvTestAsset('valid-without-header.jv'); + + const testWorkbook = await readTestWorkbook('test-without-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getColumn('index')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([0, 1, 2, 15]) as number[], + }), + ); + expect(result.right.getColumn('name')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining(['Test']) as string[], + }), + ); + expect(result.right.getColumn('flag')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([true, false]) as boolean[], + }), + ); + } + }); + + it('should diagnose no error on valid sheet with header', async () => { + const text = readJvTestAsset('valid-without-header.jv'); + + const testWorkbook = await readTestWorkbook('test-with-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getColumn('index')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([0, 1, 2, 15]) as number[], + }), + ); + expect(result.right.getColumn('name')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining(['Test']) as string[], + }), + ); + expect(result.right.getColumn('flag')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([true, false]) as boolean[], + }), + ); + } + }); + + it('should diagnose empty table on empty column parameter', async () => { + const text = readJvTestAsset('valid-empty-columns-without-header.jv'); + + const testWorkbook = await readTestWorkbook('test-without-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(0); + expect(result.right.getNumberOfRows()).toEqual(0); + } + }); + + it('should diagnose error on empty sheet', async () => { + const text = readJvTestAsset('valid-without-header.jv'); + + const testWorkbook = await readTestWorkbook('test-empty.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'There are 3 column definitions but the input sheet only has 0 columns', + ); + } + }); + + it('should diagnose skipping row on wrong cell valuetype', async () => { + const text = readJvTestAsset('valid-wrong-valuetype-without-header.jv'); + + const testWorkbook = await readTestWorkbook('test-without-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(0); + } + }); + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts index fb7da129c..77a4df8db 100644 --- a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts +++ b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts @@ -27,7 +27,7 @@ import { rowIndexToString, } from '@jvalue/jayvee-language-server'; -interface ColumnDefinitionEntry { +export interface ColumnDefinitionEntry { sheetColumnIndex: number; columnName: string; valuetype: Valuetype; diff --git a/libs/extensions/tabular/exec/src/lib/table-transformer-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/table-transformer-executor.spec.ts new file mode 100644 index 000000000..a2763b1ff --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/table-transformer-executor.spec.ts @@ -0,0 +1,220 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + PrimitiveValuetypes, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { + ReducedColumnDefinitionEntry, + createTableFromLocalExcelFile, +} from '../../test/util'; + +import { TableTransformerExecutor } from './table-transformer-executor'; + +describe('Validation of TableTransformerExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/table-transformer-executor/', + ); + + async function readTestExcelAllColumns() { + return readTestTable('test-excel.xlsx', [ + { + columnName: 'index', + sheetColumnIndex: 0, + valuetype: PrimitiveValuetypes.Integer, + }, + { + columnName: 'name', + sheetColumnIndex: 1, + valuetype: PrimitiveValuetypes.Text, + }, + { + columnName: 'flag', + sheetColumnIndex: 2, + valuetype: PrimitiveValuetypes.Boolean, + }, + ]); + } + async function readTestTable( + fileName: string, + columnDefinitions: ReducedColumnDefinitionEntry[], + ): Promise { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/table-transformer-executor/', + fileName, + ); + return await createTableFromLocalExcelFile( + absoluteFileName, + columnDefinitions, + ); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Table, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new TableTransformerExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid table transform', async () => { + const text = readJvTestAsset('valid-transfomer.jv'); + + const testTable = await readTestExcelAllColumns(); + const result = await parseAndExecuteExecutor(text, testTable); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(4); + expect(result.right.getNumberOfRows()).toEqual(6); + expect(result.right.getColumn('index')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([0, 1, 2, 5]) as number[], + }), + ); + expect(result.right.getColumn('index2')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([0, 2, 4, 10]) as number[], + }), + ); + } + }); + + it('should diagnose no error on column overwrite', async () => { + const text = readJvTestAsset('valid-column-overwrite.jv'); + + const testTable = await readTestExcelAllColumns(); + const result = await parseAndExecuteExecutor(text, testTable); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(6); + expect(result.right.getColumn('index')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([0, 2, 4, 10]) as number[], + }), + ); + } + }); + + it('should diagnose no error on column type change', async () => { + const text = readJvTestAsset('valid-column-type-change.jv'); + + const testTable = await readTestExcelAllColumns(); + const result = await parseAndExecuteExecutor(text, testTable); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(6); + expect(result.right.getColumn('index')).toEqual( + expect.objectContaining({ + values: [false, true, true, true, true, true], + valuetype: PrimitiveValuetypes.Boolean, + }), + ); + } + }); + + it('should diagnose no error on empty table', async () => { + const text = readJvTestAsset('valid-transfomer.jv'); + + const testTable = await readTestTable('test-empty.xlsx', [ + { + columnName: 'index', + sheetColumnIndex: 0, + valuetype: PrimitiveValuetypes.Integer, + }, + ]); + const result = await parseAndExecuteExecutor(text, testTable); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(2); + expect(result.right.getNumberOfRows()).toEqual(0); + expect(result.right.getColumn('index')?.values).toHaveLength(0); + expect(result.right.getColumn('index2')?.values).toHaveLength(0); + } + }); + + it('should diagnose error on missing input column', async () => { + const text = readJvTestAsset('valid-missing-input-column.jv'); + + const testTable = await readTestExcelAllColumns(); + const result = await parseAndExecuteExecutor(text, testTable); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'The specified input column "id" does not exist in the given table', + ); + } + }); + + it('should diagnose error on transform type missmatch', async () => { + const text = readJvTestAsset('valid-transform-type-missmatch.jv'); + + const testTable = await readTestExcelAllColumns(); + const result = await parseAndExecuteExecutor(text, testTable); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'Type text of column "name" is not convertible to type integer', + ); + } + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/xlsx-interpreter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/xlsx-interpreter-executor.spec.ts new file mode 100644 index 000000000..f289ea96d --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/xlsx-interpreter-executor.spec.ts @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { + createBinaryFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { XLSXInterpreterExecutor } from './xlsx-interpreter-executor'; + +describe('Validation of XLSXInterpreterExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/xlsx-interpreter-executor/', + ); + + function readTestFile(fileName: string): R.BinaryFile { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/xlsx-interpreter-executor/', + fileName, + ); + return createBinaryFileFromLocalFile(absoluteFileName); + } + + function expectSheetRowAndColumnSize( + sheet: R.Sheet | undefined, + cols: number, + rows: number, + ) { + expect(sheet?.getNumberOfColumns()).toEqual(cols); + expect(sheet?.getNumberOfRows()).toEqual(rows); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.BinaryFile, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new XLSXInterpreterExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on single sheet excel', async () => { + const text = readJvTestAsset('valid-excel-interpreter.jv'); + + const testFile = readTestFile('test-excel.xlsx'); + const result = await parseAndExecuteExecutor(text, testFile); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.WORKBOOK); + const sheets = result.right.getSheets(); + expect(sheets.size).toEqual(1); + expectSheetRowAndColumnSize(sheets.get('Sheet1'), 3, 6); + } + }); + + it('should diagnose no error on multiple sheet excel', async () => { + const text = readJvTestAsset('valid-excel-interpreter.jv'); + + const testFile = readTestFile('test-multiple-sheets.xlsx'); + const result = await parseAndExecuteExecutor(text, testFile); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.WORKBOOK); + const sheets = result.right.getSheets(); + expect(sheets.size).toEqual(3); + expectSheetRowAndColumnSize(sheets.get('Sheet1'), 3, 6); + expectSheetRowAndColumnSize(sheets.get('CustomSheet'), 0, 0); + expectSheetRowAndColumnSize(sheets.get('Sheet3'), 0, 0); + } + }); + + it('should diagnose no error on empty excel', async () => { + const text = readJvTestAsset('valid-excel-interpreter.jv'); + + const testFile = readTestFile('test-empty.xlsx'); + const result = await parseAndExecuteExecutor(text, testFile); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.WORKBOOK); + const sheets = result.right.getSheets(); + expect(sheets.size).toEqual(1); + expectSheetRowAndColumnSize(sheets.get('Sheet1'), 0, 0); + } + }); +}); diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-A1:C16.xlsx b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-A1:C16.xlsx new file mode 100644 index 000000000..0544fa281 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-A1:C16.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-A1:C16.xlsx.license b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-A1:C16.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-A1:C16.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-B1:C2.xlsx b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-B1:C2.xlsx new file mode 100644 index 000000000..fdc4858c8 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-B1:C2.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-B1:C2.xlsx.license b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-B1:C2.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-B1:C2.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-empty.xlsx b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-empty.xlsx new file mode 100644 index 000000000..af7b184be Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-empty.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-empty.xlsx.license b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-empty.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-empty.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1:A4.jv b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1:A4.jv new file mode 100644 index 000000000..4fa94a1c9 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1:A4.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype CellRangeSelector { + select: range A1:A4; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1:C*.jv b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1:C*.jv new file mode 100644 index 000000000..0ed56a14a --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1:C*.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype CellRangeSelector { + select: range A1:C*; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1:E4.jv b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1:E4.jv new file mode 100644 index 000000000..e004eb928 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1:E4.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype CellRangeSelector { + select: range A1:E4; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-A1:C16.xlsx b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-A1:C16.xlsx new file mode 100644 index 000000000..0544fa281 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-A1:C16.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-A1:C16.xlsx.license b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-A1:C16.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-A1:C16.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-empty.xlsx b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-empty.xlsx new file mode 100644 index 000000000..af7b184be Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-empty.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-empty.xlsx.license b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-empty.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-empty.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/cell-writer-executor/valid-cell-range-writer.jv b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/valid-cell-range-writer.jv new file mode 100644 index 000000000..327d9dbaa --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/valid-cell-range-writer.jv @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype CellWriter { + write: ["16", "Test2", ""]; + at: range A1:C1; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/cell-writer-executor/valid-single-cell-writer.jv b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/valid-single-cell-writer.jv new file mode 100644 index 000000000..844a8fd88 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/valid-single-cell-writer.jv @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype CellWriter { + write: ["16"]; + at: cell A1; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1:B2.xlsx b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1:B2.xlsx new file mode 100644 index 000000000..3465ea34d Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1:B2.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1:B2.xlsx.license b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1:B2.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1:B2.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1:C16.xlsx b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1:C16.xlsx new file mode 100644 index 000000000..0544fa281 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1:C16.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1:C16.xlsx.license b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1:C16.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1:C16.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-duplicate-column.jv b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-duplicate-column.jv new file mode 100644 index 000000000..b3b6a6ae4 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-duplicate-column.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype ColumnDeleter { + delete: [column A, column A]; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-multiple-column-deleter.jv b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-multiple-column-deleter.jv new file mode 100644 index 000000000..6a2aa56d2 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-multiple-column-deleter.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype ColumnDeleter { + delete: [column A, column C]; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-single-column-deleter.jv b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-single-column-deleter.jv new file mode 100644 index 000000000..ad5f6733b --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-single-column-deleter.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype ColumnDeleter { + delete: [column A]; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv-interpreter.jv b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv-interpreter.jv new file mode 100644 index 000000000..856c67eb0 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv-interpreter.jv @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestTextFileExtractor { + } + + block TestBlock oftype CSVInterpreter { + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv.csv b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv.csv new file mode 100644 index 000000000..086eb44a1 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv.csv @@ -0,0 +1,2 @@ +Test,true +Test,false diff --git a/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv.csv.license b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv.csv.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv.csv.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1:B2.xlsx b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1:B2.xlsx new file mode 100644 index 000000000..3465ea34d Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1:B2.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1:B2.xlsx.license b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1:B2.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1:B2.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1:C16.xlsx b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1:C16.xlsx new file mode 100644 index 000000000..0544fa281 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1:C16.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1:C16.xlsx.license b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1:C16.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1:C16.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-duplicate-row.jv b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-duplicate-row.jv new file mode 100644 index 000000000..cbb08f0f9 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-duplicate-row.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype RowDeleter { + delete: [row 1, row 1]; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-multiple-row-deleter.jv b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-multiple-row-deleter.jv new file mode 100644 index 000000000..d1384e505 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-multiple-row-deleter.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype RowDeleter { + delete: [row 1, row 5]; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-single-row-deleter.jv b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-single-row-deleter.jv new file mode 100644 index 000000000..cbb6610ff --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-single-row-deleter.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype RowDeleter { + delete: [row 1]; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/test-A1:C16.xlsx b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/test-A1:C16.xlsx new file mode 100644 index 000000000..0544fa281 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/test-A1:C16.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/test-A1:C16.xlsx.license b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/test-A1:C16.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/test-A1:C16.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/valid-custom-sheet-name.jv b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/valid-custom-sheet-name.jv new file mode 100644 index 000000000..11b753171 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/valid-custom-sheet-name.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestWorkbookExtractor { + } + + block TestBlock oftype SheetPicker { + sheetName: "MyCustomSheet"; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/valid-sheet-picker.jv b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/valid-sheet-picker.jv new file mode 100644 index 000000000..40654c112 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/valid-sheet-picker.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestWorkbookExtractor { + } + + block TestBlock oftype SheetPicker { + sheetName: "Sheet1"; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-empty.xlsx b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-empty.xlsx new file mode 100644 index 000000000..af7b184be Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-empty.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-empty.xlsx.license b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-empty.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-empty.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-with-header.xlsx b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-with-header.xlsx new file mode 100644 index 000000000..9417e87a4 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-with-header.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-with-header.xlsx.license b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-with-header.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-with-header.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-without-header.xlsx b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-without-header.xlsx new file mode 100644 index 000000000..880859630 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-without-header.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-without-header.xlsx.license b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-without-header.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-without-header.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv new file mode 100644 index 000000000..7e46a44c6 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype TableInterpreter { + header: true; + columns: []; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-without-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-without-header.jv new file mode 100644 index 000000000..b6e423edc --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-without-header.jv @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype TableInterpreter { + header: false; + columns: []; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv new file mode 100644 index 000000000..3d9929495 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype TableInterpreter { + header: true; + columns: [ + "Index" oftype integer, + "Name" oftype text, + "Flag" oftype boolean + ]; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv new file mode 100644 index 000000000..cd368186b --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype TableInterpreter { + header: true; + columns: [ + "index" oftype integer, + "name" oftype text, + "flag" oftype boolean + ]; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv new file mode 100644 index 000000000..6fa8f5c1d --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype TableInterpreter { + header: false; + columns: [ + "index" oftype integer, + "name" oftype text, + "flag" oftype boolean + ]; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-valuetype-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-valuetype-with-header.jv new file mode 100644 index 000000000..f8ec03446 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-valuetype-with-header.jv @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype TableInterpreter { + header: true; + columns: [ + "index" oftype integer, + "name" oftype text, + "flag" oftype integer + ]; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-valuetype-without-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-valuetype-without-header.jv new file mode 100644 index 000000000..ebc269171 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-valuetype-without-header.jv @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype TableInterpreter { + header: false; + columns: [ + "index" oftype integer, + "name" oftype text, + "flag" oftype integer + ]; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-empty.xlsx b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-empty.xlsx new file mode 100644 index 000000000..af7b184be Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-empty.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-empty.xlsx.license b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-empty.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-empty.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-excel.xlsx b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-excel.xlsx new file mode 100644 index 000000000..8869be3b3 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-excel.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-excel.xlsx.license b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-excel.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-excel.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-column-overwrite.jv b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-column-overwrite.jv new file mode 100644 index 000000000..91c301ed4 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-column-overwrite.jv @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + transform Double { + from original oftype integer; + to double oftype integer; + + double: original * 2; + } + + block TestExtractor oftype TestTableExtractor { + } + + block TestBlock oftype TableTransformer { + inputColumns: ['index']; + outputColumn: 'index'; + use: Double; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-column-type-change.jv b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-column-type-change.jv new file mode 100644 index 000000000..a85a6b2ad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-column-type-change.jv @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + transform ToBool { + from asInteger oftype integer; + to asBool oftype boolean; + + asBool: asInteger != 0; + } + + block TestExtractor oftype TestTableExtractor { + } + + block TestBlock oftype TableTransformer { + inputColumns: ['index']; + outputColumn: 'index'; + use: ToBool; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-missing-input-column.jv b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-missing-input-column.jv new file mode 100644 index 000000000..44e535ffc --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-missing-input-column.jv @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + transform Add { + from left oftype integer; + from right oftype integer; + to added oftype integer; + + added: left + right; + } + + block TestExtractor oftype TestTableExtractor { + } + + block TestBlock oftype TableTransformer { + inputColumns: ['index', 'id']; + outputColumn: 'add'; + use: Add; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-transfomer.jv b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-transfomer.jv new file mode 100644 index 000000000..856d8024b --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-transfomer.jv @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + transform Double { + from original oftype integer; + to double oftype integer; + + double: original * 2; + } + + block TestExtractor oftype TestTableExtractor { + } + + block TestBlock oftype TableTransformer { + inputColumns: ['index']; + outputColumn: 'index2'; + use: Double; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-transform-type-missmatch.jv b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-transform-type-missmatch.jv new file mode 100644 index 000000000..52675e73d --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-transform-type-missmatch.jv @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + transform Double { + from original oftype integer; + to double oftype integer; + + double: original * 2; + } + + block TestExtractor oftype TestTableExtractor { + } + + block TestBlock oftype TableTransformer { + inputColumns: ['name']; + outputColumn: 'index2'; + use: Double; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-empty.xlsx b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-empty.xlsx new file mode 100644 index 000000000..af7b184be Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-empty.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-empty.xlsx.license b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-empty.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-empty.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-excel.xlsx b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-excel.xlsx new file mode 100644 index 000000000..8869be3b3 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-excel.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-excel.xlsx.license b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-excel.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-excel.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-multiple-sheets.xlsx b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-multiple-sheets.xlsx new file mode 100644 index 000000000..3f3619fcc Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-multiple-sheets.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-multiple-sheets.xlsx.license b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-multiple-sheets.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-multiple-sheets.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/valid-excel-interpreter.jv b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/valid-excel-interpreter.jv new file mode 100644 index 000000000..a267981e2 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/valid-excel-interpreter.jv @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + block TestExtractor oftype TestFileExtractor { + } + + block TestBlock oftype XLSXInterpreter { + } + + block TestLoader oftype TestWorkbookLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/jsdom-environment-fix.ts b/libs/extensions/tabular/exec/test/jsdom-environment-fix.ts new file mode 100644 index 000000000..d5af83010 --- /dev/null +++ b/libs/extensions/tabular/exec/test/jsdom-environment-fix.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import JSDOMEnvironment from 'jest-environment-jsdom'; + +// @nrwl/jest/preset loads in jsdom environment which does not support a few node.js functions. +// This class adds these node.js functions. +export default class JSDOMEnvironmentFix extends JSDOMEnvironment { + constructor(...args: ConstructorParameters) { + super(...args); + + // https://github.com/jsdom/jsdom/issues/3363 + this.global.structuredClone = structuredClone; + // https://github.com/jestjs/jest/issues/11204 + // https://github.com/jestjs/jest/issues/12622 + this.global.setImmediate = setImmediate; + } +} diff --git a/libs/extensions/tabular/exec/test/test-extension/TestBlockTypes.jv b/libs/extensions/tabular/exec/test/test-extension/TestBlockTypes.jv new file mode 100644 index 000000000..ba08556c4 --- /dev/null +++ b/libs/extensions/tabular/exec/test/test-extension/TestBlockTypes.jv @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +builtin blocktype TestSheetExtractor { + input inPort oftype None; + output outPort oftype Sheet; +} + +builtin blocktype TestSheetLoader { + input inPort oftype Sheet; + output outPort oftype None; +} + +builtin blocktype TestTextFileExtractor { + input inPort oftype None; + output outPort oftype TextFile; +} + +builtin blocktype TestFileExtractor { + input inPort oftype None; + output outPort oftype File; +} + +builtin blocktype TestWorkbookExtractor { + input inPort oftype None; + output outPort oftype Workbook; +} + +builtin blocktype TestWorkbookLoader { + input inPort oftype Workbook; + output outPort oftype None; +} + +builtin blocktype TestTableExtractor { + input inPort oftype None; + output outPort oftype Table; +} + +builtin blocktype TestTableLoader { + input inPort oftype Table; + output outPort oftype None; +} diff --git a/libs/extensions/tabular/exec/test/util.ts b/libs/extensions/tabular/exec/test/util.ts new file mode 100644 index 000000000..68180db6f --- /dev/null +++ b/libs/extensions/tabular/exec/test/util.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import { + Table, + TableRow, + Workbook, + parseValueToInternalRepresentation, +} from '@jvalue/jayvee-execution'; +import { + InternalValueRepresentation, + Valuetype, +} from '@jvalue/jayvee-language-server'; +import * as exceljs from 'exceljs'; + +import { ColumnDefinitionEntry } from '../src/lib/table-interpreter-executor'; + +export async function createWorkbookFromLocalExcelFile( + fileName: string, +): Promise { + const workBookFromFile = new exceljs.Workbook(); + await workBookFromFile.xlsx.readFile(path.resolve(__dirname, fileName)); + + const workbook = new Workbook(); + + workBookFromFile.eachSheet((workSheet) => { + const workSheetDataArray: string[][] = []; + workSheet.eachRow((row, rowNumber) => { + const cellValues: string[] = []; + + // ExcelJS Rows and Columns are indexed from 1 + // We reduce their index to match Sheets being zero indexed + row.eachCell( + { includeEmpty: true }, + (cell: exceljs.Cell, colNumber: number) => { + cellValues[colNumber - 1] = cell.text; + }, + ); + + workSheetDataArray[rowNumber - 1] = cellValues; + }); + + workbook.addSheet(workSheetDataArray, workSheet.name); + }); + + return workbook; +} + +export type ReducedColumnDefinitionEntry = Pick< + ColumnDefinitionEntry, + 'sheetColumnIndex' | 'columnName' | 'valuetype' +>; + +/** + * Creates a Table from the first sheet of the excel file pointed to by {@link fileName} + * and parsed using the given column definitions. + * Note: The parsing is static and thus cannot detect runtime types! + * @param fileName + * @param columnDefinitions columns to be read from table (no header matching) + * @returns Table containing data of excel + */ +export async function createTableFromLocalExcelFile( + fileName: string, + columnDefinitions: ReducedColumnDefinitionEntry[], +): Promise { + const workBookFromFile = new exceljs.Workbook(); + await workBookFromFile.xlsx.readFile(path.resolve(__dirname, fileName)); + + const workSheet = workBookFromFile.worksheets[0] as exceljs.Worksheet; + const table = new Table(); + + columnDefinitions.forEach((columnDefinition) => { + table.addColumn(columnDefinition.columnName, { + values: [], + valuetype: columnDefinition.valuetype, + }); + }); + + workSheet.eachRow((row) => { + const tableRow = constructTableRow(row, columnDefinitions); + table.addRow(tableRow); + }); + + return table; +} +function constructTableRow( + row: exceljs.Row, + columnDefinitions: ReducedColumnDefinitionEntry[], +): TableRow { + const tableRow: TableRow = {}; + + row.eachCell( + { includeEmpty: true }, + (cell: exceljs.Cell, colNumber: number) => { + if (colNumber > columnDefinitions.length) return; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const columnDefinition = columnDefinitions[colNumber - 1]!; + const value = cell.text; + const valuetype = columnDefinition.valuetype; + + const parsedValue = parseAndValidatePrimitiveValue(value, valuetype); + if (parsedValue === undefined) { + return; + } + + tableRow[columnDefinition.columnName] = parsedValue; + }, + ); + return tableRow; +} +function parseAndValidatePrimitiveValue( + value: string, + valuetype: Valuetype, +): InternalValueRepresentation | undefined { + const parsedValue = parseValueToInternalRepresentation(value, valuetype); + if (parsedValue === undefined) { + return undefined; + } + + return parsedValue; +} diff --git a/libs/extensions/tabular/exec/tsconfig.spec.json b/libs/extensions/tabular/exec/tsconfig.spec.json index 6668655fc..ab8d1573c 100644 --- a/libs/extensions/tabular/exec/tsconfig.spec.json +++ b/libs/extensions/tabular/exec/tsconfig.spec.json @@ -9,6 +9,7 @@ "jest.config.ts", "src/**/*.test.ts", "src/**/*.spec.ts", - "src/**/*.d.ts" + "src/**/*.d.ts", + "test/**/*.ts" ] }