From 0945d135aa0c3f78bdf8aad2bbdea2967354b61f Mon Sep 17 00:00:00 2001 From: Stephane Geneix Date: Wed, 18 Sep 2024 07:44:44 -0700 Subject: [PATCH] destination-e2e: merge cloud and OSS --- .../airbyte/cdk/integrations/BaseConnector.kt | 8 ++ .../destination/DestinationAcceptanceTest.kt | 19 ++-- .../source/relationaldb/AbstractDbSource.kt | 4 - .../connectors/destination-dev-null/README.md | 45 +++++++--- .../destination-dev-null/build.gradle | 3 +- .../destination-dev-null/metadata.yaml | 9 +- .../dev_null/DevNullDestination.kt | 53 ----------- .../dev_null}/FailAfterNDestination.kt | 2 +- .../dev_null}/LoggingDestination.kt | 6 +- .../dev_null}/SilentDestination.kt | 2 +- .../dev_null}/TestingDestinations.kt | 38 +++++++- .../dev_null}/ThrottledDestination.kt | 4 +- .../dev_null}/logging/BaseLogger.kt | 2 +- .../dev_null}/logging/EveryNthLogger.kt | 2 +- .../dev_null}/logging/FirstNLogger.kt | 2 +- .../dev_null}/logging/LoggingConsumer.kt | 2 +- .../dev_null}/logging/RandomSamplingLogger.kt | 2 +- .../dev_null}/logging/TestingLogger.kt | 2 +- .../dev_null}/logging/TestingLoggerFactory.kt | 2 +- .../src/main/resources/spec.json | 0 ... CloudDevNullDestinationAcceptanceTest.kt} | 5 +- .../TestingSilentDestinationAcceptanceTest.kt | 9 +- .../dev_null/CloudDevNullDestinationTest.kt | 34 ++++++++ .../dev_null/DevNullDestinationTest.kt | 25 ------ .../dev_null}/ThrottledDestinationTest.kt | 2 +- ...ted_spec.json => expected_spec_cloud.json} | 0 .../connectors/destination-e2e-test/README.md | 87 ------------------- .../destination-e2e-test/build.gradle | 17 ---- .../connectors/destination-e2e-test/icon.svg | 1 - .../destination-e2e-test/metadata.yaml | 28 ------ docs/integrations/destinations/dev-null.md | 69 +++++++++++++-- docs/integrations/destinations/e2e-test.md | 69 --------------- 32 files changed, 211 insertions(+), 342 deletions(-) delete mode 100644 airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestination.kt rename airbyte-integrations/connectors/{destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test => destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null}/FailAfterNDestination.kt (97%) rename airbyte-integrations/connectors/{destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test => destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null}/LoggingDestination.kt (87%) rename airbyte-integrations/connectors/{destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test => destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null}/SilentDestination.kt (96%) rename airbyte-integrations/connectors/{destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test => destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null}/TestingDestinations.kt (64%) rename airbyte-integrations/connectors/{destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test => destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null}/ThrottledDestination.kt (94%) rename airbyte-integrations/connectors/{destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test => destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null}/logging/BaseLogger.kt (95%) rename airbyte-integrations/connectors/{destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test => destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null}/logging/EveryNthLogger.kt (93%) rename airbyte-integrations/connectors/{destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test => destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null}/logging/FirstNLogger.kt (92%) rename airbyte-integrations/connectors/{destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test => destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null}/logging/LoggingConsumer.kt (97%) rename airbyte-integrations/connectors/{destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test => destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null}/logging/RandomSamplingLogger.kt (93%) rename airbyte-integrations/connectors/{destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test => destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null}/logging/TestingLogger.kt (82%) rename airbyte-integrations/connectors/{destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test => destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null}/logging/TestingLoggerFactory.kt (96%) rename airbyte-integrations/connectors/{destination-e2e-test => destination-dev-null}/src/main/resources/spec.json (100%) rename airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/{DevNullDestinationAcceptanceTest.kt => CloudDevNullDestinationAcceptanceTest.kt} (89%) rename airbyte-integrations/connectors/{destination-e2e-test/src/test-integration/kotlin/io/airbyte/integrations/destination/e2e_test => destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null}/TestingSilentDestinationAcceptanceTest.kt (88%) create mode 100644 airbyte-integrations/connectors/destination-dev-null/src/test/kotlin/io/airbyte/integrations/destination/dev_null/CloudDevNullDestinationTest.kt delete mode 100644 airbyte-integrations/connectors/destination-dev-null/src/test/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationTest.kt rename airbyte-integrations/connectors/{destination-e2e-test/src/test/kotlin/io/airbyte/integrations/destination/e2e_test => destination-dev-null/src/test/kotlin/io/airbyte/integrations/destination/dev_null}/ThrottledDestinationTest.kt (97%) rename airbyte-integrations/connectors/destination-dev-null/src/test/resources/{expected_spec.json => expected_spec_cloud.json} (100%) delete mode 100644 airbyte-integrations/connectors/destination-e2e-test/README.md delete mode 100644 airbyte-integrations/connectors/destination-e2e-test/build.gradle delete mode 100644 airbyte-integrations/connectors/destination-e2e-test/icon.svg delete mode 100644 airbyte-integrations/connectors/destination-e2e-test/metadata.yaml delete mode 100644 docs/integrations/destinations/e2e-test.md diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/BaseConnector.kt b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/BaseConnector.kt index b3b6587d0d64..ef1d80ea635f 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/BaseConnector.kt +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/BaseConnector.kt @@ -4,11 +4,19 @@ package io.airbyte.cdk.integrations import io.airbyte.cdk.integrations.base.Integration +import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner +import io.airbyte.commons.features.EnvVariableFeatureFlags +import io.airbyte.commons.features.FeatureFlags import io.airbyte.commons.json.Jsons import io.airbyte.commons.resources.MoreResources import io.airbyte.protocol.models.v0.ConnectorSpecification abstract class BaseConnector : Integration { + open val featureFlags: FeatureFlags = EnvVariableFeatureFlags() + + val isCloudDeployment + get() = + AdaptiveSourceRunner.CLOUD_MODE.equals(featureFlags.deploymentMode(), ignoreCase = true) /** * By convention the spec is stored as a resource for java connectors. That resource is called * spec.json. diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt index 73247ce54ac6..77319b765fce 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt @@ -17,6 +17,8 @@ import io.airbyte.cdk.integrations.standardtest.destination.argproviders.util.Ar import io.airbyte.cdk.integrations.standardtest.destination.comparator.BasicTestDataComparator import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator import io.airbyte.commons.features.EnvVariableFeatureFlags +import io.airbyte.commons.features.FeatureFlags +import io.airbyte.commons.features.FeatureFlagsWrapper import io.airbyte.commons.jackson.MoreMappers import io.airbyte.commons.json.Jsons import io.airbyte.commons.lang.Exceptions @@ -98,6 +100,13 @@ abstract class DestinationAcceptanceTest( protected var testSchemas: HashSet = HashSet() private lateinit var testEnv: TestDestinationEnv + protected open val isCloudTest: Boolean = true + protected val featureFlags: FeatureFlags = + if (isCloudTest) { + FeatureFlagsWrapper.overridingDeploymentMode(EnvVariableFeatureFlags(), "CLOUD") + } else { + FeatureFlagsWrapper.overridingDeploymentMode(EnvVariableFeatureFlags(), "OSS") + } private lateinit var jobRoot: Path private lateinit var processFactory: ProcessFactory @@ -1912,7 +1921,7 @@ abstract class DestinationAcceptanceTest( null, null, false, - EnvVariableFeatureFlags() + featureFlags ) ) .run(JobGetSpecConfig().withDockerImage(imageName), jobRoot) @@ -1932,7 +1941,7 @@ abstract class DestinationAcceptanceTest( null, null, false, - EnvVariableFeatureFlags() + featureFlags ), mConnectorConfigUpdater ) @@ -1954,7 +1963,7 @@ abstract class DestinationAcceptanceTest( null, null, false, - EnvVariableFeatureFlags() + featureFlags ), mConnectorConfigUpdater ) @@ -1982,7 +1991,7 @@ abstract class DestinationAcceptanceTest( null, null, false, - EnvVariableFeatureFlags() + featureFlags ) ) } @@ -1998,7 +2007,7 @@ abstract class DestinationAcceptanceTest( null, null, false, - EnvVariableFeatureFlags() + featureFlags ) ) } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.kt b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.kt index 1e67982a5e94..6372c1be6ded 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/kotlin/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.kt @@ -22,8 +22,6 @@ import io.airbyte.cdk.integrations.util.ApmTraceUtils.addExceptionToTrace import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil import io.airbyte.commons.exceptions.ConfigErrorException import io.airbyte.commons.exceptions.ConnectionErrorException -import io.airbyte.commons.features.EnvVariableFeatureFlags -import io.airbyte.commons.features.FeatureFlags import io.airbyte.commons.functional.CheckedConsumer import io.airbyte.commons.lang.Exceptions import io.airbyte.commons.stream.AirbyteStreamUtils @@ -49,8 +47,6 @@ private val LOGGER = KotlinLogging.logger {} abstract class AbstractDbSource protected constructor(driverClassName: String) : JdbcConnector(driverClassName), Source, AutoCloseable { - // TODO: Remove when the flag is not use anymore - var featureFlags: FeatureFlags = EnvVariableFeatureFlags() @Trace(operationName = CHECK_TRACE_OPERATION_NAME) @Throws(Exception::class) diff --git a/airbyte-integrations/connectors/destination-dev-null/README.md b/airbyte-integrations/connectors/destination-dev-null/README.md index 95f0acc09cc2..c4837fe1af1e 100644 --- a/airbyte-integrations/connectors/destination-dev-null/README.md +++ b/airbyte-integrations/connectors/destination-dev-null/README.md @@ -1,6 +1,6 @@ -# Destination Dev Null +# End-to-End Testing Destination -This destination is a "safe" version of the [E2E Test destination](https://docs.airbyte.io/integrations/destinations/e2e-test). It only allows the "silent" mode. +This is the repository for the Null destination connector in Java. For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/destinations/e2e-test). ## Local development @@ -9,9 +9,13 @@ This destination is a "safe" version of the [E2E Test destination](https://docs. From the Airbyte repository root, run: ``` -./gradlew :airbyte-integrations:connectors:destination-dev-null:build +./gradlew :airbyte-integrations:connectors:destination-e2e-test:build ``` +#### Create credentials + +No credential is needed for this connector. + ### Locally running the connector docker image #### Build @@ -19,10 +23,10 @@ From the Airbyte repository root, run: Build the connector image via Gradle: ``` -./gradlew :airbyte-integrations:connectors:destination-dev-null:buildConnectorImage +./gradlew :airbyte-integrations:connectors:destination-e2e-test:buildConnectorImage ``` -Once built, the docker image name and tag on your host will be `airbyte/destination-dev-null:dev`. +Once built, the docker image name and tag on your host will be `airbyte/destination-e2e-test:dev`. the Dockerfile. #### Run @@ -30,25 +34,42 @@ the Dockerfile. Then run any of the connector commands as follows: ``` -docker run --rm airbyte/destination-dev-null:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-dev-null:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-dev-null:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-dev-null:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm airbyte/destination-e2e-test:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-e2e-test:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-e2e-test:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-e2e-test:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` +#### Cloud variant + +The cloud variant of this connector is Dev Null Destination. It only allows the "silent" mode. When this mode is changed, please make sure that the Dev Null Destination is updated and published accordingly as well. + +## Testing + +We use `JUnit` for Java tests. + +### Unit and Integration Tests + +Place unit tests under `src/test/io/airbyte/integrations/destinations/e2e-test`. + +#### Acceptance Tests + +Airbyte has a standard test suite that all destination connectors must pass. See example(s) in +`src/test-integration/java/io/airbyte/integrations/destinations/e2e-test/`. + ### Using gradle to run tests All commands should be run from airbyte project root. To run unit tests: ``` -./gradlew :airbyte-integrations:connectors:destination-dev-null:unitTest +./gradlew :airbyte-integrations:connectors:destination-e2e-test:unitTest ``` To run acceptance and custom integration tests: ``` -./gradlew :airbyte-integrations:connectors:destination-dev-null:integrationTest +./gradlew :airbyte-integrations:connectors:destination-e2e-test:integrationTest ``` ## Dependency Management @@ -57,7 +78,7 @@ To run acceptance and custom integration tests: You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-dev-null test` +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-e2e-test test` 2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). 3. Make sure the `metadata.yaml` content is up to date. 4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/e2e-test.md`). diff --git a/airbyte-integrations/connectors/destination-dev-null/build.gradle b/airbyte-integrations/connectors/destination-dev-null/build.gradle index dff97c133d29..fcd638e558c8 100644 --- a/airbyte-integrations/connectors/destination-dev-null/build.gradle +++ b/airbyte-integrations/connectors/destination-dev-null/build.gradle @@ -9,10 +9,9 @@ airbyteJavaConnector { } application { - mainClass = 'io.airbyte.integrations.destination.dev_null.DevNullDestination' + mainClass = 'io.airbyte.integrations.destination.dev_null.TestingDestinations' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-integrations:connectors:destination-e2e-test') } diff --git a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml index d01d46fd0def..4cad1d0c3256 100644 --- a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml +++ b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml @@ -2,19 +2,14 @@ data: connectorSubtype: file connectorType: destination definitionId: a7bcc9d8-13b3-4e49-b80d-d020b90045e3 - dockerImageTag: 0.5.0 + dockerImageTag: 0.6.0 dockerRepository: airbyte/destination-dev-null githubIssueLabel: destination-dev-null icon: airbyte.svg license: MIT name: End-to-End Testing (/dev/null) - registryOverrides: - cloud: - enabled: true - oss: - enabled: false releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/destinations/e2e-test + documentationUrl: https://docs.airbyte.com/integrations/destinations/dev-null tags: - language:java ab_internal: diff --git a/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestination.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestination.kt deleted file mode 100644 index 9f0ccbaca6b4..000000000000 --- a/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestination.kt +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ -package io.airbyte.integrations.destination.dev_null - -import com.fasterxml.jackson.databind.node.ArrayNode -import com.fasterxml.jackson.databind.node.ObjectNode -import io.airbyte.cdk.integrations.base.Destination -import io.airbyte.cdk.integrations.base.IntegrationRunner -import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingDestination -import io.airbyte.commons.json.Jsons -import io.airbyte.integrations.destination.e2e_test.TestingDestinations -import io.airbyte.protocol.models.v0.ConnectorSpecification -import org.slf4j.Logger -import org.slf4j.LoggerFactory - -class DevNullDestination : SpecModifyingDestination(TestingDestinations()), Destination { - /** 1. Update the title. 2. Only keep the "silent" mode. */ - override fun modifySpec(originalSpec: ConnectorSpecification): ConnectorSpecification { - val spec = Jsons.clone(originalSpec) - - (spec.connectionSpecification as ObjectNode).put("title", DEV_NULL_DESTINATION_TITLE) - - val properties = - spec.connectionSpecification["properties"]["test_destination"] as ObjectNode - val types = properties["oneOf"] as ArrayNode - val typesIterator = types.elements() - while (typesIterator.hasNext()) { - val typeNode = typesIterator.next() - if ( - !typeNode["properties"]["test_destination_type"]["const"] - .asText() - .equals("silent", ignoreCase = true) - ) { - typesIterator.remove() - } - } - return spec - } - - companion object { - private val LOGGER: Logger = LoggerFactory.getLogger(DevNullDestination::class.java) - private const val DEV_NULL_DESTINATION_TITLE = "E2E Test (/dev/null) Destination Spec" - - @Throws(Exception::class) - @JvmStatic - fun main(args: Array) { - LOGGER.info("Starting destination: {}", DevNullDestination::class.java) - IntegrationRunner(DevNullDestination()).run(args) - LOGGER.info("Completed destination: {}", DevNullDestination::class.java) - } - } -} diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/FailAfterNDestination.kt similarity index 97% rename from airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.kt rename to airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/FailAfterNDestination.kt index d9903442d189..8b750844b7b3 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/FailAfterNDestination.kt @@ -1,7 +1,7 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.e2e_test +package io.airbyte.integrations.destination.dev_null import com.fasterxml.jackson.databind.JsonNode import io.airbyte.cdk.integrations.BaseConnector diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/LoggingDestination.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/LoggingDestination.kt similarity index 87% rename from airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/LoggingDestination.kt rename to airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/LoggingDestination.kt index eee5e5077947..84c00879de74 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/LoggingDestination.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/LoggingDestination.kt @@ -1,14 +1,14 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.e2e_test +package io.airbyte.integrations.destination.dev_null import com.fasterxml.jackson.databind.JsonNode import io.airbyte.cdk.integrations.BaseConnector import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer import io.airbyte.cdk.integrations.base.Destination -import io.airbyte.integrations.destination.e2e_test.logging.LoggingConsumer -import io.airbyte.integrations.destination.e2e_test.logging.TestingLoggerFactory +import io.airbyte.integrations.destination.dev_null.logging.LoggingConsumer +import io.airbyte.integrations.destination.dev_null.logging.TestingLoggerFactory import io.airbyte.protocol.models.v0.AirbyteConnectionStatus import io.airbyte.protocol.models.v0.AirbyteMessage import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/SilentDestination.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/SilentDestination.kt similarity index 96% rename from airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/SilentDestination.kt rename to airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/SilentDestination.kt index 74181da0b1c9..d50b42989598 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/SilentDestination.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/SilentDestination.kt @@ -1,7 +1,7 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.e2e_test +package io.airbyte.integrations.destination.dev_null import com.fasterxml.jackson.databind.JsonNode import io.airbyte.cdk.integrations.BaseConnector diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/TestingDestinations.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/TestingDestinations.kt similarity index 64% rename from airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/TestingDestinations.kt rename to airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/TestingDestinations.kt index 4fe45119e713..8c14a3a6c625 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/TestingDestinations.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/TestingDestinations.kt @@ -1,17 +1,22 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.e2e_test +package io.airbyte.integrations.destination.dev_null import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.ObjectNode import com.google.common.collect.ImmutableMap import io.airbyte.cdk.integrations.BaseConnector import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer import io.airbyte.cdk.integrations.base.Destination import io.airbyte.cdk.integrations.base.IntegrationRunner +import io.airbyte.commons.features.EnvVariableFeatureFlags +import io.airbyte.commons.features.FeatureFlags import io.airbyte.protocol.models.v0.AirbyteConnectionStatus import io.airbyte.protocol.models.v0.AirbyteMessage import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog +import io.airbyte.protocol.models.v0.ConnectorSpecification import java.util.function.Consumer import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -19,13 +24,14 @@ import org.slf4j.LoggerFactory class TestingDestinations @JvmOverloads constructor( + override val featureFlags: FeatureFlags = EnvVariableFeatureFlags(), private val destinationMap: Map = ImmutableMap.builder() .put(TestDestinationType.LOGGING, LoggingDestination()) .put(TestDestinationType.THROTTLED, ThrottledDestination()) .put(TestDestinationType.SILENT, SilentDestination()) .put(TestDestinationType.FAILING, FailAfterNDestination()) - .build() + .build(), ) : BaseConnector(), Destination { enum class TestDestinationType { LOGGING, @@ -42,6 +48,33 @@ constructor( ] } + override fun spec(): ConnectorSpecification { + if (!isCloudDeployment) { + return super.spec() + } else { + /** 1. Update the title. 2. Only keep the "silent" mode. */ + val spec = super.spec() + + (spec.connectionSpecification as ObjectNode).put("title", DEV_NULL_DESTINATION_TITLE) + + val properties = + spec.connectionSpecification["properties"]["test_destination"] as ObjectNode + val types = properties["oneOf"] as ArrayNode + val typesIterator = types.elements() + while (typesIterator.hasNext()) { + val typeNode = typesIterator.next() + if ( + !typeNode["properties"]["test_destination_type"]["const"] + .asText() + .equals("silent", ignoreCase = true) + ) { + typesIterator.remove() + } + } + return spec + } + } + @Throws(Exception::class) override fun getConsumer( config: JsonNode, @@ -64,6 +97,7 @@ constructor( companion object { private val LOGGER: Logger = LoggerFactory.getLogger(TestingDestinations::class.java) + private const val DEV_NULL_DESTINATION_TITLE = "E2E Test (/dev/null) Destination Spec" @Throws(Exception::class) @JvmStatic diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/ThrottledDestination.kt similarity index 94% rename from airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.kt rename to airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/ThrottledDestination.kt index 7d95fc956ffc..aa0ab5860d14 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/ThrottledDestination.kt @@ -1,7 +1,7 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.e2e_test +package io.airbyte.integrations.destination.dev_null import com.fasterxml.jackson.databind.JsonNode import io.airbyte.cdk.integrations.BaseConnector @@ -18,7 +18,7 @@ import org.slf4j.LoggerFactory * This destination logs each record it receives. It sleeps for millis_per_record between accepting * each record. Useful for simulating backpressure / slow destination writes. */ -class ThrottledDestination : BaseConnector(), Destination { +class ThrottledDestination() : BaseConnector(), Destination { override fun check(config: JsonNode): AirbyteConnectionStatus { return AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.SUCCEEDED) } diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/BaseLogger.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/BaseLogger.kt similarity index 95% rename from airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/BaseLogger.kt rename to airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/BaseLogger.kt index 2d45dc801d8b..239fe3198dae 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/BaseLogger.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/BaseLogger.kt @@ -1,7 +1,7 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.e2e_test.logging +package io.airbyte.integrations.destination.dev_null.logging import io.airbyte.protocol.models.v0.AirbyteRecordMessage import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/EveryNthLogger.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/EveryNthLogger.kt similarity index 93% rename from airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/EveryNthLogger.kt rename to airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/EveryNthLogger.kt index b63a3682385e..f4c14c9e5b97 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/EveryNthLogger.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/EveryNthLogger.kt @@ -1,7 +1,7 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.e2e_test.logging +package io.airbyte.integrations.destination.dev_null.logging import io.airbyte.protocol.models.v0.AirbyteRecordMessage import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/FirstNLogger.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/FirstNLogger.kt similarity index 92% rename from airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/FirstNLogger.kt rename to airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/FirstNLogger.kt index 15e1ecbe002d..67a219ba3118 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/FirstNLogger.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/FirstNLogger.kt @@ -1,7 +1,7 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.e2e_test.logging +package io.airbyte.integrations.destination.dev_null.logging import io.airbyte.protocol.models.v0.AirbyteRecordMessage import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/LoggingConsumer.kt similarity index 97% rename from airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.kt rename to airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/LoggingConsumer.kt index e821a417dc0c..9d6c80209ab7 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/LoggingConsumer.kt @@ -1,7 +1,7 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.e2e_test.logging +package io.airbyte.integrations.destination.dev_null.logging import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer import io.airbyte.commons.json.Jsons diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/RandomSamplingLogger.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/RandomSamplingLogger.kt similarity index 93% rename from airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/RandomSamplingLogger.kt rename to airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/RandomSamplingLogger.kt index fa38b1e708b7..429ca5ee0ee8 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/RandomSamplingLogger.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/RandomSamplingLogger.kt @@ -1,7 +1,7 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.e2e_test.logging +package io.airbyte.integrations.destination.dev_null.logging import io.airbyte.protocol.models.v0.AirbyteRecordMessage import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/TestingLogger.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/TestingLogger.kt similarity index 82% rename from airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/TestingLogger.kt rename to airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/TestingLogger.kt index 9be54f648411..439307c42e6b 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/TestingLogger.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/TestingLogger.kt @@ -1,7 +1,7 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.e2e_test.logging +package io.airbyte.integrations.destination.dev_null.logging import io.airbyte.protocol.models.v0.AirbyteRecordMessage diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/TestingLoggerFactory.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/TestingLoggerFactory.kt similarity index 96% rename from airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/TestingLoggerFactory.kt rename to airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/TestingLoggerFactory.kt index be91958e2d47..887e3c3f6d8d 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/kotlin/io/airbyte/integrations/destination/e2e_test/logging/TestingLoggerFactory.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/logging/TestingLoggerFactory.kt @@ -1,7 +1,7 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.e2e_test.logging +package io.airbyte.integrations.destination.dev_null.logging import com.fasterxml.jackson.databind.JsonNode import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-dev-null/src/main/resources/spec.json similarity index 100% rename from airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json rename to airbyte-integrations/connectors/destination-dev-null/src/main/resources/spec.json diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.kt b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/CloudDevNullDestinationAcceptanceTest.kt similarity index 89% rename from airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.kt rename to airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/CloudDevNullDestinationAcceptanceTest.kt index db6e9aedbaac..a1c6a35a23c3 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/CloudDevNullDestinationAcceptanceTest.kt @@ -4,7 +4,6 @@ package io.airbyte.integrations.destination.dev_null import com.fasterxml.jackson.databind.JsonNode -import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest import io.airbyte.commons.json.Jsons import io.airbyte.protocol.models.v0.AirbyteMessage import io.airbyte.protocol.models.v0.AirbyteRecordMessage @@ -13,8 +12,8 @@ import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test -class DevNullDestinationAcceptanceTest : DestinationAcceptanceTest() { - override val imageName = "airbyte/destination-dev-null:dev" +class CloudDevNullDestinationAcceptanceTest : TestingSilentDestinationAcceptanceTest() { + override val isCloudTest = true override fun getConfig(): JsonNode { return Jsons.jsonNode( diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/kotlin/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.kt b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/TestingSilentDestinationAcceptanceTest.kt similarity index 88% rename from airbyte-integrations/connectors/destination-e2e-test/src/test-integration/kotlin/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.kt rename to airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/TestingSilentDestinationAcceptanceTest.kt index 85d303ef27fd..acac62617507 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/kotlin/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/TestingSilentDestinationAcceptanceTest.kt @@ -1,20 +1,21 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.e2e_test +package io.airbyte.integrations.destination.dev_null import com.fasterxml.jackson.databind.JsonNode import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest import io.airbyte.commons.json.Jsons -import io.airbyte.integrations.destination.e2e_test.TestingDestinations.TestDestinationType +import io.airbyte.integrations.destination.dev_null.TestingDestinations.TestDestinationType import io.airbyte.protocol.models.v0.* import java.util.* import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test -class TestingSilentDestinationAcceptanceTest : DestinationAcceptanceTest() { - override val imageName = "airbyte/destination-e2e-test:dev" +open class TestingSilentDestinationAcceptanceTest : DestinationAcceptanceTest() { + override val imageName = "airbyte/destination-dev-null:dev" + override val isCloudTest = false override fun getConfig(): JsonNode { return Jsons.jsonNode( diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test/kotlin/io/airbyte/integrations/destination/dev_null/CloudDevNullDestinationTest.kt b/airbyte-integrations/connectors/destination-dev-null/src/test/kotlin/io/airbyte/integrations/destination/dev_null/CloudDevNullDestinationTest.kt new file mode 100644 index 000000000000..efc31d6f3e57 --- /dev/null +++ b/airbyte-integrations/connectors/destination-dev-null/src/test/kotlin/io/airbyte/integrations/destination/dev_null/CloudDevNullDestinationTest.kt @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ +package io.airbyte.integrations.destination.dev_null + +import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner +import io.airbyte.commons.features.EnvVariableFeatureFlags +import io.airbyte.commons.features.FeatureFlagsWrapper +import io.airbyte.commons.json.Jsons +import io.airbyte.commons.resources.MoreResources +import io.airbyte.protocol.models.v0.ConnectorSpecification +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +internal class CloudDevNullDestinationTest { + @Test + @Throws(Exception::class) + fun testSpec() { + val actual = + TestingDestinations( + FeatureFlagsWrapper.overridingDeploymentMode( + EnvVariableFeatureFlags(), + AdaptiveSourceRunner.CLOUD_MODE + ) + ) + .spec() + val expected = + Jsons.deserialize( + MoreResources.readResource("expected_spec_cloud.json"), + ConnectorSpecification::class.java + ) + Assertions.assertEquals(expected, actual) + } +} diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationTest.kt b/airbyte-integrations/connectors/destination-dev-null/src/test/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationTest.kt deleted file mode 100644 index e301f29d48a1..000000000000 --- a/airbyte-integrations/connectors/destination-dev-null/src/test/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationTest.kt +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ -package io.airbyte.integrations.destination.dev_null - -import io.airbyte.commons.json.Jsons -import io.airbyte.commons.resources.MoreResources -import io.airbyte.protocol.models.v0.ConnectorSpecification -import org.junit.jupiter.api.Assertions -import org.junit.jupiter.api.Test - -internal class DevNullDestinationTest { - @Test - @Throws(Exception::class) - fun testSpec() { - val actual = DevNullDestination().spec() - val expected = - Jsons.deserialize( - MoreResources.readResource("expected_spec.json"), - ConnectorSpecification::class.java - ) - - Assertions.assertEquals(expected, actual) - } -} diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/test/kotlin/io/airbyte/integrations/destination/e2e_test/ThrottledDestinationTest.kt b/airbyte-integrations/connectors/destination-dev-null/src/test/kotlin/io/airbyte/integrations/destination/dev_null/ThrottledDestinationTest.kt similarity index 97% rename from airbyte-integrations/connectors/destination-e2e-test/src/test/kotlin/io/airbyte/integrations/destination/e2e_test/ThrottledDestinationTest.kt rename to airbyte-integrations/connectors/destination-dev-null/src/test/kotlin/io/airbyte/integrations/destination/dev_null/ThrottledDestinationTest.kt index 367b5909a21c..e7beeacdb428 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/test/kotlin/io/airbyte/integrations/destination/e2e_test/ThrottledDestinationTest.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/test/kotlin/io/airbyte/integrations/destination/dev_null/ThrottledDestinationTest.kt @@ -1,7 +1,7 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.e2e_test +package io.airbyte.integrations.destination.dev_null import com.google.common.collect.ImmutableMap import io.airbyte.commons.json.Jsons diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-dev-null/src/test/resources/expected_spec_cloud.json similarity index 100% rename from airbyte-integrations/connectors/destination-dev-null/src/test/resources/expected_spec.json rename to airbyte-integrations/connectors/destination-dev-null/src/test/resources/expected_spec_cloud.json diff --git a/airbyte-integrations/connectors/destination-e2e-test/README.md b/airbyte-integrations/connectors/destination-e2e-test/README.md deleted file mode 100644 index c4837fe1af1e..000000000000 --- a/airbyte-integrations/connectors/destination-e2e-test/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# End-to-End Testing Destination - -This is the repository for the Null destination connector in Java. For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/destinations/e2e-test). - -## Local development - -#### Building via Gradle - -From the Airbyte repository root, run: - -``` -./gradlew :airbyte-integrations:connectors:destination-e2e-test:build -``` - -#### Create credentials - -No credential is needed for this connector. - -### Locally running the connector docker image - -#### Build - -Build the connector image via Gradle: - -``` -./gradlew :airbyte-integrations:connectors:destination-e2e-test:buildConnectorImage -``` - -Once built, the docker image name and tag on your host will be `airbyte/destination-e2e-test:dev`. -the Dockerfile. - -#### Run - -Then run any of the connector commands as follows: - -``` -docker run --rm airbyte/destination-e2e-test:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-e2e-test:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-e2e-test:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-e2e-test:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json -``` - -#### Cloud variant - -The cloud variant of this connector is Dev Null Destination. It only allows the "silent" mode. When this mode is changed, please make sure that the Dev Null Destination is updated and published accordingly as well. - -## Testing - -We use `JUnit` for Java tests. - -### Unit and Integration Tests - -Place unit tests under `src/test/io/airbyte/integrations/destinations/e2e-test`. - -#### Acceptance Tests - -Airbyte has a standard test suite that all destination connectors must pass. See example(s) in -`src/test-integration/java/io/airbyte/integrations/destinations/e2e-test/`. - -### Using gradle to run tests - -All commands should be run from airbyte project root. -To run unit tests: - -``` -./gradlew :airbyte-integrations:connectors:destination-e2e-test:unitTest -``` - -To run acceptance and custom integration tests: - -``` -./gradlew :airbyte-integrations:connectors:destination-e2e-test:integrationTest -``` - -## Dependency Management - -### Publishing a new version of the connector - -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? - -1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-e2e-test test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). -3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/e2e-test.md`). -5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). -6. Pat yourself on the back for being an awesome contributor. -7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-e2e-test/build.gradle b/airbyte-integrations/connectors/destination-e2e-test/build.gradle deleted file mode 100644 index c3006ff9213c..000000000000 --- a/airbyte-integrations/connectors/destination-e2e-test/build.gradle +++ /dev/null @@ -1,17 +0,0 @@ -plugins { - id 'airbyte-java-connector' -} - -airbyteJavaConnector { - cdkVersionRequired = '0.45.0' - features = ['db-destinations'] - useLocalCdk = true -} - -application { - mainClass = 'io.airbyte.integrations.destination.e2e_test.TestingDestinations' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { -} diff --git a/airbyte-integrations/connectors/destination-e2e-test/icon.svg b/airbyte-integrations/connectors/destination-e2e-test/icon.svg deleted file mode 100644 index 36c7f62a33c1..000000000000 --- a/airbyte-integrations/connectors/destination-e2e-test/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml b/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml deleted file mode 100644 index ff58dea88f1d..000000000000 --- a/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml +++ /dev/null @@ -1,28 +0,0 @@ -data: - connectorSubtype: unknown - connectorType: destination - definitionId: 2eb65e87-983a-4fd7-b3e3-9d9dc6eb8537 - dockerImageTag: 0.5.0 - dockerRepository: airbyte/destination-e2e-test - githubIssueLabel: destination-e2e-test - icon: airbyte.svg - license: MIT - name: E2E Testing - registryOverrides: - cloud: - enabled: false - oss: - enabled: true - releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/destinations/e2e-test - tags: - - language:java - ab_internal: - sl: 100 - ql: 100 - supportLevel: community - supportsRefreshes: true - connectorTestSuitesOptions: - - suite: unitTests - - suite: integrationTests -metadataSpecVersion: "1.0" diff --git a/docs/integrations/destinations/dev-null.md b/docs/integrations/destinations/dev-null.md index ed5668910aab..b58653d71e0b 100644 --- a/docs/integrations/destinations/dev-null.md +++ b/docs/integrations/destinations/dev-null.md @@ -1,17 +1,70 @@ -# Dev Null Destination +# /dev/null Destination -The Airbyte `dev-null` Destination. This destination is for testing and debugging only. +This destination is for testing of Airbyte connections. It can be set up as a source message logger, a `/dev/null`, or to mimic specific behaviors (e.g. exception during the sync). Please use it with discretion. This destination may log your data, and expose sensitive information. + +## Features + +| Feature | Supported | Notes | +| :---------------------------- | :-------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental Sync | Yes | | +| Replicate Incremental Deletes | No | | +| SSL connection | No | | +| SSH Tunnel Support | No | | + +## Mode + +### Silent (`/dev/null`) + +**This is the only mode allowed on Airbyte Cloud.** + +This mode works as `/dev/null`. It does nothing about any data from the source connector. This is usually only useful for performance testing of the source connector. + +### Logging + +This mode logs the data from the source connector. It will log at most 1,000 data entries. + +There are the different logging modes to choose from: + +| Mode | Notes | Parameters | +| :--------------- | :-------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------- | +| First N entries | Log the first N number of data entries for each data stream. | N: how many entries to log. | +| Every N-th entry | Log every N-th entry for each data stream. When N=1, it will log every entry. When N=2, it will log every other entry. Etc. | N: the N-th entry to log. Max entry count: max number of entries to log. | +| Random sampling | Log a random percentage of the entries for each data stream. | Sampling ratio: a number in range of `[0, 1]`. Optional seed: default to system epoch time. Max entry count: max number of entries to log. | + +### Throttling + +This mode mimics a slow data sync. You can specify the time (in millisecond) of delay between each message from the source is processed. + +### Failing + +This mode throws an exception after receiving a configurable number of messages. ## Changelog
Expand to review -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :---------------- | -| 0.3.3 | 2023-05-08 | [38118](https://github.com/airbytehq/airbyte/pull/38118) | Support dedup | -| 0.3.2 | 2023-05-08 | [25776](https://github.com/airbytehq/airbyte/pull/25776) | Support Refreshes | -| 0.3.0 | 2023-05-08 | [25776](https://github.com/airbytehq/airbyte/pull/25776) | Change Schema | -| 0.2.7 | 2022-08-08 | [13932](https://github.com/airbytehq/airbyte/pull/13932) | Bump version | +The OSS and Cloud variants have the same version number starting from version `0.2.2`. + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------- | +| 0.6.0 | 2024-09-18 | [45651](https://github.com/airbytehq/airbyte/pull/45651) | merge destination-e2e(OSS) and destination-dev-null(cloud) | +| 0.5.0 | 2024-09-18 | [45650](https://github.com/airbytehq/airbyte/pull/45650) | upgrade cdk | +| 0.4.1 | 2024-09-18 | [45649](https://github.com/airbytehq/airbyte/pull/45649) | convert test code to kotlin | +| 0.4.0 | 2024-09-18 | [45648](https://github.com/airbytehq/airbyte/pull/45648) | convert production code to kotlin | +| 0.3.6 | 2024-05-09 | [38097](https://github.com/airbytehq/airbyte/pull/38097) | Support dedup | +| 0.3.5 | 2024-04-29 | [37366](https://github.com/airbytehq/airbyte/pull/37366) | Support refreshes | +| 0.3.4 | 2024-04-16 | [37366](https://github.com/airbytehq/airbyte/pull/37366) | Fix NPE | +| 0.3.3 | 2024-04-16 | [37366](https://github.com/airbytehq/airbyte/pull/37366) | Fix Log trace messages | +| 0.3.2 | 2024-02-14 | [36812](https://github.com/airbytehq/airbyte/pull/36812) | Log trace messages | +| 0.3.1 | 2024-02-14 | [35278](https://github.com/airbytehq/airbyte/pull/35278) | Adopt CDK 0.20.6 | +| 0.3.0 | 2023-05-08 | [25776](https://github.com/airbytehq/airbyte/pull/25776) | Standardize spec and change property field to non-keyword | +| 0.2.4 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | +| 0.2.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.2.2 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | +| 0.2.1 | 2021-12-19 | [\#8824](https://github.com/airbytehq/airbyte/pull/8905) | Fix documentation URL. | +| 0.2.0 | 2021-12-16 | [\#8824](https://github.com/airbytehq/airbyte/pull/8824) | Add multiple logging modes. | +| 0.1.0 | 2021-05-25 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) | Create initial version. |
\ No newline at end of file diff --git a/docs/integrations/destinations/e2e-test.md b/docs/integrations/destinations/e2e-test.md deleted file mode 100644 index 37e0294df2cc..000000000000 --- a/docs/integrations/destinations/e2e-test.md +++ /dev/null @@ -1,69 +0,0 @@ -# End-to-End Testing Destination - -This destination is for testing of Airbyte connections. It can be set up as a source message logger, a `/dev/null`, or to mimic specific behaviors (e.g. exception during the sync). Please use it with discretion. This destination may log your data, and expose sensitive information. - -## Features - -| Feature | Supported | Notes | -| :---------------------------- | :-------- | :---- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Yes | | -| Replicate Incremental Deletes | No | | -| SSL connection | No | | -| SSH Tunnel Support | No | | - -## Mode - -### Silent (`/dev/null`) - -**This is the only mode allowed on Airbyte Cloud.** - -This mode works as `/dev/null`. It does nothing about any data from the source connector. This is usually only useful for performance testing of the source connector. - -### Logging - -This mode logs the data from the source connector. It will log at most 1,000 data entries. - -There are the different logging modes to choose from: - -| Mode | Notes | Parameters | -| :--------------- | :-------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------- | -| First N entries | Log the first N number of data entries for each data stream. | N: how many entries to log. | -| Every N-th entry | Log every N-th entry for each data stream. When N=1, it will log every entry. When N=2, it will log every other entry. Etc. | N: the N-th entry to log. Max entry count: max number of entries to log. | -| Random sampling | Log a random percentage of the entries for each data stream. | Sampling ratio: a number in range of `[0, 1]`. Optional seed: default to system epoch time. Max entry count: max number of entries to log. | - -### Throttling - -This mode mimics a slow data sync. You can specify the time (in millisecond) of delay between each message from the source is processed. - -### Failing - -This mode throws an exception after receiving a configurable number of messages. - -## Changelog - -
- Expand to review - -The OSS and Cloud variants have the same version number starting from version `0.2.2`. - -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------- | -| 0.5.0 | 2024-09-18 | [45650](https://github.com/airbytehq/airbyte/pull/45650) | upgrade cdk | -| 0.4.1 | 2024-09-18 | [45649](https://github.com/airbytehq/airbyte/pull/45649) | convert test code to kotlin | -| 0.4.0 | 2024-09-18 | [45648](https://github.com/airbytehq/airbyte/pull/45648) | convert production code to kotlin | -| 0.3.6 | 2024-05-09 | [38097](https://github.com/airbytehq/airbyte/pull/38097) | Support dedup | -| 0.3.5 | 2024-04-29 | [37366](https://github.com/airbytehq/airbyte/pull/37366) | Support refreshes | -| 0.3.4 | 2024-04-16 | [37366](https://github.com/airbytehq/airbyte/pull/37366) | Fix NPE | -| 0.3.3 | 2024-04-16 | [37366](https://github.com/airbytehq/airbyte/pull/37366) | Fix Log trace messages | -| 0.3.2 | 2024-02-14 | [36812](https://github.com/airbytehq/airbyte/pull/36812) | Log trace messages | -| 0.3.1 | 2024-02-14 | [35278](https://github.com/airbytehq/airbyte/pull/35278) | Adopt CDK 0.20.6 | -| 0.3.0 | 2023-05-08 | [25776](https://github.com/airbytehq/airbyte/pull/25776) | Standardize spec and change property field to non-keyword | -| 0.2.4 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | -| 0.2.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.2.2 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | -| 0.2.1 | 2021-12-19 | [\#8824](https://github.com/airbytehq/airbyte/pull/8905) | Fix documentation URL. | -| 0.2.0 | 2021-12-16 | [\#8824](https://github.com/airbytehq/airbyte/pull/8824) | Add multiple logging modes. | -| 0.1.0 | 2021-05-25 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) | Create initial version. | - -
\ No newline at end of file