destination-e2e: merge cloud and OSS

airbytehq · Sep 19, 2024 · 0945d13 · 0945d13
1 parent 8a624a3
commit 0945d13
Show file tree

Hide file tree

Showing 32 changed files with 211 additions and 342 deletions.
diff --git a/...te-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/BaseConnector.kt b/...te-cdk/java/airbyte-cdk/core/src/main/kotlin/io/airbyte/cdk/integrations/BaseConnector.kt
@@ -4,11 +4,19 @@
 package io.airbyte.cdk.integrations
 
 import io.airbyte.cdk.integrations.base.Integration
+import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner
+import io.airbyte.commons.features.EnvVariableFeatureFlags
+import io.airbyte.commons.features.FeatureFlags
 import io.airbyte.commons.json.Jsons
 import io.airbyte.commons.resources.MoreResources
 import io.airbyte.protocol.models.v0.ConnectorSpecification
 
 abstract class BaseConnector : Integration {
+    open val featureFlags: FeatureFlags = EnvVariableFeatureFlags()
+
+    val isCloudDeployment
+        get() =
+            AdaptiveSourceRunner.CLOUD_MODE.equals(featureFlags.deploymentMode(), ignoreCase = true)
     /**
      * By convention the spec is stored as a resource for java connectors. That resource is called
      * spec.json.

diff --git a/.../kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt b/.../kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt
@@ -17,6 +17,8 @@ import io.airbyte.cdk.integrations.standardtest.destination.argproviders.util.Ar
 import io.airbyte.cdk.integrations.standardtest.destination.comparator.BasicTestDataComparator
 import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator
 import io.airbyte.commons.features.EnvVariableFeatureFlags
+import io.airbyte.commons.features.FeatureFlags
+import io.airbyte.commons.features.FeatureFlagsWrapper
 import io.airbyte.commons.jackson.MoreMappers
 import io.airbyte.commons.json.Jsons
 import io.airbyte.commons.lang.Exceptions
@@ -98,6 +100,13 @@ abstract class DestinationAcceptanceTest(
     protected var testSchemas: HashSet<String> = HashSet()
 
     private lateinit var testEnv: TestDestinationEnv
+    protected open val isCloudTest: Boolean = true
+    protected val featureFlags: FeatureFlags =
+        if (isCloudTest) {
+            FeatureFlagsWrapper.overridingDeploymentMode(EnvVariableFeatureFlags(), "CLOUD")
+        } else {
+            FeatureFlagsWrapper.overridingDeploymentMode(EnvVariableFeatureFlags(), "OSS")
+        }
 
     private lateinit var jobRoot: Path
     private lateinit var processFactory: ProcessFactory
@@ -1912,7 +1921,7 @@ abstract class DestinationAcceptanceTest(
                         null,
                         null,
                         false,
-                        EnvVariableFeatureFlags()
+                        featureFlags
                     )
                 )
                 .run(JobGetSpecConfig().withDockerImage(imageName), jobRoot)
@@ -1932,7 +1941,7 @@ abstract class DestinationAcceptanceTest(
                     null,
                     null,
                     false,
-                    EnvVariableFeatureFlags()
+                    featureFlags
                 ),
                 mConnectorConfigUpdater
             )
@@ -1954,7 +1963,7 @@ abstract class DestinationAcceptanceTest(
                             null,
                             null,
                             false,
-                            EnvVariableFeatureFlags()
+                            featureFlags
                         ),
                         mConnectorConfigUpdater
                     )
@@ -1982,7 +1991,7 @@ abstract class DestinationAcceptanceTest(
                         null,
                         null,
                         false,
-                        EnvVariableFeatureFlags()
+                        featureFlags
                     )
             )
         }
@@ -1998,7 +2007,7 @@ abstract class DestinationAcceptanceTest(
                     null,
                     null,
                     false,
-                    EnvVariableFeatureFlags()
+                    featureFlags
                 )
         )
     }

diff --git a/...urces/src/main/kotlin/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.kt b/...urces/src/main/kotlin/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.kt
@@ -22,8 +22,6 @@ import io.airbyte.cdk.integrations.util.ApmTraceUtils.addExceptionToTrace
 import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil
 import io.airbyte.commons.exceptions.ConfigErrorException
 import io.airbyte.commons.exceptions.ConnectionErrorException
-import io.airbyte.commons.features.EnvVariableFeatureFlags
-import io.airbyte.commons.features.FeatureFlags
 import io.airbyte.commons.functional.CheckedConsumer
 import io.airbyte.commons.lang.Exceptions
 import io.airbyte.commons.stream.AirbyteStreamUtils
@@ -49,8 +47,6 @@ private val LOGGER = KotlinLogging.logger {}
 abstract class AbstractDbSource<DataType, Database : AbstractDatabase?>
 protected constructor(driverClassName: String) :
     JdbcConnector(driverClassName), Source, AutoCloseable {
-    // TODO: Remove when the flag is not use anymore
-    var featureFlags: FeatureFlags = EnvVariableFeatureFlags()
 
     @Trace(operationName = CHECK_TRACE_OPERATION_NAME)
     @Throws(Exception::class)

diff --git a/airbyte-integrations/connectors/destination-dev-null/README.md b/airbyte-integrations/connectors/destination-dev-null/README.md
@@ -1,6 +1,6 @@
-# Destination Dev Null
+# End-to-End Testing Destination
 
-This destination is a "safe" version of the [E2E Test destination](https://docs.airbyte.io/integrations/destinations/e2e-test). It only allows the "silent" mode.
+This is the repository for the Null destination connector in Java. For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/destinations/e2e-test).
 
 ## Local development
 
@@ -9,46 +9,67 @@ This destination is a "safe" version of the [E2E Test destination](https://docs.
 From the Airbyte repository root, run:
 
 ```
-./gradlew :airbyte-integrations:connectors:destination-dev-null:build
+./gradlew :airbyte-integrations:connectors:destination-e2e-test:build
 ```
 
+#### Create credentials
+
+No credential is needed for this connector.
+
 ### Locally running the connector docker image
 
 #### Build
 
 Build the connector image via Gradle:
 
 ```
-./gradlew :airbyte-integrations:connectors:destination-dev-null:buildConnectorImage
+./gradlew :airbyte-integrations:connectors:destination-e2e-test:buildConnectorImage
 ```
 
-Once built, the docker image name and tag on your host will be `airbyte/destination-dev-null:dev`.
+Once built, the docker image name and tag on your host will be `airbyte/destination-e2e-test:dev`.
 the Dockerfile.
 
 #### Run
 
 Then run any of the connector commands as follows:
 
 ```
-docker run --rm airbyte/destination-dev-null:dev spec
-docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-dev-null:dev check --config /secrets/config.json
-docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-dev-null:dev discover --config /secrets/config.json
-docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-dev-null:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
+docker run --rm airbyte/destination-e2e-test:dev spec
+docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-e2e-test:dev check --config /secrets/config.json
+docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-e2e-test:dev discover --config /secrets/config.json
+docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-e2e-test:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
 ```
 
+#### Cloud variant
+
+The cloud variant of this connector is Dev Null Destination. It only allows the "silent" mode. When this mode is changed, please make sure that the Dev Null Destination is updated and published accordingly as well.
+
+## Testing
+
+We use `JUnit` for Java tests.
+
+### Unit and Integration Tests
+
+Place unit tests under `src/test/io/airbyte/integrations/destinations/e2e-test`.
+
+#### Acceptance Tests
+
+Airbyte has a standard test suite that all destination connectors must pass. See example(s) in
+`src/test-integration/java/io/airbyte/integrations/destinations/e2e-test/`.
+
 ### Using gradle to run tests
 
 All commands should be run from airbyte project root.
 To run unit tests:
 
 ```
-./gradlew :airbyte-integrations:connectors:destination-dev-null:unitTest
+./gradlew :airbyte-integrations:connectors:destination-e2e-test:unitTest
 ```
 
 To run acceptance and custom integration tests:
 
 ```
-./gradlew :airbyte-integrations:connectors:destination-dev-null:integrationTest
+./gradlew :airbyte-integrations:connectors:destination-e2e-test:integrationTest
 ```
 
 ## Dependency Management
@@ -57,7 +78,7 @@ To run acceptance and custom integration tests:
 
 You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
 
-1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-dev-null test`
+1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-e2e-test test`
 2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
 3. Make sure the `metadata.yaml` content is up to date.
 4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/e2e-test.md`).

diff --git a/airbyte-integrations/connectors/destination-dev-null/build.gradle b/airbyte-integrations/connectors/destination-dev-null/build.gradle
@@ -9,10 +9,9 @@ airbyteJavaConnector {
 }
 
 application {
-    mainClass = 'io.airbyte.integrations.destination.dev_null.DevNullDestination'
+    mainClass = 'io.airbyte.integrations.destination.dev_null.TestingDestinations'
     applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
 }
 
 dependencies {
-    implementation project(':airbyte-integrations:connectors:destination-e2e-test')
 }
diff --git a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml
@@ -2,19 +2,14 @@ data:
   connectorSubtype: file
   connectorType: destination
   definitionId: a7bcc9d8-13b3-4e49-b80d-d020b90045e3
-  dockerImageTag: 0.5.0
+  dockerImageTag: 0.6.0
   dockerRepository: airbyte/destination-dev-null
   githubIssueLabel: destination-dev-null
   icon: airbyte.svg
   license: MIT
   name: End-to-End Testing (/dev/null)
-  registryOverrides:
-    cloud:
-      enabled: true
-    oss:
-      enabled: false
   releaseStage: alpha
-  documentationUrl: https://docs.airbyte.com/integrations/destinations/e2e-test
+  documentationUrl: https://docs.airbyte.com/integrations/destinations/dev-null
   tags:
     - language:java
   ab_internal:

diff --git a/...v-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestination.kt b/...v-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestination.kt
diff --git a/...ination/e2e_test/FailAfterNDestination.kt → ...ination/dev_null/FailAfterNDestination.kt b/...ination/e2e_test/FailAfterNDestination.kt → ...ination/dev_null/FailAfterNDestination.kt
@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2023 Airbyte, Inc., all rights reserved.
  */
-package io.airbyte.integrations.destination.e2e_test
+package io.airbyte.integrations.destination.dev_null
 
 import com.fasterxml.jackson.databind.JsonNode
 import io.airbyte.cdk.integrations.BaseConnector

diff --git a/...estination/e2e_test/LoggingDestination.kt → ...estination/dev_null/LoggingDestination.kt b/...estination/e2e_test/LoggingDestination.kt → ...estination/dev_null/LoggingDestination.kt
@@ -1,14 +1,14 @@
 /*
  * Copyright (c) 2023 Airbyte, Inc., all rights reserved.
  */
-package io.airbyte.integrations.destination.e2e_test
+package io.airbyte.integrations.destination.dev_null
 
 import com.fasterxml.jackson.databind.JsonNode
 import io.airbyte.cdk.integrations.BaseConnector
 import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer
 import io.airbyte.cdk.integrations.base.Destination
-import io.airbyte.integrations.destination.e2e_test.logging.LoggingConsumer
-import io.airbyte.integrations.destination.e2e_test.logging.TestingLoggerFactory
+import io.airbyte.integrations.destination.dev_null.logging.LoggingConsumer
+import io.airbyte.integrations.destination.dev_null.logging.TestingLoggerFactory
 import io.airbyte.protocol.models.v0.AirbyteConnectionStatus
 import io.airbyte.protocol.models.v0.AirbyteMessage
 import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog

diff --git a/...destination/e2e_test/SilentDestination.kt → ...destination/dev_null/SilentDestination.kt b/...destination/e2e_test/SilentDestination.kt → ...destination/dev_null/SilentDestination.kt
@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2023 Airbyte, Inc., all rights reserved.
  */
-package io.airbyte.integrations.destination.e2e_test
+package io.airbyte.integrations.destination.dev_null
 
 import com.fasterxml.jackson.databind.JsonNode
 import io.airbyte.cdk.integrations.BaseConnector