diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml
index 670ea34..3b871e1 100644
--- a/.github/workflows/go.yaml
+++ b/.github/workflows/go.yaml
@@ -12,10 +12,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.x
+    - name: Set up Go 1.14
       uses: actions/setup-go@v2
       with:
-        go-version: ^1.14
+        go-version: 1.14
       id: go
 
     - name: Check out code into the Go module directory
diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml
new file mode 100644
index 0000000..40ba84c
--- /dev/null
+++ b/.github/workflows/integration-test.yaml
@@ -0,0 +1,25 @@
+name: Integration tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  integration-test:
+    name: Integration test
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.14
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.14
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Test
+      run: ./test/test.sh
diff --git a/test/README.md b/test/README.md
new file mode 100644
index 0000000..8c2b6f0
--- /dev/null
+++ b/test/README.md
@@ -0,0 +1,3 @@
+= Testing for the OpenTelemetry Collector Builder
+
+This is a set of end-to-end tests for the builder. As such, it includes only positive tests, based on the manifest files in this directory. Each manifest is expected to be in a working state and should yield an OpenTelemetry Collector instance that is ready within a time interval. "Ready" is defined by calling its healthcheck endpoint.
diff --git a/test/replaces.builder.yaml b/test/replaces.builder.yaml
new file mode 100644
index 0000000..8adf373
--- /dev/null
+++ b/test/replaces.builder.yaml
@@ -0,0 +1,10 @@
+dist:
+  module: github.com/observatorium/opentelemetry-collector-builder/test/replaces
+  otelcol_version: 0.12.0
+
+processors:
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/routingprocessor v0.12.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.12.0
+
+replaces:
+  - github.com/open-telemetry/opentelemetry-collector-contrib/internal/common => github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.12.0
diff --git a/test/replaces.otel.yaml b/test/replaces.otel.yaml
new file mode 100755
index 0000000..4cbcae7
--- /dev/null
+++ b/test/replaces.otel.yaml
@@ -0,0 +1,36 @@
+extensions:
+  health_check:
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: localhost:55680
+
+processors:
+  resourcedetection:
+    detectors:
+      - env
+  routing:
+    default_exporters:
+    - logging
+    from_attribute: X-Tenant
+    table:
+    - value: acme
+      exporters: 
+      - logging
+
+exporters:
+  logging:
+
+service:
+  extensions: [health_check]
+  pipelines:
+    traces:
+      receivers:
+        - otlp
+      processors:
+        - resourcedetection
+        - routing
+      exporters:
+        - logging
diff --git a/test/test.sh b/test/test.sh
new file mode 100755
index 0000000..eb09acf
--- /dev/null
+++ b/test/test.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# each attempt pauses for 100ms before retrying
+max_retries=50
+
+tests=replaces
+
+base=`mktemp -d`
+echo "Running the tests in ${base}"
+
+failed=false
+for test in $tests
+do 
+    out="${base}/${test}"
+    mkdir -p "${out}"
+    if [ $? != 0 ]; then
+        echo "❌ FAIL ${test}. Failed to create test directory for the test. Aborting tests."
+        exit 2
+    fi
+
+    echo "Starting test '${test}' at `date`" >> "${out}/test.log"
+
+    go run . --config "./test/${test}.builder.yaml" --output-path "${out}" --name otelcol-built-test > "${out}/builder.log" 2>&1
+    if [ $? != 0 ]; then
+        echo "❌ FAIL ${test}. Failed to compile the test ${test}. Build logs:"
+        cat "${out}/builder.log"
+        failed=true
+        continue
+    fi
+
+    if [ ! -f "${out}/otelcol-built-test" ]; then
+        echo "❌ FAIL ${test}. Binary not found for ${test} at '${out}/otelcol-built-test'. Build logs:"
+        cat "${out}/builder.log"
+        failed=true
+        continue
+    fi
+
+    # start the distribution
+    "${out}/otelcol-built-test" --config "./test/${test}.otel.yaml" > "${out}/otelcol.log" 2>&1 &
+    pid=$!
+
+    retries=0
+    while true
+    do
+        kill -0 "${pid}" >/dev/null 2>&1
+        if [ $? != 0 ]; then
+            echo "❌ FAIL ${test}. The OpenTelemetry Collector isn't running. Startup log:"
+            cat "${out}/otelcol.log"
+            failed=true
+            break
+        fi
+
+        curl -s localhost:13133  | grep "Server available" > /dev/null
+        if [ $? == 0 ]; then
+            echo "✅ PASS ${test}"
+
+            kill "${pid}"
+            if [ $? != 0 ]; then
+                echo "Failed to stop the running instance for test ${test}. Return code: $? . Skipping tests."
+                exit 4
+            fi
+            break
+        fi
+
+        echo "Server still unavailable for test '${test}'" >> "${out}/test.log"
+
+        let "retries++"
+        if [ "$retries" -gt "$max_retries" ]; then
+            echo "❌ FAIL ${test}. Server wasn't up after about 5s."
+            failed=true
+
+            kill "${pid}"
+            if [ $? != 0 ]; then
+                echo "Failed to stop the running instance for test ${test}. Return code: $? . Skipping tests."
+                exit 8
+            fi
+            break
+        fi
+        sleep 0.1s
+    done
+
+    echo "Stopping server for '${test}' (pid: ${pid})" >> "${out}/test.log"
+    wait ${pid}
+done
+
+if [[ "$failed" == "true" ]]; then
+    exit 1
+fi