Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[v22.3.x] Breakdown installation of ducktape dependencies in multiple bash scripts #11059

Merged
merged 5 commits into from
Jun 2, 2023
Merged

[v22.3.x] Breakdown installation of ducktape dependencies in multiple bash scripts #11059

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
884f5b2
dt/docker: divide ducktape-deps.sh in multiple ones
ivotron Apr 5, 2023
e4dfc44
dt/docker: use multi-stage build in dockerfile
ivotron Apr 5, 2023
0fdfb63
dt/docker: add helm and kubectl as dependencies
ivotron Apr 5, 2023
b205055
Add missing dependencies
gousteris May 19, 2023
4537f04
match deps with 22.3.x
gousteris May 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
249 changes: 183 additions & 66 deletions tests/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,113 +12,230 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM ubuntu:jammy
FROM ubuntu:jammy as base

ENV TZ="UTC" \
DEBIAN_FRONTEND=noninteractive

COPY --chown=0:0 tests/docker/ducktape-deps.sh /opt/ducktape-deps.sh
RUN chmod +x /opt/ducktape-deps.sh
RUN mkdir -p /opt/redpanda-tests
COPY --chown=0:0 tests/protobuf /opt/redpanda-tests/protobuf

COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/java-dev-tools /
COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/protobuf /
RUN /java-dev-tools && \
rm /java-dev-tools && \
rm -rf /var/lib/apt/lists/*

# Install dependencies of Java client builds.
RUN /opt/ducktape-deps.sh install_java_client_deps && \
# - install distro-packaged depedencies
# - install dependencies of 'rpk debug' system scan
COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/tool-pkgs /
RUN /tool-pkgs && \
rm /tool-pkgs && \
rm -rf /var/lib/apt/lists/*

#################################

FROM base as kafka-streams-examples

# Install kafka streams examples. This is a very slow step (tens of minutes), doing
# many maven dependency downloads without any parallelism. To avoid re-running it
# on unrelated changes in other steps, this step is as early on the Dockerfile as possible.
RUN /opt/ducktape-deps.sh install_kafka_streams_examples
COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/kafka-streams-examples /
RUN /kafka-streams-examples && \
rm /kafka-streams-examples

# Install Protobuf for test clients / workloads
RUN mkdir -p /opt/redpanda-tests
COPY --chown=0:0 tests/protobuf /opt/redpanda-tests/protobuf
#################################

# Install our in-tree Java test clientst
RUN mkdir -p /opt/redpanda-tests
COPY --chown=0:0 tests/java /opt/redpanda-tests/java
COPY --chown=0:0 tests/java/kafka-serde /opt/redpanda-tests/java/kafka-serde
RUN /opt/ducktape-deps.sh install_java_test_clients
FROM base as omb

# - install distro-packaged depedencies
# - allow user env variables in ssh sessions
# - install dependencies of 'rpk debug' system scan
RUN /opt/ducktape-deps.sh install_system_deps && \
rm -rf /var/lib/apt/lists/* && \
echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config && \
echo 'UsePAM yes' >> /etc/ssh/sshd_config && \
echo 'root soft nofile 65535' >> /etc/security/limits.conf && \
echo 'root hard nofile 65535' >> /etc/security/limits.conf
COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/omb /
RUN /omb && \
rm /omb

#################################

FROM base as java-verifiers

COPY --chown=0:0 tests/java/kafka-verifier /opt/redpanda-tests/java/kafka-verifier
COPY --chown=0:0 tests/java/compacted-log-verifier /opt/redpanda-tests/java/compacted-log-verifier
COPY --chown=0:0 tests/java/tx-verifier /opt/redpanda-tests/java/tx-verifier
COPY --chown=0:0 tests/java/e2e-verifiers /opt/redpanda-tests/java/e2e-verifiers
COPY --chown=0:0 tests/java/kafka-serde /opt/redpanda-tests/java/kafka-serde
COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/java-verifiers /
RUN /java-verifiers && \
rm /java-verifiers

#################################

# Install the OMB tool
RUN /opt/ducktape-deps.sh install_omb
FROM base as kafka-tools

# install kafka binary dependencies, librdkafka dev, kcat and kaf tools
ENV KAFKA_MIRROR="https://s3-us-west-2.amazonaws.com/kafka-packages"
RUN /opt/ducktape-deps.sh install_kafka_tools && \
/opt/ducktape-deps.sh install_librdkafka && \
/opt/ducktape-deps.sh install_kcat
COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/kafka-tools /
RUN /kafka-tools && \
rm /kafka-tools

# install go
RUN /opt/ducktape-deps.sh install_golang
ENV PATH="${PATH}:/usr/local/go/bin"
#################################

FROM base as librdkafka

COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/librdkafka /
RUN /librdkafka && \
rm /librdkafka

# Install `kaf`
RUN /opt/ducktape-deps.sh install_kaf
#################################

FROM librdkafka as kcat

COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/kcat /
RUN /kcat && \
rm /kcat

#################################

FROM base as golang

COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/golang /
RUN /golang && \
rm /golang
ENV PATH="${PATH}:/usr/local/go/bin"
# Compile and install rust-based workload generator.
# Install & remove compiler in the same step, to avoid bulking
# out the resulting container image.
RUN /opt/ducktape-deps.sh install_client_swarm /root

# Install golang dependencies for kafka clients such as sarama
RUN /opt/ducktape-deps.sh install_sarama_examples
#################################

# Install our in-tree go tests
COPY --chown=0:0 tests/go /opt/redpanda-tests/go
RUN /opt/ducktape-deps.sh install_go_test_clients
FROM golang as kaf

# Install franz-go
RUN /opt/ducktape-deps.sh install_franz_bench
COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/kaf /
RUN /kaf && \
rm /kaf

RUN /opt/ducktape-deps.sh install_kcl
#################################

# Install the kgo-verifier tool
RUN /opt/ducktape-deps.sh install_kgo_verifier
FROM base as client-swarm

# Expose port 8080 for any http examples within clients
EXPOSE 8080
COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/client-swarm /
RUN /client-swarm /root && \
rm /client-swarm

#################################

FROM golang as sarama-examples

# copy ssh keys
COPY --chown=0:0 tests/docker/ssh /root/.ssh
COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/sarama-examples /
RUN /sarama-examples && \
rm /sarama-examples

#################################

FROM golang as golang-test-clients

COPY --chown=0:0 --chmod=0755 tests/go /opt/redpanda-tests/go
COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/golang-test-clients /
RUN /golang-test-clients && \
rm /golang-test-clients

#################################

FROM golang as franz-bench

COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/franz-bench /
RUN /franz-bench && \
rm /franz-bench

#################################

FROM golang as kcl

COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/kcl /
RUN /kcl && \
rm /kcl

#################################

FROM golang as kgo-verifier

COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/kgo-verifier /
RUN /kgo-verifier && \
rm /kgo-verifier

#################################

FROM base as k8s

COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/k8s /
RUN /k8s && \
rm /k8s

#################################

FROM base as final

COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/arroyo /
RUN /arroyo && \
rm /arroyo

RUN mkdir -p /opt/scripts/consumer_offsets_recovery
COPY --chown=0:0 --chmod=0755 tools/consumer_offsets_recovery /opt/scripts/consumer_offsets_recovery
RUN python3 -m pip install --force --no-cache-dir -r /opt/scripts/consumer_offsets_recovery/requirements.txt

# install python dependencies and rptest package.
# rptest package installed in editable mode so it can be overridden.
# passes --force so system pip packages can be updated
COPY --chown=0:0 tests/setup.py /root/tests/
COPY --chown=0:0 --chmod=0755 tests/setup.py /root/tests/
RUN python3 -m pip install --upgrade --force pip && \
python3 -m pip install --force --no-cache-dir -e /root/tests/

# Seastar addrress to line utility
# seastar addrress to line utility depends on 'file' pkg
RUN apt update && \
apt install -y \
file && \
apt install -y file && \
rm -rf /var/lib/apt/lists/*
COPY --chown=0:0 --chmod=0755 tests/docker/ducktape-deps/addr2line /
RUN /addr2line && \
rm /addr2line

RUN /opt/ducktape-deps.sh install_arroyo
# copy offline_log_viewer
RUN mkdir -p /opt/scripts/offline_log_viewer
COPY --chown=0:0 --chmod=0755 tools/offline_log_viewer /opt/scripts/offline_log_viewer

RUN /opt/ducktape-deps.sh install_addr2line
# copy remote_scripts
RUN mkdir -p /opt/remote
COPY --chown=0:0 --chmod=0755 tests/rptest/remote_scripts /opt/remote

RUN mkdir -p /opt/scripts/offline_log_viewer
COPY --chown=0:0 tools/offline_log_viewer /opt/scripts/offline_log_viewer
# expose port 8080 for any http examples within clients
EXPOSE 8080

RUN mkdir -p /opt/scripts/consumer_offsets_recovery
COPY --chown=0:0 tools/consumer_offsets_recovery /opt/scripts/consumer_offsets_recovery
RUN python3 -m pip install --force --no-cache-dir -r /opt/scripts/consumer_offsets_recovery/requirements.txt
# copy known ssh keys
COPY --chown=0:0 --chmod=0755 tests/docker/ssh /root/.ssh

# Install remote_scripts
RUN mkdir -p /opt/remote
COPY --chown=0:0 tests/rptest/remote_scripts /opt/remote
# allow user env variables in ssh sessions and increase nofile limit
RUN echo 'PermitUserEnvironment yes' >> /etc/ssh/sshd_config && \
echo 'UsePAM yes' >> /etc/ssh/sshd_config && \
echo 'root soft nofile 65535' >> /etc/security/limits.conf && \
echo 'root hard nofile 65535' >> /etc/security/limits.conf

# copy from other images
COPY --from=kafka-streams-examples /opt/kafka-streams-examples /opt/kafka-streams-examples
COPY --from=kafka-streams-examples /root/.m2 /root/.m2
COPY --from=omb /opt/openmessaging-benchmark /opt/openmessaging-benchmark
COPY --from=omb /root/.m2 /root/.m2
COPY --from=java-verifiers /opt/redpanda-tests/java /opt/redpanda-tests/java
COPY --from=java-verifiers /opt/kafka-serde /opt/kafka-serde
COPY --from=java-verifiers /opt/kafka-verifier /opt/kafka-verifier
COPY --from=java-verifiers /opt/compacted-log-verifier /opt/compacted-log-verifier
COPY --from=java-verifiers /opt/tx-verifier /opt/tx-verifier
COPY --from=java-verifiers /root/.m2 /root/.m2
COPY --from=kafka-tools /opt /opt
COPY --from=librdkafka /opt/librdkafka /opt/librdkafka
COPY --from=librdkafka /usr/local/lib /usr/local/lib
COPY --from=kcat /usr/local/bin/kcat /usr/local/bin/
COPY --from=sarama-examples /opt/sarama /opt/sarama
COPY --from=golang-test-clients /opt/redpanda-tests/go /opt/redpanda-tests/go
COPY --from=client-swarm /usr/local/bin/client-swarm /usr/local/bin/
COPY --from=franz-bench /opt/franz-go/examples/bench /opt/franz-go/examples/bench
COPY --from=k8s /usr/local/bin/kubectl /usr/local/bin/helm /usr/local/bin/
COPY --from=kaf /usr/local/bin/kaf /usr/local/bin/
COPY --from=kcl /usr/local/bin/kcl /usr/local/bin/
COPY --from=kgo-verifier /opt/kgo-verifier /opt/kgo-verifier

RUN ldconfig

# define sshd as the entrypoint
CMD service ssh start && tail -f /dev/null
2 changes: 1 addition & 1 deletion tests/docker/Dockerfile.dockerignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@
!tests/go
!tools/offline_log_viewer
!tools/consumer_offsets_recovery
!tests/docker/ducktape-deps.sh
!tests/docker/ducktape-deps/
!tests/rptest/remote_scripts
!tests/protobuf
Loading