Skip to content

Commit

Permalink
[CI] [GHA] Add Dockerfile for U20 and use it in the Build job in Li…
Browse files Browse the repository at this point in the history
…nux workflow (#24573)

### Tickets:
 - *141573*
  • Loading branch information
akashchi committed May 22, 2024
1 parent ef6ca0a commit 8c5b357
Show file tree
Hide file tree
Showing 7 changed files with 186 additions and 82 deletions.
2 changes: 1 addition & 1 deletion .github/dockerfiles/docker_tag
Original file line number Diff line number Diff line change
@@ -1 +1 @@
pr-24598
pr-24573
72 changes: 72 additions & 0 deletions .github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04

USER root

# APT configuration
RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf

ENV DEBIAN_FRONTEND="noninteractive" \
TZ="Europe/London"

RUN apt-get update && \
apt-get install software-properties-common && \
add-apt-repository --yes --no-update ppa:git-core/ppa && \
add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \
apt-get update && \
apt-get install \
curl \
git \
ca-certificates \
gpg-agent \
tzdata \
# Pythons
python3.8-dev \
python3.8-venv \
python3.8-distutils \
python3.11-dev \
python3.11-venv \
python3.11-distutils \
# For Java API
default-jdk \
# Compiler \
gcc-10 \
g++-10 \
&& \
rm -rf /var/lib/apt/lists/*

# Install build dependencies
ADD install_build_dependencies.sh /install_build_dependencies.sh
RUN chmod +x /install_build_dependencies.sh && \
/install_build_dependencies.sh && \
rm -rf /var/lib/apt/lists/*

# Set gcc-10 as a default compiler
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30

# Install sscache
ARG SCCACHE_VERSION="v0.7.5"
ENV SCCACHE_HOME="/opt/sccache" \
SCCACHE_PATH="/opt/sccache/sccache"

RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \
SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \
curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \
tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE}

# Setup pip
ENV PIP_VERSION="24.0"
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
rm -f get-pip.py

# Use Python 3.11 as default instead of Python 3.8
# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build
RUN python3.11 -m venv venv
ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"

ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
73 changes: 73 additions & 0 deletions .github/dockerfiles/ov_build/ubuntu_22_04_x64/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04

USER root

# APT configuration
RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf

ENV DEBIAN_FRONTEND="noninteractive" \
TZ="Europe/London"

RUN apt-get update && \
apt-get install software-properties-common && \
add-apt-repository --yes --no-update ppa:git-core/ppa && \
add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \
apt-get update && \
apt-get install \
curl \
git \
ca-certificates \
gpg-agent \
tzdata \
libtbb2 \
# Pythons
python3.8-dev \
python3.8-venv \
python3.8-distutils \
python3.11-dev \
python3.11-venv \
python3.11-distutils \
# For Java API
default-jdk \
# Compiler \
gcc-10 \
g++-10 \
&& \
rm -rf /var/lib/apt/lists/*

# Install build dependencies
ADD install_build_dependencies.sh /install_build_dependencies.sh
RUN chmod +x /install_build_dependencies.sh && \
/install_build_dependencies.sh && \
rm -rf /var/lib/apt/lists/*

# Set gcc-10 as a default compiler
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30

# Install sscache
ARG SCCACHE_VERSION="v0.7.5"
ENV SCCACHE_HOME="/opt/sccache" \
SCCACHE_PATH="/opt/sccache/sccache"

RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \
SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \
curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \
tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE}

# Setup pip
ENV PIP_VERSION="24.0"
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
rm -f get-pip.py

# Use Python 3.11 as default instead of Python 3.8
# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build
RUN python3.11 -m venv venv
ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"

ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
40 changes: 1 addition & 39 deletions .github/workflows/job_onnx_runtime.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,6 @@ jobs:
ONNX_RUNTIME_UTILS: ${{ github.workspace }}/install/onnxruntime
ONNX_RUNTIME_BUILD_DIR: ${{ github.workspace }}/onnxruntime/build
steps:
- name: Set apt retries
if: runner.os == 'Linux'
run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries

- name: Download OpenVINO package
uses: actions/download-artifact@v4
with:
Expand All @@ -59,38 +55,12 @@ jobs:
echo "ONNX_RUNTIME_UTILS=$GITHUB_WORKSPACE/install/onnxruntime" >> "$GITHUB_ENV"
echo "ONNX_RUNTIME_BUILD_DIR=$GITHUB_WORKSPACE/onnxruntime/build" >> "$GITHUB_ENV"
- name: Fetch install_build_dependencies.sh and setup_python action
uses: actions/checkout@v4
with:
sparse-checkout: |
install_build_dependencies.sh
.github/actions/setup_python/action.yml
sparse-checkout-cone-mode: false
path: 'openvino'

- name: Install git
run: |
apt-get update
apt-get install --assume-yes --no-install-recommends git ca-certificates
- name: Setup Python ${{ env.PYTHON_VERSION }}
uses: ./openvino/.github/actions/setup_python
with:
version: '3.11'
should-setup-pip-paths: 'false'

- name: Extract OpenVINO package
run: |
pushd ${INSTALL_DIR}
tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
popd
- name: Install OpenVINO dependencies
run: |
${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -y
# since we are on Ubuntu 22.04, but compiled OpenVINO on Ubuntu 20.04, we need to install `libtbb2`
apt-get install --assume-yes --no-install-recommends libtbb2
- name: Clone ONNX Runtime
run: |
hash=`tr -s '\n ' < ${ONNX_RUNTIME_UTILS}/version`
Expand All @@ -102,14 +72,6 @@ jobs:
# Tests
#

- name: Install Build Dependencies
run: bash ${OPENVINO_REPO}/install_build_dependencies.sh

- name: Install sccache
uses: mozilla-actions/sccache-action@v0.0.4
with:
version: "v0.7.5"

- name: Build Lin ONNX Runtime
run: |
source ${INSTALL_DIR}/setupvars.sh
Expand All @@ -133,7 +95,7 @@ jobs:
if: ${{ runner.arch != 'ARM64' }} # Ticket: 126277
run: |
# see https://github.com/microsoft/onnxruntime/issues/13197#issuecomment-1264542497
apt-get install --assume-yes --no-install-recommends language-pack-en
apt-get update && apt-get install --assume-yes --no-install-recommends language-pack-en
locale-gen en_US.UTF-8
update-locale LANG=en_US.UTF-8
Expand Down
5 changes: 1 addition & 4 deletions .github/workflows/job_tokenizers.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ jobs:
install_build_dependencies.sh
- name: Setup Python ${{ env.PYTHON_VERSION }}
if: ${{ runner.os != 'Linux' }} # We do not need to install Python on Linux as we use Docker with it installed
uses: ./.github/actions/setup_python
with:
version: ${{ env.PYTHON_VERSION }}
Expand Down Expand Up @@ -94,10 +95,6 @@ jobs:
# Dependencies
#

- name: Install build dependencies (Linux)
if: runner.os == 'Linux'
run: ./install_build_dependencies.sh

- name: Install python dependencies
run: |
# wheel packaging
Expand Down
70 changes: 35 additions & 35 deletions .github/workflows/linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ jobs:
runs-on: ubuntu-latest
outputs:
affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
changed_components: "${{ steps.smart_ci.outputs.changed_components }}"
skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
steps:
- name: checkout action
Expand All @@ -45,15 +46,42 @@ jobs:
skip_when_only_listed_labels_set: 'docs'
skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg'

Build:
- name: Show affected components
run: |
echo "${{ toJSON(steps.smart_ci.outputs.affected_components) }}"
shell: bash

Docker:
needs: Smart_CI
runs-on: aks-linux-4-cores-16gb-docker-build
container:
image: openvinogithubactions.azurecr.io/docker_build:0.2
volumes:
- /mount:/mount
outputs:
images: "${{ steps.handle_docker.outputs.images }}"
steps:
- name: Checkout
uses: actions/checkout@v4

- uses: ./.github/actions/handle_docker
id: handle_docker
with:
images: |
ov_build/ubuntu_20_04_x64
registry: 'openvinogithubactions.azurecr.io'
dockerfiles_root_dir: '.github/dockerfiles'
changed_components: ${{ needs.smart_ci.outputs.changed_components }}

Build:
needs: [Docker, Smart_CI]
timeout-minutes: 150
defaults:
run:
shell: bash
runs-on: aks-linux-16-cores-32gb
container:
image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}
volumes:
- /mount:/mount
options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING
Expand All @@ -80,14 +108,6 @@ jobs:
if: "!needs.smart_ci.outputs.skip_workflow"

steps:
- name: Set apt retries
run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries

- name: Install git
run: |
apt-get update
apt-get install --assume-yes --no-install-recommends git ca-certificates
- name: Clone OpenVINO
uses: actions/checkout@v4
with:
Expand Down Expand Up @@ -122,26 +142,6 @@ jobs:
# Dependencies
#

- name: Install build dependencies
run: |
bash ${OPENVINO_REPO}/install_build_dependencies.sh
# default-jdk - Java API
apt install --assume-yes --no-install-recommends default-jdk
- name: Install sccache
uses: mozilla-actions/sccache-action@v0.0.4
with:
version: "v0.7.5"

- name: Setup Python ${{ env.PYTHON_VERSION }}
uses: ./openvino/.github/actions/setup_python
with:
version: ${{ env.PYTHON_VERSION }}
pip-cache-path: ${{ env.PIP_CACHE_PATH }}
should-setup-pip-paths: 'true'
self-hosted-runner: 'true'
show-cache-info: 'true'

- name: Install python dependencies
run: |
# For Python API: build and wheel packaging
Expand Down Expand Up @@ -459,12 +459,12 @@ jobs:
name: ONNX Runtime Integration
if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_RT ||
fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE
needs: [ Build, Smart_CI ]
needs: [ Build, Smart_CI, Docker ]
uses: ./.github/workflows/job_onnx_runtime.yml
with:
runner: 'aks-linux-16-cores-32gb'
container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}'
sccache-azure-key-prefix: 'ubuntu22_x86_64_onnxruntime'
container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}'
sccache-azure-key-prefix: 'ubuntu20_x86_64_onnxruntime'

ONNX_Models:
name: ONNX Models Tests
Expand Down Expand Up @@ -685,12 +685,12 @@ jobs:

Openvino_tokenizers:
name: OpenVINO tokenizers extension
needs: [ Build, Smart_CI ]
needs: [ Build, Smart_CI, Docker ]
uses: ./.github/workflows/job_tokenizers.yml
with:
runner: 'aks-linux-4-cores-16gb'
shell: bash
container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount:/mount"]}'
container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}'
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS

Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/linux_arm64.yml
Original file line number Diff line number Diff line change
Expand Up @@ -323,12 +323,12 @@ jobs:
name: ONNX Runtime Integration
if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_RT ||
fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE
needs: [ Build, Smart_CI ]
needs: [ Build, Smart_CI, Docker ]
uses: ./.github/workflows/job_onnx_runtime.yml
with:
runner: 'aks-linux-16-cores-arm'
container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}'
sccache-azure-key-prefix: 'ubuntu22_aarch64_onnxruntime'
container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_arm64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}'
sccache-azure-key-prefix: 'ubuntu20_aarch64_onnxruntime'

Openvino_tokenizers:
name: OpenVINO tokenizers extension
Expand Down

0 comments on commit 8c5b357

Please sign in to comment.