Ubuntu switch to 20.04 and add patches to minimize GLIBC requirements

.github/workflows/ci.yml

@@ -25,8 +25,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-10.15, windows-2019, ubuntu-22.04]
-        arch: [32, 64]
+        os: [macos-10.15, windows-2019, ubuntu-20.04]
+        arch: [x86, x64]
         go: [1.16.3]
         include:
           - os: macos-10.15
@@ -40,19 +40,25 @@ jobs:
           - os: windows-2019
             friendlyName: Windows
             targetPlatform: win32
-          - os: ubuntu-22.04
+          - os: ubuntu-20.04
             friendlyName: Linux
             targetPlatform: ubuntu
+          - os: ubuntu-20.04
+            friendlyName: Linux
+            targetPlatform: ubuntu
+            arch: arm64
+          - os: ubuntu-20.04
+            friendlyName: Linux
+            targetPlatform: ubuntu
+            arch: arm
         exclude:
           - os: macos-10.15
-            arch: 32
-          - os: ubuntu-22.04
-            arch: 32
+            arch: x86
     timeout-minutes: 20
     steps:
       # We need to use Xcode 10.3 for maximum compatibility with older macOS (x64)
       - name: Switch to Xcode 10.3
-        if: matrix.targetPlatform == 'macOS' && matrix.arch == 64
+        if: matrix.targetPlatform == 'macOS' && matrix.arch == 'x64'
         run: |
           sudo xcode-select -s /Applications/Xcode_10.3.app/Contents/Developer/
           # Delete the command line tools to make sure they don't get our builds
@@ -73,11 +79,35 @@ jobs:
         run: npm run prettier
       - name: Build tools
         run: npm run check
-      - name: Install extra dependencies for building Git on Ubuntu
-        if: matrix.targetPlatform == 'ubuntu'
+      - name: Install extra dependencies for building Git on Ubuntu (x64)
+        if: matrix.targetPlatform == 'ubuntu' && matrix.arch == 'x64'
         run: |
           sudo apt-get update
           sudo apt-get install libcurl4-openssl-dev libexpat1-dev gettext
+      - name: Install extra dependencies for building Git on Ubuntu (x86)
+        if: matrix.targetPlatform == 'ubuntu' && matrix.arch == 'x86'
+        run: |
+          sudo dpkg --add-architecture i386
+          sudo apt-get update
+          sudo apt-get install gcc-i686-linux-gnu binutils-i686-gnu libcurl4-openssl-dev:i386 libssl-dev:i386 zlib1g-dev:i386 gettext
+      - name: Install extra dependencies for building Git on Ubuntu (arm64)
+        if: matrix.targetPlatform == 'ubuntu' && matrix.arch == 'arm64'
+        run: |
+          sudo sed -i "s/^deb/deb [arch=amd64,i386]/g" /etc/apt/sources.list
+          echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ $(lsb_release -s -c) main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list
+          echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ $(lsb_release -s -c)-updates main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list
+          sudo dpkg --add-architecture arm64
+          sudo apt-get update
+          sudo apt-get install gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu libcurl4-openssl-dev:arm64 libssl-dev:arm64 zlib1g-dev:arm64 gettext
+      - name: Install extra dependencies for building Git on Ubuntu (arm)
+        if: matrix.targetPlatform == 'ubuntu' && matrix.arch == 'arm'
+        run: |
+          sudo sed -i "s/^deb/deb [arch=amd64,i386]/g" /etc/apt/sources.list
+          echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ $(lsb_release -s -c) main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list
+          echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ $(lsb_release -s -c)-updates main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list
+          sudo dpkg --add-architecture armhf
+          sudo apt-get update
+          sudo apt-get install gcc-arm-linux-gnueabihf binutils-arm-linux-gnueabihf libcurl4-openssl-dev:armhf libssl-dev:armhf zlib1g-dev:armhf gettext
       - name: Build (except macOS arm64)
         if: matrix.targetPlatform != 'macOS' || matrix.arch != 'arm64'
         shell: bash

README.md

@@ -37,7 +37,7 @@ Bugfixes and feature requests for Git should be discussied on the [Git mailing l
 
  - Windows 7 and later
  - macOS 10.9 and up
- - Linux (tested on Ubuntu Precise/Trusty and Fedora 24)
+ - Linux (GLIBC 2.17+ on x64/arm64 systems and GLIBC 2.28+ on x86/arm32 systems)
 
 ### Credits
 

dependencies.json

@@ -27,6 +27,24 @@
         "name": "git-lfs-linux-amd64-v3.3.0.tar.gz",
         "checksum": "6a4e6bd7d06d5c024bc70c8ee8c9da143ffc37d2646e252a17a6126d30cdebc1"
       },
+      {
+        "platform": "linux",
+        "arch": "x86",
+        "name": "git-lfs-linux-386-v3.3.0.tar.gz",
+        "checksum": "14415ebafc3ace60f178cd69d4f2e0ed42dbbf32cb2aba80e46ec3c8f7c1401f"
+      },
+      {
+        "platform": "linux",
+        "arch": "arm64",
+        "name": "git-lfs-linux-arm64-v3.3.0.tar.gz",
+        "checksum": "e97c477981a9b6a40026cadc1bf005541d973fc32df2de2f398643b15df6b5c6"
+      },
+      {
+        "platform": "linux",
+        "arch": "arm",
+        "name": "git-lfs-linux-arm-v3.3.0.tar.gz",
+        "checksum": "df8b24cf7ff6a2f105dd1a3d0a4990c53980272ea94da67d854921e21bc5444c"
+      },
       {
         "platform": "windows",
         "arch": "x86",

script/build-macos.sh

@@ -7,7 +7,7 @@ set -eu -o pipefail
 
 MACOSX_BUILD_VERSION="10.9"
 
-if [ "$TARGET_ARCH" = "64" ]; then
+if [ "$TARGET_ARCH" = "x64" ]; then
   HOST_CPU=x86_64
   TARGET_CFLAGS="-target x86_64-apple-darwin"
   GOARCH=amd64

script/build-ubuntu.sh

@@ -20,15 +20,49 @@ if [[ -z "${CURL_INSTALL_DIR}" ]]; then
   exit 1
 fi
 
-if [ "$TARGET_ARCH" = "64" ]; then
-  DEPENDENCY_ARCH="amd64"
-else
-  DEPENDENCY_ARCH="x86"
-fi
+case "$TARGET_ARCH" in
+  "x64")
+    # __GLIBC_MINOR__ is used as a feature test macro.  Replace it with the
+    # earliest supported version of glibc 2.17 as was previously the case when building on ubuntu-18.04
+    sudo sed -i 's|\(#define\s\+__GLIBC_MINOR__\)|\1 17 //|' "/usr/include/features.h"
+    # fcntl64() was introduced in glibc 2.28.  Make sure to use fcntl() instead.
+    sudo  sed -i '{N; s/#ifndef __USE_FILE_OFFSET64\(\nextern int fcntl\)/#if 1\1/}' "/usr/include/fcntl.h"
+    DEPENDENCY_ARCH="amd64"
+    export CC="gcc"
+    STRIP="strip"
+    HOST=""
+    TARGET="" ;;
+  "x86")
+    DEPENDENCY_ARCH="x86"
+    export CC="i686-linux-gnu-gcc"
+    STRIP="i686-gnu-strip"
+    HOST="--host=i686-linux-gnu"
+    TARGET="--target=i686-linux-gnu" ;;
+  "arm64")
+    # __GLIBC_MINOR__ is used as a feature test macro.  Replace it with the
+    # earliest supported version of glibc 2.17 as was previously the case when building on ubuntu-18.04
+    sudo sed -i 's|\(#define\s\+__GLIBC_MINOR__\)|\1 17 //|' "/usr/aarch64-linux-gnu/include/features.h"
+    # fcntl64() was introduced in glibc 2.28.  Make sure to use fcntl() instead.
+    sudo  sed -i '{N; s/#ifndef __USE_FILE_OFFSET64\(\nextern int fcntl\)/#if 1\1/}' "/usr/aarch64-linux-gnu/include/fcntl.h"
+    DEPENDENCY_ARCH="arm64"
+    export CC="aarch64-linux-gnu-gcc"
+    STRIP="aarch64-linux-gnu-strip"
+    HOST="--host=aarch64-linux-gnu"
+    TARGET="--target=aarch64-linux-gnu" ;;
+  "arm")
+    DEPENDENCY_ARCH="arm"
+    export CC="arm-linux-gnueabihf-gcc"
+    STRIP="arm-linux-gnueabihf-strip"
+    HOST="--host=arm-linux-gnueabihf"
+    TARGET="--target=arm-linux-gnueabihf" ;;
+  *)
+    exit 1 ;;
+esac
 
 CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 GIT_LFS_VERSION="$(jq --raw-output '.["git-lfs"].version[1:]' dependencies.json)"
 GIT_LFS_CHECKSUM="$(jq --raw-output ".\"git-lfs\".files[] | select(.arch == \"$DEPENDENCY_ARCH\" and .platform == \"linux\") | .checksum" dependencies.json)"
+GIT_LFS_FILENAME="$(jq --raw-output ".\"git-lfs\".files[] | select(.arch == \"$DEPENDENCY_ARCH\" and .platform == \"linux\") | .name" dependencies.json)"
 
 # shellcheck source=script/compute-checksum.sh
 source "$CURRENT_DIR/compute-checksum.sh"
@@ -46,7 +80,7 @@ tar -xf $CURL_FILE
 
 (
 cd $CURL_FILE_NAME || exit 1
-./configure --prefix="$CURL_INSTALL_DIR"
+./configure --prefix="$CURL_INSTALL_DIR" "$HOST" "$TARGET"
 make install
 )
 echo " -- Building git at $SOURCE to $DESTINATION"
@@ -55,12 +89,12 @@ echo " -- Building git at $SOURCE to $DESTINATION"
 cd "$SOURCE" || exit 1
 make clean
 make configure
-CC='gcc' \
-  CFLAGS='-Wall -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security -U_FORTIFY_SOURCE' \
-  LDFLAGS='-Wl,-Bsymbolic-functions -Wl,-z,relro' \
-  ./configure \
+CFLAGS='-Wall -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security -U_FORTIFY_SOURCE' \
+  LDFLAGS='-Wl,-Bsymbolic-functions -Wl,-z,relro' ac_cv_iconv_omits_bom=no ac_cv_fread_reads_directories=no ac_cv_snprintf_returns_bogus=no \
+  ./configure $HOST \
   --with-curl="$CURL_INSTALL_DIR" \
   --prefix=/
+sed -i "s/STRIP = strip/STRIP = $STRIP/" Makefile
 DESTDIR="$DESTINATION" \
   NO_TCLTK=1 \
   NO_GETTEXT=1 \
@@ -72,7 +106,7 @@ DESTDIR="$DESTINATION" \
 if [[ "$GIT_LFS_VERSION" ]]; then
   echo "-- Bundling Git LFS"
   GIT_LFS_FILE=git-lfs.tar.gz
-  GIT_LFS_URL="https://github.com/git-lfs/git-lfs/releases/download/v${GIT_LFS_VERSION}/git-lfs-linux-amd64-v${GIT_LFS_VERSION}.tar.gz"
+  GIT_LFS_URL="https://github.com/git-lfs/git-lfs/releases/download/v${GIT_LFS_VERSION}/${GIT_LFS_FILENAME}"
   echo "-- Downloading from $GIT_LFS_URL"
   curl -sL -o $GIT_LFS_FILE "$GIT_LFS_URL"
   COMPUTED_SHA256=$(compute_checksum $GIT_LFS_FILE)
@@ -128,12 +162,13 @@ check_static_linking "$DESTINATION"
 
 set -eu -o pipefail
 
+if [ "$TARGET_ARCH" == "x64" ]; then
+(
 echo "-- Testing clone operation with generated binary"
 
 TEMP_CLONE_DIR=/tmp/clones
 mkdir -p $TEMP_CLONE_DIR
 
-(
 cd "$DESTINATION/bin" || exit 1
 ./git --version
 GIT_CURL_VERBOSE=1 \
@@ -143,5 +178,6 @@ GIT_CURL_VERBOSE=1 \
   PREFIX="$DESTINATION" \
   ./git clone https://github.com/git/git.github.io "$TEMP_CLONE_DIR/git.github.io"
 )
+fi
 
 set +eu

script/build-win32.sh

@@ -10,7 +10,7 @@ if [[ -z "${DESTINATION}" ]]; then
   exit 1
 fi
 
-if [ "$TARGET_ARCH" = "64" ]; then
+if [ "$TARGET_ARCH" = "x64" ]; then
   DEPENDENCY_ARCH="amd64"
   MINGW_DIR="mingw64"
 else
@@ -20,6 +20,7 @@ fi
 
 GIT_LFS_VERSION=$(jq --raw-output ".[\"git-lfs\"].version[1:]" dependencies.json)
 GIT_LFS_CHECKSUM="$(jq --raw-output ".\"git-lfs\".files[] | select(.arch == \"$DEPENDENCY_ARCH\" and .platform == \"windows\") | .checksum" dependencies.json)"
+GIT_LFS_FILENAME="$(jq --raw-output ".\"git-lfs\".files[] | select(.arch == \"$DEPENDENCY_ARCH\" and .platform == \"windows\") | .name" dependencies.json)"
 GIT_FOR_WINDOWS_URL=$(jq --raw-output ".git.packages[] | select(.arch == \"$DEPENDENCY_ARCH\" and .platform == \"windows\") | .url" dependencies.json)
 GIT_FOR_WINDOWS_CHECKSUM=$(jq --raw-output ".git.packages[] | select(.arch == \"$DEPENDENCY_ARCH\" and .platform == \"windows\") | .checksum" dependencies.json)
 
@@ -47,8 +48,7 @@ if [[ "$GIT_LFS_VERSION" ]]; then
   # download Git LFS, verify its the right contents, and unpack it
   echo "-- Bundling Git LFS"
   GIT_LFS_FILE=git-lfs.zip
-  if [ "$TARGET_ARCH" -eq "64" ]; then GIT_LFS_ARCH="amd64"; else GIT_LFS_ARCH="386"; fi
-  GIT_LFS_URL="https://github.com/git-lfs/git-lfs/releases/download/v${GIT_LFS_VERSION}/git-lfs-windows-${GIT_LFS_ARCH}-v${GIT_LFS_VERSION}.zip"
+  GIT_LFS_URL="https://github.com/git-lfs/git-lfs/releases/download/v${GIT_LFS_VERSION}/${GIT_LFS_FILENAME}"
   echo "-- Downloading from $GIT_LFS_URL"
   curl -sL -o $GIT_LFS_FILE "$GIT_LFS_URL"
   COMPUTED_SHA256=$(compute_checksum $GIT_LFS_FILE)

script/check-static-linking.sh

@@ -11,7 +11,7 @@ check_static_linking_file() {
 
   # ermagherd there's two whitespace characters between 'LSB' and 'executable'
   # when running this on Travis - why is everything so terrible?
-  if file "$1" | grep -q 'ELF 64-bit LSB'; then
+  if file "$1" | grep -q 'ELF [36][24]-bit LSB'; then
     if readelf -d "$1" | grep -q 'Shared library'; then
       echo "File: $file"
       # this is done twice rather than storing in a bash variable because

script/package.sh

@@ -34,16 +34,14 @@ if ! [ -d "$DESTINATION" ]; then
 fi
 
 if [ "$TARGET_PLATFORM" == "ubuntu" ]; then
-  GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-ubuntu.tar.gz"
-  LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-ubuntu.lzma"
+  GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-ubuntu-$TARGET_ARCH.tar.gz"
+  LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-ubuntu-$TARGET_ARCH.lzma"
 elif [ "$TARGET_PLATFORM" == "macOS" ]; then
-  if [ "$TARGET_ARCH" -eq "64" ]; then ARCH="x64"; else ARCH="arm64"; fi
-  GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-macOS-$ARCH.tar.gz"
-  LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-macOS-$ARCH.lzma"
+  GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-macOS-$TARGET_ARCH.tar.gz"
+  LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-macOS-$TARGET_ARCH.lzma"
 elif [ "$TARGET_PLATFORM" == "win32" ]; then
-  if [ "$TARGET_ARCH" -eq "64" ]; then ARCH="x64"; else ARCH="x86"; fi
-  GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-windows-$ARCH.tar.gz"
-  LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-windows-$ARCH.lzma"
+  GZIP_FILE="dugite-native-$VERSION-$BUILD_HASH-windows-$TARGET_ARCH.tar.gz"
+  LZMA_FILE="dugite-native-$VERSION-$BUILD_HASH-windows-$TARGET_ARCH.lzma"
 else
   echo "Unable to package Git for platform $TARGET_PLATFORM"
   exit 1

script/update-git-lfs.ts

@@ -27,6 +27,12 @@ function getArch(fileName: string) {
   if (fileName.match(/-386-/)) {
     return 'x86'
   }
+  if (fileName.match(/-arm64-/)) {
+    return 'arm64'
+  }
+  if (fileName.match(/-arm-/)) {
+    return 'arm'
+  }
 
   throw new Error(`Unable to find arch for file: ${fileName}`)
 }
@@ -82,6 +88,9 @@ async function run(): Promise<boolean> {
 
   const files = [
     `git-lfs-linux-amd64-${version}.tar.gz`,
+    `git-lfs-linux-386-${version}.tar.gz`,
+    `git-lfs-linux-arm64-${version}.tar.gz`,
+    `git-lfs-linux-arm-${version}.tar.gz`,
     `git-lfs-windows-386-${version}.zip`,
     `git-lfs-windows-amd64-${version}.zip`,
   ]