start-docker-nix-build-slave

#!/usr/bin/env bash

# Requires: curl, docker, grep
cecho(){
    printf "\033[36m"
    echo "$@"
    printf "\033[0m"
}

working_dir_name=".nix-docker-build-slave"
working_dir="$HOME/$working_dir_name"
mkdir -p "$working_dir"

ssh_id_file="$working_dir/insecure_rsa"
remote_sys_conf="$working_dir/remote-systems.conf"

docker_machine_name="nix-docker-build-slave"

# -- Display info and troubleshooting tips --
cecho "## Use Docker Container as Nix Build Slave"
cecho "##"
cecho "## Usage: source $0"
cecho "##"
cecho "##     Either this is sourced in order to correctly set your env vars"
cecho "##     or you set it manually by running:"
echo  export NIX_REMOTE_SYSTEMS="$remote_sys_conf"
cecho "##"
cecho "## Troubleshooting: you can clean up the build slave artifacts and start over by:"
cecho "##   1. rm -r ~/$working_dir_name"
cecho "##   2. Delete entry in ~/.ssh/config for"
cecho "##        Host \"$docker_machine_name\""
cecho "##   3. Delete the docker container"
cecho "##        docker kill $docker_machine_name"
cecho "##        docker rm $docker_machine_name"
cecho

if [ "${BASH_SOURCE[0]}" = "$0" ]; then
    cecho "Please read the usage instructions, this file should be sourced"
    exit 1
fi


# -- Download SSH credentials for docker container --
if [ ! -f "$ssh_id_file" ]; then
    cecho ">>> Downloading SSH credentials for the docker container"
    rm -f "$ssh_id_file" "$ssh_id_file.pub"
    curl -fsSL https://raw.githubusercontent.com/LnL7/nix-docker/master/ssh/insecure_rsa -o "$ssh_id_file"
    curl -fsSL https://raw.githubusercontent.com/LnL7/nix-docker/master/ssh/insecure_rsa.pub -o "$ssh_id_file.pub"
    chmod 600 "$ssh_id_file"
fi

SSHCONF=$(cat <<CONF
Host "$docker_machine_name"
  User root
  HostName 127.0.0.1
  Port 3022
  IdentityFile "$ssh_id_file"
  SendEnv AWS_ACCESS_KEY_ID
  SendEnv AWS_SECRET_ACCESS_KEY
CONF
)

if ! test -f "$HOME/.ssh/config" || ! grep "$docker_machine_name" "$HOME/.ssh/config" > /dev/null; then
  cecho ">>> Adding an entry to $HOME/.ssh/config for $docker_machine_name"
  mkdir -p "$HOME/.ssh"
  echo "$SSHCONF" >> "$HOME/.ssh/config"
else
  cecho ">>> User SSH already contains entry for $docker_machine_name in $HOME/.ssh/config"
fi

if [ -S /nix/var/nix/daemon-socket/socket ]; then
  ROOT_HOME=~root
  if ! sudo test -f "$ROOT_HOME/.ssh/config" || ! sudo grep "$docker_machine_name" "$ROOT_HOME/.ssh/config" > /dev/null; then
    cecho ">>> Adding an entry to $ROOT_HOME/.ssh/config for $docker_machine_name"
    sudo mkdir -p "$ROOT_HOME/.ssh"
    echo "$SSHCONF" | sudo tee "$ROOT_HOME/.ssh/config" > /dev/null
  else
    cecho ">>> Root SSH already contains entry for $docker_machine_name in $ROOT_HOME/.ssh/config"
  fi
fi

# -- Start docker container --
if ( docker ps | grep --color -F "$docker_machine_name" ; ); then
    cecho ">>> Docker container $docker_machine_name already started"
else
    cecho ">>> Starting docker container: $docker_machine_name"
    cecho "    (This can and should fail if the container was already created.)"
    docker run \
       --name $docker_machine_name \
       -p 3022:22 \
       --restart always -d \
       nixos/nix \
       sh -c 'nix-env -iA nixpkgs.gnused &&
              mkdir -p /etc/ssh && \
              mkdir -p /root/.ssh && \
              sed -i 's#/bin/bash#/bin/sh#' /etc/passwd && \
              sed -i 's#!##' /etc/shadow && \
              echo "sshd:x:498:65534::/var/empty:/run/current-system/sw/bin/nologin" >> /etc/passwd && \
              ln -s /root/.nix-profile/bin/nix-store /usr/bin/nix-store && \
              echo "'"$(cat $ssh_id_file.pub)"'" > /root/.ssh/authorized_keys && \
              cat /root/.nix-profile/etc/ssh/sshd_config | grep -v PermitRoot > /etc/ssh/sshd_config && \
              echo "PermitRootLogin yes" >> /etc/ssh/sshd_config && \
              ssh-keygen -f /etc/ssh/ssh_host_rsa_key -N "" -t rsa && \
              mkdir -p /var/empty && \
              /nix/store/*ssh*/bin/sshd -D -e'
fi

# This does not work on macOS
# docker_machine_ip=$(docker inspect $docker_machine_name | jq -r '.[0].NetworkSettings.IPAddress')

# -- Write remote systems configuration --
cecho ">>> Writing remote systems configuration to $remote_sys_conf"
rm -f "$remote_sys_conf"
cat > "$remote_sys_conf" <<CONF
$docker_machine_name $(nix eval --extra-experimental-features nix-command --expr 'builtins.currentSystem' --impure --raw | cut -d'-' -f1)-linux $ssh_id_file 1
CONF

# -- Test connection --
sleep 1
cecho ">>> Running SSH test"
ssh "$docker_machine_name" echo "SSH connection is working." || echo "SSH connection failed."

if [ -S /nix/var/nix/daemon-socket/socket ]; then
    cecho ">>> Running SSH test as root"
    sudo ssh "$docker_machine_name" echo "root SSH connection is working." || echo "root SSH connection failed."
fi

#ssh "$docker_machine_name" mkdir -p /var/lib/nix-daemon
#scp -r ~/.aws "$docker_machine_name":/var/lib/nix-daemon

# -- Export environment --
# TODO: is this the right format?
cecho ">>> Setting \$NIX_REMOTE_SYSTEMS to use $remote_sys_conf"
export NIX_REMOTE_SYSTEMS="$remote_sys_conf"

# TODO: this will probably erase existing configured builders, does that matter?
cecho ">>> Setting \$NIX_CONFIG to use $remote_sys_conf"
export NIX_CONFIG="builders = @$remote_sys_conf"