Disk image building

See dib/README.md for rough usage.
tox 'dib' environement runs the element linter and then build the image.
Add CI related job and Zuul config, the job would only be run when a
file under /dib/ is changed.

Bug: T110733
Change-Id: Iab606637bd83d429865cfc4ee8709d0d56ea17fe

dib/README.md

@@ -0,0 +1,19 @@
+Usage on labs instance:
+
+    pip install --user -r requirements.txt
+    export PATH=$PATH:~/.local/bin
+    ./build_image.sh
+
+You can speed it up next run by reusing debootstrap cache:
+
+   DIB_DEBIAN_USE_DEBOOTSTRAP_CACHE=1 ./build_image.sh
+
+More aggressively:
+
+   DIB_OFFLINE=1 ./build_image.sh
+
+And maybe you will have a qcow2 image build!
+
+
+See upstream documentation:
+http://docs.openstack.org/developer/diskimage-builder/

dib/build_image.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# https://wikitech.wikimedia.org/wiki/Nodepool#Diskimage
+#
+# XXX add a switch to be able to set either of:
+# DIB_OFFLINE=1
+# DIB_DEBIAN_USE_DEBOOTSTRAP_CACHE=1
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+	set -x
+fi
+set -eu
+set -o pipefail
+
+ELEMENTS=(
+	# Built-in
+	'debian'
+	'debian-systemd'
+	'cloud-init-datasources'
+	'vm'
+	'devuser'
+	# Custom
+	'nodepool-base'
+	)
+export DIB_COMMAND=${DIB_COMMAND:-'disk-image-create'}
+export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRUBITION_MIRROR:-'http://mirrors.wikimedia.org/debian/'}
+export DIB_RELEASE=${DIB_RELEASE:-jessie}
+export DIB_IMAGE_CACHE=${DIB_IMAGE_CACHE:-/srv/dib/cache}  # XXX should be unset by default
+DATE=`date -u +'%Y%m%dT%H%M%SZ'`
+export DIB_IMAGE_NAME=${DIB_IMAGE_NAME:-"image-${DIB_RELEASE}-${DATE}"}
+export IMAGE_TYPE="qcow2"
+
+export DIB_CLOUD_INIT_DATASOURCES='Ec2'
+export DIB_DEV_USER_USERNAME='jenkins'
+export DIB_DEV_USER_AUTHORIZED_KEYS='dib_jenkins_id_rsa.pub'
+
+export QEMU_IMG_OPTIONS='compat=0.10'  # XXX might not be needed
+
+export NODEPOOL_SCRIPTDIR='../nodepool/scripts'
+
+
+ELEMENTS_PATH=${ELEMENTS_PATH:-}
+if [[ -z "$ELEMENTS_PATH" ]]; then
+	export ELEMENTS_PATH='elements'
+else
+	export ELEMENTS_PATH="${ELEMENTS_PATH}:elements"
+fi
+
+$DIB_COMMAND -t "$IMAGE_TYPE" --no-tmpfs -o "$DIB_IMAGE_NAME" ${ELEMENTS[@]}
+EXIT_CODE=$?
+if [[ $EXIT_CODE -gt 0 ]]; then
+	>&2 echo "Disk image creation failed (exit: $EXIT_CODE)"
+	exit $EXIT_CODE
+else
+	echo "Disk image successful ${DIB_IMAGE_NAME}.${IMAGE_TYPE}"
+fi

dib/dib_jenkins_id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDrERFfjRBOIoI5ASW0gx/rxSeJX/ThszByypsoA80jCfbcjrfsG94WtOGXYQw4Zjs/8u4DYMfI5aHEZKvk/K4jTAR09J9swFash9ML60AvQx/VFC5ZEDHMBa7dYyzxspDX5v73QEDYG9Hhxo6qfFOLO3IvYfat9CfwQR4/oS2lzV+oIsD68lSy/OoKCpywMs0/pExdP65RHR7xpvAlrgehzKoayfHo5Vzg9dCawj4ZoHsqwCnKG4ctMflyzyN/Lwgniv/+GSjgqf/FNXDCMDJCh+d410IXLS7szY3JTzpWekF82SxIM19CdwKh1R2zPVjUT6hvbm9kOo8Y72ORL2yj nodepool@labnodepool1001

dib/elements/nodepool-base/README.rst

@@ -0,0 +1,4 @@
+nodepool-base
+-------------
+
+NodePool element to prepare an image to be connected with Jenkins.

dib/elements/nodepool-base/element-deps

@@ -0,0 +1 @@
+package-installs

dib/elements/nodepool-base/extra-data.d/99-copy-nodepool-scripts

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copy the nodepools scripts into the image
+
+# This is 99 because it needs to depend on source-repositories having been
+# run, because the default fallthrough location for the nodepool scripts
+# is the git repo that's been cloned into the image.
+
+# During a nodepool run, it's expected that NODEPOOL_SCRIPTDIR is set, in
+# which case ordering doesn't matter - but for a developer hacking locally
+# who is not hacking on those scripts, it's an imposition to make them
+# set an env var.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+NODEPOOL_SCRIPTDIR=${NODEPOOL_SCRIPTDIR:-$TMP_MOUNT_PATH/opt/git/openstack-infra/project-config/nodepool/scripts}
+
+# /opt/nodepool-scripts is hardcoded in Nodepool daemon
+# Will be refreshed during the setup phase
+sudo mkdir -p $TMP_MOUNT_PATH/opt/nodepool-scripts
+sudo cp -av $NODEPOOL_SCRIPTDIR $TMP_MOUNT_PATH/opt/nodepool-scripts

dib/elements/nodepool-base/finalise.d/99-nodepool-dir

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+# Nodepool expects this dir to exist when it boots slaves.
+# Nodepool writes environment info to this dir. We set the mode
+# to 0777 so that any user can access this env info.
+mkdir /etc/nodepool
+chmod 0777 /etc/nodepool

dib/elements/nodepool-base/install.d/05-record-details

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+echo $DIB_IMAGE_NAME > /etc/image-hostname.txt

dib/elements/nodepool-base/install.d/package-installs-java

@@ -0,0 +1,2 @@
+# For the Jenkins client
+openjdk-7-jre-headless

dib/requirements.txt

@@ -0,0 +1 @@
+diskimage-builder==1.1.1

dib/tests.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# Wrapper to lint dib elements. Used by tox -edib
+
+set -eux
+
+cd `dirname $0`
+dib-lint
+
+# Commented out, would consume too much diskspace for now
+#DIB_DEBIAN_USE_DEBOOTSTRAP_CACHE=1 ./build_image.sh

jjb/integration.yaml

@@ -127,6 +127,8 @@
      - 'integration-zuul-layoutdiff'
      - 'integration-zuul-layoutvalidation'
      - 'integration-zuul-layoutvalidation-gate'
+     - '{name}-tox-{toxenv}-trusty':
+         toxenv: dib
 
 - project:
     name: 'integration-docroot'

nodepool/scripts/README

@@ -0,0 +1 @@
+Hold scripts used by Nodepool to prepare an instance

tox.ini

@@ -11,3 +11,7 @@ deps = -rtest-requirements.txt
 [testenv:flake8]
 commands = flake8
 deps = flake8
+
+[testenv:dib]
+commands = {toxinidir}/dib/tests.sh
+deps = -rdib/requirements.txt

zuul/layout.yaml

@@ -1639,6 +1639,9 @@ jobs:
     failure-message: 'Failed to compare your patchset with the current configuration. FAILURE'
     files: '^jjb/.*'
 
+  - name: 'integration-config-tox-dib-trusty'
+    files: '^dib/.*'
+
   - name: 'integration-zuul-layoutdiff'
     voting: false
     success-message: 'No layout difference. Build'
@@ -1975,12 +1978,14 @@ projects:
     test:
       - tox-flake8
       - tox-py27
+      - integration-config-tox-dib-trusty
       - integration-jjb-config-diff
       - integration-zuul-layoutdiff
       - integration-zuul-layoutvalidation
     gate-and-submit:
       - tox-flake8
       - tox-py27
+      - integration-config-tox-dib-trusty
       - integration-jjb-config-diff
       - integration-zuul-layoutdiff
       - integration-zuul-layoutvalidation-gate