Use a wrapper script that sanitises the environment for OpenSSL-sensi…

…tive executables (#750)

alibuild_helpers/build.py

@@ -4,7 +4,7 @@
 from alibuild_helpers.analytics import report_event
 from alibuild_helpers.log import debug, error, info, banner, warning
 from alibuild_helpers.log import dieOnError
-from alibuild_helpers.cmd import execute, getstatusoutput, DockerRunner, BASH
+from alibuild_helpers.cmd import execute, getstatusoutput, DockerRunner, BASH, install_wrapper_script
 from alibuild_helpers.utilities import star, prunePaths
 from alibuild_helpers.utilities import resolve_store_path
 from alibuild_helpers.utilities import format, parseDefaults, readDefaults
@@ -13,7 +13,7 @@
 from alibuild_helpers.utilities import Hasher
 from alibuild_helpers.utilities import yamlDump
 from alibuild_helpers.utilities import resolve_tag, resolve_version
-from alibuild_helpers.git import partialCloneFilter
+from alibuild_helpers.git import git, partialCloneFilter
 from alibuild_helpers.sync import (NoRemoteSync, HttpRemoteSync, S3RemoteSync,
                                    Boto3RemoteSync, RsyncRemoteSync)
 import yaml
@@ -37,27 +37,19 @@
 import sys
 import time
 
+
 def writeAll(fn, txt):
   f = open(fn, "w")
   f.write(txt)
   f.close()
 
+
 def readHashFile(fn):
   try:
     return open(fn).read().strip("\n")
   except IOError:
     return "0"
 
-def getDirectoryHash(d):
-  # We can't use git --git-dir=%s/.git or git -C %s here as the former requires
-  # that the directory we're inspecting to be the root of a git directory, not
-  # just contained in one (and that breaks CI tests), and the latter isn't
-  # supported by the git version we have on slc6.
-  # Silence cd as shell configuration can cause the new directory to be echoed.
-  err, out = getstatusoutput("cd %s >/dev/null 2>&1 && "
-                             "git rev-parse HEAD" % quote(d))
-  dieOnError(err, "Impossible to find reference for %s" % d)
-  return out
 
 # Creates a directory in the store which contains symlinks to the package
 # and its direct / indirect dependencies
@@ -256,7 +248,7 @@ def doBuild(args, parser):
           star(), args.configDir, star())
     return 1
 
-  err, value = getstatusoutput("GIT_DIR=%s/.git git symbolic-ref -q HEAD" % args.configDir)
+  _, value = git(("symbolic-ref", "-q", "HEAD"), directory=args.configDir, check=False)
   branch_basename = re.sub("refs/heads/", "", value)
   branch_stream = re.sub("-patches$", "", branch_basename)
   # In case the basename and the stream are the same,
@@ -273,13 +265,15 @@ def doBuild(args, parser):
   if not exists(specDir):
     makedirs(specDir)
 
-  os.environ["ALIBUILD_ALIDIST_HASH"] = getDirectoryHash(args.configDir)
+  os.environ["ALIBUILD_ALIDIST_HASH"] = git(("rev-parse", "HEAD"), directory=args.configDir)
 
   debug("Building for architecture %s", args.architecture)
   debug("Number of parallel builds: %d", args.jobs)
   debug("Using %sBuild from %sbuild@%s recipes in %sdist@%s",
         star(), star(), __version__, star(), os.environ["ALIBUILD_ALIDIST_HASH"])
 
+  install_wrapper_script("git", workDir)
+
   with DockerRunner(dockerImage, ["--network=host"]) as getstatusoutput_docker:
     my_gzip = "pigz" if getstatusoutput_docker("which pigz")[0] == 0 else "gzip"
     my_tar = ("tar --ignore-failed-read"
@@ -387,15 +381,12 @@ def downloadTask(p):
       updateReferenceRepoSpec(args.referenceSources, p, specs[p], args.fetchRepos, not args.docker)
 
       # Retrieve git heads
-      cmd = ("git", "ls-remote", "--heads", "--tags",
+      cmd = ("ls-remote", "--heads", "--tags",
              specs[p].get("reference", specs[p]["source"]))
       if specs[p]["package"] in develPkgs:
          specs[p]["source"] = join(os.getcwd(), specs[p]["package"])
-         cmd = "git", "ls-remote", "--heads", "--tags", specs[p]["source"]
-      debug("Executing %s", " ".join(cmd))
-      err, output = getstatusoutput(cmd)
-      if err:
-        raise RuntimeError("Error on %r: %s" % (" ".join(cmd), output))
+         cmd = "ls-remote", "--heads", "--tags", specs[p]["source"]
+      output = git(cmd)
       specs[p]["git_refs"] = {git_ref: git_hash for git_hash, sep, git_ref in
                               (line.partition("\t") for line in output.splitlines())
                               if sep}
@@ -433,25 +424,17 @@ def downloadTask(p):
       # different or if there are extra changes on top.
       if spec["package"] in develPkgs:
         # Devel package: we get the commit hash from the checked source, not from remote.
-        cmd = "cd %s && git rev-parse HEAD" % spec["source"]
-        err, out = getstatusoutput(cmd)
-        dieOnError(err, "Unable to detect current commit hash.")
+        out = git(("rev-parse", "HEAD"), directory=spec["source"])
         spec["commit_hash"] = out.strip()
         cmd = "cd %s && git diff -r HEAD && git status --porcelain" % spec["source"]
         h = Hasher()
         err = execute(cmd, lambda s, *a: h(s % a))
         debug("Command %s returned %d", cmd, err)
         dieOnError(err, "Unable to detect source code changes.")
         spec["devel_hash"] = spec["commit_hash"] + h.hexdigest()
-        cmd = "cd %s && git rev-parse --abbrev-ref HEAD" % spec["source"]
-        err, out = getstatusoutput(cmd)
+        out = git(("rev-parse", "--abbrev-ref", "HEAD"), directory=spec["source"])
         if out == "HEAD":
-          err, out = getstatusoutput("cd %s && git rev-parse HEAD" % spec["source"])
-          out = out[0:10]
-        if err:
-          error("Error, unable to lookup changes in development package %s. Is it a git clone?",
-                spec["source"])
-          return 1
+          out = git(("rev-parse", "HEAD"), directory=spec["source"])[:10]
         develPackageBranch = out.replace("/", "-")
         spec["tag"] = args.develPrefix if "develPrefix" in args else develPackageBranch
         spec["commit_hash"] = "0"

alibuild_helpers/build_template.sh

@@ -15,6 +15,10 @@ export WORK_DIR="${WORK_DIR_OVERRIDE:-%(workDir)s}"
 # From our dependencies
 %(dependencies)s
 
+# Insert our own wrapper scripts into $PATH, patched to use the system OpenSSL,
+# instead of the one we build ourselves.
+export PATH=$WORK_DIR/wrapper-scripts:$PATH
+
 # The following environment variables are setup by
 # the aliBuild script itself
 #

alibuild_helpers/cmd.py

@@ -1,5 +1,8 @@
+import os
+import os.path
 import sys
 from subprocess import Popen, PIPE, STDOUT
+from textwrap import dedent
 try:
   from shlex import quote  # Python 3.3+
 except ImportError:
@@ -94,3 +97,25 @@ def __exit__(self, exc_type, exc_value, traceback):
       getstatusoutput("docker container kill " + quote(self._container))
     self._container = None
     return False   # propagate any exception that may have occurred
+
+
+def install_wrapper_script(name, work_dir):
+  script_dir = os.path.join(work_dir, "wrapper-scripts")
+  try:
+    os.makedirs(script_dir)
+  except OSError as exc:
+    # Errno 17 means the directory already exists.
+    if exc.errno != 17:
+      raise
+  # Create a wrapper script that cleans up the environment, so we don't see the
+  # OpenSSL built by aliBuild.
+  with open(os.path.join(script_dir, name), "w") as scriptf:
+    # Compute the "real" executable path each time, as the wrapper script might
+    # be called on the host or in a container.
+    scriptf.write(dedent("""\
+    #!/bin/sh
+    exec env -u LD_LIBRARY_PATH -u DYLD_LIBRARY_PATH \\
+         "$(which -a "$(basename "$0")" | grep -Fxv "$0" | head -1)" "$@"
+    """))
+    os.fchmod(scriptf.fileno(), 0o755)  # make the wrapper script executable
+  os.environ["PATH"] = script_dir + ":" + os.environ["PATH"]

alibuild_helpers/git.py

@@ -1,4 +1,9 @@
+try:
+  from shlex import quote  # Python 3.3+
+except ImportError:
+  from pipes import quote  # Python 2.7
 from alibuild_helpers.cmd import getstatusoutput
+from alibuild_helpers.log import debug
 
 
 def __partialCloneFilter():
@@ -7,3 +12,21 @@ def __partialCloneFilter():
 
 
 partialCloneFilter = __partialCloneFilter()
+
+
+def git(args, directory=".", check=True):
+  debug("Executing git %s (in directory %s)", " ".join(args), directory)
+  # We can't use git --git-dir=%s/.git or git -C %s here as the former requires
+  # that the directory we're inspecting to be the root of a git directory, not
+  # just contained in one (and that breaks CI tests), and the latter isn't
+  # supported by the git version we have on slc6.
+  # Silence cd as shell configuration can cause the new directory to be echoed.
+  err, output = getstatusoutput("""\
+  set -e +x
+  cd {directory} >/dev/null 2>&1
+  exec git {args}
+  """.format(directory=quote(directory),
+             args=" ".join(map(quote, args))))
+  if check and err != 0:
+    raise RuntimeError("Error {} from git {}: {}".format(err, " ".join(args), output))
+  return output if check else (err, output)

alibuild_helpers/init.py

@@ -1,5 +1,6 @@
 from alibuild_helpers.cmd import execute
-from alibuild_helpers.utilities import format, parseRecipe, getPackageList, getRecipeReader, parseDefaults, readDefaults, validateDefaults
+from alibuild_helpers.git import git
+from alibuild_helpers.utilities import getPackageList, parseDefaults, readDefaults, validateDefaults
 from alibuild_helpers.log import debug, error, warning, banner, info
 from alibuild_helpers.log import dieOnError
 from alibuild_helpers.workarea import updateReferenceRepoSpec
@@ -37,13 +38,12 @@ def doInit(args):
   if path.exists(args.configDir):
     warning("using existing recipes from %s", args.configDir)
   else:
-    cmd = format("git clone --origin upstream %(repo)s%(branch)s %(cd)s",
-                 repo=args.dist["repo"] if ":" in args.dist["repo"] else "https://github.com/%s" % args.dist["repo"],
-                 branch=" -b "+args.dist["ver"] if args.dist["ver"] else "",
-                 cd=args.configDir)
-    debug("%s", cmd)
-    err = execute(cmd)
-    dieOnError(err!=0, "cannot clone recipes")
+    cmd = ["clone", "--origin", "upstream",
+           args.dist["repo"] if ":" in args.dist["repo"] else "https://github.com/" + args.dist["repo"]]
+    if args.dist["ver"]:
+      cmd.extend(["-b", args.dist["ver"]])
+    cmd.append(args.configDir)
+    git(cmd)
 
   # Use standard functions supporting overrides and taps. Ignore all disables
   # and system packages as they are irrelevant in this context
@@ -82,17 +82,14 @@ def doInit(args):
     debug("cloning %s%s for development", spec["package"], " version "+p["ver"] if p["ver"] else "")
 
     updateReferenceRepoSpec(args.referenceSources, spec["package"], spec, True, False)
-    cmd = format("git clone --origin upstream %(readRepo)s%(branch)s --reference %(refSource)s %(cd)s && " +
-                 "cd %(cd)s && git remote set-url --push upstream %(writeRepo)s",
-                 readRepo=spec["source"],
-                 writeRepo=writeRepo,
-                 branch=" -b "+p["ver"] if p["ver"] else "",
-                 refSource=join(args.referenceSources, spec["package"].lower()),
-                 cd=dest)
-    debug("%s", cmd)
-    err = execute(cmd)
-    dieOnError(err!=0, "cannot clone %s%s" %
-                       (spec["package"], " version "+p["ver"] if p["ver"] else ""))
+
+    cmd = ["clone", "--origin", "upstream", spec["source"],
+           "--reference", join(args.referenceSources, spec["package"].lower())]
+    if p["ver"]:
+      cmd.extend(["-b", p["ver"]])
+    cmd.append(dest)
+    git(cmd)
+    git(("remote", "set-url", "--push", "upstream", writeRepo), directory=dest)
 
     # Make it point relatively to the mirrors for relocation: as per Git specifics, the path has to
     # be relative to the repository's `.git` directory. Don't do it if no common path is found

alibuild_helpers/utilities.py

@@ -13,7 +13,8 @@
 except ImportError:
   from ordereddict import OrderedDict
 
-from alibuild_helpers.cmd import getoutput, getstatusoutput
+from alibuild_helpers.cmd import getoutput
+from alibuild_helpers.git import git
 from alibuild_helpers.log import dieOnError
 
 
@@ -279,9 +280,9 @@ def __init__(self, url, configDir):
     self.url, self.configDir = url, configDir
   def __call__(self):
     m = re.search(r'^dist:(.*)@([^@]+)$', self.url)
-    fn,gh = m.groups()
-    err,d = getstatusoutput(format("GIT_DIR=%(dist)s/.git git show %(gh)s:%(fn)s.sh",
-                                   dist=self.configDir, gh=gh, fn=fn.lower()))
+    fn, gh = m.groups()
+    err, d = git(("show", "{gh}:{fn}.sh".format(gh=gh, fn=fn.lower())),
+                 directory=self.configDir)
     if err:
       raise RuntimeError(format("Cannot read recipe %(fn)s from reference %(gh)s.\n" +
                                 "Make sure you run first (this will not alter your recipes):\n" +

alibuild_helpers/workarea.py

@@ -1,17 +1,16 @@
-from alibuild_helpers.log import dieOnError, debug
-from alibuild_helpers.cmd import execute
-from alibuild_helpers.git import partialCloneFilter
-from os.path import dirname, abspath
-from alibuild_helpers.utilities import format
-
+import codecs
 import os
-import os.path as path
+import os.path
 import tempfile
 try:
   from collections import OrderedDict
 except ImportError:
   from ordereddict import OrderedDict
 
+from alibuild_helpers.log import dieOnError, debug
+from alibuild_helpers.git import git, partialCloneFilter
+
+
 def updateReferenceRepoSpec(referenceSources, p, spec, fetch, usePartialClone=True):
   """
   Update source reference area whenever possible, and set the spec's "reference"
@@ -43,47 +42,44 @@ def updateReferenceRepo(referenceSources, p, spec, fetch=True, usePartialClone=T
   @spec             : the spec of the package to be updated (an OrderedDict)
   @fetch            : whether to fetch updates: if False, only clone if not found
   """
-  assert(type(spec) == OrderedDict)
-  if not "source" in spec:
+  assert isinstance(spec, OrderedDict)
+  if "source" not in spec:
     return
 
   debug("Updating references.")
-  referenceRepo = os.path.join(abspath(referenceSources), p.lower())
+  referenceRepo = os.path.join(os.path.abspath(referenceSources), p.lower())
 
   try:
-    os.makedirs(abspath(referenceSources))
+    os.makedirs(os.path.abspath(referenceSources))
   except:
     pass
 
   if not is_writeable(referenceSources):
-    if path.exists(referenceRepo):
+    if os.path.exists(referenceRepo):
       debug("Using %s as reference for %s", referenceRepo, p)
       return referenceRepo  # reference is read-only
     else:
       debug("Cannot create reference for %s in %s", p, referenceSources)
       return None  # no reference can be found and created (not fatal)
 
-  err = False
-  logPath = os.path.join(dirname(referenceRepo), "fetch-log.txt")
-  if not path.exists(referenceRepo):
-    cmd = ["git", "clone"] + (usePartialClone and [partialCloneFilter] or []) + ["--bare", spec["source"], referenceRepo]
-    cmd = [x for x in cmd if x]
-    debug("Cloning reference repository: %s", " ".join(cmd))
-    err = execute(cmd)
+  if not os.path.exists(referenceRepo):
+    cmd = ["clone", "--bare", spec["source"], referenceRepo]
+    if usePartialClone:
+      cmd.append(partialCloneFilter)
+    git(cmd)
   elif fetch:
-    cmd = format("cd %(referenceRepo)s && "
-                 "git fetch -f --tags %(source)s >%(logPath)s 2>&1 && "
-                 "git fetch -f %(source)s '+refs/heads/*:refs/heads/*' >>%(logPath)s 2>&1",
-                 referenceRepo=referenceRepo,
-                 source=spec["source"],
-                 logPath=logPath)
-    debug("Updating reference repository: %s", cmd)
-    err = execute(cmd)
-  if os.path.exists(logPath):
-    execute("cat " + logPath)
-  dieOnError(err, "Error while updating reference repos %s." % spec["source"])
+    with codecs.open(os.path.join(os.path.dirname(referenceRepo),
+                                  "fetch-log.txt"),
+                     "w", encoding="utf-8", errors="replace") as logf:
+      err, output = git(("fetch", "-f", "--tags", spec["source"],
+                         "+refs/heads/*:refs/heads/*"),
+                        directory=referenceRepo, check=False)
+      logf.write(output)
+      debug(output)
+      dieOnError(err, "Error while updating reference repo for %s." % spec["source"])
   return referenceRepo  # reference is read-write
 
+
 def is_writeable(dirpath):
   try:
     with tempfile.NamedTemporaryFile(dir=dirpath):