cmspm

#!/usr/bin/env python3
"""CMS Package Management script. Feedback and Help: mojedasa@cern.ch

This script must be able to run in Python 2.4.

Note: All commands that checkout/export tags try to make a 'src' folder
      if it does not exist. Then they checkout inside that folder.

Syntax:
    cmspm <command> [parameters]


Commands:
    co        Checkouts/exports tags read from stdin.

    corel     Checkouts/exports a full release using the tags
              from Tag Collector (i.e. works with non-tagged releases).

    frombase  1. Runs 'scram p CMSSW <project>'.
              2. Checkouts the diff tags between <release> and <baserelease>.
              3. Runs 'checkdeps'.
              4. Checkouts the packages from the 'checkdeps' output.
              5. Optionally, removes the CVS folders.
              The packages are checked-out one by one.
              The diff tags are fetched from Tag Collector.

    query     Query the tags of a release from Tag Collector.

    tag       Tags a release in CVS (using rtag).
              The packages are tagged one by one.
              The tags are fetched from Tag Collector.


Checkout (co) Command:
  Syntax:
    cmspm co [options]

  Options:
    -e   --export           Use CVS export instead of CVS checkout
                            (export does not download CVS folders
                            and is much faster).

    -n   --dry-run          Do not do anything, just print.

    -j   --n-threads        Number of threads used (default: 10, max: 99).


Checkout Release (corel) Command:
  Syntax:
    cmspm corel <release> [options]

  Options:
    -e   --export           Use CVS export instead of CVS checkout
                            (export does not download CVS folders
                            and is much faster).

    -o   --dumptags <file>  Dump the release's tags to a file.

    -n   --dry-run          Do not do anything, just print.

    -j   --n-threads        Number of threads used (default: 10, max: 99).


Frombase (frombase) Command:
  Syntax:
    cmspm frombase <release> <baserelease> <project> [options]

  Options:
    -e   --export           Remove the CVS folders after checkdeps has run
                            (i.e. the final results will look like using
                            the -e option in the 'corel' command).

    -o   --dumptags <file>  Dump the release's tags to a file.

    -j   --n-threads        Number of threads used (default: 10, max: 99).


Query (query) Command:
  Syntax:
    cmspm query <release>


Tag (tag) Command:
  Syntax:
    cmspm tag <release> [options]

  Options:
    -n   --dry-run          Do not do anything, just print.

    -j   --n-threads        Number of threads used (default: 10, max: 99).


Examples:
    cmspm co       < tags.txt
    cmspm corel    CMSSW_4_2_9_HLT1 -e -o tags.txt
    cmspm frombase CMSSW_4_2_9_HLT1_hltpatch1 CMSSW_4_2_9_HLT1 CMSSW_4_2_9_HLT1
    cmspm query    CMSSW_4_2_9_HLT1
    cmspm tag      CMSSW_4_2_9_HLT1
"""

__author__ = "Miguel Ojeda"
__copyright__ = "Copyright 2011, CERN CMS"
__credits__ = ["Miguel Ojeda"]
__license__ = "Unknown"
__maintainer__ = "Miguel Ojeda"
__email__ = "mojedasa@cern.ch"
__status__ = "Staging"

import time
import os
import sys
import shutil
import fnmatch
import optparse
import subprocess
import threading
from queue import Queue
from subprocess import getstatusoutput
from os import getenv

try:
  from subprocess import CalledProcessError
except ImportError:
  # Python 2.4 doesn't implement CalledProcessError
  class CalledProcessError(Exception):
    """This exception is raised when a process run by check_call() returns
       a non-zero exit status. The exit status will be stored in the
       returncode attribute."""
    def __init__(self, returncode, cmd):
      self.returncode = returncode
      self.cmd = cmd
    def __str__(self):
      return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)

_maxtaglen = len('V01-01-01-01')

# Simple class similar to Pool in Python 2.6, but for threads and supporting
# only the useful map() method.
# Note: it is not a true pool as the threads are not kept between map() calls,
# i.e. they are not created in __init__().
class ThreadPool(object):
  def __init__(self, n_threads):
    self.n_threads = n_threads

  def map(self, function, arguments, common_arguments = ()):
    threads = []
    queue = Queue.Queue()
    for a in arguments:
      queue.put(a)

    for i in range(self.n_threads):
      t = threading.Thread(target = self._target, args = (i, function, queue, common_arguments, ))
      threads.append(t)
      t.start()

    for t in threads:
      t.join()

  def _target(self, tid, function, queue, common_arguments):
    while True:
      try:
        if function(tid, queue.get_nowait(), *common_arguments):
          return
      except Queue.Empty:
        return

# A recursive globbing helper (from cmsBuild).
def recursive_glob(treeroot, pattern):
  results = []
  for base, dirs, files in os.walk(treeroot):
    goodfiles = fnmatch.filter(files, pattern) + fnmatch.filter(dirs, pattern)
    results.extend(os.path.join(base, f) for f in goodfiles)
  return results

def die(msg):
  print(msg)
  sys.exit(1)

def deleteCVSFolders(treeroot):
  map(shutil.rmtree, recursive_glob(treeroot, 'CVS'))

def deleteGITFolders(treeroot):
  map(shutil.rmtree, recursive_glob(treeroot, '.git'))

def parseJSONTag(tag):
  if tag == 'null':
    return None
  return tag.strip('"')

def parseJSONTags(tags):
  tags = tags[1:-1]
  real_tags = []
  while True:
    start = tags.find('[')
    if start == -1:
      break
    end = tags.find(']')
    if end == -1:
      raise Exception('Error while parsing tags: Found the start of a tag ("[") but not the end ("]")')
    row = tags[start+1:end].split(', ')
    real_tag = [row[0].strip('"')]
    for tag in row[1:]:
      real_tag.append(parseJSONTag(tag))
    real_tags.append(real_tag)
    tags = tags[end+1:]

  return real_tags

def getTcData(url):
  tcurl = 'https://cmstags.cern.ch/tc/'+url
  err, output = getstatusoutput('which wget')
  cmd = "wget  --no-check-certificate -nv -o /dev/null -O- '"+tcurl+"'"
  if err: cmd = "curl -L -k --stderr /dev/null '"+tcurl.replace('[','\[').replace(']','\]')+"'"
  err, output =  getstatusoutput(cmd)
  if err: raise Exception("Error: running command"+cmd+"\n"+output)
  return output

def getReleasesTags(releases, diff):
  if diff:
    diff = 'true'
  else:
    diff = 'false'
  packs = getTcData ('public/py_getReleasesTags?releases=["' + '","'.join(releases) + '"]&diff=' + diff)
  return parseJSONTags(packs)
  
# Get return a dictionary with package => tag for the
# specified @a tagset
def getTagSetTags(tagsets):
  extratags = {}
  for tagset in tagsets:
    packs = getTcData('public/py_getTagsetTags?tagset=' + tagset)
    extratags.update(dict(parseJSONTags(packs)))
  return extratags

def formatTag(tag, maxlen = _maxtaglen):
  return tag + ' ' * (maxlen - len(tag))

class TagsDumper:
  def __init__(self, filename):
    try:
      self._dumptags_fd = None
      if filename:
        self._dumptags_fd = open(filename, 'w')
    except OSError:
      die("Cannot open file %s for writing" % opts.dumptags)
  
  def dump(self, tags):
    if self._dumptags_fd:
      for tag in tags:
        self._dumptags_fd.write(tag[0] + " " + tag[1] + "\n")
      self._dumptags_fd.close()

def popen(args, merge_stderr = True, shell = False):
  stderr = None
  if merge_stderr:
    stderr = subprocess.STDOUT
  print(args)
  p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=stderr, close_fds=True, shell=shell)
  r = p.communicate()[0]
  print (r.decode("utf-8", errors="replace"))
  if p.returncode != 0:
    if isinstance(args, str):
      str_args = args
    else:
      str_args = ' '.join(args)
    raise CalledProcessError(p.returncode, str_args)
  return r

def queryCVS(args, cvsroot, tries=5):
  if tries<1: tries=1
  while tries>0:
    tries -= 1
    try:
      return popen(['cvs', '-Q', '-d', cvsroot] + args)
    except Exception as ex:
      if tries==0: raise ex
    time.sleep(5)

def exportCVS(tag, args, cvsroot):
  return queryCVS(['export', '-r', tag] + args, cvsroot)

def checkoutCVS(tag, args, cvsroot):
  return queryCVS(['co', '-r', tag] + args, cvsroot)

def rtagCVS(tag, args, cvsroot):
  return queryCVS(['rtag', '-r', tag] + args, cvsroot)

def mergeTags(tags):
  new_tags = {}
  for (package, tag) in tags:
    if tag not in new_tags:
      new_tags[tag] = []
    new_tags[tag].append(package)
  return new_tags

def checkoutTag(tid, tag, lock, export, dryrun, cvsroot, errors):
  (tag, packages) = tag
  action = 'Checking out'
  if export:
    action = 'Exporting'
  try:
    lock.acquire()
    if len(errors) != 0:
      print ("[Thread %2i] Stopping because something else failed." % tid)
      return True
    print ("[Thread %2i]" % tid, action + ":", formatTag(tag), ' '.join(packages))
  finally:
    lock.release()

  if dryrun:
    return

  try:
    if export:
      exportCVS(tag, packages, cvsroot)
    else:
      checkoutCVS(tag, packages, cvsroot)
  except:        
    try:
      lock.acquire()
      print ("[Thread %2i]" % tid, action, "failed:", formatTag(tag), ' '.join(packages))
      errors.append(True)
      return True
    finally:
      lock.release()

def checkoutSubsystem(tid, tags, lock, export, dryrun, cvsroot, errors):
  tags = mergeTags(tags)
  failed = []
  for tag in tags:
    if checkoutTag(tid, (tag, tags[tag]), lock, export, dryrun, cvsroot, errors):
      return True

def checkNThreads(nthreads):
  if nthreads < 1:
    raise Exception("Error: The number of threads should be at least 1.")
  if nthreads > 99:
    raise Exception("Error: The number of threads should not be more than 99, in order to prevent loading the CVS server too much.")

# This function checkouts a list of tags using the best 'algorithm' to checkout them:
#   When checkout out, merge tags per subsystem and checkout the subsystems in parallel.
#   When exporting, merge all tags and export them in parallel.
def checkoutTags(tags, nthreads, export, dryrun, cvsroot):
  checkNThreads(nthreads)
  lock = threading.Lock()
  errors = []
  pool = ThreadPool(nthreads)

  if export:
    tags = mergeTags(tags)
    pool.map(checkoutTag, tags.items(), (lock, export, dryrun, cvsroot, errors))
  else:
    # When checking out, we can't run cvs in parallel for the same subsystem.
    # Otherwise, cvs would randomly access the CVS/* files.
    subsystems = {}
    for tag in tags:
      subsystem = tag[0].split('/')[0]
      if subsystem not in subsystems:
        subsystems[subsystem] = []
      subsystems[subsystem].append(tag)
    pool.map(checkoutSubsystem, subsystems.values(), (lock, export, dryrun, cvsroot, errors))

  if len(errors) != 0:
    die('Error: A checkout/export failed.')

def rtag(tid, tag, lock, release, dryrun, errors, cvsroot):
  (tag, packages) = tag

  try:
    lock.acquire()
    if len(errors) != 0:
      print ("[Thread %2i] Stopping because something else failed." % tid)
      return True
    print ("[Thread %2i]" % tid, "Tagging:", formatTag(tag), release, ' '.join(packages))
  finally:
    lock.release()

  if dryrun:
    return

  try:
    rtagCVS(tag, [release] + packages, cvsroot)
  except:
    try:
      lock.acquire()
      print ("[Thread %2i]" % tid, "Tagging failed:", formatTag(tag), release, ' '.join(packages))
      errors.append(True)
      return True
    finally:
      lock.release()

def cmd_co(args):
  parser = optparse.OptionParser(usage="cmspm co [options]")
  parser.add_option("-e", "--export", action="store_true", dest="export", default=False)
  parser.add_option("-n", "--dry-run", action="store_true", dest="dryrun", default=False)
  parser.add_option("-j", "--n-threads", action="store", type="int", dest="nthreads", default=10)
  parser.add_option("-d", "--cvsroot", dest="cvsroot", default=getenv("CVSROOT"))
  (opts, args) = parser.parse_args(args)

  if len(args) != 0:
    raise Exception("Error: No arguments expected.")
  
  folder = 'src'
  if not opts.dryrun:
    try:
      os.mkdir(folder)
    except OSError as e:
      pass
    os.chdir(folder)

  tags = {}
  for line in sys.stdin.readlines():
    line = line.split()
    if len(line) == 0:
      continue
    if len(line) != 2:
      raise Exception("Error: The input must be one (package, tag) pair per line.")
    (package, tag) = line
    if package in tags and tag != tags[package]:
      raise Exception("Error: The input contains a duplicate package with a different tag.")
    tags[package] = tag

  removedTags = [k.strip("/").strip(".").strip("/") for (k,v) in tags.items() if v == "-" and k]
  for tag in removedTags:
    error, out = getstatusoutput("rm -rf %s" % tag)
    if error:
      print (out)
      exit(1)
  addedTag = [(k,v) for (k,v) in tags.items() if v != "-"]
  checkoutTags(addedTag, opts.nthreads, opts.export, opts.dryrun, opts.cvsroot)

def cmd_corel(args):
  parser = optparse.OptionParser(usage="cmspm corel <release> [options]")
  parser.add_option("-e", "--export", action="store_true", dest="export", default=False)
  parser.add_option("-o", "--dump-tags", action="store", type="string", dest="dumptags", default=None)
  parser.add_option("-n", "--dry-run", action="store_true", dest="dryrun", default=False)
  parser.add_option("-j", "--n-threads", action="store", type="int", dest="nthreads", default=10)
  parser.add_option("-t", "--additional-tagsets", metavar="TAGSET1[+TAGSET2[+...]]", action="store", type="string", dest="additionalTagSetsSpec", default="")
  parser.add_option("-d", "--cvsroot", dest="cvsroot", default=getenv("CVSROOT"))
  (opts, args) = parser.parse_args(args)

  if len(args) != 1:
    raise Exception("Error: Expected <release>.")
  
  release = args[0]

  # Create destination directory, if needed.
  folder = 'src'
  if not opts.dryrun:
    try:
      os.mkdir(folder)
    except OSError as e:
      pass

  # Open the file before doing anything else to catch errors earlier
  dumper = TagsDumper(opts.dumptags)
  tags = getReleasesTags([release], False)

  # If additional tagsets are specified, we replace them int the final list of tags                                                                                                                                                                                                       
  # and in the tags to be checked out.
  additionalTagSets = [x for x in opts.additionalTagSetsSpec.split("+") if x]
  extraTagsetTags = getTagSetTags(additionalTagSets)
  tags = dict(tags)
  tags.update(extraTagsetTags)
  tags = tags.items()
  tags.sort()

  # Dump the tags first so that we can tell the difference.
  dumper.dump(tags)

  if opts.dryrun:
    sys.exit(0)

  os.chdir(folder)
  checkoutTags(tags, opts.nthreads, opts.export, opts.dryrun, opts.cvsroot)

def getSCRAMVariable(variable):
  err, out = getstatusoutput("scram build -f echo_%s | grep %s | sed 's|^%s\s*=\s*||'" % (variable, variable, variable))
  return out

def checkBigProducts(release):
  arch = getenv("SCRAM_ARCH")
  err, releaseTop = getstatusoutput("cat .SCRAM/%s/Environment | sed 's|^RELEASETOP=||'" % arch)
  releaseTop = releaseTop.rstrip()
  if not releaseTop:
    raise Exception("Error: Unable to find the base release path.")

  if not os.path.exists(releaseTop):
    raise Exception("Error: Release top does not exist: %s" % releaseTop)

  if not os.path.exists(os.path.join(releaseTop, ".SCRAM" , arch, "subpackage-debug")):
    return

  print ("Checking for Big products packages...")
  getstatusoutput("scram build -r echo_CXX")
  doReadTree=False
  for bigprod in getSCRAMVariable("ALL_BIGPRODS").split(" "):
    if not bigprod: continue
    print ("Checking big product "+bigprod)
    checkout_all=False
    pkgs = []
    for pkg in getSCRAMVariable(bigprod+"_PACKAGES").split(" "):
      if (not pkg) or (not "/" in pkg): continue
      if os.path.exists (os.path.join("src",pkg)):
        checkout_all=True
        print ("  Rebuild all packages of big product %s due to %s" % (bigprod, pkg))
      else:
        pkgs.append(pkg)

    if checkout_all:
      if not pkgs:
        print ("  All packages for %s are already checked out" % bigprod)
        continue
      doReadTree=True
      f=open ("src/.git/info/sparse-checkout", "a")
      for pkg in pkgs:
        print (pkg+"\n")
        f.write(pkg+"\n")
      f.close()
  if doReadTree:
    popen('eval `scram run -sh` && cd src && git read-tree -mu %s' % (release), False, True)
  print ("Checking for Big products packages... DONE")

def cmd_frombase(args):
  parser = optparse.OptionParser(usage="cmspm frombase <base-release> <release> <project>")
  parser.add_option("-e", "--export", action="store_true", dest="export", default=False)
  parser.add_option("-o", "--dump-tags", action="store", type="string", dest="dumptags", default=None)
  parser.add_option("-j", "--n-threads", action="store", type="int", dest="nthreads", default=10)
  parser.add_option("-n", "--dry-run", action="store_true", dest="dryrun", default=False)
  parser.add_option("-t", "--additional-tagsets", metavar="TAGSET1[+TAGSET2[+...]]", action="store", type="string", dest="additionalTagSetsSpec", default="")
  parser.add_option("-d", "--cvsroot", dest="cvsroot", default=getenv("CVSROOT"))
  (opts, args) = parser.parse_args(args)

  if len(args) != 3:
    raise Exception("Error: Expected <release> <baserelease> <project>.")
  
  release = args[0]
  baserelease = args[1]
  project = args[2]

  if opts.dryrun:
    sys.exit(0)
    
  print ("Running scram project...")
  popen(['scram', '-a', getenv("SCRAM_ARCH"), 'p', 'CMSSW', project])
  os.chdir(project)
  print ("Running scram project... DONE")

  e, o = getstatusoutput("which git-cms-addpkg")
  if e:
    print (o)
    sys.exit(1)
  if not os.path.exists(os.path.join(os.path.dirname(o),"git-cms-checkout-topic")):
    print ("Running cms-sparse-checkout...")
    popen('eval `scram run -sh` && git cms-addpkg --ssh -y FWCore/Version && cd src && git checkout %s && git cms-sparse-checkout --exclude-cvs-keywords %s %s && git read-tree -mu %s  && git cms-checkdeps -A -a' % (release, baserelease, release, release), False, True)
    print ("Running cms-sparse-checkout... DONE")
  else:
    print ("Running cms-checkout-topic...")
    popen('eval `scram run -sh` && git cms-checkout-topic --ssh %s && git cms-addpkg -y FWCore/Version && git cms-checkdeps -A -a' % (release), False, True)
    print ("Running cms-checkout-topic... DONE")

  checkBigProducts(release)

  if opts.export:
    deleteGITFolders('src')

  os.chdir('..')
  shutil.move(os.path.join(project, 'src'), 'src')
  if os.path.exists (os.path.join(project, 'poison')):
    shutil.move(os.path.join(project, 'poison'), 'poison')
  shutil.rmtree(project)

def cmd_query(args):
  parser = optparse.OptionParser(usage="cmspm query <release>")
  (opts, args) = parser.parse_args(args)

  if len(args) != 1:
    raise Exception("Error: Expected <release>.")

  release = args[0]

  tags = getReleasesTags([release], False)

  for tag in tags:
    print (tag[0]+" "+tag[1])

def cmd_tag(args):
  parser = optparse.OptionParser(usage="cmspm tag <release> [options]")
  parser.add_option("-n", "--dry-run", action="store_true", dest="dryrun", default=False)
  parser.add_option("-j", "--n-threads", action="store", type="int", dest="nthreads", default=10)
  parser.add_option("-d", "--cvsroot", dest="cvsroot", default=getenv("CVSROOT"))
  (opts, args) = parser.parse_args(args)

  if len(args) != 1:
    raise Exception("Error: Expected <release>.")

  release = args[0]

  checkNThreads(opts.nthreads)

  print ("Getting tags from TagCollector...",)
  tags = getReleasesTags([release], False)
  print ("DONE")

  lock = threading.Lock()
  errors = []
  pool = ThreadPool(opts.nthreads)
  tags = mergeTags(tags)
  pool.map(rtag, tags.items(), (lock, release, opts.dryrun, errors, opts.cvsroot))

  if len(errors) != 0:
    raise Exception('Error: A tagging failed.')

def main():
  try:
    args = sys.argv[1:]
    cmd = globals()['cmd_' + args[0]]
  except:
    print (__doc__)
    print ("Exception: Expected <command>.")
    sys.exit(-1)
  cmd(args[1:])

if __name__ == '__main__':
  main()