annotated_steps.py

#!/usr/bin/python

# Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.

import imp
import os
import re
import shutil
import shlex
import subprocess
import sys
import tempfile
import zipfile

import config_parser

# We expect the tools directory from the dart repo to be checked out into:
# ../../tools
DART_DIR = os.path.abspath(
    os.path.normpath(os.path.join(__file__, '..', '..', '..')))
UTILS_PATH = os.path.join(DART_DIR, 'tools', 'utils.py')
BOT_UTILS_PATH = os.path.join(DART_DIR, 'tools', 'bots', 'bot_utils.py')
if os.path.isfile(UTILS_PATH):
  utils = imp.load_source('utils', UTILS_PATH)
else:
  print 'error: %s not found' % UTILS_PATH
  exit(1)
if os.path.isfile(BOT_UTILS_PATH):
  bot_utils = imp.load_source('bot_utils', BOT_UTILS_PATH)
else:
  print 'error: %s not found' % BOT_UTILS_PATH
  exit(1)


# We are deliberately not using bot utils from the dart repo.
PACKAGES_BUILDER = r'packages-(windows|linux|mac)(-repo)?(-sample)?-(.*)'

NAME_OVERRIDES = {
  'dart-protobuf' : 'protobuf',
  'polymer-dart' : 'polymer',
  'serialization.dart' : 'serialization',
  'unittest-stable' : 'unittest'
}

# Some packages need all tests run sequentially, due to side effects.
# This list only affects tests run with 'pub run test'.
SERIALIZED_PACKAGES = [
  'analyzer_cli'
]

class BotInfo(object):
  """
  Stores the info extracted from the bot name
  - system: windows, linux, mac
  - package-name
  """
  def __init__(self, system, package_name, is_repo, is_sample):
      self.system = system
      self.package_name = NAME_OVERRIDES.get(package_name,
                                             package_name.replace('-', '_'))
      self.is_repo = is_repo
      self.is_sample = is_sample

  def __str__(self):
    return "System: %s, Package-name: %s, Repo: %s, Sample: %s" % (
      self.system, self.package_name, self.is_repo, self.is_sample)

def GetBotInfo():
  name = os.environ.get('BUILDBOT_BUILDERNAME')
  if not name:
    print ("BUILDBOT_BUILDERNAME not defined. "
        "Expected pattern of the form: %s" % PACKAGES_BUILDER)
    exit(1)
  builder_pattern = re.match(PACKAGES_BUILDER, name)
  if builder_pattern:
    is_repo = builder_pattern.group(2) is not None
    is_sample = builder_pattern.group(3) is not None
    return BotInfo(builder_pattern.group(1),
                   builder_pattern.group(4),
                   is_repo,
                   is_sample)

class BuildStep(object):
  """
  A context manager for handling build steps.

  When the context manager is entered, it prints the "@@@BUILD_STEP __@@@"
  message. If it exits from an error being raised it displays the
  "@@@STEP_FAILURE@@@" message.

  If swallow_error is True, then this will catch and discard any OSError that
  is thrown. This lets you run later BuildSteps if the current one fails.
  """
  def __init__(self, name, swallow_error=False):
    self.name = name
    self.swallow_error = swallow_error

  def __enter__(self):
    print '@@@BUILD_STEP %s@@@' % self.name
    sys.stdout.flush()

  def __exit__(self, type, value, traceback):
    if value:
      print '@@@STEP_FAILURE@@@'
      sys.stdout.flush()
      if self.swallow_error and isinstance(value, OSError):
        return True

class TempDir(object):
  def __init__(self, prefix=''):
    self._temp_dir = None
    self._prefix = prefix

  def __enter__(self):
    self._temp_dir = tempfile.mkdtemp(self._prefix)
    return self._temp_dir

  def __exit__(self, *_):
    shutil.rmtree(self._temp_dir, ignore_errors=True)

class ChangedWorkingDirectory(object):
  def __init__(self, working_directory):
    self._working_directory = working_directory

  def __enter__(self):
    self._old_cwd = os.getcwd()
    print "Enter directory = ", self._working_directory
    os.chdir(self._working_directory)

  def __exit__(self, *_):
    print "Enter directory = ", self._old_cwd
    os.chdir(self._old_cwd)

def RunProcess(command, shell=False, extra_env=None):
  """
  Runs command.

  If a non-zero exit code is returned, raises an OSError with errno as the exit
  code.
  """
  env = dict(os.environ)
  env['TERM'] = 'nocolor'
  if 'GIT_USER_AGENT' in env:
    del env['GIT_USER_AGENT']
  if extra_env:
    env.update(extra_env)
  print "Running: %s" % ' '.join(command)
  print "env: %s" % str(env)
  sys.stdout.flush()
  exit_code = subprocess.call(command, env=env, shell=shell)
  if exit_code != 0:
    raise OSError(exit_code)

def GetSDK(bot_info):
  with BuildStep('Get sdk'):
    namer = bot_utils.GCSNamer(channel=bot_utils.Channel.DEV)
    # TODO(ricow): Be smarter here, only download if new.
    build_root = GetBuildRoot(bot_info)
    SafeDelete(os.path.join(build_root, 'dart-sdk'), bot_info)
    if not os.path.exists(build_root):
      os.makedirs(build_root)
    local_zip = os.path.join(build_root, 'sdk.zip')
    gsutils = bot_utils.GSUtil()
    gsutils.execute(['cp',
                    namer.sdk_zipfilepath('latest', bot_info.system,
                                          'ia32', 'release'),
                    local_zip])
    if bot_info.system == 'windows':
      with zipfile.ZipFile(local_zip, 'r') as zip_file:
        zip_file.extractall(path=build_root)
    else:
      # We don't keep the execution bit if we use python's zipfile on posix.
      RunProcess(['unzip', local_zip, '-d', build_root])
    pub = GetPub(bot_info)
    RunProcess([pub, '--version'])

def GetPackagePath(bot_info):
  if bot_info.is_repo:
    return os.path.join('pkg', bot_info.package_name)
  return os.path.join('third_party', 'pkg', bot_info.package_name)

def GetBuildRoot(bot_info):
  system = bot_info.system
  if system == 'windows':
    system = 'win32'
  if system == 'mac':
    system = 'macos'
  return utils.GetBuildRoot(system, mode='release', arch='ia32',
                            target_os=system)

def SafeDelete(path, bot_info):
  if bot_info.system == 'windows':
    if os.path.exists(path):
      args = ['cmd.exe', '/c', 'rmdir', '/q', '/s', path]
      RunProcess(args)
  else:
    shutil.rmtree(path, ignore_errors=True)


def GetPackageCopy(bot_info):
  build_root = GetBuildRoot(bot_info)
  package_copy = os.path.join(build_root, 'package_copy')
  package_path = GetPackagePath(bot_info)
  copy_path = os.path.join(package_copy, bot_info.package_name)
  SafeDelete(package_copy, bot_info)
  no_git = shutil.ignore_patterns('.git')
  shutil.copytree(package_path, copy_path, symlinks=False, ignore=no_git)
  return copy_path

def GetSdkBin(bot_info):
  return os.path.join(os.getcwd(), GetBuildRoot(bot_info),
                      'dart-sdk', 'bin')

def GetPub(bot_info):
  executable = 'pub.bat' if bot_info.system == 'windows' else 'pub'
  return os.path.join(GetSdkBin(bot_info), executable)

def GetPubEnv(bot_info):
  pub_cache = os.path.join(os.getcwd(), GetBuildRoot(bot_info), 'pub_cache')
  return {'PUB_CACHE': pub_cache, 'PUB_ENVIRONMENT': 'dart_bots'}

# _RunPubCacheRepair and _CheckPubCacheCorruption are not used right now, but we
# keep them around because they provide an easy way to diagnose and fix issues
# in the bots.
def _RunPubCacheRepair(bot_info, path):
  pub = GetPub(bot_info)
  extra_env = GetPubEnv(bot_info)
  with BuildStep('Pub cache repair'):
    # For now, assume pub
    with ChangedWorkingDirectory(path):
      args = [pub, 'cache', 'repair']
      RunProcess(args, extra_env=extra_env)

corruption_checks = 0
def _CheckPubCacheCorruption(bot_info, path):
  extra_env = GetPubEnv(bot_info)
  global corruption_checks
  corruption_checks += 1
  with BuildStep('Check pub cache corruption %d' % corruption_checks):
    with ChangedWorkingDirectory(path):
      packages = os.path.join(
        extra_env['PUB_CACHE'], 'hosted', 'pub.dartlang.org')
      print '\nLooking for packages in %s:' % str(packages)
      if not os.path.exists(packages):
        print "cache directory doesn't exist"
        return
      for package in os.listdir(packages):
        if 'unittest-' in package:
          exists = os.path.exists(
                  os.path.join(packages, package, 'lib', 'unittest.dart'))
          print '- ok:  ' if exists else '- bad: ',
          print os.path.join(package, 'lib', 'unittest.dart')
      print ''

def RunPubUpgrade(bot_info, path):
  pub = GetPub(bot_info)
  extra_env = GetPubEnv(bot_info)
  with BuildStep('Pub upgrade'):
    # For now, assume pub
    with ChangedWorkingDirectory(path):
      args = [pub, 'upgrade', '--no-packages-dir']
      RunProcess(args, extra_env=extra_env)

def RunPubBuild(bot_info, path, folder, mode=None):
  skip_pub_build = ['dart-protobuf', 'rpc']
  with BuildStep('Pub build on %s' % folder):
    if bot_info.package_name in skip_pub_build:
      print "Not running pub build"
      return
    pub = GetPub(bot_info)
    extra_env = GetPubEnv(bot_info)
    with ChangedWorkingDirectory(path):
      # run pub-build on the web folder
      if os.path.exists(folder):
        args = [pub, 'build']
        if mode:
            args.append('--mode=%s' % mode)
        if folder != 'web':
            args.append(folder)
        RunProcess(args, extra_env=extra_env)

# Major hack
def FixupTestControllerJS(package_path):
  if os.path.exists(os.path.join(package_path, 'packages', 'unittest')):
    test_controller = os.path.join(package_path, 'packages', 'unittest',
                                   'test_controller.js')
    dart_controller = os.path.join('tools', 'testing', 'dart',
                                   'test_controller.js')
    print 'Hack test controller by copying of  %s to %s' % (dart_controller,
                                                            test_controller)
    shutil.copy(dart_controller, test_controller)
  else:
    print "No unittest to patch, do you even have tests"

JS_RUNTIMES = {
  'windows': ['ff', 'chrome', 'ie10'],
  'linux': ['d8', 'ff', 'chrome'],
  'mac': ['safari'],
}

is_first_test_run = True
def LogsArgument():
  global is_first_test_run
  if is_first_test_run:
    is_first_test_run = False
    return []
  return ['--append_logs']

def RunPackageTesting(bot_info, package_path, folder='test'):
  package_name = os.path.basename(package_path)
  if package_name == '':
    # when package_path had a trailing slash
    package_name = os.path.basename(os.path.dirname(package_path))
  if folder == 'build/test':
    suffix = ' under build'
    package_root = os.path.join(package_path, folder, 'packages')
    package_arg = '--package-root=%s' % package_root
  else:
    suffix = ''
    package_spec_file = os.path.join(package_path, '.packages')
    package_arg = '--packages=%s' % package_spec_file


  # Note: we use package_name/package_name/folder and not package_name/folder on
  # purpose. The first package_name denotes the suite, the second is part of the
  # path we want to match. Without the second package_name, we may match tests
  # that contain "folder" further down. So if folder is "test",
  # "package_name/test" matches "package_name/build/test", but
  # "package_name/package_name/test" does not.
  standard_args = ['--arch=ia32',
                   '--suite-dir=%s' % package_path,
                   '--use-sdk', '--report', '--progress=buildbot',
                   '--reset-browser-configuration',
                   package_arg,
                   '--write-debug-log', '-v',
                   '--time',
                   '%s/%s/%s/' % (package_name, package_name, folder)]
  with BuildStep('Test vm release mode%s' % suffix, swallow_error=True):
    args = [sys.executable, 'tools/test.py',
            '-mrelease', '-rvm', '-cnone'] + standard_args
    args.extend(LogsArgument())
    # For easy integration testing we give access to the sdk bin directory.
    # This only makes sense on vm testing.
    extra_env = { 'DART_SDK_BIN' : GetSdkBin(bot_info) }
    RunProcess(args, extra_env=extra_env)
  with BuildStep('Test analyzer%s' % suffix, swallow_error=True):
    args = [sys.executable, 'tools/test.py',
            '-mrelease', '-rnone', '-cdart2analyzer'] + standard_args
    args.extend(LogsArgument())
    RunProcess(args)
  # TODO(27065): Restore Dartium testing once it works on test.py again.
  for runtime in JS_RUNTIMES[bot_info.system]:
    with BuildStep('dart2js-%s%s' % (runtime, suffix), swallow_error=True):
      test_args = [sys.executable, 'tools/test.py',
                   '-mrelease', '-r%s' % runtime, '-cdart2js', '-j4',
                   '--dart2js-batch']
      args = test_args + standard_args
      args.extend(LogsArgument())
      _RunWithXvfb(bot_info, args)

def FillMagicMarkers(v, replacements):
  def replace(match):
    word = match.group(1)
    if not word in replacements:
      raise Exception("Unknown magic marker %s. Known mappings are: %s" %
                      (word, replacements))
    return replacements[word]
  return re.sub(r"\$(\w+)", replace, v)

def RunTestRunner(bot_info, test_package, package_path):
  package_name = os.path.basename(package_path)
  if package_name == '':
    # when package_path had a trailing slash
    package_name = os.path.basename(os.path.dirname(package_path))

  pub = GetPub(bot_info)
  extra_env = GetPubEnv(bot_info)
  with BuildStep('pub run test', swallow_error=True):
    # TODO(nweiz): include dartium here once sdk#23816 is fixed.
    platforms = set(['vm', 'chrome', 'firefox'])
    if bot_info.system == 'windows':
      platforms.add('ie')
      # TODO(nweiz): remove dartium here once sdk#23816 is fixed.
    elif bot_info.system == 'mac':
      platforms.add('safari')
      platforms.remove('firefox')

    if 'platforms' in test_package:
      platforms = platforms.intersection(set(test_package['platforms']))

    with utils.ChangedWorkingDirectory(package_path):
      test_args = [pub, 'run', 'test', '--reporter', 'expanded', '--no-color',
                   '--platform', ','.join(platforms)]
      if bot_info.package_name in SERIALIZED_PACKAGES:
          test_args.append('-j1')
      # TODO(6): If barback is needed, use --pub-serve option and pub serve test
      if test_package.get('barback'): test_args.append('build/test')
      _RunWithXvfb(bot_info, test_args, extra_env=extra_env)

def _RunWithXvfb(bot_info, args, **kwargs):
  if bot_info.system == 'linux':
    args = ['xvfb-run', '-a', '--server-args=-screen 0 1024x768x24'] + args
  RunProcess(args, **kwargs)

# Runs the script given by test_config.get_config if it exists, does nothing
# otherwise.
# Returns `True` if the script was run.
def RunCustomScript(test_config):
  custom_script = test_config.get_custom_script()
  if custom_script:
    command_string = FillMagicMarkers(custom_script, test_config.replacements)
    with BuildStep('Running custom script'):
      args = shlex.split(command_string, posix=False)
      print 'Running command: %s' % args
      sys.stdout.flush()
      exit_code = subprocess.call(args)
      if exit_code != 0:
        print "Custom script failed"
    return True
  else:
    return False

def RunDefaultScript(bot_info, test_config, copy_path):
  print "No custom script found, running default steps."

  GetSDK(bot_info)
  print 'Running testing in copy of package in %s' % copy_path
  RunPrePubUpgradeHooks(test_config)
  RunPubUpgrade(bot_info, copy_path)

  test_package = test_config.get_test_package()
  if test_package is None or test_package.get('barback'):
    RunPrePubBuildHooks(test_config)
    RunPubBuild(bot_info, copy_path, 'web')
    RunPubBuild(bot_info, copy_path, 'test', 'debug')
    RunPostPubBuildHooks(test_config)

  if test_package is not None:
    print 'Running the test package runner'
    RunPreTestHooks(test_config)
    RunTestRunner(bot_info, test_package, copy_path)
  else:
    print 'Running tests manually'
    FixupTestControllerJS(copy_path)
    RunPreTestHooks(test_config)
    RunPackageTesting(bot_info, copy_path, 'test')
    # TODO(6): Packages that need barback should use the test package runner,
    # instead of trying to run from the build/test directory.
    RunPackageTesting(bot_info, copy_path, 'build/test')

  RunPostTestHooks(test_config)


def RunHooks(hooks, section_name, replacements):
  for name, command in hooks.iteritems():
    command = FillMagicMarkers(command, replacements)
    with BuildStep('%s: %s' % (section_name, name), swallow_error=True):
      RunProcess(command, shell=True)

def RunPrePubUpgradeHooks(test_config):
  RunHooks(test_config.get_pre_pub_upgrade_hooks(), "Pre pub upgrade hooks",
           test_config.replacements)

def RunPrePubBuildHooks(test_config):
  RunHooks(test_config.get_pre_pub_build_hooks(), "Pre pub build hooks",
           test_config.replacements)

def RunPostPubBuildHooks(test_config):
  RunHooks(test_config.get_post_pub_build_hooks(), "Pre pub build hooks",
           test_config.replacements)

def RunPreTestHooks(test_config):
  RunHooks(test_config.get_pre_test_hooks(), "Pre test hooks",
           test_config.replacements)

def RunPostTestHooks(test_config):
  RunHooks(test_config.get_post_test_hooks(), "Post test hooks",
           test_config.replacements)

def main():
  bot_info = GetBotInfo()

  print 'Bot info: %s' % bot_info
  copy_path = GetPackageCopy(bot_info)
  config_file = os.path.join(copy_path, '.test_config')
  test_config = config_parser.ConfigParser(config_file)
  test_config.replacements = {
    'dart': utils.CheckedInSdkExecutable(),
    'project_root': copy_path,
    'python': sys.executable
  }

  RunCustomScript(test_config) or \
    RunDefaultScript(bot_info, test_config, copy_path)

if __name__ == '__main__':
  main()