git-cms-checkdeps

#!/usr/bin/env perl
#
# $Id: git-checkdeps,v 1.8 2013/03/25 13:26:43 eulisse Exp $
#
# Created by: David Lange
#             LLNL
#             12 September 207
#
use strict;
use warnings;
use Getopt::Long;
use Cwd;

#Idea: diff checked out packages.
#     looking for changed headers
#     find all packages that depend on those headers
#     print that list.

#if any argument, print help and exit 

my $poisondir="poison";
my $checkout=0;
my $checkpython=0;
my $checkheader=0;
my $checkbuildfile=0;
my $printFileNames=0;
my $poison_includes=undef;
my $poison_only=0;
my $jobs=4;
 
while (my $arg=shift) {
    if ( $arg eq "-a" ) {
	$checkout=1;
	if (!defined $poison_includes) {$poison_includes=1;}
    }
    elsif ( $arg eq "-p" ) {
	$checkpython=1;
    }
    elsif ( $arg eq "-h" ) {
	$checkheader=1;
    }
    elsif ( $arg eq "-b" ) {
	$checkbuildfile=1;
    }
    elsif ( $arg eq "-f" ) {
	$printFileNames=1;
    }
    elsif ( $arg eq "-d" ) {
	$poison_includes=0;
    }
    elsif ( $arg eq "-D" ) {
	$poison_only=1;
    }
    elsif ( $arg eq "-j" ) {
        $arg = shift;
	if ($arg!~/^\d+$/){die "Error: Invalid value '$arg' use with -j command-line argument.\n";}
	$jobs=int($arg);
        if ($jobs<1){$jobs=1;}
    }
    elsif ( $arg eq "-A" ) {
        $checkpython=1;
        $checkheader=1;
        $checkbuildfile=1;
    }
    else{
	print "checkdeps:\n",
              "   Utility to check your local development area against the CMSSW release.\n",
              "   Any modified header files or python modules are found and the package\n",
              "   dependencies of these files will be returned. addpkg-ing these packages\n",
              "   and rebuilding them should provide a full and consistent build.\n",
              "\nUsage: \n",
              "   -p look for python modules and their dependencies  (ON by default)\n",
              "   -h look for header files and their dependencies    (ON by default)\n",
              "   -b look for BuildFile files and their dependencies (OFF by default)\n",
              "   -f print name of changed file name instead of type of dependency (OFF by default)\n",
              "   -a will add/checkout the packages into your development area\n",
              "   -A will check all dependencies i.e. header (-h), python(-p) and BuildFile(-b)\n",
              "   -d do not create dummy copy of deleted files in ${poisondir} directory. Dummy copies\n",
              "      are useful to find out if deleted headers files are included by other source files.\n",
              "   -D only dummy copy of deleted files in ${poisondir} directory created.\n",
              "      NOTE: all other dependency check/add command-line args are ignored.\n",
              "   -j <N> Run N threads to checkout packages. Default is ${jobs}\n",
              "   -q will pass -Q option to cvs to make it quiet\n\n";
	exit;
    }
}

if ($poison_only) {
    $checkpython=0;
    $checkheader=0;
    $checkbuildfile=0;
    $checkout=0;
    $poison_includes=1;
}
elsif ((!$checkpython) && (!$checkheader) && (!$checkbuildfile)) {
    $checkpython=1;
    $checkheader=1;
}

if(!defined $poison_includes) { $poison_includes = 0; }

my $localtop=$ENV{CMSSW_BASE};
die "ERROR: Could not find developer area base path. Please run \"cmsenv\" in a developer area.\n" if ($localtop eq "");

my $releasetop=$ENV{CMSSW_RELEASE_BASE};
die "ERROR: Could not find release base path. Please run this script from a developer area.\n" if ($releasetop eq "");

my %vals=();
if ($checkheader){
    my $depfile = "${releasetop}/etc/dependencies/usedby.out.gz";
    die "ERROR: This release appears not to support the functionality of this script (170pre4 and higher). Sorry\n" if (!-e "$depfile");
    &readDependencyInfo("$depfile",\%vals);
}

if ($checkpython) {
    my $depfile = "${releasetop}/etc/dependencies/pyusedby.out.gz";
    &readDependencyInfo("$depfile",\%vals) if (-e "$depfile");
}

if ($checkbuildfile)
{
    my $depfile = "${releasetop}/etc/dependencies/bfusedby.out.gz";
    &readDependencyInfo("$depfile",\%vals) if (-e "$depfile");
}

chdir("${localtop}/src");

my @changedFiles;
my @deletedFiles;
my $reltag;
if ($ENV{CMSSW_GIT_HASH}) {
    $reltag=$ENV{CMSSW_GIT_HASH};
} else {
    $reltag=$ENV{CMSSW_VERSION};
}
my %upackages=();

foreach my $diff (`git diff -G '^([^\$]+\$|[^\$]*[\$][^\$]*\$|([^\$]*[\$])(\$|[^RAIDLNSH]|(R[^eC]|A[^u]|I[^d]|D[^a]|L[^o]|N[^a]|S[^o]|H[^e])|(Re[^v]|RC[^S]|Au[^t]|Id[^:]|Da[^t]|Lo[^g]|Na[^m]|So[^u]|He[^a])|(Rev[^i]|RCS[^f]|Aut[^h]|Dat[^e]|Log[^:]|Nam[^e]|Sou[^r]|Hea[^d])|(Revi[^s]|RCSf[^i]|Auth[^o]|Date[^:]|Name[^:]|Sour[^c]|Head[^e])|(Revis[^i]|RCSfi[^l]|Autho[^r]|Sourc[^e]|Heade[^r])|(Revisi[^o]|RCSfil[^e]|Author[^:]|Source[^:]|Header[^:])|(Revisio[^n]|RCSfile[^:])|(Revision[^:])))' --name-status -r $reltag | grep -v .gitignore | sed -e 's/[ \t]\+/ /'`)
{
  chomp $diff;
  if ($diff=~/^.\s+([^\/]+\/[^\/]+)\/[^\s]+$/o)
  {
    my $pack=$1;
    if (!exists $upackages{$pack})
    {
      $upackages{$pack}=[];
      if (!-e $pack){print ">> Package removed $pack\n";}
    }
    push @{$upackages{$pack}},$diff;
  }
}
my @packages=sort keys %upackages;

foreach my $package (@packages) {
    next if ($package =~/^UserCode.*/o);    
    print ">> Checking $package $reltag\n";
    my %msgs=("-" => {}, "x" => {});
    foreach my $diff (@{$upackages{$package}}) {
        my $del = 0;
        if ( ($diff =~/^(M)/o) || ($diff =~/^(U)/o) || ($diff =~/^(D)/o) ) {
            if ($1 eq "D"){$del =1;}
            my @sp2=split(' ', $diff);
            $diff = $sp2[1];
        }
        else {$diff = "";}
        if ($diff) {$msgs{"x"}{$diff}=1;}
        if ($del)  {$msgs{"-"}{$diff}=1;}
    }
    my @files=sort keys %{$msgs{"-"}};
    push(@deletedFiles,@files);
    push(@changedFiles,@files);
    foreach my $diff (@files){print "   - $diff\n"; delete $msgs{"x"}{$diff};}
    @files=sort keys %{$msgs{"x"}};
    push(@changedFiles,@files);
    foreach my $diff (@files){print "   x $diff\n";}
}

&poisonIncludes(\@deletedFiles,$localtop) if ($poison_includes);
if ($poison_only) {exit 0;}

my %recompileList;
my $big_bf="";
foreach my $file (@changedFiles) {
    if ($file=~/^BigProducts\/[^\/]+\/BuildFile\.xml$/o){$big_bf="${big_bf} ${file}";}
    next if (!(exists $vals{$file}));
    foreach my $dep (@{$vals{$file}}) {
        my @sp=split('/',$dep);
        my $recompile="$sp[0]/$sp[1]";
        if (!(grep {$_ eq $recompile} @packages)) {
            my $scope="header";
            if ($file=~/\.py$/o){$scope="python";}
            elsif($file=~/\/BuildFile(\.xml|)$/o){$scope="buildfile";}
            $recompileList{$recompile}{$scope}=$file;
        }
    }
}

if ($big_bf ne "")
{
  foreach my $diff (`git diff -r $reltag -- $big_bf | sed -e 's/[ \t]\+/ /' | grep '^+' | grep -i '< *use  *name *='`)
  {
    chomp $diff;
    $diff=~s/^[+]//o;
    foreach my $p (split(">",$diff))
    {
      if ($p=~/^\s*<\s*use\s+name\s*=\s*["]([^"]+)["]\s*/)
      {
	if (-e "${releasetop}/src/${1}"){$recompileList{$1}{biglib}="BigProducts";}
      }
    }
  }
}

my @t=sort(keys %recompileList);
my $len=scalar(@t);
if ( $checkout == 0 ) {
    print "Packages to check out and compile: $len\n";
}
else{
    print "Checking out these packages: $len\n";
}

my $exitcode=0;
if ( $len > 0 ) {
    foreach my $pk (@t) {
	if ( $printFileNames == 1 ) {
	    print "$pk (",join(", ",sort values %{$recompileList{$pk}}),")\n";
	} else {
	    print "$pk (",join(", ",sort keys %{$recompileList{$pk}}),")\n";
	}
    }
    if ( $checkout == 1 ) {
        chdir ($localtop);
        use File::Temp qw(tempfile);
        my ($fh,$fname) = tempfile($ENV{CMSSW_BASE} . '/tmp/checkdepsXXXXX', UNLINK => 1);;
        foreach my $pk (@t) {
            print $fh "$pk/\n";
        }
        system("cd $ENV{CMSSW_BASE}/src; cat $fname >> $ENV{CMSSW_BASE}/src/.git/info/sparse-checkout; git read-tree -mu HEAD");
        $exitcode+=$?;
        close($fh);
    }
}
if ($exitcode>0){$exitcode=1;}
exit($exitcode);

sub readDependencyInfo()
{
  my $file=shift;
  my $cache=shift;
  my $ref;
  open($ref,"gunzip -c $file |") || die "Can not open file for reading: $file\n";
  while  (my $line=<$ref>) { 
    chomp($line);
    my ($file1,$rest)=split(' ',$line,2);
    if ($file1 ne "")
    {
      if (!exists $vals{$file1}){$vals{$file1}=[];}
      push @{$vals{$file1}},split(' ',$rest);
    }
  }
  close($ref);
}

sub poisonIncludes() {
    my ($deletedFiles,$topdir)=@_;
    use File::Path qw (mkpath rmtree);
    use File::Basename qw(dirname);
    my $poison="${topdir}/${poisondir}";
    if (-d $poison){rmtree($poison);}
    if (scalar(@$deletedFiles)>0) {print ">> Creating dummy files under $poison directory.\n";}
    foreach my $file (@$deletedFiles) {
        my $f="${poison}/${file}";
        my $dir=dirname($f);
        if (!-d $dir){mkpath($dir);}
        my $ref;
        if(!open($ref,">${f}")){die "ERROR: Can not open file for writing: $f\n";}
        print $ref "#error THIS FILE HAS BEEN REMOVED FROM THE PACKAGE.\n";
        close($ref);
        print "   $file\n";
    }
}