[contrib cms] Add option to include additional files in bundling.

law/contrib/cms/scripts/bundle_cmssw.sh

@@ -7,6 +7,7 @@
 # 1. The path to the CMSSW checkout, i.e., the value of the CMSSW_BASE variable.
 # 2. The path where the bundle should be stored, should be absolute and end with .tgz.
 # 3. A regex for excluding files or directories in src, should start with (e.g.) ^src/. Optional.
+# 4. Space separated names of files or directories in CMSSW_BASE to include. Optional.
 
 action() {
     local cmssw_base="$1"
@@ -27,17 +28,27 @@ action() {
     fi
 
     # choose a default value the the exclusion regex that really should not match any path in src
-    local exclude="${3:-???}"
+    local exclude="${3:-${RANDOM}${RANDOM}${RANDOM}}"
 
+    # handle additional files or directories to include, which are all considered optional so filter
+    local include=""
+    for f in ${4:-}; do
+        if [ -f "${cmssw_base}/${f}" ] || [ -d "${cmssw_base}/${f}" ]; then
+            include="${include} ${f}"
+        fi
+    done
+
+    # create parent directories
+    local dst_dir="$( dirname "${dst_path}" )"
+    [ ! -z "${dst_dir}" ] && [ ! -d "${dst_dir}" ] && mkdir -p "${dst_dir}"
+
+    # create the bundle
     (
         cd "${cmssw_base}" && \
         find src -maxdepth 3 -type d \
             | grep -e "^src/.*/.*/\(interface\|data\|python\)" \
             | grep -v -e "${exclude}" \
-            | tar -czf "${dst_path}" --dereference lib biglib bin cfipython --exclude="*.pyc" --files-from -
+            | tar -czf "${dst_path}" --dereference lib biglib bin cfipython ${include} --exclude "*.pyc" --exclude "__pycache__" --files-from -
     )
-    local ret="$?"
-
-    return "${ret}"
 }
 action "$@"

law/contrib/cms/tasks.py

@@ -18,7 +18,7 @@
 from law.task.base import Task
 from law.target.file import get_path
 from law.target.local import LocalFileTarget
-from law.parameter import NO_STR
+from law.parameter import NO_STR, CSVParameter
 from law.decorator import log
 from law.util import rel_path, interruptable_popen, quote_cmd
 
@@ -29,9 +29,16 @@ class BundleCMSSW(Task):
 
     exclude = luigi.Parameter(
         default=NO_STR,
+        significant=False,
         description="regular expression for excluding files or directories relative to CMSSW_BASE; "
         "default: empty",
     )
+    include = CSVParameter(
+        default=(),
+        significant=False,
+        description="comma-separated list of files or directories relative to CMSSW_BASE to "
+        "include; default: empty",
+    )
     custom_checksum = luigi.Parameter(
         default=NO_STR,
         description="a custom checksum to use; default: empty",
@@ -87,16 +94,17 @@ def run(self):
             with self.publish_step("bundle CMSSW ..."):
                 self.bundle(tmp.path)
 
-    def bundle(self, dst_path):
-        cmd = [
+    def get_cmssw_bundle_command(self, dst_path):
+        return [
             rel_path(__file__, "scripts", "bundle_cmssw.sh"),
             get_path(self.get_cmssw_path()),
             get_path(dst_path),
+            self.exclude if self.exclude not in (None, NO_STR) else "",
+            " ".join(self.include),
         ]
-        if self.exclude != NO_STR:
-            cmd += [self.exclude]
-        cmd = quote_cmd(cmd)
 
-        code = interruptable_popen(cmd, shell=True, executable="/bin/bash")[0]
+    def bundle(self, dst_path):
+        cmd = self.get_cmssw_bundle_command(dst_path)
+        code = interruptable_popen(quote_cmd(cmd), shell=True, executable="/bin/bash")[0]
         if code != 0:
             raise Exception("cmssw bundling failed")

law/job/law_job.sh

@@ -75,7 +75,6 @@ law_job() {
     echo "${_law_job_start_time}"
 
 
-
     #
     # store arguments
     #