Merge pull request #43701 from jsamudio/alpakaPFWorkflow

Introduce matrix workflows for Alpaka-based Particle Flow

Configuration/PyReleaseValidation/README.md

@@ -34,6 +34,10 @@ The offsets currently in use are:
 * 0.403: Alpaka, pixel only quadruplets, portable vs. CPU validation
 * 0.404: Alpaka, pixel only quadruplets, portable profiling
 * 0.412: Alpaka, ECAL only, portable
+* 0.422: Alpaka, HCAL only, portable
+* 0.423: Alpaka, HCAL only, portable vs CPU validation
+* 0.424: Alpaka, HCAL only, portable profiling
+* 0.492: Alpaka, full reco with pixel quadruplets
 * 0.5: Pixel tracking only + 0.1
 * 0.501: Patatrack, pixel only quadruplets, on CPU
 * 0.502: Patatrack, pixel only quadruplets, with automatic offload to GPU if available

Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py

@@ -1069,7 +1069,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
 upgradeWFs['PatatrackECALOnlyAlpaka'] = PatatrackWorkflow(
     digi = {
         # customize the ECAL Local Reco part of the HLT menu for Alpaka
-        '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka 
+        '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
     },
     reco = {
         '-s': 'RAW2DIGI:RawToDigi_ecalOnly,RECO:reconstruction_ecalOnly,VALIDATION:@ecalOnlyValidation,DQM:@ecalOnly',
@@ -1238,6 +1238,78 @@ def setup_(self, step, stepName, stepDict, k, properties):
     offset = 0.524,
 )
 
+# HCAL-PF Only workflow running HCAL local reco on GPU and PF with Alpaka with DQM and Validation
+# - HLT-alpaka
+# - HCAL-only reconstruction using Alpaka with DQM and Validation
+upgradeWFs['PatatrackHCALOnlyAlpakaValidation'] = PatatrackWorkflow(
+    digi = {
+        '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
+    },
+    reco = {
+        '-s': 'RAW2DIGI:RawToDigi_hcalOnly,RECO:reconstruction_hcalOnly,VALIDATION:@hcalOnlyValidation,DQM:@hcalOnly+@hcal2Only',
+        '--procModifiers': 'alpaka'
+    },
+    harvest = {
+        '-s': 'HARVESTING:@hcalOnlyValidation'
+    },
+    suffix = 'Patatrack_HCALOnlyAlpaka_Validation',
+    offset = 0.422,
+)
+
+# HCAL-PF Only workflow running HCAL local reco and PF with Alpaka with cluster level-validation
+# - HLT-alpaka
+# - HCAL-only reconstruction using GPU and Alpaka with DQM and Validation for PF Alpaka vs CPU comparisons
+upgradeWFs['PatatrackHCALOnlyGPUandAlpakaValidation'] = PatatrackWorkflow(
+    digi = {
+        '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
+    },
+    reco = {
+        '-s': 'RAW2DIGI:RawToDigi_hcalOnly,RECO:reconstruction_hcalOnlyLegacy+reconstruction_hcalOnly,VALIDATION:@hcalOnlyValidation+pfClusterHBHEOnlyAlpakaComparisonSequence,DQM:@hcalOnly+@hcal2Only',
+        '--procModifiers': 'alpaka'
+    },
+    harvest = {
+        '-s': 'HARVESTING:@hcalOnlyValidation'
+    },
+    suffix = 'Patatrack_HCALOnlyGPUandAlpaka_Validation',
+    offset = 0.423,
+)
+
+# HCAL-PF Only workflow running HCAL local reco on CPU and PF with Alpaka slimmed for benchmarking
+# - HLT-alpaka
+# - HCAL-only reconstruction using Alpaka
+upgradeWFs['PatatrackHCALOnlyAlpakaProfiling'] = PatatrackWorkflow(
+    digi = {
+        '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
+    },
+    reco = {
+        '-s': 'RAW2DIGI:RawToDigi_hcalOnly,RECO:reconstruction_hcalOnly',
+        '--procModifiers': 'alpaka'
+    },
+    harvest = None,
+    suffix = 'Patatrack_HCALOnlyAlpaka_Profiling',
+    offset = 0.424,
+)
+
+# Workflow running the Pixel quadruplets, ECAL and HCAL reconstruction on GPU (optional), PF using Alpaka, together with the full offline reconstruction on CPU
+#  - HLT on GPU (optional)
+#  - reconstruction on Alpaka, with DQM and validation
+#  - harvesting
+upgradeWFs['PatatrackFullRecoAlpaka'] = PatatrackWorkflow(
+    digi = {
+        '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
+    },
+    reco = {
+        # skip the @pixelTrackingOnlyValidation which cannot run together with the full reconstruction
+        '-s': 'RAW2DIGI:RawToDigi+RawToDigi_pixelOnly,L1Reco,RECO:reconstruction+reconstruction_pixelTrackingOnly,RECOSIM,PAT,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM+@pixelTrackingOnlyDQM',
+        '--procModifiers': 'alpaka,pixelNtupletFit'
+    },
+    harvest = {
+        # skip the @pixelTrackingOnlyDQM harvesting
+    },
+    suffix = 'Patatrack_FullRecoAlpaka',
+    offset = 0.492,
+)
+
 # Workflow running the Pixel quadruplets, ECAL and HCAL reconstruction on CPU
 #  - HLT on CPU
 #  - reconstruction on CPU, with DQM and validation
@@ -1547,7 +1619,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
 
 upgradeWFs['PatatrackPixelOnlyAlpaka'] = PatatrackWorkflow(
     digi = {
-        '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka 
+        '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
     },
     reco = {
         '-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM',
@@ -1562,7 +1634,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
 
 upgradeWFs['PatatrackPixelOnlyAlpakaValidation'] = PatatrackWorkflow(
     digi = {
-        '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka 
+        '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
     },
     reco = {
         '-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly,VALIDATION:@pixelTrackingOnlyValidation,DQM:@pixelTrackingOnlyDQM',
@@ -1577,7 +1649,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
 
 upgradeWFs['PatatrackPixelOnlyAlpakaProfiling'] = PatatrackWorkflow(
     digi = {
-        '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka 
+        '--procModifiers': 'alpaka', # alpaka modifier activates customiseHLTForAlpaka
     },
     reco = {
         '-s': 'RAW2DIGI:RawToDigi_pixelOnly,RECO:reconstruction_pixelTrackingOnly',

Configuration/StandardSequences/python/Reconstruction_cff.py

@@ -243,7 +243,16 @@
     pfClusteringHBHEHFOnlyTask
 )
 
+# define secondary validation task running only Legacy
+reconstruction_hcalOnlyLegacyTask = cms.Task(
+        bunchSpacingProducer,
+        offlineBeamSpot,
+        hcalOnlyLegacyLocalRecoTask,
+        hcalOnlyLegacyGlobalRecoTask,
+        pfClusteringHBHEHFOnlyLegacyTask)
+
 reconstruction_hcalOnly = cms.Sequence(reconstruction_hcalOnlyTask)
+reconstruction_hcalOnlyLegacy = cms.Sequence(reconstruction_hcalOnlyLegacyTask)
 
 #need a fully expanded sequence copy
 modulesToRemove = list() # copy does not work well

RecoLocalCalo/Configuration/python/hcalGlobalReco_cff.py

@@ -19,6 +19,9 @@
 run3_HB.toReplaceWith(hbhereco.cpu, _phase1_hbheprereco)
 run3_HB.toReplaceWith(hcalOnlyGlobalRecoTask, cms.Task(hbhereco))
 
+#-- Legacy HCAL Only Task
+hcalOnlyLegacyGlobalRecoTask = hcalOnlyGlobalRecoTask.copy()
+
 #--- for Run 3 on GPU
 from Configuration.ProcessModifiers.gpu_cff import gpu
 

RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py

@@ -58,6 +58,9 @@
 from Configuration.Eras.Modifier_run3_HB_cff import run3_HB
 run3_HB.toReplaceWith(hcalLocalRecoTask, _run3_hcalLocalRecoTask)
 
+#--- Legacy HCAL Only Task
+hcalOnlyLegacyLocalRecoTask = hcalLocalRecoTask.copyAndExclude([zdcreco])
+
 #--- for Run 3 on GPU
 from Configuration.ProcessModifiers.gpu_cff import gpu
 

RecoParticleFlow/PFClusterProducer/python/particleFlowCluster_cff.py

@@ -44,6 +44,9 @@
                                       particleFlowClusterHF,
                                       particleFlowClusterHCALOnly)
 
+#--- Legacy HCAL Only Task
+pfClusteringHBHEHFOnlyLegacyTask = pfClusteringHBHEHFOnlyTask.copy()
+
 pfClusteringHOTask = cms.Task(particleFlowRecHitHO,particleFlowClusterHO)
 pfClusteringHO = cms.Sequence(pfClusteringHOTask)
 

RecoParticleFlow/PFClusterProducer/python/pfClusterHBHEAlpaka_cff.py

@@ -11,9 +11,12 @@
 from RecoParticleFlow.PFClusterProducer.pfClusterSoAProducer_cfi import pfClusterSoAProducer as _pfClusterSoAProducer
 from RecoParticleFlow.PFClusterProducer.legacyPFClusterProducer_cfi import legacyPFClusterProducer as _legacyPFClusterProducer
 
-from RecoParticleFlow.PFClusterProducer.particleFlowCluster_cff import pfClusteringHBHEHFTask, particleFlowClusterHBHE, particleFlowRecHitHBHE, particleFlowClusterHCAL
-
+from RecoParticleFlow.PFClusterProducer.particleFlowCluster_cff import pfClusteringHBHEHFTask, pfClusteringHBHEHFOnlyTask, particleFlowClusterHBHE, particleFlowRecHitHBHE, particleFlowClusterHCAL, particleFlowClusterHBHEOnly, particleFlowRecHitHBHEOnly, particleFlowClusterHCALOnly
+#Full Reco
 _alpaka_pfClusteringHBHEHFTask = pfClusteringHBHEHFTask.copy()
+#HCAL Only
+_alpaka_pfClusteringHBHEHFOnlyTask = pfClusteringHBHEHFOnlyTask.copy()
+
 
 pfRecHitHCALParamsRecordSource = cms.ESSource('EmptyESSource',
             recordName = cms.string('PFRecHitHCALParamsRecord'),
@@ -75,23 +78,86 @@
         PFRecHitsLabelIn = 'pfRecHitSoAProducerHCAL'
     )
 
-
+#Full Reco
 _alpaka_pfClusteringHBHEHFTask.add(pfRecHitHCALParamsRecordSource)
 _alpaka_pfClusteringHBHEHFTask.add(pfRecHitHCALTopologyRecordSource)
 _alpaka_pfClusteringHBHEHFTask.add(pfClusterParamsRecordSource)
 _alpaka_pfClusteringHBHEHFTask.add(hbheRecHitToSoA)
 _alpaka_pfClusteringHBHEHFTask.add(pfRecHitHCALParamsESProducer)
+_alpaka_pfClusteringHBHEHFTask.add(pfRecHitHCALTopologyESProducer)
 _alpaka_pfClusteringHBHEHFTask.add(pfRecHitSoAProducerHCAL)
 _alpaka_pfClusteringHBHEHFTask.add(legacyPFRecHitProducer)
 _alpaka_pfClusteringHBHEHFTask.add(pfClusterParamsESProducer)
 _alpaka_pfClusteringHBHEHFTask.add(pfClusterSoAProducer)
+_alpaka_pfClusteringHBHEHFTask.add(legacyPFClusterProducer)
 
 _alpaka_pfClusteringHBHEHFTask.remove(particleFlowRecHitHBHE)
 _alpaka_pfClusteringHBHEHFTask.remove(particleFlowClusterHBHE)
 _alpaka_pfClusteringHBHEHFTask.remove(particleFlowClusterHCAL)
-_alpaka_pfClusteringHBHEHFTask.add(particleFlowClusterHBHE)
 _alpaka_pfClusteringHBHEHFTask.add(particleFlowClusterHCAL)
 
-alpaka.toReplaceWith(particleFlowClusterHBHE, legacyPFClusterProducer)
+alpaka.toModify(particleFlowClusterHCAL, clustersSource = "legacyPFClusterProducer")
 
 alpaka.toReplaceWith(pfClusteringHBHEHFTask, _alpaka_pfClusteringHBHEHFTask)
+
+#HCAL Only
+
+from Configuration.Eras.Modifier_run3_HB_cff import run3_HB
+
+hbheOnlyRecHitToSoA = _hcalRecHitSoAProducer.clone(
+        src = "hbheprereco"
+    )
+run3_HB.toModify(hbheOnlyRecHitToSoA,
+        src = "hbhereco"
+    )
+
+pfRecHitSoAProducerHBHEOnly = _pfRecHitSoAProducerHCAL.clone(
+        producers = cms.VPSet(
+            cms.PSet(
+                src = cms.InputTag("hbheOnlyRecHitToSoA"),
+                params = cms.ESInputTag("pfRecHitHCALParamsESProducer:"),
+            )
+        ),
+        topology = "pfRecHitHCALTopologyESProducer:",
+        synchronise = cms.untracked.bool(False)
+    )
+
+legacyPFRecHitProducerHBHEOnly = _legacyPFRecHitProducer.clone(
+        src = "pfRecHitSoAProducerHBHEOnly"
+    )
+
+pfClusterSoAProducerHBHEOnly = _pfClusterSoAProducer.clone(
+        pfRecHits = 'pfRecHitSoAProducerHBHEOnly',
+        topology = "pfRecHitHCALTopologyESProducer:",
+        pfClusterParams = 'pfClusterParamsESProducer:',
+        synchronise = cms.bool(False)
+    )
+
+legacyPFClusterProducerHBHEOnly = _legacyPFClusterProducer.clone(
+        src = 'pfClusterSoAProducerHBHEOnly',
+        pfClusterParams = 'pfClusterParamsESProducer:',
+        pfClusterBuilder = particleFlowClusterHBHE.pfClusterBuilder,
+        recHitsSource = 'legacyPFRecHitProducerHBHEOnly',
+        PFRecHitsLabelIn = 'pfRecHitSoAProducerHBHEOnly'
+    )
+
+_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitHCALParamsRecordSource)
+_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitHCALTopologyRecordSource)
+_alpaka_pfClusteringHBHEHFOnlyTask.add(pfClusterParamsRecordSource)
+_alpaka_pfClusteringHBHEHFOnlyTask.add(hbheOnlyRecHitToSoA)
+_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitHCALParamsESProducer)
+_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitHCALTopologyESProducer)
+_alpaka_pfClusteringHBHEHFOnlyTask.add(pfRecHitSoAProducerHBHEOnly)
+_alpaka_pfClusteringHBHEHFOnlyTask.add(legacyPFRecHitProducerHBHEOnly)
+_alpaka_pfClusteringHBHEHFOnlyTask.add(pfClusterParamsESProducer)
+_alpaka_pfClusteringHBHEHFOnlyTask.add(pfClusterSoAProducerHBHEOnly)
+_alpaka_pfClusteringHBHEHFOnlyTask.add(legacyPFClusterProducerHBHEOnly)
+
+_alpaka_pfClusteringHBHEHFOnlyTask.remove(particleFlowRecHitHBHEOnly)
+_alpaka_pfClusteringHBHEHFOnlyTask.remove(particleFlowClusterHBHEOnly)
+_alpaka_pfClusteringHBHEHFOnlyTask.remove(particleFlowClusterHCALOnly)
+_alpaka_pfClusteringHBHEHFOnlyTask.add(particleFlowClusterHCALOnly)
+
+alpaka.toModify(particleFlowClusterHCALOnly, clustersSource = "legacyPFClusterProducerHBHEOnly")
+
+alpaka.toReplaceWith(pfClusteringHBHEHFOnlyTask, _alpaka_pfClusteringHBHEHFOnlyTask)

Validation/RecoParticleFlow/python/PFClusterValidation_cff.py

@@ -1,10 +1,15 @@
 import FWCore.ParameterSet.Config as cms
 from Validation.RecoParticleFlow.pfClusterValidation_cfi import pfClusterValidation
+from Validation.RecoParticleFlow.pfCaloGPUComparisonTask_cfi import pfClusterHBHEOnlyAlpakaComparison, pfClusterHBHEAlpakaComparison
 
 pfClusterValidationSequence = cms.Sequence( pfClusterValidation )
 
+pfClusterAlpakaComparisonSequence = cms.Sequence( pfClusterHBHEAlpakaComparison )
+
 pfClusterCaloOnlyValidation = pfClusterValidation.clone(
     pflowClusterHCAL = 'particleFlowClusterHCALOnly'
 )
 
 pfClusterCaloOnlyValidationSequence = cms.Sequence( pfClusterCaloOnlyValidation )
+
+pfClusterHBHEOnlyAlpakaComparisonSequence = cms.Sequence( pfClusterHBHEOnlyAlpakaComparison )

Validation/RecoParticleFlow/python/pfCaloGPUComparisonTask_cfi.py

@@ -0,0 +1,14 @@
+import FWCore.ParameterSet.Config as cms
+
+from DQMServices.Core.DQMEDAnalyzer import DQMEDAnalyzer
+pfClusterHBHEOnlyAlpakaComparison = DQMEDAnalyzer("PFCaloGPUComparisonTask",
+                                                    pfClusterToken_ref = cms.untracked.InputTag('particleFlowClusterHBHEOnly'),
+                                                    pfClusterToken_target = cms.untracked.InputTag('legacyPFClusterProducerHBHEOnly'),
+                                                    pfCaloGPUCompDir = cms.untracked.string("pfClusterHBHEAlpakaV")
+)
+
+pfClusterHBHEAlpakaComparison = DQMEDAnalyzer("PFCaloGPUComparisonTask",
+                                                    pfClusterToken_ref = cms.untracked.InputTag('particleFlowClusterHBHE'),
+                                                    pfClusterToken_target = cms.untracked.InputTag('legacyPFClusterProducer'),
+                                                    pfCaloGPUCompDir = cms.untracked.string("pfClusterHBHEAlpakaV")
+)