Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Private ips setup #2

Draft
wants to merge 37 commits into
base: stage-cloud-storage-files
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
43d8cfb
Update nextflow.config
cgpu Mar 10, 2021
d207346
Rename s3Location to be cloud agnostic
cgpu Mar 10, 2021
1947089
Adds google.config
cgpu Mar 11, 2021
016346b
Update nextflow.config
cgpu Mar 11, 2021
41d0089
Adds gls info in main.nf
cgpu Mar 11, 2021
c76c322
Update google.config
cgpu Mar 11, 2021
0688754
Inits processAMachineType to false
cgpu Mar 11, 2021
0586b93
Removes machineType
cgpu Mar 11, 2021
b522e7b
Adds google.lifeSciences.usePrivateAddress
cgpu Mar 11, 2021
19589e1
Replaces s3Location with cloudStorageLocation
cgpu Mar 11, 2021
41d2be9
Update nextflow.config
cgpu Mar 11, 2021
f0a00e0
Update nextflow.config
cgpu Mar 11, 2021
a186a42
Updates publishDir; Updates s3 -> agnostic
cgpu Mar 12, 2021
a142cb2
Updates branch
cgpu Mar 12, 2021
015de36
Adds option to gls_sshDaemon
cgpu Mar 14, 2021
4cad33c
Absorbs latest changes from stage-cloud-storage-files
cgpu Apr 12, 2021
bea3be8
Adds private ip params in log.info
cgpu Apr 12, 2021
13e1fd0
Adds empty line
cgpu Apr 12, 2021
edc502b
Removes defaultBranch
cgpu Apr 23, 2021
28f6019
Removes log.info "defaultBranch"
cgpu May 12, 2021
13c1cc4
Sets default gls copyImage to gcr.io
cgpu May 12, 2021
1f7bc34
Adds sshImage and copyImage as params
cgpu May 12, 2021
db87f42
Updates container for google cloud config
cgpu May 14, 2021
4479be5
Adds awsbatch.config
cgpu May 20, 2021
fd4e2c0
Parameterises batch options
cgpu May 20, 2021
6f1ab7c
Adds aws batch params in log.info header
cgpu May 20, 2021
8308a1f
Fix param name typo
cgpu May 20, 2021
b9bafa9
Fix params name
cgpu May 21, 2021
38a3873
Update google.config
cgpu Jul 1, 2021
1561fbb
Adds aws_ignite.config in conf/
cgpu Aug 18, 2021
e18bf80
Experimentally add execution vars in print header
cgpu Aug 18, 2021
909119a
Adds defaults for ignite on AWS
cgpu Aug 18, 2021
94ff60a
Adds defaults for ignite on AWS
cgpu Aug 18, 2021
d72c3a2
Adds command-logs folder in results (.command*)
cgpu Aug 24, 2021
161df30
Adds qa.config
cgpu Sep 28, 2021
e66ab54
Update qa.config
cgpu Sep 28, 2021
6cc20ea
Updates regex * -> ** [traverse nested folders]
cgpu Oct 6, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion conf/google.config
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,6 @@ google {
zone = params.zone
network = params.network
subnetwork = params.subnetwork
lifeSciences.usePrivateAddress = params.gls_usePrivateAddress
google.lifeSciences.sshDaemon = params.gls_sshDaemon
}
cgpu marked this conversation as resolved.
Show resolved Hide resolved

8 changes: 8 additions & 0 deletions main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,14 @@ log.info "container : ${params.container}"
log.info "maxForks : ${params.maxForks}"
log.info "queueSize : ${params.queueSize}"
log.info "executor : ${params.executor}"
if(params.executor == 'google-lifesciences') {
log.info "gls_bootDiskSize : ${params.gls_bootDiskSize}"
log.info "gls_preemptible : ${params.gls_preemptible}"
log.info "gls_usePrivateAddress : ${params.gls_usePrivateAddress}"
log.info "zone : ${params.zone}"
log.info "network : ${params.network}"
log.info "subnetwork : ${params.subnetwork}"
cgpu marked this conversation as resolved.
Show resolved Hide resolved
}
log.info ""

numberRepetitionsForProcessA = params.repsProcessA
Expand Down
10 changes: 8 additions & 2 deletions nextflow.config
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ docker.enabled = true
// NOTE:
// Initialise the values of the params to the preferred default value or to false
params {
processA_memory = '4.GB'
processA_cpus = 1
processA_disk = '20.GB'
mainScript = 'main.nf'
defaultBranch = 'stage-s3-files'
config = 'conf/standard.config'
Expand All @@ -23,7 +26,6 @@ params {

echo = false
cpus = 1
processA_cpus = 1
errorStrategy = 'ignore'
container = 'quay.io/lifebitai/ubuntu:18.10'
maxForks = 200
Expand All @@ -32,6 +34,8 @@ params {
executor = 'ignite'

// google-lifesciences
gls_usePrivateAddress = false
gls_sshDaemon = false
gls_bootDiskSize = '50.GB'
gls_preemptible = true
zone = 'us-east1-b'
Expand All @@ -56,7 +60,9 @@ process {
errorStrategy = params.errorStrategy

withName: processA {
disk = params.processA_disk
cpus = params.processA_cpus
memory = params.processA_memory
}
}

Expand All @@ -70,4 +76,4 @@ manifest {
description = 'A stampede of processes at your fingertips'
mainScript = params.mainScript
defaultBranch = params.defaultBranch
}
}