azurebatch Configuration

To be used with the azurebatchdev profile by specifying the -profile azurebatchdev when running nf-core pipelines. Custom queue and storage need to be supplied with params.az_location, params.batch_name, params.storage_name, params.principal_id, params.principal_secret, params.tenant_id

Required Parameters

--storage_name

Name of Azure blob storage account.

--az_location

The Azure Batch region where the computation is executed in VMs. Default (westus2).

--principal_id

The service principal client ID.

--principal_secret

The service principal client secret.

--tenant_id

The Azure tenant ID.

-w

The Azure Blob container to be used as Nextflow work directory (-w az://work).

--autopoolmode

Whether to use Nextflow autopool mode which creates an autoscaling pool for running Nextflow jobs. Defaults to false.

--allowpoolcreation

Allow Nextflow to create a pool for running Nextflow jobs. Defaults to false.

--deletejobs

Allow Nextflow to delete pools after completion. Defaults to true.

--acr_registry

URL to Azure container registry for private docker images.

--acr_username

Username to access private Azure container registry.

--acr_password

Password to access private Azure container registry.

Azure Batch Setup

Please refer to the Nextflow documentation which describe how to setup the Azure Batch environment.

Config file

See config file on GitHub

azurebatchdev.config
// Nextflow config file for running on Azure batch
params {
    config_profile_description = 'Azure BATCH Dev Cloud Profile'
    config_profile_contact     = "Venkat Malladi (@vsmalladi)'; Abhinav Sharma (@abhi18av)"
    config_profile_url         = 'https://azure.microsoft.com/services/batch/'
 
    // Active Directory
    principal_id               = null
    principal_secret           = null
    tenant_id                  = null
 
    // Storage
    storage_name               = null
 
    // Batch
    az_location                = "westus2"
    batch_name                 = null
 
    vm_type                    = "Standard_D8s_v3"
    autopoolmode               = false
    allowpoolcreation          = true
    deletejobs                 = true
    deletepools                = false
 
    // ACR
    acr_registry               = null
    acr_username               = null
    acr_password               = null
}
 
process {
    executor = "azurebatch"
}
 
azure {
    process {
        queue = 'Standard_D2d_v4'
        withLabel: process_low {
            queue = 'Standard_D4d_v4'
        }
        withLabel: process_medium {
            queue = 'Standard_D16d_v4'
        }
        withLabel: process_high {
            queue = 'Standard_D32d_v4'
        }
        withLabel: process_high_memory {
            queue = 'Standard_D48d_v4'
        }
    }
    activeDirectory {
        servicePrincipalId     = params.principal_id
        servicePrincipalSecret = params.principal_secret
        tenantId               = params.tenant_id
    }
    storage {
        accountName = params.storage_name
    }
    batch {
        location                = params.az_location
        accountName             = params.batch_name
        tokenDuration           = "24h"
        autoPoolMode            = params.autopoolmode
        allowPoolCreation       = params.allowpoolcreation
        deleteJobsOnCompletion  = params.deletejobs
        deletePoolsOnCompletion = params.deletepools
        pools {
            Standard_D2d_v4 {
                autoScale    = true
                vmType       = 'Standard_D2d_v4'
                vmCount      = 2
                maxVmCount   = 20
                scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
                '''
            }
            Standard_D4d_v4 {
                autoScale    = true
                vmType       = 'Standard_D4d_v4'
                vmCount      = 2
                maxVmCount   = 20
                scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
                '''
            }
            Standard_D16d_v4 {
                autoScale    = true
                vmType       = 'Standard_D16d_v4'
                vmCount      = 2
                maxVmCount   = 20
                scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
                '''
            }
            Standard_D32d_v4 {
                autoScale    = true
                vmType       = 'Standard_D32d_v4'
                vmCount      = 2
                maxVmCount   = 20
                scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
                '''
            }
            Standard_D48d_v4 {
                autoScale    = true
                vmType       = 'Standard_D48d_v4'
                vmCount      = 2
                maxVmCount   = 10
                scaleFormula = '''
                $TargetLowPriorityNodes = 1;
                $TargetDedicatedNodes   = 0;
                $NodeDeallocationOption = taskcompletion;
                '''
            }
        }
    }
    registry {
        server   = params.acr_registry
        userName = params.acr_username
        password = params.acr_password
    }
}