feat: Add support for new GCP and Azure features [COMP-1463, COMP-1464]

gradle/libs.versions.toml

@@ -14,7 +14,7 @@ mockserverVersion = "5.15.0"
 picocliVersion = "4.6.3"
 shadowVersion = "9.3.1"
 slf4jVersion = "2.0.17"
-towerJavaSdkVersion = "1.114.0"
+towerJavaSdkVersion = "1.133.0"
 xzVersion = "1.10"
 
 [libraries]

src/main/java/io/seqera/tower/cli/commands/computeenvs/platforms/AzBatchForgePlatform.java

@@ -21,9 +21,9 @@
 import io.seqera.tower.cli.exceptions.CredentialsNotFoundException;
 import io.seqera.tower.model.AzBatchConfig;
 import io.seqera.tower.model.AzBatchForgeConfig;
+import io.seqera.tower.model.AzBatchPoolConfig;
 import io.seqera.tower.model.ComputeEnvComputeConfig.PlatformEnum;
 import io.seqera.tower.model.Credentials;
-import io.seqera.tower.model.JobCleanupPolicy;
 import picocli.CommandLine.ArgGroup;
 import picocli.CommandLine.Option;
 
@@ -40,15 +40,24 @@ public class AzBatchForgePlatform extends AbstractPlatform<AzBatchConfig> {
     @Option(names = {"-l", "--location"}, description = "Azure region where compute resources will be deployed (e.g., eastus, westeurope).", required = true)
     public String location;
 
-    @Option(names = {"--vm-type"}, description = "Azure VM size for compute pool. Must be a valid Azure Batch VM type. If absent, Platform defaults to Standard_D4_v3.")
+    @Option(names = {"--dual-pool"}, description = "Enable dual pool mode with separate head and worker pools. Head pool runs the Nextflow launcher on a small VM; worker pool scales independently for pipeline tasks.")
+    public boolean dualPool;
+
+    @Option(names = {"--vm-type"}, description = "Azure VM size for compute pool (single pool mode). Must be a valid Azure Batch VM type. If absent, Platform defaults to Standard_D4_v3.")
     public String vmType;
 
-    @Option(names = {"--vm-count"}, description = "Number of VMs in the Batch pool. With autoscaling enabled, this is the maximum capacity. Pool scales to zero when unused.", required = true)
+    @Option(names = {"--vm-count"}, description = "Number of VMs in the Batch pool (single pool mode). With autoscaling enabled, this is the maximum capacity. Pool scales to zero when unused.")
     public Integer vmCount;
 
-    @Option(names = {"--no-auto-scale"}, description = "Disable pool autoscaling. When disabled, pool maintains fixed VM count and does not scale based on workload.")
+    @Option(names = {"--no-auto-scale"}, description = "Disable pool autoscaling (single pool mode). When disabled, pool maintains fixed VM count and does not scale based on workload.")
     public boolean noAutoScale;
 
+    @ArgGroup(heading = "%nDual pool options (head pool):%n", validate = false)
+    public HeadPoolOptions headPoolOpts;
+
+    @ArgGroup(heading = "%nDual pool options (worker pool):%n", validate = false)
+    public WorkerPoolOptions workerPoolOpts;
+
     @Option(names = {"--preserve-resources"}, description = "Preserve Azure Batch pool resources on deletion. Keeps the compute pool and related resources when the compute environment is deleted from Seqera Platform.")
     public boolean preserveResources;
 
@@ -61,6 +70,12 @@ public class AzBatchForgePlatform extends AbstractPlatform<AzBatchConfig> {
     @Option(names = {"--wave"}, description = "Enable Wave containers. Allows access to private container repositories and on-demand container provisioning.")
     public boolean wave;
 
+    @Option(names = {"--subnet-id"}, description = "Azure VNet subnet resource ID for private network isolation. Requires Entra (service principal) credentials. Format: /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/virtualNetworks/{vnet}/subnets/{subnet}")
+    public String subnetId;
+
+    @ArgGroup(heading = "%nManaged identity options:%n", validate = false)
+    public ManagedIdentityOptions managedIdentity;
+
     @ArgGroup(heading = "%nAdvanced options:%n", validate = false)
     public AdvancedOptions adv;
 
@@ -86,17 +101,40 @@ public AzBatchConfig computeConfig(Long workspaceId, CredentialsApi credentialsA
 
         if (adv != null) {
             config
-                    .deleteJobsOnCompletion(adv.jobsCleanup)
-                    .tokenDuration(adv.tokenDuration);
+                    .deleteJobsOnCompletion(adv.deleteJobsOnCompletion)
+                    .deleteTasksOnCompletion(adv.deleteTasksOnCompletion)
+                    .terminateJobsOnCompletion(adv.terminateJobsOnCompletion)
+                    .tokenDuration(adv.tokenDuration)
+                    .jobMaxWallClockTime(adv.jobMaxWallClockTime);
         }
 
 
         AzBatchForgeConfig forge = new AzBatchForgeConfig()
-                .vmType(vmType)
-                .vmCount(vmCount)
-                .autoScale(!noAutoScale)
                 .disposeOnDeletion(!preserveResources);
 
+        if (dualPool) {
+            AzBatchPoolConfig headPool = new AzBatchPoolConfig();
+            if (headPoolOpts != null) {
+                headPool.vmType(headPoolOpts.headVmType);
+                headPool.vmCount(headPoolOpts.headVmCount);
+                headPool.autoScale(headPoolOpts.headNoAutoScale != null ? !headPoolOpts.headNoAutoScale : null);
+            }
+
+            AzBatchPoolConfig workerPool = new AzBatchPoolConfig();
+            if (workerPoolOpts != null) {
+                workerPool.vmType(workerPoolOpts.workerVmType);
+                workerPool.vmCount(workerPoolOpts.workerVmCount);
+                workerPool.autoScale(workerPoolOpts.workerNoAutoScale != null ? !workerPoolOpts.workerNoAutoScale : null);
+            }
+
+            forge.headPool(headPool);
+            forge.workerPool(workerPool);
+        } else {
+            forge.vmType(vmType)
+                    .vmCount(vmCount)
+                    .autoScale(!noAutoScale);
+        }
+
         if (registryCredentials != null && registryCredentials.size() > 0) {
 
             Map<String, String> credentialsNameToId = new HashMap<>();
@@ -119,17 +157,76 @@ public AzBatchConfig computeConfig(Long workspaceId, CredentialsApi credentialsA
         }
 
         config.forge(forge);
+        config.subnetId(subnetId);
+
+        if (managedIdentity != null) {
+            config.managedIdentityClientId(managedIdentity.managedIdentityHeadClientId);
+            config.managedIdentityPoolClientId(managedIdentity.managedIdentityPoolClientId);
+            config.managedIdentityHeadResourceId(managedIdentity.managedIdentityHeadResourceId);
+            config.managedIdentityPoolResourceId(managedIdentity.managedIdentityPoolResourceId);
+        }
 
         return config;
     }
 
+    public static class HeadPoolOptions {
+
+        @Option(names = {"--head-vm-type"}, description = "Azure VM size for the head pool (dual pool mode). If absent, defaults to Standard_D2s_v3.")
+        public String headVmType;
+
+        @Option(names = {"--head-vm-count"}, description = "Number of VMs in the head pool (dual pool mode). If absent, defaults to 1.")
+        public Integer headVmCount;
+
+        @Option(names = {"--head-no-auto-scale"}, description = "Disable autoscaling for the head pool (dual pool mode).")
+        public Boolean headNoAutoScale;
+
+    }
+
+    public static class WorkerPoolOptions {
+
+        @Option(names = {"--worker-vm-type"}, description = "Azure VM size for the worker pool (dual pool mode). If absent, defaults to Standard_D4s_v3.")
+        public String workerVmType;
+
+        @Option(names = {"--worker-vm-count"}, description = "Max number of VMs in the worker pool (dual pool mode).")
+        public Integer workerVmCount;
+
+        @Option(names = {"--worker-no-auto-scale"}, description = "Disable autoscaling for the worker pool (dual pool mode).")
+        public Boolean workerNoAutoScale;
+
+    }
+
+    public static class ManagedIdentityOptions {
+
+        @Option(names = {"--managed-identity-head-client-id"}, description = "Head job managed identity client ID (UUID). The user-assigned managed identity used by the Nextflow launcher (head job).")
+        public String managedIdentityHeadClientId;
+
+        @Option(names = {"--managed-identity-pool-client-id"}, description = "Compute job managed identity client ID (UUID). The user-assigned managed identity used by compute tasks running on Batch pool nodes.")
+        public String managedIdentityPoolClientId;
+
+        @Option(names = {"--managed-identity-head-resource-id"}, description = "Head job managed identity resource ID. Full Azure resource ID of the user-assigned managed identity for the head job. Required in Forge mode when head job managed identity client ID is specified. Format: /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{name}")
+        public String managedIdentityHeadResourceId;
+
+        @Option(names = {"--managed-identity-pool-resource-id"}, description = "Compute job managed identity resource ID. Full Azure resource ID of the user-assigned managed identity for compute jobs. Required in Forge mode when compute job managed identity client ID is specified. Format: /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{name}")
+        public String managedIdentityPoolResourceId;
+
+    }
+
     public static class AdvancedOptions {
 
-        @Option(names = {"--jobs-cleanup"}, description = "Automatic deletion of Azure Batch jobs after pipeline execution. ON_SUCCESS deletes only successful jobs. ALWAYS deletes all jobs. NEVER keeps all jobs.")
-        public JobCleanupPolicy jobsCleanup;
+        @Option(names = {"--delete-jobs-on-completion"}, description = "Delete Azure Batch jobs when the workflow completes successfully. Failed jobs are always preserved. Default: false.")
+        public Boolean deleteJobsOnCompletion;
+
+        @Option(names = {"--delete-tasks-on-completion"}, description = "Delete individual Azure Batch tasks when they complete successfully. Failed tasks are preserved. Default: true.")
+        public Boolean deleteTasksOnCompletion;
+
+        @Option(names = {"--terminate-jobs-on-completion"}, description = "Terminate Azure Batch jobs when all tasks complete. Default: true.")
+        public Boolean terminateJobsOnCompletion;
 
         @Option(names = {"--token-duration"}, description = "Duration of the SAS (shared access signature) token for Azure Blob Storage access. If absent, Platform defaults to 12h.")
         public String tokenDuration;
 
+        @Option(names = {"--job-max-wall-clock-time"}, description = "Maximum elapsed time for an Azure Batch job before automatic termination. Accepts duration syntax (e.g., '7d', '1d12h', '168h'). Defaults to 7d. Maximum: 180 days.")
+        public String jobMaxWallClockTime;
+
     }
 }

src/main/java/io/seqera/tower/cli/commands/computeenvs/platforms/AzBatchManualPlatform.java

@@ -19,7 +19,6 @@
 import io.seqera.tower.ApiException;
 import io.seqera.tower.model.AzBatchConfig;
 import io.seqera.tower.model.ComputeEnvComputeConfig.PlatformEnum;
-import io.seqera.tower.model.JobCleanupPolicy;
 import picocli.CommandLine.ArgGroup;
 import picocli.CommandLine.Option;
 
@@ -33,15 +32,24 @@ public class AzBatchManualPlatform extends AbstractPlatform<AzBatchConfig> {
     @Option(names = {"-l", "--location"}, description = "Azure region where compute resources will be deployed (e.g., eastus, westeurope).", required = true)
     public String location;
 
-    @Option(names = {"--compute-pool-name"}, description = "Pre-configured Azure Batch compute pool for running Nextflow jobs. Must include azcopy command-line tool. See Nextflow documentation for pool requirements.", required = true)
+    @Option(names = {"--compute-pool-name"}, description = "Pre-configured Azure Batch pool for the Nextflow head job. When used with --worker-pool, this pool handles only the launcher. Must include azcopy command-line tool.", required = true)
     public String computePoolName;
 
+    @Option(names = {"--worker-pool"}, description = "Pre-configured Azure Batch pool for pipeline worker tasks. When specified, the head job runs on --compute-pool-name and worker tasks run on this pool. Must be different from the head pool.")
+    public String workerPool;
+
     @Option(names = {"--fusion-v2"}, description = "Enable Fusion file system. Provides native access to Azure Blob Storage with low-latency I/O. Requires Wave containers.")
     public boolean fusionV2;
 
     @Option(names = {"--wave"}, description = "Enable Wave containers. Allows access to private container repositories and on-demand container provisioning.")
     public boolean wave;
 
+    @Option(names = {"--subnet-id"}, description = "Azure VNet subnet resource ID for private network isolation. Requires Entra (service principal) credentials. Format: /subscriptions/{sub}/resourceGroups/{rg}/providers/Microsoft.Network/virtualNetworks/{vnet}/subnets/{subnet}")
+    public String subnetId;
+
+    @ArgGroup(heading = "%nManaged identity options:%n", validate = false)
+    public ManagedIdentityOptions managedIdentity;
+
     @ArgGroup(heading = "%nAdvanced options:%n", validate = false)
     public AdvancedOptions adv;
 
@@ -57,12 +65,16 @@ public AzBatchConfig computeConfig() throws ApiException, IOException {
                 .fusion2Enabled(fusionV2)
                 .waveEnabled(wave)
                 .region(location)
-                .headPool(computePoolName);
+                .headPool(computePoolName)
+                .workerPool(workerPool);
 
         if (adv != null) {
             config
-                    .deleteJobsOnCompletion(adv.jobsCleanup)
-                    .tokenDuration(adv.tokenDuration);
+                    .deleteJobsOnCompletion(adv.deleteJobsOnCompletion)
+                    .deleteTasksOnCompletion(adv.deleteTasksOnCompletion)
+                    .terminateJobsOnCompletion(adv.terminateJobsOnCompletion)
+                    .tokenDuration(adv.tokenDuration)
+                    .jobMaxWallClockTime(adv.jobMaxWallClockTime);
         }
 
         // Common
@@ -72,16 +84,42 @@ public AzBatchConfig computeConfig() throws ApiException, IOException {
                 .nextflowConfig(nextflowConfigString())
                 .environment(environmentVariables());
 
+        config.subnetId(subnetId);
+
+        if (managedIdentity != null) {
+            config.managedIdentityClientId(managedIdentity.managedIdentityHeadClientId);
+            config.managedIdentityPoolClientId(managedIdentity.managedIdentityPoolClientId);
+        }
+
         return config;
     }
 
+    public static class ManagedIdentityOptions {
+
+        @Option(names = {"--managed-identity-head-client-id"}, description = "Head job managed identity client ID (UUID). The user-assigned managed identity used by the Nextflow launcher (head job).")
+        public String managedIdentityHeadClientId;
+
+        @Option(names = {"--managed-identity-pool-client-id"}, description = "Compute job managed identity client ID (UUID). The user-assigned managed identity used by compute tasks running on Batch pool nodes.")
+        public String managedIdentityPoolClientId;
+
+    }
+
     public static class AdvancedOptions {
 
-        @Option(names = {"--jobs-cleanup"}, description = "Automatic deletion of Azure Batch jobs after pipeline execution. ON_SUCCESS deletes only successful jobs. ALWAYS deletes all jobs. NEVER keeps all jobs.")
-        public JobCleanupPolicy jobsCleanup;
+        @Option(names = {"--delete-jobs-on-completion"}, description = "Delete Azure Batch jobs when the workflow completes successfully. Failed jobs are always preserved. Default: false.")
+        public Boolean deleteJobsOnCompletion;
+
+        @Option(names = {"--delete-tasks-on-completion"}, description = "Delete individual Azure Batch tasks when they complete successfully. Failed tasks are preserved. Default: true.")
+        public Boolean deleteTasksOnCompletion;
+
+        @Option(names = {"--terminate-jobs-on-completion"}, description = "Terminate Azure Batch jobs when all tasks complete. Default: true.")
+        public Boolean terminateJobsOnCompletion;
 
         @Option(names = {"--token-duration"}, description = "Duration of the SAS (shared access signature) token for Azure Blob Storage access. If absent, Platform defaults to 12h.")
         public String tokenDuration;
 
+        @Option(names = {"--job-max-wall-clock-time"}, description = "Maximum elapsed time for an Azure Batch job before automatic termination. Accepts duration syntax (e.g., '7d', '1d12h', '168h'). Defaults to 7d. Maximum: 180 days.")
+        public String jobMaxWallClockTime;
+
     }
 }