From eec534d425fdd5b76dbb7b2069fe67e3d1b6a976 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Tue, 25 Mar 2025 15:42:44 +0100 Subject: [PATCH 01/63] feat: setup for WOW file system --- .../src/main/nextflow/cws/CWSPlugin.groovy | 3 + .../nextflow/cws/wow/fs/WOWFileSystem.groovy | 73 ++++++++++++ .../cws/wow/fs/WOWFileSystemProvider.groovy | 109 ++++++++++++++++++ 3 files changed, 185 insertions(+) create mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystem.groovy create mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystemProvider.groovy diff --git a/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy b/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy index 069c002..b3d447a 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy @@ -1,6 +1,8 @@ package nextflow.cws import groovy.transform.CompileStatic +import nextflow.cws.wow.fs.WOWFileSystemProvider +import nextflow.file.FileHelper import nextflow.plugin.BasePlugin import nextflow.trace.TraceRecord import org.pf4j.PluginWrapper @@ -40,6 +42,7 @@ class CWSPlugin extends BasePlugin { void start() { super.start() registerTraceFields() + FileHelper.getOrInstallProvider(WOWFileSystemProvider) } } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystem.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystem.groovy new file mode 100644 index 0000000..14ce475 --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystem.groovy @@ -0,0 +1,73 @@ +package nextflow.cws.wow.fs + +import java.nio.file.FileStore +import java.nio.file.FileSystem +import java.nio.file.Path +import java.nio.file.PathMatcher +import java.nio.file.WatchService +import java.nio.file.attribute.UserPrincipalLookupService +import java.nio.file.spi.FileSystemProvider + +class WOWFileSystem extends FileSystem { + + static final WOWFileSystem INSTANCE = new WOWFileSystem() + + @Override + FileSystemProvider provider() { + return WOWFileSystemProvider.INSTANCE + } + + @Override + void close() throws IOException { + } + + @Override + boolean isOpen() { + return true + } + + @Override + boolean isReadOnly() { + return false + } + + @Override + String getSeparator() { + return "/" + } + + @Override + Iterable getRootDirectories() { + throw new UnsupportedOperationException("Root directories not supported by ${provider().getScheme().toUpperCase()} file system") + } + + @Override + Iterable getFileStores() { + throw new UnsupportedOperationException("File stores not supported by ${provider().getScheme().toUpperCase()} file system") + } + + @Override + Set supportedFileAttributeViews() { + return new HashSet<>() + } + + @Override + Path getPath(String s, String... strings) { + throw new UnsupportedOperationException("Path get not supported by ${provider().getScheme().toUpperCase()} file system") + } + + @Override + PathMatcher getPathMatcher(String s) { + throw new UnsupportedOperationException("Path matcher not supported by ${provider().getScheme().toUpperCase()} file system") + } + + @Override + UserPrincipalLookupService getUserPrincipalLookupService() { + throw new UnsupportedOperationException("User principal lookup service not supported by ${provider().getScheme().toUpperCase()} file system") + } + + @Override + WatchService newWatchService() throws IOException { + throw new UnsupportedOperationException("Watch service not supported by ${provider().getScheme().toUpperCase()} file system") + } +} diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystemProvider.groovy new file mode 100644 index 0000000..199c208 --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystemProvider.groovy @@ -0,0 +1,109 @@ +package nextflow.cws.wow.fs + +import java.nio.channels.SeekableByteChannel +import java.nio.file.AccessMode +import java.nio.file.CopyOption +import java.nio.file.DirectoryStream +import java.nio.file.FileStore +import java.nio.file.FileSystem +import java.nio.file.LinkOption +import java.nio.file.OpenOption +import java.nio.file.Path +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.attribute.FileAttribute +import java.nio.file.attribute.FileAttributeView +import java.nio.file.spi.FileSystemProvider + +class WOWFileSystemProvider extends FileSystemProvider { + + static final WOWFileSystemProvider INSTANCE = new WOWFileSystemProvider() + + @Override + String getScheme() { + return "wow" + } + + @Override + FileSystem newFileSystem(URI uri, Map map) throws IOException { + return getFileSystem(uri) + } + + @Override + FileSystem getFileSystem(URI uri) { + return WOWFileSystem.INSTANCE + } + + @Override + Path getPath(URI uri) { + return getFileSystem(uri).getPath(uri.path) + } + + @Override + SeekableByteChannel newByteChannel(Path path, Set set, FileAttribute... fileAttributes) throws IOException { + throw new UnsupportedOperationException("Byte channel not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + DirectoryStream newDirectoryStream(Path path, DirectoryStream.Filter filter) throws IOException { + throw new UnsupportedOperationException("Directory stream not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + void createDirectory(Path path, FileAttribute... fileAttributes) throws IOException { + throw new UnsupportedOperationException("Create directory not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + void delete(Path path) throws IOException { + throw new UnsupportedOperationException("Delete not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + void copy(Path path, Path path1, CopyOption... copyOptions) throws IOException { + throw new UnsupportedOperationException("Copy not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + void move(Path path, Path path1, CopyOption... copyOptions) throws IOException { + throw new UnsupportedOperationException("move not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + boolean isSameFile(Path path1, Path path2) throws IOException { + return path1 == path2 + } + + @Override + boolean isHidden(Path path) throws IOException { + return path.getFileName().startsWith(".") + } + + @Override + FileStore getFileStore(Path path) throws IOException { + throw new UnsupportedOperationException("File store not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + void checkAccess(Path path, AccessMode... accessModes) throws IOException { + } + + @Override + def V getFileAttributeView(Path path, Class aClass, LinkOption... linkOptions) { + throw new UnsupportedOperationException("File attribute view not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + def A readAttributes(Path path, Class aClass, LinkOption... linkOptions) throws IOException { + throw new UnsupportedOperationException("Read attributes not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + Map readAttributes(Path path, String s, LinkOption... linkOptions) throws IOException { + throw new UnsupportedOperationException("Read attributes not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + void setAttribute(Path path, String s, Object o, LinkOption... linkOptions) throws IOException { + throw new UnsupportedOperationException("Set attribute not supported by ${getScheme().toUpperCase()} file system provider") + } +} From e3709a7f0fc97d5ec2b62d4aaee971baf83ec4d2 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Tue, 25 Mar 2025 16:00:23 +0100 Subject: [PATCH 02/63] feat: register new trace fields --- .../src/main/nextflow/cws/CWSPlugin.groovy | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy b/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy index b3d447a..d127150 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy @@ -16,6 +16,10 @@ class CWSPlugin extends BasePlugin { private static void registerTraceFields() { TraceRecord.FIELDS.putAll( [ + infiles_time: 'num', + outfiles_time: 'num', + create_bash_wrapper_time: 'num', + create_request_time: 'num', submit_to_scheduler_time: 'num', submit_to_k8s_time: 'num', scheduler_time_in_queue: 'num', @@ -35,6 +39,19 @@ class CWSPlugin extends BasePlugin { scheduler_delta_submitted_batch_end: 'num', memory_adapted: 'mem', input_size: 'num', + out_label: 'str', + scheduler_files_bytes: 'num', + scheduler_files_node_bytes: 'num', + scheduler_files_node_other_task_bytes: 'num', + scheduler_files: 'num', + scheduler_files_node: 'num', + scheduler_files_node_other_task: 'num', + scheduler_depending_task: 'num', + scheduler_location_count: 'num', + scheduler_nodes_to_copy_from: 'num', + scheduler_no_alignment_found: 'num', + scheduler_time_delta_phase_three: 'str', + scheduler_copy_tasks: 'num', ] ) } From 74fb76bf30f7679fbd6aa603c6e97e35949a9c7d Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Tue, 25 Mar 2025 16:25:08 +0100 Subject: [PATCH 03/63] feat: WOW scheduler - configuration, k8s requests, scheduler requests --- .../src/main/nextflow/cws/CWSConfig.groovy | 4 + .../main/nextflow/cws/k8s/CWSK8sClient.groovy | 88 +++++++- .../main/nextflow/cws/k8s/CWSK8sConfig.groovy | 89 +++++++- .../nextflow/cws/k8s/CWSK8sExecutor.groovy | 205 ++++++++++++++++-- .../cws/k8s/K8sSchedulerClient.groovy | 27 +-- .../cws/k8s/WOWK8sWrapperBuilder.groovy | 119 ++++++++++ .../k8s/model/PodMountConfigWithMode.groovy | 21 ++ .../getStatsAndResolveSymlinks_linux_aarch64 | Bin 0 -> 25312 bytes .../getStatsAndResolveSymlinks_linux_x86 | Bin 0 -> 21632 bytes 9 files changed, 513 insertions(+), 40 deletions(-) create mode 100644 plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy create mode 100644 plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy create mode 100755 plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_linux_aarch64 create mode 100755 plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_linux_x86 diff --git a/plugins/nf-cws/src/main/nextflow/cws/CWSConfig.groovy b/plugins/nf-cws/src/main/nextflow/cws/CWSConfig.groovy index e83e398..3ec6f62 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/CWSConfig.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/CWSConfig.groovy @@ -30,6 +30,10 @@ class CWSConfig { return s ? new MemoryUnit(s) : null } + Integer getMaxCopyTasksPerNode() { target.maxCopyTasksPerNode as Integer } + + Integer getMaxWaitingCopyTasksPerNode() { target.maxWaitingCopyTasksPerNode as Integer } + int getBatchSize() { String s = target.batchSize as String //Default: 1 -> No batching diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sClient.groovy index 083149a..99c5603 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sClient.groovy @@ -1,15 +1,99 @@ package nextflow.cws.k8s +import groovy.json.JsonOutput import groovy.util.logging.Slf4j +import nextflow.k8s.client.ClientConfig import nextflow.k8s.client.K8sClient import nextflow.k8s.client.K8sResponseJson import nextflow.util.MemoryUnit +import org.yaml.snakeyaml.Yaml + +import java.nio.file.Path @Slf4j class CWSK8sClient extends K8sClient { - CWSK8sClient( K8sClient k8sClient ) { - super( k8sClient.config ) + CWSK8sClient(K8sClient k8sClient) { + super(k8sClient.config) + } + + CWSK8sClient(ClientConfig config) { + super(config) + } + + static private void trace(String method, String path, String text) { + log.trace "[CWS-K8s] API response $method $path \n${prettyPrint(text).indent()}" + } + + /** + * Create a pod + * + * See + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#create-55 + * https://v1-8.docs.kubernetes.io/docs/api-reference/v1.8/#pod-v1-core + * + * @param spec + * @return + */ + K8sResponseJson podCreate(String req, namespace = config.namespace) { + assert req + final action = "/api/v1/namespaces/$namespace/pods" + final resp = post(action, req) + trace('POST', action, resp.text) + return new K8sResponseJson(resp.text) + } + + K8sResponseJson podCreate(Map req, Path saveYamlPath=null, namespace = config.namespace) { + + if( saveYamlPath ) try { + saveYamlPath.text = new Yaml().dump(req).toString() + } + catch( Exception e ) { + log.debug "WARN: unable to save request yaml -- cause: ${e.message ?: e}" + } + + podCreate(JsonOutput.toJson(req), namespace) + } + + K8sResponseJson daemonSetCreate(Map req, Path saveYamlPath=null) { + if (saveYamlPath) { + try { + saveYamlPath.text = new Yaml().dump(req).toString() + } + catch (Exception e) { + log.debug "WARN: unable to save request yaml -- cause: ${e.message ?: e}" + } + } + daemonSetCreate(JsonOutput.toJson(req)) + } + + K8sResponseJson daemonSetCreate(String req) { + assert req + final action = "/apis/apps/v1/namespaces/$config.namespace/daemonsets" + log.debug "TRYING... $action" + final resp = post(action, req) + trace('POST', action, resp.text) + new K8sResponseJson(resp.text) + } + + K8sResponseJson daemonSetDelete(String name) { + assert name + final action = "/apis/apps/v1/namespaces/$config.namespace/daemonsets/$name" + final resp = delete(action) + trace('DELETE', action, resp.text) + new K8sResponseJson(resp.text) + } + + K8sResponseJson configCreateBinary(String name, Map data) { + + final spec = [ + apiVersion: 'v1', + kind: 'ConfigMap', + metadata: [ name: name, namespace: config.namespace ], + binaryData: data + ] + + configCreate0(spec) } /** diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy index 964b561..7bca2c0 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy @@ -3,9 +3,13 @@ package nextflow.cws.k8s import groovy.transform.CompileStatic import groovy.transform.Memoized import groovy.transform.PackageScope +import nextflow.exception.AbortOperationException import nextflow.k8s.K8sConfig +import nextflow.k8s.client.K8sClient +import nextflow.k8s.model.PodHostMount import nextflow.k8s.model.PodNodeSelector +import java.nio.file.Path import java.util.stream.Collectors class CWSK8sConfig extends K8sConfig { @@ -15,10 +19,50 @@ class CWSK8sConfig extends K8sConfig { CWSK8sConfig(Map config) { super(config) this.target = config + if (getLocalPath()) { + final name = getLocalPath() + final mount = getLocalStorageMountPath() + getPodOptions().mountHostPaths.add(new PodHostMount(name, mount)) + } } K8sScheduler getScheduler(){ - return target.scheduler ? new K8sScheduler( (Map)target.scheduler ) : null + return target.scheduler || locationAwareScheduling() ? new K8sScheduler( (Map)target.scheduler ) : null + } + + Storage getStorage() { + locationAwareScheduling() ? new Storage((Map) target.storage, getLocalClaimPaths()) : null + } + + String getLocalPath() { + target.localPath as String + } + + String getLocalStorageMountPath() { + target.localStorageMountPath ?: '/workspace' as String + } + + boolean locationAwareScheduling() { + getLocalClaimPaths().size() > 0 + } + + Collection getLocalClaimPaths() { + getPodOptions().mountHostPaths.collect { it.mountPath } + } + + String findLocalVolumeClaimByPath(String path) { + def result = getPodOptions().mountHostPaths.find { path.startsWith(it.mountPath) } + return result ? result.hostPath : null + } + + void checkStorageAndPaths(K8sClient client, String pipelineName) { + super.checkStorageAndPaths(client) + //The nextflow project/workflow has to be on a shared drive + if (pipelineName && pipelineName[0] == '/' && !findVolumeClaimByPath(pipelineName)) + throw new AbortOperationException("Kubernetes `pipelineName` must be a path mounted as a persistent volume -- projectDir=$pipelineName; volumes=${getClaimPaths().join(', ')}") + + if (getStorage() && !findLocalVolumeClaimByPath(getStorage().getWorkdir())) + throw new AbortOperationException("Kubernetes `storage.workdir` must be a path mounted as a local volume -- storage.workdir=${getStorage().getWorkdir()}; volumes=${getLocalClaimPaths().join(', ')}") } @CompileStatic @@ -91,4 +135,45 @@ class CWSK8sConfig extends K8sConfig { } -} + @CompileStatic + @PackageScope + static class Storage { + + @Delegate + Map target + Collection localClaims + + Storage(Map scheduler, Collection localClaims ) { + this.target = scheduler + this.localClaims = localClaims + } + + String getCopyStrategy() { + target.copyStrategy as String ?: 'ftp' + } + + String getWorkdir() { + Path workdir = (target.workdir ?: localClaims[0]) as Path + if( ! workdir.getName().equalsIgnoreCase('localWork') ){ + workdir = workdir.resolve( 'localWork' ) + } + return workdir.toString() + } + + PodNodeSelector getNodeSelector(){ + return target.nodeSelector ? new PodNodeSelector( target.nodeSelector ) : null + } + + boolean deleteIntermediateData(){ + target.deleteIntermediateData as Boolean ?: false + } + + String getImageName() { + target.imageName ?: 'commonworkflowscheduler/ftpdaemon:v1.0' + } + + String getCmd() { + target.cmd as String + } + } +} \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy index c64026a..3165fb7 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy @@ -1,5 +1,6 @@ package nextflow.cws.k8s +import com.google.common.hash.Hashing import groovy.transform.CompileStatic import groovy.transform.Memoized import groovy.transform.PackageScope @@ -7,11 +8,15 @@ import groovy.util.logging.Slf4j import nextflow.cws.CWSConfig import nextflow.cws.CWSSchedulerBatch import nextflow.cws.SchedulerClient +import nextflow.cws.k8s.model.PodMountConfigWithMode import nextflow.cws.processor.CWSTaskPollingMonitor import nextflow.k8s.K8sConfig import nextflow.k8s.K8sExecutor import nextflow.k8s.client.K8sClient +import nextflow.k8s.client.K8sResponseException +import nextflow.k8s.model.PodHostMount import nextflow.k8s.model.PodOptions +import nextflow.k8s.model.PodVolumeClaim import nextflow.processor.TaskHandler import nextflow.processor.TaskMonitor import nextflow.processor.TaskRun @@ -19,30 +24,32 @@ import nextflow.util.Duration import nextflow.util.ServiceName import org.pf4j.ExtensionPoint +import java.nio.file.Paths + @Slf4j @CompileStatic @ServiceName('k8s') class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { - @PackageScope SchedulerClient schedulerClient @PackageScope CWSSchedulerBatch schedulerBatch protected CWSK8sClient client + /** + * Name of the created daemonSet + */ + private String daemonSet = null @Override @Memoized protected K8sConfig getK8sConfig() { return new CWSK8sConfig( (Map)session.config.k8s ) } - @Memoized @PackageScope CWSConfig getCWSConfig(){ new CWSConfig(session.config.navigate('cws') as Map) } - @PackageScope CWSK8sClient getCWSK8sClient() { client } - /** * @return A {@link nextflow.processor.TaskMonitor} associated to this executor type */ @@ -54,7 +61,6 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { } return CWSTaskPollingMonitor.create( session, name, 100, Duration.of('5 sec'), this.schedulerBatch ) } - /** * Creates a {@link nextflow.processor.TaskHandler} for the given {@link nextflow.processor.TaskRun} instance * @@ -69,41 +75,63 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { return new CWSK8sTaskHandler( task, this ) } + @Override + protected K8sClient getClient() { + return client + } + @Override protected void register() { super.register() - this.client = new CWSK8sClient(super.getClient()) + final k8sConfig = getK8sConfig() + final clientConfig = k8sConfig.getClient() + this.client = new CWSK8sClient(clientConfig) + log.debug "[K8s] config=$k8sConfig; API client config=$clientConfig" - CWSK8sConfig.K8sScheduler cwsK8sConfig = (k8sConfig as CWSK8sConfig).getScheduler() - CWSConfig cwsConfig = new CWSConfig(session.config.navigate('cws') as Map) + final CWSK8sConfig cwsK8sConfig = k8sConfig as CWSK8sConfig + + if ( cwsK8sConfig.locationAwareScheduling() ) { + createDaemonSet() + registerGetStatsConfigMap() + } + + CWSK8sConfig.K8sScheduler k8sSchedulerConfig = cwsK8sConfig.getScheduler() + final CWSConfig cwsConfig = new CWSConfig(session.config.navigate('cws') as Map) Map data - if ( !cwsK8sConfig && !cwsConfig.dns ) { + if ( !k8sSchedulerConfig && !cwsConfig.dns ) { //Use default configuration - cwsK8sConfig = CWSK8sConfig.K8sScheduler.defaultConfig( k8sConfig ) + k8sSchedulerConfig = CWSK8sConfig.K8sScheduler.defaultConfig( k8sConfig ) } - if( cwsK8sConfig ) { + if( k8sSchedulerConfig ) { + final PodOptions podOptions = cwsK8sConfig.getPodOptions() schedulerClient = new K8sSchedulerClient( cwsConfig, + k8sSchedulerConfig, cwsK8sConfig, - k8sConfig, - k8sConfig.getNamespace(), + cwsK8sConfig.getNamespace(), session.runName, client, - k8sConfig.getPodOptions().getVolumeClaims() + podOptions.getVolumeClaims(), + podOptions.getMountHostPaths() ) - final PodOptions podOptions = k8sConfig.getPodOptions() Boolean traceEnabled = session.config.navigate('trace.enabled') as Boolean + CWSK8sConfig.Storage storage = cwsK8sConfig.getStorage() data = [ - volumeClaims : podOptions.volumeClaims, + volumeClaims : podOptions.getVolumeClaims(), traceEnabled : traceEnabled, costFunction : cwsConfig.getCostFunction(), memoryPredictor : cwsConfig.getMemoryPredictor(), maxMemory : cwsConfig.getMaxMemory()?.toBytes(), minMemory : cwsConfig.getMinMemory()?.toBytes(), - additional : cwsK8sConfig.getAdditional() + additional : k8sSchedulerConfig.getAdditional(), + workDir : storage?.getWorkdir(), + copyStrategy : storage?.getCopyStrategy(), + locationAware : cwsK8sConfig.locationAwareScheduling(), + maxCopyTasksPerNode : cwsConfig.getMaxCopyTasksPerNode(), + maxWaitingCopyTasksPerNode : cwsConfig.getMaxWaitingCopyTasksPerNode() ] } else { data = [ @@ -116,7 +144,6 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { this.schedulerBatch?.setSchedulerClient( schedulerClient ) schedulerClient.registerScheduler( data ) } - @Override void shutdown() { final CWSK8sConfig.K8sScheduler schedulerConfig = (k8sConfig as CWSK8sConfig).getScheduler() @@ -127,6 +154,146 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { log.error( "Error while closing scheduler", e) } } + + if( daemonSet ){ + try { + def result = client.daemonSetDelete( daemonSet ) + log.trace "$result" + } catch (K8sResponseException e){ + log.error("Couldn't delete daemonset: $daemonSet", e) + } + } + log.trace "Close K8s Executor" + } + + protected void registerGetStatsConfigMap() { + Map configMap = [:] + + String architectureStatFileName + final String architecture = System.getProperty("os.arch"); + if (architecture == "amd64" || architecture == "x86_64") { + architectureStatFileName = "/nf-cws/getStatsAndResolveSymlinks_linux_x86" + } else if (architecture == "aarch64" || architecture == "arm64") { + architectureStatFileName = "/nf-cws/getStatsAndResolveSymlinks_linux_aarch64" + } else { + throw new RuntimeException("The ${architecture} architecture is by default not supported for WOW." + + "You may compile the getStatsAndResolveSymlinks.c yourself and add it to the resources directory.") + } + final contentStream = CWSK8sExecutor.class.getResourceAsStream(architectureStatFileName) + final content = contentStream.bytes.encodeBase64().toString() + configMap[WOWK8sWrapperBuilder.statFileName] = content + + String configMapName = makeConfigMapName(content) + tryCreateConfigMap(configMapName, configMap) + log.debug "Created K8s configMap with name: $configMapName" + k8sConfig.getPodOptions().getMountConfigMaps().add( new PodMountConfigWithMode(configMapName, '/etc/nextflow', 0111) ) + } + + protected void tryCreateConfigMap(String name, Map data) { + try { + client.configCreateBinary(name, data) + } + catch( K8sResponseException e ) { + if( e.response.reason != 'AlreadyExists' ) + throw e + } + } + + protected static String makeConfigMapName(String content ) { + "nf-get-stat-${hash(content)}" + } + + protected static String hash(String text) { + def hasher = Hashing .murmur3_32() .newHasher() + hasher.putUnencodedChars(text) + return hasher.hash().toString() + } + + private void createDaemonSet(){ + + final K8sConfig k8sConfig = getK8sConfig() + final PodOptions podOptions = (k8sConfig as CWSK8sConfig).getPodOptions() + final mounts = [] + final volumes = [] + int volume = 1 + + // host mounts + for( PodHostMount entry : podOptions.mountHostPaths ) { + final name = 'vol-' + volume++ + mounts << [name: name, mountPath: entry.mountPath] + volumes << [name: name, hostPath: [path: entry.hostPath]] + } + + final namesMap = [:] + + // creates a volume name for each unique claim name + for( String claimName : podOptions.volumeClaims.collect { it.claimName }.unique() ) { + final volName = 'vol-' + volume++ + namesMap[claimName] = volName + volumes << [name: volName, persistentVolumeClaim: [claimName: claimName]] + } + + // -- volume claims + for( PodVolumeClaim entry : podOptions.volumeClaims ) { + //check if we already have a volume for the pvc + final name = namesMap.get(entry.claimName) + final claim = [name: name, mountPath: entry.mountPath ] + if( entry.subPath ) + claim.subPath = entry.subPath + if( entry.readOnly ) + claim.readOnly = entry.readOnly + mounts << claim + } + + String name = "mount-${session.runName.replace('_', '-')}" + def spec = [ + containers: [ [ + name: name, + image: (k8sConfig as CWSK8sConfig).getStorage().getImageName(), + volumeMounts: mounts, + imagePullPolicy : 'IfNotPresent' + ] ], + volumes: volumes, + serviceAccount: client.config.serviceAccount + ] + + CWSK8sConfig.Storage storage = (k8sConfig as CWSK8sConfig).getStorage() + if( storage.getNodeSelector() ) + spec.put( 'nodeSelector', storage.getNodeSelector().toSpec() as Serializable ) + + def pod = [ + apiVersion: 'apps/v1', + kind: 'DaemonSet', + metadata: [ + labels: [ + app: 'nextflow' + ], + name: name, + namespace: k8sConfig.getNamespace() ?: 'default' + ], + spec : [ + restartPolicy: 'Always', + template: [ + metadata: [ + labels: [ + name : name, + app: 'nextflow' + ] + ], + spec: spec, + ], + selector: [ + matchLabels: [ + name: name + ] + ] + ] + ] + + daemonSet = name + client.daemonSetCreate(pod, Paths.get('.nextflow-daemonset.yaml') ) + log.trace "Created daemonSet: $name" + } -} +} \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy index 6aa3ac6..a47c6ef 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy @@ -5,21 +5,20 @@ import nextflow.cws.CWSConfig import nextflow.cws.SchedulerClient import nextflow.exception.NodeTerminationException import nextflow.k8s.K8sConfig -import nextflow.k8s.client.K8sClient import nextflow.k8s.client.K8sResponseException +import nextflow.k8s.model.PodHostMount import nextflow.k8s.model.PodSecurityContext import nextflow.k8s.model.PodSpecBuilder import nextflow.k8s.model.PodVolumeClaim - import java.nio.file.Paths - @Slf4j class K8sSchedulerClient extends SchedulerClient { private final CWSK8sConfig.K8sScheduler schedulerConfig - private final K8sClient k8sClient + private final CWSK8sClient k8sClient private final K8sConfig k8sConfig private final String namespace + private final Collection hostMounts private final Collection volumeClaims private String ip @@ -29,10 +28,12 @@ class K8sSchedulerClient extends SchedulerClient { K8sConfig k8sConfig, String namespace, String runName, - K8sClient k8sClient, - Collection volumeClaims + CWSK8sClient k8sClient, + Collection volumeClaims, + Collection hostMounts ) { super( config, runName ) + this.hostMounts = hostMounts ?: [] this.volumeClaims = volumeClaims this.k8sClient = k8sClient this.k8sConfig = k8sConfig @@ -52,14 +53,13 @@ class K8sSchedulerClient extends SchedulerClient { } data.dns = getDNS() data.namespace = namespace + data.localClaims = hostMounts super.registerScheduler(data) } private void startScheduler(){ - boolean start = false Map state - try{ //If no pod with the name exists an exceptions is thrown state = k8sClient.podState( schedulerConfig.getName() ) @@ -69,7 +69,6 @@ class K8sSchedulerClient extends SchedulerClient { log.info "Scheduler ${schedulerConfig.getName()} is terminated" } else if( state.running || state.waiting ) log.trace "Scheduler ${schedulerConfig.getName()} is already running" else log.error "Unknown state for ${schedulerConfig.getName()}: ${state.toString()}" - } catch ( K8sResponseException e ) { log.error( "Got unexpected HTTP error ${e.response} while checking scheduler's state", e.message ) } catch ( NodeTerminationException ignored){ @@ -77,7 +76,6 @@ class K8sSchedulerClient extends SchedulerClient { start = true log.info "Scheduler ${schedulerConfig.getName()} can not be found and will be started..." } - if( start ){ log.trace "Scheduler ${schedulerConfig.getName()} is not running, let's start" final builder = new PodSpecBuilder() @@ -91,25 +89,21 @@ class K8sSchedulerClient extends SchedulerClient { .withLabel('component', 'scheduler') .withLabel('tier', 'control-plane') .withLabel('app', 'nextflow') + .withHostMounts( hostMounts ) .withVolumeClaims( volumeClaims ) if( schedulerConfig.getNodeSelector() ) builder.setNodeSelector( schedulerConfig.getNodeSelector() ) - if ( schedulerConfig.getWorkDir() ) builder.withWorkDir( schedulerConfig.getWorkDir() ) - if( schedulerConfig.getCommand() ) builder.withCommand( schedulerConfig.getCommand() ) - if( schedulerConfig.runAsUser() != null ){ builder.securityContext = new PodSecurityContext( schedulerConfig.runAsUser() ) } - //This is required to use the PodSpecBuilder as it is builder.command = [ "delete this" ] Map pod = builder.build() - List env = [[ name: 'SCHEDULER_NAME', value: schedulerConfig.getName() @@ -117,13 +111,12 @@ class K8sSchedulerClient extends SchedulerClient { name: 'AUTOCLOSE', value: schedulerConfig.autoClose() as String ]] - Map container = pod.spec.containers.get(0) as Map container.put('env', env) container.remove( 'command' ) (container.resources as Map)?.remove( 'limits' ) - k8sClient.podCreate( pod, Paths.get('.nextflow-scheduler.yaml') ) + k8sClient.podCreate( pod, Paths.get('.nextflow-scheduler.yaml'), namespace) } //wait for scheduler to get ready diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy new file mode 100644 index 0000000..e249cd9 --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy @@ -0,0 +1,119 @@ +package nextflow.cws.k8s + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.file.FileHelper +import nextflow.k8s.K8sWrapperBuilder +import nextflow.processor.TaskRun +import nextflow.util.Escape +import java.nio.file.Path + +/** + * Implements a BASH wrapper for tasks executed by kubernetes cluster + * + * @author Paolo Di Tommaso + */ +@CompileStatic +@Slf4j +class WOWK8sWrapperBuilder extends K8sWrapperBuilder { + + CWSK8sConfig.Storage storage + Path localWorkDir + + static String statFileName = "getStatsAndSymlinks" + + WOWK8sWrapperBuilder(TaskRun task, CWSK8sConfig.Storage storage) { + this(task) + this.storage = storage + if( storage ){ + switch (storage.getCopyStrategy().toLowerCase()) { + case 'copy': + case 'ftp': + if ( this.scratch == null || this.scratch == true ){ + //Reduce amount of local data + this.scratch = (storage.getWorkdir() as Path).resolve( "scratch" ).toString() + this.stageOutMode = 'move' + } + break + } + if ( !this.targetDir || workDir == targetDir ) { + this.localWorkDir = FileHelper.getWorkFolder(storage.getWorkdir() as Path, task.getHash()) + this.targetDir = this.localWorkDir + } + } + } + + WOWK8sWrapperBuilder(TaskRun task) { + super(task) + this.headerScript = "NXF_CHDIR=${Escape.path(task.workDir)}" + } + /** + * only for testing purpose -- do not use + */ + protected K8sWrapperBuilder() {} + + @Override + protected boolean shouldUnstageOutputs() { + return localWorkDir || super.shouldUnstageOutputs() + } + + private String getStorageLocalWorkDir() { + String localWorkDir = storage.getWorkdir() + if ( !localWorkDir.endsWith("/") ){ + localWorkDir += "/" + } + localWorkDir + } + + @Override + protected Map makeBinding() { + final Map binding = super.makeBinding() + + binding.K8sResolveSymlinks = null + + if ( binding.stage_inputs && storage && localWorkDir ) { + final String cmd = """\ + # create symlinks + if test -f "${workDir.toString()}/.command.symlinks"; then + bash "${workDir.toString()}/.command.symlinks" || true + fi + """.stripIndent() + binding.stage_inputs = cmd + binding.stage_inputs + } + + if ( localWorkDir ) { + binding.unstage_outputs = copyStrategy.getUnstageOutputFilesScript(outputFiles, localWorkDir) + } + + return binding + } + + @Override + protected String getLaunchCommand(String interpreter, String env) { + String cmd = '' + if( storage && localWorkDir ){ + cmd += "local INFILESTIME=\$(/etc/nextflow/${statFileName} infiles \"${workDir.toString()}/.command.infiles\" \"${getStorageLocalWorkDir()}\" \"\$PWD/\" || true)\n" + } + cmd += super.getLaunchCommand(interpreter, env) + if( storage && localWorkDir && isTraceRequired() ){ + cmd += "\nlocal exitCode=\$?" + cmd += """\necho \"infiles_time=\${INFILESTIME}" >> ${TaskRun.CMD_TRACE}\n""" + cmd += "return \$exitCode\n" + } + return cmd + } + + @Override + String getCleanupCmd(String scratch) { + String cmd = super.getCleanupCmd( scratch ) + if( storage && localWorkDir ){ + cmd += "mkdir -p \"${localWorkDir.toString()}/\" || true\n" + cmd += "local OUTFILESTIME=\$(/etc/nextflow/${statFileName} outfiles \"${workDir.toString()}/.command.outfiles\" \"${getStorageLocalWorkDir()}\" \"${localWorkDir.toString()}/\" || true)\n" + if ( isTraceRequired() ) { + cmd += "echo \"outfiles_time=\${OUTFILESTIME}\" >> ${workDir.resolve(TaskRun.CMD_TRACE)}" + } + } + return cmd + } + +} \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy new file mode 100644 index 0000000..d3b8372 --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy @@ -0,0 +1,21 @@ +package nextflow.cws.k8s.model; + +import nextflow.k8s.model.PodMountConfig; + +public class PodMountConfigWithMode extends PodMountConfig { + private Integer mode; + + public PodMountConfigWithMode(String config, String mount, Integer mode = null) { + super(config, mount); + this.mode = mode; + } + + public Integer getMode() { + return mode; + } + + public void setMode(Integer mode) { + this.mode = mode; + } + +} \ No newline at end of file diff --git a/plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_linux_aarch64 b/plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_linux_aarch64 new file mode 100755 index 0000000000000000000000000000000000000000..44c4e37c7f335418a7742cb39761856684465c6b GIT binary patch literal 25312 zcmeHv3zSsVneIMys=BIQ)lX;;rB6d6i1veryd$TZmyGy=j+il-qPnWOyQI3Rsj7xX z$j~6sOvs(k1B!+y1)7OsGQ*0=)t*Tr(KzE8$pl5c#HeY|j7$>OMKla$#G3ElkE&DE zRT|g5v)0^o-QDNx{eS!a|NZZOzs}jG&e^tX)pDDrF%@j=2}YDzZgNPR85{WmCnYwI z6*H4HvdOFnh#iN?b0`)m6{I%1NWXA{6Yo5>R@o*-bRVr3!44w)w+Q&J*s zoGM5aGhQ2UCR@)miTpE7Z9Jjkabys}G#M*7GLPggNVyAAPQ@eAFcqu%QQhe1koD`Z znV^U^DPjgcXJX=|aMN$%e(sk>LC+4-l2O3AodmRHa((*AKW zmepk~pASfqJ|s5j{+-W4}|o|i;!b@G%rRUtBPxCP1T z5Rb=kiKH(>JPt=K4yxCsI7)Gh!9ltv;;6upFGy3fvZ!5(vycoyc9q~DUzX!gbtD_> zag^bp`c&hn!7&yG^-F6%q%w(@WO+LXCEOIk=bzg}RVWx%BZu_ohIRn?a8jQSzq
IecXuul^MSRTL8Hqw79`kS($`3)90I>L3 zZ=lx)_AZgzD}+e6&l~SyR1j$;_E;beX@4LZCFOohUF$_@D86e`G#C$%6~6VJ?m#>q z>tXb*gjJXi&?u~?{n1?)N`x}x)(!MXCEL< z*gj6%n1dz|gpn4}J{l&!toc;78%>Hdf2BjK;I{#nDtMO5g7ZmF3RGI~^wNsFaTc8B zT@?%qZk=DJS#UnTN`Y1jZe3sIS@3khTNwEn1Ys|BaIOoeq8 zoYoQ*dMvnF^C+*+f?NGRV8K;e691qDr}asNuUc^H{I=VI)7quNeHNVNL=_SioYq&tMHr!SJwcPcie)jYeB+KT5#*aI%2`;8bXCL z7M!jfRPZ$FFJIj9)T0-5Hgt%wy%)Fq^pT^w=M|NI7J0j2M%70vl9i_y4;i-lSLzM@ zWpyC$Y?aMSR@xZP<8%?|Nfx>Ubi0LigYLA@)u4A+=&_&=S?EhZKW(8Wf*Y?<wL92dHD5m~%85?|YB97na?C>ewOrD<1hW-ux39~CM6ONu^uEaUky5pPCAnPw2F9MB^x?J>ZDrecHGJ5%=cnMue0{wAEi>84 zWo>C$D)Zu&pYXBqUMWlNL7D3&Eu9aYSHA%}4K}!X-6Z?`_e#y=#Vt?saq|}Jy}0Ga zTxO)64Kr=<#oNF;Qpb`bb!JjygVnHgc%*)CnEih1i4NqQg5P+aUGSW$H;1tv4HI^8 z%QIYWY6*K`#EE>TnZztUTyJMHM(Pe+f*XTsFK+oqQ7+0lRl|~}D}+AchrTM( z$L!`Xe73$#XB~=W1NCc8kd0FXzs+uPTZxx#HTSSHTTXBrb=%?D%HFA{i?mmR{t=?7 zUMS-_u8VYxE}!$%n91`WrS@TM9jYbX$t-DOJ6=2md}OkjTtirenS4j`loDMpd4P}9 zo5?>TjQpu=g$>+R_|#x**D2^hS-c(vd7wwt;o_EOxt~>gkC5+B)*+OG^?$>l#yZgM z$EokZpUF4TW>4D&Uu;0XXN(FDKN}i*ZK4u3W@#x^8*am%E#v~d0;d>!N zb-GcOt)WijKl0JYLWL2L4Coh5*WzoEZc6@go+KAHU38(oK{{BS{54AO}m1_s( ze8Kw#jRR^ER?6G_a@auaH;>v6`c6X!&p%p*zKQWdIx**wFQ{A%^EIxgvvjKR(_>;p z=&q2u8*=zhLoY951Z80CI3!;k*)Ub~`vlffp0=|M40WY4Ps4^&u%F7qwfo+Y8`&`K zHL~y?P2e=P$>zTy`)?#4t9kXXSVz}rkN)8{+oS*C(;rQ)w?F#FnB&pZGwg-LVwOCJ zetT#xOETNw>ckRl*oLuma7o*6VzHUL3;GfVSTb=y_~W2utdM_+huR|Ti*@C^j>ouP z;b)E2Y8dAv|2XN|4_*6(E-&g#b3hAto?5F78%UE66MxT=gVzkcxSsg;vE(JlPwYWC zKQcGhk7FC!AUB4vwQYRPSO=QqvG-D&*vECxPxH584MD%CM_lGc4F3*-zupjjUJrj# z+h9jnqI-4GxrpIqCd|seRN|2a%@o#zKzf$pmi;T2CvuA6ivCrC+F>Yb@>kmeg}fuQ^P5 z>{8DN#!EYVco6MHZ9a+YqPByN$ab0!$WGoD(9Yc7Xgk~^X6!Z04X03V-Y(MrG_E+q)i9QSjQnX_w#nc`~v%_-bYJW2aPG>|A1^oe5aIa zK))gSm!NswQIAVe55isoM*aR!+7Fi=9)T~OmU_2eOwks6+Lo}?+2+DH$2IzU(hpCQ z9<0;+I*c&*=w0YgKKmM_C!jx|%qL`-Um>hc8$LJzW3;XfO<&@8Op4M%&!BFB(jA8gfBVW{QAI5yQOO;J>)V~sUqQ8@!HQR@Ie*;Z*I}B`_ z4LTt2B6;c?v(2Q#k}d+Ra5fKP3p@?L8XDnmq}!2hpJpZ>LfVJ4?-c8eeiDL(++3mBVN9Kc;x!wLEnVz zdD8P3{Q20YTR#@T7WlV%7V5+2PsmRu`!Gi*(2mqLYW4gRd39YN+4A&P%C|!~^Yjy9h7^K=8uQu zx}(amtUJf@tUDVZ!{;-aU!RckV7E5BQQqo>^BCj|7g>XU5`+H)_Me*%fXKY*CUkijuoGtUx z>k;^G;WnAcU&}nc9s$1^IIsHwxgG&)1x91~qYtwC?nEh@wH|^0UnHKj9s!?7daUaa zd~`i{mCybS`V%ea58$nlWu7AJN~}jMZNrHx<$5#~JX3Slqi_6QUXL*MqVEkQc8T%8 z*P|=tdL+wEuSdJk-^tFYay}X(FMku zxYxJd2*!*+Z(n>fE(|^&^L7Uo7*k@#ZNYF?FcgSwjr7NP;!DP&Xdu=fihGC*dwT=d z7>n>6%p3CH$r;i(d*Ts1KBK4n*Oc&LWjXOk#OMlaGQ835{@y@19%~TUjOQEtL0Y)m zi1!2xdZK1Tx(vT}GkByCOc|l-BK2+zL}S57*y!z##f{E@F?C98su79un3pgm6be#8 z*?|JM$p(xK_~Mc1W+CH`1Y$-w5;ySNF&1x@)@8|MB;pZ6RI;$_CTXasA3YQd%KDM> z;9psS$d5<8ye5(*Q&O5Q4N1MxKzD!08#PE9L9xxfosm$`XYi+$(r$wuU*<1290@n& zID%{L53dhLHig-g*up8ng>(+7h^IoqOhi=xh5r)piwb4r1fWgO7h*iHDjcqVPo=(# zcmVNH#E&3;3Go@kXAqD3Z7Nl!;Wr$J4a9wjO~iW<--`I;?^3AlplO5KO}eZ)^9 zzV@9|>LlW0|2vhsfVlPDREmCwa^!p}^+m*IK1`(&h4B05i)}XwLCSv%$F1;P2r=@xJztP|Dc=schmn88l7G7;zY+Pf;d9{& zA+PKrc`COD`8&y{ZSFzsfyxIfYzMdo%J(Dx7}|#7w0&zu0VbfGz$?*agY*Em%{^j! z;Hn3&Y%9NP`JU-}uefL0y;JvHe&3Y)8^1Yu|Dg4^pYENJEGDfa^hL`<|k`jwScl@70&>v)!km+M--)6yE-j`8s4&JCf*U;WH0> z=7G;V@RU84AZ`g08%tWPvdazfw0kB9V}<2S402 zx^SaOcdTtqebETNt!u-LFMb~Ci}yzZ#*EhHSxqxrXU=GDooh_b(kn?VW23Xc{**){L|2aoJ2F|6L5H{Q0Z6{0v-qi~POi zCRd>PJCp5A*SZwc_%0LVsI#ztOk&V5F;Va6ol;tUFk4Sk%=c<{` zOna)_uPU*8+;0Wu=5REm*xjjW?fkvCl zd4?jbh~7NWi*V8|ucKcI+H8&(a`YaYv}u<%0@8m=*;mxAN7`<8;J3g`Uqac_$6iSi zF2@3{%e9zv{T?UVB{d!rbP+JF#-#M^q;-7l8cHvt^d+@XN9GdT+map9jS5`o!z?#Tu0!&9Hg^5LEyvCTk;qO>FoX?fkz2E%0W82 z{{<(PNwzu9x$2NvOdA5u_oZ#8fH?n9aTE-;;z}aU*M1v>y?7}R$(m*85H42oGS^6F z_oc|FyalIAotoy_ibPS2cy&$aEP9*@uxmnT(G5g7G&i-1y$Cy!%vq$3nNDhpzD9XY zkvEQ&Ohsn7i_Y%tI8|JZ?2@l@kk0OV3H&mF2RTS*_hST(C-8rAkj`%MLdCNLKF>iq zyI&^oiv*5vkj`%EkQKDoQSt{4(%GHjNoq-V37k>xx{H!ycyd1_X;hTE@Y@UKzLF=Y zEcYCqBqzJuc#@psrcPDvq7mb!K2q-bTS|8FuPkunjzeZB0 zdO0WZ&h4r~+U~4(QL`0?D5Ig`ex$+nK2ck$A4a;W()p_61Hjc}qw}>Ap8f!qI$tj> zK?79R68=W%B}lW<`*0$Z_sI%YN-8La(@s`O4M@$#X`Q(SQBtMzq>Fk;@sCOCuZxdy zBT8xXe_pXVS&iXpci%;9e+M?#Qa6oDRy-aGT+3X$h_N^fN!Rky{U9n?2?_H-LbNQf z^y?6MEJN3lyIm`EQfx2QJ8UNM5op4+vx!uan8MxwA5CFb0kXwFXmzCvKo{#zHkgRv zH8B<0XAwa^X11|4T%OJphNKEHNq!5codr0@Z;(^=s?5%v%W0C3bDI7W@Yr6cTLx|R z**K|rQ_Z5*X~q!(106-o>7uF4bvI7M{Pfq9q7xd6PoPfMUJy>#H*v-cO*3cdV<=2( zs_+6Tr72x&;pwtx316or8cgI+B}CA^ReqSbmTIa7ThlpvPzQH{=Z+-<)ODo$A2>Fd zOt}@uY8>vDN&nJT!Vcqj7)QmoiEnMCiSz{==W$eiQ5rytA%(A?kg6clZml5KOa?Up zNA;j&tI=+<-NkCOn{9XFyxR6vFx4s@WFQ;!1C&CCHm_W^;5VpyVOJHe0D|E-@-oYu zR)x8`N~_j%_XPJ8cMVo!rd4T3l@d~ki&tACD=obSB)Z2~rn?uEE-am2x)P7i>@o){ zuA@dPEk%xFeyN6>qH@6qa%IgtTv0ovBnMzl4GQD}P(=d8Qh;kFfs)G^6;Y~bD{F|M ztO0ZjGRiR`dFk%CoTH)!MJ>WjL}gm1G@`0b*d&Xnwibcc?WDyrw|2IrYG2h#ZFKpa z>j^8*;_<|yXIx+x7DEUw{d`e=hr-p)PjISG<4_eZYJee5RcY54ZR&XLFbA)FNv8bL zZ0pD+WzB5yc-3N4$5Y|vxP9%&DR0*D=8$T)!{Q2C-4pHT1n7=R&&nE!RhGg()lL$r zLbgkF44Ga1S35K?z|&vbfoDqNhT7q;T7dZZ$_&ViA>$T}54)9O?(kgStp%zxy5O~I ze{`2K4S^W(-`w01+Z^);`eH5eEm1twpqHeY*HAt>l}%ma=}p??hROKX#~RmNck^UC z-R?`Bd931g^wJA+Ml z0u^ZXQFYm596F;lXHJ2+cwJ~m6$(k}Hyf6Fo`=7j#d}{ z`f;XB%E0O#K!1~9&b%C6(AoF$mEPj_#=R}K2g2)v;aJP2NOXOy&+7}c;2x?8j|0U$ zRExMRYU%99TXO34V0a-MkZ%lkhx=O~AIIx@{8i#^pHKCVTsFGjYMudZU#Q<7U<|HD z(Z-y~8z{dPBi@JQQf6w=BJ3(CF=-3-!&JZ8UWoO*hUon$SR>Lu*60sSEME z`Di?h!vM(MK(ss1gf7tJi%?&{^JAdVNI#T?yP+%ILtKcO0?}wV0wNrNLOk21TXVW0 z$E|u(S3DGJ>W%mVq0dpj2zoeL~;xlH>Gy21^V0Sp+w^*wnUO3F278@|S#I)&n5P}H1g1K!itLp$s3#B!qa>P#GuXh)tGOf7&JwKz zX+lU^)hQ!`*Ll;PnKjco|EA%2;6P9H#J$}aIlQwg8tMk#M$)WIzL$(4M=;OI8}Z66 zI;p20FD=u077zNGq5(X4$Ml_NK+5TvT00o`c%xD8X7SE1DtAW!)sU|bW8~pi7~vob zgg5qCXS)A31MsGmOFrc=#BW0|7Ew)WCCLvAIU505e1d7Dfct1x%nl1Bsj-_>vM&vF zsie9!u(S8Hn+{}a$fcATjoqx0xG>;BW8Erg)p+xtta0^djXn7^t{$aPIv& z(3oE&GG|4m032{dO)Q+B&IQ+O0JWT&hFfHKoLC|KAFly*w^MkA?ixw5xLK( zE-}=~rLHa1s-@&qoU2dCjg$L^N>1JTD!D%;*5RmmHggm!y+hxT7uVT9UfiC3PGME1 zw*b_l2|Z3YM=ObnJns@6%-SSFz)brWN989be#3&ylQ|I;&yo`B64ri|_BW zSs|%s3_B^G?^rW+e4#}>_sPcV*jf1=cs7oEo*DV{tEXO8Zn`j5PrGvQ3E6<8biS)U zI!oLI@P8-GjwL8(<(t%!*=#=O3^AVd z`J?-f&|k&g%4v5U2IbQu_(%7v#scNeDu63{)O^Iv!-;mzY}m6_c&*G3{G-QDCCaI#Z9XO?Q1&@+0xKg0Em#sz-`JKA6>6+9Mr z6}ynruLPd4UkRN?qHueavkteZSb4036)4=*T{pu!E6{74vd3m;;Lj6JU z8&k9SOVB{rn_;;)m6Ok&alApRN6o8Oa6O~v)tQn%cYa>R@zL}Atz6IOdANZLLhI+Q zJ7Mrw^StcSAjivDLi*n>vv(KZKU4sJ4!bo5z9yDC&wn5Mm6q=jsnU-Y(DPCO{Otlb zIzL(^vyNc5m4_DKRBmO?`ce(N8sp`vDwFf63o?3aN4V;r3a7_RmGqUg96j_*s@gVs zzG^N|?gFk~*k(PiDu92f0KTCBerExk9){&>_s0w1{{)=so7;{j3h=)T+^|+irq37P zFVfLow$bxs4e)$+HWEL}tW&Cg&MLsajN|3(n4D*pqMazLF2L`X{A%7%HyLq>KO*%s z3N>tN0X>ZhNQZ%gR`yUr7F~vfLvwkDgiOYnOK&(N{lX1x*RNcXodehKO*(A9c$Qr&HC(%NHS?@kbz^(mD$k9}m#Lv>aT`As`3l=a;@ zYX)|yuytxhuuY1#fXjUASvassBs>AWb0^BiCJa0zKB#AvdL6?I7x`)1C z=q2YTczL|emY zN=S&AOGv0R6k~fFd%@ii=)$R)7a){jfU{=IiuA@Dj16@5cSFt_?#2`4|nUXvxE60uS?Sri^*Sf21H%;w+cPG&C70Ck`dw zVKYJDohk_!oXVWAjAaFUZ-<_trsW50CMdjbCFPHCO7KupU3V)z^h_GBi$zrV>i5Ri zVjZQ>C?Z}4C6&B{8SUC)wN00=_WSyzJUs%YK)*Agx+{6y3GlE4XJS$1t9`*f5e5;htwO_)l8wv8I zl27kP%8Dv^dD%6p{u`uxyHu!tU)?I@)$g%Mek_h$|7`(AewFUZ>~B(|ASgczO4zE> z6mP^~l~?zphLq1#Oh{FVD1F|ZBd>m|V9FoJ`)kvzJYi;&>~7H3^3{I$!wrHlN1XfG zs^Wi0dF6k#|N5kq|FX)ELdt(i2i{ZWR(G(%z&TX6b#5vrPaI6rZ3nw8Ec72K5~!ulAAE z_uNOQaS&AhQgW&v{44T^MairE=94WXMfXD4LCLH51hTF2YG3+{lo!8p&gf8bDt;|T zUfmC$l>@I(`AXwyBuJOCKYhPK6C=VW$-fOAvR@WQA|gonFDY8a(w$s}ptCp>CRup2 zPRh6DWaipK`)oNZTz>aVp>2$u#{@N+w-mzp`5&!ZflPAB$=*rcK%wLxX`t7g+*(q6sRTOjQ$b-{g-EHu>Z z-3~p9%m1|KVcw+L^BM=g>bHg}tm4Kb4DA1jen0XnHt{IwY>`5pRm(lYHgPFQ+SCJNWmHUcSBRo!2e|zmV&?mt@Fq z(jgftWKUf}j8lCk{z-=8;X6R|BBEV~|1+Tdc*tJ{I$bC3@^2ta z5?2X74Frl_A4S}ZwKWD!5ePKy2?mPt1MY9-SSZ*e@DXVb znk~>{B`hP20n}q=pfwn7BXf)#S_+q$|{s{;!vsutPRh51^( znrBztl;`KE3n~_iKwZQ70DKPZ49Cq-tYQ7?NVF}~5Zn<7!LOaI(KZ#mfD|275aU&Z zm9hwbKK?A`&myeOM05W6uxgIO=oZo>M&s@ScRc14t;19v{OmtqnJpDl6zz&|7#YSm zajn9?#r&~iuEM7mI^=OZ9?jHjVI_E=DOk^DNsftEm$U&zyT$d3CeWoS?>g$JWJFl~m z4N_(~)*j)P8Xf-aQ~`glzQ zT=$AX>w>s!$4*GTnUC&8kw>apN9P;fBx^rFMbCtID68@X<;AE{=1FSfa<)ID5N|)51NRt+8=hu!pl%Hckum zaCWAR(*ix5on+&*Fb`*aHeSZ~>+k9Q(n5^>+c+)2=)aBA!i)afI4!v7zm3yEi~id< zZMf*ajnjgQ{@XY$wCKN$(*leB+c+((=)aBAf{OmzI4z{;zs50|e_BDKxkm7K0>7}l z5P#K$|I&s3hYNqfg@4b5cf0T>UHCU#_}5+dmtFV~7oK+Eoi2R83s1Q4Ru>*};oDsJ z78kzWh1a_96)t?S3!m@8XS?ugT=-NcUUi|)I&+UK!`5=%W?x{*LINHvkF2!OU(4* zR|w2~&#;D!lds)soE#Zr_?|P)yk|~y9*)`yN2-C&Evye&p@#V4q0 zSg)EBjMR!x0?CB`l*?tBaIrmCas;?<8+7gVNk6+AOsZ^w9u_OVwg3Fnv<758jVWzd zwIfEVcBEH8DwnD~Z#-n294I!d-b|F)&k)zJ>aoVvj-?#WhPCB1RQppRUGL*cb@8%i5&>29^`OlLI02V3ipI{A z7#2CvZ=|YA4eKZ5+BtOENNvd&)++L=KUI#;j76VlsyEa0F7aW;A~i>oECNq@Gbc$T z+2f;nCz-Qqhm6$vK8O#Upc@#&`XTY=N>it)o_6TmE_KlL_0+2or_P&y+BAh&5jv%5b8j0S_ConLWZu5;)$(L|4VOS?m(3LEC+9&CR*lI*#{U(#mD3l`q(>-0$6rfe65(7vABxjOK?N z+}_MTD-oI`G=w8o`QeLw9K`gOu~l$!gRK z?4Ihd_8-D5?PxMm#k1#7BynX_^(6NnD!zXc%ybwu$x1%Flwv&bgV;pm1nGjP$c*6; zFPT7G3pQdr%$7Q!|oE7xrmm9=O&?&-+6!nq+MxqnEQ=pn5gy_ser^8&&>`|Tf`QriY^KqQij(d!h} z+w{b0_J}5gbZRTKyLcY<(|k}iCrg_>7`%O~z=L;`sI)^Q_m7Cgon)HUExHh9QVZ#c zSGlae183O!;C=|byNKO@)roUlf@f^OhUESig;|PEpE+ajCW@9?RI!%xM6TkoKR-Bq z6y7Gxd#m>RVR*)ir{6%|GTpEaf**wJ zv$!pQ{W++YeY<)`b8=*Cq9pqbP?Ozk#w^R*%N~)^r1@NdUfH*Mk|Pt$P53DL%u3vv zBu9Ma_~b~bd0qBKpqTQXLqd*LOQGA_nSqCJMPwIB=KZ5G-!GZbJX5)fZJ^Rf9U(u| zR{#8)9Aly&7!b?*NHsflXCa5}++q)s-Q-C8FwDe!J9#@Xe-!Xf6u63UwboKX@Oyj*)QW3E!DXdjM-vHh!vt5 z_XnLXBP>vt^R72j{RY%6Bm`-uXuCY&Pr)>HvMUf>lbzt`W`9K%(j_zVB-rEWEce$7 z81w`Gb{@}9m%(v|2a696`fwSmc6u{kLbyA7$~teR$&%s- zoS=J6Tvkii26yTU!z}f{6i5wT=TM(T7gOHadbXCpDBBGjrC!L$eG69VtW#jhes4v; zZ8=lQt-~#X^)g+jv$@U0=L-ED#DJ2&pQ1Tu4H(w54jYwh!_Xt*%oclJXGdkjnEq&B z78I;n1TzRMK)q|CMvClS9xf_tQrM!K=D&iY9W z=NiPZjvR-W(c9StfT$uRJV8qytN8l9OLboHj6b|(?CL3}E%3luF$5jNVea>hWiOhO zv8ZyYoIEsZgl_W(vzUGdUdf?u%*A%5>zzL~%V#q+nJG1V14goxLzL_R=v)f-QtZ3( zT91NJnc|Kid-k0~BuG%rz-JINsY9JfrI}T54OLQDtzSS*MYd&KYU}m_v>vIEgKN|m zpw)&0HB#;E1*p;DK-ZO*OaI)ZcM??#WUJ zthdhAS^d@-x@OZQk5SaD=j1N^O$v1KR5h&91xkaykz+3#Np@3PSX8d$q?Q*&lG*j2 z)Sx|cPth8aw{QVmHjjO@-Ta4XhV>-R9qY*B zz+9WBk?yQyhgDiT7pASw7a^GJrcYs3V4?l>4b<1dOzkFI+|-BL(9aZWWSg+L+k-2o zajKIVMPm9DoHIdOhAS!i&E8RZqJ$c*fJtdakXpT!V)z{05tIQ7i$rY*Q$PRu3lbF`LF(kVQ$n3!OuH@W2_@17)n0uV} zt{aM75YJ_I$?Sl1?H%q`@?Gw;NIGc#*xA8y&JKP+;YE?z!DVu~41p-KgNLPC0py+-hWt_4qJA8+9>ScP9_A79Or;EtRXkimJ8HtCouvd?UF=y!6XR@sM~M0}Ar zvyY!c${sGxKG=jztki%#%yW`cEji0*PIzP=Ih<~*4siCdEbo1OJ~^`w9nfodR@>PJ zO+cM}bg@nD)O6J;JNwY;kJ81az3$EY2!n@fgt8AUF@q&Ia*JU_W*;&toPB7yx2U%% zk(Zo!p(Xy!NB24^1>PG|N}2Wxh~W%ltXNEY8~INXg&H)xW>PR0WylK^0Z`WC;DJK}l zCigws0QKhe6|$M%+;8P3r2S+!HG_QMdYKQ%>#jF**Kb)x&x0rA8p1mh2J)BSj+zfV zL2Ax?03$@_z5HF5{p}Ww+of_T0+%9iDFT0;5uoSgsMN+{(b&!Y&GaCkKOFamTHDRN zGlkd^5AF=z?4KR?-xY3a4o5=qebI!;m2Ljzu~0k_G4WI+9!3ndhF1EQ?2sLhPeL`K8y`}tYJww->nCFG}v4*k() ze^YSpc%j-Bx@~Ri-cT$aj<)$*6LHhOBjmq+cKmvOG{$hem>r3PVcXF^GR!Xe;asTE zjK=m#ou+6g?r)2lemo%=H!GBu9T(EeYnV~L?39P@Zq*%n>N2dlLoq@awPD#`GZy3? zD4BcTeNMytFI!_wyQ0=yG@AM z@mpqxZ=uhK23(7T^MC?KC?MR)YfvYA)5qMh4}qmeB{ILv<&Fb({VJF12Rscp1bFal zF82`N$6v|iN|E870;F$U&H@^MA3B%IwE!LhJP26%YA*LU;6cFCfW=4;GJs9URZ1~U z9~sK!D&=(LXX#;D`2KE_@a-x0O`kBX^e|*`9?QQO$qeLjA0mM@wE^8B1=RLHK0|31)rNMDt!0T1Cn zh`w$nTJ7cKpDbEEact2?;F0v@UqOBle11Mn^Bb7WzWxCG&w^h|b~S&@avw>M{qy*L z3NdR2wE0EbZ2J?S_b&KfE8u_0=HCE*$FFm_tp)szHh&%XUEuGd*xK#Ki*)-D@TdGH zm)j1Q=ikZxQCts!-hg~-1$}|1^*4NG%%_VFl_bZG`Or8RBDp6Z7sfq9Iem?HvE17v z2hOE(DFT-wa47lNIt zV7r1H3LaMQn1aU@>`|~!!2tz_6vU4Txs)sDSFlpSY6a^R+^%4|f*lGTR`8gD#}({R zuus7O1>ODq?{NLQvD;R!zS%!#%Z@~wnebOtRV=EgoL`k-wCcVE3o0rX%~j+-Z;_^N zv3Q+N#+{NhEqve{QUmT+Ogs=WsyxlNB2g}eNWfA2s1x0ua@733fb4|ho&E7`Kk8r^fdi3R7UHsSHDiNohjn*B5ffLN3-)| zmj8fo&*zs|?FylPN3X5C#`4qbbwSgM(Q!O_a#5M;L0{lOU+Y2t80h|stP9bJ^~f)< znyUC~k$XMt&?6|L`T4L1y^HDPqD3ume6Nc}PAU37sd=(lHT(kTXJ&86N{W~7?S`YfiJm_s6^d#sM2lu!f_K^Ro2ffRK z{sYkcIw%hKl85{t=#z@B6nZ_+4p7jF*c9 zY7NCT)@%}i+tzJdRkJRzaqZg8wGDxWnpNv+1N_E;YW-t>{;__*^*RHayKs?wVL|&| zn?wry@`Tp*0@|PZ#lL!;LV>(~O+f*%;MD{+2d^1u3Q@56YYfpA+AS-AP*czhsy8dx z(&V6eFG2y02SmMLp^)KvPlC-{lz-ELO(W~n9df^bs9q>hKtp%pA)NOY5HX+NwS4^n zxr6sc6!7)S9}0=`#S#TH*LxJ=m@qbf;X;~~@o1nW*w%z^7vL_xzalT7UM67^@YaVw z-A1^DZyW*%xGmZf@s?fi;6W0L_?$ExLv`h=gM8q5@xAg(`NoB`Vrun9ngpI#1e>z{^GE zhnp0sW>wvMTFY3xB^YlJ6-|5F(1e6$OmfuMY6VCDyjUm_Bn4G#kC>u@yHtT{#m*>D zs>DN$qQVUA0ZNM-q@#QRR)kvAa@NuWYZ{X_-#DV{RPr=k5X9f1d0c{quPIcPNCr(C9cmY-2Ml_r!xwzug@_wG_c=O zAs;EuT3?@Mgh3+_t?ws66b90}-^|_3O$CP2xrlUyJGccn#?s!dcwy)BV3wziWY%*6AWSGTXvH|+NxD*x58VB6E<_%F~U5v{M!K?hcXh~oF? zzixjl_)dNMKDe6I?fEXTy#E@X0A;eJ+t=?uDzBCj+D|R-*A)dPsrD;fbge(nMJHdW zIH8_9k|RH>t0nAL3j^J^(j}$)t=C^VkJG+O4f#3kHVF^9CY)|p%hKgOn*OoBkrG$A I6kMqIFB2Q18~^|S literal 0 HcmV?d00001 From 190376c6fe37e04a4a39107f94bd7168dca3111d Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Thu, 27 Mar 2025 14:47:10 +0100 Subject: [PATCH 04/63] feat: all file related changes - trying TaskProcessor inheritance --- .../main/nextflow/cws/SchedulerClient.groovy | 56 ++ .../nextflow/cws/k8s/CWSK8sTaskHandler.groovy | 11 +- .../cws/k8s/localdata/LocalFile.groovy | 18 + .../cws/k8s/localdata/LocalPath.groovy | 666 ++++++++++++++++++ .../cws/processor/WOWTaskProcessor.groovy | 96 +++ .../nextflow/cws/wow/file/DateParser.groovy | 35 + .../cws/wow/file/LocalFileWalker.groovy | 217 ++++++ .../cws/wow/file/WOWFileHelper.groovy | 105 +++ .../src/resources/META-INF/extensions.idx | 3 +- 9 files changed, 1205 insertions(+), 2 deletions(-) create mode 100644 plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalFile.groovy create mode 100644 plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalPath.groovy create mode 100644 plugins/nf-cws/src/main/nextflow/cws/processor/WOWTaskProcessor.groovy create mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy create mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy create mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileHelper.groovy diff --git a/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy index 20722a4..dbc6632 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy @@ -177,4 +177,60 @@ class SchedulerClient { } } + ///* File location */ + + Map getFileLocation( String path ){ + + String pathEncoded = URLEncoder.encode(path,'utf-8') + HttpURLConnection get = new URL("${getDNS()}/file/$runName?path=$pathEncoded").openConnection() as HttpURLConnection + get.setRequestMethod( "GET" ) + get.setDoOutput(true) + int responseCode = get.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while requesting file location: $path (${get.responseMessage})" ) + } + Map response = new JsonSlurper().parse(get.getInputStream()) as Map + return response + + } + + String getDaemonOnNode( String node ){ + + HttpURLConnection get = new URL("${getDNS()}/daemon/$runName/$node").openConnection() as HttpURLConnection + get.setRequestMethod( "GET" ) + get.setDoOutput(true) + int responseCode = get.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while requesting daemon on node: $node" ) + } + String response = new JsonSlurper().parse(get.getInputStream()) as String + return response + + } + + void addFileLocation( String path, long size, long timestamp, long locationWrapperID, boolean overwrite, String node = null ){ + + String method = overwrite ? 'overwrite' : 'add' + + HttpURLConnection get = new URL("${getDNS()}/file/$runName/location/${method}${ node ? "/$node" : ''}").openConnection() as HttpURLConnection + get.setRequestMethod( "POST" ) + get.setDoOutput(true) + Map data = [ + path : path, + size : size, + timestamp : timestamp, + locationWrapperID : locationWrapperID + ] + if ( node ){ + data.node = node + } + String message = JsonOutput.toJson( data ) + get.setRequestProperty("Content-Type", "application/json") + get.getOutputStream().write(message.getBytes("UTF-8")); + int responseCode = get.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while updating file location: $path: $node (${get.responseMessage})" ) + } + + } } \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy index be46394..ea02b1e 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy @@ -84,6 +84,8 @@ class CWSK8sTaskHandler extends K8sTaskHandler { booleanInputs.add( [ name : key, value : input] ) } else if ( input instanceof Number ) { numberInputs.add( [ name : key, value : input] ) + } else if ( input instanceof Character ) { + stringInputs.add( [ name : key, value : input as String] ) } else if ( input instanceof String ) { stringInputs.add( [ name : key, value : input] ) } else if ( input instanceof GStringImpl ) { @@ -137,7 +139,12 @@ class CWSK8sTaskHandler extends K8sTaskHandler { return schedulerClient.registerTask( config, task.id.intValue() ) } + @Override protected BashWrapperBuilder createBashWrapper(TaskRun task) { + CWSK8sConfig cwsK8sConfig = k8sConfig as CWSK8sConfig + if ( cwsK8sConfig?.locationAwareScheduling() ) { + return new WOWK8sWrapperBuilder( task , cwsK8sConfig.getStorage() ) + } return fusionEnabled() ? fusionLauncher() : new CWSK8sWrapperBuilder( task, executor.getCWSConfig().memoryPredictor as boolean ) @@ -213,13 +220,15 @@ class CWSK8sTaskHandler extends K8sTaskHandler { continue switch( name ) { case "scheduler_nodes_cost" : - case "scheduler_init_throughput": traceRecord.put( name, value ) break case "scheduler_best_cost" : double val = parseDouble( value, file, name ) traceRecord.put( name, val ) break + case "input_size" : + traceRecord.put( name, inputSize ) + break default: long val = parseLong( value, file, name ) traceRecord.put( name, val ) diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalFile.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalFile.groovy new file mode 100644 index 0000000..78b2ce9 --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalFile.groovy @@ -0,0 +1,18 @@ +package nextflow.cws.k8s.localdata + +import java.nio.file.Path + +class LocalFile extends File { + + private final LocalPath localPath; + + LocalFile( LocalPath localPath ){ + super( localPath.toString() ) + this.localPath = localPath; + } + + @Override + Path toPath() { + return localPath + } +} \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalPath.groovy new file mode 100644 index 0000000..a118afe --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalPath.groovy @@ -0,0 +1,666 @@ +package nextflow.cws.k8s.localdata + +import groovy.util.logging.Slf4j +import nextflow.file.FileHelper +import nextflow.cws.wow.file.LocalFileWalker +import nextflow.cws.k8s.K8sSchedulerClient +import org.codehaus.groovy.runtime.IOGroovyMethods +import sun.net.ftp.FtpClient + +import java.nio.charset.Charset +import java.nio.file.* +import java.nio.file.attribute.BasicFileAttributes + +@Slf4j +class LocalPath implements Path { + + private final Path path + private transient final LocalFileWalker.FileAttributes attributes + private static transient K8sSchedulerClient client = null + private boolean wasDownloaded = false + private Path workDir + private boolean createdSymlinks = false + private transient final Object createSymlinkHelper = new Object() + + private LocalPath(Path path, LocalFileWalker.FileAttributes attributes, Path workDir ) { + this.path = path + this.attributes = attributes + this.workDir = workDir + } + + private LocalPath(){ + path = null + this.attributes = null + this.workDir = null + } + + LocalPath toLocalPath( Path path, LocalFileWalker.FileAttributes attributes = null ){ + toLocalPath( path, attributes, workDir ) + } + + static LocalPath toLocalPath( Path path, LocalFileWalker.FileAttributes attributes, Path workDir ){ + ( path instanceof LocalPath ) ? path as LocalPath : new LocalPath( path, attributes, workDir ) + } + + static void setClient( K8sSchedulerClient client ){ + if ( !this.client ) this.client = client + else throw new IllegalStateException("Client was already set.") + } + + private FtpClient getConnection( final String node, String daemon ){ + int trial = 0 + while ( true ) { + try { + FtpClient ftpClient = FtpClient.create(daemon) + ftpClient.login("root", "password".toCharArray() ) + ftpClient.enablePassiveMode( true ) + return ftpClient + } catch ( IOException e ) { + if ( trial > 5 ) throw e + log.error("Cannot create FTP client: $daemon on $node", e) + sleep(Math.pow(2, trial++) as long) + daemon = client.getDaemonOnNode(node) + } + } + } + + private Map getLocation( String absolutePath ){ + Map response = client.getFileLocation( absolutePath ) + synchronized ( createSymlinkHelper ) { + if ( !createdSymlinks ) { + for ( Map link : (response.symlinks as List)) { + Path src = link.src as Path + Path dst = link.dst as Path + if (Files.exists(src, LinkOption.NOFOLLOW_LINKS)) { + try { + if (src.isDirectory()) src.deleteDir() + else Files.delete(src) + } catch ( Exception ignored){ + log.warn( "Unable to delete " + src ) + } + } else { + src.parent.toFile().mkdirs() + } + try{ + Files.createSymbolicLink(src, dst) + } catch ( Exception ignored){ + log.warn( "Unable to create symlink: " + src + " -> " + dst ) + } + } + createdSymlinks = true + } + } + response + } + + private void checkParentDirectoryExists() { + if (!path.getParent().exists()) { + try { + log.trace("Creating directory locally ${path.getParent().toAbsolutePath().toString()}") + Files.createDirectories(path.getParent()) + } catch (IOException e) { + log.error("Couldn't create directory ${path.getParent().toAbsolutePath().toString()}", e) + } + } + } + + /** + * + * @return A path located in the original workdir + */ + Path fakePath(){ + Path fake = FileHelper.fakePath( path, workDir ) + return fake + } + + String getText(){ + getText( Charset.defaultCharset().toString() ) + } + + String getText( String charset ){ + final String absolutePath = path.toAbsolutePath().toString() + final def location = getLocation( absolutePath ) + if ( wasDownloaded || location.sameAsEngine ){ + log.trace("Read locally $absolutePath") + return path.getText( charset ) + } + try (FtpClient ftpClient = getConnection(location.node as String, location.daemon as String)) { + try (InputStream fileStream = ftpClient.getFileStream(location.path as String)) { + log.trace("Read remote $absolutePath") + return fileStream.getText( charset ) + } + } + } + + void setText( String text ){ + setText( text, Charset.defaultCharset().toString() ) + } + + void setText( String text, String charset ){ + final String absolutePath = path.toAbsolutePath().toString() + final def location = client.getFileLocation( absolutePath ) + checkParentDirectoryExists() + log.trace("Write locally $absolutePath") + path.setText( text, charset ) + def file = path.toFile() + client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) + } + + byte[] getBytes(){ + final String absolutePath = path.toAbsolutePath().toString() + final def location = getLocation( absolutePath ) + if ( wasDownloaded || location.sameAsEngine ){ + log.trace("Read locally $absolutePath") + return path.getBytes() + } + try (FtpClient ftpClient = getConnection(location.node as String, location.daemon as String)) { + try (InputStream fileStream = ftpClient.getFileStream( location.path as String )) { + log.trace("Read remote $absolutePath") + return fileStream.getBytes() + } + } + } + + void setBytes( byte[] bytes ){ + final String absolutePath = path.toAbsolutePath().toString() + final def location = client.getFileLocation( absolutePath ) + checkParentDirectoryExists() + log.trace("Write locally $absolutePath") + path.setBytes( bytes ) + def file = path.toFile() + client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) + } + + Object withReader( Closure closure ){ + withReader ( Charset.defaultCharset().toString(), closure ) + } + + Object withReader( String charset, Closure closure ){ + final String absolutePath = path.toAbsolutePath().toString() + final def location = getLocation( absolutePath ) + if ( wasDownloaded || location.sameAsEngine ){ + log.trace("Read locally $absolutePath") + return path.withReader( charset, closure ) + } + try (FtpClient ftpClient = getConnection( location.node as String , location.daemon as String )) { + try (InputStream fileStream = ftpClient.getFileStream( location.path as String )) { + log.trace("Read remote $absolutePath") + return IOGroovyMethods.withReader(fileStream, closure) + } + } + } + + def T withWriter( Closure closure ){ + return withWriter( Charset.defaultCharset().toString(), closure ) + } + + def T withWriter( String charset, Closure closure ) { + final String absolutePath = path.toAbsolutePath().toString() + final def location = client.getFileLocation( absolutePath ) + checkParentDirectoryExists() + log.trace("Write locally $absolutePath") + T rv = path.withWriter( charset, closure ) + def file = path.toFile() + client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) + return rv + } + + def T withPrintWriter( String charset, Closure closure ){ + final String absolutePath = path.toAbsolutePath().toString() + final def location = client.getFileLocation( absolutePath ) + checkParentDirectoryExists() + log.trace("Write locally $absolutePath") + T rv = path.withPrintWriter( charset, closure ) + def file = path.toFile() + client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) + return rv + } + + List readLines(){ + IOGroovyMethods.readLines( newReader() ) + } + + List readLines( String charset ){ + IOGroovyMethods.readLines( newReader( charset ) ) + } + + public T eachLine( Closure closure ) throws IOException { + eachLine( Charset.defaultCharset().toString(), 1, closure ) + } + + public T eachLine( int firstLine, Closure closure ) throws IOException { + eachLine( Charset.defaultCharset().toString(), firstLine, closure ) + } + + public T eachLine( String charset, Closure closure ) throws IOException { + eachLine( charset, 1, closure ) + } + + public T eachLine( String charset, int firstLine, Closure closure ) throws IOException { + final String absolutePath = path.toAbsolutePath().toString() + final def location = getLocation( absolutePath ) + if ( wasDownloaded || location.sameAsEngine ){ + log.trace("Read locally $absolutePath") + return path.eachLine( charset, firstLine, closure ) + } + try (FtpClient ftpClient = getConnection( location.node as String, location.daemon as String )) { + try (InputStream fileStream = ftpClient.getFileStream( location.path as String )) { + log.trace("Read remote $absolutePath") + return IOGroovyMethods.eachLine( fileStream, charset, firstLine, closure ) + } + } + } + + BufferedReader newReader(){ + return newReader( Charset.defaultCharset().toString() ) + } + + BufferedReader newReader( String charset ){ + final String absolutePath = path.toAbsolutePath().toString() + final def location = getLocation( absolutePath ) + if ( wasDownloaded || location.sameAsEngine ){ + log.trace("Read locally $absolutePath") + return path.newReader() + } + try (FtpClient ftpClient = getConnection( location.node as String , location.daemon as String )) { + InputStream fileStream = ftpClient.getFileStream( location.path as String ) + log.trace("Read remote $absolutePath") + InputStreamReader isr = new InputStreamReader( fileStream, charset ) + return new BufferedReader(isr) + } + } + + BufferedWriter newWriter(){ + return newWriter( Charset.defaultCharset().toString(), false, false ) + } + + BufferedWriter newWriter( String charset ) { + return newWriter( charset, false, false ) + } + + BufferedWriter newWriter( boolean append ) { + return newWriter( Charset.defaultCharset().toString(), append, false ) + } + + BufferedWriter newWriter( String charset, boolean append ){ + return newWriter( charset, append, false ) + } + + BufferedWriter newWriter( String charset, boolean append, boolean writeBom ){ + final String absolutePath = path.toAbsolutePath().toString() + final def location = client.getFileLocation( absolutePath ) + checkParentDirectoryExists() + log.trace("Write locally $absolutePath") + BufferedWriter bufferedWriter = path.newWriter( charset, append, writeBom ) + def file = path.toFile() + client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) + return bufferedWriter + } + + PrintWriter newPrintWriter( String charset ){ + final String absolutePath = path.toAbsolutePath().toString() + final def location = client.getFileLocation( absolutePath ) + checkParentDirectoryExists() + log.trace("Write locally $absolutePath") + PrintWriter printWriter = path.newPrintWriter( charset ) + def file = path.toFile() + client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) + return printWriter + } + + public T eachByte( Closure closure ) throws IOException { + final String absolutePath = path.toAbsolutePath().toString() + final def location = getLocation( absolutePath ) + if ( wasDownloaded || location.sameAsEngine ){ + log.trace("Read locally $absolutePath") + return path.eachByte( closure ) + } + try (FtpClient ftpClient = getConnection( location.node as String , location.daemon as String )) { + try (InputStream fileStream = ftpClient.getFileStream( location.path as String )) { + log.trace("Read remote $absolutePath") + return IOGroovyMethods.eachByte( new BufferedInputStream( fileStream ), closure) + } + } + } + + public T eachByte( int bufferLen, Closure closure ) throws IOException { + final String absolutePath = path.toAbsolutePath().toString() + final def location = getLocation( absolutePath ) + if ( wasDownloaded || location.sameAsEngine ){ + log.trace("Read locally $absolutePath") + return path.eachByte( bufferLen, closure ) + } + try (FtpClient ftpClient = getConnection( location.node as String , location.daemon as String )) { + try (InputStream fileStream = ftpClient.getFileStream( location.path as String )) { + log.trace("Read remote $absolutePath") + return IOGroovyMethods.eachByte( new BufferedInputStream( fileStream ), bufferLen, closure) + } + } + } + + public T withInputStream( Closure closure) throws IOException { + final String absolutePath = path.toAbsolutePath().toString() + final def location = getLocation( absolutePath ) + if ( wasDownloaded || location.sameAsEngine ){ + log.trace("Read locally $absolutePath") + return path.withInputStream( closure ) + } + try (FtpClient ftpClient = getConnection( location.node as String , location.daemon as String )) { + InputStream fileStream = ftpClient.getFileStream( location.path as String ) + log.trace("Read remote $absolutePath") + return IOGroovyMethods.withStream(new BufferedInputStream( fileStream ), closure) + } + } + + def T withOutputStream( Closure closure ){ + final String absolutePath = path.toAbsolutePath().toString() + final def location = client.getFileLocation( absolutePath ) + checkParentDirectoryExists() + log.trace("Write locally $absolutePath") + T rv = path.withOutputStream( closure ) + def file = path.toFile() + client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) + return rv + } + + public BufferedInputStream newInputStream() throws IOException { + final String absolutePath = path.toAbsolutePath().toString() + final def location = getLocation( absolutePath ) + if ( wasDownloaded || location.sameAsEngine ){ + log.trace("Read locally $absolutePath") + return path.newInputStream() + } + try (FtpClient ftpClient = getConnection( location.node as String , location.daemon as String )) { + InputStream fileStream = ftpClient.getFileStream( location.path as String ) + log.trace("Read remote $absolutePath") + return new BufferedInputStream( fileStream ) + } + } + + BufferedOutputStream newOutputStream(){ + final String absolutePath = path.toAbsolutePath().toString() + final def location = client.getFileLocation( absolutePath ) + checkParentDirectoryExists() + log.trace("Write locally $absolutePath") + BufferedOutputStream bufferedOutputStream = path.newOutputStream() + def file = path.toFile() + client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) + return bufferedOutputStream + } + + void write( String text ){ + write(text, Charset.defaultCharset().toString(), false) + } + + void write( String text, String charset ){ + write(text, charset, false) + } + + void write( String text, boolean writeBom ){ + write(text, Charset.defaultCharset().toString(), writeBom) + } + + void write( String text, String charset, boolean writeBom ){ + final String absolutePath = path.toAbsolutePath().toString() + final def location = client.getFileLocation( absolutePath ) + checkParentDirectoryExists() + log.trace("Write locally $absolutePath") + path.write( text, charset, writeBom ) + def file = path.toFile() + client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) + def loc = client.getFileLocation( absolutePath ) + } + + void leftShift(Object text) { + append(text) + } + + void append( Object text ){ + append(text, Charset.defaultCharset().toString(), false) + } + + void append( Object text, String charset ){ + append(text, charset, false) + } + + void append( Object text, boolean writeBom ){ + append(text, Charset.defaultCharset().toString(), writeBom) + } + + void append( Object text, String charset, boolean writeBom ){ + if ( !text ) { + return + } + final String absolutePath = path.toAbsolutePath().toString() + final def location = client.getFileLocation( absolutePath ) + if ( wasDownloaded || location.sameAsEngine ) { + log.info("Append locally $absolutePath") + path.append( text, charset, writeBom ) + } else { + try (FtpClient ftpClient = getConnection(location.node.toString(), location.daemon.toString())) { + byte[] bytes = text.toString().getBytes( charset ) + InputStream stream = new ByteArrayInputStream( bytes ) + log.info("Append remote $absolutePath") + ftpClient.appendFile( absolutePath, stream ) + } + } + def file = path.toFile() + client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) + def loc = client.getFileLocation( absolutePath ) + } + + private Map download(){ + final String absolutePath = path.toAbsolutePath().toString() + final def location = getLocation( absolutePath ) + synchronized ( this ) { + if ( this.wasDownloaded || location.sameAsEngine ) { + log.trace("No download") + return [ wasDownloaded : false, location : location ] + } + try (FtpClient ftpClient = getConnection(location.node as String, location.daemon as String )) { + try (InputStream fileStream = ftpClient.getFileStream( location.path as String )) { + log.trace("Download remote $absolutePath") + final def file = toFile() + path.parent.toFile().mkdirs() + OutputStream outStream = new FileOutputStream(file) + byte[] buffer = new byte[8 * 1024] + int bytesRead + while ((bytesRead = fileStream.read(buffer)) != -1) { + outStream.write(buffer, 0, bytesRead) + } + fileStream.closeQuietly() + outStream.closeQuietly() + this.wasDownloaded = true + return [ wasDownloaded : true, location : location ] + } catch (Exception e) { + throw e + } + } catch (Exception e) { + throw e + } + } + } + + T asType( Class c ) { + if ( c == Path.class ) return this + if ( c == File.class ) return toFile() + if ( c == String.class ) return toString() + log.info("Invoke method asType $c on $this") + return super.asType( c ) + } + + @Override + Object invokeMethod(String name, Object args) { + Map downloadResult = download() + def file = path.toFile() + def lastModified = file.lastModified() + Object result = path.invokeMethod(name, args) + if( lastModified != file.lastModified() ){ + //Update location in scheduler (overwrite all others) + client.addFileLocation( downloadResult.location.path as String , file.size(), file.lastModified(), downloadResult.location.locationWrapperID as long, true ) + } else if ( downloadResult.wasDownloaded ){ + //Add location to scheduler + client.addFileLocation( downloadResult.location.path as String , file.size(), file.lastModified(), downloadResult.location.locationWrapperID as long, false ) + } + return result + } + + String getBaseName() { + path.getBaseName() + } + + boolean isDirectory( LinkOption... options ) { + attributes ? attributes.isDirectory() : 0 + } + + long size() { + attributes ? attributes.size() : 0 + } + + boolean empty(){ + //TODO empty file? + this.size() == 0 + } + + boolean asBoolean(){ + true + } + + @Override + FileSystem getFileSystem() { + LocalFileSystem.INSTANCE + } + + @Override + boolean isAbsolute() { + path.isAbsolute() + } + + @Override + Path getRoot() { + path.getRoot() + } + + @Override + Path getFileName() { + path.getFileName() + } + + @Override + Path getParent() { + toLocalPath( path.getParent() ) + } + + @Override + int getNameCount() { + path.getNameCount() + } + + @Override + Path getName(int index) { + path.getName( index ) + } + + @Override + Path subpath(int beginIndex, int endIndex) { + toLocalPath( path.subpath( beginIndex, endIndex ) ) + } + + @Override + boolean startsWith(Path other) { + path.startsWith( other ) + } + + @Override + boolean startsWith(String other) { + path.startsWith(other) + } + + @Override + boolean endsWith(Path other) { + path.endsWith( other ) + } + + @Override + boolean endsWith(String other) { + path.endsWith( other ) + } + + @Override + Path normalize() { + toLocalPath( path.normalize() ) + } + + @Override + Path resolve(Path other) { + //TODO other attributes + toLocalPath( path.resolve( other ) ) + } + + @Override + Path resolve(String other) { + //TODO other attributes + toLocalPath( path.resolve( other ) ) + } + + @Override + Path resolveSibling(Path other) { + path.resolveSibling( other ) + } + + @Override + Path resolveSibling(String other) { + path.resolveSibling( other ) + } + + @Override + Path relativize(Path other) { + path.relativize( other ) + } + + @Override + URI toUri() { + path.toUri() + } + + Path toAbsolutePath(){ + toLocalPath( path.toAbsolutePath() ) + } + + @Override + Path toRealPath(LinkOption... options) throws IOException { + attributes.destination ? toLocalPath( attributes.destination ) : toLocalPath( path.toRealPath( options ) ) + } + + @Override + File toFile() { + path.toFile() + } + + @Override + WatchKey register(WatchService watcher, WatchEvent.Kind[] events, WatchEvent.Modifier... modifiers) throws IOException { + path.register( watcher, events, modifiers ) + } + + @Override + WatchKey register(WatchService watcher, WatchEvent.Kind... events) throws IOException { + path.register( watcher, events ) + } + + @Override + int compareTo(Path other) { + if ( other instanceof LocalPath ){ + return path.compareTo( ((LocalPath) other).path ) + } + path.compareTo( other ) + } + + @Override + String toString() { + path.toString() + } + + BasicFileAttributes getAttributes(){ + attributes + } +} \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/processor/WOWTaskProcessor.groovy b/plugins/nf-cws/src/main/nextflow/cws/processor/WOWTaskProcessor.groovy new file mode 100644 index 0000000..e6c7bc4 --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/processor/WOWTaskProcessor.groovy @@ -0,0 +1,96 @@ +package nextflow.cws.processor + +import groovy.transform.PackageScope +import groovy.util.logging.Slf4j +import nextflow.Session +import nextflow.cws.wow.file.WOWFileHelper +import nextflow.exception.MissingFileException +import nextflow.executor.Executor +import nextflow.file.FilePatternSplitter +import nextflow.processor.TaskProcessor +import nextflow.processor.TaskRun +import nextflow.script.BodyDef +import nextflow.script.BaseScript +import nextflow.script.ProcessConfig +import nextflow.script.params.FileOutParam + +import java.nio.file.LinkOption +import java.nio.file.NoSuchFileException +import java.nio.file.Path + +@Slf4j +class WOWTaskProcessor extends TaskProcessor { + + WOWTaskProcessor(String name, Executor executor, Session session, BaseScript script, ProcessConfig config, BodyDef taskBody ) { + super(name, executor, session, script, config, taskBody) + } + + @PackageScope + List fetchResultFiles( FileOutParam param, String namePattern, Path workDir ) { + assert namePattern + assert workDir + + List files = [] + def opts = visitOptions(param, namePattern) + // scan to find the file with that name + try { + WOWFileHelper.visitFiles(opts, workDir, namePattern) { Path it -> files.add(it) } + } + catch( NoSuchFileException e ) { + throw new MissingFileException("Cannot access directory: '$workDir'", e) + } + return files.sort() + } + + + @Override + protected void collectOutFiles(TaskRun task, FileOutParam param, Path workDir, Map context ) { + final List allFiles = [] + // type file parameter can contain a multiple files pattern separating them with a special character + def entries = param.getFilePatterns(context, task.workDir) + boolean inputsRemovedFlag = false + // for each of them collect the produced files + for( String filePattern : entries ) { + List result = null + + def splitter = param.glob ? FilePatternSplitter.glob().parse(filePattern) : null + if( splitter?.isPattern() ) { + result = fetchResultFiles(param, filePattern, workDir) + // filter the inputs + if( result && !param.includeInputs ) { + result = filterByRemovingStagedInputs(task, result, workDir) + log.trace "Process ${safeTaskName(task)} > after removing staged inputs: ${result}" + inputsRemovedFlag |= (result.size()==0) + } + } + else { + def path = param.glob ? splitter.strip(filePattern) : filePattern + def file = workDir.resolve(path) + def origFile = file + def outfiles = workDir.resolve( ".command.outfiles" ).toFile() + def exists + if( outfiles.exists() ){ + file = LocalFileWalker.exists( outfiles, file, workDir, param.followLinks ? LinkOption.NOFOLLOW_LINKS : null ) + exists = file != null + } else { + exists = param.followLinks ? file.exists() : file.exists(LinkOption.NOFOLLOW_LINKS) + } + if( exists ) + result = [file] + else + log.debug "Process `${safeTaskName(task)}` is unable to find [${origFile.class.simpleName}]: `$file` (pattern: `$filePattern`)" + } + + if( result ) + allFiles.addAll(result) + + else if( !param.optional ) { + def msg = "Missing output file(s) `$filePattern` expected by process `${safeTaskName(task)}`" + if( inputsRemovedFlag ) + msg += " (note: input files are not included in the default matching set)" + throw new MissingFileException(msg) + } + } + task.setOutput( param, allFiles.size()==1 ? allFiles[0] : allFiles ) + } +} diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy new file mode 100644 index 0000000..221bf19 --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy @@ -0,0 +1,35 @@ +package nextflow.cws.wow.file + +import java.nio.file.attribute.FileTime; +import java.text.SimpleDateFormat; + +public final class DateParser { + + private DateParser(){} + + public static Long millisFromString( String date ) { + if( date == null || date.isEmpty() || date.equals("-") || date.equals("w") ) { + return null; + } + try { + if (Character.isLetter(date.charAt(0))) { + // if ls was used, date has the format "Nov 2 08:49:30 2021" + return new SimpleDateFormat("MMM dd HH:mm:ss yyyy").parse(date).getTime(); + } else { + // if stat was used, date has the format "2021-11-02 08:49:30.955691861 +0000" + String[] parts = date.split(" "); + parts[1] = parts[1].substring(0, 12); + // parts[1] now has milliseconds as smallest units e.g. "08:49:30.955" + String shortenedDate = String.join(" ", parts); + return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z").parse(shortenedDate).getTime(); + } + } catch ( Exception e ){ + return null; + } + } + + public static FileTime fileTimeFromString( String date ) { + final Long millisFromSring = millisFromString( date ); + return millisFromSring == null ? null : FileTime.fromMillis( millisFromSring ); + } +} diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy new file mode 100644 index 0000000..a04539d --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy @@ -0,0 +1,217 @@ +package nextflow.cws.wow.file + +import groovy.util.logging.Slf4j +import nextflow.extension.FilesEx + +import java.nio.file.FileVisitOption +import java.nio.file.FileVisitResult +import java.nio.file.FileVisitor +import java.nio.file.Files +import java.nio.file.LinkOption +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.attribute.FileTime + +@Slf4j +class LocalFileWalker { + + static final int VIRTUAL_PATH = 0 + static final int FILE_EXISTS = 1 + static final int REAL_PATH = 2 + static final int SIZE = 3 + static final int FILE_TYPE = 4 + static final int CREATION_DATE = 5 + static final int ACCESS_DATE = 6 + static final int MODIFICATION_DATE = 7 + + public static TriFunction createLocalPath + + static Path walkFileTree(Path start, + Set options, + int maxDepth, + FileVisitor visitor, + Path workDir + ) + { + + String parent = null + boolean skip = false + + File file = new File( start.toString() + File.separatorChar + ".command.outfiles" ) + String line + file.withReader { reader -> + while ((line = reader.readLine()) != null) { + String[] data = line.split(';') + + String path = data[ VIRTUAL_PATH ] + + //If Symlink & not followLinks + //split path + //save it, and ignore everything at the path behind that + + if ( parent && path.startsWith(parent) ) { + if ( skip ) { + log.trace "Skip $path" + continue + } + } else { + skip = false + } + + FileAttributes attributes = new FileAttributes( data ) + Path currentPath = Paths.get(path) + if ( !attributes.local ) { + //If task did not run on local machine, create symbolic link + if ( !Files.isSymbolicLink( currentPath ) ) { + Files.createDirectories( currentPath.getParent() ) + Files.createSymbolicLink( currentPath, attributes.destination ) + } + Files.walkFileTree( currentPath, options, maxDepth, visitor) + } else { + Path p = createLocalPath.apply( currentPath, attributes, workDir ) + if ( attributes.isDirectory() ) { + def visitDirectory = visitor.preVisitDirectory( p, attributes ) + if( visitDirectory == FileVisitResult.SKIP_SUBTREE ){ + skip = true + parent = path + } + } else { + visitor.visitFile( p, attributes) + } + } + } + } + + return start + } + + static class FileAttributes implements BasicFileAttributes { + + private final boolean directory + private final boolean link + private final long size + private final String fileType + private final FileTime creationDate + private final FileTime accessDate + private final FileTime modificationDate + private final Path destination + private final boolean local + + FileAttributes( String[] data ) { + if ( data.length != 8 && data[ FILE_EXISTS ] != "0" ) throw new RuntimeException( "Cannot parse row (8 columns required): ${data.join(',')}" ) + boolean fileExists = data[ FILE_EXISTS ] == "1" + destination = data.length > REAL_PATH && data[ REAL_PATH ] ? data[ REAL_PATH ] as Path : null + if ( data.length != 8 ) { + this.link = true + this.size = 0 + this.fileType = null + this.creationDate = null + this.accessDate = null + this.modificationDate = null + return + } + this.link = data[ REAL_PATH ].isEmpty() + this.size = data[ SIZE ] as Long + this.fileType = data[ FILE_TYPE ] + this.accessDate = DateParser.fileTimeFromString(data[ ACCESS_DATE ]) + this.modificationDate = DateParser.fileTimeFromString(data[ MODIFICATION_DATE ]) + this.creationDate = DateParser.fileTimeFromString(data[ CREATION_DATE ]) ?: this.modificationDate + if ( fileType.startsWith("non-local ") ) { + this.local = false + this.fileType = fileType.substring( 10 ) + } else { + this.local = true + } + this.directory = fileType == 'directory' + if ( !directory && !fileType.contains( 'file' ) ){ + log.error( "Unknown type: $fileType" ) + } + } + + @Override + FileTime lastModifiedTime() { + return modificationDate + } + + @Override + FileTime lastAccessTime() { + return accessDate + } + + @Override + FileTime creationTime() { + return creationDate + } + + @Override + boolean isRegularFile() { + return !directory + } + + @Override + boolean isDirectory() { + return directory + } + + @Override + boolean isSymbolicLink() { + return link + } + + @Override + boolean isOther() { + return false + } + + @Override + long size() { + return size + } + + @Override + Object fileKey() { + return null + } + + Path getDestination(){ + destination + } + + } + + static Path exists( final File outfile, final Path file, final Path workDir, final LinkOption option ){ + + String rootDirString + Scanner sc = new Scanner( outfile ) + if( sc.hasNext() ) + rootDirString = sc.next().split(";")[0] + else + return null + + final Path rootDir = rootDirString as Path + final Path fakePath = WOWFileHelper.fakePath( file, rootDir ) + + //TODO ignore link + final Optional first = Files.lines( outfile.toPath() ) + .parallel() + .map {line -> + String[] data = line.split(';') + Path currentPath = data[ VIRTUAL_PATH ] as Path + log.trace "Compare $currentPath and $fakePath match: ${(currentPath == fakePath)}" + return ( currentPath == fakePath ) + ? createLocalPath.apply( currentPath, new FileAttributes( data ), workDir ) + : null + } + .filter{ it != null } + .findFirst() + + return first.isPresent() ? first.get() : null + + } + + static interface TriFunction { + Path apply( Path path, FileAttributes attributes, Path workDir ); + } + +} \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileHelper.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileHelper.groovy new file mode 100644 index 0000000..ab1011e --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileHelper.groovy @@ -0,0 +1,105 @@ +package nextflow.cws.wow.file + +import groovy.util.logging.Slf4j +import nextflow.extension.FilesEx +import nextflow.file.FileHelper +import nextflow.file.FilePatternSplitter + +import java.nio.file.FileSystemLoopException +import java.nio.file.FileVisitResult +import java.nio.file.FileVisitOption +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes + +@Slf4j +class WOWFileHelper extends FileHelper { + + static Path fakePath(Path path, Path destination ) { + String pathHelper = path.toString() + String n1 = FilesEx.getName(destination.getParent()) + String n2 = FilesEx.getName(destination) + String cdir = (n1 as Path).resolve(n2).toString() + + int index = pathHelper.indexOf( cdir ) + + if( index == -1 ){ + log.error("Cannot calculate fake path for path: $path to dest.: $destination cdir: $cdir") + return path + } + + return destination.getParent().getParent().resolve(pathHelper.substring( index )) + } + + static void visitFiles( Map options = null, Path folder, String filePattern, Closure action ) { + assert folder + assert filePattern + assert action + if (options == null) options = Map.of() + final type = options.type ?: 'any' + final walkOptions = options.followLinks == false ? EnumSet.noneOf(FileVisitOption.class) : EnumSet.of(FileVisitOption.FOLLOW_LINKS) + final int maxDepth = getMaxDepth(options.maxDepth, filePattern) + final includeHidden = options.hidden as Boolean ?: filePattern.startsWith('.') + final includeDir = type in ['dir', 'any'] + final includeFile = type in ['file', 'any'] + final syntax = options.syntax ?: 'glob' + final relative = options.relative == true + final matcher = getPathMatcherFor("$syntax:${filePattern}", folder.fileSystem) + final singleParam = action.getMaximumNumberOfParameters() == 1 + + final boolean outFileExists = new File(folder.resolve(".command.outfiles").toString()).exists() + + def visitor = new SimpleFileVisitor() { + + @Override + FileVisitResult preVisitDirectory(Path fullPath, BasicFileAttributes attrs) throws IOException { + fullPath = outFileExists ? fakePath(fullPath, folder) : fullPath + final int depth = fullPath.nameCount - folder.nameCount + final path = relativize0(folder, fullPath) + log.trace "visitFiles > dir=$path; depth=$depth; includeDir=$includeDir; matches=${matcher.matches(path)}; isDir=${attrs.isDirectory()}" + if (depth > 0 && includeDir && matcher.matches(path) && attrs.isDirectory() && (includeHidden || !isHidden(fullPath))) { + def result = relative ? path : fullPath + singleParam ? action.call(result) : action.call(result, attrs) + } + return depth > maxDepth ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE + } + + @Override + FileVisitResult visitFile(Path fullPath, BasicFileAttributes attrs) throws IOException { + final Path fakePath = outFileExists ? fakePath(fullPath, folder) : null + final path = folder.relativize(fakePath ?: fullPath) + log.trace "visitFiles > file=$path; includeFile=$includeFile; matches=${matcher.matches(path)}; isRegularFile=${attrs.isRegularFile()}" + + if (includeFile && matcher.matches(path) && (attrs.isRegularFile() || (options.followLinks == false && attrs.isSymbolicLink())) && (includeHidden || !isHidden(fullPath))) { + def result = relative ? path : fullPath + singleParam ? action.call(result) : action.call(result, attrs) + } + return FileVisitResult.CONTINUE + } + + FileVisitResult visitFileFailed(Path currentPath, IOException e) { + if (e instanceof FileSystemLoopException) { + final Path fakePath = outFileExists ? fakePath(currentPath, folder) : null + final path = folder.relativize(fakePath ?: currentPath).toString() + final capture = FilePatternSplitter.glob().parse(filePattern).getParent() + final message = "Circular file path detected -- Files in the following directory will be ignored: $currentPath" + // show a warning message only when offending path is contained + // by the capture path specified by the user + if (capture == './' || path.startsWith(capture)) + log.warn(message) + else + log.debug(message) + return FileVisitResult.SKIP_SUBTREE + } + throw e + } + } + + if (outFileExists) { + LocalFileWalker.walkFileTree(folder, walkOptions, Integer.MAX_VALUE, visitor, folder) + } else { + Files.walkFileTree(folder, walkOptions, Integer.MAX_VALUE, visitor) + } + } +} diff --git a/plugins/nf-cws/src/resources/META-INF/extensions.idx b/plugins/nf-cws/src/resources/META-INF/extensions.idx index a8d9308..b3d0ac5 100644 --- a/plugins/nf-cws/src/resources/META-INF/extensions.idx +++ b/plugins/nf-cws/src/resources/META-INF/extensions.idx @@ -1,2 +1,3 @@ nextflow.cws.CWSFactory -nextflow.cws.k8s.CWSK8sExecutor \ No newline at end of file +nextflow.cws.k8s.CWSK8sExecutor +nextflow.cws.processor.WOWTaskProcessor \ No newline at end of file From 1e795b41bf4031322999c73263cc1188cadf51dd Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Thu, 27 Mar 2025 17:31:00 +0100 Subject: [PATCH 05/63] fix: properly copy and call script in bash wrapper --- Makefile | 5 +- .../cws/k8s/WOWK8sWrapperBuilder.groovy | 7 +- .../cws/processor/WOWTaskProcessor.groovy | 1 + .../nf-cws/getStatsAndResolveSymlinks.c | 425 ++++++++++++++++++ .../getStatsAndResolveSymlinks_linux_aarch64 | Bin 25312 -> 0 bytes .../getStatsAndResolveSymlinks_linux_x86 | Bin 21632 -> 0 bytes 6 files changed, 435 insertions(+), 3 deletions(-) create mode 100644 plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c delete mode 100755 plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_linux_aarch64 delete mode 100755 plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_linux_x86 diff --git a/Makefile b/Makefile index 97f0a81..e922eb9 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,9 @@ assemble: # you can install the plugin copying manually these files to $HOME/.nextflow/plugins # buildPlugins: + # TODO: add targets + # clang -static -target arm-linux-gnu plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c -o plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_linux_aarch86 + clang -static -target x86_64-linux-gnu plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c -o plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_linux_x86 ./gradlew copyPluginZip # @@ -69,4 +72,4 @@ upload-plugins: ./gradlew plugins:upload publish-index: - ./gradlew plugins:publishIndex \ No newline at end of file + ./gradlew plugins:publishIndex diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy index e249cd9..f678b44 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy @@ -92,7 +92,9 @@ class WOWK8sWrapperBuilder extends K8sWrapperBuilder { protected String getLaunchCommand(String interpreter, String env) { String cmd = '' if( storage && localWorkDir ){ - cmd += "local INFILESTIME=\$(/etc/nextflow/${statFileName} infiles \"${workDir.toString()}/.command.infiles\" \"${getStorageLocalWorkDir()}\" \"\$PWD/\" || true)\n" + cmd += "cp -u \"/etc/nextflow/${statFileName}\" \"${getStorageLocalWorkDir()}\"\n" + cmd += "chmod +x \"${getStorageLocalWorkDir()}/${statFileName}\"\n" + cmd += "local INFILESTIME=\$(\"${getStorageLocalWorkDir()}/${statFileName}\" infiles \"${workDir.toString()}/.command.infiles\" \"${getStorageLocalWorkDir()}\" \"\$PWD/\" || true)\n" } cmd += super.getLaunchCommand(interpreter, env) if( storage && localWorkDir && isTraceRequired() ){ @@ -108,7 +110,8 @@ class WOWK8sWrapperBuilder extends K8sWrapperBuilder { String cmd = super.getCleanupCmd( scratch ) if( storage && localWorkDir ){ cmd += "mkdir -p \"${localWorkDir.toString()}/\" || true\n" - cmd += "local OUTFILESTIME=\$(/etc/nextflow/${statFileName} outfiles \"${workDir.toString()}/.command.outfiles\" \"${getStorageLocalWorkDir()}\" \"${localWorkDir.toString()}/\" || true)\n" + cmd += "\"${getStorageLocalWorkDir()}/${statFileName}\" outfiles \"${workDir.toString()}/.command.outfiles\" \"${getStorageLocalWorkDir()}\" \"${localWorkDir.toString()}/\" > \"${localWorkDir.toString()}/.command.getStatsOut\" 2> \"${localWorkDir.toString()}/.command.getStatsErr\"\n" + cmd += "local OUTFILESTIME=\$(\"${getStorageLocalWorkDir()}/${statFileName}\" outfiles \"${workDir.toString()}/.command.outfiles\" \"${getStorageLocalWorkDir()}\" \"${localWorkDir.toString()}/\" || true)\n" if ( isTraceRequired() ) { cmd += "echo \"outfiles_time=\${OUTFILESTIME}\" >> ${workDir.resolve(TaskRun.CMD_TRACE)}" } diff --git a/plugins/nf-cws/src/main/nextflow/cws/processor/WOWTaskProcessor.groovy b/plugins/nf-cws/src/main/nextflow/cws/processor/WOWTaskProcessor.groovy index e6c7bc4..373db1e 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/processor/WOWTaskProcessor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/processor/WOWTaskProcessor.groovy @@ -4,6 +4,7 @@ import groovy.transform.PackageScope import groovy.util.logging.Slf4j import nextflow.Session import nextflow.cws.wow.file.WOWFileHelper +import nextflow.cws.wow.file.LocalFileWalker import nextflow.exception.MissingFileException import nextflow.executor.Executor import nextflow.file.FilePatternSplitter diff --git a/plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c b/plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c new file mode 100644 index 0000000..19d6d70 --- /dev/null +++ b/plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c @@ -0,0 +1,425 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#define FULL_DESCR 0 +#define SHORT_DESCR_WITH_TIMESTAMP 1 + +#define INFILES_NAME "infiles" +#define OUTFILES_NAME "outfiles" + +#define STACK_MIN_SIZE 1 + +#define SYMLINK_TARGET_BUFFER_SIZE 500 + +struct symlink { + char * src; + char * dst; +}; + +struct stack { + int top; + int size; + struct symlink * items; +}; + +struct stack * newStack(int size) { + struct stack * ptr = (struct stack *) malloc(sizeof(struct stack *)); + ptr->top = -1; + ptr->items = (struct symlink *) malloc(sizeof(struct symlink) * size); + ptr->size = size; + return ptr; +} + +int isEmpty(struct stack * ptr) { + return (ptr->top == -1); +} + +struct stack * push_symlink(struct stack * ptr, char * const src, char * const dst) { + if (ptr->top + 1 == ptr->size) { + struct stack * new_ptr = newStack(ptr->size * 2); + new_ptr->items = (struct symlink *) realloc(ptr->items, sizeof(struct symlink) * ptr->size * 2); + new_ptr->top = ptr->top; + free(ptr); + ptr = new_ptr; + } + ptr->top++; + char * source = (char *) malloc(strlen(src) + 1); + strcpy(source, src); + char * destination = (char *) malloc(strlen(dst) + 1); + strcpy(destination, dst); + ptr->items[ptr->top].src = source; + ptr->items[ptr->top].dst = destination; + return ptr; +} + +const struct symlink * const head(struct stack * ptr) { + if (isEmpty(ptr)) { + fprintf(stderr, "Error: Stack is empty!"); + exit(EXIT_FAILURE); + } + return &(ptr->items[ptr->top]); +} + +void delete_top(struct stack * ptr) { + if (isEmpty(ptr)) { + fprintf(stderr, "Error: Stack is empty!"); + exit(EXIT_FAILURE); + } + free(ptr->items[ptr->top].src); + free(ptr->items[ptr->top].dst); + ptr->top--; +} + +void deleteStack(struct stack * ptr) { + while (!isEmpty(ptr)) { + delete_top(ptr); + } + ptr->size = -1; + free(ptr->items); + free(ptr); +} + +int collectFileInformation(char * const * dir_to_search, const int version, + const char * const local_dir, + const char * const result_filename); +int getFullDescr(char * const * dir, + const char * const local_dir, + FILE * file_ptr); +int getShortDescrAndTimestamp(char * const * dir, + const char * const local_dir, + FILE * file_ptr); + + +int main(int arc, char * const argv[]) { + + if (arc < 5) { + fprintf(stderr, "Error: too few arguments!\n"); + return -1; + } + argv++; + struct timespec start_time, end_time; + int gettimeofday_rc; + if ((gettimeofday_rc = clock_gettime(CLOCK_REALTIME, &start_time)) != 0) { + fprintf(stderr, "Error getting the time of day\n"); + return gettimeofday_rc; + } + + if (strcmp(argv[0], INFILES_NAME) != 0 && strcmp(argv[0], OUTFILES_NAME) != 0) { + fprintf(stderr, "Error: version must be '%s' or '%s'\n", INFILES_NAME, OUTFILES_NAME); + return -1; + } + const char * const name = argv++[0]; + const int version = strcmp(name, INFILES_NAME) == 0 + ? SHORT_DESCR_WITH_TIMESTAMP : FULL_DESCR; + const char * const result_filename = argv++[0]; + const char * const local_dir = argv++[0]; + int rc = collectFileInformation(argv, version, local_dir, result_filename); + + if ((gettimeofday_rc = clock_gettime(CLOCK_REALTIME, &end_time)) != 0) { + fprintf(stderr, "Error getting the time of day\n"); + return gettimeofday_rc; + } + long long int start = start_time.tv_sec * 1000000000 + start_time.tv_nsec; + long long int end = end_time.tv_sec * 1000000000 + end_time.tv_nsec; + printf("%lli\n", (end - start) / 1000000); + + return rc; +} + +int collectFileInformation(char * const * dir_to_search, const int version, + const char * const local_dir, + const char * const result_filename) { + + DIR* dir_ptr = opendir(local_dir); + if (dir_ptr) { + closedir(dir_ptr); + } else { + fprintf(stderr, "Error: the local directory '%s' does not exist.\n", local_dir); + return -1; + } + + dir_ptr = opendir(dir_to_search[0]); + if (dir_ptr) { + closedir(dir_ptr); + } else { + fprintf(stderr, "Error: the directory to search '%s' does not exist.\n", dir_to_search[0]); + return -1; + } + + FILE * file_ptr = fopen(result_filename, "w"); + if (file_ptr == NULL) { + fprintf(stderr, "Error opening the file %s\n", result_filename); + return -1; + } + int rc; + switch (version) { + case FULL_DESCR: + rc = getFullDescr(dir_to_search, local_dir, file_ptr); + break; + case SHORT_DESCR_WITH_TIMESTAMP: + rc = getShortDescrAndTimestamp(dir_to_search, local_dir, file_ptr); + break; + + default: + break; + } + fclose(file_ptr); + return rc; +} + +int getFullDescr(char * const * dir, + const char * const local_dir, + FILE * file_ptr) { + + FTS * fts_ptr; + FTSENT * ptr, * ch_ptr; + int fts_options = FTS_PHYSICAL; + + if ((fts_ptr = fts_open(dir, fts_options, NULL)) == NULL) { + fprintf(stderr, "Error traversing the directory %s\n", dir[0]); + return -1; + } + ch_ptr = fts_children(fts_ptr, 0); + if (ch_ptr == NULL) { + return 0; + } + int skip_next = 0; + struct stack * symlink_stack = newStack(STACK_MIN_SIZE); + char * symlink_target_path = (char *) malloc(SYMLINK_TARGET_BUFFER_SIZE); + while ((ptr = fts_read(fts_ptr)) != NULL) { + if (ptr->fts_info == FTS_DP) { + continue; + } + if (skip_next) { + skip_next = 0; + continue; + } + char * file_type; + strcpy(symlink_target_path, ""); + int exists; + switch (ptr->fts_info) { + case FTS_D: + file_type = "directory"; + exists = 1; + break; + case FTS_F: + file_type = "regular file"; + exists = 1; + break; + case FTS_SL: + file_type = "symbolic link"; + realpath(ptr->fts_path, symlink_target_path); + if (symlink_target_path == NULL) { + strcpy(symlink_target_path, ""); + } + exists = 1 + access(symlink_target_path, F_OK); // test for file existence + if (!exists) { + break; + } + // checking whether it is a directory: + struct stat target_file_stat; + int stat_rv; + if ((stat_rv = stat(symlink_target_path, &target_file_stat)) != 0) { + fprintf(stderr, "Error reading the file %s\n", symlink_target_path); + return stat_rv; + } + if (S_ISDIR(target_file_stat.st_mode)) { + // if target is not local, we skip it + if (strncmp(symlink_target_path, local_dir, strlen(local_dir)) != 0) { + file_type = "non-local directory"; + break; + } + file_type = "directory"; + // if target is within the directory we are searching, we skip it, + // to prevent searching a directory more than once + if (strncmp(symlink_target_path, dir[0], strlen(dir[0])) == 0) { + break; + } + fts_set(fts_ptr, ptr, FTS_FOLLOW); + symlink_stack = push_symlink(symlink_stack, ptr->fts_path, symlink_target_path); + skip_next = 1; + } else if (S_ISREG(target_file_stat.st_mode)) { + file_type = "regular file"; + } + break; + + default: + file_type = "unknown"; + exists = 1; + break; + } + if (!isEmpty(symlink_stack) && strcmp(file_type, "symbolic link") != 0) { + while (strncmp(head(symlink_stack)->src, ptr->fts_path, strlen(head(symlink_stack)->src)) != 0) { + delete_top(symlink_stack); + if (isEmpty(symlink_stack)) { + break; + } + } + if (!isEmpty(symlink_stack)) { + strcpy(symlink_target_path, head(symlink_stack)->dst); + int fts_len = strlen(ptr->fts_path); + int to_replace_len = strlen(head(symlink_stack)->src); + int rel_len = fts_len - to_replace_len; + char rel_path[rel_len+1]; + memcpy(rel_path, &ptr->fts_path[to_replace_len], rel_len+1); + strcat(symlink_target_path, rel_path); + } + } + fprintf( + file_ptr, + "%s;%i;%s;%li;%s;%li%li;%li%li;%li%li\n", + ptr->fts_path, + exists, + symlink_target_path, + ptr->fts_statp->st_size, + file_type, + ptr->fts_statp->st_ctim.tv_sec, ptr->fts_statp->st_ctim.tv_nsec, + // ctim - time of last status change which is used as an approximation of the creation time + ptr->fts_statp->st_atim.tv_sec, ptr->fts_statp->st_atim.tv_nsec, + ptr->fts_statp->st_mtim.tv_sec, ptr->fts_statp->st_mtim.tv_nsec + ); + } + free(symlink_target_path); + deleteStack(symlink_stack); + fts_close(fts_ptr); + return 0; +} + +int getShortDescrAndTimestamp(char * const * dir, + const char * const local_dir, + FILE * file_ptr) { + + struct timespec time_now; + int gettimeofday_rc; + if ((gettimeofday_rc = clock_gettime(CLOCK_REALTIME, &time_now)) != 0) { + fprintf(stderr, "Error getting the time of day\n"); + return gettimeofday_rc; + } + fprintf(file_ptr, "%li%li\n", time_now.tv_sec, time_now.tv_nsec); + + fprintf(file_ptr, "%s\n", dir[0]); + + int prefix_len = strlen(dir[0]); + if (dir[0][prefix_len-1] != '/') { + prefix_len++; + } + + FTS * fts_ptr; + FTSENT * ptr, * ch_ptr; + int fts_options = FTS_PHYSICAL; + + if ((fts_ptr = fts_open(dir, fts_options, NULL)) == NULL) { + fprintf(stderr, "Error traversing the directory %s\n", dir[0]); + return -1; + } + ch_ptr = fts_children(fts_ptr, 0); + if (ch_ptr == NULL) { + return 0; + } + int skip_next = 0; + struct stack * symlink_stack = newStack(STACK_MIN_SIZE); + char * symlink_target_path = (char *) malloc(SYMLINK_TARGET_BUFFER_SIZE); + while ((ptr = fts_read(fts_ptr)) != NULL) { + if (ptr->fts_info == FTS_DP) { + continue; + } + if (strncmp(ptr->fts_path, dir[0], strlen(ptr->fts_path)) == 0) { + continue; + } + if (skip_next) { + skip_next = 0; + continue; + } + char * file_type; + strcpy(symlink_target_path, ""); + int exists; + switch (ptr->fts_info) { + case FTS_D: + file_type = "directory"; + exists = 1; + break; + case FTS_F: + file_type = "regular file"; + exists = 1; + break; + case FTS_SL: + file_type = "symbolic link"; + int bytes_written = readlink(ptr->fts_path, symlink_target_path, SYMLINK_TARGET_BUFFER_SIZE); + symlink_target_path[bytes_written] = '\0'; + // realpath(ptr->fts_path, symlink_target_path); + if (symlink_target_path == NULL) { + strcpy(symlink_target_path, ""); + } + exists = 1 + access(symlink_target_path, F_OK); // test for file existence + if (!exists) { + break; + } + // checking whether it is a directory: + struct stat target_file_stat; + int stat_rv; + if ((stat_rv = stat(symlink_target_path, &target_file_stat)) != 0) { + fprintf(stderr, "Error reading the file %s\n", symlink_target_path); + return stat_rv; + } + if (S_ISDIR(target_file_stat.st_mode)) { + // if target is not local, we skip it + if (strncmp(symlink_target_path, local_dir, strlen(local_dir)) != 0) { + file_type = "non-local directory"; + break; + } + file_type = "directory"; + // if target is within the directory we are searching, we skip it, + // to prevent searching a directory more than once + if (strncmp(symlink_target_path, dir[0], strlen(dir[0])) == 0) { + break; + } + fts_set(fts_ptr, ptr, FTS_FOLLOW); + symlink_stack = push_symlink(symlink_stack, ptr->fts_path, symlink_target_path); + skip_next = 1; + } else if (S_ISREG(target_file_stat.st_mode)) { + file_type = "regular file"; + } + break; + + default: + file_type = "unknown"; + exists = 1; + break; + } + if (!isEmpty(symlink_stack) && strcmp(file_type, "symbolic link") != 0) { + while (strncmp(head(symlink_stack)->src, ptr->fts_path, strlen(head(symlink_stack)->src)) != 0) { + delete_top(symlink_stack); + if (isEmpty(symlink_stack)) { + break; + } + } + if (!isEmpty(symlink_stack)) { + strcpy(symlink_target_path, head(symlink_stack)->dst); + int fts_len = strlen(ptr->fts_path); + int to_replace_len = strlen(head(symlink_stack)->src); + int rel_len = fts_len - to_replace_len; + char rel_path[rel_len+1]; + memcpy(rel_path, &ptr->fts_path[to_replace_len], rel_len+1); + strcat(symlink_target_path, rel_path); + } + } + fprintf( + file_ptr, + "%s;%i;%s;%s\n", + ptr->fts_path + prefix_len, + exists, + symlink_target_path, + file_type + ); + } + free(symlink_target_path); + deleteStack(symlink_stack); + fts_close(fts_ptr); + return 0; +} diff --git a/plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_linux_aarch64 b/plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_linux_aarch64 deleted file mode 100755 index 44c4e37c7f335418a7742cb39761856684465c6b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25312 zcmeHv3zSsVneIMys=BIQ)lX;;rB6d6i1veryd$TZmyGy=j+il-qPnWOyQI3Rsj7xX z$j~6sOvs(k1B!+y1)7OsGQ*0=)t*Tr(KzE8$pl5c#HeY|j7$>OMKla$#G3ElkE&DE zRT|g5v)0^o-QDNx{eS!a|NZZOzs}jG&e^tX)pDDrF%@j=2}YDzZgNPR85{WmCnYwI z6*H4HvdOFnh#iN?b0`)m6{I%1NWXA{6Yo5>R@o*-bRVr3!44w)w+Q&J*s zoGM5aGhQ2UCR@)miTpE7Z9Jjkabys}G#M*7GLPggNVyAAPQ@eAFcqu%QQhe1koD`Z znV^U^DPjgcXJX=|aMN$%e(sk>LC+4-l2O3AodmRHa((*AKW zmepk~pASfqJ|s5j{+-W4}|o|i;!b@G%rRUtBPxCP1T z5Rb=kiKH(>JPt=K4yxCsI7)Gh!9ltv;;6upFGy3fvZ!5(vycoyc9q~DUzX!gbtD_> zag^bp`c&hn!7&yG^-F6%q%w(@WO+LXCEOIk=bzg}RVWx%BZu_ohIRn?a8jQSzq
IecXuul^MSRTL8Hqw79`kS($`3)90I>L3 zZ=lx)_AZgzD}+e6&l~SyR1j$;_E;beX@4LZCFOohUF$_@D86e`G#C$%6~6VJ?m#>q z>tXb*gjJXi&?u~?{n1?)N`x}x)(!MXCEL< z*gj6%n1dz|gpn4}J{l&!toc;78%>Hdf2BjK;I{#nDtMO5g7ZmF3RGI~^wNsFaTc8B zT@?%qZk=DJS#UnTN`Y1jZe3sIS@3khTNwEn1Ys|BaIOoeq8 zoYoQ*dMvnF^C+*+f?NGRV8K;e691qDr}asNuUc^H{I=VI)7quNeHNVNL=_SioYq&tMHr!SJwcPcie)jYeB+KT5#*aI%2`;8bXCL z7M!jfRPZ$FFJIj9)T0-5Hgt%wy%)Fq^pT^w=M|NI7J0j2M%70vl9i_y4;i-lSLzM@ zWpyC$Y?aMSR@xZP<8%?|Nfx>Ubi0LigYLA@)u4A+=&_&=S?EhZKW(8Wf*Y?<wL92dHD5m~%85?|YB97na?C>ewOrD<1hW-ux39~CM6ONu^uEaUky5pPCAnPw2F9MB^x?J>ZDrecHGJ5%=cnMue0{wAEi>84 zWo>C$D)Zu&pYXBqUMWlNL7D3&Eu9aYSHA%}4K}!X-6Z?`_e#y=#Vt?saq|}Jy}0Ga zTxO)64Kr=<#oNF;Qpb`bb!JjygVnHgc%*)CnEih1i4NqQg5P+aUGSW$H;1tv4HI^8 z%QIYWY6*K`#EE>TnZztUTyJMHM(Pe+f*XTsFK+oqQ7+0lRl|~}D}+AchrTM( z$L!`Xe73$#XB~=W1NCc8kd0FXzs+uPTZxx#HTSSHTTXBrb=%?D%HFA{i?mmR{t=?7 zUMS-_u8VYxE}!$%n91`WrS@TM9jYbX$t-DOJ6=2md}OkjTtirenS4j`loDMpd4P}9 zo5?>TjQpu=g$>+R_|#x**D2^hS-c(vd7wwt;o_EOxt~>gkC5+B)*+OG^?$>l#yZgM z$EokZpUF4TW>4D&Uu;0XXN(FDKN}i*ZK4u3W@#x^8*am%E#v~d0;d>!N zb-GcOt)WijKl0JYLWL2L4Coh5*WzoEZc6@go+KAHU38(oK{{BS{54AO}m1_s( ze8Kw#jRR^ER?6G_a@auaH;>v6`c6X!&p%p*zKQWdIx**wFQ{A%^EIxgvvjKR(_>;p z=&q2u8*=zhLoY951Z80CI3!;k*)Ub~`vlffp0=|M40WY4Ps4^&u%F7qwfo+Y8`&`K zHL~y?P2e=P$>zTy`)?#4t9kXXSVz}rkN)8{+oS*C(;rQ)w?F#FnB&pZGwg-LVwOCJ zetT#xOETNw>ckRl*oLuma7o*6VzHUL3;GfVSTb=y_~W2utdM_+huR|Ti*@C^j>ouP z;b)E2Y8dAv|2XN|4_*6(E-&g#b3hAto?5F78%UE66MxT=gVzkcxSsg;vE(JlPwYWC zKQcGhk7FC!AUB4vwQYRPSO=QqvG-D&*vECxPxH584MD%CM_lGc4F3*-zupjjUJrj# z+h9jnqI-4GxrpIqCd|seRN|2a%@o#zKzf$pmi;T2CvuA6ivCrC+F>Yb@>kmeg}fuQ^P5 z>{8DN#!EYVco6MHZ9a+YqPByN$ab0!$WGoD(9Yc7Xgk~^X6!Z04X03V-Y(MrG_E+q)i9QSjQnX_w#nc`~v%_-bYJW2aPG>|A1^oe5aIa zK))gSm!NswQIAVe55isoM*aR!+7Fi=9)T~OmU_2eOwks6+Lo}?+2+DH$2IzU(hpCQ z9<0;+I*c&*=w0YgKKmM_C!jx|%qL`-Um>hc8$LJzW3;XfO<&@8Op4M%&!BFB(jA8gfBVW{QAI5yQOO;J>)V~sUqQ8@!HQR@Ie*;Z*I}B`_ z4LTt2B6;c?v(2Q#k}d+Ra5fKP3p@?L8XDnmq}!2hpJpZ>LfVJ4?-c8eeiDL(++3mBVN9Kc;x!wLEnVz zdD8P3{Q20YTR#@T7WlV%7V5+2PsmRu`!Gi*(2mqLYW4gRd39YN+4A&P%C|!~^Yjy9h7^K=8uQu zx}(amtUJf@tUDVZ!{;-aU!RckV7E5BQQqo>^BCj|7g>XU5`+H)_Me*%fXKY*CUkijuoGtUx z>k;^G;WnAcU&}nc9s$1^IIsHwxgG&)1x91~qYtwC?nEh@wH|^0UnHKj9s!?7daUaa zd~`i{mCybS`V%ea58$nlWu7AJN~}jMZNrHx<$5#~JX3Slqi_6QUXL*MqVEkQc8T%8 z*P|=tdL+wEuSdJk-^tFYay}X(FMku zxYxJd2*!*+Z(n>fE(|^&^L7Uo7*k@#ZNYF?FcgSwjr7NP;!DP&Xdu=fihGC*dwT=d z7>n>6%p3CH$r;i(d*Ts1KBK4n*Oc&LWjXOk#OMlaGQ835{@y@19%~TUjOQEtL0Y)m zi1!2xdZK1Tx(vT}GkByCOc|l-BK2+zL}S57*y!z##f{E@F?C98su79un3pgm6be#8 z*?|JM$p(xK_~Mc1W+CH`1Y$-w5;ySNF&1x@)@8|MB;pZ6RI;$_CTXasA3YQd%KDM> z;9psS$d5<8ye5(*Q&O5Q4N1MxKzD!08#PE9L9xxfosm$`XYi+$(r$wuU*<1290@n& zID%{L53dhLHig-g*up8ng>(+7h^IoqOhi=xh5r)piwb4r1fWgO7h*iHDjcqVPo=(# zcmVNH#E&3;3Go@kXAqD3Z7Nl!;Wr$J4a9wjO~iW<--`I;?^3AlplO5KO}eZ)^9 zzV@9|>LlW0|2vhsfVlPDREmCwa^!p}^+m*IK1`(&h4B05i)}XwLCSv%$F1;P2r=@xJztP|Dc=schmn88l7G7;zY+Pf;d9{& zA+PKrc`COD`8&y{ZSFzsfyxIfYzMdo%J(Dx7}|#7w0&zu0VbfGz$?*agY*Em%{^j! z;Hn3&Y%9NP`JU-}uefL0y;JvHe&3Y)8^1Yu|Dg4^pYENJEGDfa^hL`<|k`jwScl@70&>v)!km+M--)6yE-j`8s4&JCf*U;WH0> z=7G;V@RU84AZ`g08%tWPvdazfw0kB9V}<2S402 zx^SaOcdTtqebETNt!u-LFMb~Ci}yzZ#*EhHSxqxrXU=GDooh_b(kn?VW23Xc{**){L|2aoJ2F|6L5H{Q0Z6{0v-qi~POi zCRd>PJCp5A*SZwc_%0LVsI#ztOk&V5F;Va6ol;tUFk4Sk%=c<{` zOna)_uPU*8+;0Wu=5REm*xjjW?fkvCl zd4?jbh~7NWi*V8|ucKcI+H8&(a`YaYv}u<%0@8m=*;mxAN7`<8;J3g`Uqac_$6iSi zF2@3{%e9zv{T?UVB{d!rbP+JF#-#M^q;-7l8cHvt^d+@XN9GdT+map9jS5`o!z?#Tu0!&9Hg^5LEyvCTk;qO>FoX?fkz2E%0W82 z{{<(PNwzu9x$2NvOdA5u_oZ#8fH?n9aTE-;;z}aU*M1v>y?7}R$(m*85H42oGS^6F z_oc|FyalIAotoy_ibPS2cy&$aEP9*@uxmnT(G5g7G&i-1y$Cy!%vq$3nNDhpzD9XY zkvEQ&Ohsn7i_Y%tI8|JZ?2@l@kk0OV3H&mF2RTS*_hST(C-8rAkj`%MLdCNLKF>iq zyI&^oiv*5vkj`%EkQKDoQSt{4(%GHjNoq-V37k>xx{H!ycyd1_X;hTE@Y@UKzLF=Y zEcYCqBqzJuc#@psrcPDvq7mb!K2q-bTS|8FuPkunjzeZB0 zdO0WZ&h4r~+U~4(QL`0?D5Ig`ex$+nK2ck$A4a;W()p_61Hjc}qw}>Ap8f!qI$tj> zK?79R68=W%B}lW<`*0$Z_sI%YN-8La(@s`O4M@$#X`Q(SQBtMzq>Fk;@sCOCuZxdy zBT8xXe_pXVS&iXpci%;9e+M?#Qa6oDRy-aGT+3X$h_N^fN!Rky{U9n?2?_H-LbNQf z^y?6MEJN3lyIm`EQfx2QJ8UNM5op4+vx!uan8MxwA5CFb0kXwFXmzCvKo{#zHkgRv zH8B<0XAwa^X11|4T%OJphNKEHNq!5codr0@Z;(^=s?5%v%W0C3bDI7W@Yr6cTLx|R z**K|rQ_Z5*X~q!(106-o>7uF4bvI7M{Pfq9q7xd6PoPfMUJy>#H*v-cO*3cdV<=2( zs_+6Tr72x&;pwtx316or8cgI+B}CA^ReqSbmTIa7ThlpvPzQH{=Z+-<)ODo$A2>Fd zOt}@uY8>vDN&nJT!Vcqj7)QmoiEnMCiSz{==W$eiQ5rytA%(A?kg6clZml5KOa?Up zNA;j&tI=+<-NkCOn{9XFyxR6vFx4s@WFQ;!1C&CCHm_W^;5VpyVOJHe0D|E-@-oYu zR)x8`N~_j%_XPJ8cMVo!rd4T3l@d~ki&tACD=obSB)Z2~rn?uEE-am2x)P7i>@o){ zuA@dPEk%xFeyN6>qH@6qa%IgtTv0ovBnMzl4GQD}P(=d8Qh;kFfs)G^6;Y~bD{F|M ztO0ZjGRiR`dFk%CoTH)!MJ>WjL}gm1G@`0b*d&Xnwibcc?WDyrw|2IrYG2h#ZFKpa z>j^8*;_<|yXIx+x7DEUw{d`e=hr-p)PjISG<4_eZYJee5RcY54ZR&XLFbA)FNv8bL zZ0pD+WzB5yc-3N4$5Y|vxP9%&DR0*D=8$T)!{Q2C-4pHT1n7=R&&nE!RhGg()lL$r zLbgkF44Ga1S35K?z|&vbfoDqNhT7q;T7dZZ$_&ViA>$T}54)9O?(kgStp%zxy5O~I ze{`2K4S^W(-`w01+Z^);`eH5eEm1twpqHeY*HAt>l}%ma=}p??hROKX#~RmNck^UC z-R?`Bd931g^wJA+Ml z0u^ZXQFYm596F;lXHJ2+cwJ~m6$(k}Hyf6Fo`=7j#d}{ z`f;XB%E0O#K!1~9&b%C6(AoF$mEPj_#=R}K2g2)v;aJP2NOXOy&+7}c;2x?8j|0U$ zRExMRYU%99TXO34V0a-MkZ%lkhx=O~AIIx@{8i#^pHKCVTsFGjYMudZU#Q<7U<|HD z(Z-y~8z{dPBi@JQQf6w=BJ3(CF=-3-!&JZ8UWoO*hUon$SR>Lu*60sSEME z`Di?h!vM(MK(ss1gf7tJi%?&{^JAdVNI#T?yP+%ILtKcO0?}wV0wNrNLOk21TXVW0 z$E|u(S3DGJ>W%mVq0dpj2zoeL~;xlH>Gy21^V0Sp+w^*wnUO3F278@|S#I)&n5P}H1g1K!itLp$s3#B!qa>P#GuXh)tGOf7&JwKz zX+lU^)hQ!`*Ll;PnKjco|EA%2;6P9H#J$}aIlQwg8tMk#M$)WIzL$(4M=;OI8}Z66 zI;p20FD=u077zNGq5(X4$Ml_NK+5TvT00o`c%xD8X7SE1DtAW!)sU|bW8~pi7~vob zgg5qCXS)A31MsGmOFrc=#BW0|7Ew)WCCLvAIU505e1d7Dfct1x%nl1Bsj-_>vM&vF zsie9!u(S8Hn+{}a$fcATjoqx0xG>;BW8Erg)p+xtta0^djXn7^t{$aPIv& z(3oE&GG|4m032{dO)Q+B&IQ+O0JWT&hFfHKoLC|KAFly*w^MkA?ixw5xLK( zE-}=~rLHa1s-@&qoU2dCjg$L^N>1JTD!D%;*5RmmHggm!y+hxT7uVT9UfiC3PGME1 zw*b_l2|Z3YM=ObnJns@6%-SSFz)brWN989be#3&ylQ|I;&yo`B64ri|_BW zSs|%s3_B^G?^rW+e4#}>_sPcV*jf1=cs7oEo*DV{tEXO8Zn`j5PrGvQ3E6<8biS)U zI!oLI@P8-GjwL8(<(t%!*=#=O3^AVd z`J?-f&|k&g%4v5U2IbQu_(%7v#scNeDu63{)O^Iv!-;mzY}m6_c&*G3{G-QDCCaI#Z9XO?Q1&@+0xKg0Em#sz-`JKA6>6+9Mr z6}ynruLPd4UkRN?qHueavkteZSb4036)4=*T{pu!E6{74vd3m;;Lj6JU z8&k9SOVB{rn_;;)m6Ok&alApRN6o8Oa6O~v)tQn%cYa>R@zL}Atz6IOdANZLLhI+Q zJ7Mrw^StcSAjivDLi*n>vv(KZKU4sJ4!bo5z9yDC&wn5Mm6q=jsnU-Y(DPCO{Otlb zIzL(^vyNc5m4_DKRBmO?`ce(N8sp`vDwFf63o?3aN4V;r3a7_RmGqUg96j_*s@gVs zzG^N|?gFk~*k(PiDu92f0KTCBerExk9){&>_s0w1{{)=so7;{j3h=)T+^|+irq37P zFVfLow$bxs4e)$+HWEL}tW&Cg&MLsajN|3(n4D*pqMazLF2L`X{A%7%HyLq>KO*%s z3N>tN0X>ZhNQZ%gR`yUr7F~vfLvwkDgiOYnOK&(N{lX1x*RNcXodehKO*(A9c$Qr&HC(%NHS?@kbz^(mD$k9}m#Lv>aT`As`3l=a;@ zYX)|yuytxhuuY1#fXjUASvassBs>AWb0^BiCJa0zKB#AvdL6?I7x`)1C z=q2YTczL|emY zN=S&AOGv0R6k~fFd%@ii=)$R)7a){jfU{=IiuA@Dj16@5cSFt_?#2`4|nUXvxE60uS?Sri^*Sf21H%;w+cPG&C70Ck`dw zVKYJDohk_!oXVWAjAaFUZ-<_trsW50CMdjbCFPHCO7KupU3V)z^h_GBi$zrV>i5Ri zVjZQ>C?Z}4C6&B{8SUC)wN00=_WSyzJUs%YK)*Agx+{6y3GlE4XJS$1t9`*f5e5;htwO_)l8wv8I zl27kP%8Dv^dD%6p{u`uxyHu!tU)?I@)$g%Mek_h$|7`(AewFUZ>~B(|ASgczO4zE> z6mP^~l~?zphLq1#Oh{FVD1F|ZBd>m|V9FoJ`)kvzJYi;&>~7H3^3{I$!wrHlN1XfG zs^Wi0dF6k#|N5kq|FX)ELdt(i2i{ZWR(G(%z&TX6b#5vrPaI6rZ3nw8Ec72K5~!ulAAE z_uNOQaS&AhQgW&v{44T^MairE=94WXMfXD4LCLH51hTF2YG3+{lo!8p&gf8bDt;|T zUfmC$l>@I(`AXwyBuJOCKYhPK6C=VW$-fOAvR@WQA|gonFDY8a(w$s}ptCp>CRup2 zPRh6DWaipK`)oNZTz>aVp>2$u#{@N+w-mzp`5&!ZflPAB$=*rcK%wLxX`t7g+*(q6sRTOjQ$b-{g-EHu>Z z-3~p9%m1|KVcw+L^BM=g>bHg}tm4Kb4DA1jen0XnHt{IwY>`5pRm(lYHgPFQ+SCJNWmHUcSBRo!2e|zmV&?mt@Fq z(jgftWKUf}j8lCk{z-=8;X6R|BBEV~|1+Tdc*tJ{I$bC3@^2ta z5?2X74Frl_A4S}ZwKWD!5ePKy2?mPt1MY9-SSZ*e@DXVb znk~>{B`hP20n}q=pfwn7BXf)#S_+q$|{s{;!vsutPRh51^( znrBztl;`KE3n~_iKwZQ70DKPZ49Cq-tYQ7?NVF}~5Zn<7!LOaI(KZ#mfD|275aU&Z zm9hwbKK?A`&myeOM05W6uxgIO=oZo>M&s@ScRc14t;19v{OmtqnJpDl6zz&|7#YSm zajn9?#r&~iuEM7mI^=OZ9?jHjVI_E=DOk^DNsftEm$U&zyT$d3CeWoS?>g$JWJFl~m z4N_(~)*j)P8Xf-aQ~`glzQ zT=$AX>w>s!$4*GTnUC&8kw>apN9P;fBx^rFMbCtID68@X<;AE{=1FSfa<)ID5N|)51NRt+8=hu!pl%Hckum zaCWAR(*ix5on+&*Fb`*aHeSZ~>+k9Q(n5^>+c+)2=)aBA!i)afI4!v7zm3yEi~id< zZMf*ajnjgQ{@XY$wCKN$(*leB+c+((=)aBAf{OmzI4z{;zs50|e_BDKxkm7K0>7}l z5P#K$|I&s3hYNqfg@4b5cf0T>UHCU#_}5+dmtFV~7oK+Eoi2R83s1Q4Ru>*};oDsJ z78kzWh1a_96)t?S3!m@8XS?ugT=-NcUUi|)I&+UK!`5=%W?x{*LINHvkF2!OU(4* zR|w2~&#;D!lds)soE#Zr_?|P)yk|~y9*)`yN2-C&Evye&p@#V4q0 zSg)EBjMR!x0?CB`l*?tBaIrmCas;?<8+7gVNk6+AOsZ^w9u_OVwg3Fnv<758jVWzd zwIfEVcBEH8DwnD~Z#-n294I!d-b|F)&k)zJ>aoVvj-?#WhPCB1RQppRUGL*cb@8%i5&>29^`OlLI02V3ipI{A z7#2CvZ=|YA4eKZ5+BtOENNvd&)++L=KUI#;j76VlsyEa0F7aW;A~i>oECNq@Gbc$T z+2f;nCz-Qqhm6$vK8O#Upc@#&`XTY=N>it)o_6TmE_KlL_0+2or_P&y+BAh&5jv%5b8j0S_ConLWZu5;)$(L|4VOS?m(3LEC+9&CR*lI*#{U(#mD3l`q(>-0$6rfe65(7vABxjOK?N z+}_MTD-oI`G=w8o`QeLw9K`gOu~l$!gRK z?4Ihd_8-D5?PxMm#k1#7BynX_^(6NnD!zXc%ybwu$x1%Flwv&bgV;pm1nGjP$c*6; zFPT7G3pQdr%$7Q!|oE7xrmm9=O&?&-+6!nq+MxqnEQ=pn5gy_ser^8&&>`|Tf`QriY^KqQij(d!h} z+w{b0_J}5gbZRTKyLcY<(|k}iCrg_>7`%O~z=L;`sI)^Q_m7Cgon)HUExHh9QVZ#c zSGlae183O!;C=|byNKO@)roUlf@f^OhUESig;|PEpE+ajCW@9?RI!%xM6TkoKR-Bq z6y7Gxd#m>RVR*)ir{6%|GTpEaf**wJ zv$!pQ{W++YeY<)`b8=*Cq9pqbP?Ozk#w^R*%N~)^r1@NdUfH*Mk|Pt$P53DL%u3vv zBu9Ma_~b~bd0qBKpqTQXLqd*LOQGA_nSqCJMPwIB=KZ5G-!GZbJX5)fZJ^Rf9U(u| zR{#8)9Aly&7!b?*NHsflXCa5}++q)s-Q-C8FwDe!J9#@Xe-!Xf6u63UwboKX@Oyj*)QW3E!DXdjM-vHh!vt5 z_XnLXBP>vt^R72j{RY%6Bm`-uXuCY&Pr)>HvMUf>lbzt`W`9K%(j_zVB-rEWEce$7 z81w`Gb{@}9m%(v|2a696`fwSmc6u{kLbyA7$~teR$&%s- zoS=J6Tvkii26yTU!z}f{6i5wT=TM(T7gOHadbXCpDBBGjrC!L$eG69VtW#jhes4v; zZ8=lQt-~#X^)g+jv$@U0=L-ED#DJ2&pQ1Tu4H(w54jYwh!_Xt*%oclJXGdkjnEq&B z78I;n1TzRMK)q|CMvClS9xf_tQrM!K=D&iY9W z=NiPZjvR-W(c9StfT$uRJV8qytN8l9OLboHj6b|(?CL3}E%3luF$5jNVea>hWiOhO zv8ZyYoIEsZgl_W(vzUGdUdf?u%*A%5>zzL~%V#q+nJG1V14goxLzL_R=v)f-QtZ3( zT91NJnc|Kid-k0~BuG%rz-JINsY9JfrI}T54OLQDtzSS*MYd&KYU}m_v>vIEgKN|m zpw)&0HB#;E1*p;DK-ZO*OaI)ZcM??#WUJ zthdhAS^d@-x@OZQk5SaD=j1N^O$v1KR5h&91xkaykz+3#Np@3PSX8d$q?Q*&lG*j2 z)Sx|cPth8aw{QVmHjjO@-Ta4XhV>-R9qY*B zz+9WBk?yQyhgDiT7pASw7a^GJrcYs3V4?l>4b<1dOzkFI+|-BL(9aZWWSg+L+k-2o zajKIVMPm9DoHIdOhAS!i&E8RZqJ$c*fJtdakXpT!V)z{05tIQ7i$rY*Q$PRu3lbF`LF(kVQ$n3!OuH@W2_@17)n0uV} zt{aM75YJ_I$?Sl1?H%q`@?Gw;NIGc#*xA8y&JKP+;YE?z!DVu~41p-KgNLPC0py+-hWt_4qJA8+9>ScP9_A79Or;EtRXkimJ8HtCouvd?UF=y!6XR@sM~M0}Ar zvyY!c${sGxKG=jztki%#%yW`cEji0*PIzP=Ih<~*4siCdEbo1OJ~^`w9nfodR@>PJ zO+cM}bg@nD)O6J;JNwY;kJ81az3$EY2!n@fgt8AUF@q&Ia*JU_W*;&toPB7yx2U%% zk(Zo!p(Xy!NB24^1>PG|N}2Wxh~W%ltXNEY8~INXg&H)xW>PR0WylK^0Z`WC;DJK}l zCigws0QKhe6|$M%+;8P3r2S+!HG_QMdYKQ%>#jF**Kb)x&x0rA8p1mh2J)BSj+zfV zL2Ax?03$@_z5HF5{p}Ww+of_T0+%9iDFT0;5uoSgsMN+{(b&!Y&GaCkKOFamTHDRN zGlkd^5AF=z?4KR?-xY3a4o5=qebI!;m2Ljzu~0k_G4WI+9!3ndhF1EQ?2sLhPeL`K8y`}tYJww->nCFG}v4*k() ze^YSpc%j-Bx@~Ri-cT$aj<)$*6LHhOBjmq+cKmvOG{$hem>r3PVcXF^GR!Xe;asTE zjK=m#ou+6g?r)2lemo%=H!GBu9T(EeYnV~L?39P@Zq*%n>N2dlLoq@awPD#`GZy3? zD4BcTeNMytFI!_wyQ0=yG@AM z@mpqxZ=uhK23(7T^MC?KC?MR)YfvYA)5qMh4}qmeB{ILv<&Fb({VJF12Rscp1bFal zF82`N$6v|iN|E870;F$U&H@^MA3B%IwE!LhJP26%YA*LU;6cFCfW=4;GJs9URZ1~U z9~sK!D&=(LXX#;D`2KE_@a-x0O`kBX^e|*`9?QQO$qeLjA0mM@wE^8B1=RLHK0|31)rNMDt!0T1Cn zh`w$nTJ7cKpDbEEact2?;F0v@UqOBle11Mn^Bb7WzWxCG&w^h|b~S&@avw>M{qy*L z3NdR2wE0EbZ2J?S_b&KfE8u_0=HCE*$FFm_tp)szHh&%XUEuGd*xK#Ki*)-D@TdGH zm)j1Q=ikZxQCts!-hg~-1$}|1^*4NG%%_VFl_bZG`Or8RBDp6Z7sfq9Iem?HvE17v z2hOE(DFT-wa47lNIt zV7r1H3LaMQn1aU@>`|~!!2tz_6vU4Txs)sDSFlpSY6a^R+^%4|f*lGTR`8gD#}({R zuus7O1>ODq?{NLQvD;R!zS%!#%Z@~wnebOtRV=EgoL`k-wCcVE3o0rX%~j+-Z;_^N zv3Q+N#+{NhEqve{QUmT+Ogs=WsyxlNB2g}eNWfA2s1x0ua@733fb4|ho&E7`Kk8r^fdi3R7UHsSHDiNohjn*B5ffLN3-)| zmj8fo&*zs|?FylPN3X5C#`4qbbwSgM(Q!O_a#5M;L0{lOU+Y2t80h|stP9bJ^~f)< znyUC~k$XMt&?6|L`T4L1y^HDPqD3ume6Nc}PAU37sd=(lHT(kTXJ&86N{W~7?S`YfiJm_s6^d#sM2lu!f_K^Ro2ffRK z{sYkcIw%hKl85{t=#z@B6nZ_+4p7jF*c9 zY7NCT)@%}i+tzJdRkJRzaqZg8wGDxWnpNv+1N_E;YW-t>{;__*^*RHayKs?wVL|&| zn?wry@`Tp*0@|PZ#lL!;LV>(~O+f*%;MD{+2d^1u3Q@56YYfpA+AS-AP*czhsy8dx z(&V6eFG2y02SmMLp^)KvPlC-{lz-ELO(W~n9df^bs9q>hKtp%pA)NOY5HX+NwS4^n zxr6sc6!7)S9}0=`#S#TH*LxJ=m@qbf;X;~~@o1nW*w%z^7vL_xzalT7UM67^@YaVw z-A1^DZyW*%xGmZf@s?fi;6W0L_?$ExLv`h=gM8q5@xAg(`NoB`Vrun9ngpI#1e>z{^GE zhnp0sW>wvMTFY3xB^YlJ6-|5F(1e6$OmfuMY6VCDyjUm_Bn4G#kC>u@yHtT{#m*>D zs>DN$qQVUA0ZNM-q@#QRR)kvAa@NuWYZ{X_-#DV{RPr=k5X9f1d0c{quPIcPNCr(C9cmY-2Ml_r!xwzug@_wG_c=O zAs;EuT3?@Mgh3+_t?ws66b90}-^|_3O$CP2xrlUyJGccn#?s!dcwy)BV3wziWY%*6AWSGTXvH|+NxD*x58VB6E<_%F~U5v{M!K?hcXh~oF? zzixjl_)dNMKDe6I?fEXTy#E@X0A;eJ+t=?uDzBCj+D|R-*A)dPsrD;fbge(nMJHdW zIH8_9k|RH>t0nAL3j^J^(j}$)t=C^VkJG+O4f#3kHVF^9CY)|p%hKgOn*OoBkrG$A I6kMqIFB2Q18~^|S From cfc9e45b36b85cab3b5efa8aa096dcc08549562e Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Tue, 1 Apr 2025 14:29:32 +0200 Subject: [PATCH 06/63] feat: register files in WOW file system --- .../src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy | 5 +++++ .../main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy | 3 +-- .../src/main/nextflow/cws/k8s/localdata/LocalPath.groovy | 5 +++-- .../main/nextflow/cws/wow/file/LocalFileWalker.groovy | 3 +-- .../nextflow/cws/wow/fs/WOWFileSystemProvider.groovy | 9 ++++++++- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy index a47c6ef..a0b3922 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy @@ -3,6 +3,8 @@ package nextflow.cws.k8s import groovy.util.logging.Slf4j import nextflow.cws.CWSConfig import nextflow.cws.SchedulerClient +import nextflow.cws.k8s.localdata.LocalPath +import nextflow.cws.wow.file.LocalFileWalker import nextflow.exception.NodeTerminationException import nextflow.k8s.K8sConfig import nextflow.k8s.client.K8sResponseException @@ -10,6 +12,8 @@ import nextflow.k8s.model.PodHostMount import nextflow.k8s.model.PodSecurityContext import nextflow.k8s.model.PodSpecBuilder import nextflow.k8s.model.PodVolumeClaim + +import java.nio.file.Path import java.nio.file.Paths @Slf4j class K8sSchedulerClient extends SchedulerClient { @@ -39,6 +43,7 @@ class K8sSchedulerClient extends SchedulerClient { this.k8sConfig = k8sConfig this.schedulerConfig = schedulerConfig this.namespace = namespace ?: 'default' + LocalFileWalker.createLocalPath = (Path path, LocalFileWalker.FileAttributes attr, Path workDir) -> LocalPath.toLocalPath( path, attr, workDir ) } protected String getDNS(){ diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy index f678b44..93885aa 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy @@ -99,7 +99,7 @@ class WOWK8sWrapperBuilder extends K8sWrapperBuilder { cmd += super.getLaunchCommand(interpreter, env) if( storage && localWorkDir && isTraceRequired() ){ cmd += "\nlocal exitCode=\$?" - cmd += """\necho \"infiles_time=\${INFILESTIME}" >> ${TaskRun.CMD_TRACE}\n""" + cmd += """\necho \"infiles_time=\${INFILESTIME}" >> ${workDir.resolve(TaskRun.CMD_TRACE)}\n""" cmd += "return \$exitCode\n" } return cmd @@ -110,7 +110,6 @@ class WOWK8sWrapperBuilder extends K8sWrapperBuilder { String cmd = super.getCleanupCmd( scratch ) if( storage && localWorkDir ){ cmd += "mkdir -p \"${localWorkDir.toString()}/\" || true\n" - cmd += "\"${getStorageLocalWorkDir()}/${statFileName}\" outfiles \"${workDir.toString()}/.command.outfiles\" \"${getStorageLocalWorkDir()}\" \"${localWorkDir.toString()}/\" > \"${localWorkDir.toString()}/.command.getStatsOut\" 2> \"${localWorkDir.toString()}/.command.getStatsErr\"\n" cmd += "local OUTFILESTIME=\$(\"${getStorageLocalWorkDir()}/${statFileName}\" outfiles \"${workDir.toString()}/.command.outfiles\" \"${getStorageLocalWorkDir()}\" \"${localWorkDir.toString()}/\" || true)\n" if ( isTraceRequired() ) { cmd += "echo \"outfiles_time=\${OUTFILESTIME}\" >> ${workDir.resolve(TaskRun.CMD_TRACE)}" diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalPath.groovy index a118afe..c89b3e1 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalPath.groovy @@ -1,6 +1,7 @@ package nextflow.cws.k8s.localdata import groovy.util.logging.Slf4j +import nextflow.cws.wow.fs.WOWFileSystem import nextflow.file.FileHelper import nextflow.cws.wow.file.LocalFileWalker import nextflow.cws.k8s.K8sSchedulerClient @@ -528,7 +529,7 @@ class LocalPath implements Path { @Override FileSystem getFileSystem() { - LocalFileSystem.INSTANCE + WOWFileSystem.INSTANCE } @Override @@ -620,7 +621,7 @@ class LocalPath implements Path { @Override URI toUri() { - path.toUri() + return getFileSystem().provider().getScheme() + "://" + path.toAbsolutePath() as URI } Path toAbsolutePath(){ diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy index a04539d..f09987f 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy @@ -1,7 +1,6 @@ package nextflow.cws.wow.file import groovy.util.logging.Slf4j -import nextflow.extension.FilesEx import java.nio.file.FileVisitOption import java.nio.file.FileVisitResult @@ -111,7 +110,7 @@ class LocalFileWalker { this.modificationDate = null return } - this.link = data[ REAL_PATH ].isEmpty() + this.link = !data[ REAL_PATH ].isEmpty() this.size = data[ SIZE ] as Long this.fileType = data[ FILE_TYPE ] this.accessDate = DateParser.fileTimeFromString(data[ ACCESS_DATE ]) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystemProvider.groovy index 199c208..4819b70 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystemProvider.groovy @@ -1,11 +1,14 @@ package nextflow.cws.wow.fs +import nextflow.cws.k8s.localdata.LocalPath + import java.nio.channels.SeekableByteChannel import java.nio.file.AccessMode import java.nio.file.CopyOption import java.nio.file.DirectoryStream import java.nio.file.FileStore import java.nio.file.FileSystem +import java.nio.file.Files import java.nio.file.LinkOption import java.nio.file.OpenOption import java.nio.file.Path @@ -94,7 +97,11 @@ class WOWFileSystemProvider extends FileSystemProvider { @Override def A readAttributes(Path path, Class aClass, LinkOption... linkOptions) throws IOException { - throw new UnsupportedOperationException("Read attributes not supported by ${getScheme().toUpperCase()} file system provider") + if ( path instanceof LocalPath ) { + return path.getAttributes() as A + } else { + return Files.readAttributes(path, BasicFileAttributes.class, linkOptions) as A + } } @Override From 9668dc4196ca517b257145e3f14bba8f1ef8c49d Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Thu, 10 Apr 2025 18:50:23 +0200 Subject: [PATCH 07/63] fix: solves dependency issues with FTP --- .../src/main/nextflow/cws/CWSPlugin.groovy | 2 +- .../cws/k8s/K8sSchedulerClient.groovy | 3 +- .../localdata => wow/file}/LocalFile.groovy | 2 +- .../localdata => wow/file}/LocalPath.groovy | 88 ++++++++++--------- .../cws/wow/{fs => file}/WOWFileSystem.groovy | 2 +- .../{fs => file}/WOWFileSystemProvider.groovy | 12 ++- 6 files changed, 61 insertions(+), 48 deletions(-) rename plugins/nf-cws/src/main/nextflow/cws/{k8s/localdata => wow/file}/LocalFile.groovy (89%) rename plugins/nf-cws/src/main/nextflow/cws/{k8s/localdata => wow/file}/LocalPath.groovy (89%) rename plugins/nf-cws/src/main/nextflow/cws/wow/{fs => file}/WOWFileSystem.groovy (98%) rename plugins/nf-cws/src/main/nextflow/cws/wow/{fs => file}/WOWFileSystemProvider.groovy (91%) diff --git a/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy b/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy index d127150..f38ce85 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy @@ -1,7 +1,7 @@ package nextflow.cws import groovy.transform.CompileStatic -import nextflow.cws.wow.fs.WOWFileSystemProvider +import nextflow.cws.wow.file.WOWFileSystemProvider import nextflow.file.FileHelper import nextflow.plugin.BasePlugin import nextflow.trace.TraceRecord diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy index a0b3922..378eba9 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy @@ -3,7 +3,7 @@ package nextflow.cws.k8s import groovy.util.logging.Slf4j import nextflow.cws.CWSConfig import nextflow.cws.SchedulerClient -import nextflow.cws.k8s.localdata.LocalPath +import nextflow.cws.wow.file.LocalPath import nextflow.cws.wow.file.LocalFileWalker import nextflow.exception.NodeTerminationException import nextflow.k8s.K8sConfig @@ -43,6 +43,7 @@ class K8sSchedulerClient extends SchedulerClient { this.k8sConfig = k8sConfig this.schedulerConfig = schedulerConfig this.namespace = namespace ?: 'default' + LocalPath.setClient(this) LocalFileWalker.createLocalPath = (Path path, LocalFileWalker.FileAttributes attr, Path workDir) -> LocalPath.toLocalPath( path, attr, workDir ) } diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalFile.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy similarity index 89% rename from plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalFile.groovy rename to plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy index 78b2ce9..0b17d09 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalFile.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy @@ -1,4 +1,4 @@ -package nextflow.cws.k8s.localdata +package nextflow.cws.wow.file import java.nio.file.Path diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy similarity index 89% rename from plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalPath.groovy rename to plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index c89b3e1..805c59f 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/localdata/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -1,9 +1,7 @@ -package nextflow.cws.k8s.localdata +package nextflow.cws.wow.file import groovy.util.logging.Slf4j -import nextflow.cws.wow.fs.WOWFileSystem import nextflow.file.FileHelper -import nextflow.cws.wow.file.LocalFileWalker import nextflow.cws.k8s.K8sSchedulerClient import org.codehaus.groovy.runtime.IOGroovyMethods import sun.net.ftp.FtpClient @@ -35,6 +33,10 @@ class LocalPath implements Path { this.workDir = null } + Path getInner() { + return path + } + LocalPath toLocalPath( Path path, LocalFileWalker.FileAttributes attributes = null ){ toLocalPath( path, attributes, workDir ) } @@ -48,8 +50,16 @@ class LocalPath implements Path { else throw new IllegalStateException("Client was already set.") } - private FtpClient getConnection( final String node, String daemon ){ + static InputStream getFileStream(final String node, String daemon, String path) { + URL url = new URL("ftp://$daemon/$path") + URLConnection con = url.openConnection() + InputStream is = con.getInputStream() + return is + } + + static FtpClient getConnection(final String node, String daemon ){ int trial = 0 + log.info("FRIEDRICH $node -- $daemon") while ( true ) { try { FtpClient ftpClient = FtpClient.create(daemon) @@ -65,7 +75,7 @@ class LocalPath implements Path { } } - private Map getLocation( String absolutePath ){ + Map getLocation( String absolutePath ){ Map response = client.getFileLocation( absolutePath ) synchronized ( createSymlinkHelper ) { if ( !createdSymlinks ) { @@ -154,8 +164,8 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.getBytes() } - try (FtpClient ftpClient = getConnection(location.node as String, location.daemon as String)) { - try (InputStream fileStream = ftpClient.getFileStream( location.path as String )) { + try (FtpClient ftpClient = getConnection(location.node.toString(), location.daemon.toString())) { + try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { log.trace("Read remote $absolutePath") return fileStream.getBytes() } @@ -183,8 +193,8 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.withReader( charset, closure ) } - try (FtpClient ftpClient = getConnection( location.node as String , location.daemon as String )) { - try (InputStream fileStream = ftpClient.getFileStream( location.path as String )) { + try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { + try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { log.trace("Read remote $absolutePath") return IOGroovyMethods.withReader(fileStream, closure) } @@ -244,8 +254,8 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.eachLine( charset, firstLine, closure ) } - try (FtpClient ftpClient = getConnection( location.node as String, location.daemon as String )) { - try (InputStream fileStream = ftpClient.getFileStream( location.path as String )) { + try (FtpClient ftpClient = getConnection( location.node.toString(), location.daemon.toString() )) { + try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { log.trace("Read remote $absolutePath") return IOGroovyMethods.eachLine( fileStream, charset, firstLine, closure ) } @@ -263,8 +273,8 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.newReader() } - try (FtpClient ftpClient = getConnection( location.node as String , location.daemon as String )) { - InputStream fileStream = ftpClient.getFileStream( location.path as String ) + try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { + InputStream fileStream = ftpClient.getFileStream( location.path.toString() ) log.trace("Read remote $absolutePath") InputStreamReader isr = new InputStreamReader( fileStream, charset ) return new BufferedReader(isr) @@ -316,8 +326,8 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.eachByte( closure ) } - try (FtpClient ftpClient = getConnection( location.node as String , location.daemon as String )) { - try (InputStream fileStream = ftpClient.getFileStream( location.path as String )) { + try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { + try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { log.trace("Read remote $absolutePath") return IOGroovyMethods.eachByte( new BufferedInputStream( fileStream ), closure) } @@ -331,8 +341,8 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.eachByte( bufferLen, closure ) } - try (FtpClient ftpClient = getConnection( location.node as String , location.daemon as String )) { - try (InputStream fileStream = ftpClient.getFileStream( location.path as String )) { + try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { + try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { log.trace("Read remote $absolutePath") return IOGroovyMethods.eachByte( new BufferedInputStream( fileStream ), bufferLen, closure) } @@ -346,8 +356,8 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.withInputStream( closure ) } - try (FtpClient ftpClient = getConnection( location.node as String , location.daemon as String )) { - InputStream fileStream = ftpClient.getFileStream( location.path as String ) + try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { + InputStream fileStream = ftpClient.getFileStream( location.path.toString() ) log.trace("Read remote $absolutePath") return IOGroovyMethods.withStream(new BufferedInputStream( fileStream ), closure) } @@ -371,8 +381,8 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.newInputStream() } - try (FtpClient ftpClient = getConnection( location.node as String , location.daemon as String )) { - InputStream fileStream = ftpClient.getFileStream( location.path as String ) + try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { + InputStream fileStream = ftpClient.getFileStream( location.path.toString() ) log.trace("Read remote $absolutePath") return new BufferedInputStream( fileStream ) } @@ -450,7 +460,7 @@ class LocalPath implements Path { def loc = client.getFileLocation( absolutePath ) } - private Map download(){ + Map download(){ final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) synchronized ( this ) { @@ -458,24 +468,20 @@ class LocalPath implements Path { log.trace("No download") return [ wasDownloaded : false, location : location ] } - try (FtpClient ftpClient = getConnection(location.node as String, location.daemon as String )) { - try (InputStream fileStream = ftpClient.getFileStream( location.path as String )) { - log.trace("Download remote $absolutePath") - final def file = toFile() - path.parent.toFile().mkdirs() - OutputStream outStream = new FileOutputStream(file) - byte[] buffer = new byte[8 * 1024] - int bytesRead - while ((bytesRead = fileStream.read(buffer)) != -1) { - outStream.write(buffer, 0, bytesRead) - } - fileStream.closeQuietly() - outStream.closeQuietly() - this.wasDownloaded = true - return [ wasDownloaded : true, location : location ] - } catch (Exception e) { - throw e + try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { + log.trace("Download remote $absolutePath") + final def file = toFile() + path.parent.toFile().mkdirs() + OutputStream outStream = new FileOutputStream(file) + byte[] buffer = new byte[8 * 1024] + int bytesRead + while ((bytesRead = fileStream.read(buffer)) != -1) { + outStream.write(buffer, 0, bytesRead) } + fileStream.close() + outStream.close() + this.wasDownloaded = true + return [ wasDownloaded : true, location : location ] } catch (Exception e) { throw e } @@ -498,10 +504,10 @@ class LocalPath implements Path { Object result = path.invokeMethod(name, args) if( lastModified != file.lastModified() ){ //Update location in scheduler (overwrite all others) - client.addFileLocation( downloadResult.location.path as String , file.size(), file.lastModified(), downloadResult.location.locationWrapperID as long, true ) + client.addFileLocation( downloadResult.location.path.toString() , file.size(), file.lastModified(), downloadResult.location.locationWrapperID as long, true ) } else if ( downloadResult.wasDownloaded ){ //Add location to scheduler - client.addFileLocation( downloadResult.location.path as String , file.size(), file.lastModified(), downloadResult.location.locationWrapperID as long, false ) + client.addFileLocation( downloadResult.location.path.toString() , file.size(), file.lastModified(), downloadResult.location.locationWrapperID as long, false ) } return result } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystem.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystem.groovy similarity index 98% rename from plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystem.groovy rename to plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystem.groovy index 14ce475..3a7d15b 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystem.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystem.groovy @@ -1,4 +1,4 @@ -package nextflow.cws.wow.fs +package nextflow.cws.wow.file import java.nio.file.FileStore import java.nio.file.FileSystem diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy similarity index 91% rename from plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystemProvider.groovy rename to plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy index 4819b70..9bfa155 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/fs/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy @@ -1,6 +1,7 @@ -package nextflow.cws.wow.fs +package nextflow.cws.wow.file -import nextflow.cws.k8s.localdata.LocalPath +import groovy.util.logging.Slf4j +import sun.net.ftp.FtpClient import java.nio.channels.SeekableByteChannel import java.nio.file.AccessMode @@ -17,6 +18,7 @@ import java.nio.file.attribute.FileAttribute import java.nio.file.attribute.FileAttributeView import java.nio.file.spi.FileSystemProvider +@Slf4j class WOWFileSystemProvider extends FileSystemProvider { static final WOWFileSystemProvider INSTANCE = new WOWFileSystemProvider() @@ -43,7 +45,11 @@ class WOWFileSystemProvider extends FileSystemProvider { @Override SeekableByteChannel newByteChannel(Path path, Set set, FileAttribute... fileAttributes) throws IOException { - throw new UnsupportedOperationException("Byte channel not supported by ${getScheme().toUpperCase()} file system provider") + LocalPath localPath = (LocalPath) path + // TODO: find an alternative to always downloading + Map downloadResult = localPath.download() + assert (downloadResult.wasDownloaded || downloadResult.location.sameAsEngine) + return Files.newByteChannel(localPath.getInner()) } @Override From 9470602c3ebccc8f2a2dc1b97939bf85f9c3234f Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Thu, 10 Apr 2025 17:51:15 +0200 Subject: [PATCH 08/63] feat: (empty) file transfer for WOW file system --- .../cws/wow/file/WOWFileSystemProvider.groovy | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy index 9bfa155..bbeccf6 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy @@ -1,9 +1,12 @@ package nextflow.cws.wow.file import groovy.util.logging.Slf4j +import nextflow.file.FileSystemTransferAware +import nextflow.file.http.XFileAttributes import sun.net.ftp.FtpClient import java.nio.channels.SeekableByteChannel +import java.nio.file.AccessDeniedException import java.nio.file.AccessMode import java.nio.file.CopyOption import java.nio.file.DirectoryStream @@ -19,7 +22,7 @@ import java.nio.file.attribute.FileAttributeView import java.nio.file.spi.FileSystemProvider @Slf4j -class WOWFileSystemProvider extends FileSystemProvider { +class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTransferAware { static final WOWFileSystemProvider INSTANCE = new WOWFileSystemProvider() @@ -94,6 +97,7 @@ class WOWFileSystemProvider extends FileSystemProvider { @Override void checkAccess(Path path, AccessMode... accessModes) throws IOException { + // TODO: re-check that this may be empty } @Override @@ -119,4 +123,25 @@ class WOWFileSystemProvider extends FileSystemProvider { void setAttribute(Path path, String s, Object o, LinkOption... linkOptions) throws IOException { throw new UnsupportedOperationException("Set attribute not supported by ${getScheme().toUpperCase()} file system provider") } + + @Override + boolean canUpload(Path source, Path target) { + return false + } + + @Override + boolean canDownload(Path source, Path target) { + return true + } + + @Override + void download(Path source, Path target, CopyOption... copyOptions) throws IOException { + log.warn("Work in progress: Implementation for downloading functionality may not be correct or complete in ${getScheme().toUpperCase()} file system provider") + // do nothing, as data downloading is handled by the WOW scheduler + } + + @Override + void upload(Path source, Path target, CopyOption... copyOptions) throws IOException { + throw new UnsupportedOperationException("Uploading not supported by ${getScheme().toUpperCase()} file system provider") + } } From 48a6ec3be66c2208929caf30f59d86d4586de7af Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Thu, 10 Apr 2025 18:08:45 +0200 Subject: [PATCH 09/63] debug: log usage of overwritten methods --- .../nextflow/cws/wow/file/LocalPath.groovy | 24 +++++++++++++++++++ .../cws/wow/file/WOWFileSystemProvider.groovy | 3 --- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index 805c59f..ca20a3d 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -58,6 +58,7 @@ class LocalPath implements Path { } static FtpClient getConnection(final String node, String daemon ){ + log.info("FRIEDRICH getConnection") int trial = 0 log.info("FRIEDRICH $node -- $daemon") while ( true ) { @@ -76,6 +77,7 @@ class LocalPath implements Path { } Map getLocation( String absolutePath ){ + log.info("FRIEDRICH getLocation") Map response = client.getFileLocation( absolutePath ) synchronized ( createSymlinkHelper ) { if ( !createdSymlinks ) { @@ -129,6 +131,7 @@ class LocalPath implements Path { } String getText( String charset ){ + log.info("FRIEDRICH getText") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) if ( wasDownloaded || location.sameAsEngine ){ @@ -148,6 +151,7 @@ class LocalPath implements Path { } void setText( String text, String charset ){ + log.info("FRIEDRICH setText") final String absolutePath = path.toAbsolutePath().toString() final def location = client.getFileLocation( absolutePath ) checkParentDirectoryExists() @@ -158,6 +162,7 @@ class LocalPath implements Path { } byte[] getBytes(){ + log.info("FRIEDRICH getBytes") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) if ( wasDownloaded || location.sameAsEngine ){ @@ -173,6 +178,7 @@ class LocalPath implements Path { } void setBytes( byte[] bytes ){ + log.info("FRIEDRICH setBytes") final String absolutePath = path.toAbsolutePath().toString() final def location = client.getFileLocation( absolutePath ) checkParentDirectoryExists() @@ -187,6 +193,7 @@ class LocalPath implements Path { } Object withReader( String charset, Closure closure ){ + log.info("FRIEDRICH withReader") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) if ( wasDownloaded || location.sameAsEngine ){ @@ -206,6 +213,7 @@ class LocalPath implements Path { } def T withWriter( String charset, Closure closure ) { + log.info("FRIEDRICH withWriter") final String absolutePath = path.toAbsolutePath().toString() final def location = client.getFileLocation( absolutePath ) checkParentDirectoryExists() @@ -217,6 +225,7 @@ class LocalPath implements Path { } def T withPrintWriter( String charset, Closure closure ){ + log.info("FRIEDRICH withPrintWriter") final String absolutePath = path.toAbsolutePath().toString() final def location = client.getFileLocation( absolutePath ) checkParentDirectoryExists() @@ -248,6 +257,7 @@ class LocalPath implements Path { } public T eachLine( String charset, int firstLine, Closure closure ) throws IOException { + log.info("FRIEDRICH eachLine") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) if ( wasDownloaded || location.sameAsEngine ){ @@ -267,6 +277,7 @@ class LocalPath implements Path { } BufferedReader newReader( String charset ){ + log.info("FRIEDRICH newReader") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) if ( wasDownloaded || location.sameAsEngine ){ @@ -298,6 +309,7 @@ class LocalPath implements Path { } BufferedWriter newWriter( String charset, boolean append, boolean writeBom ){ + log.info("FRIEDRICH newWriter") final String absolutePath = path.toAbsolutePath().toString() final def location = client.getFileLocation( absolutePath ) checkParentDirectoryExists() @@ -309,6 +321,7 @@ class LocalPath implements Path { } PrintWriter newPrintWriter( String charset ){ + log.info("FRIEDRICH newPrintWriter") final String absolutePath = path.toAbsolutePath().toString() final def location = client.getFileLocation( absolutePath ) checkParentDirectoryExists() @@ -320,6 +333,7 @@ class LocalPath implements Path { } public T eachByte( Closure closure ) throws IOException { + log.info("FRIEDRICH eachByte1") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) if ( wasDownloaded || location.sameAsEngine ){ @@ -335,6 +349,7 @@ class LocalPath implements Path { } public T eachByte( int bufferLen, Closure closure ) throws IOException { + log.info("FRIEDRICH eachByte2") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) if ( wasDownloaded || location.sameAsEngine ){ @@ -350,6 +365,7 @@ class LocalPath implements Path { } public T withInputStream( Closure closure) throws IOException { + log.info("FRIEDRICH withInputStream") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) if ( wasDownloaded || location.sameAsEngine ){ @@ -364,6 +380,7 @@ class LocalPath implements Path { } def T withOutputStream( Closure closure ){ + log.info("FRIEDRICH withOutputStream") final String absolutePath = path.toAbsolutePath().toString() final def location = client.getFileLocation( absolutePath ) checkParentDirectoryExists() @@ -375,6 +392,7 @@ class LocalPath implements Path { } public BufferedInputStream newInputStream() throws IOException { + log.info("FRIEDRICH newInputStream") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) if ( wasDownloaded || location.sameAsEngine ){ @@ -389,6 +407,7 @@ class LocalPath implements Path { } BufferedOutputStream newOutputStream(){ + log.info("FRIEDRICH newOutputStream") final String absolutePath = path.toAbsolutePath().toString() final def location = client.getFileLocation( absolutePath ) checkParentDirectoryExists() @@ -412,6 +431,7 @@ class LocalPath implements Path { } void write( String text, String charset, boolean writeBom ){ + log.info("FRIEDRICH write") final String absolutePath = path.toAbsolutePath().toString() final def location = client.getFileLocation( absolutePath ) checkParentDirectoryExists() @@ -439,6 +459,7 @@ class LocalPath implements Path { } void append( Object text, String charset, boolean writeBom ){ + log.info("FRIEDRICH append") if ( !text ) { return } @@ -461,6 +482,7 @@ class LocalPath implements Path { } Map download(){ + log.info("FRIEDRICH download") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) synchronized ( this ) { @@ -489,6 +511,7 @@ class LocalPath implements Path { } T asType( Class c ) { + log.info("FRIEDRICH asType") if ( c == Path.class ) return this if ( c == File.class ) return toFile() if ( c == String.class ) return toString() @@ -498,6 +521,7 @@ class LocalPath implements Path { @Override Object invokeMethod(String name, Object args) { + log.info("FRIEDRICH invokeMethod") Map downloadResult = download() def file = path.toFile() def lastModified = file.lastModified() diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy index bbeccf6..8d84948 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy @@ -2,11 +2,8 @@ package nextflow.cws.wow.file import groovy.util.logging.Slf4j import nextflow.file.FileSystemTransferAware -import nextflow.file.http.XFileAttributes -import sun.net.ftp.FtpClient import java.nio.channels.SeekableByteChannel -import java.nio.file.AccessDeniedException import java.nio.file.AccessMode import java.nio.file.CopyOption import java.nio.file.DirectoryStream From d40a9e15c57610a4c2301c9a546ea7b3ed718237 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 23 Apr 2025 19:19:11 +0200 Subject: [PATCH 10/63] avoid download for wow files Signed-off-by: Lehmann_Fabian --- .../nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy index 3165fb7..c523a83 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy @@ -24,6 +24,7 @@ import nextflow.util.Duration import nextflow.util.ServiceName import org.pf4j.ExtensionPoint +import java.nio.file.Path import java.nio.file.Paths @Slf4j @@ -296,4 +297,9 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { } + @Override + boolean isForeignFile( Path path ) { + if ( path.getScheme() == 'wow' ) return false + return super.isForeignFile(path) + } } \ No newline at end of file From b40a2ab33986606800d657942d57ea9dbbc5b23c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 16:14:06 +0200 Subject: [PATCH 11/63] Remove the need for the TaskProcessor Signed-off-by: Lehmann_Fabian --- .../cws/k8s/K8sSchedulerClient.groovy | 4 +- .../processor/CWSTaskPollingMonitor.groovy | 14 +++ .../cws/processor/WOWTaskProcessor.groovy | 97 ---------------- .../cws/wow/file/LocalFileWalker.groovy | 102 ++++------------- .../nextflow/cws/wow/file/LocalPath.groovy | 6 +- .../cws/wow/file/OfflineLocalPath.groovy | 18 +++ .../cws/wow/file/WOWFileHelper.groovy | 105 ------------------ .../cws/wow/file/WOWFileSystem.groovy | 15 ++- .../cws/wow/file/WOWFileSystemProvider.groovy | 14 +-- .../cws/wow/file/WorkdirHelper.groovy | 68 ++++++++++++ .../nextflow/cws/wow/file/WorkdirPath.groovy | 72 ++++++++++++ .../src/resources/META-INF/extensions.idx | 3 +- .../cws/wow/file/WorkdirHelperTest.groovy | 49 ++++++++ .../cws/wow/file/WorkdirPathTest.groovy | 42 +++++++ 14 files changed, 305 insertions(+), 304 deletions(-) delete mode 100644 plugins/nf-cws/src/main/nextflow/cws/processor/WOWTaskProcessor.groovy create mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/file/OfflineLocalPath.groovy delete mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileHelper.groovy create mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy create mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy create mode 100644 plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirHelperTest.groovy create mode 100644 plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirPathTest.groovy diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy index 378eba9..dd15460 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy @@ -4,7 +4,6 @@ import groovy.util.logging.Slf4j import nextflow.cws.CWSConfig import nextflow.cws.SchedulerClient import nextflow.cws.wow.file.LocalPath -import nextflow.cws.wow.file.LocalFileWalker import nextflow.exception.NodeTerminationException import nextflow.k8s.K8sConfig import nextflow.k8s.client.K8sResponseException @@ -13,8 +12,8 @@ import nextflow.k8s.model.PodSecurityContext import nextflow.k8s.model.PodSpecBuilder import nextflow.k8s.model.PodVolumeClaim -import java.nio.file.Path import java.nio.file.Paths + @Slf4j class K8sSchedulerClient extends SchedulerClient { @@ -44,7 +43,6 @@ class K8sSchedulerClient extends SchedulerClient { this.schedulerConfig = schedulerConfig this.namespace = namespace ?: 'default' LocalPath.setClient(this) - LocalFileWalker.createLocalPath = (Path path, LocalFileWalker.FileAttributes attr, Path workDir) -> LocalPath.toLocalPath( path, attr, workDir ) } protected String getDNS(){ diff --git a/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy b/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy index 7784618..c37df3e 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy @@ -2,6 +2,10 @@ package nextflow.cws.processor import groovy.util.logging.Slf4j import nextflow.Session +import nextflow.cws.wow.file.LocalFileWalker +import nextflow.cws.wow.file.OfflineLocalPath +import nextflow.cws.wow.file.WorkdirPath +import nextflow.processor.TaskHandler import nextflow.processor.TaskPollingMonitor import nextflow.util.Duration @@ -54,4 +58,14 @@ class CWSTaskPollingMonitor extends TaskPollingMonitor { return pendingTasks } + @Override + protected void finalizeTask(TaskHandler handler) { + def workDir = handler.task.workDir + def helper = LocalFileWalker.createWorkdirHelper( workDir ) + def attributes = new LocalFileWalker.FileAttributes(workDir) + OfflineLocalPath path = new WorkdirPath( workDir, attributes, workDir, helper ) + handler.task.workDir = path + super.finalizeTask(handler) + helper.validate() + } } diff --git a/plugins/nf-cws/src/main/nextflow/cws/processor/WOWTaskProcessor.groovy b/plugins/nf-cws/src/main/nextflow/cws/processor/WOWTaskProcessor.groovy deleted file mode 100644 index 373db1e..0000000 --- a/plugins/nf-cws/src/main/nextflow/cws/processor/WOWTaskProcessor.groovy +++ /dev/null @@ -1,97 +0,0 @@ -package nextflow.cws.processor - -import groovy.transform.PackageScope -import groovy.util.logging.Slf4j -import nextflow.Session -import nextflow.cws.wow.file.WOWFileHelper -import nextflow.cws.wow.file.LocalFileWalker -import nextflow.exception.MissingFileException -import nextflow.executor.Executor -import nextflow.file.FilePatternSplitter -import nextflow.processor.TaskProcessor -import nextflow.processor.TaskRun -import nextflow.script.BodyDef -import nextflow.script.BaseScript -import nextflow.script.ProcessConfig -import nextflow.script.params.FileOutParam - -import java.nio.file.LinkOption -import java.nio.file.NoSuchFileException -import java.nio.file.Path - -@Slf4j -class WOWTaskProcessor extends TaskProcessor { - - WOWTaskProcessor(String name, Executor executor, Session session, BaseScript script, ProcessConfig config, BodyDef taskBody ) { - super(name, executor, session, script, config, taskBody) - } - - @PackageScope - List fetchResultFiles( FileOutParam param, String namePattern, Path workDir ) { - assert namePattern - assert workDir - - List files = [] - def opts = visitOptions(param, namePattern) - // scan to find the file with that name - try { - WOWFileHelper.visitFiles(opts, workDir, namePattern) { Path it -> files.add(it) } - } - catch( NoSuchFileException e ) { - throw new MissingFileException("Cannot access directory: '$workDir'", e) - } - return files.sort() - } - - - @Override - protected void collectOutFiles(TaskRun task, FileOutParam param, Path workDir, Map context ) { - final List allFiles = [] - // type file parameter can contain a multiple files pattern separating them with a special character - def entries = param.getFilePatterns(context, task.workDir) - boolean inputsRemovedFlag = false - // for each of them collect the produced files - for( String filePattern : entries ) { - List result = null - - def splitter = param.glob ? FilePatternSplitter.glob().parse(filePattern) : null - if( splitter?.isPattern() ) { - result = fetchResultFiles(param, filePattern, workDir) - // filter the inputs - if( result && !param.includeInputs ) { - result = filterByRemovingStagedInputs(task, result, workDir) - log.trace "Process ${safeTaskName(task)} > after removing staged inputs: ${result}" - inputsRemovedFlag |= (result.size()==0) - } - } - else { - def path = param.glob ? splitter.strip(filePattern) : filePattern - def file = workDir.resolve(path) - def origFile = file - def outfiles = workDir.resolve( ".command.outfiles" ).toFile() - def exists - if( outfiles.exists() ){ - file = LocalFileWalker.exists( outfiles, file, workDir, param.followLinks ? LinkOption.NOFOLLOW_LINKS : null ) - exists = file != null - } else { - exists = param.followLinks ? file.exists() : file.exists(LinkOption.NOFOLLOW_LINKS) - } - if( exists ) - result = [file] - else - log.debug "Process `${safeTaskName(task)}` is unable to find [${origFile.class.simpleName}]: `$file` (pattern: `$filePattern`)" - } - - if( result ) - allFiles.addAll(result) - - else if( !param.optional ) { - def msg = "Missing output file(s) `$filePattern` expected by process `${safeTaskName(task)}`" - if( inputsRemovedFlag ) - msg += " (note: input files are not included in the default matching set)" - throw new MissingFileException(msg) - } - } - task.setOutput( param, allFiles.size()==1 ? allFiles[0] : allFiles ) - } -} diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy index f09987f..f04b2f5 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy @@ -2,11 +2,7 @@ package nextflow.cws.wow.file import groovy.util.logging.Slf4j -import java.nio.file.FileVisitOption -import java.nio.file.FileVisitResult -import java.nio.file.FileVisitor import java.nio.file.Files -import java.nio.file.LinkOption import java.nio.file.Path import java.nio.file.Paths import java.nio.file.attribute.BasicFileAttributes @@ -24,40 +20,19 @@ class LocalFileWalker { static final int ACCESS_DATE = 6 static final int MODIFICATION_DATE = 7 - public static TriFunction createLocalPath - - static Path walkFileTree(Path start, - Set options, - int maxDepth, - FileVisitor visitor, - Path workDir - ) - { - - String parent = null - boolean skip = false - + static WorkdirHelper createWorkdirHelper(Path start){ + Map files = new HashMap<>() + Path rootPath = null File file = new File( start.toString() + File.separatorChar + ".command.outfiles" ) String line + WorkdirHelper workdirHelper file.withReader { reader -> + line = reader.readLine() + rootPath = Paths.get(line.split(';')[ VIRTUAL_PATH ]) + workdirHelper = new WorkdirHelper( rootPath, files ) while ((line = reader.readLine()) != null) { String[] data = line.split(';') - String path = data[ VIRTUAL_PATH ] - - //If Symlink & not followLinks - //split path - //save it, and ignore everything at the path behind that - - if ( parent && path.startsWith(parent) ) { - if ( skip ) { - log.trace "Skip $path" - continue - } - } else { - skip = false - } - FileAttributes attributes = new FileAttributes( data ) Path currentPath = Paths.get(path) if ( !attributes.local ) { @@ -66,23 +41,14 @@ class LocalFileWalker { Files.createDirectories( currentPath.getParent() ) Files.createSymbolicLink( currentPath, attributes.destination ) } - Files.walkFileTree( currentPath, options, maxDepth, visitor) } else { - Path p = createLocalPath.apply( currentPath, attributes, workDir ) - if ( attributes.isDirectory() ) { - def visitDirectory = visitor.preVisitDirectory( p, attributes ) - if( visitDirectory == FileVisitResult.SKIP_SUBTREE ){ - skip = true - parent = path - } - } else { - visitor.visitFile( p, attributes) - } + def localPath = new OfflineLocalPath(currentPath, attributes, start, workdirHelper) + def pathOnSharedFs = start.resolve(rootPath.relativize(currentPath)) + files.put( pathOnSharedFs, localPath ) } } + return workdirHelper } - - return start } static class FileAttributes implements BasicFileAttributes { @@ -128,6 +94,18 @@ class LocalFileWalker { } } + FileAttributes( Path path ) { + directory = true + link = false + size = 4096 + fileType = 'directory' + creationDate = FileTime.fromMillis( 0 ) + accessDate = FileTime.fromMillis( 0 ) + modificationDate = FileTime.fromMillis( 0 ) + destination = path + local = false + } + @Override FileTime lastModifiedTime() { return modificationDate @@ -179,38 +157,4 @@ class LocalFileWalker { } - static Path exists( final File outfile, final Path file, final Path workDir, final LinkOption option ){ - - String rootDirString - Scanner sc = new Scanner( outfile ) - if( sc.hasNext() ) - rootDirString = sc.next().split(";")[0] - else - return null - - final Path rootDir = rootDirString as Path - final Path fakePath = WOWFileHelper.fakePath( file, rootDir ) - - //TODO ignore link - final Optional first = Files.lines( outfile.toPath() ) - .parallel() - .map {line -> - String[] data = line.split(';') - Path currentPath = data[ VIRTUAL_PATH ] as Path - log.trace "Compare $currentPath and $fakePath match: ${(currentPath == fakePath)}" - return ( currentPath == fakePath ) - ? createLocalPath.apply( currentPath, new FileAttributes( data ), workDir ) - : null - } - .filter{ it != null } - .findFirst() - - return first.isPresent() ? first.get() : null - - } - - static interface TriFunction { - Path apply( Path path, FileAttributes attributes, Path workDir ); - } - } \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index ca20a3d..e85d14a 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -1,8 +1,8 @@ package nextflow.cws.wow.file import groovy.util.logging.Slf4j -import nextflow.file.FileHelper import nextflow.cws.k8s.K8sSchedulerClient +import nextflow.file.FileHelper import org.codehaus.groovy.runtime.IOGroovyMethods import sun.net.ftp.FtpClient @@ -13,7 +13,7 @@ import java.nio.file.attribute.BasicFileAttributes @Slf4j class LocalPath implements Path { - private final Path path + protected final Path path private transient final LocalFileWalker.FileAttributes attributes private static transient K8sSchedulerClient client = null private boolean wasDownloaded = false @@ -21,7 +21,7 @@ class LocalPath implements Path { private boolean createdSymlinks = false private transient final Object createSymlinkHelper = new Object() - private LocalPath(Path path, LocalFileWalker.FileAttributes attributes, Path workDir ) { + protected LocalPath(Path path, LocalFileWalker.FileAttributes attributes, Path workDir ) { this.path = path this.attributes = attributes this.workDir = workDir diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/OfflineLocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/OfflineLocalPath.groovy new file mode 100644 index 0000000..23772fe --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/OfflineLocalPath.groovy @@ -0,0 +1,18 @@ +package nextflow.cws.wow.file + +import java.nio.file.Path + +/** + * We need this class to be able to iterate through a subdirectory of a workdir + */ +class OfflineLocalPath extends LocalPath { + + final protected WorkdirHelper workdirHelper + + + OfflineLocalPath( Path path, LocalFileWalker.FileAttributes attributes, Path workDir, WorkdirHelper workdirHelper ) { + super(path, attributes, workDir) + this.workdirHelper = workdirHelper + } + +} diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileHelper.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileHelper.groovy deleted file mode 100644 index ab1011e..0000000 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileHelper.groovy +++ /dev/null @@ -1,105 +0,0 @@ -package nextflow.cws.wow.file - -import groovy.util.logging.Slf4j -import nextflow.extension.FilesEx -import nextflow.file.FileHelper -import nextflow.file.FilePatternSplitter - -import java.nio.file.FileSystemLoopException -import java.nio.file.FileVisitResult -import java.nio.file.FileVisitOption -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.SimpleFileVisitor -import java.nio.file.attribute.BasicFileAttributes - -@Slf4j -class WOWFileHelper extends FileHelper { - - static Path fakePath(Path path, Path destination ) { - String pathHelper = path.toString() - String n1 = FilesEx.getName(destination.getParent()) - String n2 = FilesEx.getName(destination) - String cdir = (n1 as Path).resolve(n2).toString() - - int index = pathHelper.indexOf( cdir ) - - if( index == -1 ){ - log.error("Cannot calculate fake path for path: $path to dest.: $destination cdir: $cdir") - return path - } - - return destination.getParent().getParent().resolve(pathHelper.substring( index )) - } - - static void visitFiles( Map options = null, Path folder, String filePattern, Closure action ) { - assert folder - assert filePattern - assert action - if (options == null) options = Map.of() - final type = options.type ?: 'any' - final walkOptions = options.followLinks == false ? EnumSet.noneOf(FileVisitOption.class) : EnumSet.of(FileVisitOption.FOLLOW_LINKS) - final int maxDepth = getMaxDepth(options.maxDepth, filePattern) - final includeHidden = options.hidden as Boolean ?: filePattern.startsWith('.') - final includeDir = type in ['dir', 'any'] - final includeFile = type in ['file', 'any'] - final syntax = options.syntax ?: 'glob' - final relative = options.relative == true - final matcher = getPathMatcherFor("$syntax:${filePattern}", folder.fileSystem) - final singleParam = action.getMaximumNumberOfParameters() == 1 - - final boolean outFileExists = new File(folder.resolve(".command.outfiles").toString()).exists() - - def visitor = new SimpleFileVisitor() { - - @Override - FileVisitResult preVisitDirectory(Path fullPath, BasicFileAttributes attrs) throws IOException { - fullPath = outFileExists ? fakePath(fullPath, folder) : fullPath - final int depth = fullPath.nameCount - folder.nameCount - final path = relativize0(folder, fullPath) - log.trace "visitFiles > dir=$path; depth=$depth; includeDir=$includeDir; matches=${matcher.matches(path)}; isDir=${attrs.isDirectory()}" - if (depth > 0 && includeDir && matcher.matches(path) && attrs.isDirectory() && (includeHidden || !isHidden(fullPath))) { - def result = relative ? path : fullPath - singleParam ? action.call(result) : action.call(result, attrs) - } - return depth > maxDepth ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE - } - - @Override - FileVisitResult visitFile(Path fullPath, BasicFileAttributes attrs) throws IOException { - final Path fakePath = outFileExists ? fakePath(fullPath, folder) : null - final path = folder.relativize(fakePath ?: fullPath) - log.trace "visitFiles > file=$path; includeFile=$includeFile; matches=${matcher.matches(path)}; isRegularFile=${attrs.isRegularFile()}" - - if (includeFile && matcher.matches(path) && (attrs.isRegularFile() || (options.followLinks == false && attrs.isSymbolicLink())) && (includeHidden || !isHidden(fullPath))) { - def result = relative ? path : fullPath - singleParam ? action.call(result) : action.call(result, attrs) - } - return FileVisitResult.CONTINUE - } - - FileVisitResult visitFileFailed(Path currentPath, IOException e) { - if (e instanceof FileSystemLoopException) { - final Path fakePath = outFileExists ? fakePath(currentPath, folder) : null - final path = folder.relativize(fakePath ?: currentPath).toString() - final capture = FilePatternSplitter.glob().parse(filePattern).getParent() - final message = "Circular file path detected -- Files in the following directory will be ignored: $currentPath" - // show a warning message only when offending path is contained - // by the capture path specified by the user - if (capture == './' || path.startsWith(capture)) - log.warn(message) - else - log.debug(message) - return FileVisitResult.SKIP_SUBTREE - } - throw e - } - } - - if (outFileExists) { - LocalFileWalker.walkFileTree(folder, walkOptions, Integer.MAX_VALUE, visitor, folder) - } else { - Files.walkFileTree(folder, walkOptions, Integer.MAX_VALUE, visitor) - } - } -} diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystem.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystem.groovy index 3a7d15b..9aa8274 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystem.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystem.groovy @@ -1,10 +1,6 @@ package nextflow.cws.wow.file -import java.nio.file.FileStore -import java.nio.file.FileSystem -import java.nio.file.Path -import java.nio.file.PathMatcher -import java.nio.file.WatchService +import java.nio.file.* import java.nio.file.attribute.UserPrincipalLookupService import java.nio.file.spi.FileSystemProvider @@ -58,7 +54,14 @@ class WOWFileSystem extends FileSystem { @Override PathMatcher getPathMatcher(String s) { - throw new UnsupportedOperationException("Path matcher not supported by ${provider().getScheme().toUpperCase()} file system") + return new PathMatcher() { + private final def matcher = FileSystems.getDefault().getPathMatcher( s ) + @Override + boolean matches(Path path) { + // Make this a Unix Path and use the default matcher + return matcher.matches( Path.of(path.toString()) ) + } + } } @Override diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy index 8d84948..4cd7aec 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy @@ -4,15 +4,7 @@ import groovy.util.logging.Slf4j import nextflow.file.FileSystemTransferAware import java.nio.channels.SeekableByteChannel -import java.nio.file.AccessMode -import java.nio.file.CopyOption -import java.nio.file.DirectoryStream -import java.nio.file.FileStore -import java.nio.file.FileSystem -import java.nio.file.Files -import java.nio.file.LinkOption -import java.nio.file.OpenOption -import java.nio.file.Path +import java.nio.file.* import java.nio.file.attribute.BasicFileAttributes import java.nio.file.attribute.FileAttribute import java.nio.file.attribute.FileAttributeView @@ -54,6 +46,10 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran @Override DirectoryStream newDirectoryStream(Path path, DirectoryStream.Filter filter) throws IOException { + if ( path instanceof OfflineLocalPath && !(path as OfflineLocalPath).workdirHelper.isValidated() ) { + return (path as OfflineLocalPath).workdirHelper.getDirectoryStream( path ) + } + throw new UnsupportedOperationException("Directory stream not supported by ${getScheme().toUpperCase()} file system provider") } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy new file mode 100644 index 0000000..cc643af --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy @@ -0,0 +1,68 @@ +package nextflow.cws.wow.file + +import groovy.util.logging.Slf4j + +import java.nio.file.DirectoryStream +import java.nio.file.Path + +@Slf4j +class WorkdirHelper { + + private final Map paths + private final Path rootPath + private boolean validated = false + + WorkdirHelper( Path rootPath, Map paths) { + this.rootPath = rootPath + this.paths = paths + } + + void validate() { + validated = true + } + + boolean isValidated() { + return validated + } + + LocalPath get( Path path ) { + paths.get( path ) + } + + Path relativeToWorkdir( Path path ) { + rootPath.relativize( path ) + } + + DirectoryStream getDirectoryStream(Path path) { + if( validated ) { + throw new IllegalStateException("WorkdirHelper validated") + } + // If this is a local path, we need to check if the path is relative to the rootPath + boolean useLocalPath = path.startsWith(rootPath) + return new DirectoryStream() { + @Override + Iterator iterator() { + final def cp = path as LocalPath + def all = paths.entrySet().findAll { + Path toCompareAgainst = useLocalPath ? it.value.getInner() : it.key + def result = toCompareAgainst.parent == cp.getInner() + return result + }.collect { + it.value + } + return all.iterator() + } + @Override + void close() throws IOException {} + } + } + + @Override + String toString() { + return "WorkdirHelper{" + + "paths=" + paths + + ", rootPath=" + rootPath + + ", validated=" + validated + + '}' + } +} diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy new file mode 100644 index 0000000..26d467a --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy @@ -0,0 +1,72 @@ +package nextflow.cws.wow.file + +import groovy.util.logging.Slf4j +import nextflow.extension.FilesEx + +import java.nio.file.Path + +/** This special path is used to represent the workdir path in the local file system. + * This is required to calculate the correct relative path to the workdir. + */ +@Slf4j +class WorkdirPath extends OfflineLocalPath { + + WorkdirPath(Path path, LocalFileWalker.FileAttributes attributes, Path workDir, WorkdirHelper workdirHelper) { + super(path, attributes, workDir, workdirHelper) + } + + @Override + Path relativize(Path other) { + if ( other instanceof LocalPath ) { + def otherPath = ((LocalPath) other).path + if ( this.path == otherPath ) { + return Path.of("") + } + return workdirHelper.relativeToWorkdir( otherPath ) + } + return this.path.relativize( other ) + } + + /** + * This method considers that the current path != the local path. + * If a file is found in the local paths, it returns the local path. + * @param other + * @return + */ + Path resolve( String other ) { + def file = this.path.resolve( other ) + workdirHelper.get( file ) ?: file + } + + /** + * This method is used to get the relative path of a file in the local file system. + * @param workdir + * @param localPath + * @return + */ + static Path getRelativePathOnFake(Path workdir, Path localPath ) { + String n1 = FilesEx.getName(workdir.getParent()) + String n2 = FilesEx.getName(workdir) + Path p = null + boolean foundN1 = false + boolean foundN2 = false + for (final def part in localPath) { + if ( part.toString() == n1 && !foundN1 ) { + foundN1 = true + } else if ( part.toString() == n2 && foundN1 && !foundN2 ) { + foundN2 = true + } else if ( foundN1 && !foundN2 ) { + return localPath + } else if ( !foundN1 ) { + // n1 not found yet, ignore part before + } else if ( !p ) { + p = part + } else { + p = p.resolve(part) + } + } + return p ?: workdir.relativize( localPath ) + } + + +} diff --git a/plugins/nf-cws/src/resources/META-INF/extensions.idx b/plugins/nf-cws/src/resources/META-INF/extensions.idx index b3d0ac5..a8d9308 100644 --- a/plugins/nf-cws/src/resources/META-INF/extensions.idx +++ b/plugins/nf-cws/src/resources/META-INF/extensions.idx @@ -1,3 +1,2 @@ nextflow.cws.CWSFactory -nextflow.cws.k8s.CWSK8sExecutor -nextflow.cws.processor.WOWTaskProcessor \ No newline at end of file +nextflow.cws.k8s.CWSK8sExecutor \ No newline at end of file diff --git a/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirHelperTest.groovy b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirHelperTest.groovy new file mode 100644 index 0000000..ebcd5aa --- /dev/null +++ b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirHelperTest.groovy @@ -0,0 +1,49 @@ +package nextflow.cws.wow.file + +import spock.lang.Specification + +import java.nio.file.Path + +class WorkdirHelperTest extends Specification { + + def "GetDirectoryStream"() { + String localPath = "/localdata/localwork/be/8292aaebea2ddf9ae8ad4952882dcb" + String sharedPath = "/input/data/work/be/8292aaebea2ddf9ae8ad4952882dcb" + def localPath1 = LocalPath.toLocalPath(Path.of(localPath, "file.txt"), null, null) + def localPath2 = LocalPath.toLocalPath(Path.of(localPath, "a/file2.txt"), null, null) + def localPath3 = LocalPath.toLocalPath(Path.of(localPath, "b/file3.txt"), null, null) + def localPath4 = LocalPath.toLocalPath(Path.of(localPath, "b/c/"), null, null) + def localPath5 = LocalPath.toLocalPath(Path.of(localPath, "b/c/file4.txt"), null, null) + def localPath6 = LocalPath.toLocalPath(Path.of(localPath, "a/"), null, null) + def localPath7 = LocalPath.toLocalPath(Path.of(localPath, "b/"), null, null) + def sharedPath1 = LocalPath.toLocalPath(Path.of(sharedPath, "file.txt"), null, null) + def sharedPath2 = LocalPath.toLocalPath(Path.of(sharedPath, "a/file2.txt"), null, null) + def sharedPath3 = LocalPath.toLocalPath(Path.of(sharedPath, "b/file3.txt"), null, null) + def sharedPath4 = LocalPath.toLocalPath(Path.of(sharedPath, "b/c/"), null, null) + def sharedPath5 = LocalPath.toLocalPath(Path.of(sharedPath, "b/c/file4.txt"), null, null) + def sharedPath6 = LocalPath.toLocalPath(Path.of(sharedPath, "a/"), null, null) + def sharedPath7 = LocalPath.toLocalPath(Path.of(sharedPath, "b/"), null, null) + Map files = new HashMap<>() + files.put( sharedPath1, localPath1 ) + files.put( sharedPath2, localPath2 ) + files.put( sharedPath3, localPath3 ) + files.put( sharedPath4, localPath4 ) + files.put( sharedPath5, localPath5 ) + files.put( sharedPath6, localPath6 ) + files.put( sharedPath7, localPath7 ) + def helper = new WorkdirHelper( Path.of(localPath), files ) + def workDir = Path.of("/input/data/work/be/8292aaebea2ddf9ae8ad4952882dcb") + def attributes = new LocalFileWalker.FileAttributes(workDir) + WorkdirPath path = new WorkdirPath( workDir, attributes, workDir, helper ) + def stream = helper.getDirectoryStream( path ) + def result = stream.collect() + + expect: + result.size() == 3 + localPath1 in result + localPath6 in result + localPath7 in result + } + + +} diff --git a/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirPathTest.groovy b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirPathTest.groovy new file mode 100644 index 0000000..28d25f2 --- /dev/null +++ b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirPathTest.groovy @@ -0,0 +1,42 @@ +package nextflow.cws.wow.file + +import spock.lang.Specification + +import java.nio.file.Path + +class WorkdirPathTest extends Specification { + + def "FakePath"() { + when: + def path = Path.of("/input/data/work/be/8292aaebea2ddf9ae8ad4952882dcb") + def localPath = Path.of("/localdata/localwork/be/8292aaebea2ddf9ae8ad4952882dcb/file.txt") + def fakePath = WorkdirPath.getRelativePathOnFake( path, localPath ) + + then: + fakePath == Path.of("file.txt") + } + + def "matches"() { + when: + def path = Path.of("file.txt").getFileName() + + then: + path.getFileSystem().getPathMatcher( "glob:*.txt" ).matches( path ) + } + + def "resolve"() { + when: + def workDir = Path.of("/input/data/work/be/8292aaebea2ddf9ae8ad4952882dcb") + def localPath = LocalPath.toLocalPath(Path.of("/localdata/localwork/be/8292aaebea2ddf9ae8ad4952882dcb/file.txt"), null, null) + Map files = new HashMap<>() + files.put(workDir.resolve("file.txt"), localPath) + def helper = new WorkdirHelper( Path.of("/localdata/localwork/be/8292aaebea2ddf9ae8ad4952882dcb/"), files ) + def attributes = new LocalFileWalker.FileAttributes(workDir) + WorkdirPath path = new WorkdirPath( workDir, attributes, workDir, helper ) + + then: + def resolve = path.resolve("file.txt") + resolve == localPath + } + +} From 4a7212b893a5884be7ea9d5e8d715de4fc718691 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 16:19:59 +0200 Subject: [PATCH 12/63] asType on LocalPath returns itself Signed-off-by: Lehmann_Fabian --- plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index e85d14a..da27b2b 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -512,7 +512,7 @@ class LocalPath implements Path { T asType( Class c ) { log.info("FRIEDRICH asType") - if ( c == Path.class ) return this + if ( c == Path.class || c == LocalPath.class ) return this if ( c == File.class ) return toFile() if ( c == String.class ) return toString() log.info("Invoke method asType $c on $this") From 3389c60f0c89db7cac2662afa6a53beefe9f3422 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 16:20:51 +0200 Subject: [PATCH 13/63] relativize to localpath uses internal path Signed-off-by: Lehmann_Fabian --- plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index da27b2b..364ff04 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -646,6 +646,9 @@ class LocalPath implements Path { @Override Path relativize(Path other) { + if ( other instanceof LocalPath ){ + return toLocalPath( path.relativize( ((LocalPath) other).path ) ) + } path.relativize( other ) } From 55de3d60061cb499bdac7653efd07daef28cef14 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 16:21:10 +0200 Subject: [PATCH 14/63] add equals and hashCode to LocalPath Signed-off-by: Lehmann_Fabian --- .../main/nextflow/cws/wow/file/LocalPath.groovy | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index 364ff04..484b082 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -697,4 +697,20 @@ class LocalPath implements Path { BasicFileAttributes getAttributes(){ attributes } + + @Override + boolean equals(Object obj) { + if ( obj instanceof LocalPath ){ + return path == ((LocalPath) obj).path + } + if ( obj instanceof Path ){ + return path == obj + } + return false + } + + @Override + int hashCode() { + return path.hashCode() * 2 + 1 + } } \ No newline at end of file From b4e889b457a7928b93957b17950740b27ad66021 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 16:22:09 +0200 Subject: [PATCH 15/63] ignore compiled getStatsAndResolveSymlinks Signed-off-by: Lehmann_Fabian --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ff34859..fa803fb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ out /gradle.properties /logs.tmp *.jar +/plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_* From 516b47ce734cf896fa89244425d3fd5921fbd9f4 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 16:34:31 +0200 Subject: [PATCH 16/63] Remove old Java artefacts Signed-off-by: Lehmann_Fabian --- .../main/nextflow/cws/SchedulerClient.groovy | 2 +- .../nextflow/cws/k8s/CWSK8sExecutor.groovy | 2 +- .../cws/k8s/WOWK8sWrapperBuilder.groovy | 1 + .../k8s/model/PodMountConfigWithMode.groovy | 22 +++++++------- .../nextflow/cws/wow/file/DateParser.groovy | 30 +++++++++---------- .../nextflow/cws/wow/file/LocalFile.groovy | 4 +-- .../nextflow/cws/wow/file/LocalPath.groovy | 20 ++++++------- .../cws/wow/file/WOWFileSystemProvider.groovy | 4 +-- 8 files changed, 43 insertions(+), 42 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy index dbc6632..0b54224 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy @@ -226,7 +226,7 @@ class SchedulerClient { } String message = JsonOutput.toJson( data ) get.setRequestProperty("Content-Type", "application/json") - get.getOutputStream().write(message.getBytes("UTF-8")); + get.getOutputStream().write(message.getBytes("UTF-8")) int responseCode = get.getResponseCode() if( responseCode != 200 ){ throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while updating file location: $path: $node (${get.responseMessage})" ) diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy index c523a83..5ee9062 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy @@ -171,7 +171,7 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { Map configMap = [:] String architectureStatFileName - final String architecture = System.getProperty("os.arch"); + final String architecture = System.getProperty("os.arch") if (architecture == "amd64" || architecture == "x86_64") { architectureStatFileName = "/nf-cws/getStatsAndResolveSymlinks_linux_x86" } else if (architecture == "aarch64" || architecture == "arm64") { diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy index 93885aa..d847e07 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy @@ -6,6 +6,7 @@ import nextflow.file.FileHelper import nextflow.k8s.K8sWrapperBuilder import nextflow.processor.TaskRun import nextflow.util.Escape + import java.nio.file.Path /** diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy index d3b8372..4c5a722 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy @@ -1,21 +1,21 @@ -package nextflow.cws.k8s.model; +package nextflow.cws.k8s.model -import nextflow.k8s.model.PodMountConfig; +import nextflow.k8s.model.PodMountConfig -public class PodMountConfigWithMode extends PodMountConfig { - private Integer mode; +class PodMountConfigWithMode extends PodMountConfig { + private Integer mode - public PodMountConfigWithMode(String config, String mount, Integer mode = null) { - super(config, mount); - this.mode = mode; + PodMountConfigWithMode(String config, String mount, Integer mode = null) { + super(config, mount) + this.mode = mode } - public Integer getMode() { - return mode; + Integer getMode() { + return mode } - public void setMode(Integer mode) { - this.mode = mode; + void setMode(Integer mode) { + this.mode = mode } } \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy index 221bf19..5f41554 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy @@ -1,35 +1,35 @@ package nextflow.cws.wow.file -import java.nio.file.attribute.FileTime; -import java.text.SimpleDateFormat; +import java.nio.file.attribute.FileTime +import java.text.SimpleDateFormat -public final class DateParser { +final class DateParser { private DateParser(){} - public static Long millisFromString( String date ) { - if( date == null || date.isEmpty() || date.equals("-") || date.equals("w") ) { - return null; + static Long millisFromString( String date ) { + if( date == null || date.isEmpty() || date == "-" || date == "w" ) { + return null } try { if (Character.isLetter(date.charAt(0))) { // if ls was used, date has the format "Nov 2 08:49:30 2021" - return new SimpleDateFormat("MMM dd HH:mm:ss yyyy").parse(date).getTime(); + return new SimpleDateFormat("MMM dd HH:mm:ss yyyy").parse(date).getTime() } else { // if stat was used, date has the format "2021-11-02 08:49:30.955691861 +0000" - String[] parts = date.split(" "); - parts[1] = parts[1].substring(0, 12); + String[] parts = date.split(" ") + parts[1] = parts[1].substring(0, 12) // parts[1] now has milliseconds as smallest units e.g. "08:49:30.955" - String shortenedDate = String.join(" ", parts); - return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z").parse(shortenedDate).getTime(); + String shortenedDate = String.join(" ", parts) + return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z").parse(shortenedDate).getTime() } } catch ( Exception e ){ - return null; + return null } } - public static FileTime fileTimeFromString( String date ) { - final Long millisFromSring = millisFromString( date ); - return millisFromSring == null ? null : FileTime.fromMillis( millisFromSring ); + static FileTime fileTimeFromString( String date ) { + final Long millisFromSring = millisFromString( date ) + return millisFromSring == null ? null : FileTime.fromMillis( millisFromSring ) } } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy index 0b17d09..e58c04d 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy @@ -4,11 +4,11 @@ import java.nio.file.Path class LocalFile extends File { - private final LocalPath localPath; + private final LocalPath localPath LocalFile( LocalPath localPath ){ super( localPath.toString() ) - this.localPath = localPath; + this.localPath = localPath } @Override diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index 484b082..7cd64e3 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -244,19 +244,19 @@ class LocalPath implements Path { IOGroovyMethods.readLines( newReader( charset ) ) } - public T eachLine( Closure closure ) throws IOException { + T eachLine( Closure closure ) throws IOException { eachLine( Charset.defaultCharset().toString(), 1, closure ) } - public T eachLine( int firstLine, Closure closure ) throws IOException { + T eachLine( int firstLine, Closure closure ) throws IOException { eachLine( Charset.defaultCharset().toString(), firstLine, closure ) } - public T eachLine( String charset, Closure closure ) throws IOException { + T eachLine( String charset, Closure closure ) throws IOException { eachLine( charset, 1, closure ) } - public T eachLine( String charset, int firstLine, Closure closure ) throws IOException { + T eachLine( String charset, int firstLine, Closure closure ) throws IOException { log.info("FRIEDRICH eachLine") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) @@ -332,7 +332,7 @@ class LocalPath implements Path { return printWriter } - public T eachByte( Closure closure ) throws IOException { + T eachByte( Closure closure ) throws IOException { log.info("FRIEDRICH eachByte1") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) @@ -348,7 +348,7 @@ class LocalPath implements Path { } } - public T eachByte( int bufferLen, Closure closure ) throws IOException { + T eachByte( int bufferLen, Closure closure ) throws IOException { log.info("FRIEDRICH eachByte2") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) @@ -364,7 +364,7 @@ class LocalPath implements Path { } } - public T withInputStream( Closure closure) throws IOException { + T withInputStream( Closure closure) throws IOException { log.info("FRIEDRICH withInputStream") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) @@ -391,7 +391,7 @@ class LocalPath implements Path { return rv } - public BufferedInputStream newInputStream() throws IOException { + BufferedInputStream newInputStream() throws IOException { log.info("FRIEDRICH newInputStream") final String absolutePath = path.toAbsolutePath().toString() final def location = getLocation( absolutePath ) @@ -684,9 +684,9 @@ class LocalPath implements Path { @Override int compareTo(Path other) { if ( other instanceof LocalPath ){ - return path.compareTo( ((LocalPath) other).path ) + return path <=> ((LocalPath) other).path } - path.compareTo( other ) + path <=> other } @Override diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy index 4cd7aec..c8d7394 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy @@ -94,12 +94,12 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran } @Override - def V getFileAttributeView(Path path, Class aClass, LinkOption... linkOptions) { + V getFileAttributeView(Path path, Class aClass, LinkOption... linkOptions) { throw new UnsupportedOperationException("File attribute view not supported by ${getScheme().toUpperCase()} file system provider") } @Override - def A readAttributes(Path path, Class aClass, LinkOption... linkOptions) throws IOException { + A readAttributes(Path path, Class aClass, LinkOption... linkOptions) throws IOException { if ( path instanceof LocalPath ) { return path.getAttributes() as A } else { From 722769fde5cd3e4b61cc0c4655768735ad871383 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 16:39:45 +0200 Subject: [PATCH 17/63] Replace deprecated new URL Signed-off-by: Lehmann_Fabian --- .../main/nextflow/cws/SchedulerClient.groovy | 22 +++++++++---------- .../nextflow/cws/wow/file/LocalPath.groovy | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy index 0b54224..d4b8f5e 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy @@ -33,7 +33,7 @@ class SchedulerClient { int trials = 0 while ( trials++ < 50 ) { try { - HttpURLConnection post = new URL(url).openConnection() as HttpURLConnection + HttpURLConnection post = URI.create(url).toURL().openConnection() as HttpURLConnection post.setRequestMethod( "POST" ) post.setDoOutput(true) post.setRequestProperty("Content-Type", "application/json") @@ -59,14 +59,14 @@ class SchedulerClient { synchronized void closeScheduler(){ if ( closed ) return closed = true - HttpURLConnection post = new URL("${getDNS()}/scheduler/$runName").openConnection() as HttpURLConnection + HttpURLConnection post = URI.create("${getDNS()}/scheduler/$runName").toURL().openConnection() as HttpURLConnection post.setRequestMethod( "DELETE" ) int responseCode = post.getResponseCode() log.trace "Delete scheduler code was: ${responseCode}" } void submitMetrics( Map metrics, int id ){ - HttpURLConnection post = new URL("${getDNS()}/scheduler/$runName/metrics/task/$id").openConnection() as HttpURLConnection + HttpURLConnection post = URI.create("${getDNS()}/scheduler/$runName/metrics/task/$id").toURL().openConnection() as HttpURLConnection post.setRequestMethod( "POST" ) String message = JsonOutput.toJson( metrics ) post.setDoOutput(true) @@ -80,7 +80,7 @@ class SchedulerClient { Map registerTask( Map config, int id ){ - HttpURLConnection post = new URL("${getDNS()}/scheduler/$runName/task/$id").openConnection() as HttpURLConnection + HttpURLConnection post = URI.create("${getDNS()}/scheduler/$runName/task/$id").toURL().openConnection() as HttpURLConnection post.setRequestMethod( "POST" ) String message = JsonOutput.toJson( config ) post.setDoOutput(true) @@ -97,7 +97,7 @@ class SchedulerClient { } private void batch( String command ){ - HttpURLConnection put = new URL("${getDNS()}/scheduler/$runName/${command}Batch").openConnection() as HttpURLConnection + HttpURLConnection put = URI.create("${getDNS()}/scheduler/$runName/${command}Batch").toURL().openConnection() as HttpURLConnection put.setRequestMethod( "PUT" ) if ( command == 'end' ){ put.setDoOutput(true) @@ -121,7 +121,7 @@ class SchedulerClient { Map getTaskState( int id ){ - HttpURLConnection get = new URL("${getDNS()}/scheduler/$runName/task/$id").openConnection() as HttpURLConnection + HttpURLConnection get = URI.create("${getDNS()}/scheduler/$runName/task/$id").toURL().openConnection() as HttpURLConnection get.setRequestMethod( "GET" ) get.setDoOutput(true) int responseCode = get.getResponseCode() @@ -144,7 +144,7 @@ class SchedulerClient { uid : it.getId() ] as Map } - HttpURLConnection put = new URL("${getDNS()}/scheduler/$runName/DAG/vertices").openConnection() as HttpURLConnection + HttpURLConnection put = URI.create("${getDNS()}/scheduler/$runName/DAG/vertices").toURL().openConnection() as HttpURLConnection put.setRequestMethod( "POST" ) String message = JsonOutput.toJson( verticesToSubmit ) put.setDoOutput(true) @@ -165,7 +165,7 @@ class SchedulerClient { to : it.getTo()?.getId() ] as Map } - HttpURLConnection put = new URL("${getDNS()}/scheduler/$runName/DAG/edges").openConnection() as HttpURLConnection + HttpURLConnection put = URI.create("${getDNS()}/scheduler/$runName/DAG/edges").toURL().openConnection() as HttpURLConnection put.setRequestMethod( "POST" ) String message = JsonOutput.toJson( edgesToSubmit ) put.setDoOutput(true) @@ -182,7 +182,7 @@ class SchedulerClient { Map getFileLocation( String path ){ String pathEncoded = URLEncoder.encode(path,'utf-8') - HttpURLConnection get = new URL("${getDNS()}/file/$runName?path=$pathEncoded").openConnection() as HttpURLConnection + HttpURLConnection get = URI.create("${getDNS()}/file/$runName?path=$pathEncoded").toURL().openConnection() as HttpURLConnection get.setRequestMethod( "GET" ) get.setDoOutput(true) int responseCode = get.getResponseCode() @@ -196,7 +196,7 @@ class SchedulerClient { String getDaemonOnNode( String node ){ - HttpURLConnection get = new URL("${getDNS()}/daemon/$runName/$node").openConnection() as HttpURLConnection + HttpURLConnection get = URI.create("${getDNS()}/daemon/$runName/$node").toURL().openConnection() as HttpURLConnection get.setRequestMethod( "GET" ) get.setDoOutput(true) int responseCode = get.getResponseCode() @@ -212,7 +212,7 @@ class SchedulerClient { String method = overwrite ? 'overwrite' : 'add' - HttpURLConnection get = new URL("${getDNS()}/file/$runName/location/${method}${ node ? "/$node" : ''}").openConnection() as HttpURLConnection + HttpURLConnection get = URI.create("${getDNS()}/file/$runName/location/${method}${ node ? "/$node" : ''}").toURL().openConnection() as HttpURLConnection get.setRequestMethod( "POST" ) get.setDoOutput(true) Map data = [ diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index 7cd64e3..82ecd32 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -51,7 +51,7 @@ class LocalPath implements Path { } static InputStream getFileStream(final String node, String daemon, String path) { - URL url = new URL("ftp://$daemon/$path") + URL url = URI.create("ftp://$daemon/$path").toURL() URLConnection con = url.openConnection() InputStream is = con.getInputStream() return is From 2fcc174c4b836f68f386485d416cdab2bd747b83 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 16:40:41 +0200 Subject: [PATCH 18/63] Warning if copy strategy is unsupported Signed-off-by: Lehmann_Fabian --- .../src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy index d847e07..57e9acf 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy @@ -36,6 +36,8 @@ class WOWK8sWrapperBuilder extends K8sWrapperBuilder { this.stageOutMode = 'move' } break + default : + throw new IllegalArgumentException("Unsupported copy strategy: ${storage.getCopyStrategy()}") } if ( !this.targetDir || workDir == targetDir ) { this.localWorkDir = FileHelper.getWorkFolder(storage.getWorkdir() as Path, task.getHash()) From e0f65eb854d9c56aa096273967b7163ed4546ee6 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 16:43:24 +0200 Subject: [PATCH 19/63] Use fixed hasher Signed-off-by: Lehmann_Fabian --- plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy index 5ee9062..1b1bfd2 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy @@ -205,7 +205,7 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { } protected static String hash(String text) { - def hasher = Hashing .murmur3_32() .newHasher() + def hasher = Hashing.murmur3_32_fixed().newHasher() hasher.putUnencodedChars(text) return hasher.hash().toString() } From d3bbc23dd1c99fdbd7bfd6bbe7d73a4553e9357b Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 16:45:08 +0200 Subject: [PATCH 20/63] Remove unused method Signed-off-by: Lehmann_Fabian --- .../src/main/nextflow/cws/wow/file/LocalPath.groovy | 9 --------- 1 file changed, 9 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index 82ecd32..735d851 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -117,15 +117,6 @@ class LocalPath implements Path { } } - /** - * - * @return A path located in the original workdir - */ - Path fakePath(){ - Path fake = FileHelper.fakePath( path, workDir ) - return fake - } - String getText(){ getText( Charset.defaultCharset().toString() ) } From 0852d2314e0a19ad2256006d1d4c7cfad34438d1 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 16:56:13 +0200 Subject: [PATCH 21/63] Remove unused import Signed-off-by: Lehmann_Fabian --- plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index 735d851..71c46c2 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -2,7 +2,6 @@ package nextflow.cws.wow.file import groovy.util.logging.Slf4j import nextflow.cws.k8s.K8sSchedulerClient -import nextflow.file.FileHelper import org.codehaus.groovy.runtime.IOGroovyMethods import sun.net.ftp.FtpClient From 23654481b27004634ec928b01931058801962d89 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 17:35:48 +0200 Subject: [PATCH 22/63] Create LocalFile Signed-off-by: Lehmann_Fabian --- plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index 71c46c2..6dd6cb4 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -658,7 +658,7 @@ class LocalPath implements Path { @Override File toFile() { - path.toFile() + new LocalFile( this ) } @Override From bdf8d3005f0ea1387e240b99553c6d43c057688e Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 17:38:11 +0200 Subject: [PATCH 23/63] Remove unused method getRelativePathonFake Signed-off-by: Lehmann_Fabian --- .../nextflow/cws/wow/file/WorkdirPath.groovy | 31 ------------------- .../cws/wow/file/WorkdirPathTest.groovy | 10 ------ 2 files changed, 41 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy index 26d467a..bac1190 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy @@ -38,35 +38,4 @@ class WorkdirPath extends OfflineLocalPath { workdirHelper.get( file ) ?: file } - /** - * This method is used to get the relative path of a file in the local file system. - * @param workdir - * @param localPath - * @return - */ - static Path getRelativePathOnFake(Path workdir, Path localPath ) { - String n1 = FilesEx.getName(workdir.getParent()) - String n2 = FilesEx.getName(workdir) - Path p = null - boolean foundN1 = false - boolean foundN2 = false - for (final def part in localPath) { - if ( part.toString() == n1 && !foundN1 ) { - foundN1 = true - } else if ( part.toString() == n2 && foundN1 && !foundN2 ) { - foundN2 = true - } else if ( foundN1 && !foundN2 ) { - return localPath - } else if ( !foundN1 ) { - // n1 not found yet, ignore part before - } else if ( !p ) { - p = part - } else { - p = p.resolve(part) - } - } - return p ?: workdir.relativize( localPath ) - } - - } diff --git a/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirPathTest.groovy b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirPathTest.groovy index 28d25f2..81124d9 100644 --- a/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirPathTest.groovy +++ b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirPathTest.groovy @@ -6,16 +6,6 @@ import java.nio.file.Path class WorkdirPathTest extends Specification { - def "FakePath"() { - when: - def path = Path.of("/input/data/work/be/8292aaebea2ddf9ae8ad4952882dcb") - def localPath = Path.of("/localdata/localwork/be/8292aaebea2ddf9ae8ad4952882dcb/file.txt") - def fakePath = WorkdirPath.getRelativePathOnFake( path, localPath ) - - then: - fakePath == Path.of("file.txt") - } - def "matches"() { when: def path = Path.of("file.txt").getFileName() From ecdd8f068d4288d0e22be1e949f4283d23cc2957 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 18:25:01 +0200 Subject: [PATCH 24/63] Check also for inherited classes on asType call. Signed-off-by: Lehmann_Fabian --- .../main/nextflow/cws/wow/file/LocalPath.groovy | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index 6dd6cb4..5973bfa 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -502,13 +502,24 @@ class LocalPath implements Path { T asType( Class c ) { log.info("FRIEDRICH asType") - if ( c == Path.class || c == LocalPath.class ) return this - if ( c == File.class ) return toFile() + if ( isSuperClass(getClass(), c) ) return this + if ( isSuperClass(LocalPath.class, c) ) return toFile() if ( c == String.class ) return toString() log.info("Invoke method asType $c on $this") return super.asType( c ) } + private static boolean isSuperClass(Class clazz, Class clazzToCheck) { + Class current = clazz + while (current != null) { + if ( current == clazzToCheck ) { + return true + } + current = current.getSuperclass() + } + return false + } + @Override Object invokeMethod(String name, Object args) { log.info("FRIEDRICH invokeMethod") From d5d718dbc1d2be3a6bff0dc9964bb2d0c3700355 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 18:31:33 +0200 Subject: [PATCH 25/63] Cast instead of asType Signed-off-by: Lehmann_Fabian --- .../main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy index c8d7394..6d9014e 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy @@ -46,8 +46,8 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran @Override DirectoryStream newDirectoryStream(Path path, DirectoryStream.Filter filter) throws IOException { - if ( path instanceof OfflineLocalPath && !(path as OfflineLocalPath).workdirHelper.isValidated() ) { - return (path as OfflineLocalPath).workdirHelper.getDirectoryStream( path ) + if ( path instanceof OfflineLocalPath && !((OfflineLocalPath) path).workdirHelper.isValidated() ) { + return ((OfflineLocalPath) path).workdirHelper.getDirectoryStream( path ) } throw new UnsupportedOperationException("Directory stream not supported by ${getScheme().toUpperCase()} file system provider") From 1649baaed6af97f334d6890998f1714a61efdccc Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 18:31:50 +0200 Subject: [PATCH 26/63] Remove unused import Signed-off-by: Lehmann_Fabian --- plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy index bac1190..f27eb61 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy @@ -1,7 +1,6 @@ package nextflow.cws.wow.file import groovy.util.logging.Slf4j -import nextflow.extension.FilesEx import java.nio.file.Path From d2c11e349e3517c55071f7915e1d977804e7afd6 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 18:38:30 +0200 Subject: [PATCH 27/63] Use Java native methods to check also for inherited classes on asType call. Signed-off-by: Lehmann_Fabian --- .../main/nextflow/cws/wow/file/LocalPath.groovy | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index 5973bfa..2d080a4 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -502,24 +502,13 @@ class LocalPath implements Path { T asType( Class c ) { log.info("FRIEDRICH asType") - if ( isSuperClass(getClass(), c) ) return this - if ( isSuperClass(LocalPath.class, c) ) return toFile() + if ( c.isAssignableFrom( getClass() ) ) return this + if ( c.isAssignableFrom( LocalPath.class ) ) return toFile() if ( c == String.class ) return toString() log.info("Invoke method asType $c on $this") return super.asType( c ) } - private static boolean isSuperClass(Class clazz, Class clazzToCheck) { - Class current = clazz - while (current != null) { - if ( current == clazzToCheck ) { - return true - } - current = current.getSuperclass() - } - return false - } - @Override Object invokeMethod(String name, Object args) { log.info("FRIEDRICH invokeMethod") From 62a2b1a89cff3b7d6952225761c6043516eb4675 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 18:54:57 +0200 Subject: [PATCH 28/63] Hint user to NXF_OPTS="--add-exports=java.base/sun.net.ftp=ALL-UNNAMED" Signed-off-by: Lehmann_Fabian --- .../nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index 2d080a4..45673c7 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -71,6 +71,10 @@ class LocalPath implements Path { log.error("Cannot create FTP client: $daemon on $node", e) sleep(Math.pow(2, trial++) as long) daemon = client.getDaemonOnNode(node) + } catch ( IllegalAccessError e ) { + log.error( "Cannot use FTP for data download. Please make sure to start Nextflow with: " + + "'export NXF_OPTS=\"--add-exports=java.base/sun.net.ftp=ALL-UNNAMED\"'") + throw e } } } From 35b44cdb928a511c4cde2ded094cc36a78f12133 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 25 Apr 2025 19:01:40 +0200 Subject: [PATCH 29/63] Clear data when no needed to free up memory Signed-off-by: Lehmann_Fabian --- .../src/main/nextflow/cws/wow/file/WorkdirHelper.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy index cc643af..78c6556 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy @@ -19,6 +19,8 @@ class WorkdirHelper { void validate() { validated = true + // Garbage collector might remove unused paths + paths.clear() } boolean isValidated() { @@ -26,6 +28,9 @@ class WorkdirHelper { } LocalPath get( Path path ) { + if ( validated ) { + throw new IllegalStateException("WorkdirHelper validated") + } paths.get( path ) } From aaa318686781aeb48c68799cf6420ee5e5aa6385 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Sat, 26 Apr 2025 15:36:17 +0200 Subject: [PATCH 30/63] refactor: replace FtpClient with FtpURLConnection --- .../nextflow/cws/wow/file/LocalPath.groovy | 76 +++++-------------- 1 file changed, 21 insertions(+), 55 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index e85d14a..61a474d 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -57,25 +57,6 @@ class LocalPath implements Path { return is } - static FtpClient getConnection(final String node, String daemon ){ - log.info("FRIEDRICH getConnection") - int trial = 0 - log.info("FRIEDRICH $node -- $daemon") - while ( true ) { - try { - FtpClient ftpClient = FtpClient.create(daemon) - ftpClient.login("root", "password".toCharArray() ) - ftpClient.enablePassiveMode( true ) - return ftpClient - } catch ( IOException e ) { - if ( trial > 5 ) throw e - log.error("Cannot create FTP client: $daemon on $node", e) - sleep(Math.pow(2, trial++) as long) - daemon = client.getDaemonOnNode(node) - } - } - } - Map getLocation( String absolutePath ){ log.info("FRIEDRICH getLocation") Map response = client.getFileLocation( absolutePath ) @@ -138,11 +119,9 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.getText( charset ) } - try (FtpClient ftpClient = getConnection(location.node as String, location.daemon as String)) { - try (InputStream fileStream = ftpClient.getFileStream(location.path as String)) { - log.trace("Read remote $absolutePath") - return fileStream.getText( charset ) - } + try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { + log.trace("Read remote $absolutePath") + return fileStream.getText( charset ) } } @@ -169,11 +148,9 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.getBytes() } - try (FtpClient ftpClient = getConnection(location.node.toString(), location.daemon.toString())) { - try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { - log.trace("Read remote $absolutePath") - return fileStream.getBytes() - } + try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { + log.trace("Read remote $absolutePath") + return fileStream.getBytes() } } @@ -200,11 +177,9 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.withReader( charset, closure ) } - try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { - try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { - log.trace("Read remote $absolutePath") - return IOGroovyMethods.withReader(fileStream, closure) - } + try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { + log.trace("Read remote $absolutePath") + return IOGroovyMethods.withReader(fileStream, closure) } } @@ -264,11 +239,9 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.eachLine( charset, firstLine, closure ) } - try (FtpClient ftpClient = getConnection( location.node.toString(), location.daemon.toString() )) { - try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { - log.trace("Read remote $absolutePath") - return IOGroovyMethods.eachLine( fileStream, charset, firstLine, closure ) - } + try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { + log.trace("Read remote $absolutePath") + return IOGroovyMethods.eachLine( fileStream, charset, firstLine, closure ) } } @@ -284,8 +257,7 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.newReader() } - try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { - InputStream fileStream = ftpClient.getFileStream( location.path.toString() ) + try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { log.trace("Read remote $absolutePath") InputStreamReader isr = new InputStreamReader( fileStream, charset ) return new BufferedReader(isr) @@ -340,11 +312,9 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.eachByte( closure ) } - try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { - try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { - log.trace("Read remote $absolutePath") - return IOGroovyMethods.eachByte( new BufferedInputStream( fileStream ), closure) - } + try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { + log.trace("Read remote $absolutePath") + return IOGroovyMethods.eachByte( new BufferedInputStream( fileStream ), closure) } } @@ -356,11 +326,9 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.eachByte( bufferLen, closure ) } - try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { - try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { - log.trace("Read remote $absolutePath") - return IOGroovyMethods.eachByte( new BufferedInputStream( fileStream ), bufferLen, closure) - } + try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { + log.trace("Read remote $absolutePath") + return IOGroovyMethods.eachByte( new BufferedInputStream( fileStream ), bufferLen, closure) } } @@ -372,8 +340,7 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.withInputStream( closure ) } - try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { - InputStream fileStream = ftpClient.getFileStream( location.path.toString() ) + try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { log.trace("Read remote $absolutePath") return IOGroovyMethods.withStream(new BufferedInputStream( fileStream ), closure) } @@ -399,8 +366,7 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.newInputStream() } - try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { - InputStream fileStream = ftpClient.getFileStream( location.path.toString() ) + try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { log.trace("Read remote $absolutePath") return new BufferedInputStream( fileStream ) } From 62ac1d2d44bb54a6ab20434ea01b9d05abb31805 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Sat, 26 Apr 2025 18:48:43 +0200 Subject: [PATCH 31/63] fix: revert FTP changes --- .../nextflow/cws/wow/file/LocalPath.groovy | 109 +++++++++++------- 1 file changed, 70 insertions(+), 39 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index efcefdb..fe1f6dd 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -49,11 +49,23 @@ class LocalPath implements Path { else throw new IllegalStateException("Client was already set.") } - static InputStream getFileStream(final String node, String daemon, String path) { - URL url = URI.create("ftp://$daemon/$path").toURL() - URLConnection con = url.openConnection() - InputStream is = con.getInputStream() - return is + static FtpClient getConnection(final String node, String daemon ){ + log.info("FRIEDRICH getConnection") + int trial = 0 + log.info("FRIEDRICH $node -- $daemon") + while ( true ) { + try { + FtpClient ftpClient = FtpClient.create(daemon) + ftpClient.login("root", "password".toCharArray() ) + ftpClient.enablePassiveMode( true ) + return ftpClient + } catch ( IOException e ) { + if ( trial > 5 ) throw e + log.error("Cannot create FTP client: $daemon on $node", e) + sleep(Math.pow(2, trial++) as long) + daemon = client.getDaemonOnNode(node) + } + } } Map getLocation( String absolutePath ){ @@ -109,9 +121,11 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.getText( charset ) } - try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { - log.trace("Read remote $absolutePath") - return fileStream.getText( charset ) + try (FtpClient ftpClient = getConnection(location.node as String, location.daemon as String)) { + try (InputStream fileStream = ftpClient.getFileStream(location.path as String)) { + log.trace("Read remote $absolutePath") + return fileStream.getText( charset ) + } } } @@ -138,9 +152,11 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.getBytes() } - try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { - log.trace("Read remote $absolutePath") - return fileStream.getBytes() + try (FtpClient ftpClient = getConnection(location.node.toString(), location.daemon.toString())) { + try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { + log.trace("Read remote $absolutePath") + return fileStream.getBytes() + } } } @@ -167,9 +183,11 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.withReader( charset, closure ) } - try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { - log.trace("Read remote $absolutePath") - return IOGroovyMethods.withReader(fileStream, closure) + try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { + try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { + log.trace("Read remote $absolutePath") + return IOGroovyMethods.withReader(fileStream, closure) + } } } @@ -229,9 +247,11 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.eachLine( charset, firstLine, closure ) } - try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { - log.trace("Read remote $absolutePath") - return IOGroovyMethods.eachLine( fileStream, charset, firstLine, closure ) + try (FtpClient ftpClient = getConnection( location.node.toString(), location.daemon.toString() )) { + try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { + log.trace("Read remote $absolutePath") + return IOGroovyMethods.eachLine( fileStream, charset, firstLine, closure ) + } } } @@ -247,7 +267,8 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.newReader() } - try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { + try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { + InputStream fileStream = ftpClient.getFileStream( location.path.toString() ) log.trace("Read remote $absolutePath") InputStreamReader isr = new InputStreamReader( fileStream, charset ) return new BufferedReader(isr) @@ -302,9 +323,11 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.eachByte( closure ) } - try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { - log.trace("Read remote $absolutePath") - return IOGroovyMethods.eachByte( new BufferedInputStream( fileStream ), closure) + try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { + try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { + log.trace("Read remote $absolutePath") + return IOGroovyMethods.eachByte( new BufferedInputStream( fileStream ), closure) + } } } @@ -316,9 +339,11 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.eachByte( bufferLen, closure ) } - try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { - log.trace("Read remote $absolutePath") - return IOGroovyMethods.eachByte( new BufferedInputStream( fileStream ), bufferLen, closure) + try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { + try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { + log.trace("Read remote $absolutePath") + return IOGroovyMethods.eachByte( new BufferedInputStream( fileStream ), bufferLen, closure) + } } } @@ -330,7 +355,8 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.withInputStream( closure ) } - try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { + try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { + InputStream fileStream = ftpClient.getFileStream( location.path.toString() ) log.trace("Read remote $absolutePath") return IOGroovyMethods.withStream(new BufferedInputStream( fileStream ), closure) } @@ -356,7 +382,8 @@ class LocalPath implements Path { log.trace("Read locally $absolutePath") return path.newInputStream() } - try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { + try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { + InputStream fileStream = ftpClient.getFileStream( location.path.toString() ) log.trace("Read remote $absolutePath") return new BufferedInputStream( fileStream ) } @@ -446,20 +473,24 @@ class LocalPath implements Path { log.trace("No download") return [ wasDownloaded : false, location : location ] } - try (InputStream fileStream = getFileStream( location.node.toString(), location.daemon.toString(), location.path.toString() )) { - log.trace("Download remote $absolutePath") - final def file = toFile() - path.parent.toFile().mkdirs() - OutputStream outStream = new FileOutputStream(file) - byte[] buffer = new byte[8 * 1024] - int bytesRead - while ((bytesRead = fileStream.read(buffer)) != -1) { - outStream.write(buffer, 0, bytesRead) + try (FtpClient ftpClient = getConnection(location.node as String, location.daemon as String)) { + try (InputStream fileStream = ftpClient.getFileStream(location.path as String)) { + log.trace("Download remote $absolutePath") + final def file = toFile() + path.parent.toFile().mkdirs() + OutputStream outStream = new FileOutputStream(file) + byte[] buffer = new byte[8 * 1024] + int bytesRead + while ((bytesRead = fileStream.read(buffer)) != -1) { + outStream.write(buffer, 0, bytesRead) + } + fileStream.close() + outStream.close() + this.wasDownloaded = true + return [wasDownloaded: true, location: location] + } catch (Exception e) { + throw e } - fileStream.close() - outStream.close() - this.wasDownloaded = true - return [ wasDownloaded : true, location : location ] } catch (Exception e) { throw e } From e94a84e5a432097c020db2f1e9eb8fa589ad8be2 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Sat, 26 Apr 2025 18:53:13 +0200 Subject: [PATCH 32/63] feat: generalize InputStream and OutputStream for LocalPath --- .../main/nextflow/cws/SchedulerClient.groovy | 2 + .../nextflow/cws/wow/file/LocalPath.groovy | 370 +----------------- .../cws/wow/file/WOWFileSystemProvider.groovy | 44 ++- .../cws/wow/file/WOWInputStream.groovy | 52 +++ .../cws/wow/file/WOWOutputStream.groovy | 29 ++ 5 files changed, 126 insertions(+), 371 deletions(-) create mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy create mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWOutputStream.groovy diff --git a/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy index d4b8f5e..961028a 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy @@ -3,6 +3,7 @@ package nextflow.cws import groovy.json.JsonOutput import groovy.json.JsonSlurper import groovy.util.logging.Slf4j +import nextflow.cws.wow.file.WOWFileSystemProvider import nextflow.dag.DAG @Slf4j @@ -20,6 +21,7 @@ class SchedulerClient { this.runName = runName this.dns = config.dns?.endsWith('/') ? config.dns[0..-2] : config.dns CWSSession.INSTANCE.addSchedulerClient( this ) + WOWFileSystemProvider.INSTANCE.registerSchedulerClient( this ) } protected String getDNS() { diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index fe1f6dd..853f752 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -2,10 +2,8 @@ package nextflow.cws.wow.file import groovy.util.logging.Slf4j import nextflow.cws.k8s.K8sSchedulerClient -import org.codehaus.groovy.runtime.IOGroovyMethods import sun.net.ftp.FtpClient -import java.nio.charset.Charset import java.nio.file.* import java.nio.file.attribute.BasicFileAttributes @@ -68,6 +66,8 @@ class LocalPath implements Path { } } + Map getLocation() { getLocation(path.toAbsolutePath().toString()) } + Map getLocation( String absolutePath ){ log.info("FRIEDRICH getLocation") Map response = client.getFileLocation( absolutePath ) @@ -98,372 +98,6 @@ class LocalPath implements Path { response } - private void checkParentDirectoryExists() { - if (!path.getParent().exists()) { - try { - log.trace("Creating directory locally ${path.getParent().toAbsolutePath().toString()}") - Files.createDirectories(path.getParent()) - } catch (IOException e) { - log.error("Couldn't create directory ${path.getParent().toAbsolutePath().toString()}", e) - } - } - } - - String getText(){ - getText( Charset.defaultCharset().toString() ) - } - - String getText( String charset ){ - log.info("FRIEDRICH getText") - final String absolutePath = path.toAbsolutePath().toString() - final def location = getLocation( absolutePath ) - if ( wasDownloaded || location.sameAsEngine ){ - log.trace("Read locally $absolutePath") - return path.getText( charset ) - } - try (FtpClient ftpClient = getConnection(location.node as String, location.daemon as String)) { - try (InputStream fileStream = ftpClient.getFileStream(location.path as String)) { - log.trace("Read remote $absolutePath") - return fileStream.getText( charset ) - } - } - } - - void setText( String text ){ - setText( text, Charset.defaultCharset().toString() ) - } - - void setText( String text, String charset ){ - log.info("FRIEDRICH setText") - final String absolutePath = path.toAbsolutePath().toString() - final def location = client.getFileLocation( absolutePath ) - checkParentDirectoryExists() - log.trace("Write locally $absolutePath") - path.setText( text, charset ) - def file = path.toFile() - client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) - } - - byte[] getBytes(){ - log.info("FRIEDRICH getBytes") - final String absolutePath = path.toAbsolutePath().toString() - final def location = getLocation( absolutePath ) - if ( wasDownloaded || location.sameAsEngine ){ - log.trace("Read locally $absolutePath") - return path.getBytes() - } - try (FtpClient ftpClient = getConnection(location.node.toString(), location.daemon.toString())) { - try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { - log.trace("Read remote $absolutePath") - return fileStream.getBytes() - } - } - } - - void setBytes( byte[] bytes ){ - log.info("FRIEDRICH setBytes") - final String absolutePath = path.toAbsolutePath().toString() - final def location = client.getFileLocation( absolutePath ) - checkParentDirectoryExists() - log.trace("Write locally $absolutePath") - path.setBytes( bytes ) - def file = path.toFile() - client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) - } - - Object withReader( Closure closure ){ - withReader ( Charset.defaultCharset().toString(), closure ) - } - - Object withReader( String charset, Closure closure ){ - log.info("FRIEDRICH withReader") - final String absolutePath = path.toAbsolutePath().toString() - final def location = getLocation( absolutePath ) - if ( wasDownloaded || location.sameAsEngine ){ - log.trace("Read locally $absolutePath") - return path.withReader( charset, closure ) - } - try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { - try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { - log.trace("Read remote $absolutePath") - return IOGroovyMethods.withReader(fileStream, closure) - } - } - } - - def T withWriter( Closure closure ){ - return withWriter( Charset.defaultCharset().toString(), closure ) - } - - def T withWriter( String charset, Closure closure ) { - log.info("FRIEDRICH withWriter") - final String absolutePath = path.toAbsolutePath().toString() - final def location = client.getFileLocation( absolutePath ) - checkParentDirectoryExists() - log.trace("Write locally $absolutePath") - T rv = path.withWriter( charset, closure ) - def file = path.toFile() - client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) - return rv - } - - def T withPrintWriter( String charset, Closure closure ){ - log.info("FRIEDRICH withPrintWriter") - final String absolutePath = path.toAbsolutePath().toString() - final def location = client.getFileLocation( absolutePath ) - checkParentDirectoryExists() - log.trace("Write locally $absolutePath") - T rv = path.withPrintWriter( charset, closure ) - def file = path.toFile() - client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) - return rv - } - - List readLines(){ - IOGroovyMethods.readLines( newReader() ) - } - - List readLines( String charset ){ - IOGroovyMethods.readLines( newReader( charset ) ) - } - - T eachLine( Closure closure ) throws IOException { - eachLine( Charset.defaultCharset().toString(), 1, closure ) - } - - T eachLine( int firstLine, Closure closure ) throws IOException { - eachLine( Charset.defaultCharset().toString(), firstLine, closure ) - } - - T eachLine( String charset, Closure closure ) throws IOException { - eachLine( charset, 1, closure ) - } - - T eachLine( String charset, int firstLine, Closure closure ) throws IOException { - log.info("FRIEDRICH eachLine") - final String absolutePath = path.toAbsolutePath().toString() - final def location = getLocation( absolutePath ) - if ( wasDownloaded || location.sameAsEngine ){ - log.trace("Read locally $absolutePath") - return path.eachLine( charset, firstLine, closure ) - } - try (FtpClient ftpClient = getConnection( location.node.toString(), location.daemon.toString() )) { - try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { - log.trace("Read remote $absolutePath") - return IOGroovyMethods.eachLine( fileStream, charset, firstLine, closure ) - } - } - } - - BufferedReader newReader(){ - return newReader( Charset.defaultCharset().toString() ) - } - - BufferedReader newReader( String charset ){ - log.info("FRIEDRICH newReader") - final String absolutePath = path.toAbsolutePath().toString() - final def location = getLocation( absolutePath ) - if ( wasDownloaded || location.sameAsEngine ){ - log.trace("Read locally $absolutePath") - return path.newReader() - } - try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { - InputStream fileStream = ftpClient.getFileStream( location.path.toString() ) - log.trace("Read remote $absolutePath") - InputStreamReader isr = new InputStreamReader( fileStream, charset ) - return new BufferedReader(isr) - } - } - - BufferedWriter newWriter(){ - return newWriter( Charset.defaultCharset().toString(), false, false ) - } - - BufferedWriter newWriter( String charset ) { - return newWriter( charset, false, false ) - } - - BufferedWriter newWriter( boolean append ) { - return newWriter( Charset.defaultCharset().toString(), append, false ) - } - - BufferedWriter newWriter( String charset, boolean append ){ - return newWriter( charset, append, false ) - } - - BufferedWriter newWriter( String charset, boolean append, boolean writeBom ){ - log.info("FRIEDRICH newWriter") - final String absolutePath = path.toAbsolutePath().toString() - final def location = client.getFileLocation( absolutePath ) - checkParentDirectoryExists() - log.trace("Write locally $absolutePath") - BufferedWriter bufferedWriter = path.newWriter( charset, append, writeBom ) - def file = path.toFile() - client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) - return bufferedWriter - } - - PrintWriter newPrintWriter( String charset ){ - log.info("FRIEDRICH newPrintWriter") - final String absolutePath = path.toAbsolutePath().toString() - final def location = client.getFileLocation( absolutePath ) - checkParentDirectoryExists() - log.trace("Write locally $absolutePath") - PrintWriter printWriter = path.newPrintWriter( charset ) - def file = path.toFile() - client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) - return printWriter - } - - T eachByte( Closure closure ) throws IOException { - log.info("FRIEDRICH eachByte1") - final String absolutePath = path.toAbsolutePath().toString() - final def location = getLocation( absolutePath ) - if ( wasDownloaded || location.sameAsEngine ){ - log.trace("Read locally $absolutePath") - return path.eachByte( closure ) - } - try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { - try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { - log.trace("Read remote $absolutePath") - return IOGroovyMethods.eachByte( new BufferedInputStream( fileStream ), closure) - } - } - } - - T eachByte( int bufferLen, Closure closure ) throws IOException { - log.info("FRIEDRICH eachByte2") - final String absolutePath = path.toAbsolutePath().toString() - final def location = getLocation( absolutePath ) - if ( wasDownloaded || location.sameAsEngine ){ - log.trace("Read locally $absolutePath") - return path.eachByte( bufferLen, closure ) - } - try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { - try (InputStream fileStream = ftpClient.getFileStream( location.path.toString() )) { - log.trace("Read remote $absolutePath") - return IOGroovyMethods.eachByte( new BufferedInputStream( fileStream ), bufferLen, closure) - } - } - } - - T withInputStream( Closure closure) throws IOException { - log.info("FRIEDRICH withInputStream") - final String absolutePath = path.toAbsolutePath().toString() - final def location = getLocation( absolutePath ) - if ( wasDownloaded || location.sameAsEngine ){ - log.trace("Read locally $absolutePath") - return path.withInputStream( closure ) - } - try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { - InputStream fileStream = ftpClient.getFileStream( location.path.toString() ) - log.trace("Read remote $absolutePath") - return IOGroovyMethods.withStream(new BufferedInputStream( fileStream ), closure) - } - } - - def T withOutputStream( Closure closure ){ - log.info("FRIEDRICH withOutputStream") - final String absolutePath = path.toAbsolutePath().toString() - final def location = client.getFileLocation( absolutePath ) - checkParentDirectoryExists() - log.trace("Write locally $absolutePath") - T rv = path.withOutputStream( closure ) - def file = path.toFile() - client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) - return rv - } - - BufferedInputStream newInputStream() throws IOException { - log.info("FRIEDRICH newInputStream") - final String absolutePath = path.toAbsolutePath().toString() - final def location = getLocation( absolutePath ) - if ( wasDownloaded || location.sameAsEngine ){ - log.trace("Read locally $absolutePath") - return path.newInputStream() - } - try (FtpClient ftpClient = getConnection( location.node.toString() , location.daemon.toString() )) { - InputStream fileStream = ftpClient.getFileStream( location.path.toString() ) - log.trace("Read remote $absolutePath") - return new BufferedInputStream( fileStream ) - } - } - - BufferedOutputStream newOutputStream(){ - log.info("FRIEDRICH newOutputStream") - final String absolutePath = path.toAbsolutePath().toString() - final def location = client.getFileLocation( absolutePath ) - checkParentDirectoryExists() - log.trace("Write locally $absolutePath") - BufferedOutputStream bufferedOutputStream = path.newOutputStream() - def file = path.toFile() - client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) - return bufferedOutputStream - } - - void write( String text ){ - write(text, Charset.defaultCharset().toString(), false) - } - - void write( String text, String charset ){ - write(text, charset, false) - } - - void write( String text, boolean writeBom ){ - write(text, Charset.defaultCharset().toString(), writeBom) - } - - void write( String text, String charset, boolean writeBom ){ - log.info("FRIEDRICH write") - final String absolutePath = path.toAbsolutePath().toString() - final def location = client.getFileLocation( absolutePath ) - checkParentDirectoryExists() - log.trace("Write locally $absolutePath") - path.write( text, charset, writeBom ) - def file = path.toFile() - client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) - def loc = client.getFileLocation( absolutePath ) - } - - void leftShift(Object text) { - append(text) - } - - void append( Object text ){ - append(text, Charset.defaultCharset().toString(), false) - } - - void append( Object text, String charset ){ - append(text, charset, false) - } - - void append( Object text, boolean writeBom ){ - append(text, Charset.defaultCharset().toString(), writeBom) - } - - void append( Object text, String charset, boolean writeBom ){ - log.info("FRIEDRICH append") - if ( !text ) { - return - } - final String absolutePath = path.toAbsolutePath().toString() - final def location = client.getFileLocation( absolutePath ) - if ( wasDownloaded || location.sameAsEngine ) { - log.info("Append locally $absolutePath") - path.append( text, charset, writeBom ) - } else { - try (FtpClient ftpClient = getConnection(location.node.toString(), location.daemon.toString())) { - byte[] bytes = text.toString().getBytes( charset ) - InputStream stream = new ByteArrayInputStream( bytes ) - log.info("Append remote $absolutePath") - ftpClient.appendFile( absolutePath, stream ) - } - } - def file = path.toFile() - client.addFileLocation( path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true ) - def loc = client.getFileLocation( absolutePath ) - } - Map download(){ log.info("FRIEDRICH download") final String absolutePath = path.toAbsolutePath().toString() diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy index 6d9014e..7a4fd03 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy @@ -1,7 +1,9 @@ package nextflow.cws.wow.file import groovy.util.logging.Slf4j +import nextflow.cws.SchedulerClient import nextflow.file.FileSystemTransferAware +import sun.net.ftp.FtpClient import java.nio.channels.SeekableByteChannel import java.nio.file.* @@ -15,6 +17,41 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran static final WOWFileSystemProvider INSTANCE = new WOWFileSystemProvider() + protected SchedulerClient schedulerClient = null // TODO: support multiple clients + + void registerSchedulerClient(SchedulerClient schedulerClient) throws UnsupportedOperationException { + if (schedulerClient != null) { + throw new UnsupportedOperationException("WOW file system does not support multiple scheduler clients") + } + this.schedulerClient = schedulerClient + } + + @Override + InputStream newInputStream(Path path, OpenOption... options) { + if (schedulerClient != null) { + throw new RuntimeException("WOW file system has no registered scheduler client") + } + assert path instanceof LocalPath + Map location = (path as LocalPath).getLocation() + + try (FtpClient ftpClient = path.getConnection(location.node.toString(), location.daemon.toString())) { + try (InputStream is = ftpClient.getFileStream(location.path.toString())) { + return new WOWInputStream(is, schedulerClient, path) + } + } + } + + @Override + OutputStream newOutputStream(Path path, OpenOption... options) { + if (schedulerClient != null) { + throw new RuntimeException("WOW file system has no registered scheduler client") + } + assert path instanceof LocalPath + + OutputStream os = super.newOutputStream(path.getInner(), options) + return new WOWOutputStream(os, schedulerClient, path) + } + @Override String getScheme() { return "wow" @@ -37,6 +74,7 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran @Override SeekableByteChannel newByteChannel(Path path, Set set, FileAttribute... fileAttributes) throws IOException { + log.info("FRIEDRICH: newByteChannel") LocalPath localPath = (LocalPath) path // TODO: find an alternative to always downloading Map downloadResult = localPath.download() @@ -46,8 +84,8 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran @Override DirectoryStream newDirectoryStream(Path path, DirectoryStream.Filter filter) throws IOException { - if ( path instanceof OfflineLocalPath && !((OfflineLocalPath) path).workdirHelper.isValidated() ) { - return ((OfflineLocalPath) path).workdirHelper.getDirectoryStream( path ) + if (path instanceof OfflineLocalPath && !((OfflineLocalPath) path).workdirHelper.isValidated()) { + return ((OfflineLocalPath) path).workdirHelper.getDirectoryStream(path) } throw new UnsupportedOperationException("Directory stream not supported by ${getScheme().toUpperCase()} file system provider") @@ -100,7 +138,7 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran @Override A readAttributes(Path path, Class aClass, LinkOption... linkOptions) throws IOException { - if ( path instanceof LocalPath ) { + if (path instanceof LocalPath) { return path.getAttributes() as A } else { return Files.readAttributes(path, BasicFileAttributes.class, linkOptions) as A diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy new file mode 100644 index 0000000..4bd8a95 --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy @@ -0,0 +1,52 @@ +package nextflow.cws.wow.file + +import nextflow.cws.SchedulerClient + +class WOWInputStream extends InputStream { + + private InputStream inner + private SchedulerClient client + private LocalPath path + + private File temporaryFile + private OutputStream temporaryFileStream + private boolean transferredTemporaryFile + + WOWInputStream(InputStream inner, SchedulerClient client, LocalPath path) { + super() + this.inner = inner + this.client = client + this.path = path + this.temporaryFile = File.createTempFile("local", "buffer") + this.temporaryFileStream = temporaryFile.newOutputStream() + this.transferredTemporaryFile = false + } + + private void checkTemporaryFileTransferal() { + if (transferredTemporaryFile || inner.available() > 0) { + return + } + temporaryFileStream.flush() + temporaryFileStream.close() + + File file = path.getInner().toFile() + temporaryFile.moveTo(file) + transferredTemporaryFile = true + + Map location = path.getLocation() + client.addFileLocation(path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, false) + } + + @Override + int read() throws IOException { + int b = inner.read() + temporaryFileStream.write(b) + return b + } + + @Override + void close() throws IOException { + inner.close() + checkTemporaryFileTransferal() + } +} diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWOutputStream.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWOutputStream.groovy new file mode 100644 index 0000000..c3fea9d --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWOutputStream.groovy @@ -0,0 +1,29 @@ +package nextflow.cws.wow.file + +import nextflow.cws.SchedulerClient + +class WOWOutputStream extends OutputStream { + + private OutputStream inner + private SchedulerClient client + private LocalPath path + + WOWOutputStream(OutputStream inner, SchedulerClient client, LocalPath path) { + super() + this.inner = inner + this.client = client + this.path = path + } + + @Override + void write(int b) throws IOException { + super.write(b) + } + + @Override + void close() throws IOException { + Map location = path.getLocation() + File file = path.getInner().toFile() + client.addFileLocation(path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true) + } +} From a04860aa3897990d33f076fbbe31c0faf5223558 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Sat, 26 Apr 2025 18:59:33 +0200 Subject: [PATCH 33/63] fix: check for scheduler client existence --- .../main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy index 7a4fd03..80e147d 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy @@ -20,7 +20,7 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran protected SchedulerClient schedulerClient = null // TODO: support multiple clients void registerSchedulerClient(SchedulerClient schedulerClient) throws UnsupportedOperationException { - if (schedulerClient != null) { + if (this.schedulerClient != null) { throw new UnsupportedOperationException("WOW file system does not support multiple scheduler clients") } this.schedulerClient = schedulerClient @@ -28,7 +28,7 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran @Override InputStream newInputStream(Path path, OpenOption... options) { - if (schedulerClient != null) { + if (schedulerClient == null) { throw new RuntimeException("WOW file system has no registered scheduler client") } assert path instanceof LocalPath @@ -43,7 +43,7 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran @Override OutputStream newOutputStream(Path path, OpenOption... options) { - if (schedulerClient != null) { + if (schedulerClient == null) { throw new RuntimeException("WOW file system has no registered scheduler client") } assert path instanceof LocalPath From 1383dd4fc5aee4e795f57b59fac9eb368910617c Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Sat, 26 Apr 2025 19:08:52 +0200 Subject: [PATCH 34/63] fix: check for fully read FTP stream --- .../src/main/nextflow/cws/wow/file/WOWInputStream.groovy | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy index 4bd8a95..583631d 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy @@ -7,6 +7,7 @@ class WOWInputStream extends InputStream { private InputStream inner private SchedulerClient client private LocalPath path + private boolean fullyRead private File temporaryFile private OutputStream temporaryFileStream @@ -17,13 +18,14 @@ class WOWInputStream extends InputStream { this.inner = inner this.client = client this.path = path + this.fullyRead = false this.temporaryFile = File.createTempFile("local", "buffer") this.temporaryFileStream = temporaryFile.newOutputStream() this.transferredTemporaryFile = false } private void checkTemporaryFileTransferal() { - if (transferredTemporaryFile || inner.available() > 0) { + if (transferredTemporaryFile || !fullyRead) { return } temporaryFileStream.flush() @@ -41,11 +43,16 @@ class WOWInputStream extends InputStream { int read() throws IOException { int b = inner.read() temporaryFileStream.write(b) + if (b == -1) { + fullyRead = true + checkTemporaryFileTransferal() + } return b } @Override void close() throws IOException { + fullyRead = inner.read() == -1 inner.close() checkTemporaryFileTransferal() } From 2d089afddfe3ecfb7efb906db98b4767d1030c02 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Sat, 26 Apr 2025 19:15:15 +0200 Subject: [PATCH 35/63] fix: implement available() --- .../src/main/nextflow/cws/wow/file/WOWInputStream.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy index 583631d..d071caf 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy @@ -39,6 +39,11 @@ class WOWInputStream extends InputStream { client.addFileLocation(path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, false) } + @Override + int available() throws IOException { + return inner.available() + } + @Override int read() throws IOException { int b = inner.read() From 584836759a053571f1642ac4fbbbe8b54724cb46 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Sat, 26 Apr 2025 19:29:25 +0200 Subject: [PATCH 36/63] fix: do not close ftp client before starting --- .../cws/wow/file/WOWFileSystemProvider.groovy | 8 +++----- .../main/nextflow/cws/wow/file/WOWInputStream.groovy | 12 ++++++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy index 80e147d..2763139 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy @@ -34,11 +34,9 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran assert path instanceof LocalPath Map location = (path as LocalPath).getLocation() - try (FtpClient ftpClient = path.getConnection(location.node.toString(), location.daemon.toString())) { - try (InputStream is = ftpClient.getFileStream(location.path.toString())) { - return new WOWInputStream(is, schedulerClient, path) - } - } + FtpClient ftpClient = path.getConnection(location.node.toString(), location.daemon.toString()) + InputStream is = ftpClient.getFileStream(location.path.toString()) + return new WOWInputStream(is, schedulerClient, path, ftpClient) } @Override diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy index d071caf..a3c5918 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy @@ -1,24 +1,27 @@ package nextflow.cws.wow.file import nextflow.cws.SchedulerClient +import sun.net.ftp.FtpClient class WOWInputStream extends InputStream { private InputStream inner - private SchedulerClient client + private SchedulerClient schedulerClient private LocalPath path private boolean fullyRead + private FtpClient ftpClient private File temporaryFile private OutputStream temporaryFileStream private boolean transferredTemporaryFile - WOWInputStream(InputStream inner, SchedulerClient client, LocalPath path) { + WOWInputStream(InputStream inner, SchedulerClient schedulerClient, LocalPath path, FtpClient ftpClient) { super() this.inner = inner - this.client = client + this.schedulerClient = schedulerClient this.path = path this.fullyRead = false + this.ftpClient = ftpClient this.temporaryFile = File.createTempFile("local", "buffer") this.temporaryFileStream = temporaryFile.newOutputStream() this.transferredTemporaryFile = false @@ -36,7 +39,7 @@ class WOWInputStream extends InputStream { transferredTemporaryFile = true Map location = path.getLocation() - client.addFileLocation(path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, false) + schedulerClient.addFileLocation(path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, false) } @Override @@ -59,6 +62,7 @@ class WOWInputStream extends InputStream { void close() throws IOException { fullyRead = inner.read() == -1 inner.close() + ftpClient.close() checkTemporaryFileTransferal() } } From 7802c8c0049c9e143ab71922916b281d1d3359fd Mon Sep 17 00:00:00 2001 From: Fabian Lehmann <46564585+Lehmann-Fabian@users.noreply.github.com> Date: Wed, 30 Apr 2025 13:31:18 +0200 Subject: [PATCH 37/63] More Fixes (#4) --- .../src/main/nextflow/cws/CWSPlugin.groovy | 5 +++ .../nextflow/cws/CWSSchedulerBatch.groovy | 2 + .../src/main/nextflow/cws/CWSSession.groovy | 3 ++ .../main/nextflow/cws/SchedulerClient.groovy | 6 ++- .../main/nextflow/cws/k8s/CWSK8sClient.groovy | 6 ++- .../nextflow/cws/k8s/CWSK8sTaskHandler.groovy | 17 +++++++-- .../cws/k8s/CWSK8sWrapperBuilder.groovy | 2 + .../cws/k8s/K8sSchedulerClient.groovy | 4 +- .../cws/k8s/MemoryScalingFailure.groovy | 2 + .../k8s/model/PodMountConfigWithMode.groovy | 2 + .../processor/CWSTaskPollingMonitor.groovy | 2 + .../cws/processor/SchedulerBatch.groovy | 3 ++ .../nextflow/cws/wow/file/DateParser.groovy | 3 ++ .../nextflow/cws/wow/file/LocalFile.groovy | 3 ++ .../cws/wow/file/LocalFileWalker.groovy | 2 + .../nextflow/cws/wow/file/LocalPath.groovy | 35 ++++++++++++------ .../cws/wow/file/OfflineLocalPath.groovy | 6 +++ .../cws/wow/file/WOWFileSystem.groovy | 4 ++ .../wow/file/WOWFileSystemPathFactory.groovy | 37 +++++++++++++++++++ .../cws/wow/file/WOWFileSystemProvider.groovy | 7 +++- .../cws/wow/file/WOWInputStream.groovy | 5 ++- .../cws/wow/file/WOWOutputStream.groovy | 5 ++- .../cws/wow/file/WorkdirHelper.groovy | 18 ++++++--- .../nextflow/cws/wow/file/WorkdirPath.groovy | 8 +++- .../src/resources/META-INF/extensions.idx | 3 +- .../cws/wow/file/WOWFileSystemTest.groovy | 19 ++++++++++ 26 files changed, 179 insertions(+), 30 deletions(-) create mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemPathFactory.groovy create mode 100644 plugins/nf-cws/src/test/nextflow/cws/wow/file/WOWFileSystemTest.groovy diff --git a/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy b/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy index f38ce85..ffaf5bf 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy @@ -1,10 +1,13 @@ package nextflow.cws import groovy.transform.CompileStatic +import nextflow.cws.wow.file.OfflineLocalPath import nextflow.cws.wow.file.WOWFileSystemProvider +import nextflow.cws.wow.file.WorkdirHelper import nextflow.file.FileHelper import nextflow.plugin.BasePlugin import nextflow.trace.TraceRecord +import nextflow.util.KryoHelper import org.pf4j.PluginWrapper @CompileStatic @@ -59,6 +62,8 @@ class CWSPlugin extends BasePlugin { void start() { super.start() registerTraceFields() + KryoHelper.register( OfflineLocalPath ) + KryoHelper.register( WorkdirHelper ) FileHelper.getOrInstallProvider(WOWFileSystemProvider) } diff --git a/plugins/nf-cws/src/main/nextflow/cws/CWSSchedulerBatch.groovy b/plugins/nf-cws/src/main/nextflow/cws/CWSSchedulerBatch.groovy index 783fec3..94fc908 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/CWSSchedulerBatch.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/CWSSchedulerBatch.groovy @@ -1,7 +1,9 @@ package nextflow.cws +import groovy.transform.CompileStatic import nextflow.cws.processor.SchedulerBatch +@CompileStatic class CWSSchedulerBatch extends SchedulerBatch { private SchedulerClient schedulerClient diff --git a/plugins/nf-cws/src/main/nextflow/cws/CWSSession.groovy b/plugins/nf-cws/src/main/nextflow/cws/CWSSession.groovy index 1b4408c..984d36c 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/CWSSession.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/CWSSession.groovy @@ -1,8 +1,11 @@ package nextflow.cws +import groovy.transform.CompileStatic + /** * Central, global instance to manage all generated Executor instances */ +@CompileStatic class CWSSession { static final CWSSession INSTANCE = new CWSSession() diff --git a/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy index 961028a..5958b60 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy @@ -2,11 +2,14 @@ package nextflow.cws import groovy.json.JsonOutput import groovy.json.JsonSlurper +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.cws.wow.file.WOWFileSystemProvider import nextflow.dag.DAG @Slf4j +@CompileStatic class SchedulerClient { private final CWSConfig config @@ -137,7 +140,7 @@ class SchedulerClient { ///* DAG */ - + @CompileDynamic void submitVertices( List vertices ){ List> verticesToSubmit = vertices.collect { [ @@ -158,6 +161,7 @@ class SchedulerClient { } } + @CompileDynamic void submitEdges( List edges ){ List> edgesToSubmit = edges.collect { [ diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sClient.groovy index 99c5603..3ed27a0 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sClient.groovy @@ -1,6 +1,7 @@ package nextflow.cws.k8s import groovy.json.JsonOutput +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.k8s.client.ClientConfig import nextflow.k8s.client.K8sClient @@ -11,6 +12,7 @@ import org.yaml.snakeyaml.Yaml import java.nio.file.Path @Slf4j +@CompileStatic class CWSK8sClient extends K8sClient { CWSK8sClient(K8sClient k8sClient) { @@ -106,7 +108,7 @@ class CWSK8sClient extends K8sClient { final K8sResponseJson resp = podStatus0(podName) //If this label is not set, the memory was not scaled - if ( resp?.metadata?.labels?."commonworkflowscheduler/memoryscaled" != 'true' ) { + if ( ((resp?.metadata as Map)?.labels as Map)?."commonworkflowscheduler/memoryscaled" != 'true' ) { return null } @@ -114,7 +116,7 @@ class CWSK8sClient extends K8sClient { if ( containers == null || containers.size() == 0 ) { return null } - String memory = containers[0]?.resources?.limits?.memory as String + String memory = ((containers[0]?.resources as Map)?.limits as Map)?.memory as String if ( memory == null ) { return null } else { diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy index ea02b1e..5919bd0 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy @@ -1,6 +1,7 @@ package nextflow.cws.k8s import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.cws.SchedulerClient import nextflow.executor.BashWrapperBuilder @@ -15,6 +16,7 @@ import java.nio.file.NoSuchFileException import java.nio.file.Path @Slf4j +@CompileStatic class CWSK8sTaskHandler extends K8sTaskHandler { static final public String CMD_TRACE_SCHEDULER = '.command.scheduler.trace' @@ -177,22 +179,29 @@ class CWSK8sTaskHandler extends K8sTaskHandler { } } + boolean schedulerPostProcessingHasFinished(){ + Map state = schedulerClient.getTaskState(task.id.intValue()) + return (!state.state) ?: ["FINISHED", "FINISHED_WITH_ERROR", "INIT_WITH_ERRORS", "DELETED"].contains( state.state.toString() ) + } + @Override boolean checkIfCompleted() { Map state = getState() - if( !state || !state.terminated ) { + if( !state || !state.terminated || ( (k8sConfig as CWSK8sConfig)?.locationAwareScheduling() && !schedulerPostProcessingHasFinished() ) ) { return false } if( executor.getCWSConfig().memoryPredictor ) { memoryAdapted = client.getAdaptedPodMemory( podName ) if ( memoryAdapted != null ) { //only use a special failure logic if the memory was adapted - if (state.terminated.exitCode == 128 - && (state.terminated.reason as String) == "StartError") { + + def terminated = state.terminated as Map + if (terminated.exitCode == 128 + && (terminated.reason as String) == "StartError") { failedOOM = true log.info("The memory was choosen too small for the pod ${podName} to be started. More memory is tried next time.") task.error = new MemoryScalingFailure() - } else if ((state.terminated.reason as String) == "OOMKilled") { + } else if ((terminated.reason as String) == "OOMKilled") { failedOOM = true log.info("The memory was choosen too small for the pod ${podName} to be executed. More memory is tried next time.") task.error = new MemoryScalingFailure() diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sWrapperBuilder.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sWrapperBuilder.groovy index a9f096b..c4f7dca 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sWrapperBuilder.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sWrapperBuilder.groovy @@ -1,8 +1,10 @@ package nextflow.cws.k8s +import groovy.transform.CompileStatic import nextflow.k8s.K8sWrapperBuilder import nextflow.processor.TaskRun +@CompileStatic class CWSK8sWrapperBuilder extends K8sWrapperBuilder { CWSK8sWrapperBuilder(TaskRun task, boolean memoryPredictorEnabled) { diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy index dd15460..caf5b0d 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy @@ -1,5 +1,6 @@ package nextflow.cws.k8s +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.cws.CWSConfig import nextflow.cws.SchedulerClient @@ -15,6 +16,7 @@ import nextflow.k8s.model.PodVolumeClaim import java.nio.file.Paths @Slf4j +@CompileStatic class K8sSchedulerClient extends SchedulerClient { private final CWSK8sConfig.K8sScheduler schedulerConfig @@ -115,7 +117,7 @@ class K8sSchedulerClient extends SchedulerClient { name: 'AUTOCLOSE', value: schedulerConfig.autoClose() as String ]] - Map container = pod.spec.containers.get(0) as Map + Map container = ((pod.spec as Map).containers as List).get(0) as Map container.put('env', env) container.remove( 'command' ) (container.resources as Map)?.remove( 'limits' ) diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/MemoryScalingFailure.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/MemoryScalingFailure.groovy index 937a023..96bbfe1 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/MemoryScalingFailure.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/MemoryScalingFailure.groovy @@ -1,8 +1,10 @@ package nextflow.cws.k8s +import groovy.transform.CompileStatic import groovy.transform.InheritConstructors import nextflow.exception.ProcessRetryableException @InheritConstructors +@CompileStatic class MemoryScalingFailure extends RuntimeException implements ProcessRetryableException { } diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy index 4c5a722..f88a61e 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy @@ -1,7 +1,9 @@ package nextflow.cws.k8s.model +import groovy.transform.CompileStatic import nextflow.k8s.model.PodMountConfig +@CompileStatic class PodMountConfigWithMode extends PodMountConfig { private Integer mode diff --git a/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy b/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy index c37df3e..e0133b3 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy @@ -1,5 +1,6 @@ package nextflow.cws.processor +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.Session import nextflow.cws.wow.file.LocalFileWalker @@ -10,6 +11,7 @@ import nextflow.processor.TaskPollingMonitor import nextflow.util.Duration @Slf4j +@CompileStatic class CWSTaskPollingMonitor extends TaskPollingMonitor { /** diff --git a/plugins/nf-cws/src/main/nextflow/cws/processor/SchedulerBatch.groovy b/plugins/nf-cws/src/main/nextflow/cws/processor/SchedulerBatch.groovy index ae46ddb..cd5726b 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/processor/SchedulerBatch.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/processor/SchedulerBatch.groovy @@ -1,5 +1,8 @@ package nextflow.cws.processor +import groovy.transform.CompileStatic + +@CompileStatic abstract class SchedulerBatch { private final int batchSize diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy index 5f41554..8a9e5d2 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy @@ -1,8 +1,11 @@ package nextflow.cws.wow.file +import groovy.transform.CompileStatic + import java.nio.file.attribute.FileTime import java.text.SimpleDateFormat +@CompileStatic final class DateParser { private DateParser(){} diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy index e58c04d..7d75729 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy @@ -1,7 +1,10 @@ package nextflow.cws.wow.file +import groovy.transform.CompileStatic + import java.nio.file.Path +@CompileStatic class LocalFile extends File { private final LocalPath localPath diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy index f04b2f5..cdad294 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy @@ -1,5 +1,6 @@ package nextflow.cws.wow.file +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import java.nio.file.Files @@ -9,6 +10,7 @@ import java.nio.file.attribute.BasicFileAttributes import java.nio.file.attribute.FileTime @Slf4j +@CompileStatic class LocalFileWalker { static final int VIRTUAL_PATH = 0 diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index 853f752..3b13fa2 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -1,5 +1,6 @@ package nextflow.cws.wow.file +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.cws.k8s.K8sSchedulerClient import sun.net.ftp.FtpClient @@ -8,13 +9,14 @@ import java.nio.file.* import java.nio.file.attribute.BasicFileAttributes @Slf4j -class LocalPath implements Path { +@CompileStatic +class LocalPath implements Path, Serializable { protected final Path path private transient final LocalFileWalker.FileAttributes attributes private static transient K8sSchedulerClient client = null private boolean wasDownloaded = false - private Path workDir + protected Path workDir private boolean createdSymlinks = false private transient final Object createSymlinkHelper = new Object() @@ -56,6 +58,7 @@ class LocalPath implements Path { FtpClient ftpClient = FtpClient.create(daemon) ftpClient.login("root", "password".toCharArray() ) ftpClient.enablePassiveMode( true ) + ftpClient.setBinaryType() return ftpClient } catch ( IOException e ) { if ( trial > 5 ) throw e @@ -101,7 +104,13 @@ class LocalPath implements Path { Map download(){ log.info("FRIEDRICH download") final String absolutePath = path.toAbsolutePath().toString() - final def location = getLocation( absolutePath ) + def location + def trial = 0 + do { + if ( trial > 0 ) Thread.sleep( trial * 1000 ) + location = getLocation( absolutePath ) + trial++ + } while ( (location.node == null || location.daemon == null) && trial < 5 ) synchronized ( this ) { if ( this.wasDownloaded || location.sameAsEngine ) { log.trace("No download") @@ -132,11 +141,10 @@ class LocalPath implements Path { } T asType( Class c ) { - log.info("FRIEDRICH asType") - if ( c.isAssignableFrom( getClass() ) ) return this - if ( c.isAssignableFrom( LocalPath.class ) ) return toFile() - if ( c == String.class ) return toString() - log.info("Invoke method asType $c on $this") + if ( c.isAssignableFrom( getClass() ) ) return (T) this + if ( c.isAssignableFrom( LocalPath.class ) ) return (T) toFile() + if ( c == String.class ) return (T) toString() + log.info("Invoke method asType $c on ${this.class}") return super.asType( c ) } @@ -149,10 +157,10 @@ class LocalPath implements Path { Object result = path.invokeMethod(name, args) if( lastModified != file.lastModified() ){ //Update location in scheduler (overwrite all others) - client.addFileLocation( downloadResult.location.path.toString() , file.size(), file.lastModified(), downloadResult.location.locationWrapperID as long, true ) + client.addFileLocation( (downloadResult.location as Map).path.toString() , file.size(), file.lastModified(), (downloadResult.location as Map).locationWrapperID as long, true ) } else if ( downloadResult.wasDownloaded ){ //Add location to scheduler - client.addFileLocation( downloadResult.location.path.toString() , file.size(), file.lastModified(), downloadResult.location.locationWrapperID as long, false ) + client.addFileLocation( (downloadResult.location as Map).path.toString() , file.size(), file.lastModified(), (downloadResult.location as Map).locationWrapperID as long, false ) } return result } @@ -268,7 +276,8 @@ class LocalPath implements Path { @Override Path relativize(Path other) { if ( other instanceof LocalPath ){ - return toLocalPath( path.relativize( ((LocalPath) other).path ) ) + def localPath = (LocalPath) other + return toLocalPath( path.relativize( localPath.path), (LocalFileWalker.FileAttributes) localPath.attributes, localPath.workDir ) } path.relativize( other ) } @@ -278,6 +287,10 @@ class LocalPath implements Path { return getFileSystem().provider().getScheme() + "://" + path.toAbsolutePath() as URI } + String toUriString() { + return getFileSystem().provider().getScheme() + ":/" + path.toAbsolutePath() + } + Path toAbsolutePath(){ toLocalPath( path.toAbsolutePath() ) } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/OfflineLocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/OfflineLocalPath.groovy index 23772fe..44979c7 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/OfflineLocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/OfflineLocalPath.groovy @@ -1,14 +1,20 @@ package nextflow.cws.wow.file +import groovy.transform.CompileStatic import java.nio.file.Path /** * We need this class to be able to iterate through a subdirectory of a workdir */ +@CompileStatic class OfflineLocalPath extends LocalPath { final protected WorkdirHelper workdirHelper + private OfflineLocalPath(){ + this(null, null, null, null) + } + OfflineLocalPath( Path path, LocalFileWalker.FileAttributes attributes, Path workDir, WorkdirHelper workdirHelper ) { super(path, attributes, workDir) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystem.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystem.groovy index 9aa8274..f993231 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystem.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystem.groovy @@ -1,9 +1,12 @@ package nextflow.cws.wow.file +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic import java.nio.file.* import java.nio.file.attribute.UserPrincipalLookupService import java.nio.file.spi.FileSystemProvider +@CompileStatic class WOWFileSystem extends FileSystem { static final WOWFileSystem INSTANCE = new WOWFileSystem() @@ -53,6 +56,7 @@ class WOWFileSystem extends FileSystem { } @Override + @CompileDynamic PathMatcher getPathMatcher(String s) { return new PathMatcher() { private final def matcher = FileSystems.getDefault().getPathMatcher( s ) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemPathFactory.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemPathFactory.groovy new file mode 100644 index 0000000..fd8890a --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemPathFactory.groovy @@ -0,0 +1,37 @@ +package nextflow.cws.wow.file + +import nextflow.file.FileSystemPathFactory + +import java.nio.file.Path + +class WOWFileSystemPathFactory extends FileSystemPathFactory { + + + @Override + protected Path parseUri(String uri) { + if ( uri.startsWith("wow://") ) { + def of = Path.of(uri.substring(5)) + return LocalPath.toLocalPath(of, null, null ) + } + return null + } + + @Override + protected String toUriString(Path path) { + if ( path instanceof LocalPath ) { + return path.toUriString() + } + return null + } + + @Override + protected String getBashLib(Path target) { + null + } + + @Override + protected String getUploadCmd(String source, Path target) { + throw new UnsupportedOperationException("WOWFileSystemPathFactory does not support getUploadCmd") + } + +} diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy index 2763139..7de57a0 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy @@ -34,6 +34,10 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran assert path instanceof LocalPath Map location = (path as LocalPath).getLocation() + if ( location?.sameAsEngine ) { + return Files.newInputStream(path.getInner(), options) + } + FtpClient ftpClient = path.getConnection(location.node.toString(), location.daemon.toString()) InputStream is = ftpClient.getFileStream(location.path.toString()) return new WOWInputStream(is, schedulerClient, path, ftpClient) @@ -165,8 +169,7 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran @Override void download(Path source, Path target, CopyOption... copyOptions) throws IOException { - log.warn("Work in progress: Implementation for downloading functionality may not be correct or complete in ${getScheme().toUpperCase()} file system provider") - // do nothing, as data downloading is handled by the WOW scheduler + log.warn( "Not Implemented: Download from ${source} (${source.class.name}) to ${target} (${target.class.name}) with options ${copyOptions}" ) } @Override diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy index a3c5918..1cef9af 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy @@ -1,8 +1,10 @@ package nextflow.cws.wow.file +import groovy.transform.CompileStatic import nextflow.cws.SchedulerClient import sun.net.ftp.FtpClient +@CompileStatic class WOWInputStream extends InputStream { private InputStream inner @@ -50,10 +52,11 @@ class WOWInputStream extends InputStream { @Override int read() throws IOException { int b = inner.read() - temporaryFileStream.write(b) if (b == -1) { fullyRead = true checkTemporaryFileTransferal() + } else { + temporaryFileStream.write(b) } return b } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWOutputStream.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWOutputStream.groovy index c3fea9d..d4a8c6f 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWOutputStream.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWOutputStream.groovy @@ -1,7 +1,9 @@ package nextflow.cws.wow.file +import groovy.transform.CompileStatic import nextflow.cws.SchedulerClient +@CompileStatic class WOWOutputStream extends OutputStream { private OutputStream inner @@ -17,11 +19,12 @@ class WOWOutputStream extends OutputStream { @Override void write(int b) throws IOException { - super.write(b) + inner.write(b) } @Override void close() throws IOException { + inner.close() Map location = path.getLocation() File file = path.getInner().toFile() client.addFileLocation(path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy index 78c6556..8a51a46 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy @@ -1,11 +1,13 @@ package nextflow.cws.wow.file +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import java.nio.file.DirectoryStream import java.nio.file.Path @Slf4j +@CompileStatic class WorkdirHelper { private final Map paths @@ -17,10 +19,12 @@ class WorkdirHelper { this.paths = paths } + private WorkdirHelper() { + this(null, null) + } + void validate() { validated = true - // Garbage collector might remove unused paths - paths.clear() } boolean isValidated() { @@ -34,8 +38,12 @@ class WorkdirHelper { paths.get( path ) } - Path relativeToWorkdir( Path path ) { - rootPath.relativize( path ) + Path relativeToWorkdir( LocalPath path ) { + new LocalPath( rootPath.relativize( path.path ), (LocalFileWalker.FileAttributes) path.getAttributes(), path.workDir ) + } + + int getNameCount() { + rootPath.getNameCount() } DirectoryStream getDirectoryStream(Path path) { @@ -53,7 +61,7 @@ class WorkdirHelper { def result = toCompareAgainst.parent == cp.getInner() return result }.collect { - it.value + (Path) it.value } return all.iterator() } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy index f27eb61..7e1822b 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy @@ -1,5 +1,6 @@ package nextflow.cws.wow.file +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import java.nio.file.Path @@ -8,6 +9,7 @@ import java.nio.file.Path * This is required to calculate the correct relative path to the workdir. */ @Slf4j +@CompileStatic class WorkdirPath extends OfflineLocalPath { WorkdirPath(Path path, LocalFileWalker.FileAttributes attributes, Path workDir, WorkdirHelper workdirHelper) { @@ -21,7 +23,7 @@ class WorkdirPath extends OfflineLocalPath { if ( this.path == otherPath ) { return Path.of("") } - return workdirHelper.relativeToWorkdir( otherPath ) + return workdirHelper.relativeToWorkdir( (LocalPath) other ) } return this.path.relativize( other ) } @@ -37,4 +39,8 @@ class WorkdirPath extends OfflineLocalPath { workdirHelper.get( file ) ?: file } + @Override + int getNameCount() { + return workdirHelper.getNameCount() + } } diff --git a/plugins/nf-cws/src/resources/META-INF/extensions.idx b/plugins/nf-cws/src/resources/META-INF/extensions.idx index a8d9308..e48d4d8 100644 --- a/plugins/nf-cws/src/resources/META-INF/extensions.idx +++ b/plugins/nf-cws/src/resources/META-INF/extensions.idx @@ -1,2 +1,3 @@ nextflow.cws.CWSFactory -nextflow.cws.k8s.CWSK8sExecutor \ No newline at end of file +nextflow.cws.k8s.CWSK8sExecutor +nextflow.cws.wow.file.WOWFileSystemPathFactory \ No newline at end of file diff --git a/plugins/nf-cws/src/test/nextflow/cws/wow/file/WOWFileSystemTest.groovy b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WOWFileSystemTest.groovy new file mode 100644 index 0000000..182b2ba --- /dev/null +++ b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WOWFileSystemTest.groovy @@ -0,0 +1,19 @@ +package nextflow.cws.wow.file + +import spock.lang.Specification + +import java.nio.file.Path + +class OWFileSystemTest extends Specification { + + def "GetPathMatcher"() { + given: + def fileSystem = new WOWFileSystem() + + when: + def matcher = fileSystem.getPathMatcher("glob:*_data") + + then: + matcher.matches(Path.of("/localdata/localwork/c4/6274337c27f9979441ab60afdd145d/multiqc_data").getFileName()) + } +} From 4f3a5c445cfe2158dfe7171a1c1c38a987d14775 Mon Sep 17 00:00:00 2001 From: Fabian Lehmann <46564585+Lehmann-Fabian@users.noreply.github.com> Date: Fri, 2 May 2025 12:40:36 +0200 Subject: [PATCH 38/63] Fix Compilation Errors and Missing Files (#5) --- .../src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy | 2 +- .../main/nextflow/cws/k8s/K8sSchedulerClient.groovy | 2 +- .../cws/processor/CWSTaskPollingMonitor.groovy | 11 +++++++---- .../main/nextflow/cws/wow/file/LocalFileWalker.groovy | 4 ++++ 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy index 1b1bfd2..0c22839 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy @@ -278,7 +278,7 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { metadata: [ labels: [ name : name, - app: 'nextflow' + "nextflow.io/app" : 'nextflow' ] ], spec: spec, diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy index caf5b0d..40b4f7b 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy @@ -94,7 +94,7 @@ class K8sSchedulerClient extends SchedulerClient { .withNamespace( namespace ) .withLabel('component', 'scheduler') .withLabel('tier', 'control-plane') - .withLabel('app', 'nextflow') + .withLabel('nextflow.io/app', 'nextflow') .withHostMounts( hostMounts ) .withVolumeClaims( volumeClaims ) diff --git a/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy b/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy index e0133b3..e112342 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy @@ -64,10 +64,13 @@ class CWSTaskPollingMonitor extends TaskPollingMonitor { protected void finalizeTask(TaskHandler handler) { def workDir = handler.task.workDir def helper = LocalFileWalker.createWorkdirHelper( workDir ) - def attributes = new LocalFileWalker.FileAttributes(workDir) - OfflineLocalPath path = new WorkdirPath( workDir, attributes, workDir, helper ) - handler.task.workDir = path + if ( helper ) { + def attributes = new LocalFileWalker.FileAttributes(workDir) + OfflineLocalPath path = new WorkdirPath( workDir, attributes, workDir, helper ) + handler.task.workDir = path + } super.finalizeTask(handler) - helper.validate() + helper?.validate() } + } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy index cdad294..40db20b 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy @@ -26,6 +26,10 @@ class LocalFileWalker { Map files = new HashMap<>() Path rootPath = null File file = new File( start.toString() + File.separatorChar + ".command.outfiles" ) + if ( !file.exists() ) { + log.warn( "File ${file} does not exist" ) + return null + } String line WorkdirHelper workdirHelper file.withReader { reader -> From dc2198628b7870907e25e5a41f87a598bf80bdb2 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 2 May 2025 14:23:35 +0200 Subject: [PATCH 39/63] fix: do not check for .command.outfiles in CWS --- plugins/nf-cws/src/main/nextflow/cws/CWSConfig.groovy | 8 ++++++++ .../nextflow/cws/processor/CWSTaskPollingMonitor.groovy | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/plugins/nf-cws/src/main/nextflow/cws/CWSConfig.groovy b/plugins/nf-cws/src/main/nextflow/cws/CWSConfig.groovy index 3ec6f62..c445daf 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/CWSConfig.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/CWSConfig.groovy @@ -16,6 +16,14 @@ class CWSConfig { String getStrategy() { target.strategy as String ?: 'FIFO' } + boolean strategyIsLocationAware() { + switch(target.strategy) { + case "wow": return true + default: + return false + } + } + String getCostFunction() { target.costFunction as String } String getMemoryPredictor() { target.memoryPredictor as String } diff --git a/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy b/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy index e112342..817d110 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy @@ -3,6 +3,7 @@ package nextflow.cws.processor import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.Session +import nextflow.cws.CWSConfig import nextflow.cws.wow.file.LocalFileWalker import nextflow.cws.wow.file.OfflineLocalPath import nextflow.cws.wow.file.WorkdirPath @@ -18,6 +19,7 @@ class CWSTaskPollingMonitor extends TaskPollingMonitor { * Object to batch the task submission to achieve a better scheduling plan */ private final SchedulerBatch schedulerBatch + private final CWSConfig cwsConfig /** * Create the task polling monitor with the provided named parameters object. @@ -34,6 +36,7 @@ class CWSTaskPollingMonitor extends TaskPollingMonitor { protected CWSTaskPollingMonitor(Map params) { super(params) this.schedulerBatch = params.schedulerBatch as SchedulerBatch + cwsConfig = new CWSConfig(session.config.navigate('cws') as Map) } static TaskPollingMonitor create(Session session, String name, int defQueueSize, Duration defPollInterval, SchedulerBatch schedulerBatch ) { @@ -62,6 +65,10 @@ class CWSTaskPollingMonitor extends TaskPollingMonitor { @Override protected void finalizeTask(TaskHandler handler) { + if (!cwsConfig.strategyIsLocationAware()) { + super.finalizeTask(handler) + return + } def workDir = handler.task.workDir def helper = LocalFileWalker.createWorkdirHelper( workDir ) if ( helper ) { From edafe7d9c1ef41e719ec10a534f52236abf220f1 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 2 May 2025 14:24:00 +0200 Subject: [PATCH 40/63] fix: remove need for 'cp -u' to be implemented --- .../src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy index 57e9acf..92ab81f 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy @@ -95,7 +95,7 @@ class WOWK8sWrapperBuilder extends K8sWrapperBuilder { protected String getLaunchCommand(String interpreter, String env) { String cmd = '' if( storage && localWorkDir ){ - cmd += "cp -u \"/etc/nextflow/${statFileName}\" \"${getStorageLocalWorkDir()}\"\n" + cmd += "[[ -f \"${getStorageLocalWorkDir()}/${statFileName}\" ]] || cp \"/etc/nextflow/${statFileName}\" \"${getStorageLocalWorkDir()}\"\n" cmd += "chmod +x \"${getStorageLocalWorkDir()}/${statFileName}\"\n" cmd += "local INFILESTIME=\$(\"${getStorageLocalWorkDir()}/${statFileName}\" infiles \"${workDir.toString()}/.command.infiles\" \"${getStorageLocalWorkDir()}\" \"\$PWD/\" || true)\n" } From f1d67515494fcccd1567ad6a3b2c9b35009ecf70 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 2 May 2025 16:43:31 +0200 Subject: [PATCH 41/63] refactor: simplify LocalPath --- .../nextflow/cws/wow/file/LocalPath.groovy | 60 ------------------- .../cws/wow/file/WOWFileSystemProvider.groovy | 15 ++--- 2 files changed, 5 insertions(+), 70 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index 3b13fa2..49db2bc 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -15,7 +15,6 @@ class LocalPath implements Path, Serializable { protected final Path path private transient final LocalFileWalker.FileAttributes attributes private static transient K8sSchedulerClient client = null - private boolean wasDownloaded = false protected Path workDir private boolean createdSymlinks = false private transient final Object createSymlinkHelper = new Object() @@ -50,9 +49,7 @@ class LocalPath implements Path, Serializable { } static FtpClient getConnection(final String node, String daemon ){ - log.info("FRIEDRICH getConnection") int trial = 0 - log.info("FRIEDRICH $node -- $daemon") while ( true ) { try { FtpClient ftpClient = FtpClient.create(daemon) @@ -72,7 +69,6 @@ class LocalPath implements Path, Serializable { Map getLocation() { getLocation(path.toAbsolutePath().toString()) } Map getLocation( String absolutePath ){ - log.info("FRIEDRICH getLocation") Map response = client.getFileLocation( absolutePath ) synchronized ( createSymlinkHelper ) { if ( !createdSymlinks ) { @@ -101,45 +97,6 @@ class LocalPath implements Path, Serializable { response } - Map download(){ - log.info("FRIEDRICH download") - final String absolutePath = path.toAbsolutePath().toString() - def location - def trial = 0 - do { - if ( trial > 0 ) Thread.sleep( trial * 1000 ) - location = getLocation( absolutePath ) - trial++ - } while ( (location.node == null || location.daemon == null) && trial < 5 ) - synchronized ( this ) { - if ( this.wasDownloaded || location.sameAsEngine ) { - log.trace("No download") - return [ wasDownloaded : false, location : location ] - } - try (FtpClient ftpClient = getConnection(location.node as String, location.daemon as String)) { - try (InputStream fileStream = ftpClient.getFileStream(location.path as String)) { - log.trace("Download remote $absolutePath") - final def file = toFile() - path.parent.toFile().mkdirs() - OutputStream outStream = new FileOutputStream(file) - byte[] buffer = new byte[8 * 1024] - int bytesRead - while ((bytesRead = fileStream.read(buffer)) != -1) { - outStream.write(buffer, 0, bytesRead) - } - fileStream.close() - outStream.close() - this.wasDownloaded = true - return [wasDownloaded: true, location: location] - } catch (Exception e) { - throw e - } - } catch (Exception e) { - throw e - } - } - } - T asType( Class c ) { if ( c.isAssignableFrom( getClass() ) ) return (T) this if ( c.isAssignableFrom( LocalPath.class ) ) return (T) toFile() @@ -148,23 +105,6 @@ class LocalPath implements Path, Serializable { return super.asType( c ) } - @Override - Object invokeMethod(String name, Object args) { - log.info("FRIEDRICH invokeMethod") - Map downloadResult = download() - def file = path.toFile() - def lastModified = file.lastModified() - Object result = path.invokeMethod(name, args) - if( lastModified != file.lastModified() ){ - //Update location in scheduler (overwrite all others) - client.addFileLocation( (downloadResult.location as Map).path.toString() , file.size(), file.lastModified(), (downloadResult.location as Map).locationWrapperID as long, true ) - } else if ( downloadResult.wasDownloaded ){ - //Add location to scheduler - client.addFileLocation( (downloadResult.location as Map).path.toString() , file.size(), file.lastModified(), (downloadResult.location as Map).locationWrapperID as long, false ) - } - return result - } - String getBaseName() { path.getBaseName() } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy index 7de57a0..28e2237 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy @@ -32,13 +32,13 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran throw new RuntimeException("WOW file system has no registered scheduler client") } assert path instanceof LocalPath - Map location = (path as LocalPath).getLocation() + Map location = path.getLocation() if ( location?.sameAsEngine ) { return Files.newInputStream(path.getInner(), options) } - FtpClient ftpClient = path.getConnection(location.node.toString(), location.daemon.toString()) + FtpClient ftpClient = LocalPath.getConnection(location.node.toString(), location.daemon.toString()) InputStream is = ftpClient.getFileStream(location.path.toString()) return new WOWInputStream(is, schedulerClient, path, ftpClient) } @@ -75,13 +75,8 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran } @Override - SeekableByteChannel newByteChannel(Path path, Set set, FileAttribute... fileAttributes) throws IOException { - log.info("FRIEDRICH: newByteChannel") - LocalPath localPath = (LocalPath) path - // TODO: find an alternative to always downloading - Map downloadResult = localPath.download() - assert (downloadResult.wasDownloaded || downloadResult.location.sameAsEngine) - return Files.newByteChannel(localPath.getInner()) + SeekableByteChannel newByteChannel(Path path, Set options, FileAttribute... attrs) throws IOException { + throw new UnsupportedOperationException("New byte channel not supported by ${getScheme().toUpperCase()} file system provider") } @Override @@ -110,7 +105,7 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran @Override void move(Path path, Path path1, CopyOption... copyOptions) throws IOException { - throw new UnsupportedOperationException("move not supported by ${getScheme().toUpperCase()} file system provider") + throw new UnsupportedOperationException("Move not supported by ${getScheme().toUpperCase()} file system provider") } @Override From 84151c6df30cc3245180245a5bfa4c5fade8a3ef Mon Sep 17 00:00:00 2001 From: Fabian Lehmann <46564585+Lehmann-Fabian@users.noreply.github.com> Date: Mon, 5 May 2025 01:44:01 +0200 Subject: [PATCH 42/63] Use ConfigMap and make BashWrapper Ponder compatible (#6) --- .../main/nextflow/cws/k8s/CWSK8sConfig.groovy | 1 + .../nextflow/cws/k8s/CWSK8sExecutor.groovy | 10 ++++--- .../nextflow/cws/k8s/CWSK8sTaskHandler.groovy | 28 +++++++++++++++---- .../cws/k8s/WOWK8sWrapperBuilder.groovy | 17 ++++------- .../k8s/model/PodMountConfigWithMode.groovy | 23 --------------- .../wow/file/WOWFileSystemPathFactory.groovy | 2 ++ .../cws/wow/file/WOWFileSystemProvider.groovy | 2 ++ 7 files changed, 40 insertions(+), 43 deletions(-) delete mode 100644 plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy index 7bca2c0..d1f7002 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy @@ -12,6 +12,7 @@ import nextflow.k8s.model.PodNodeSelector import java.nio.file.Path import java.util.stream.Collectors +@CompileStatic class CWSK8sConfig extends K8sConfig { private Map target diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy index 0c22839..0bea2e0 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy @@ -8,13 +8,13 @@ import groovy.util.logging.Slf4j import nextflow.cws.CWSConfig import nextflow.cws.CWSSchedulerBatch import nextflow.cws.SchedulerClient -import nextflow.cws.k8s.model.PodMountConfigWithMode import nextflow.cws.processor.CWSTaskPollingMonitor import nextflow.k8s.K8sConfig import nextflow.k8s.K8sExecutor import nextflow.k8s.client.K8sClient import nextflow.k8s.client.K8sResponseException import nextflow.k8s.model.PodHostMount +import nextflow.k8s.model.PodMountConfig import nextflow.k8s.model.PodOptions import nextflow.k8s.model.PodVolumeClaim import nextflow.processor.TaskHandler @@ -39,6 +39,8 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { */ private String daemonSet = null + private String configMapName = null + @Override @Memoized protected K8sConfig getK8sConfig() { @@ -73,7 +75,7 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { assert task assert task.workDir log.trace "[K8s] launching process > ${task.name} -- work folder: ${task.workDirStr}" - return new CWSK8sTaskHandler( task, this ) + return new CWSK8sTaskHandler( task, this, configMapName ) } @Override @@ -184,10 +186,10 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { final content = contentStream.bytes.encodeBase64().toString() configMap[WOWK8sWrapperBuilder.statFileName] = content - String configMapName = makeConfigMapName(content) + configMapName = makeConfigMapName(content) tryCreateConfigMap(configMapName, configMap) log.debug "Created K8s configMap with name: $configMapName" - k8sConfig.getPodOptions().getMountConfigMaps().add( new PodMountConfigWithMode(configMapName, '/etc/nextflow', 0111) ) + k8sConfig.getPodOptions().getMountConfigMaps().add( new PodMountConfig(configMapName, '/etc/nextflow') ) } protected void tryCreateConfigMap(String name, Map data) { diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy index 5919bd0..fb1768b 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy @@ -39,12 +39,15 @@ class CWSK8sTaskHandler extends K8sTaskHandler { private boolean failedOOM = false - CWSK8sTaskHandler( TaskRun task, CWSK8sExecutor executor ) { + private final String configMapName + + CWSK8sTaskHandler( TaskRun task, CWSK8sExecutor executor, String configMapName ) { super( task, executor ) this.client = executor.getCWSK8sClient() this.schedulerClient = executor.schedulerClient this.executor = executor this.syntheticPodName = super.getSyntheticPodName(task) + this.configMapName = configMapName } @Override @@ -58,6 +61,20 @@ class CWSK8sTaskHandler extends K8sTaskHandler { if ( (k8sConfig as CWSK8sConfig)?.getScheduler() ){ (pod.spec as Map).schedulerName = (k8sConfig as CWSK8sConfig).getScheduler().getName() + "-" + getRunName() } + + //Set default mode for configMap + Map specs = pod.spec as Map + List volumes = specs?.volumes as List + if ( volumes ) { + for ( Map vol : volumes ) { + if ( (vol.configMap as Map)?.name == configMapName ) { + Map configMap = vol.configMap as Map + configMap.defaultMode = 0755 + break + } + } + } + return pod } @@ -145,7 +162,7 @@ class CWSK8sTaskHandler extends K8sTaskHandler { protected BashWrapperBuilder createBashWrapper(TaskRun task) { CWSK8sConfig cwsK8sConfig = k8sConfig as CWSK8sConfig if ( cwsK8sConfig?.locationAwareScheduling() ) { - return new WOWK8sWrapperBuilder( task , cwsK8sConfig.getStorage() ) + return new WOWK8sWrapperBuilder( task , cwsK8sConfig.getStorage(), executor.getCWSConfig().memoryPredictor as boolean ) } return fusionEnabled() ? fusionLauncher() @@ -228,14 +245,15 @@ class CWSK8sTaskHandler extends K8sTaskHandler { if( value == null ) continue switch( name ) { - case "scheduler_nodes_cost" : + case 'scheduler_nodes_cost' : + case 'scheduler_time_delta_phase_three' : traceRecord.put( name, value ) break - case "scheduler_best_cost" : + case 'scheduler_best_cost' : double val = parseDouble( value, file, name ) traceRecord.put( name, val ) break - case "input_size" : + case 'input_size' : traceRecord.put( name, inputSize ) break default: diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy index 92ab81f..e59c90e 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy @@ -16,15 +16,16 @@ import java.nio.file.Path */ @CompileStatic @Slf4j -class WOWK8sWrapperBuilder extends K8sWrapperBuilder { +class WOWK8sWrapperBuilder extends CWSK8sWrapperBuilder { CWSK8sConfig.Storage storage Path localWorkDir static String statFileName = "getStatsAndSymlinks" - WOWK8sWrapperBuilder(TaskRun task, CWSK8sConfig.Storage storage) { - this(task) + WOWK8sWrapperBuilder(TaskRun task, CWSK8sConfig.Storage storage, boolean memoryPredictorEnabled) { + super(task, memoryPredictorEnabled) + this.headerScript = "NXF_CHDIR=${Escape.path(task.workDir)}" this.storage = storage if( storage ){ switch (storage.getCopyStrategy().toLowerCase()) { @@ -46,10 +47,6 @@ class WOWK8sWrapperBuilder extends K8sWrapperBuilder { } } - WOWK8sWrapperBuilder(TaskRun task) { - super(task) - this.headerScript = "NXF_CHDIR=${Escape.path(task.workDir)}" - } /** * only for testing purpose -- do not use */ @@ -95,9 +92,7 @@ class WOWK8sWrapperBuilder extends K8sWrapperBuilder { protected String getLaunchCommand(String interpreter, String env) { String cmd = '' if( storage && localWorkDir ){ - cmd += "[[ -f \"${getStorageLocalWorkDir()}/${statFileName}\" ]] || cp \"/etc/nextflow/${statFileName}\" \"${getStorageLocalWorkDir()}\"\n" - cmd += "chmod +x \"${getStorageLocalWorkDir()}/${statFileName}\"\n" - cmd += "local INFILESTIME=\$(\"${getStorageLocalWorkDir()}/${statFileName}\" infiles \"${workDir.toString()}/.command.infiles\" \"${getStorageLocalWorkDir()}\" \"\$PWD/\" || true)\n" + cmd += "local INFILESTIME=\$(\"/etc/nextflow/${statFileName}\" infiles \"${workDir.toString()}/.command.infiles\" \"${getStorageLocalWorkDir()}\" \"\$PWD/\" || true)\n" } cmd += super.getLaunchCommand(interpreter, env) if( storage && localWorkDir && isTraceRequired() ){ @@ -113,7 +108,7 @@ class WOWK8sWrapperBuilder extends K8sWrapperBuilder { String cmd = super.getCleanupCmd( scratch ) if( storage && localWorkDir ){ cmd += "mkdir -p \"${localWorkDir.toString()}/\" || true\n" - cmd += "local OUTFILESTIME=\$(\"${getStorageLocalWorkDir()}/${statFileName}\" outfiles \"${workDir.toString()}/.command.outfiles\" \"${getStorageLocalWorkDir()}\" \"${localWorkDir.toString()}/\" || true)\n" + cmd += "local OUTFILESTIME=\$(\"/etc/nextflow/${statFileName}\" outfiles \"${workDir.toString()}/.command.outfiles\" \"${getStorageLocalWorkDir()}\" \"${localWorkDir.toString()}/\" || true)\n" if ( isTraceRequired() ) { cmd += "echo \"outfiles_time=\${OUTFILESTIME}\" >> ${workDir.resolve(TaskRun.CMD_TRACE)}" } diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy deleted file mode 100644 index f88a61e..0000000 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/model/PodMountConfigWithMode.groovy +++ /dev/null @@ -1,23 +0,0 @@ -package nextflow.cws.k8s.model - -import groovy.transform.CompileStatic -import nextflow.k8s.model.PodMountConfig - -@CompileStatic -class PodMountConfigWithMode extends PodMountConfig { - private Integer mode - - PodMountConfigWithMode(String config, String mount, Integer mode = null) { - super(config, mount) - this.mode = mode - } - - Integer getMode() { - return mode - } - - void setMode(Integer mode) { - this.mode = mode - } - -} \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemPathFactory.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemPathFactory.groovy index fd8890a..3c508e7 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemPathFactory.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemPathFactory.groovy @@ -1,9 +1,11 @@ package nextflow.cws.wow.file +import groovy.transform.CompileStatic import nextflow.file.FileSystemPathFactory import java.nio.file.Path +@CompileStatic class WOWFileSystemPathFactory extends FileSystemPathFactory { diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy index 28e2237..d817bb4 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy @@ -1,5 +1,6 @@ package nextflow.cws.wow.file +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.cws.SchedulerClient import nextflow.file.FileSystemTransferAware @@ -13,6 +14,7 @@ import java.nio.file.attribute.FileAttributeView import java.nio.file.spi.FileSystemProvider @Slf4j +@CompileStatic class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTransferAware { static final WOWFileSystemProvider INSTANCE = new WOWFileSystemProvider() From 8c6b406de79f3441eb8bff82327bd8877091e267 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 9 May 2025 18:16:36 +0200 Subject: [PATCH 43/63] Module Restructure and TODOs (#7) --- Makefile | 38 +- .../src/main/nextflow/cws/CWSPlugin.groovy | 11 +- .../main/nextflow/cws/SchedulerClient.groovy | 41 +- .../main/nextflow/cws/k8s/CWSK8sClient.groovy | 4 - .../nextflow/cws/k8s/CWSK8sExecutor.groovy | 23 +- .../nextflow/cws/k8s/CWSK8sTaskHandler.groovy | 7 +- .../cws/k8s/K8sSchedulerClient.groovy | 8 +- .../cws/k8s/WOWK8sWrapperBuilder.groovy | 6 - .../processor/CWSTaskPollingMonitor.groovy | 24 +- .../cws/processor/SchedulerBatch.groovy | 1 + .../nextflow/cws/wow/file/LocalFile.groovy | 1 + .../cws/wow/file/LocalFileWalker.groovy | 166 ------- .../nextflow/cws/wow/file/LocalPath.groovy | 103 +---- .../cws/wow/file/OfflineLocalPath.groovy | 10 +- .../cws/wow/file/WOWFileAttributes.groovy | 155 +++++++ .../cws/wow/file/WorkdirHelper.groovy | 9 +- .../nextflow/cws/wow/file/WorkdirPath.groovy | 2 +- .../{file => filesystem}/WOWFileSystem.groovy | 162 +++---- .../WOWFileSystemPathFactory.groovy | 78 ++-- .../WOWFileSystemProvider.groovy | 411 ++++++++++-------- .../WOWInputStream.groovy | 6 +- .../WOWOutputStream.groovy | 8 +- .../wow/serializer/LocalPathSerializer.groovy | 32 ++ .../cws/wow/{file => util}/DateParser.groovy | 7 +- .../cws/wow/util/LocalFileWalker.groovy | 55 +++ .../src/resources/META-INF/extensions.idx | 2 +- .../cws/wow/file/WOWFileSystemTest.groovy | 1 + .../cws/wow/file/WorkdirHelperTest.groovy | 3 +- .../cws/wow/file/WorkdirPathTest.groovy | 3 +- 29 files changed, 781 insertions(+), 596 deletions(-) delete mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy create mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy rename plugins/nf-cws/src/main/nextflow/cws/wow/{file => filesystem}/WOWFileSystem.groovy (94%) rename plugins/nf-cws/src/main/nextflow/cws/wow/{file => filesystem}/WOWFileSystemPathFactory.groovy (88%) rename plugins/nf-cws/src/main/nextflow/cws/wow/{file => filesystem}/WOWFileSystemProvider.groovy (68%) rename plugins/nf-cws/src/main/nextflow/cws/wow/{file => filesystem}/WOWInputStream.groovy (92%) rename plugins/nf-cws/src/main/nextflow/cws/wow/{file => filesystem}/WOWOutputStream.groovy (83%) create mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/serializer/LocalPathSerializer.groovy rename plugins/nf-cws/src/main/nextflow/cws/wow/{file => util}/DateParser.groovy (86%) create mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/util/LocalFileWalker.groovy diff --git a/Makefile b/Makefile index e922eb9..0b1cf5b 100644 --- a/Makefile +++ b/Makefile @@ -7,11 +7,44 @@ else mm = endif +C_SCRIPT_SRC = plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c +C_SCRIPT_AARCH64 = plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_linux_aarch64 +C_SCRIPT_X86_64 = plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_linux_x86_64 +C_SCRIPT_ALL_TARGETS = $(C_SCRIPT_X86_64) $(C_SCRIPT_AARCH64) + +arch:=$(shell uname -m) + +ifneq ($(arch),x86_64) +$(info ====================================================================================) +$(info This Makefile assumes to be run on an x86_64 build system. Found: $(arch)) +$(info As an alternative you can adjust the Makefile or build $(C_SCRIPT_SRC) yourself for the following targets:) +$(info - target aarch64-linux-gnu: saved to $(C_SCRIPT_AARCH64)) +$(info - target x86_64-linux-gnu: saved to $(C_SCRIPT_X86_64)) +$(error Aborting) +endif + +$(C_SCRIPT_AARCH64): $(C_SRIPT_SRC) + clang -static \ + -target aarch64-linux-gnu \ + --sysroot=/usr/aarch64-linux-gnu \ + plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c \ + -fuse-ld=lld \ + -o $@ + + +$(C_SCRIPT_X86_64): $(C_SCRIPT_SRC) + clang -static \ + -target x86_64-linux-gnu \ + plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c \ + -fuse-ld=lld \ + -o $@ + clean: rm -rf .nextflow* rm -rf work rm -rf build rm -rf plugins/*/build + rm $(C_SCRIPT_ALL_TARGETS) ./gradlew clean compile: @@ -55,10 +88,7 @@ assemble: # generate build zips under build/plugins # you can install the plugin copying manually these files to $HOME/.nextflow/plugins # -buildPlugins: - # TODO: add targets - # clang -static -target arm-linux-gnu plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c -o plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_linux_aarch86 - clang -static -target x86_64-linux-gnu plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c -o plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks_linux_x86 +buildPlugins: $(C_SCRIPT_ALL_TARGETS) ./gradlew copyPluginZip # diff --git a/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy b/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy index ffaf5bf..2b0ed98 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy @@ -1,9 +1,11 @@ package nextflow.cws import groovy.transform.CompileStatic +import nextflow.cws.wow.file.LocalPath import nextflow.cws.wow.file.OfflineLocalPath -import nextflow.cws.wow.file.WOWFileSystemProvider -import nextflow.cws.wow.file.WorkdirHelper +import nextflow.cws.wow.file.WorkdirPath +import nextflow.cws.wow.filesystem.WOWFileSystemProvider +import nextflow.cws.wow.serializer.LocalPathSerializer import nextflow.file.FileHelper import nextflow.plugin.BasePlugin import nextflow.trace.TraceRecord @@ -62,8 +64,9 @@ class CWSPlugin extends BasePlugin { void start() { super.start() registerTraceFields() - KryoHelper.register( OfflineLocalPath ) - KryoHelper.register( WorkdirHelper ) + KryoHelper.register( LocalPath, LocalPathSerializer ) + KryoHelper.register( OfflineLocalPath, LocalPathSerializer ) + KryoHelper.register( WorkdirPath, LocalPathSerializer ) FileHelper.getOrInstallProvider(WOWFileSystemProvider) } diff --git a/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy index 5958b60..7be4055 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy @@ -5,18 +5,25 @@ import groovy.json.JsonSlurper import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import nextflow.cws.wow.file.WOWFileSystemProvider +import nextflow.cws.wow.filesystem.WOWFileSystemProvider import nextflow.dag.DAG +import java.nio.file.Path + @Slf4j @CompileStatic class SchedulerClient { private final CWSConfig config + private final String runName + private boolean registered = false + private boolean closed = false + private int tasksInBatch = 0 + protected String dns SchedulerClient( CWSConfig config, String runName ) { @@ -138,8 +145,8 @@ class SchedulerClient { } - ///* DAG */ + @CompileDynamic void submitVertices( List vertices ){ List> verticesToSubmit = vertices.collect { @@ -239,4 +246,34 @@ class SchedulerClient { } } + + void publish(Path source, Path destination, String mode ) { + HttpURLConnection get = URI.create("${getDNS()}/file/$runName/publish").toURL().openConnection() as HttpURLConnection + get.setRequestMethod( "PUT" ) + get.setDoOutput(true) + Map data = [ + 'source' : source.toString(), + 'destination' : destination.toString(), + 'mode' : mode.toUpperCase(), + ] + String message = JsonOutput.toJson( data ) + get.setRequestProperty("Content-Type", "application/json") + get.getOutputStream().write(message.getBytes("UTF-8")) + int responseCode = get.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while publishing file: $source (${get.responseMessage}) -- ${get.getURL()}" ) + } + } + + int getRemainingToPublish() { + HttpURLConnection get = URI.create("${getDNS()}/file/$runName/publish").toURL().openConnection() as HttpURLConnection + get.setRequestMethod( "GET" ) + get.setRequestProperty("Content-Type", "application/json") + int responseCode = get.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while getting remaining to publish (${get.responseMessage})" ) + } + get.getInputStream().text as int + } + } \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sClient.groovy index 3ed27a0..48b41a6 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sClient.groovy @@ -15,10 +15,6 @@ import java.nio.file.Path @CompileStatic class CWSK8sClient extends K8sClient { - CWSK8sClient(K8sClient k8sClient) { - super(k8sClient.config) - } - CWSK8sClient(ClientConfig config) { super(config) } diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy index 0bea2e0..fc856dd 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy @@ -31,9 +31,13 @@ import java.nio.file.Paths @CompileStatic @ServiceName('k8s') class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { + @PackageScope SchedulerClient schedulerClient + @PackageScope CWSSchedulerBatch schedulerBatch + protected CWSK8sClient client + /** * Name of the created daemonSet */ @@ -46,13 +50,16 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { protected K8sConfig getK8sConfig() { return new CWSK8sConfig( (Map)session.config.k8s ) } + @Memoized @PackageScope CWSConfig getCWSConfig(){ new CWSConfig(session.config.navigate('cws') as Map) } + @PackageScope CWSK8sClient getCWSK8sClient() { client } + /** * @return A {@link nextflow.processor.TaskMonitor} associated to this executor type */ @@ -64,6 +71,7 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { } return CWSTaskPollingMonitor.create( session, name, 100, Duration.of('5 sec'), this.schedulerBatch ) } + /** * Creates a {@link nextflow.processor.TaskHandler} for the given {@link nextflow.processor.TaskRun} instance * @@ -147,9 +155,20 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { this.schedulerBatch?.setSchedulerClient( schedulerClient ) schedulerClient.registerScheduler( data ) } + @Override void shutdown() { final CWSK8sConfig.K8sScheduler schedulerConfig = (k8sConfig as CWSK8sConfig).getScheduler() + + int remaining + do { + remaining = schedulerClient.getRemainingToPublish() + if( remaining > 0 ) { + log.info "Waiting for $remaining files to be published" + sleep( 1000 ) + } + } while( remaining > 0 ) + if( schedulerConfig ) { try{ schedulerClient.closeScheduler() @@ -175,11 +194,11 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { String architectureStatFileName final String architecture = System.getProperty("os.arch") if (architecture == "amd64" || architecture == "x86_64") { - architectureStatFileName = "/nf-cws/getStatsAndResolveSymlinks_linux_x86" + architectureStatFileName = "/nf-cws/getStatsAndResolveSymlinks_linux_x86_64" } else if (architecture == "aarch64" || architecture == "arm64") { architectureStatFileName = "/nf-cws/getStatsAndResolveSymlinks_linux_aarch64" } else { - throw new RuntimeException("The ${architecture} architecture is by default not supported for WOW." + + throw new RuntimeException("The ${architecture} architecture is currently not supported for WOW. " + "You may compile the getStatsAndResolveSymlinks.c yourself and add it to the resources directory.") } final contentStream = CWSK8sExecutor.class.getResourceAsStream(architectureStatFileName) diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy index fb1768b..ae3e023 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy @@ -67,6 +67,7 @@ class CWSK8sTaskHandler extends K8sTaskHandler { List volumes = specs?.volumes as List if ( volumes ) { for ( Map vol : volumes ) { + if ( vol.configMap == null ) continue if ( (vol.configMap as Map)?.name == configMapName ) { Map configMap = vol.configMap as Map configMap.defaultMode = 0755 @@ -115,7 +116,7 @@ class CWSK8sTaskHandler extends K8sTaskHandler { } } - private long calculateInputSize( List> fileInputs ){ + private static long calculateInputSize(List> fileInputs ){ return fileInputs .parallelStream() .mapToLong { @@ -263,7 +264,7 @@ class CWSK8sTaskHandler extends K8sTaskHandler { } } - private double parseDouble( String str, Path file , String row ) { + private static double parseDouble(String str, Path file, String row ) { try { return str.toDouble() } @@ -273,7 +274,7 @@ class CWSK8sTaskHandler extends K8sTaskHandler { } } - private long parseLong( String str, Path file , String row ) { + private static long parseLong(String str, Path file, String row ) { try { return str.toLong() } diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy index 40b4f7b..b507689 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/K8sSchedulerClient.groovy @@ -4,7 +4,6 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.cws.CWSConfig import nextflow.cws.SchedulerClient -import nextflow.cws.wow.file.LocalPath import nextflow.exception.NodeTerminationException import nextflow.k8s.K8sConfig import nextflow.k8s.client.K8sResponseException @@ -20,11 +19,17 @@ import java.nio.file.Paths class K8sSchedulerClient extends SchedulerClient { private final CWSK8sConfig.K8sScheduler schedulerConfig + private final CWSK8sClient k8sClient + private final K8sConfig k8sConfig + private final String namespace + private final Collection hostMounts + private final Collection volumeClaims + private String ip K8sSchedulerClient( @@ -44,7 +49,6 @@ class K8sSchedulerClient extends SchedulerClient { this.k8sConfig = k8sConfig this.schedulerConfig = schedulerConfig this.namespace = namespace ?: 'default' - LocalPath.setClient(this) } protected String getDNS(){ diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy index e59c90e..98b83cd 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy @@ -3,7 +3,6 @@ package nextflow.cws.k8s import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.file.FileHelper -import nextflow.k8s.K8sWrapperBuilder import nextflow.processor.TaskRun import nextflow.util.Escape @@ -47,11 +46,6 @@ class WOWK8sWrapperBuilder extends CWSK8sWrapperBuilder { } } - /** - * only for testing purpose -- do not use - */ - protected K8sWrapperBuilder() {} - @Override protected boolean shouldUnstageOutputs() { return localWorkDir || super.shouldUnstageOutputs() diff --git a/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy b/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy index 817d110..a908936 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/processor/CWSTaskPollingMonitor.groovy @@ -4,9 +4,11 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.Session import nextflow.cws.CWSConfig -import nextflow.cws.wow.file.LocalFileWalker import nextflow.cws.wow.file.OfflineLocalPath +import nextflow.cws.wow.file.WOWFileAttributes import nextflow.cws.wow.file.WorkdirPath +import nextflow.cws.wow.util.LocalFileWalker +import nextflow.processor.PublishDir import nextflow.processor.TaskHandler import nextflow.processor.TaskPollingMonitor import nextflow.util.Duration @@ -19,6 +21,7 @@ class CWSTaskPollingMonitor extends TaskPollingMonitor { * Object to batch the task submission to achieve a better scheduling plan */ private final SchedulerBatch schedulerBatch + private final CWSConfig cwsConfig /** @@ -63,6 +66,22 @@ class CWSTaskPollingMonitor extends TaskPollingMonitor { return pendingTasks } + private static void checkPublishDirMode(TaskHandler handler ) { + def publishDirs = handler.task.config.get('publishDir') + if ( publishDirs && publishDirs instanceof List ) { + for( Object params : publishDirs ) { + if( !params ) continue + if( params instanceof Map ) { + def mode = PublishDir.create(params).getMode() + // We only support COPY and MOVE, if the user uses a different mode, we set it to COPY + if ( !(mode in [ PublishDir.Mode.COPY, PublishDir.Mode.MOVE]) ) { + params.mode = PublishDir.Mode.COPY + } + } + } + } + } + @Override protected void finalizeTask(TaskHandler handler) { if (!cwsConfig.strategyIsLocationAware()) { @@ -72,10 +91,11 @@ class CWSTaskPollingMonitor extends TaskPollingMonitor { def workDir = handler.task.workDir def helper = LocalFileWalker.createWorkdirHelper( workDir ) if ( helper ) { - def attributes = new LocalFileWalker.FileAttributes(workDir) + def attributes = new WOWFileAttributes(workDir) OfflineLocalPath path = new WorkdirPath( workDir, attributes, workDir, helper ) handler.task.workDir = path } + checkPublishDirMode(handler) super.finalizeTask(handler) helper?.validate() } diff --git a/plugins/nf-cws/src/main/nextflow/cws/processor/SchedulerBatch.groovy b/plugins/nf-cws/src/main/nextflow/cws/processor/SchedulerBatch.groovy index cd5726b..480e906 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/processor/SchedulerBatch.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/processor/SchedulerBatch.groovy @@ -6,6 +6,7 @@ import groovy.transform.CompileStatic abstract class SchedulerBatch { private final int batchSize + private int currentlySubmitted = 0 SchedulerBatch( int batchSize ) { diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy index 7d75729..730bd6a 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFile.groovy @@ -18,4 +18,5 @@ class LocalFile extends File { Path toPath() { return localPath } + } \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy deleted file mode 100644 index 40db20b..0000000 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalFileWalker.groovy +++ /dev/null @@ -1,166 +0,0 @@ -package nextflow.cws.wow.file - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j - -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.nio.file.attribute.BasicFileAttributes -import java.nio.file.attribute.FileTime - -@Slf4j -@CompileStatic -class LocalFileWalker { - - static final int VIRTUAL_PATH = 0 - static final int FILE_EXISTS = 1 - static final int REAL_PATH = 2 - static final int SIZE = 3 - static final int FILE_TYPE = 4 - static final int CREATION_DATE = 5 - static final int ACCESS_DATE = 6 - static final int MODIFICATION_DATE = 7 - - static WorkdirHelper createWorkdirHelper(Path start){ - Map files = new HashMap<>() - Path rootPath = null - File file = new File( start.toString() + File.separatorChar + ".command.outfiles" ) - if ( !file.exists() ) { - log.warn( "File ${file} does not exist" ) - return null - } - String line - WorkdirHelper workdirHelper - file.withReader { reader -> - line = reader.readLine() - rootPath = Paths.get(line.split(';')[ VIRTUAL_PATH ]) - workdirHelper = new WorkdirHelper( rootPath, files ) - while ((line = reader.readLine()) != null) { - String[] data = line.split(';') - String path = data[ VIRTUAL_PATH ] - FileAttributes attributes = new FileAttributes( data ) - Path currentPath = Paths.get(path) - if ( !attributes.local ) { - //If task did not run on local machine, create symbolic link - if ( !Files.isSymbolicLink( currentPath ) ) { - Files.createDirectories( currentPath.getParent() ) - Files.createSymbolicLink( currentPath, attributes.destination ) - } - } else { - def localPath = new OfflineLocalPath(currentPath, attributes, start, workdirHelper) - def pathOnSharedFs = start.resolve(rootPath.relativize(currentPath)) - files.put( pathOnSharedFs, localPath ) - } - } - return workdirHelper - } - } - - static class FileAttributes implements BasicFileAttributes { - - private final boolean directory - private final boolean link - private final long size - private final String fileType - private final FileTime creationDate - private final FileTime accessDate - private final FileTime modificationDate - private final Path destination - private final boolean local - - FileAttributes( String[] data ) { - if ( data.length != 8 && data[ FILE_EXISTS ] != "0" ) throw new RuntimeException( "Cannot parse row (8 columns required): ${data.join(',')}" ) - boolean fileExists = data[ FILE_EXISTS ] == "1" - destination = data.length > REAL_PATH && data[ REAL_PATH ] ? data[ REAL_PATH ] as Path : null - if ( data.length != 8 ) { - this.link = true - this.size = 0 - this.fileType = null - this.creationDate = null - this.accessDate = null - this.modificationDate = null - return - } - this.link = !data[ REAL_PATH ].isEmpty() - this.size = data[ SIZE ] as Long - this.fileType = data[ FILE_TYPE ] - this.accessDate = DateParser.fileTimeFromString(data[ ACCESS_DATE ]) - this.modificationDate = DateParser.fileTimeFromString(data[ MODIFICATION_DATE ]) - this.creationDate = DateParser.fileTimeFromString(data[ CREATION_DATE ]) ?: this.modificationDate - if ( fileType.startsWith("non-local ") ) { - this.local = false - this.fileType = fileType.substring( 10 ) - } else { - this.local = true - } - this.directory = fileType == 'directory' - if ( !directory && !fileType.contains( 'file' ) ){ - log.error( "Unknown type: $fileType" ) - } - } - - FileAttributes( Path path ) { - directory = true - link = false - size = 4096 - fileType = 'directory' - creationDate = FileTime.fromMillis( 0 ) - accessDate = FileTime.fromMillis( 0 ) - modificationDate = FileTime.fromMillis( 0 ) - destination = path - local = false - } - - @Override - FileTime lastModifiedTime() { - return modificationDate - } - - @Override - FileTime lastAccessTime() { - return accessDate - } - - @Override - FileTime creationTime() { - return creationDate - } - - @Override - boolean isRegularFile() { - return !directory - } - - @Override - boolean isDirectory() { - return directory - } - - @Override - boolean isSymbolicLink() { - return link - } - - @Override - boolean isOther() { - return false - } - - @Override - long size() { - return size - } - - @Override - Object fileKey() { - return null - } - - Path getDestination(){ - destination - } - - } - -} \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index 49db2bc..8ef494e 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -2,101 +2,40 @@ package nextflow.cws.wow.file import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import nextflow.cws.k8s.K8sSchedulerClient -import sun.net.ftp.FtpClient +import nextflow.cws.wow.filesystem.WOWFileSystem import java.nio.file.* -import java.nio.file.attribute.BasicFileAttributes @Slf4j @CompileStatic class LocalPath implements Path, Serializable { protected final Path path - private transient final LocalFileWalker.FileAttributes attributes - private static transient K8sSchedulerClient client = null + + private transient final WOWFileAttributes attributes + + boolean createdSymlinks = false + protected Path workDir - private boolean createdSymlinks = false - private transient final Object createSymlinkHelper = new Object() - protected LocalPath(Path path, LocalFileWalker.FileAttributes attributes, Path workDir ) { + protected LocalPath(Path path, WOWFileAttributes attributes, Path workDir ) { this.path = path this.attributes = attributes this.workDir = workDir } - private LocalPath(){ - path = null - this.attributes = null - this.workDir = null - } - Path getInner() { - return path + path } - LocalPath toLocalPath( Path path, LocalFileWalker.FileAttributes attributes = null ){ + LocalPath toLocalPath( Path path, WOWFileAttributes attributes = null ){ toLocalPath( path, attributes, workDir ) } - static LocalPath toLocalPath( Path path, LocalFileWalker.FileAttributes attributes, Path workDir ){ + static LocalPath toLocalPath( Path path, WOWFileAttributes attributes, Path workDir ){ ( path instanceof LocalPath ) ? path as LocalPath : new LocalPath( path, attributes, workDir ) } - static void setClient( K8sSchedulerClient client ){ - if ( !this.client ) this.client = client - else throw new IllegalStateException("Client was already set.") - } - - static FtpClient getConnection(final String node, String daemon ){ - int trial = 0 - while ( true ) { - try { - FtpClient ftpClient = FtpClient.create(daemon) - ftpClient.login("root", "password".toCharArray() ) - ftpClient.enablePassiveMode( true ) - ftpClient.setBinaryType() - return ftpClient - } catch ( IOException e ) { - if ( trial > 5 ) throw e - log.error("Cannot create FTP client: $daemon on $node", e) - sleep(Math.pow(2, trial++) as long) - daemon = client.getDaemonOnNode(node) - } - } - } - - Map getLocation() { getLocation(path.toAbsolutePath().toString()) } - - Map getLocation( String absolutePath ){ - Map response = client.getFileLocation( absolutePath ) - synchronized ( createSymlinkHelper ) { - if ( !createdSymlinks ) { - for ( Map link : (response.symlinks as List)) { - Path src = link.src as Path - Path dst = link.dst as Path - if (Files.exists(src, LinkOption.NOFOLLOW_LINKS)) { - try { - if (src.isDirectory()) src.deleteDir() - else Files.delete(src) - } catch ( Exception ignored){ - log.warn( "Unable to delete " + src ) - } - } else { - src.parent.toFile().mkdirs() - } - try{ - Files.createSymbolicLink(src, dst) - } catch ( Exception ignored){ - log.warn( "Unable to create symlink: " + src + " -> " + dst ) - } - } - createdSymlinks = true - } - } - response - } - T asType( Class c ) { if ( c.isAssignableFrom( getClass() ) ) return (T) this if ( c.isAssignableFrom( LocalPath.class ) ) return (T) toFile() @@ -118,7 +57,6 @@ class LocalPath implements Path, Serializable { } boolean empty(){ - //TODO empty file? this.size() == 0 } @@ -188,19 +126,17 @@ class LocalPath implements Path, Serializable { @Override Path normalize() { - toLocalPath( path.normalize() ) + toLocalPath( path.normalize(), attributes ) } @Override Path resolve(Path other) { - //TODO other attributes - toLocalPath( path.resolve( other ) ) + toLocalPath( path.resolve( other ), new WOWFileAttributes( other ) ) } @Override Path resolve(String other) { - //TODO other attributes - toLocalPath( path.resolve( other ) ) + resolve( Path.of( other ) ) } @Override @@ -217,22 +153,22 @@ class LocalPath implements Path, Serializable { Path relativize(Path other) { if ( other instanceof LocalPath ){ def localPath = (LocalPath) other - return toLocalPath( path.relativize( localPath.path), (LocalFileWalker.FileAttributes) localPath.attributes, localPath.workDir ) + return toLocalPath( path.relativize( localPath.path), (WOWFileAttributes) localPath.attributes, localPath.workDir ) } path.relativize( other ) } @Override URI toUri() { - return getFileSystem().provider().getScheme() + "://" + path.toAbsolutePath() as URI + getFileSystem().provider().getScheme() + "://" + path.toAbsolutePath() as URI } String toUriString() { - return getFileSystem().provider().getScheme() + ":/" + path.toAbsolutePath() + getFileSystem().provider().getScheme() + ":/" + path.toAbsolutePath() } Path toAbsolutePath(){ - toLocalPath( path.toAbsolutePath() ) + toLocalPath( path.toAbsolutePath(), attributes ) } @Override @@ -268,7 +204,7 @@ class LocalPath implements Path, Serializable { path.toString() } - BasicFileAttributes getAttributes(){ + WOWFileAttributes getAttributes(){ attributes } @@ -285,6 +221,7 @@ class LocalPath implements Path, Serializable { @Override int hashCode() { - return path.hashCode() * 2 + 1 + path.hashCode() * 2 + 1 } + } \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/OfflineLocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/OfflineLocalPath.groovy index 44979c7..ae50514 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/OfflineLocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/OfflineLocalPath.groovy @@ -1,6 +1,7 @@ package nextflow.cws.wow.file import groovy.transform.CompileStatic + import java.nio.file.Path /** @@ -9,14 +10,9 @@ import java.nio.file.Path @CompileStatic class OfflineLocalPath extends LocalPath { - final protected WorkdirHelper workdirHelper - - private OfflineLocalPath(){ - this(null, null, null, null) - } - + final WorkdirHelper workdirHelper - OfflineLocalPath( Path path, LocalFileWalker.FileAttributes attributes, Path workDir, WorkdirHelper workdirHelper ) { + OfflineLocalPath(Path path, WOWFileAttributes attributes, Path workDir, WorkdirHelper workdirHelper ) { super(path, attributes, workDir) this.workdirHelper = workdirHelper } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy new file mode 100644 index 0000000..ec1f54c --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy @@ -0,0 +1,155 @@ +package nextflow.cws.wow.file + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.cws.wow.util.DateParser + +import java.nio.file.Path +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.attribute.FileTime + +@Slf4j +@CompileStatic +class WOWFileAttributes implements BasicFileAttributes { + + /** + * Helper to parse the file attributes returned by the getStatsAndResolveSymlinks.c + */ + static private final int FILE_EXISTS = 1 + static private final int REAL_PATH = 2 + static private final int SIZE = 3 + static private final int FILE_TYPE = 4 + static private final int CREATION_DATE = 5 + static private final int ACCESS_DATE = 6 + static private final int MODIFICATION_DATE = 7 + + private final boolean directory + + private final boolean link + + private final long size + + private final String fileType + + private final FileTime creationDate + + private final FileTime accessDate + + private final FileTime modificationDate + + private final Path destination + + private final boolean local + + WOWFileAttributes( String[] data ) { + boolean fileExists = data[ FILE_EXISTS ] == "1" + if ( fileExists && data.length != 8 ) throw new RuntimeException( "Cannot parse row (8 columns required): ${data.join(',')}" ) + destination = data.length > REAL_PATH && data[ REAL_PATH ] ? data[ REAL_PATH ] as Path : null + if ( data.length != 8 ) { + this.directory = false + this.link = true + this.size = 0 + this.fileType = null + this.creationDate = null + this.accessDate = null + this.modificationDate = null + this.local = false + return + } + this.link = !data[ REAL_PATH ].isEmpty() + this.size = data[ SIZE ] as Long + String fileType = data[ FILE_TYPE ] + this.accessDate = DateParser.fileTimeFromString(data[ ACCESS_DATE ]) + this.modificationDate = DateParser.fileTimeFromString(data[ MODIFICATION_DATE ]) + this.creationDate = DateParser.fileTimeFromString(data[ CREATION_DATE ]) ?: this.modificationDate + if ( fileType.startsWith("non-local ") ) { + this.local = false + this.fileType = fileType.substring( 10 ) + } else { + this.local = true + this.fileType = fileType + } + this.directory = fileType == 'directory' + if ( !directory && !fileType.contains( 'file' ) ){ + log.error( "Unknown type: $fileType" ) + } + } + + WOWFileAttributes(Path path ) { + if (path.isDirectory()) { + directory = true + link = false + size = 4096 + fileType = 'directory' + creationDate = FileTime.fromMillis(0) + accessDate = FileTime.fromMillis(0) + modificationDate = FileTime.fromMillis(0) + destination = path + local = false + } else { + directory = false + link = path.isLink() + size = path.size() + fileType = link ? 'symbolic link' : 'regular file' + creationDate = null + accessDate = null + modificationDate = FileTime.fromMillis(path.lastModified()) + destination = link ? path.toRealPath() : path + local = !link + } + } + + @Override + FileTime lastModifiedTime() { + modificationDate + } + + @Override + FileTime lastAccessTime() { + accessDate + } + + @Override + FileTime creationTime() { + creationDate + } + + @Override + boolean isRegularFile() { + !directory + } + + @Override + boolean isDirectory() { + directory + } + + @Override + boolean isSymbolicLink() { + link + } + + @Override + boolean isOther() { + false + } + + @Override + long size() { + size + } + + @Override + Object fileKey() { + null + } + + Path getDestination(){ + destination + } + + boolean isLocal() { + local + } + +} \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy index 8a51a46..3b3420d 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirHelper.groovy @@ -11,7 +11,9 @@ import java.nio.file.Path class WorkdirHelper { private final Map paths + private final Path rootPath + private boolean validated = false WorkdirHelper( Path rootPath, Map paths) { @@ -19,10 +21,6 @@ class WorkdirHelper { this.paths = paths } - private WorkdirHelper() { - this(null, null) - } - void validate() { validated = true } @@ -39,7 +37,7 @@ class WorkdirHelper { } Path relativeToWorkdir( LocalPath path ) { - new LocalPath( rootPath.relativize( path.path ), (LocalFileWalker.FileAttributes) path.getAttributes(), path.workDir ) + new LocalPath( rootPath.relativize( path.path ), path.getAttributes(), path.workDir ) } int getNameCount() { @@ -78,4 +76,5 @@ class WorkdirHelper { ", validated=" + validated + '}' } + } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy index 7e1822b..43541a3 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WorkdirPath.groovy @@ -12,7 +12,7 @@ import java.nio.file.Path @CompileStatic class WorkdirPath extends OfflineLocalPath { - WorkdirPath(Path path, LocalFileWalker.FileAttributes attributes, Path workDir, WorkdirHelper workdirHelper) { + WorkdirPath(Path path, WOWFileAttributes attributes, Path workDir, WorkdirHelper workdirHelper) { super(path, attributes, workDir, workdirHelper) } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystem.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystem.groovy similarity index 94% rename from plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystem.groovy rename to plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystem.groovy index f993231..502aaed 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystem.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystem.groovy @@ -1,80 +1,82 @@ -package nextflow.cws.wow.file - -import groovy.transform.CompileDynamic -import groovy.transform.CompileStatic -import java.nio.file.* -import java.nio.file.attribute.UserPrincipalLookupService -import java.nio.file.spi.FileSystemProvider - -@CompileStatic -class WOWFileSystem extends FileSystem { - - static final WOWFileSystem INSTANCE = new WOWFileSystem() - - @Override - FileSystemProvider provider() { - return WOWFileSystemProvider.INSTANCE - } - - @Override - void close() throws IOException { - } - - @Override - boolean isOpen() { - return true - } - - @Override - boolean isReadOnly() { - return false - } - - @Override - String getSeparator() { - return "/" - } - - @Override - Iterable getRootDirectories() { - throw new UnsupportedOperationException("Root directories not supported by ${provider().getScheme().toUpperCase()} file system") - } - - @Override - Iterable getFileStores() { - throw new UnsupportedOperationException("File stores not supported by ${provider().getScheme().toUpperCase()} file system") - } - - @Override - Set supportedFileAttributeViews() { - return new HashSet<>() - } - - @Override - Path getPath(String s, String... strings) { - throw new UnsupportedOperationException("Path get not supported by ${provider().getScheme().toUpperCase()} file system") - } - - @Override - @CompileDynamic - PathMatcher getPathMatcher(String s) { - return new PathMatcher() { - private final def matcher = FileSystems.getDefault().getPathMatcher( s ) - @Override - boolean matches(Path path) { - // Make this a Unix Path and use the default matcher - return matcher.matches( Path.of(path.toString()) ) - } - } - } - - @Override - UserPrincipalLookupService getUserPrincipalLookupService() { - throw new UnsupportedOperationException("User principal lookup service not supported by ${provider().getScheme().toUpperCase()} file system") - } - - @Override - WatchService newWatchService() throws IOException { - throw new UnsupportedOperationException("Watch service not supported by ${provider().getScheme().toUpperCase()} file system") - } -} +package nextflow.cws.wow.filesystem + +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic + +import java.nio.file.* +import java.nio.file.attribute.UserPrincipalLookupService +import java.nio.file.spi.FileSystemProvider + +@CompileStatic +class WOWFileSystem extends FileSystem { + + static final WOWFileSystem INSTANCE = new WOWFileSystem() + + @Override + FileSystemProvider provider() { + return WOWFileSystemProvider.INSTANCE + } + + @Override + void close() throws IOException { + } + + @Override + boolean isOpen() { + return true + } + + @Override + boolean isReadOnly() { + return false + } + + @Override + String getSeparator() { + return "/" + } + + @Override + Iterable getRootDirectories() { + throw new UnsupportedOperationException("Root directories not supported by ${provider().getScheme().toUpperCase()} file system") + } + + @Override + Iterable getFileStores() { + throw new UnsupportedOperationException("File stores not supported by ${provider().getScheme().toUpperCase()} file system") + } + + @Override + Set supportedFileAttributeViews() { + return new HashSet<>() + } + + @Override + Path getPath(String s, String... strings) { + throw new UnsupportedOperationException("Path get not supported by ${provider().getScheme().toUpperCase()} file system") + } + + @Override + @CompileDynamic + PathMatcher getPathMatcher(String s) { + return new PathMatcher() { + private final def matcher = FileSystems.getDefault().getPathMatcher( s ) + @Override + boolean matches(Path path) { + // Make this a Unix Path and use the default matcher + return matcher.matches( Path.of(path.toString()) ) + } + } + } + + @Override + UserPrincipalLookupService getUserPrincipalLookupService() { + throw new UnsupportedOperationException("User principal lookup service not supported by ${provider().getScheme().toUpperCase()} file system") + } + + @Override + WatchService newWatchService() throws IOException { + throw new UnsupportedOperationException("Watch service not supported by ${provider().getScheme().toUpperCase()} file system") + } + +} diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemPathFactory.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystemPathFactory.groovy similarity index 88% rename from plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemPathFactory.groovy rename to plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystemPathFactory.groovy index 3c508e7..fe5731e 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemPathFactory.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystemPathFactory.groovy @@ -1,39 +1,39 @@ -package nextflow.cws.wow.file - -import groovy.transform.CompileStatic -import nextflow.file.FileSystemPathFactory - -import java.nio.file.Path - -@CompileStatic -class WOWFileSystemPathFactory extends FileSystemPathFactory { - - - @Override - protected Path parseUri(String uri) { - if ( uri.startsWith("wow://") ) { - def of = Path.of(uri.substring(5)) - return LocalPath.toLocalPath(of, null, null ) - } - return null - } - - @Override - protected String toUriString(Path path) { - if ( path instanceof LocalPath ) { - return path.toUriString() - } - return null - } - - @Override - protected String getBashLib(Path target) { - null - } - - @Override - protected String getUploadCmd(String source, Path target) { - throw new UnsupportedOperationException("WOWFileSystemPathFactory does not support getUploadCmd") - } - -} +package nextflow.cws.wow.filesystem + +import groovy.transform.CompileStatic +import nextflow.cws.wow.file.LocalPath +import nextflow.file.FileSystemPathFactory + +import java.nio.file.Path + +@CompileStatic +class WOWFileSystemPathFactory extends FileSystemPathFactory { + + @Override + protected Path parseUri(String uri) { + if ( uri.startsWith("wow://") ) { + def of = Path.of(uri.substring(5)) + return LocalPath.toLocalPath(of, null, null ) + } + return null + } + + @Override + protected String toUriString(Path path) { + if ( path instanceof LocalPath ) { + return path.toUriString() + } + return null + } + + @Override + protected String getBashLib(Path target) { + null + } + + @Override + protected String getUploadCmd(String source, Path target) { + throw new UnsupportedOperationException("WOWFileSystemPathFactory does not support getUploadCmd") + } + +} diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystemProvider.groovy similarity index 68% rename from plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy rename to plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystemProvider.groovy index d817bb4..4b4fbd8 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystemProvider.groovy @@ -1,176 +1,235 @@ -package nextflow.cws.wow.file - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import nextflow.cws.SchedulerClient -import nextflow.file.FileSystemTransferAware -import sun.net.ftp.FtpClient - -import java.nio.channels.SeekableByteChannel -import java.nio.file.* -import java.nio.file.attribute.BasicFileAttributes -import java.nio.file.attribute.FileAttribute -import java.nio.file.attribute.FileAttributeView -import java.nio.file.spi.FileSystemProvider - -@Slf4j -@CompileStatic -class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTransferAware { - - static final WOWFileSystemProvider INSTANCE = new WOWFileSystemProvider() - - protected SchedulerClient schedulerClient = null // TODO: support multiple clients - - void registerSchedulerClient(SchedulerClient schedulerClient) throws UnsupportedOperationException { - if (this.schedulerClient != null) { - throw new UnsupportedOperationException("WOW file system does not support multiple scheduler clients") - } - this.schedulerClient = schedulerClient - } - - @Override - InputStream newInputStream(Path path, OpenOption... options) { - if (schedulerClient == null) { - throw new RuntimeException("WOW file system has no registered scheduler client") - } - assert path instanceof LocalPath - Map location = path.getLocation() - - if ( location?.sameAsEngine ) { - return Files.newInputStream(path.getInner(), options) - } - - FtpClient ftpClient = LocalPath.getConnection(location.node.toString(), location.daemon.toString()) - InputStream is = ftpClient.getFileStream(location.path.toString()) - return new WOWInputStream(is, schedulerClient, path, ftpClient) - } - - @Override - OutputStream newOutputStream(Path path, OpenOption... options) { - if (schedulerClient == null) { - throw new RuntimeException("WOW file system has no registered scheduler client") - } - assert path instanceof LocalPath - - OutputStream os = super.newOutputStream(path.getInner(), options) - return new WOWOutputStream(os, schedulerClient, path) - } - - @Override - String getScheme() { - return "wow" - } - - @Override - FileSystem newFileSystem(URI uri, Map map) throws IOException { - return getFileSystem(uri) - } - - @Override - FileSystem getFileSystem(URI uri) { - return WOWFileSystem.INSTANCE - } - - @Override - Path getPath(URI uri) { - return getFileSystem(uri).getPath(uri.path) - } - - @Override - SeekableByteChannel newByteChannel(Path path, Set options, FileAttribute... attrs) throws IOException { - throw new UnsupportedOperationException("New byte channel not supported by ${getScheme().toUpperCase()} file system provider") - } - - @Override - DirectoryStream newDirectoryStream(Path path, DirectoryStream.Filter filter) throws IOException { - if (path instanceof OfflineLocalPath && !((OfflineLocalPath) path).workdirHelper.isValidated()) { - return ((OfflineLocalPath) path).workdirHelper.getDirectoryStream(path) - } - - throw new UnsupportedOperationException("Directory stream not supported by ${getScheme().toUpperCase()} file system provider") - } - - @Override - void createDirectory(Path path, FileAttribute... fileAttributes) throws IOException { - throw new UnsupportedOperationException("Create directory not supported by ${getScheme().toUpperCase()} file system provider") - } - - @Override - void delete(Path path) throws IOException { - throw new UnsupportedOperationException("Delete not supported by ${getScheme().toUpperCase()} file system provider") - } - - @Override - void copy(Path path, Path path1, CopyOption... copyOptions) throws IOException { - throw new UnsupportedOperationException("Copy not supported by ${getScheme().toUpperCase()} file system provider") - } - - @Override - void move(Path path, Path path1, CopyOption... copyOptions) throws IOException { - throw new UnsupportedOperationException("Move not supported by ${getScheme().toUpperCase()} file system provider") - } - - @Override - boolean isSameFile(Path path1, Path path2) throws IOException { - return path1 == path2 - } - - @Override - boolean isHidden(Path path) throws IOException { - return path.getFileName().startsWith(".") - } - - @Override - FileStore getFileStore(Path path) throws IOException { - throw new UnsupportedOperationException("File store not supported by ${getScheme().toUpperCase()} file system provider") - } - - @Override - void checkAccess(Path path, AccessMode... accessModes) throws IOException { - // TODO: re-check that this may be empty - } - - @Override - V getFileAttributeView(Path path, Class aClass, LinkOption... linkOptions) { - throw new UnsupportedOperationException("File attribute view not supported by ${getScheme().toUpperCase()} file system provider") - } - - @Override - A readAttributes(Path path, Class aClass, LinkOption... linkOptions) throws IOException { - if (path instanceof LocalPath) { - return path.getAttributes() as A - } else { - return Files.readAttributes(path, BasicFileAttributes.class, linkOptions) as A - } - } - - @Override - Map readAttributes(Path path, String s, LinkOption... linkOptions) throws IOException { - throw new UnsupportedOperationException("Read attributes not supported by ${getScheme().toUpperCase()} file system provider") - } - - @Override - void setAttribute(Path path, String s, Object o, LinkOption... linkOptions) throws IOException { - throw new UnsupportedOperationException("Set attribute not supported by ${getScheme().toUpperCase()} file system provider") - } - - @Override - boolean canUpload(Path source, Path target) { - return false - } - - @Override - boolean canDownload(Path source, Path target) { - return true - } - - @Override - void download(Path source, Path target, CopyOption... copyOptions) throws IOException { - log.warn( "Not Implemented: Download from ${source} (${source.class.name}) to ${target} (${target.class.name}) with options ${copyOptions}" ) - } - - @Override - void upload(Path source, Path target, CopyOption... copyOptions) throws IOException { - throw new UnsupportedOperationException("Uploading not supported by ${getScheme().toUpperCase()} file system provider") - } -} +package nextflow.cws.wow.filesystem + +import groovy.transform.CompileStatic +import groovy.transform.PackageScope +import groovy.util.logging.Slf4j +import nextflow.cws.SchedulerClient +import nextflow.cws.wow.file.LocalPath +import nextflow.cws.wow.file.OfflineLocalPath +import nextflow.file.FileSystemTransferAware +import sun.net.ftp.FtpClient + +import java.nio.channels.SeekableByteChannel +import java.nio.file.* +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.attribute.FileAttribute +import java.nio.file.attribute.FileAttributeView +import java.nio.file.spi.FileSystemProvider + +@Slf4j +@CompileStatic +class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTransferAware { + + static final WOWFileSystemProvider INSTANCE = new WOWFileSystemProvider() + + protected SchedulerClient schedulerClient = null + + private transient final Object createSymlinkHelper = new Object() + + void registerSchedulerClient(SchedulerClient schedulerClient) throws UnsupportedOperationException { + if (this.schedulerClient != null) { + throw new UnsupportedOperationException("WOW file system does not support multiple scheduler clients") + } + this.schedulerClient = schedulerClient + } + + private FtpClient getConnection(final String node, String daemon ){ + int trial = 0 + while ( true ) { + try { + FtpClient ftpClient = FtpClient.create(daemon) + ftpClient.login("root", "password".toCharArray() ) + ftpClient.enablePassiveMode( true ) + ftpClient.setBinaryType() + return ftpClient + } catch ( IOException e ) { + if ( trial > 5 ) throw e + log.error("Cannot create FTP client: $daemon on $node", e) + sleep(Math.pow(2, trial++) as long) + daemon = schedulerClient.getDaemonOnNode(node) + } + } + } + + @PackageScope Map getLocation(LocalPath path ){ + String absolutePath = path.toAbsolutePath().toString() + Map response = schedulerClient.getFileLocation( absolutePath ) + synchronized ( createSymlinkHelper ) { + if ( !path.createdSymlinks ) { + for (Map link : (response.symlinks as List)) { + Path src = link.src as Path + Path dst = link.dst as Path + if (Files.exists(src, LinkOption.NOFOLLOW_LINKS)) { + try { + if (src.isDirectory()) src.deleteDir() + else Files.delete(src) + } catch (Exception ignored) { + log.warn("Unable to delete " + src) + } + } else { + src.parent.toFile().mkdirs() + } + try { + Files.createSymbolicLink(src, dst) + } catch (Exception ignored) { + log.warn("Unable to create symlink: " + src + " -> " + dst) + } + } + path.createdSymlinks = true + } + } + response + } + + @Override + InputStream newInputStream(Path path, OpenOption... options) { + if (schedulerClient == null) { + throw new RuntimeException("WOW file system has no registered scheduler client") + } + assert path instanceof LocalPath + Map location = getLocation( path ) + + if ( location?.sameAsEngine ) { + return Files.newInputStream(path.getInner(), options) + } + + FtpClient ftpClient = getConnection(location.node.toString(), location.daemon.toString()) + InputStream is = ftpClient.getFileStream(location.path.toString()) + return new WOWInputStream(is, schedulerClient, path, ftpClient) + } + + @Override + OutputStream newOutputStream(Path path, OpenOption... options) { + if (schedulerClient == null) { + throw new RuntimeException("WOW file system has no registered scheduler client") + } + assert path instanceof LocalPath + + OutputStream os = super.newOutputStream(path.getInner(), options) + return new WOWOutputStream(os, schedulerClient, path) + } + + @Override + String getScheme() { + return "wow" + } + + @Override + FileSystem newFileSystem(URI uri, Map map) throws IOException { + return getFileSystem(uri) + } + + @Override + FileSystem getFileSystem(URI uri) { + return WOWFileSystem.INSTANCE + } + + @Override + Path getPath(URI uri) { + return getFileSystem(uri).getPath(uri.path) + } + + @Override + SeekableByteChannel newByteChannel(Path path, Set options, FileAttribute... attrs) throws IOException { + throw new UnsupportedOperationException("New byte channel not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + DirectoryStream newDirectoryStream(Path path, DirectoryStream.Filter filter) throws IOException { + if (path instanceof OfflineLocalPath && !((OfflineLocalPath) path).workdirHelper.isValidated()) { + return ((OfflineLocalPath) path).workdirHelper.getDirectoryStream(path) + } + + throw new UnsupportedOperationException("Directory stream not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + void createDirectory(Path path, FileAttribute... fileAttributes) throws IOException { + throw new UnsupportedOperationException("Create directory not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + void delete(Path path) throws IOException { + throw new UnsupportedOperationException("Delete not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + void copy(Path path, Path path1, CopyOption... copyOptions) throws IOException { + throw new UnsupportedOperationException("Copy not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + void move(Path path, Path path1, CopyOption... copyOptions) throws IOException { + throw new UnsupportedOperationException("Move not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + boolean isSameFile(Path path1, Path path2) throws IOException { + return path1 == path2 + } + + @Override + boolean isHidden(Path path) throws IOException { + return path.getFileName().startsWith(".") + } + + @Override + FileStore getFileStore(Path path) throws IOException { + throw new UnsupportedOperationException("File store not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + void checkAccess(Path path, AccessMode... accessModes) throws IOException { + // all access is allowed + } + + @Override + V getFileAttributeView(Path path, Class aClass, LinkOption... linkOptions) { + throw new UnsupportedOperationException("File attribute view not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + A readAttributes(Path path, Class aClass, LinkOption... linkOptions) throws IOException { + if (path instanceof LocalPath) { + return path.getAttributes() as A + } else { + return Files.readAttributes(path, BasicFileAttributes.class, linkOptions) as A + } + } + + @Override + Map readAttributes(Path path, String s, LinkOption... linkOptions) throws IOException { + throw new UnsupportedOperationException("Read attributes not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + void setAttribute(Path path, String s, Object o, LinkOption... linkOptions) throws IOException { + throw new UnsupportedOperationException("Set attribute not supported by ${getScheme().toUpperCase()} file system provider") + } + + @Override + boolean canUpload(Path source, Path target) { + return false + } + + @Override + boolean canDownload(Path source, Path target) { + return true + } + + @Override + void download(Path source, Path target, CopyOption... copyOptions) throws IOException { + try { + schedulerClient.publish( source, target, "COPY" ) + } catch ( Exception e ) { + log.error("Error downloading file from ${source} to ${target}", e) + throw e + } + } + + @Override + void upload(Path source, Path target, CopyOption... copyOptions) throws IOException { + throw new UnsupportedOperationException("Uploading not supported by ${getScheme().toUpperCase()} file system provider") + } + +} diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWInputStream.groovy similarity index 92% rename from plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy rename to plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWInputStream.groovy index 1cef9af..6433da5 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWInputStream.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWInputStream.groovy @@ -1,7 +1,8 @@ -package nextflow.cws.wow.file +package nextflow.cws.wow.filesystem import groovy.transform.CompileStatic import nextflow.cws.SchedulerClient +import nextflow.cws.wow.file.LocalPath import sun.net.ftp.FtpClient @CompileStatic @@ -40,7 +41,7 @@ class WOWInputStream extends InputStream { temporaryFile.moveTo(file) transferredTemporaryFile = true - Map location = path.getLocation() + Map location = WOWFileSystemProvider.INSTANCE.getLocation( path ) schedulerClient.addFileLocation(path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, false) } @@ -68,4 +69,5 @@ class WOWInputStream extends InputStream { ftpClient.close() checkTemporaryFileTransferal() } + } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWOutputStream.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWOutputStream.groovy similarity index 83% rename from plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWOutputStream.groovy rename to plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWOutputStream.groovy index d4a8c6f..56e7e8d 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWOutputStream.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWOutputStream.groovy @@ -1,13 +1,16 @@ -package nextflow.cws.wow.file +package nextflow.cws.wow.filesystem import groovy.transform.CompileStatic import nextflow.cws.SchedulerClient +import nextflow.cws.wow.file.LocalPath @CompileStatic class WOWOutputStream extends OutputStream { private OutputStream inner + private SchedulerClient client + private LocalPath path WOWOutputStream(OutputStream inner, SchedulerClient client, LocalPath path) { @@ -25,8 +28,9 @@ class WOWOutputStream extends OutputStream { @Override void close() throws IOException { inner.close() - Map location = path.getLocation() + Map location = WOWFileSystemProvider.INSTANCE.getLocation( path ) File file = path.getInner().toFile() client.addFileLocation(path.toString(), file.size(), file.lastModified(), location.locationWrapperID as long, true) } + } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/serializer/LocalPathSerializer.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/serializer/LocalPathSerializer.groovy new file mode 100644 index 0000000..e06efb5 --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/serializer/LocalPathSerializer.groovy @@ -0,0 +1,32 @@ +package nextflow.cws.wow.serializer + +import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.Serializer +import com.esotericsoftware.kryo.io.Input +import com.esotericsoftware.kryo.io.Output +import groovy.transform.CompileStatic +import nextflow.cws.wow.file.LocalPath + +@CompileStatic +class LocalPathSerializer extends Serializer { + + private static final Map storage = [:] + + @Override + void write(Kryo kryo, Output output, LocalPath object) { + def content = object.getInner().toString() + synchronized (storage) { + storage.put(content, object) + } + output.writeString( content ) + } + + @Override + LocalPath read(Kryo kryo, Input input, Class type) { + def content = input.readString() + synchronized (storage) { + return storage.get(content) + } + } + +} diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/util/DateParser.groovy similarity index 86% rename from plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy rename to plugins/nf-cws/src/main/nextflow/cws/wow/util/DateParser.groovy index 8a9e5d2..672bc86 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/DateParser.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/util/DateParser.groovy @@ -1,4 +1,4 @@ -package nextflow.cws.wow.file +package nextflow.cws.wow.util import groovy.transform.CompileStatic @@ -32,7 +32,8 @@ final class DateParser { } static FileTime fileTimeFromString( String date ) { - final Long millisFromSring = millisFromString( date ) - return millisFromSring == null ? null : FileTime.fromMillis( millisFromSring ) + final Long millisFromString = millisFromString( date ) + return millisFromString == null ? null : FileTime.fromMillis( millisFromString ) } + } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/util/LocalFileWalker.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/util/LocalFileWalker.groovy new file mode 100644 index 0000000..4011bdf --- /dev/null +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/util/LocalFileWalker.groovy @@ -0,0 +1,55 @@ +package nextflow.cws.wow.util + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.cws.wow.file.LocalPath +import nextflow.cws.wow.file.OfflineLocalPath +import nextflow.cws.wow.file.WOWFileAttributes +import nextflow.cws.wow.file.WorkdirHelper + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +@Slf4j +@CompileStatic +class LocalFileWalker { + + static private final int VIRTUAL_PATH = 0 + + static WorkdirHelper createWorkdirHelper(Path start){ + Map files = new HashMap<>() + Path rootPath = null + File file = new File( start.toString() + File.separatorChar + ".command.outfiles" ) + if ( !file.exists() ) { + log.warn( "File ${file} does not exist" ) + return null + } + String line + WorkdirHelper workdirHelper + file.withReader { reader -> + line = reader.readLine() + rootPath = Paths.get(line.split(';')[ VIRTUAL_PATH ]) + workdirHelper = new WorkdirHelper( rootPath, files ) + while ((line = reader.readLine()) != null) { + String[] data = line.split(';') + String path = data[ VIRTUAL_PATH ] + WOWFileAttributes attributes = new WOWFileAttributes( data ) + Path currentPath = Paths.get(path) + if ( !attributes.local ) { + //If task did not run on local machine, create symbolic link + if ( !Files.isSymbolicLink( currentPath ) ) { + Files.createDirectories( currentPath.getParent() ) + Files.createSymbolicLink( currentPath, attributes.destination ) + } + } else { + def localPath = new OfflineLocalPath(currentPath, attributes, start, workdirHelper) + def pathOnSharedFs = start.resolve(rootPath.relativize(currentPath)) + files.put( pathOnSharedFs, localPath ) + } + } + return workdirHelper + } + } + +} \ No newline at end of file diff --git a/plugins/nf-cws/src/resources/META-INF/extensions.idx b/plugins/nf-cws/src/resources/META-INF/extensions.idx index e48d4d8..557340d 100644 --- a/plugins/nf-cws/src/resources/META-INF/extensions.idx +++ b/plugins/nf-cws/src/resources/META-INF/extensions.idx @@ -1,3 +1,3 @@ nextflow.cws.CWSFactory nextflow.cws.k8s.CWSK8sExecutor -nextflow.cws.wow.file.WOWFileSystemPathFactory \ No newline at end of file +nextflow.cws.wow.filesystem.WOWFileSystemPathFactory \ No newline at end of file diff --git a/plugins/nf-cws/src/test/nextflow/cws/wow/file/WOWFileSystemTest.groovy b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WOWFileSystemTest.groovy index 182b2ba..8acc2a3 100644 --- a/plugins/nf-cws/src/test/nextflow/cws/wow/file/WOWFileSystemTest.groovy +++ b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WOWFileSystemTest.groovy @@ -1,5 +1,6 @@ package nextflow.cws.wow.file +import nextflow.cws.wow.filesystem.WOWFileSystem import spock.lang.Specification import java.nio.file.Path diff --git a/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirHelperTest.groovy b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirHelperTest.groovy index ebcd5aa..a9a403c 100644 --- a/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirHelperTest.groovy +++ b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirHelperTest.groovy @@ -1,5 +1,6 @@ package nextflow.cws.wow.file + import spock.lang.Specification import java.nio.file.Path @@ -33,7 +34,7 @@ class WorkdirHelperTest extends Specification { files.put( sharedPath7, localPath7 ) def helper = new WorkdirHelper( Path.of(localPath), files ) def workDir = Path.of("/input/data/work/be/8292aaebea2ddf9ae8ad4952882dcb") - def attributes = new LocalFileWalker.FileAttributes(workDir) + def attributes = new WOWFileAttributes(workDir) WorkdirPath path = new WorkdirPath( workDir, attributes, workDir, helper ) def stream = helper.getDirectoryStream( path ) def result = stream.collect() diff --git a/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirPathTest.groovy b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirPathTest.groovy index 81124d9..13abf10 100644 --- a/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirPathTest.groovy +++ b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirPathTest.groovy @@ -1,5 +1,6 @@ package nextflow.cws.wow.file + import spock.lang.Specification import java.nio.file.Path @@ -21,7 +22,7 @@ class WorkdirPathTest extends Specification { Map files = new HashMap<>() files.put(workDir.resolve("file.txt"), localPath) def helper = new WorkdirHelper( Path.of("/localdata/localwork/be/8292aaebea2ddf9ae8ad4952882dcb/"), files ) - def attributes = new LocalFileWalker.FileAttributes(workDir) + def attributes = new WOWFileAttributes(workDir) WorkdirPath path = new WorkdirPath( workDir, attributes, workDir, helper ) then: From baa95ce11b8240d71fd2a3ede5ba598b1ac13ae1 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Tue, 13 May 2025 11:12:48 +0200 Subject: [PATCH 44/63] WOW scheduler README (#8) --- README.md | 92 ++++++++++++++++++- .../src/main/nextflow/cws/CWSPlugin.groovy | 1 - 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ceca44a..b8baca5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,11 @@ -# nf-cws plugin +# nf-cws plugin -This plugin enables Nextflow to communicate with a Common Workflow Scheduler instance and transfer the required information. +This plugin enables Nextflow to communicate with a Common Workflow Scheduler instance and transfer the required +information. + +For more information on the scheduling, +see the [scheduler repository](https://github.com/CommonWorkflowScheduler/KubernetesScheduler) and the [respective +papers](#citation): ### Supported Executors @@ -10,6 +15,7 @@ This plugin enables Nextflow to communicate with a Common Workflow Scheduler ins To run Nextflow with this plugin, you need version >=`23.03.0-edge`. To activate the plugin, add `-plugins nf-cws` to your `nextflow` call or add the following to your `nextflow.config`: + ``` plugins { id 'nf-cws' @@ -28,7 +34,8 @@ plugins { | minMemory | - | The minimum memory to size a task to. Only used if memory prediction is performed. | | maxMemory | - | The maximum memory to size a task to. Only used if memory prediction is performed. | -##### Example: +##### Example: + ``` cws { dns = 'http://cws-scheduler/' @@ -43,7 +50,8 @@ cws { #### K8s Executor -The `k8s` executor allows starting a Common Workflow Scheduler instance on demand. This will happen if you do not define any CWS-related config. Otherwise, you can configure the following: +The `k8s` executor allows starting a Common Workflow Scheduler instance on demand. This will happen if you do not define +any CWS-related config. Otherwise, you can configure the following: ``` k8s { @@ -79,8 +87,37 @@ k8s { | autoClose | - | Stop the pod after the workflow is finished | | nodeSelector | - | A node selector for the CWS pod | +#### WOW + +WOW is a new scheduling approach for dynamic scientific workflow systems that steers both data movement and task +scheduling to reduce network congestion and overall runtime. + +WOW requires some additional configuration due to its use of the local file system in addition to the distributed file +system. + +``` +k8s { + localPath = '/localdata' + localStorageMountPath = '/localdata' + storage { + copyStrategy = 'ftp' + workdir = '/localdata/localwork/' + } +} +``` + +| Attribute | Required | Explanation | +|:----------------------|----------|-------------------------------------------------------------------------------------------------| +| localPath | yes | Host path for the local mount +| localStorageMountPath | no | Container path for the local mount +| storage.copyStrategy | no | Strategy to copy the files between nodes - currently only supports 'ftp' (and its alias 'copy') +| storage.workdir | no | Working directory to use - must be inside of the locally mounted directory + ### Tracing -This plugin adds additional fields to the trace report. Therefore, you have to add the required fields to the `trace.fields` field in your Nextflow config (also check the official [documentation](https://www.nextflow.io/docs/latest/tracing.html#trace-report)). + +This plugin adds additional fields to the trace report. Therefore, you have to add the required fields to +the `trace.fields` field in your Nextflow config (also check the +official [documentation](https://www.nextflow.io/docs/latest/tracing.html#trace-report)). The following fields can be used: | Name | Description | @@ -106,3 +143,48 @@ The following fields can be used: | scheduler_delta_submitted_batch_end | Time delta between a task was submitted, and the batch became schedulable | | memory_adapted | The memory used for a task when sizing is active | | input_size | The sum of the input size of all task inputs | +| infiles_time: | (WOW) Time to walk through and retrieve stats of all local (input) files at task start | +| outfiles_time: | (WOW) Time to walk through and retrieve stats of all local (output) files at task start | +| scheduler_time_delta_phase_three: | (WOW) List of time instances taken to calculcate step 3 of the WOW scheduling algorithm (see paper for details) | +| scheduler_copy_tasks: | (WOW) Number of times copy tasks were started for this task | + +--- + +## Citation + +If you use this software or artifacts in a publication, please cite it as: + +#### Text + +Lehmann Fabian, Jonathan Bader, Friedrich Tschirpke, Lauritz Thamsen, and Ulf Leser. **How Workflow Engines Should Talk +to Resource Managers: A Proposal for a Common Workflow Scheduling Interface**. In 2023 IEEE/ACM 23rd International +Symposium on Cluster, Cloud and Internet Computing (CCGrid). Bangalore, India, 2023. + +([https://arxiv.org/pdf/2302.07652.pdf](https://arxiv.org/pdf/2302.07652.pdf)) + +#### BibTeX + +``` +@inproceedings{lehmannHowWorkflowEngines2023, + author = {Lehmann, Fabian and Bader, Jonathan and Tschirpke, Friedrich and Thamsen, Lauritz and Leser, Ulf}, + booktitle = {2023 IEEE/ACM 23rd International Symposium on Cluster, Cloud and Internet Computing (CCGrid)}, + title = {How Workflow Engines Should Talk to Resource Managers: A Proposal for a Common Workflow Scheduling Interface}, + year = {2023}, + address = {{Bangalore, India}}, + doi = {10.1109/CCGrid57682.2023.00025} +} +``` + +#### Strategy-specific Citation + +Please note that the following strategies originated in individual papers: + +- PONDER: [https://arxiv.org/pdf/2408.00047.pdf](https://arxiv.org/pdf/2408.00047.pdf) +- WOW: [https://arxiv.org/pdf/2503.13072.pdf](https://arxiv.org/pdf/2503.13072.pdf) + +--- + +#### Acknowledgement: + +This work was funded by the German Research Foundation (DFG), CRC 1404: "FONDA: Foundations of Workflows for Large-Scale +Scientific Data Analysis." \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy b/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy index 2b0ed98..bc90ae9 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/CWSPlugin.groovy @@ -44,7 +44,6 @@ class CWSPlugin extends BasePlugin { scheduler_delta_submitted_batch_end: 'num', memory_adapted: 'mem', input_size: 'num', - out_label: 'str', scheduler_files_bytes: 'num', scheduler_files_node_bytes: 'num', scheduler_files_node_other_task_bytes: 'num', From 6550209a62befaeb37354c982a66bd9650739199 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 14 May 2025 16:43:17 +0200 Subject: [PATCH 45/63] Transfer workdir and localworkdir to scheduler Signed-off-by: Lehmann_Fabian --- plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy index fc856dd..a110a00 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy @@ -138,7 +138,8 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { maxMemory : cwsConfig.getMaxMemory()?.toBytes(), minMemory : cwsConfig.getMinMemory()?.toBytes(), additional : k8sSchedulerConfig.getAdditional(), - workDir : storage?.getWorkdir(), + workDir : session.workDir as String, + localWorkDir : storage?.getWorkdir(), copyStrategy : storage?.getCopyStrategy(), locationAware : cwsK8sConfig.locationAwareScheduling(), maxCopyTasksPerNode : cwsConfig.getMaxCopyTasksPerNode(), From c059b37331885a30c6eac9e0a6bb4e7af7ec3926 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 14 May 2025 22:39:29 +0200 Subject: [PATCH 46/63] Trigger publish all remaining data when workflow has finished Signed-off-by: Lehmann_Fabian --- .../main/nextflow/cws/SchedulerClient.groovy | 565 +++++++++--------- .../nextflow/cws/k8s/CWSK8sExecutor.groovy | 2 + 2 files changed, 289 insertions(+), 278 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy b/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy index 7be4055..46151fb 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/SchedulerClient.groovy @@ -1,279 +1,288 @@ -package nextflow.cws - -import groovy.json.JsonOutput -import groovy.json.JsonSlurper -import groovy.transform.CompileDynamic -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import nextflow.cws.wow.filesystem.WOWFileSystemProvider -import nextflow.dag.DAG - -import java.nio.file.Path - -@Slf4j -@CompileStatic -class SchedulerClient { - - private final CWSConfig config - - private final String runName - - private boolean registered = false - - private boolean closed = false - - private int tasksInBatch = 0 - - protected String dns - - SchedulerClient( CWSConfig config, String runName ) { - this.config = config - this.runName = runName - this.dns = config.dns?.endsWith('/') ? config.dns[0..-2] : config.dns - CWSSession.INSTANCE.addSchedulerClient( this ) - WOWFileSystemProvider.INSTANCE.registerSchedulerClient( this ) - } - - protected String getDNS() { - return dns ? dns + "/v1" : null - } - - synchronized void registerScheduler( Map data ) { - if ( registered ) return - String url = "${getDNS()}/scheduler/$runName" - registered = true - int trials = 0 - while ( trials++ < 50 ) { - try { - HttpURLConnection post = URI.create(url).toURL().openConnection() as HttpURLConnection - post.setRequestMethod( "POST" ) - post.setDoOutput(true) - post.setRequestProperty("Content-Type", "application/json") - data.strategy = config.getStrategy() - String message = JsonOutput.toJson( data ) - post.getOutputStream().write(message.getBytes("UTF-8")) - int responseCode = post.getResponseCode() - if( responseCode != 200 ){ - throw new IllegalStateException( "Got code: ${responseCode} from k8s scheduler while registering" ) - } - return - } catch ( UnknownHostException ignored ) { - throw new IllegalArgumentException("The scheduler was not found under '$url', is the url correct and the scheduler running?") - } catch ( ConnectException ignored ) { - Thread.sleep( 3000 ) - }catch (IOException e) { - throw new IllegalStateException("Cannot register scheduler under $url, got ${e.class.toString()}: ${e.getMessage()}", e) - } - } - throw new IllegalStateException("Cannot connect to scheduler under $url" ) - } - - synchronized void closeScheduler(){ - if ( closed ) return - closed = true - HttpURLConnection post = URI.create("${getDNS()}/scheduler/$runName").toURL().openConnection() as HttpURLConnection - post.setRequestMethod( "DELETE" ) - int responseCode = post.getResponseCode() - log.trace "Delete scheduler code was: ${responseCode}" - } - - void submitMetrics( Map metrics, int id ){ - HttpURLConnection post = URI.create("${getDNS()}/scheduler/$runName/metrics/task/$id").toURL().openConnection() as HttpURLConnection - post.setRequestMethod( "POST" ) - String message = JsonOutput.toJson( metrics ) - post.setDoOutput(true) - post.setRequestProperty("Content-Type", "application/json") - post.getOutputStream().write(message.getBytes("UTF-8")) - int responseCode = post.getResponseCode() - if( responseCode != 200 ){ - throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while submitting metrics" ) - } - } - - Map registerTask( Map config, int id ){ - - HttpURLConnection post = URI.create("${getDNS()}/scheduler/$runName/task/$id").toURL().openConnection() as HttpURLConnection - post.setRequestMethod( "POST" ) - String message = JsonOutput.toJson( config ) - post.setDoOutput(true) - post.setRequestProperty("Content-Type", "application/json") - post.getOutputStream().write(message.getBytes("UTF-8")) - int responseCode = post.getResponseCode() - if( responseCode != 200 ){ - throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while registering task: ${config.name}" ) - } - tasksInBatch++ - Map response = new JsonSlurper().parse(post.getInputStream()) as Map - return response - - } - - private void batch( String command ){ - HttpURLConnection put = URI.create("${getDNS()}/scheduler/$runName/${command}Batch").toURL().openConnection() as HttpURLConnection - put.setRequestMethod( "PUT" ) - if ( command == 'end' ){ - put.setDoOutput(true) - put.setRequestProperty("Content-Type", "application/json") - put.getOutputStream().write("$tasksInBatch".getBytes("UTF-8")) - } - int responseCode = put.getResponseCode() - if( responseCode != 200 ){ - throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while ${command}ing batch" ) - } - } - - void startBatch(){ - tasksInBatch = 0 - if ( !closed ) batch('start') - } - - void endBatch(){ - if ( !closed ) batch('end') - } - - Map getTaskState( int id ){ - - HttpURLConnection get = URI.create("${getDNS()}/scheduler/$runName/task/$id").toURL().openConnection() as HttpURLConnection - get.setRequestMethod( "GET" ) - get.setDoOutput(true) - int responseCode = get.getResponseCode() - if( responseCode != 200 ){ - throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while requesting task state: $id" ) - } - Map response = new JsonSlurper().parse(get.getInputStream()) as Map - return response - - } - - ///* DAG */ - - @CompileDynamic - void submitVertices( List vertices ){ - List> verticesToSubmit = vertices.collect { - [ - label : it.label, - type : it.type.toString(), - uid : it.getId() - ] as Map - } - HttpURLConnection put = URI.create("${getDNS()}/scheduler/$runName/DAG/vertices").toURL().openConnection() as HttpURLConnection - put.setRequestMethod( "POST" ) - String message = JsonOutput.toJson( verticesToSubmit ) - put.setDoOutput(true) - put.setRequestProperty("Content-Type", "application/json") - put.getOutputStream().write(message.getBytes("UTF-8")) - int responseCode = put.getResponseCode() - if( responseCode != 200 ){ - throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while submitting vertices: ${vertices}" ) - } - } - - @CompileDynamic - void submitEdges( List edges ){ - List> edgesToSubmit = edges.collect { - [ - uid : it.getId(), - label : it.getLabel(), - from : it.getFrom()?.getId(), - to : it.getTo()?.getId() - ] as Map - } - HttpURLConnection put = URI.create("${getDNS()}/scheduler/$runName/DAG/edges").toURL().openConnection() as HttpURLConnection - put.setRequestMethod( "POST" ) - String message = JsonOutput.toJson( edgesToSubmit ) - put.setDoOutput(true) - put.setRequestProperty("Content-Type", "application/json") - put.getOutputStream().write(message.getBytes("UTF-8")) - int responseCode = put.getResponseCode() - if( responseCode != 200 ){ - throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while submitting edges: ${edges}" ) - } - } - - ///* File location */ - - Map getFileLocation( String path ){ - - String pathEncoded = URLEncoder.encode(path,'utf-8') - HttpURLConnection get = URI.create("${getDNS()}/file/$runName?path=$pathEncoded").toURL().openConnection() as HttpURLConnection - get.setRequestMethod( "GET" ) - get.setDoOutput(true) - int responseCode = get.getResponseCode() - if( responseCode != 200 ){ - throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while requesting file location: $path (${get.responseMessage})" ) - } - Map response = new JsonSlurper().parse(get.getInputStream()) as Map - return response - - } - - String getDaemonOnNode( String node ){ - - HttpURLConnection get = URI.create("${getDNS()}/daemon/$runName/$node").toURL().openConnection() as HttpURLConnection - get.setRequestMethod( "GET" ) - get.setDoOutput(true) - int responseCode = get.getResponseCode() - if( responseCode != 200 ){ - throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while requesting daemon on node: $node" ) - } - String response = new JsonSlurper().parse(get.getInputStream()) as String - return response - - } - - void addFileLocation( String path, long size, long timestamp, long locationWrapperID, boolean overwrite, String node = null ){ - - String method = overwrite ? 'overwrite' : 'add' - - HttpURLConnection get = URI.create("${getDNS()}/file/$runName/location/${method}${ node ? "/$node" : ''}").toURL().openConnection() as HttpURLConnection - get.setRequestMethod( "POST" ) - get.setDoOutput(true) - Map data = [ - path : path, - size : size, - timestamp : timestamp, - locationWrapperID : locationWrapperID - ] - if ( node ){ - data.node = node - } - String message = JsonOutput.toJson( data ) - get.setRequestProperty("Content-Type", "application/json") - get.getOutputStream().write(message.getBytes("UTF-8")) - int responseCode = get.getResponseCode() - if( responseCode != 200 ){ - throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while updating file location: $path: $node (${get.responseMessage})" ) - } - - } - - void publish(Path source, Path destination, String mode ) { - HttpURLConnection get = URI.create("${getDNS()}/file/$runName/publish").toURL().openConnection() as HttpURLConnection - get.setRequestMethod( "PUT" ) - get.setDoOutput(true) - Map data = [ - 'source' : source.toString(), - 'destination' : destination.toString(), - 'mode' : mode.toUpperCase(), - ] - String message = JsonOutput.toJson( data ) - get.setRequestProperty("Content-Type", "application/json") - get.getOutputStream().write(message.getBytes("UTF-8")) - int responseCode = get.getResponseCode() - if( responseCode != 200 ){ - throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while publishing file: $source (${get.responseMessage}) -- ${get.getURL()}" ) - } - } - - int getRemainingToPublish() { - HttpURLConnection get = URI.create("${getDNS()}/file/$runName/publish").toURL().openConnection() as HttpURLConnection - get.setRequestMethod( "GET" ) - get.setRequestProperty("Content-Type", "application/json") - int responseCode = get.getResponseCode() - if( responseCode != 200 ){ - throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while getting remaining to publish (${get.responseMessage})" ) - } - get.getInputStream().text as int - } - +package nextflow.cws + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.cws.wow.filesystem.WOWFileSystemProvider +import nextflow.dag.DAG + +import java.nio.file.Path + +@Slf4j +@CompileStatic +class SchedulerClient { + + private final CWSConfig config + + private final String runName + + private boolean registered = false + + private boolean closed = false + + private int tasksInBatch = 0 + + protected String dns + + SchedulerClient( CWSConfig config, String runName ) { + this.config = config + this.runName = runName + this.dns = config.dns?.endsWith('/') ? config.dns[0..-2] : config.dns + CWSSession.INSTANCE.addSchedulerClient( this ) + WOWFileSystemProvider.INSTANCE.registerSchedulerClient( this ) + } + + protected String getDNS() { + return dns ? dns + "/v1" : null + } + + synchronized void registerScheduler( Map data ) { + if ( registered ) return + String url = "${getDNS()}/scheduler/$runName" + registered = true + int trials = 0 + while ( trials++ < 50 ) { + try { + HttpURLConnection post = URI.create(url).toURL().openConnection() as HttpURLConnection + post.setRequestMethod( "POST" ) + post.setDoOutput(true) + post.setRequestProperty("Content-Type", "application/json") + data.strategy = config.getStrategy() + String message = JsonOutput.toJson( data ) + post.getOutputStream().write(message.getBytes("UTF-8")) + int responseCode = post.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from k8s scheduler while registering" ) + } + return + } catch ( UnknownHostException ignored ) { + throw new IllegalArgumentException("The scheduler was not found under '$url', is the url correct and the scheduler running?") + } catch ( ConnectException ignored ) { + Thread.sleep( 3000 ) + }catch (IOException e) { + throw new IllegalStateException("Cannot register scheduler under $url, got ${e.class.toString()}: ${e.getMessage()}", e) + } + } + throw new IllegalStateException("Cannot connect to scheduler under $url" ) + } + + synchronized void closeScheduler(){ + if ( closed ) return + closed = true + HttpURLConnection post = URI.create("${getDNS()}/scheduler/$runName").toURL().openConnection() as HttpURLConnection + post.setRequestMethod( "DELETE" ) + int responseCode = post.getResponseCode() + log.trace "Delete scheduler code was: ${responseCode}" + } + + void submitMetrics( Map metrics, int id ){ + HttpURLConnection post = URI.create("${getDNS()}/scheduler/$runName/metrics/task/$id").toURL().openConnection() as HttpURLConnection + post.setRequestMethod( "POST" ) + String message = JsonOutput.toJson( metrics ) + post.setDoOutput(true) + post.setRequestProperty("Content-Type", "application/json") + post.getOutputStream().write(message.getBytes("UTF-8")) + int responseCode = post.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while submitting metrics" ) + } + } + + Map registerTask( Map config, int id ){ + + HttpURLConnection post = URI.create("${getDNS()}/scheduler/$runName/task/$id").toURL().openConnection() as HttpURLConnection + post.setRequestMethod( "POST" ) + String message = JsonOutput.toJson( config ) + post.setDoOutput(true) + post.setRequestProperty("Content-Type", "application/json") + post.getOutputStream().write(message.getBytes("UTF-8")) + int responseCode = post.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while registering task: ${config.name}" ) + } + tasksInBatch++ + Map response = new JsonSlurper().parse(post.getInputStream()) as Map + return response + + } + + private void batch( String command ){ + HttpURLConnection put = URI.create("${getDNS()}/scheduler/$runName/${command}Batch").toURL().openConnection() as HttpURLConnection + put.setRequestMethod( "PUT" ) + if ( command == 'end' ){ + put.setDoOutput(true) + put.setRequestProperty("Content-Type", "application/json") + put.getOutputStream().write("$tasksInBatch".getBytes("UTF-8")) + } + int responseCode = put.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while ${command}ing batch" ) + } + } + + void startBatch(){ + tasksInBatch = 0 + if ( !closed ) batch('start') + } + + void endBatch(){ + if ( !closed ) batch('end') + } + + Map getTaskState( int id ){ + + HttpURLConnection get = URI.create("${getDNS()}/scheduler/$runName/task/$id").toURL().openConnection() as HttpURLConnection + get.setRequestMethod( "GET" ) + get.setDoOutput(true) + int responseCode = get.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while requesting task state: $id" ) + } + Map response = new JsonSlurper().parse(get.getInputStream()) as Map + return response + + } + + ///* DAG */ + + @CompileDynamic + void submitVertices( List vertices ){ + List> verticesToSubmit = vertices.collect { + [ + label : it.label, + type : it.type.toString(), + uid : it.getId() + ] as Map + } + HttpURLConnection put = URI.create("${getDNS()}/scheduler/$runName/DAG/vertices").toURL().openConnection() as HttpURLConnection + put.setRequestMethod( "POST" ) + String message = JsonOutput.toJson( verticesToSubmit ) + put.setDoOutput(true) + put.setRequestProperty("Content-Type", "application/json") + put.getOutputStream().write(message.getBytes("UTF-8")) + int responseCode = put.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while submitting vertices: ${vertices}" ) + } + } + + @CompileDynamic + void submitEdges( List edges ){ + List> edgesToSubmit = edges.collect { + [ + uid : it.getId(), + label : it.getLabel(), + from : it.getFrom()?.getId(), + to : it.getTo()?.getId() + ] as Map + } + HttpURLConnection put = URI.create("${getDNS()}/scheduler/$runName/DAG/edges").toURL().openConnection() as HttpURLConnection + put.setRequestMethod( "POST" ) + String message = JsonOutput.toJson( edgesToSubmit ) + put.setDoOutput(true) + put.setRequestProperty("Content-Type", "application/json") + put.getOutputStream().write(message.getBytes("UTF-8")) + int responseCode = put.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while submitting edges: ${edges}" ) + } + } + + ///* File location */ + + Map getFileLocation( String path ){ + + String pathEncoded = URLEncoder.encode(path,'utf-8') + HttpURLConnection get = URI.create("${getDNS()}/file/$runName?path=$pathEncoded").toURL().openConnection() as HttpURLConnection + get.setRequestMethod( "GET" ) + get.setDoOutput(true) + int responseCode = get.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while requesting file location: $path (${get.responseMessage})" ) + } + Map response = new JsonSlurper().parse(get.getInputStream()) as Map + return response + + } + + String getDaemonOnNode( String node ){ + + HttpURLConnection get = URI.create("${getDNS()}/daemon/$runName/$node").toURL().openConnection() as HttpURLConnection + get.setRequestMethod( "GET" ) + get.setDoOutput(true) + int responseCode = get.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while requesting daemon on node: $node" ) + } + String response = new JsonSlurper().parse(get.getInputStream()) as String + return response + + } + + void addFileLocation( String path, long size, long timestamp, long locationWrapperID, boolean overwrite, String node = null ){ + + String method = overwrite ? 'overwrite' : 'add' + + HttpURLConnection get = URI.create("${getDNS()}/file/$runName/location/${method}${ node ? "/$node" : ''}").toURL().openConnection() as HttpURLConnection + get.setRequestMethod( "POST" ) + get.setDoOutput(true) + Map data = [ + path : path, + size : size, + timestamp : timestamp, + locationWrapperID : locationWrapperID + ] + if ( node ){ + data.node = node + } + String message = JsonOutput.toJson( data ) + get.setRequestProperty("Content-Type", "application/json") + get.getOutputStream().write(message.getBytes("UTF-8")) + int responseCode = get.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while updating file location: $path: $node (${get.responseMessage})" ) + } + + } + + void publish(Path source, Path destination, String mode ) { + HttpURLConnection get = URI.create("${getDNS()}/file/$runName/publish").toURL().openConnection() as HttpURLConnection + get.setRequestMethod( "PUT" ) + get.setDoOutput(true) + Map data = [ + 'source' : source.toString(), + 'destination' : destination.toString(), + 'mode' : mode.toUpperCase(), + ] + String message = JsonOutput.toJson( data ) + get.setRequestProperty("Content-Type", "application/json") + get.getOutputStream().write(message.getBytes("UTF-8")) + int responseCode = get.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while publishing file: $source (${get.responseMessage}) -- ${get.getURL()}" ) + } + } + + void publishRemaining() { + HttpURLConnection get = URI.create("${getDNS()}/file/$runName/publish").toURL().openConnection() as HttpURLConnection + get.setRequestMethod( "POST" ) + int responseCode = get.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, when triggering publish (${get.responseMessage}) -- ${get.getURL()}" ) + } + } + + int getRemainingToPublish() { + HttpURLConnection get = URI.create("${getDNS()}/file/$runName/publish").toURL().openConnection() as HttpURLConnection + get.setRequestMethod( "GET" ) + get.setRequestProperty("Content-Type", "application/json") + int responseCode = get.getResponseCode() + if( responseCode != 200 ){ + throw new IllegalStateException( "Got code: ${responseCode} from nextflow scheduler, while getting remaining to publish (${get.responseMessage})" ) + } + get.getInputStream().text as int + } + } \ No newline at end of file diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy index a110a00..e664484 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy @@ -161,6 +161,8 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { void shutdown() { final CWSK8sConfig.K8sScheduler schedulerConfig = (k8sConfig as CWSK8sConfig).getScheduler() + schedulerClient.publishRemaining(); + int remaining do { remaining = schedulerClient.getRemainingToPublish() From 3b974ff959fefca695221932c1975a54e1e36882 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Wed, 14 May 2025 22:59:19 +0200 Subject: [PATCH 47/63] Require daemon in version 2.0 Signed-off-by: Lehmann_Fabian --- .../main/nextflow/cws/k8s/CWSK8sConfig.groovy | 358 +++++++++--------- 1 file changed, 179 insertions(+), 179 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy index d1f7002..2d3bae8 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy @@ -1,180 +1,180 @@ -package nextflow.cws.k8s - -import groovy.transform.CompileStatic -import groovy.transform.Memoized -import groovy.transform.PackageScope -import nextflow.exception.AbortOperationException -import nextflow.k8s.K8sConfig -import nextflow.k8s.client.K8sClient -import nextflow.k8s.model.PodHostMount -import nextflow.k8s.model.PodNodeSelector - -import java.nio.file.Path -import java.util.stream.Collectors - -@CompileStatic -class CWSK8sConfig extends K8sConfig { - - private Map target - - CWSK8sConfig(Map config) { - super(config) - this.target = config - if (getLocalPath()) { - final name = getLocalPath() - final mount = getLocalStorageMountPath() - getPodOptions().mountHostPaths.add(new PodHostMount(name, mount)) - } - } - - K8sScheduler getScheduler(){ - return target.scheduler || locationAwareScheduling() ? new K8sScheduler( (Map)target.scheduler ) : null - } - - Storage getStorage() { - locationAwareScheduling() ? new Storage((Map) target.storage, getLocalClaimPaths()) : null - } - - String getLocalPath() { - target.localPath as String - } - - String getLocalStorageMountPath() { - target.localStorageMountPath ?: '/workspace' as String - } - - boolean locationAwareScheduling() { - getLocalClaimPaths().size() > 0 - } - - Collection getLocalClaimPaths() { - getPodOptions().mountHostPaths.collect { it.mountPath } - } - - String findLocalVolumeClaimByPath(String path) { - def result = getPodOptions().mountHostPaths.find { path.startsWith(it.mountPath) } - return result ? result.hostPath : null - } - - void checkStorageAndPaths(K8sClient client, String pipelineName) { - super.checkStorageAndPaths(client) - //The nextflow project/workflow has to be on a shared drive - if (pipelineName && pipelineName[0] == '/' && !findVolumeClaimByPath(pipelineName)) - throw new AbortOperationException("Kubernetes `pipelineName` must be a path mounted as a persistent volume -- projectDir=$pipelineName; volumes=${getClaimPaths().join(', ')}") - - if (getStorage() && !findLocalVolumeClaimByPath(getStorage().getWorkdir())) - throw new AbortOperationException("Kubernetes `storage.workdir` must be a path mounted as a local volume -- storage.workdir=${getStorage().getWorkdir()}; volumes=${getLocalClaimPaths().join(', ')}") - } - - @CompileStatic - @PackageScope - static class K8sScheduler { - - Map target - - private final String[] fields = [ - 'name', - 'serviceAccount', - 'cpu', - 'memory', - 'container', - 'command', - 'port', - 'workDir', - 'runAsUser', - 'autoClose', - 'nodeSelector', - 'imagePullPolicy' - ] - - K8sScheduler(Map scheduler) { - this.target = scheduler - } - - String getName() { target.name as String ?: 'workflow-scheduler' } - - String getServiceAccount() { target.serviceAccount as String } - - // If no container is specified pull the latest image - String getImagePullPolicy() { target.container ? target.imagePullPolicy as String : "Always" } - - Integer getCPUs() { target.cpu as Integer ?: 1 } - - String getMemory() { target.memory as String ?: "1400Mi" } - - String getContainer() { target.container as String ?: 'commonworkflowscheduler/kubernetesscheduler:latest' } - - String getCommand() { target.command as String } - - Integer getPort() { target.port as Integer ?: 8080 } - - String getWorkDir() { target.workDir as String } - - Integer runAsUser() { target.runAsUser as Integer } - - Boolean autoClose() { target.autoClose == null ? true : target.autoClose as Boolean } - - PodNodeSelector getNodeSelector(){ - return target.nodeSelector ? new PodNodeSelector( target.nodeSelector ) : null - } - - Map getAdditional() { - return target.entrySet() - .stream() - .filter{!(it.getKey() in fields) } - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) - } - - @Memoized - static K8sScheduler defaultConfig( K8sConfig k8sConfig ){ - return new K8sScheduler([ - "serviceAccount" : k8sConfig.getServiceAccount(), - "runAsUser" : 0, - "autoClose" : true - ] as Map) - } - - } - - @CompileStatic - @PackageScope - static class Storage { - - @Delegate - Map target - Collection localClaims - - Storage(Map scheduler, Collection localClaims ) { - this.target = scheduler - this.localClaims = localClaims - } - - String getCopyStrategy() { - target.copyStrategy as String ?: 'ftp' - } - - String getWorkdir() { - Path workdir = (target.workdir ?: localClaims[0]) as Path - if( ! workdir.getName().equalsIgnoreCase('localWork') ){ - workdir = workdir.resolve( 'localWork' ) - } - return workdir.toString() - } - - PodNodeSelector getNodeSelector(){ - return target.nodeSelector ? new PodNodeSelector( target.nodeSelector ) : null - } - - boolean deleteIntermediateData(){ - target.deleteIntermediateData as Boolean ?: false - } - - String getImageName() { - target.imageName ?: 'commonworkflowscheduler/ftpdaemon:v1.0' - } - - String getCmd() { - target.cmd as String - } - } +package nextflow.cws.k8s + +import groovy.transform.CompileStatic +import groovy.transform.Memoized +import groovy.transform.PackageScope +import nextflow.exception.AbortOperationException +import nextflow.k8s.K8sConfig +import nextflow.k8s.client.K8sClient +import nextflow.k8s.model.PodHostMount +import nextflow.k8s.model.PodNodeSelector + +import java.nio.file.Path +import java.util.stream.Collectors + +@CompileStatic +class CWSK8sConfig extends K8sConfig { + + private Map target + + CWSK8sConfig(Map config) { + super(config) + this.target = config + if (getLocalPath()) { + final name = getLocalPath() + final mount = getLocalStorageMountPath() + getPodOptions().mountHostPaths.add(new PodHostMount(name, mount)) + } + } + + K8sScheduler getScheduler(){ + return target.scheduler || locationAwareScheduling() ? new K8sScheduler( (Map)target.scheduler ) : null + } + + Storage getStorage() { + locationAwareScheduling() ? new Storage((Map) target.storage, getLocalClaimPaths()) : null + } + + String getLocalPath() { + target.localPath as String + } + + String getLocalStorageMountPath() { + target.localStorageMountPath ?: '/workspace' as String + } + + boolean locationAwareScheduling() { + getLocalClaimPaths().size() > 0 + } + + Collection getLocalClaimPaths() { + getPodOptions().mountHostPaths.collect { it.mountPath } + } + + String findLocalVolumeClaimByPath(String path) { + def result = getPodOptions().mountHostPaths.find { path.startsWith(it.mountPath) } + return result ? result.hostPath : null + } + + void checkStorageAndPaths(K8sClient client, String pipelineName) { + super.checkStorageAndPaths(client) + //The nextflow project/workflow has to be on a shared drive + if (pipelineName && pipelineName[0] == '/' && !findVolumeClaimByPath(pipelineName)) + throw new AbortOperationException("Kubernetes `pipelineName` must be a path mounted as a persistent volume -- projectDir=$pipelineName; volumes=${getClaimPaths().join(', ')}") + + if (getStorage() && !findLocalVolumeClaimByPath(getStorage().getWorkdir())) + throw new AbortOperationException("Kubernetes `storage.workdir` must be a path mounted as a local volume -- storage.workdir=${getStorage().getWorkdir()}; volumes=${getLocalClaimPaths().join(', ')}") + } + + @CompileStatic + @PackageScope + static class K8sScheduler { + + Map target + + private final String[] fields = [ + 'name', + 'serviceAccount', + 'cpu', + 'memory', + 'container', + 'command', + 'port', + 'workDir', + 'runAsUser', + 'autoClose', + 'nodeSelector', + 'imagePullPolicy' + ] + + K8sScheduler(Map scheduler) { + this.target = scheduler + } + + String getName() { target.name as String ?: 'workflow-scheduler' } + + String getServiceAccount() { target.serviceAccount as String } + + // If no container is specified pull the latest image + String getImagePullPolicy() { target.container ? target.imagePullPolicy as String : "Always" } + + Integer getCPUs() { target.cpu as Integer ?: 1 } + + String getMemory() { target.memory as String ?: "1400Mi" } + + String getContainer() { target.container as String ?: 'commonworkflowscheduler/kubernetesscheduler:latest' } + + String getCommand() { target.command as String } + + Integer getPort() { target.port as Integer ?: 8080 } + + String getWorkDir() { target.workDir as String } + + Integer runAsUser() { target.runAsUser as Integer } + + Boolean autoClose() { target.autoClose == null ? true : target.autoClose as Boolean } + + PodNodeSelector getNodeSelector(){ + return target.nodeSelector ? new PodNodeSelector( target.nodeSelector ) : null + } + + Map getAdditional() { + return target.entrySet() + .stream() + .filter{!(it.getKey() in fields) } + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) + } + + @Memoized + static K8sScheduler defaultConfig( K8sConfig k8sConfig ){ + return new K8sScheduler([ + "serviceAccount" : k8sConfig.getServiceAccount(), + "runAsUser" : 0, + "autoClose" : true + ] as Map) + } + + } + + @CompileStatic + @PackageScope + static class Storage { + + @Delegate + Map target + Collection localClaims + + Storage(Map scheduler, Collection localClaims ) { + this.target = scheduler + this.localClaims = localClaims + } + + String getCopyStrategy() { + target.copyStrategy as String ?: 'ftp' + } + + String getWorkdir() { + Path workdir = (target.workdir ?: localClaims[0]) as Path + if( ! workdir.getName().equalsIgnoreCase('localWork') ){ + workdir = workdir.resolve( 'localWork' ) + } + return workdir.toString() + } + + PodNodeSelector getNodeSelector(){ + return target.nodeSelector ? new PodNodeSelector( target.nodeSelector ) : null + } + + boolean deleteIntermediateData(){ + target.deleteIntermediateData as Boolean ?: false + } + + String getImageName() { + target.imageName ?: 'commonworkflowscheduler/ftpdaemon:v2.0' + } + + String getCmd() { + target.cmd as String + } + } } \ No newline at end of file From 0905e35c7220a21c8cc6eabbb758fb6f5fb0227e Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 15 May 2025 14:29:15 +0200 Subject: [PATCH 48/63] Require kubernetesscheduler in version 2.1 Signed-off-by: Lehmann_Fabian --- plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy index 2d3bae8..746bdac 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy @@ -102,7 +102,7 @@ class CWSK8sConfig extends K8sConfig { String getMemory() { target.memory as String ?: "1400Mi" } - String getContainer() { target.container as String ?: 'commonworkflowscheduler/kubernetesscheduler:latest' } + String getContainer() { target.container as String ?: 'commonworkflowscheduler/kubernetesscheduler:v2.1' } String getCommand() { target.command as String } From dc6b1a533d6b9e8f508450f2013a0c0b5830a642 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Thu, 15 May 2025 14:31:04 +0200 Subject: [PATCH 49/63] Set kubernetesscheduler to version 2.1 in readme Signed-off-by: Lehmann_Fabian --- README.md | 378 +++++++++++++++++++++++++++--------------------------- 1 file changed, 189 insertions(+), 189 deletions(-) diff --git a/README.md b/README.md index b8baca5..5d0ccdb 100644 --- a/README.md +++ b/README.md @@ -1,190 +1,190 @@ -# nf-cws plugin - -This plugin enables Nextflow to communicate with a Common Workflow Scheduler instance and transfer the required -information. - -For more information on the scheduling, -see the [scheduler repository](https://github.com/CommonWorkflowScheduler/KubernetesScheduler) and the [respective -papers](#citation): - -### Supported Executors - -- k8s - -### How to use - -To run Nextflow with this plugin, you need version >=`23.03.0-edge`. -To activate the plugin, add `-plugins nf-cws` to your `nextflow` call or add the following to your `nextflow.config`: - -``` -plugins { - id 'nf-cws' -} -``` - -### Configuration - -| Attribute | Required | Explanation | -|:---------------:|---------:|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| dns | - | Provide the link to the running CWS instance.
NOTE: If you provide an address here, the `k8s` executor will not try to start a Common Workflow Scheduler instance on demand. | -| strategy | - | Which strategy should be used for scheduling; available strategies depend on the CWS instance | -| costFunction | - | Which cost function should be used for scheduling; available strategies depend on the CWS instance | -| batchSize | - | Number of tasks to submit together (only if more than this are ready to run); default: 1 | -| memoryPredictor | - | The memory predictor that shall be used for task scaling.
If not set, task scaling is disabled. See Common Workflow Scheduler for supported predictors. | -| minMemory | - | The minimum memory to size a task to. Only used if memory prediction is performed. | -| maxMemory | - | The maximum memory to size a task to. Only used if memory prediction is performed. | - -##### Example: - -``` -cws { - dns = 'http://cws-scheduler/' - strategy = 'rank_max-fair' - costFunction = 'MinSize' - batchSize = 10 - memoryPredictor = '' - minMemory = 128.MB - maxMemory = 64.GB -} -``` - -#### K8s Executor - -The `k8s` executor allows starting a Common Workflow Scheduler instance on demand. This will happen if you do not define -any CWS-related config. Otherwise, you can configure the following: - -``` -k8s { - scheduler { - name = 'workflow-scheduler' - serviceAccount = 'nextflowscheduleraccount' - imagePullPolicy = 'IfNotPresent' - cpu = '2' - memory = '1400Mi' - container = 'commonworkflowscheduler/kubernetesscheduler:v1.0' - command = null - port = 8080 - workDir = '/scheduler' - runAsUser = 0 - autoClose = false - nodeSelector = null - } -} -``` - -| Attribute | Required | Explanation | -|:----------------|----------|------------------------------------------------------------------------------------------------------------------------------| -| name | - | The name of the pod created | -| serviceAccount | - | Service account used by the scheduler | -| imagePullPolicy | - | Image pull policy for the created pod ([k8s docs](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy)) | -| cpu | - | Number of cores to use for the scheduler pod | -| memory | - | Memory to use for the scheduler pod | -| container | - | Container image to use for the scheduler pod | -| command | - | Command to start the CWS in the pod. If you need to overwrite the original ENTRYPOINT | -| port | - | Port where to reach the CWS Rest API | -| workDir | - | Workdir within the pod | -| runAsUser | - | Run the scheduler as a specific user | -| autoClose | - | Stop the pod after the workflow is finished | -| nodeSelector | - | A node selector for the CWS pod | - -#### WOW - -WOW is a new scheduling approach for dynamic scientific workflow systems that steers both data movement and task -scheduling to reduce network congestion and overall runtime. - -WOW requires some additional configuration due to its use of the local file system in addition to the distributed file -system. - -``` -k8s { - localPath = '/localdata' - localStorageMountPath = '/localdata' - storage { - copyStrategy = 'ftp' - workdir = '/localdata/localwork/' - } -} -``` - -| Attribute | Required | Explanation | -|:----------------------|----------|-------------------------------------------------------------------------------------------------| -| localPath | yes | Host path for the local mount -| localStorageMountPath | no | Container path for the local mount -| storage.copyStrategy | no | Strategy to copy the files between nodes - currently only supports 'ftp' (and its alias 'copy') -| storage.workdir | no | Working directory to use - must be inside of the locally mounted directory - -### Tracing - -This plugin adds additional fields to the trace report. Therefore, you have to add the required fields to -the `trace.fields` field in your Nextflow config (also check the -official [documentation](https://www.nextflow.io/docs/latest/tracing.html#trace-report)). -The following fields can be used: - -| Name | Description | -|:---------------------------------------|:-----------------------------------------------------------------------------------------------------------------:| -| input_size | The accumulated size of the input files | -| memory_adapted | The memory that was used after adaption by the scheduler | -| submit_to_scheduler_time | Time in ms to register the task at CWS | -| submit_to_k8s_time | Time to create and submit pod to k8s | -| scheduler_time_in_queue | How long was the task in the queue until it got scheduled | -| scheduler_place_in_queue | At which place was the task in the queue when it got scheduled | -| scheduler_tried_to_schedule | How often was a scheduling plan calculated until the task was assigned | -| scheduler_time_to_schedule | How long did it take to calculate the location for this task | -| scheduler_nodes_tried | How many nodes have been compared | -| scheduler_nodes_cost | Cost value to schedule on the different nodes (only available for some algorithms) | -| scheduler_could_stop_fetching | How often could the scheduler skip a node | -| scheduler_best_cost | Cost on the selected node (only available for some algorithms) | -| scheduler_delta_schedule_submitted | Time delta between starting to calculate the scheduling plan and submitting the task to the target node | -| scheduler_delta_schedule_alignment | Time delta between beginning to calculate the scheduling plan and finding the target node | -| scheduler_batch_id | The id of the batch the task belongs to | -| scheduler_delta_batch_start_submitted | Time delta between a batch was started, and the scheduler received this task from the workflow engine | -| scheduler_delta_batch_start_received | Time delta between a batch was started, and the scheduler received the pod from the k8s API | -| scheduler_delta_batch_closed_batch_end | Time delta between a batch was closed by the workflow engine, and the scheduler received the pod from the k8s API | -| scheduler_delta_submitted_batch_end | Time delta between a task was submitted, and the batch became schedulable | -| memory_adapted | The memory used for a task when sizing is active | -| input_size | The sum of the input size of all task inputs | -| infiles_time: | (WOW) Time to walk through and retrieve stats of all local (input) files at task start | -| outfiles_time: | (WOW) Time to walk through and retrieve stats of all local (output) files at task start | -| scheduler_time_delta_phase_three: | (WOW) List of time instances taken to calculcate step 3 of the WOW scheduling algorithm (see paper for details) | -| scheduler_copy_tasks: | (WOW) Number of times copy tasks were started for this task | - ---- - -## Citation - -If you use this software or artifacts in a publication, please cite it as: - -#### Text - -Lehmann Fabian, Jonathan Bader, Friedrich Tschirpke, Lauritz Thamsen, and Ulf Leser. **How Workflow Engines Should Talk -to Resource Managers: A Proposal for a Common Workflow Scheduling Interface**. In 2023 IEEE/ACM 23rd International -Symposium on Cluster, Cloud and Internet Computing (CCGrid). Bangalore, India, 2023. - -([https://arxiv.org/pdf/2302.07652.pdf](https://arxiv.org/pdf/2302.07652.pdf)) - -#### BibTeX - -``` -@inproceedings{lehmannHowWorkflowEngines2023, - author = {Lehmann, Fabian and Bader, Jonathan and Tschirpke, Friedrich and Thamsen, Lauritz and Leser, Ulf}, - booktitle = {2023 IEEE/ACM 23rd International Symposium on Cluster, Cloud and Internet Computing (CCGrid)}, - title = {How Workflow Engines Should Talk to Resource Managers: A Proposal for a Common Workflow Scheduling Interface}, - year = {2023}, - address = {{Bangalore, India}}, - doi = {10.1109/CCGrid57682.2023.00025} -} -``` - -#### Strategy-specific Citation - -Please note that the following strategies originated in individual papers: - -- PONDER: [https://arxiv.org/pdf/2408.00047.pdf](https://arxiv.org/pdf/2408.00047.pdf) -- WOW: [https://arxiv.org/pdf/2503.13072.pdf](https://arxiv.org/pdf/2503.13072.pdf) - ---- - -#### Acknowledgement: - -This work was funded by the German Research Foundation (DFG), CRC 1404: "FONDA: Foundations of Workflows for Large-Scale +# nf-cws plugin + +This plugin enables Nextflow to communicate with a Common Workflow Scheduler instance and transfer the required +information. + +For more information on the scheduling, +see the [scheduler repository](https://github.com/CommonWorkflowScheduler/KubernetesScheduler) and the [respective +papers](#citation): + +### Supported Executors + +- k8s + +### How to use + +To run Nextflow with this plugin, you need version >=`23.03.0-edge`. +To activate the plugin, add `-plugins nf-cws` to your `nextflow` call or add the following to your `nextflow.config`: + +``` +plugins { + id 'nf-cws' +} +``` + +### Configuration + +| Attribute | Required | Explanation | +|:---------------:|---------:|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| dns | - | Provide the link to the running CWS instance.
NOTE: If you provide an address here, the `k8s` executor will not try to start a Common Workflow Scheduler instance on demand. | +| strategy | - | Which strategy should be used for scheduling; available strategies depend on the CWS instance | +| costFunction | - | Which cost function should be used for scheduling; available strategies depend on the CWS instance | +| batchSize | - | Number of tasks to submit together (only if more than this are ready to run); default: 1 | +| memoryPredictor | - | The memory predictor that shall be used for task scaling.
If not set, task scaling is disabled. See Common Workflow Scheduler for supported predictors. | +| minMemory | - | The minimum memory to size a task to. Only used if memory prediction is performed. | +| maxMemory | - | The maximum memory to size a task to. Only used if memory prediction is performed. | + +##### Example: + +``` +cws { + dns = 'http://cws-scheduler/' + strategy = 'rank_max-fair' + costFunction = 'MinSize' + batchSize = 10 + memoryPredictor = '' + minMemory = 128.MB + maxMemory = 64.GB +} +``` + +#### K8s Executor + +The `k8s` executor allows starting a Common Workflow Scheduler instance on demand. This will happen if you do not define +any CWS-related config. Otherwise, you can configure the following: + +``` +k8s { + scheduler { + name = 'workflow-scheduler' + serviceAccount = 'nextflowscheduleraccount' + imagePullPolicy = 'IfNotPresent' + cpu = '2' + memory = '1400Mi' + container = 'commonworkflowscheduler/kubernetesscheduler:v2.1' + command = null + port = 8080 + workDir = '/scheduler' + runAsUser = 0 + autoClose = false + nodeSelector = null + } +} +``` + +| Attribute | Required | Explanation | +|:----------------|----------|------------------------------------------------------------------------------------------------------------------------------| +| name | - | The name of the pod created | +| serviceAccount | - | Service account used by the scheduler | +| imagePullPolicy | - | Image pull policy for the created pod ([k8s docs](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy)) | +| cpu | - | Number of cores to use for the scheduler pod | +| memory | - | Memory to use for the scheduler pod | +| container | - | Container image to use for the scheduler pod | +| command | - | Command to start the CWS in the pod. If you need to overwrite the original ENTRYPOINT | +| port | - | Port where to reach the CWS Rest API | +| workDir | - | Workdir within the pod | +| runAsUser | - | Run the scheduler as a specific user | +| autoClose | - | Stop the pod after the workflow is finished | +| nodeSelector | - | A node selector for the CWS pod | + +#### WOW + +WOW is a new scheduling approach for dynamic scientific workflow systems that steers both data movement and task +scheduling to reduce network congestion and overall runtime. + +WOW requires some additional configuration due to its use of the local file system in addition to the distributed file +system. + +``` +k8s { + localPath = '/localdata' + localStorageMountPath = '/localdata' + storage { + copyStrategy = 'ftp' + workdir = '/localdata/localwork/' + } +} +``` + +| Attribute | Required | Explanation | +|:----------------------|----------|-------------------------------------------------------------------------------------------------| +| localPath | yes | Host path for the local mount +| localStorageMountPath | no | Container path for the local mount +| storage.copyStrategy | no | Strategy to copy the files between nodes - currently only supports 'ftp' (and its alias 'copy') +| storage.workdir | no | Working directory to use - must be inside of the locally mounted directory + +### Tracing + +This plugin adds additional fields to the trace report. Therefore, you have to add the required fields to +the `trace.fields` field in your Nextflow config (also check the +official [documentation](https://www.nextflow.io/docs/latest/tracing.html#trace-report)). +The following fields can be used: + +| Name | Description | +|:---------------------------------------|:-----------------------------------------------------------------------------------------------------------------:| +| input_size | The accumulated size of the input files | +| memory_adapted | The memory that was used after adaption by the scheduler | +| submit_to_scheduler_time | Time in ms to register the task at CWS | +| submit_to_k8s_time | Time to create and submit pod to k8s | +| scheduler_time_in_queue | How long was the task in the queue until it got scheduled | +| scheduler_place_in_queue | At which place was the task in the queue when it got scheduled | +| scheduler_tried_to_schedule | How often was a scheduling plan calculated until the task was assigned | +| scheduler_time_to_schedule | How long did it take to calculate the location for this task | +| scheduler_nodes_tried | How many nodes have been compared | +| scheduler_nodes_cost | Cost value to schedule on the different nodes (only available for some algorithms) | +| scheduler_could_stop_fetching | How often could the scheduler skip a node | +| scheduler_best_cost | Cost on the selected node (only available for some algorithms) | +| scheduler_delta_schedule_submitted | Time delta between starting to calculate the scheduling plan and submitting the task to the target node | +| scheduler_delta_schedule_alignment | Time delta between beginning to calculate the scheduling plan and finding the target node | +| scheduler_batch_id | The id of the batch the task belongs to | +| scheduler_delta_batch_start_submitted | Time delta between a batch was started, and the scheduler received this task from the workflow engine | +| scheduler_delta_batch_start_received | Time delta between a batch was started, and the scheduler received the pod from the k8s API | +| scheduler_delta_batch_closed_batch_end | Time delta between a batch was closed by the workflow engine, and the scheduler received the pod from the k8s API | +| scheduler_delta_submitted_batch_end | Time delta between a task was submitted, and the batch became schedulable | +| memory_adapted | The memory used for a task when sizing is active | +| input_size | The sum of the input size of all task inputs | +| infiles_time: | (WOW) Time to walk through and retrieve stats of all local (input) files at task start | +| outfiles_time: | (WOW) Time to walk through and retrieve stats of all local (output) files at task start | +| scheduler_time_delta_phase_three: | (WOW) List of time instances taken to calculcate step 3 of the WOW scheduling algorithm (see paper for details) | +| scheduler_copy_tasks: | (WOW) Number of times copy tasks were started for this task | + +--- + +## Citation + +If you use this software or artifacts in a publication, please cite it as: + +#### Text + +Lehmann Fabian, Jonathan Bader, Friedrich Tschirpke, Lauritz Thamsen, and Ulf Leser. **How Workflow Engines Should Talk +to Resource Managers: A Proposal for a Common Workflow Scheduling Interface**. In 2023 IEEE/ACM 23rd International +Symposium on Cluster, Cloud and Internet Computing (CCGrid). Bangalore, India, 2023. + +([https://arxiv.org/pdf/2302.07652.pdf](https://arxiv.org/pdf/2302.07652.pdf)) + +#### BibTeX + +``` +@inproceedings{lehmannHowWorkflowEngines2023, + author = {Lehmann, Fabian and Bader, Jonathan and Tschirpke, Friedrich and Thamsen, Lauritz and Leser, Ulf}, + booktitle = {2023 IEEE/ACM 23rd International Symposium on Cluster, Cloud and Internet Computing (CCGrid)}, + title = {How Workflow Engines Should Talk to Resource Managers: A Proposal for a Common Workflow Scheduling Interface}, + year = {2023}, + address = {{Bangalore, India}}, + doi = {10.1109/CCGrid57682.2023.00025} +} +``` + +#### Strategy-specific Citation + +Please note that the following strategies originated in individual papers: + +- PONDER: [https://arxiv.org/pdf/2408.00047.pdf](https://arxiv.org/pdf/2408.00047.pdf) +- WOW: [https://arxiv.org/pdf/2503.13072.pdf](https://arxiv.org/pdf/2503.13072.pdf) + +--- + +#### Acknowledgement: + +This work was funded by the German Research Foundation (DFG), CRC 1404: "FONDA: Foundations of Workflows for Large-Scale Scientific Data Analysis." \ No newline at end of file From aef4df0a70c6620c678457622738db004a595519 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 16 May 2025 10:56:52 +0200 Subject: [PATCH 50/63] Fix problem that fileType was not correctly parsed Signed-off-by: Lehmann_Fabian --- .../cws/wow/file/WOWFileAttributes.groovy | 308 +++++++++--------- 1 file changed, 154 insertions(+), 154 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy index ec1f54c..21e953e 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy @@ -1,155 +1,155 @@ -package nextflow.cws.wow.file - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import nextflow.cws.wow.util.DateParser - -import java.nio.file.Path -import java.nio.file.attribute.BasicFileAttributes -import java.nio.file.attribute.FileTime - -@Slf4j -@CompileStatic -class WOWFileAttributes implements BasicFileAttributes { - - /** - * Helper to parse the file attributes returned by the getStatsAndResolveSymlinks.c - */ - static private final int FILE_EXISTS = 1 - static private final int REAL_PATH = 2 - static private final int SIZE = 3 - static private final int FILE_TYPE = 4 - static private final int CREATION_DATE = 5 - static private final int ACCESS_DATE = 6 - static private final int MODIFICATION_DATE = 7 - - private final boolean directory - - private final boolean link - - private final long size - - private final String fileType - - private final FileTime creationDate - - private final FileTime accessDate - - private final FileTime modificationDate - - private final Path destination - - private final boolean local - - WOWFileAttributes( String[] data ) { - boolean fileExists = data[ FILE_EXISTS ] == "1" - if ( fileExists && data.length != 8 ) throw new RuntimeException( "Cannot parse row (8 columns required): ${data.join(',')}" ) - destination = data.length > REAL_PATH && data[ REAL_PATH ] ? data[ REAL_PATH ] as Path : null - if ( data.length != 8 ) { - this.directory = false - this.link = true - this.size = 0 - this.fileType = null - this.creationDate = null - this.accessDate = null - this.modificationDate = null - this.local = false - return - } - this.link = !data[ REAL_PATH ].isEmpty() - this.size = data[ SIZE ] as Long - String fileType = data[ FILE_TYPE ] - this.accessDate = DateParser.fileTimeFromString(data[ ACCESS_DATE ]) - this.modificationDate = DateParser.fileTimeFromString(data[ MODIFICATION_DATE ]) - this.creationDate = DateParser.fileTimeFromString(data[ CREATION_DATE ]) ?: this.modificationDate - if ( fileType.startsWith("non-local ") ) { - this.local = false - this.fileType = fileType.substring( 10 ) - } else { - this.local = true - this.fileType = fileType - } - this.directory = fileType == 'directory' - if ( !directory && !fileType.contains( 'file' ) ){ - log.error( "Unknown type: $fileType" ) - } - } - - WOWFileAttributes(Path path ) { - if (path.isDirectory()) { - directory = true - link = false - size = 4096 - fileType = 'directory' - creationDate = FileTime.fromMillis(0) - accessDate = FileTime.fromMillis(0) - modificationDate = FileTime.fromMillis(0) - destination = path - local = false - } else { - directory = false - link = path.isLink() - size = path.size() - fileType = link ? 'symbolic link' : 'regular file' - creationDate = null - accessDate = null - modificationDate = FileTime.fromMillis(path.lastModified()) - destination = link ? path.toRealPath() : path - local = !link - } - } - - @Override - FileTime lastModifiedTime() { - modificationDate - } - - @Override - FileTime lastAccessTime() { - accessDate - } - - @Override - FileTime creationTime() { - creationDate - } - - @Override - boolean isRegularFile() { - !directory - } - - @Override - boolean isDirectory() { - directory - } - - @Override - boolean isSymbolicLink() { - link - } - - @Override - boolean isOther() { - false - } - - @Override - long size() { - size - } - - @Override - Object fileKey() { - null - } - - Path getDestination(){ - destination - } - - boolean isLocal() { - local - } - +package nextflow.cws.wow.file + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.cws.wow.util.DateParser + +import java.nio.file.Path +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.attribute.FileTime + +@Slf4j +@CompileStatic +class WOWFileAttributes implements BasicFileAttributes { + + /** + * Helper to parse the file attributes returned by the getStatsAndResolveSymlinks.c + */ + static private final int FILE_EXISTS = 1 + static private final int REAL_PATH = 2 + static private final int SIZE = 3 + static private final int FILE_TYPE = 4 + static private final int CREATION_DATE = 5 + static private final int ACCESS_DATE = 6 + static private final int MODIFICATION_DATE = 7 + + private final boolean directory + + private final boolean link + + private final long size + + private final String fileType + + private final FileTime creationDate + + private final FileTime accessDate + + private final FileTime modificationDate + + private final Path destination + + private final boolean local + + WOWFileAttributes( String[] data ) { + boolean fileExists = data[ FILE_EXISTS ] == "1" + if ( fileExists && data.length != 8 ) throw new RuntimeException( "Cannot parse row (8 columns required): ${data.join(',')}" ) + destination = data.length > REAL_PATH && data[ REAL_PATH ] ? data[ REAL_PATH ] as Path : null + if ( data.length != 8 ) { + this.directory = false + this.link = true + this.size = 0 + this.fileType = null + this.creationDate = null + this.accessDate = null + this.modificationDate = null + this.local = false + return + } + this.link = !data[ REAL_PATH ].isEmpty() + this.size = data[ SIZE ] as Long + String fileType = data[ FILE_TYPE ] + this.accessDate = DateParser.fileTimeFromString(data[ ACCESS_DATE ]) + this.modificationDate = DateParser.fileTimeFromString(data[ MODIFICATION_DATE ]) + this.creationDate = DateParser.fileTimeFromString(data[ CREATION_DATE ]) ?: this.modificationDate + if ( fileType.startsWith("non-local ") ) { + this.local = false + this.fileType = fileType.substring( 10 ) + } else { + this.local = true + this.fileType = fileType + } + this.directory = this.fileType == 'directory' + if ( !directory && !this.fileType.contains( 'file' ) ){ + log.error( "Unknown type: ${this.fileType}" ) + } + } + + WOWFileAttributes(Path path ) { + if (path.isDirectory()) { + directory = true + link = false + size = 4096 + fileType = 'directory' + creationDate = FileTime.fromMillis(0) + accessDate = FileTime.fromMillis(0) + modificationDate = FileTime.fromMillis(0) + destination = path + local = false + } else { + directory = false + link = path.isLink() + size = path.size() + fileType = link ? 'symbolic link' : 'regular file' + creationDate = null + accessDate = null + modificationDate = FileTime.fromMillis(path.lastModified()) + destination = link ? path.toRealPath() : path + local = !link + } + } + + @Override + FileTime lastModifiedTime() { + modificationDate + } + + @Override + FileTime lastAccessTime() { + accessDate + } + + @Override + FileTime creationTime() { + creationDate + } + + @Override + boolean isRegularFile() { + !directory + } + + @Override + boolean isDirectory() { + directory + } + + @Override + boolean isSymbolicLink() { + link + } + + @Override + boolean isOther() { + false + } + + @Override + long size() { + size + } + + @Override + Object fileKey() { + null + } + + Path getDestination(){ + destination + } + + boolean isLocal() { + local + } + } \ No newline at end of file From 9c73ca5d295d69749c1c62fcc168ae255c66d267 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 16 May 2025 15:42:40 +0200 Subject: [PATCH 51/63] fix: parse FileTime from long int --- .../cws/wow/file/WOWFileAttributes.groovy | 17 ++++++-- .../nextflow/cws/wow/util/DateParser.groovy | 39 ------------------- .../nf-cws/getStatsAndResolveSymlinks.c | 2 +- 3 files changed, 14 insertions(+), 44 deletions(-) delete mode 100644 plugins/nf-cws/src/main/nextflow/cws/wow/util/DateParser.groovy diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy index 21e953e..0046255 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy @@ -2,7 +2,6 @@ package nextflow.cws.wow.file import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import nextflow.cws.wow.util.DateParser import java.nio.file.Path import java.nio.file.attribute.BasicFileAttributes @@ -41,6 +40,16 @@ class WOWFileAttributes implements BasicFileAttributes { private final boolean local + private static FileTime fileTimeFromString( String date ) { + for (int i = 0; i < date.length(); i++) { + if ( !date.charAt(i).isDigit() ) { + return null + } + } + long nanos = Long.parseLong(date) + return FileTime.fromMillis(nanos / 1e6 as long) + } + WOWFileAttributes( String[] data ) { boolean fileExists = data[ FILE_EXISTS ] == "1" if ( fileExists && data.length != 8 ) throw new RuntimeException( "Cannot parse row (8 columns required): ${data.join(',')}" ) @@ -59,9 +68,9 @@ class WOWFileAttributes implements BasicFileAttributes { this.link = !data[ REAL_PATH ].isEmpty() this.size = data[ SIZE ] as Long String fileType = data[ FILE_TYPE ] - this.accessDate = DateParser.fileTimeFromString(data[ ACCESS_DATE ]) - this.modificationDate = DateParser.fileTimeFromString(data[ MODIFICATION_DATE ]) - this.creationDate = DateParser.fileTimeFromString(data[ CREATION_DATE ]) ?: this.modificationDate + this.accessDate = fileTimeFromString(data[ ACCESS_DATE ]) + this.modificationDate = fileTimeFromString(data[ MODIFICATION_DATE ]) + this.creationDate = fileTimeFromString(data[ CREATION_DATE ]) ?: this.modificationDate if ( fileType.startsWith("non-local ") ) { this.local = false this.fileType = fileType.substring( 10 ) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/util/DateParser.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/util/DateParser.groovy deleted file mode 100644 index 672bc86..0000000 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/util/DateParser.groovy +++ /dev/null @@ -1,39 +0,0 @@ -package nextflow.cws.wow.util - -import groovy.transform.CompileStatic - -import java.nio.file.attribute.FileTime -import java.text.SimpleDateFormat - -@CompileStatic -final class DateParser { - - private DateParser(){} - - static Long millisFromString( String date ) { - if( date == null || date.isEmpty() || date == "-" || date == "w" ) { - return null - } - try { - if (Character.isLetter(date.charAt(0))) { - // if ls was used, date has the format "Nov 2 08:49:30 2021" - return new SimpleDateFormat("MMM dd HH:mm:ss yyyy").parse(date).getTime() - } else { - // if stat was used, date has the format "2021-11-02 08:49:30.955691861 +0000" - String[] parts = date.split(" ") - parts[1] = parts[1].substring(0, 12) - // parts[1] now has milliseconds as smallest units e.g. "08:49:30.955" - String shortenedDate = String.join(" ", parts) - return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS Z").parse(shortenedDate).getTime() - } - } catch ( Exception e ){ - return null - } - } - - static FileTime fileTimeFromString( String date ) { - final Long millisFromString = millisFromString( date ) - return millisFromString == null ? null : FileTime.fromMillis( millisFromString ) - } - -} diff --git a/plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c b/plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c index 19d6d70..8c67994 100644 --- a/plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c +++ b/plugins/nf-cws/src/resources/nf-cws/getStatsAndResolveSymlinks.c @@ -273,7 +273,7 @@ int getFullDescr(char * const * dir, } fprintf( file_ptr, - "%s;%i;%s;%li;%s;%li%li;%li%li;%li%li\n", + "%s;%i;%s;%li;%s;%li%09li;%li%09li;%li%09li\n", ptr->fts_path, exists, symlink_target_path, From d7acc4b35a9e56a0c0376258d66d18add61ba8f8 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 16 May 2025 16:05:27 +0200 Subject: [PATCH 52/63] fix: WOW-only checks and small chores --- Makefile | 3 +-- .../main/nextflow/cws/k8s/CWSK8sConfig.groovy | 18 ++++++++----- .../nextflow/cws/k8s/CWSK8sExecutor.groovy | 26 +++++++++--------- .../nextflow/cws/k8s/CWSK8sTaskHandler.groovy | 27 ++++++++++--------- .../cws/k8s/WOWK8sWrapperBuilder.groovy | 23 ++++++++-------- .../nextflow/cws/wow/file/LocalPath.groovy | 2 +- .../filesystem/WOWFileSystemProvider.groovy | 22 ++++++++------- 7 files changed, 66 insertions(+), 55 deletions(-) diff --git a/Makefile b/Makefile index 0b1cf5b..dfd04d8 100644 --- a/Makefile +++ b/Makefile @@ -44,14 +44,13 @@ clean: rm -rf work rm -rf build rm -rf plugins/*/build - rm $(C_SCRIPT_ALL_TARGETS) + rm -f $(C_SCRIPT_ALL_TARGETS) ./gradlew clean compile: ./gradlew :nextflow:exportClasspath compileGroovy @echo "DONE `date`" - check: ./gradlew check diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy index 746bdac..11dedc0 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy @@ -28,13 +28,17 @@ class CWSK8sConfig extends K8sConfig { } K8sScheduler getScheduler(){ - return target.scheduler || locationAwareScheduling() ? new K8sScheduler( (Map)target.scheduler ) : null + target.scheduler || locationAwareScheduling() ? new K8sScheduler( (Map)target.scheduler ) : null } Storage getStorage() { locationAwareScheduling() ? new Storage((Map) target.storage, getLocalClaimPaths()) : null } + String getArchitecture() { + target.architecture ?: System.getProperty("os.arch") + } + String getLocalPath() { target.localPath as String } @@ -58,12 +62,14 @@ class CWSK8sConfig extends K8sConfig { void checkStorageAndPaths(K8sClient client, String pipelineName) { super.checkStorageAndPaths(client) - //The nextflow project/workflow has to be on a shared drive - if (pipelineName && pipelineName[0] == '/' && !findVolumeClaimByPath(pipelineName)) - throw new AbortOperationException("Kubernetes `pipelineName` must be a path mounted as a persistent volume -- projectDir=$pipelineName; volumes=${getClaimPaths().join(', ')}") + if ( locationAwareScheduling() ) { + //The nextflow project/workflow has to be on a shared drive + if (pipelineName && pipelineName[0] == '/' && !findVolumeClaimByPath(pipelineName)) + throw new AbortOperationException("Kubernetes `pipelineName` must be a path mounted as a persistent volume -- projectDir=$pipelineName; volumes=${getClaimPaths().join(', ')}") - if (getStorage() && !findLocalVolumeClaimByPath(getStorage().getWorkdir())) - throw new AbortOperationException("Kubernetes `storage.workdir` must be a path mounted as a local volume -- storage.workdir=${getStorage().getWorkdir()}; volumes=${getLocalClaimPaths().join(', ')}") + if (getStorage() && !findLocalVolumeClaimByPath(getStorage().getWorkdir())) + throw new AbortOperationException("Kubernetes `storage.workdir` must be a path mounted as a local volume -- storage.workdir=${getStorage().getWorkdir()}; volumes=${getLocalClaimPaths().join(', ')}") + } } @CompileStatic diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy index e664484..533f11e 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy @@ -104,7 +104,7 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { if ( cwsK8sConfig.locationAwareScheduling() ) { createDaemonSet() - registerGetStatsConfigMap() + registerGetStatsConfigMap( cwsK8sConfig.getArchitecture() ) } CWSK8sConfig.K8sScheduler k8sSchedulerConfig = cwsK8sConfig.getScheduler() @@ -159,18 +159,21 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { @Override void shutdown() { - final CWSK8sConfig.K8sScheduler schedulerConfig = (k8sConfig as CWSK8sConfig).getScheduler() + final CWSK8sConfig cwsK8sConfig = k8sConfig as CWSK8sConfig + final CWSK8sConfig.K8sScheduler schedulerConfig = cwsK8sConfig.getScheduler() schedulerClient.publishRemaining(); - int remaining - do { - remaining = schedulerClient.getRemainingToPublish() - if( remaining > 0 ) { - log.info "Waiting for $remaining files to be published" - sleep( 1000 ) - } - } while( remaining > 0 ) + if ( cwsK8sConfig.locationAwareScheduling() ) { + int remaining + do { + remaining = schedulerClient.getRemainingToPublish() + if (remaining > 0) { + log.info "Waiting for $remaining files to be published" + sleep(1000) + } + } while (remaining > 0) + } if( schedulerConfig ) { try{ @@ -191,11 +194,10 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { log.trace "Close K8s Executor" } - protected void registerGetStatsConfigMap() { + protected void registerGetStatsConfigMap( String architecture ) { Map configMap = [:] String architectureStatFileName - final String architecture = System.getProperty("os.arch") if (architecture == "amd64" || architecture == "x86_64") { architectureStatFileName = "/nf-cws/getStatsAndResolveSymlinks_linux_x86_64" } else if (architecture == "aarch64" || architecture == "arm64") { diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy index ae3e023..cb4a896 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy @@ -58,20 +58,23 @@ class CWSK8sTaskHandler extends K8sTaskHandler { @Override protected Map newSubmitRequest0(TaskRun task, String imageName) { Map pod = super.newSubmitRequest0(task, imageName) - if ( (k8sConfig as CWSK8sConfig)?.getScheduler() ){ - (pod.spec as Map).schedulerName = (k8sConfig as CWSK8sConfig).getScheduler().getName() + "-" + getRunName() + final CWSK8sConfig cwsK8sConfig = k8sConfig as CWSK8sConfig + if ( cwsK8sConfig?.getScheduler() ){ + (pod.spec as Map).schedulerName = cwsK8sConfig.getScheduler().getName() + "-" + getRunName() } - //Set default mode for configMap - Map specs = pod.spec as Map - List volumes = specs?.volumes as List - if ( volumes ) { - for ( Map vol : volumes ) { - if ( vol.configMap == null ) continue - if ( (vol.configMap as Map)?.name == configMapName ) { - Map configMap = vol.configMap as Map - configMap.defaultMode = 0755 - break + if ( cwsK8sConfig.locationAwareScheduling() ) { + //Set default mode for configMap + Map specs = pod.spec as Map + List volumes = specs?.volumes as List + if (volumes) { + for (Map vol : volumes) { + if (vol.configMap == null) continue + if ((vol.configMap as Map)?.name == configMapName) { + Map configMap = vol.configMap as Map + configMap.defaultMode = 0755 + break + } } } } diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy index 98b83cd..e2eacbf 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/WOWK8sWrapperBuilder.groovy @@ -31,7 +31,7 @@ class WOWK8sWrapperBuilder extends CWSK8sWrapperBuilder { case 'copy': case 'ftp': if ( this.scratch == null || this.scratch == true ){ - //Reduce amount of local data + //Reduce amount of local data - only keep necessary outputs this.scratch = (storage.getWorkdir() as Path).resolve( "scratch" ).toString() this.stageOutMode = 'move' } @@ -48,7 +48,8 @@ class WOWK8sWrapperBuilder extends CWSK8sWrapperBuilder { @Override protected boolean shouldUnstageOutputs() { - return localWorkDir || super.shouldUnstageOutputs() + assert localWorkDir + return super.shouldUnstageOutputs() } private String getStorageLocalWorkDir() { @@ -84,12 +85,11 @@ class WOWK8sWrapperBuilder extends CWSK8sWrapperBuilder { @Override protected String getLaunchCommand(String interpreter, String env) { + assert storage && localWorkDir String cmd = '' - if( storage && localWorkDir ){ - cmd += "local INFILESTIME=\$(\"/etc/nextflow/${statFileName}\" infiles \"${workDir.toString()}/.command.infiles\" \"${getStorageLocalWorkDir()}\" \"\$PWD/\" || true)\n" - } + cmd += "local INFILESTIME=\$(\"/etc/nextflow/${statFileName}\" infiles \"${workDir.toString()}/.command.infiles\" \"${getStorageLocalWorkDir()}\" \"\$PWD/\" || true)\n" cmd += super.getLaunchCommand(interpreter, env) - if( storage && localWorkDir && isTraceRequired() ){ + if( isTraceRequired() ){ cmd += "\nlocal exitCode=\$?" cmd += """\necho \"infiles_time=\${INFILESTIME}" >> ${workDir.resolve(TaskRun.CMD_TRACE)}\n""" cmd += "return \$exitCode\n" @@ -99,13 +99,12 @@ class WOWK8sWrapperBuilder extends CWSK8sWrapperBuilder { @Override String getCleanupCmd(String scratch) { + assert storage && localWorkDir String cmd = super.getCleanupCmd( scratch ) - if( storage && localWorkDir ){ - cmd += "mkdir -p \"${localWorkDir.toString()}/\" || true\n" - cmd += "local OUTFILESTIME=\$(\"/etc/nextflow/${statFileName}\" outfiles \"${workDir.toString()}/.command.outfiles\" \"${getStorageLocalWorkDir()}\" \"${localWorkDir.toString()}/\" || true)\n" - if ( isTraceRequired() ) { - cmd += "echo \"outfiles_time=\${OUTFILESTIME}\" >> ${workDir.resolve(TaskRun.CMD_TRACE)}" - } + cmd += "mkdir -p \"${localWorkDir.toString()}/\" || true\n" + cmd += "local OUTFILESTIME=\$(\"/etc/nextflow/${statFileName}\" outfiles \"${workDir.toString()}/.command.outfiles\" \"${getStorageLocalWorkDir()}\" \"${localWorkDir.toString()}/\" || true)\n" + if ( isTraceRequired() ) { + cmd += "echo \"outfiles_time=\${OUTFILESTIME}\" >> ${workDir.resolve(TaskRun.CMD_TRACE)}" } return cmd } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy index 8ef494e..57cbe87 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/LocalPath.groovy @@ -40,7 +40,7 @@ class LocalPath implements Path, Serializable { if ( c.isAssignableFrom( getClass() ) ) return (T) this if ( c.isAssignableFrom( LocalPath.class ) ) return (T) toFile() if ( c == String.class ) return (T) toString() - log.info("Invoke method asType $c on ${this.class}") + log.debug("Invoke method asType $c on ${this.class}") return super.asType( c ) } diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystemProvider.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystemProvider.groovy index 4b4fbd8..057da10 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystemProvider.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystemProvider.groovy @@ -43,8 +43,10 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran ftpClient.setBinaryType() return ftpClient } catch ( IOException e ) { - if ( trial > 5 ) throw e - log.error("Cannot create FTP client: $daemon on $node", e) + if ( trial > 5 ) { + log.error("Cannot create FTP client: $daemon on $node", e) + throw e + } sleep(Math.pow(2, trial++) as long) daemon = schedulerClient.getDaemonOnNode(node) } @@ -111,22 +113,22 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran @Override String getScheme() { - return "wow" + "wow" } @Override FileSystem newFileSystem(URI uri, Map map) throws IOException { - return getFileSystem(uri) + getFileSystem(uri) } @Override FileSystem getFileSystem(URI uri) { - return WOWFileSystem.INSTANCE + WOWFileSystem.INSTANCE } @Override Path getPath(URI uri) { - return getFileSystem(uri).getPath(uri.path) + getFileSystem(uri).getPath(uri.path) } @Override @@ -165,12 +167,12 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran @Override boolean isSameFile(Path path1, Path path2) throws IOException { - return path1 == path2 + path1 == path2 } @Override boolean isHidden(Path path) throws IOException { - return path.getFileName().startsWith(".") + path.getFileName().startsWith(".") } @Override @@ -209,12 +211,12 @@ class WOWFileSystemProvider extends FileSystemProvider implements FileSystemTran @Override boolean canUpload(Path source, Path target) { - return false + false } @Override boolean canDownload(Path source, Path target) { - return true + true } @Override From 68127633b3f8287d1992f474e294d31a844149be Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 16 May 2025 17:20:36 +0200 Subject: [PATCH 53/63] fix: move location-awareness check to CWSConfig --- .../main/nextflow/cws/k8s/CWSK8sConfig.groovy | 16 +++++++--------- .../main/nextflow/cws/k8s/CWSK8sExecutor.groovy | 10 +++++----- .../nextflow/cws/k8s/CWSK8sTaskHandler.groovy | 14 +++++++++----- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy index 11dedc0..39a2057 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sConfig.groovy @@ -16,11 +16,13 @@ import java.util.stream.Collectors class CWSK8sConfig extends K8sConfig { private Map target + final boolean schedulingIsLocationAware - CWSK8sConfig(Map config) { + CWSK8sConfig(Map config, boolean isLocationAware) { super(config) this.target = config - if (getLocalPath()) { + this.schedulingIsLocationAware = isLocationAware + if ( getLocalPath() ) { final name = getLocalPath() final mount = getLocalStorageMountPath() getPodOptions().mountHostPaths.add(new PodHostMount(name, mount)) @@ -28,11 +30,11 @@ class CWSK8sConfig extends K8sConfig { } K8sScheduler getScheduler(){ - target.scheduler || locationAwareScheduling() ? new K8sScheduler( (Map)target.scheduler ) : null + target.scheduler || schedulingIsLocationAware ? new K8sScheduler( (Map)target.scheduler ) : null } Storage getStorage() { - locationAwareScheduling() ? new Storage((Map) target.storage, getLocalClaimPaths()) : null + schedulingIsLocationAware ? new Storage((Map) target.storage, getLocalClaimPaths()) : null } String getArchitecture() { @@ -47,10 +49,6 @@ class CWSK8sConfig extends K8sConfig { target.localStorageMountPath ?: '/workspace' as String } - boolean locationAwareScheduling() { - getLocalClaimPaths().size() > 0 - } - Collection getLocalClaimPaths() { getPodOptions().mountHostPaths.collect { it.mountPath } } @@ -62,7 +60,7 @@ class CWSK8sConfig extends K8sConfig { void checkStorageAndPaths(K8sClient client, String pipelineName) { super.checkStorageAndPaths(client) - if ( locationAwareScheduling() ) { + if ( schedulingIsLocationAware ) { //The nextflow project/workflow has to be on a shared drive if (pipelineName && pipelineName[0] == '/' && !findVolumeClaimByPath(pipelineName)) throw new AbortOperationException("Kubernetes `pipelineName` must be a path mounted as a persistent volume -- projectDir=$pipelineName; volumes=${getClaimPaths().join(', ')}") diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy index 533f11e..1c30f5d 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sExecutor.groovy @@ -48,7 +48,7 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { @Override @Memoized protected K8sConfig getK8sConfig() { - return new CWSK8sConfig( (Map)session.config.k8s ) + return new CWSK8sConfig( (Map)session.config.k8s, getCWSConfig().strategyIsLocationAware() ) } @Memoized @@ -100,15 +100,15 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { this.client = new CWSK8sClient(clientConfig) log.debug "[K8s] config=$k8sConfig; API client config=$clientConfig" + final CWSConfig cwsConfig = getCWSConfig() final CWSK8sConfig cwsK8sConfig = k8sConfig as CWSK8sConfig - if ( cwsK8sConfig.locationAwareScheduling() ) { + if ( cwsConfig.strategyIsLocationAware() ) { createDaemonSet() registerGetStatsConfigMap( cwsK8sConfig.getArchitecture() ) } CWSK8sConfig.K8sScheduler k8sSchedulerConfig = cwsK8sConfig.getScheduler() - final CWSConfig cwsConfig = new CWSConfig(session.config.navigate('cws') as Map) Map data if ( !k8sSchedulerConfig && !cwsConfig.dns ) { @@ -141,7 +141,7 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { workDir : session.workDir as String, localWorkDir : storage?.getWorkdir(), copyStrategy : storage?.getCopyStrategy(), - locationAware : cwsK8sConfig.locationAwareScheduling(), + locationAware : cwsConfig.strategyIsLocationAware(), maxCopyTasksPerNode : cwsConfig.getMaxCopyTasksPerNode(), maxWaitingCopyTasksPerNode : cwsConfig.getMaxWaitingCopyTasksPerNode() ] @@ -164,7 +164,7 @@ class CWSK8sExecutor extends K8sExecutor implements ExtensionPoint { schedulerClient.publishRemaining(); - if ( cwsK8sConfig.locationAwareScheduling() ) { + if ( getCWSConfig().strategyIsLocationAware() ) { int remaining do { remaining = schedulerClient.getRemainingToPublish() diff --git a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy index cb4a896..6ca3e0e 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/k8s/CWSK8sTaskHandler.groovy @@ -3,6 +3,7 @@ package nextflow.cws.k8s import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import nextflow.cws.CWSConfig import nextflow.cws.SchedulerClient import nextflow.executor.BashWrapperBuilder import nextflow.extension.GroupKey @@ -63,7 +64,8 @@ class CWSK8sTaskHandler extends K8sTaskHandler { (pod.spec as Map).schedulerName = cwsK8sConfig.getScheduler().getName() + "-" + getRunName() } - if ( cwsK8sConfig.locationAwareScheduling() ) { + final CWSConfig cwsConfig = executor.getCWSConfig() + if ( cwsConfig.strategyIsLocationAware() ) { //Set default mode for configMap Map specs = pod.spec as Map List volumes = specs?.volumes as List @@ -164,9 +166,10 @@ class CWSK8sTaskHandler extends K8sTaskHandler { @Override protected BashWrapperBuilder createBashWrapper(TaskRun task) { - CWSK8sConfig cwsK8sConfig = k8sConfig as CWSK8sConfig - if ( cwsK8sConfig?.locationAwareScheduling() ) { - return new WOWK8sWrapperBuilder( task , cwsK8sConfig.getStorage(), executor.getCWSConfig().memoryPredictor as boolean ) + final CWSConfig cwsConfig = executor.getCWSConfig() + final CWSK8sConfig cwsK8sConfig = k8sConfig as CWSK8sConfig + if ( cwsConfig.strategyIsLocationAware() ) { + return new WOWK8sWrapperBuilder( task , cwsK8sConfig.getStorage(), cwsConfig.memoryPredictor as boolean ) } return fusionEnabled() ? fusionLauncher() @@ -208,7 +211,8 @@ class CWSK8sTaskHandler extends K8sTaskHandler { @Override boolean checkIfCompleted() { Map state = getState() - if( !state || !state.terminated || ( (k8sConfig as CWSK8sConfig)?.locationAwareScheduling() && !schedulerPostProcessingHasFinished() ) ) { + final CWSConfig cwsConfig = executor.getCWSConfig() + if( !state || !state.terminated || ( cwsConfig.strategyIsLocationAware() && !schedulerPostProcessingHasFinished() ) ) { return false } if( executor.getCWSConfig().memoryPredictor ) { From e6199746aba8fbabb7e3499d20836a1a25ae141b Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 16 May 2025 17:25:25 +0200 Subject: [PATCH 54/63] ci: build c script in GitHub actions --- .github/workflows/build.yml | 21 +++++++++++++++++++++ .github/workflows/compile-c-script.yml | 6 ++++++ Makefile | 2 ++ 3 files changed, 29 insertions(+) create mode 100644 .github/workflows/compile-c-script.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 680cc42..5e8fd37 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,8 +9,29 @@ on: branches: - '*' jobs: + compile-c-binaries: + name: Compile C binaries + if: "!contains(github.event.head_commit.message, '[ci skip]')" + runs-on: ubuntu-latest + timeout-minutes: 4 + steps: + - name: Checkout source + uses: actions/checkout@v3 + with: + fetch-depth: 1 + submodules: true + + - name: Install dependencies for (cross-)compilation + run: | + sudo apt-get update + sudo apt-get install -y clang lld gcc-aarch64-linux-gnu libc6-dev-arm64-cross + + - name: Compile using make + run: make c_scripts + build: name: Build nf-cws + needs: compile-c-binaries if: "!contains(github.event.head_commit.message, '[ci skip]')" runs-on: ubuntu-latest timeout-minutes: 10 diff --git a/.github/workflows/compile-c-script.yml b/.github/workflows/compile-c-script.yml new file mode 100644 index 0000000..0b74647 --- /dev/null +++ b/.github/workflows/compile-c-script.yml @@ -0,0 +1,6 @@ +name: compile-c-script.yml + +jobs: + - name: Compile C scripts + if: steps.cache-c.outputs.cache-hit != 'true' + run: make c_scripts diff --git a/Makefile b/Makefile index dfd04d8..b80f6ee 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,8 @@ $(info - target x86_64-linux-gnu: saved to $(C_SCRIPT_X86_64)) $(error Aborting) endif +c_scripts: $(C_SCRIPT_ALL_TARGETS) + $(C_SCRIPT_AARCH64): $(C_SRIPT_SRC) clang -static \ -target aarch64-linux-gnu \ From d3df860fcd7aa2ef295b235f6e435d0841630c6b Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 16 May 2025 17:32:29 +0200 Subject: [PATCH 55/63] fix: nextflow version --- .github/workflows/build.yml | 4 ++++ plugins/nf-cws/build.gradle | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5e8fd37..d772ad7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,6 +57,10 @@ jobs: architecture: x64 distribution: 'temurin' + - name: Check files + run: | + ls -lisah plugins/nf-cws/src/resources/nf-cws/ + - name: Compile run: ./gradlew assemble diff --git a/plugins/nf-cws/build.gradle b/plugins/nf-cws/build.gradle index 3463a22..3d0aa6b 100644 --- a/plugins/nf-cws/build.gradle +++ b/plugins/nf-cws/build.gradle @@ -34,7 +34,7 @@ sourceSets { } ext{ - nextflowVersion = '24.11.0-edge' + nextflowVersion = '25.02.0-edge' } dependencies { From 83811fbaafd13cef9dd03a90ff169ffbfab29b8d Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 16 May 2025 17:36:22 +0200 Subject: [PATCH 56/63] fix: CI and newest Nextflow version --- .github/workflows/build.yml | 29 ++++++++--------------------- plugins/nf-cws/build.gradle | 2 +- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d772ad7..0f92702 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,29 +9,8 @@ on: branches: - '*' jobs: - compile-c-binaries: - name: Compile C binaries - if: "!contains(github.event.head_commit.message, '[ci skip]')" - runs-on: ubuntu-latest - timeout-minutes: 4 - steps: - - name: Checkout source - uses: actions/checkout@v3 - with: - fetch-depth: 1 - submodules: true - - - name: Install dependencies for (cross-)compilation - run: | - sudo apt-get update - sudo apt-get install -y clang lld gcc-aarch64-linux-gnu libc6-dev-arm64-cross - - - name: Compile using make - run: make c_scripts - build: name: Build nf-cws - needs: compile-c-binaries if: "!contains(github.event.head_commit.message, '[ci skip]')" runs-on: ubuntu-latest timeout-minutes: 10 @@ -50,6 +29,14 @@ jobs: fetch-depth: 1 submodules: true + - name: Install dependencies for C (cross-)compilation + run: | + sudo apt-get update + sudo apt-get install -y clang lld gcc-aarch64-linux-gnu libc6-dev-arm64-cross + + - name: Compile using make + run: make c_scripts + - name: Setup Java ${{ matrix.java_version }} uses: actions/setup-java@v3 with: diff --git a/plugins/nf-cws/build.gradle b/plugins/nf-cws/build.gradle index 3d0aa6b..ece34a5 100644 --- a/plugins/nf-cws/build.gradle +++ b/plugins/nf-cws/build.gradle @@ -34,7 +34,7 @@ sourceSets { } ext{ - nextflowVersion = '25.02.0-edge' + nextflowVersion = '25.04.2' } dependencies { From 20aa7404ce43ca47f80c1e18242170ba70ae995b Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 16 May 2025 17:47:35 +0200 Subject: [PATCH 57/63] fix: final CI and Nextflow version available on Maven --- .github/workflows/build.yml | 8 +++----- plugins/nf-cws/build.gradle | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0f92702..e9a3477 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,7 +35,9 @@ jobs: sudo apt-get install -y clang lld gcc-aarch64-linux-gnu libc6-dev-arm64-cross - name: Compile using make - run: make c_scripts + run: | + make c_scripts + ls -la plugins/nf-cws/src/resources/nf-cws - name: Setup Java ${{ matrix.java_version }} uses: actions/setup-java@v3 @@ -44,10 +46,6 @@ jobs: architecture: x64 distribution: 'temurin' - - name: Check files - run: | - ls -lisah plugins/nf-cws/src/resources/nf-cws/ - - name: Compile run: ./gradlew assemble diff --git a/plugins/nf-cws/build.gradle b/plugins/nf-cws/build.gradle index ece34a5..0ed4f34 100644 --- a/plugins/nf-cws/build.gradle +++ b/plugins/nf-cws/build.gradle @@ -34,7 +34,7 @@ sourceSets { } ext{ - nextflowVersion = '25.04.2' + nextflowVersion = '24.10.0' } dependencies { From 2f1ab0d44d0350bb5a9daf12fc67fef9f7affeb2 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 16 May 2025 18:10:40 +0200 Subject: [PATCH 58/63] fix: CI and tests --- .github/workflows/build.yml | 6 ++---- .github/workflows/compile-c-script.yml | 6 ------ .../nextflow/cws/wow/file/WOWFileAttributes.groovy | 12 +++++++++++- .../nextflow/cws/wow/file/WorkdirHelperTest.groovy | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) delete mode 100644 .github/workflows/compile-c-script.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e9a3477..7a76d1b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,10 +34,8 @@ jobs: sudo apt-get update sudo apt-get install -y clang lld gcc-aarch64-linux-gnu libc6-dev-arm64-cross - - name: Compile using make - run: | - make c_scripts - ls -la plugins/nf-cws/src/resources/nf-cws + - name: Compile C script using make + run: make c_scripts - name: Setup Java ${{ matrix.java_version }} uses: actions/setup-java@v3 diff --git a/.github/workflows/compile-c-script.yml b/.github/workflows/compile-c-script.yml deleted file mode 100644 index 0b74647..0000000 --- a/.github/workflows/compile-c-script.yml +++ /dev/null @@ -1,6 +0,0 @@ -name: compile-c-script.yml - -jobs: - - name: Compile C scripts - if: steps.cache-c.outputs.cache-hit != 'true' - run: make c_scripts diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy index 0046255..939fc30 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/file/WOWFileAttributes.groovy @@ -85,7 +85,17 @@ class WOWFileAttributes implements BasicFileAttributes { } WOWFileAttributes(Path path ) { - if (path.isDirectory()) { + if ( !path.exists() ) { + directory = true + link = false + size = 4096 + fileType = 'directory' + creationDate = FileTime.fromMillis( 0 ) + accessDate = FileTime.fromMillis( 0 ) + modificationDate = FileTime.fromMillis( 0 ) + destination = path + local = false + } else if ( path.isDirectory() ) { directory = true link = false size = 4096 diff --git a/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirHelperTest.groovy b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirHelperTest.groovy index a9a403c..f45eefd 100644 --- a/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirHelperTest.groovy +++ b/plugins/nf-cws/src/test/nextflow/cws/wow/file/WorkdirHelperTest.groovy @@ -34,7 +34,7 @@ class WorkdirHelperTest extends Specification { files.put( sharedPath7, localPath7 ) def helper = new WorkdirHelper( Path.of(localPath), files ) def workDir = Path.of("/input/data/work/be/8292aaebea2ddf9ae8ad4952882dcb") - def attributes = new WOWFileAttributes(workDir) + def attributes = new WOWFileAttributes(workDir) WorkdirPath path = new WorkdirPath( workDir, attributes, workDir, helper ) def stream = helper.getDirectoryStream( path ) def result = stream.collect() From e04bd5a52d934efa93717d6e1cbf7b05b0d1b16f Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 16 May 2025 18:22:41 +0200 Subject: [PATCH 59/63] fix: more one-line functions --- .../nextflow/cws/wow/filesystem/WOWFileSystem.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystem.groovy b/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystem.groovy index 502aaed..ac0a3dd 100644 --- a/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystem.groovy +++ b/plugins/nf-cws/src/main/nextflow/cws/wow/filesystem/WOWFileSystem.groovy @@ -14,7 +14,7 @@ class WOWFileSystem extends FileSystem { @Override FileSystemProvider provider() { - return WOWFileSystemProvider.INSTANCE + WOWFileSystemProvider.INSTANCE } @Override @@ -23,17 +23,17 @@ class WOWFileSystem extends FileSystem { @Override boolean isOpen() { - return true + true } @Override boolean isReadOnly() { - return false + false } @Override String getSeparator() { - return "/" + "/" } @Override @@ -48,7 +48,7 @@ class WOWFileSystem extends FileSystem { @Override Set supportedFileAttributeViews() { - return new HashSet<>() + new HashSet<>() } @Override @@ -59,7 +59,7 @@ class WOWFileSystem extends FileSystem { @Override @CompileDynamic PathMatcher getPathMatcher(String s) { - return new PathMatcher() { + new PathMatcher() { private final def matcher = FileSystems.getDefault().getPathMatcher( s ) @Override boolean matches(Path path) { From 84d43c66858fefe481c85c3c35895c90c65bccdc Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 16 May 2025 19:45:12 +0200 Subject: [PATCH 60/63] chore: add nf-hello changes from ee8371fb7c46b82d9e705a1b3281736aeaea3ad1 --- .../io.nextflow.groovy-common-conventions.gradle | 10 ++-------- gradle/wrapper/gradle-wrapper.properties | 2 +- plugins/nf-cws/build.gradle | 12 ++++++------ 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/buildSrc/src/main/groovy/io.nextflow.groovy-common-conventions.gradle b/buildSrc/src/main/groovy/io.nextflow.groovy-common-conventions.gradle index 70a6450..9fd33fa 100644 --- a/buildSrc/src/main/groovy/io.nextflow.groovy-common-conventions.gradle +++ b/buildSrc/src/main/groovy/io.nextflow.groovy-common-conventions.gradle @@ -16,15 +16,9 @@ java { toolchain { languageVersion = JavaLanguageVersion.of(21) } -} - -compileJava { - options.release.set(17) -} -tasks.withType(GroovyCompile) { - sourceCompatibility = '17' - targetCompatibility = '17' + sourceCompatibility = 17 + targetCompatibility = 17 } tasks.withType(Test) { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e411586..2733ed5 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/plugins/nf-cws/build.gradle b/plugins/nf-cws/build.gradle index 0ed4f34..c161373 100644 --- a/plugins/nf-cws/build.gradle +++ b/plugins/nf-cws/build.gradle @@ -34,21 +34,21 @@ sourceSets { } ext{ - nextflowVersion = '24.10.0' + nextflowVersion = '25.01.0-edge' } dependencies { // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" - compileOnly 'org.slf4j:slf4j-api:1.7.10' - compileOnly 'org.pf4j:pf4j:3.4.1' + compileOnly 'org.slf4j:slf4j-api:2.0.16' + compileOnly 'org.pf4j:pf4j:3.12.0' // add here plugins depepencies // test configuration - testImplementation "org.apache.groovy:groovy:4.0.18" - testImplementation "org.apache.groovy:groovy-nio:4.0.18" + testImplementation "org.apache.groovy:groovy:4.0.26" + testImplementation "org.apache.groovy:groovy-nio:4.0.26" testImplementation "io.nextflow:nextflow:$nextflowVersion" - testImplementation ("org.apache.groovy:groovy-test:4.0.18") { exclude group: 'org.apache.groovy' } + testImplementation ("org.apache.groovy:groovy-test:4.0.26") { exclude group: 'org.apache.groovy' } testImplementation ("cglib:cglib-nodep:3.3.0") testImplementation ("org.objenesis:objenesis:3.1") testImplementation ("org.spockframework:spock-core:2.3-groovy-4.0") { exclude group: 'org.apache.groovy'; exclude group: 'net.bytebuddy' } From 6211338c492b1fbd842769f8ff8f59c4104cc661 Mon Sep 17 00:00:00 2001 From: Friedrich Tschirpke Date: Fri, 16 May 2025 20:30:08 +0200 Subject: [PATCH 61/63] fix: update README with Nextflow versions --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5d0ccdb..5bd803a 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,14 @@ # nf-cws plugin This plugin enables Nextflow to communicate with a Common Workflow Scheduler instance and transfer the required -information. +information, which simplifies development of scheduling algorithms. + +Additionally, this plugin also provides scheduling algorithms for: +- dynamic task resizing For more information on the scheduling, see the [scheduler repository](https://github.com/CommonWorkflowScheduler/KubernetesScheduler) and the [respective -papers](#citation): +papers](#citation). ### Supported Executors @@ -13,7 +16,8 @@ papers](#citation): ### How to use -To run Nextflow with this plugin, you need version >=`23.03.0-edge`. +To run Nextflow with this plugin, you need version >=`24.04.0` and <=`25.02.3-edge` +(because Nextflow's Kubernetes refactor in `25.03.0-edge` is not supported *yet*). To activate the plugin, add `-plugins nf-cws` to your `nextflow` call or add the following to your `nextflow.config`: ``` @@ -187,4 +191,4 @@ Please note that the following strategies originated in individual papers: #### Acknowledgement: This work was funded by the German Research Foundation (DFG), CRC 1404: "FONDA: Foundations of Workflows for Large-Scale -Scientific Data Analysis." \ No newline at end of file +Scientific Data Analysis." From 9aa979754264f6dacf7aa9d48f0824b54f4a299c Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 16 May 2025 20:46:08 +0200 Subject: [PATCH 62/63] Update Readme Signed-off-by: Lehmann_Fabian --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5bd803a..3290918 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,15 @@ # nf-cws plugin This plugin enables Nextflow to communicate with a Common Workflow Scheduler instance and transfer the required -information, which simplifies development of scheduling algorithms. +information. -Additionally, this plugin also provides scheduling algorithms for: -- dynamic task resizing +Together with the Common Workflow Scheduler, the plugin enables you: +- to use more sophisticated scheduling strategies [(More information)](https://arxiv.org/pdf/2302.07652.pdf) +- automatically resize the memory of your memory if your estimation is too high [(More information)](https://arxiv.org/pdf/2408.00047.pdf) +- keep your intermediate data locally at the worker node - this saves 18% of makespan for RNA-Seq, and 95% of makespan for I/O intensive task chaining [(More information)](https://arxiv.org/pdf/2503.13072.pdf) For more information on the scheduling, -see the [scheduler repository](https://github.com/CommonWorkflowScheduler/KubernetesScheduler) and the [respective -papers](#citation). +see the [scheduler repository](https://github.com/CommonWorkflowScheduler/KubernetesScheduler). ### Supported Executors From 9e26f2f57aab9b79b8d7ed20ca832eb221d14ef5 Mon Sep 17 00:00:00 2001 From: Lehmann_Fabian Date: Fri, 16 May 2025 20:46:33 +0200 Subject: [PATCH 63/63] Release version 2.0.0 Signed-off-by: Lehmann_Fabian --- plugins/nf-cws/src/resources/META-INF/MANIFEST.MF | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-cws/src/resources/META-INF/MANIFEST.MF b/plugins/nf-cws/src/resources/META-INF/MANIFEST.MF index ace7073..9097f88 100644 --- a/plugins/nf-cws/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-cws/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Id: nf-cws -Plugin-Version: 1.0.6 +Plugin-Version: 2.0.0 Plugin-Class: nextflow.cws.CWSPlugin Plugin-Provider: nextflow Plugin-Requires: >=23.05.0-edge