diff --git a/.travis.yml b/.travis.yml index 9611dd4a53..a5ee33aa48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,8 +26,9 @@ before_script: - make assemble script: - | - set -e; + (set -e; make test install; - ./integration-tests.sh -after_script: - - if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then bash pub-tests.sh travis; fi + ./integration-tests.sh) + test_status=$? + if [[ "$test_status" != 0 && "$TRAVIS_PULL_REQUEST" == false ]]; then bash pub-tests.sh travis; fi + exit $test_status \ No newline at end of file diff --git a/docs/channel.rst b/docs/channel.rst index 9095e4c851..9af074b374 100644 --- a/docs/channel.rst +++ b/docs/channel.rst @@ -418,8 +418,11 @@ Binding values Since in `Nextflow` channels are implemented using `dataflow` variables or queues. Thus sending a message is equivalent to `bind` a value to object representing the communication channel. -bind( ) -------- + +.. _channel-bind1: + +bind +---- Channel objects provide a `bind( )` method which is the basic operation to send a message over the channel. For example:: @@ -428,10 +431,12 @@ For example:: myChannel.bind( 'Hello world' ) +.. _channel-bind2: + operator << ----------- -The operator ``<<`` is just a syntax sugar for the `bind( )` method. Thus, the following example produce +The operator ``<<`` is just a syntax sugar for the `bind` method. Thus, the following example produce an identical result as the previous one:: myChannel = Channel.create() @@ -445,10 +450,10 @@ Observing events .. _channel-subscribe: -subscribe( ) ------------- +subscribe +--------- -The ``subscribe( )`` method permits to execute a user define function each time a new value is emitted by the source channel. +The ``subscribe`` method permits to execute a user define function each time a new value is emitted by the source channel. The emitted value is passed implicitly to the specified function. For example:: @@ -489,10 +494,10 @@ Read :ref:`script-closure` paragraph to learn more about `closure` feature. onNext, onComplete, and onError ------------------------------- -The ``subscribe()`` method may accept one or more of the following event handlers: +The ``subscribe`` method may accept one or more of the following event handlers: * ``onNext``: registers a function that is invoked whenever the channel emits a value. - This is the same as using the ``subscribe( )`` with a `plain` closure as describe in the examples above. + This is the same as using the ``subscribe`` with a `plain` closure as describe in the examples above. * ``onComplete``: registers a function that is invoked after the `last` value is emitted by the channel. @@ -505,19 +510,14 @@ For example:: Channel .from( 1, 2, 3 ) - .subscribe onNext: { println it }, onComplete: { println 'Done.' } + .subscribe onNext: { println it }, onComplete: { println 'Done' } :: 1 2 3 - Done. - - -.. Special messages -.. STOP -.. VOID + Done diff --git a/docs/dsl2.rst b/docs/dsl2.rst new file mode 100644 index 0000000000..6d89dd384e --- /dev/null +++ b/docs/dsl2.rst @@ -0,0 +1,330 @@ +.. _dsl2-page: + +****** +DSL 2 +****** + +Nextflow implements an experimental syntax that implements new features and enhancements that +simplifies the implementation of data analysis applications. + +To enable this feature you need to defined the following directive at the beginning of +your workflow script:: + + nextflow.preview.dsl=2 + + +.. warning:: THIS IS AN EXPERIMENT FEATURE UNDER DEVELOPMENT. SYNTAX MAY CHANGE IN FUTURE RELEASE. + + +Function +======== + +Nextflow allows the definition of custom function in the workflow script using the following syntax:: + + def ( arg1, arg, .. ) { + + } + +For example:: + + def foo() { + 'Hello world' + } + + def bar(alpha, omega) { + alpha + omega + } + + +The above snippet defines two simple functions, that can be invoked in the workflow script as `foo()` which +returns the ``Hello world`` string and ``bar(10,20)`` which return the sum of two parameters. + +.. tip:: Functions implicitly return the result of the function last evaluated statement. + +The keyword ``return`` can be used to explicitly exit from a function returning the specified value. +for example:: + + def fib( x ) { + if( x <= 1 ) + return x + else + fib(x-1) + fib(x-2) + } + +Process +======= + +Process definition +------------------ + +The new DSL separates the definition of a process by its invocation. The process definition follows the usual +for syntax as described in the :ref:`process documentation `. The only difference is that the +``from`` and ``into`` channel declaration has to be omitted. + +Then processes can be invoked as a function in the ``workflow`` scope, passing the expected +input channels as parameters. + +For example:: + + nextflow.preview.dsl=2 + + process foo { + output: + file 'foo.txt' + script: + """ + your_command > foo.txt + """ + } + + process bar { + input: + file x + output: + file 'bar.txt' + script: + """ + your_command > bar.txt + """ + } + + workflow { + data = Channel.fromPath('/some/path/*.txt') + foo() + bar(data) + } + + +Process invocation +------------------ + + +Process composition +------------------- + +Processes having matching input-output declaration can be composed so that the output +of the first process is passed as input to the following process. Take in consideration +the previous process definition, it's possible to write the following:: + + workflow { + foo(bar()) + } + +Process outputs +--------------- + +A process output can also be accessed using the ``output`` attribute for the respective +process object. For example:: + + workflow { + foo() + bar( foo.output ) + bar.output.println() + } + + +When a process defines two or more output channels, each of them can be accessed +using the array element operator e.g. ``output[0]``, etc or using the ``first``, ``second``, etc +sub-properties e.g. ``output.first``. + +Workflow +======== + +Workflow definition +-------------------- + +The ``workflow`` keyword allows the definition of sub-workflow components that enclose the +invocation of two or more processes or operators. For example:: + + workflow my_pipeline { + foo() + bar( foo.output.collect() ) + } + + +Once defined it can be invoked from another (sub) workflow component definition. + +Workflow parameters +------------------- + +A workflow component can be define one or more parameter in a similar manner as for a function +definition. For example:: + + workflow my_pipeline( data ) { + foo() + bar( data.mix( foo.output ) ) + } + +The result channel of the last evaluated process is implicitly returned as the workflow output. + + +Main workflow +------------- + +A workflow definition which does not define any name is assumed to be the main workflow and it's +implicitly executed. Therefore it's the entry point of the workflow application. + +Modules +======= + +The new DSL allows the definition module scripts that +can be included and shared across workflow applications. + +A module can contain the definition of function, process and workflow definitions +as described above. + +Modules include +--------------- + +A module script can be included from another Nextflow script using the ``include`` keyword. +Then it's possible to reference of components (eg. functions, processes and workflow ) defined in the module +from the importing script. + +For example:: + + nextflow.preview.dsl=2 + + include 'modules/libx' + + workflow { + data = Channel.fromPath('/some/data/*.txt') + my_pipeline(data) + } + +Nextflow implicitly looks for the module script ``modules/libx.nf`` resolving the path +against the main script location. + +Selective inclusion +------------------- + +The module inclusion implicitly imports all the components defined in the module script. +It's possible to selective include only a specific component by its name using the +inclusion extended syntax as shown below:: + + nextflow.preview.dsl=2 + + include my_pipeline from 'modules/libx' + + workflow { + data = Channel.fromPath('/some/data/*.txt') + my_pipeline(data) + } + +The module component can be included using a name alias as shown below:: + + + nextflow.preview.dsl=2 + + include my_pipeline as my_tool from 'modules/libx' + + workflow { + data = Channel.fromPath('/some/data/*.txt') + my_tool(data) + } + + +Module aliases +-------------- + +When including a module component it's possible to specify a name alias. +This allows the import and the invocation of the same component multiple times +in your script using different names. For example:: + + nextflow.preview.dsl=2 + + include foo from 'modules/my-library' + include for as bar from 'modules/my-library' + + workflow { + foo(some_data) + bar(other_data) + } + + +Module parameters +----------------- + +A module script can define one or more parameters as any other Nextflow script.:: + + params.foo = 'hello' + params.bar = 'world' + + def sayHello() { + "$params.foo $params.bar" + } + + +Then, parameters can be specified when the module is imported with the ``include`` statement:: + + + nextflow.preview.dsl=2 + + include 'modules/library.nf' params(foo: 'Hola', bar: 'mundo') + + + +Channel forking +=============== + +Using the new DSL Nextflow channels are automatically forked when connecting two or more consumers. +This means that, for example, a process output can be used by two or more processes without the +need to fork them using the :ref:`operator-into` operator, making the writing of workflow script +much fluent and readable. + +Pipes +===== + +Nextflow processes and operators can be composed using the ``|`` *pipe* operator. For example:: + + process foo { + input: val data + output: val result + exec: + result = "$data mundo" + } + + workflow { + Channel.from('Hello','world') | foo + } + + +The above snippet defines a process named ``foo`` then invoke it passing the content of the +``data`` channel. + +The ``&`` *and* operator allow the feed of two or more processes with the content of the same +channel e.g.:: + + process foo { + input: val data + output: val result + exec: + result = "$data mundo" + } + + process bar { + input: val data + output: val result + exec: + result = data.toUpperCase() + } + + + workflow { + Channel.from('Hello') | map { it.reverse() } | (foo & bar) + } + + +Deprecated methods and operators +================================ + +The following methods are not allowed any more when using Nextflow DSL 2: + +* :ref:`channel-create` +* :ref:`channel-bind1` +* :ref:`channel-bind2` +* :ref:`operator-close` +* :ref:`operator-countby` +* :ref:`operator-route` +* :ref:`operator-separate` +* :ref:`operator-into` +* :ref:`operator-merge` diff --git a/docs/index.rst b/docs/index.rst index bff0b3c260..42649a6f8c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,6 +19,7 @@ Contents: operator executor config + dsl2 awscloud amazons3 google diff --git a/docs/metrics.rst b/docs/metrics.rst index 0d646f3240..7a628fd2e8 100644 --- a/docs/metrics.rst +++ b/docs/metrics.rst @@ -1,4 +1,3 @@ - .. _metrics-page: ******* diff --git a/docs/operator.rst b/docs/operator.rst index 2ae404ec5a..10210f896e 100644 --- a/docs/operator.rst +++ b/docs/operator.rst @@ -1684,6 +1684,7 @@ the others into ``queue2`` queue1.subscribe { println it } +.. _operator-separate: separate ------------ @@ -1774,6 +1775,8 @@ The output will look like the following fragment:: See also: `into`_, `choice`_ and `map`_ operators. +.. _operator-route: + route ----- @@ -1868,6 +1871,8 @@ a literal value, a Java class, or a `boolean predicate` that needs to be satisfi // -> 4 +.. _operator-countby: + countBy ---------- diff --git a/modules/nextflow/src/main/groovy/nextflow/Channel.groovy b/modules/nextflow/src/main/groovy/nextflow/Channel.groovy index 4ddf49ca8e..60c430e0a2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Channel.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Channel.groovy @@ -28,6 +28,7 @@ import groovy.transform.PackageScope import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowChannel import groovyx.gpars.dataflow.DataflowQueue +import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowVariable import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.ControlMessage @@ -35,6 +36,7 @@ import groovyx.gpars.dataflow.operator.PoisonPill import nextflow.dag.NodeMarker import nextflow.datasource.SraExplorer import nextflow.exception.AbortOperationException +import nextflow.extension.ChannelFactory import nextflow.extension.GroupTupleOp import nextflow.extension.MapOp import nextflow.file.DirWatcher @@ -61,13 +63,20 @@ class Channel { // only for testing purpose ! private static CompletableFuture fromPath0Future + static private Session getSession() { Global.session as Session } + + /** * Create an new channel * * @return The channel instance */ - static DataflowChannel create() { - new DataflowQueue() + @Deprecated + static DataflowChannel create() { + if( NF.isDsl2() ) + throw new DeprecationException("Channel `create` method is not supported any more") + log.warn("The channel `create` method is deprecated -- it will be removed in a future release") + new DataflowQueue() } /** @@ -75,8 +84,8 @@ class Channel { * * @return The channel instance */ - static DataflowChannel empty() { - def result = new DataflowQueue() + static DataflowChannel empty() { + final result = new DataflowQueue() result.bind(STOP) NodeMarker.addSourceNode('Channel.empty', result) return result @@ -88,20 +97,42 @@ class Channel { * @param items * @return */ - static DataflowChannel from( Collection items ) { - final result = Nextflow.channel(items) + static DataflowWriteChannel from( Collection items ) { + final result = from0(items) NodeMarker.addSourceNode('Channel.from', result) return result } + static private DataflowWriteChannel from0( Collection items ) { + final result = ChannelFactory.create() + if( NF.isDsl2() ) { + session.addIgniter { bindValues0(items, result) } + } + else { + bindValues0(items, result) + } + return result + } + + static private void bindValues0(Collection values, DataflowWriteChannel result) { + if ( values != null ) { + // bind all the items in the provided collection + for( def item : values ) { + result.bind(item) + } + // bind a stop signal to 'terminate' the result + result.bind(Channel.STOP) + } + } + /** * Creates a channel sending the items in the collection over it * * @param items * @return */ - static DataflowChannel from( Object... items ) { - final result = Nextflow.channel(items) + static DataflowWriteChannel from( Object... items ) { + final result = from0(items as List) NodeMarker.addSourceNode('Channel.from', result) return result } @@ -132,10 +163,8 @@ class Channel { * @param duration * @return */ - static DataflowChannel interval(String duration) { - - final result = interval( duration, { index -> index }) - + static DataflowWriteChannel interval(String duration) { + final result = interval0( duration, { index -> index }) NodeMarker.addSourceNode('Channel.interval', result) return result } @@ -149,11 +178,16 @@ class Channel { * @return */ - static DataflowChannel interval(String duration, Closure closure ) { + static DataflowWriteChannel interval(String duration, Closure closure ) { + final result = interval0(duration, closure) + NodeMarker.addSourceNode('Channel.interval', result) + return result + } + static private DataflowWriteChannel interval0(String duration, Closure closure) { def millis = Duration.of(duration).toMillis() def timer = new Timer() - def result = create() + def result = ChannelFactory.create() long index = 0 def task = { @@ -164,12 +198,14 @@ class Channel { } } - timer.schedule( task as TimerTask, millis ) - - NodeMarker.addSourceNode('Channel.interval', result) + if( NF.isDsl2() ) + session.addIgniter { timer.schedule( task as TimerTask, millis ) } + else + timer.schedule( task as TimerTask, millis ) + return result } - + /* * valid parameters for fromPath operator */ @@ -194,7 +230,7 @@ class Channel { * @return * A channel emitting the matching files */ - static DataflowChannel fromPath( Map opts = null, pattern ) { + static DataflowWriteChannel fromPath( Map opts = null, pattern ) { if( !pattern ) throw new AbortOperationException("Missing `fromPath` parameter") // verify that the 'type' parameter has a valid value @@ -205,20 +241,29 @@ class Channel { return result } - private static DataflowChannel fromPath0( Map opts, List allPatterns ) { + private static DataflowWriteChannel fromPath0( Map opts, List allPatterns ) { - final result = new DataflowQueue() + final result = ChannelFactory.create() + if( NF.isDsl2() ) { + session.addIgniter { pumpFiles0(result, opts, allPatterns) } + } + else { + pumpFiles0(result, opts, allPatterns) + } + return result + } + + private static void pumpFiles0(DataflowWriteChannel result, Map opts, List allPatterns) { + def future = CompletableFuture.completedFuture(null) for( int index=0; index result.bind(file.toAbsolutePath()) } + static private DataflowWriteChannel watchImpl( String syntax, String folder, String pattern, boolean skipHidden, String events, FileSystem fs ) { + + final result = ChannelFactory.create() + final watcher = new DirWatcher(syntax,folder,pattern,skipHidden,events, fs) + .setOnComplete { result.bind(STOP) } + + if( NF.isDsl2() ) { + session.addIgniter { + watcher.apply { Path file -> result.bind(file.toAbsolutePath()) } + } + } + else { + watcher.apply { Path file -> result.bind(file.toAbsolutePath()) } + } return result } @@ -254,7 +307,7 @@ class Channel { * @return A dataflow channel that will emit the matching files * */ - static DataflowChannel watchPath( Pattern filePattern, String events = 'create' ) { + static DataflowWriteChannel watchPath( Pattern filePattern, String events = 'create' ) { assert filePattern // split the folder and the pattern final splitter = FilePatternSplitter.regex().parse(filePattern.toString()) @@ -281,7 +334,7 @@ class Channel { * @return A dataflow channel that will emit the matching files * */ - static DataflowChannel watchPath( String filePattern, String events = 'create' ) { + static DataflowWriteChannel watchPath( String filePattern, String events = 'create' ) { if( filePattern.endsWith('/') ) filePattern += '*' @@ -298,7 +351,7 @@ class Channel { return result } - static DataflowChannel watchPath( Path path, String events = 'create' ) { + static DataflowWriteChannel watchPath( Path path, String events = 'create' ) { final fs = path.getFileSystem() final splitter = FilePatternSplitter.glob().parse(path.toString()) final folder = splitter.parent @@ -325,7 +378,7 @@ class Channel { * @return * A channel emitting the file pairs matching the specified pattern(s) */ - static DataflowChannel fromFilePairs(Map options = null, pattern) { + static DataflowWriteChannel fromFilePairs(Map options = null, pattern) { final allPatterns = pattern instanceof List ? pattern : [pattern] final allGrouping = new ArrayList(allPatterns.size()) for( int i=0; i grouping) { + private static DataflowWriteChannel fromFilePairs0(Map options, List allPatterns, List grouping) { assert allPatterns.size() == grouping.size() if( !allPatterns ) throw new AbortOperationException("Missing `fromFilePairs` parameter") if( !grouping ) throw new AbortOperationException("Missing `fromFilePairs` grouping parameter") - boolean anyPattern=false // -- a channel from the path - final fromOpts = fetchParams(VALID_FROM_PATH_PARAMS, options) + final fromOpts = fetchParams0(VALID_FROM_PATH_PARAMS, options) final files = new DataflowQueue() - def future = CompletableFuture.completedFuture(null) - for( int index=0; index def prefix = grouping[index].call(path) return [ prefix, path ] } - def mapChannel = new MapOp(files, mapper).apply() + def mapChannel = (DataflowReadChannel)new MapOp(files, mapper) + .setTarget(new DataflowQueue()) + .apply() - // -- result the files having the same ID + boolean anyPattern=false + for( int index=0; index() : ChannelFactory.create() + + new GroupTupleOp(groupOpts, mapChannel) + .setTarget(groupChannel) + .apply() // -- flat the group resulting tuples - DataflowChannel result - if( options?.flat == true ) { + DataflowWriteChannel result + if( isFlat ) { def makeFlat = { id, List items -> def tuple = new ArrayList(items.size()+1); tuple.add(id) tuple.addAll(items) return tuple } - result = new MapOp(groupChannel,makeFlat).apply() + result = new MapOp((DataflowReadChannel)groupChannel,makeFlat).apply() } else { result = groupChannel @@ -420,7 +478,19 @@ class Channel { return result } - static private Map fetchParams( Map valid, Map actual ) { + static private void pumpFilePairs0(DataflowWriteChannel files, Map fromOpts, List allPatterns) { + def future = CompletableFuture.completedFuture(null) + for( int index=0; index + */ +class NF { + + static private Session session() { + return (Session)Global.session + } + + static void init() { + ChannelFactory.init() + WorkflowBinding.init() + } + + static boolean isDsl2() { + NextflowMeta.instance.isDsl2() + } + + static Binding getBinding() { + isDsl2() ? ExecutionStack.binding() : session().getBinding() + } + + static String lookupVariable(value) { + if( isDsl2() ) + return WorkflowBinding.lookup(value) + return session().getBinding().getVariableName(value) + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy b/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy index d9905d49ac..48cfc34a7a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy @@ -43,6 +43,8 @@ import static nextflow.file.FileHelper.isGlobAllowed */ class Nextflow { + static private Session getSession() { Global.session as Session } + // note: groovy `Slf4j` annotation causes a bizarre issue // https://issues.apache.org/jira/browse/GROOVY-7371 // declare public so that can be accessed from the user script @@ -226,6 +228,11 @@ class Nextflow { * @param message The message that will be reported in the log file (optional) */ static void exit(int exitCode, String message = null) { + if( session.aborted ) { + log.debug "Ignore exit because execution is already aborted -- message=$message" + return + } + if ( exitCode && message ) { log.error message } diff --git a/modules/nextflow/src/main/groovy/nextflow/NextflowMeta.groovy b/modules/nextflow/src/main/groovy/nextflow/NextflowMeta.groovy new file mode 100644 index 0000000000..d570337c1a --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/NextflowMeta.groovy @@ -0,0 +1,61 @@ +package nextflow + +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString +import groovy.util.logging.Slf4j +import nextflow.util.VersionNumber + +/** + * Models nextflow script properties and metadata + * + * @author Paolo Di Tommaso + */ +@Singleton(strict = false) +@ToString(includeNames = true) +@EqualsAndHashCode +class NextflowMeta { + + static public final Closure JSON_CONVERTER = { meta, key -> key=='timestamp' ? new Date(Const.APP_TIMESTAMP) : meta.getProperty(key) } + + @Slf4j + static class Preview { + float dsl + + void setDsl( float num ) { + if( num != 2 && num != 1 ) + throw new IllegalArgumentException("Not a valid DSL version number: $num") + if( num == 2 ) + log.warn "DSL 2 IS AN EXPERIMENTAL FEATURE UNDER DEVELOPMENT -- SYNTAX MAY CHANGE IN FUTURE RELEASE" + dsl = num + } + } + + final VersionNumber version + final int build + final String timestamp + final Preview preview = new Preview() + + private NextflowMeta() { + version = new VersionNumber(Const.APP_VER) + build = Const.APP_BUILDNUM + timestamp = Const.APP_TIMESTAMP_UTC + } + + protected NextflowMeta(String ver, int build, String timestamp ) { + this.version = new VersionNumber(ver) + this.build = build + this.timestamp = timestamp + } + + boolean isDsl2() { + preview.dsl == 2 + } + + void enableDsl2() { + preview.dsl = 2 + } + + void disableDsl2() { + preview.dsl = 1 + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index ffe0f44da9..eb3eeb4578 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -26,6 +26,7 @@ import java.util.concurrent.Executors import com.google.common.hash.HashCode import com.upplication.s3fs.S3OutputStream +import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.transform.Memoized import groovy.transform.PackageScope @@ -39,15 +40,21 @@ import nextflow.exception.AbortOperationException import nextflow.exception.AbortSignalException import nextflow.exception.IllegalConfigException import nextflow.exception.MissingLibraryException +import nextflow.executor.ExecutorFactory +import nextflow.extension.ChannelFactory import nextflow.file.FileHelper import nextflow.file.FilePorter import nextflow.processor.ErrorStrategy -import nextflow.processor.ProcessConfig -import nextflow.processor.TaskDispatcher import nextflow.processor.TaskFault import nextflow.processor.TaskHandler import nextflow.processor.TaskProcessor +import nextflow.script.BaseScript +import nextflow.script.ProcessConfig +import nextflow.script.ProcessFactory import nextflow.script.ScriptBinding +import nextflow.script.ScriptFile +import nextflow.script.ScriptRunner +import nextflow.script.WorkflowMetadata import nextflow.trace.AnsiLogObserver import nextflow.trace.GraphObserver import nextflow.trace.ReportObserver @@ -66,7 +73,6 @@ import nextflow.util.NameGenerator import sun.misc.Signal import sun.misc.SignalHandler import static nextflow.Const.S3_UPLOADER_CLASS - /** * Holds the information on the current execution * @@ -81,10 +87,17 @@ class Session implements ISession { */ final Collection allOperators = new ConcurrentLinkedQueue<>() + final List igniters = new ArrayList<>(10) + + /** + * Creates process executors + */ + ExecutorFactory executorFactory + /** - * Dispatch tasks for executions + * Script binding */ - TaskDispatcher dispatcher + ScriptBinding binding /** * Holds the configuration object @@ -126,6 +139,10 @@ class Session implements ISession { */ String scriptClassName + Class scriptClass + + BaseScript script + /** * Mnemonic name of this run instance */ @@ -141,6 +158,10 @@ class Session implements ISession { */ List configFiles + String profile + + String commandLine + /** * Local path where script generated classes are saved */ @@ -175,20 +196,18 @@ class Session implements ISession { private volatile Throwable error - private ScriptBinding binding - - private ClassLoader classLoader - private Queue> shutdownCallbacks = new ConcurrentLinkedQueue<>() private int poolSize - private List observers + private List observers = Collections.emptyList() private Closure errorAction private boolean statsEnabled + private WorkflowMetadata workflowMetadata + private WorkflowStats workflowStats private FilePorter filePorter @@ -209,6 +228,8 @@ class Session implements ISession { WorkflowStats getWorkflowStats() { workflowStats } + WorkflowMetadata getWorkflowMetadata() { workflowMetadata } + Path getClassesDir() { classesDir } boolean ansiLog @@ -223,26 +244,18 @@ class Session implements ISession { * Creates a new session with an 'empty' (default) configuration */ Session() { - create(new ScriptBinding([:])) + create(new LinkedHashMap(10)) } - /** - * Create a new session given the {@link ScriptBinding} object - * - * @param binding - */ - Session(ScriptBinding binding) { - create(binding) - } /** * Create a new session given the configuration specified * * @param config */ - Session(Map cfg) { - final config = cfg instanceof ConfigObject ? cfg.toMap() : cfg - create(new ScriptBinding(config)) + Session(Map obj) { + final config = obj instanceof ConfigObject ? obj.toMap() : obj + create(config) } /** @@ -255,11 +268,6 @@ class Session implements ISession { */ int getPoolSize() { poolSize } - /** - * @return The session {@link TaskDispatcher} - */ - TaskDispatcher getDispatcher() { dispatcher } - CacheDB getCache() { cache } /** @@ -267,17 +275,19 @@ class Session implements ISession { * * @param binding */ - private void create( ScriptBinding binding ) { - assert binding != null + private void create( Map config ) { + assert config != null - this.binding = binding - this.config = binding.config + this.config = config this.dumpHashes = config.dumpHashes this.dumpChannels = (List)config.dumpChannels + this.binding = new ScriptBinding() // -- poor man session object dependency injection Global.setSession(this) Global.setConfig(config) + // -- init static structs + NF.init() // -- cacheable flag cacheable = config.cacheable @@ -309,11 +319,8 @@ class Session implements ISession { this.poolSize = config.poolSize as int log.debug "Executor pool size: ${poolSize}" - // -- create the task dispatcher instance - this.dispatcher = new TaskDispatcher(this) - - // -- DGA object - this.dag = new DAG(session:this) + // -- DAG object + this.dag = new DAG() // -- init work dir this.workDir = ((config.workDir ?: 'work') as Path).complete() @@ -321,12 +328,13 @@ class Session implements ISession { // -- file porter config this.filePorter = new FilePorter(this) + } /** * Initialize the session workDir, libDir, baseDir and scriptName variables */ - void init( Path scriptPath ) { + Session init( ScriptFile scriptFile, List args=null ) { if(!workDir.mkdirs()) throw new AbortOperationException("Cannot create work-dir: $workDir -- Make sure you have write permissions or specify a different directory by using the `-w` command line option") log.debug "Work-dir: ${workDir.toUriString()} [${FileHelper.getPathFsType(workDir)}]" @@ -336,20 +344,36 @@ class Session implements ISession { log.debug "Bucket-dir: ${bucketDir.toUriString()}" } - if( scriptPath ) { + if( scriptFile ) { // the folder that contains the main script - this.setBaseDir(scriptPath.parent) + this.setBaseDir(scriptFile.main.parent) // set the script name attribute - this.setScriptName(scriptPath.name) + this.setScriptName(scriptFile.main.name) } // set the byte-code target directory this.classesDir = FileHelper.createLocalDir() - + this.executorFactory = new ExecutorFactory() this.observers = createObservers() this.statsEnabled = observers.any { it.enableMetrics() } + this.workflowMetadata = new WorkflowMetadata(this, scriptFile) + + // configure script params + binding.setParams( (Map)config.params ) + binding.setArgs( new ScriptRunner.ArgsList(args) ) cache = new CacheDB(uniqueId,runName).open() + + return this + } + + Session setBinding(ScriptBinding binding ) { + this.binding = binding + return this + } + + ProcessFactory newProcessFactory(BaseScript script) { + new ProcessFactory(script, this) } /** @@ -486,6 +510,30 @@ class Session implements ISession { Signal.handle( new Signal("HUP"), abort_h) } + void addIgniter( Closure action ) { + igniters.add(action) + } + + void fireDataflowNetwork() { + if( !NextflowMeta.instance.isDsl2() ) + return + + // bridge any dataflow queue into a broadcast channel + ChannelFactory.broadcast() + + log.debug "Ignite dataflow network (${igniters.size()})" + for( def action : igniters ) { + try { + action.call() + } + catch( Exception e ) { + log.error(e.message ?: "Failed to trigger dataflow network", e) + abort(e) + break + } + } + } + /** * Dump the current dataflow network listing * the status of active processes and operators @@ -502,7 +550,6 @@ class Session implements ISession { } } - Session start() { log.debug "Session start invoked" @@ -511,8 +558,6 @@ class Session implements ISession { // create tasks executor execService = Executors.newFixedThreadPool(poolSize) - // signal start to tasks dispatcher - dispatcher.start() // signal start to trace observers observers.each { trace -> trace.onFlowStart(this) } @@ -521,11 +566,22 @@ class Session implements ISession { ScriptBinding getBinding() { binding } - ClassLoader getClassLoader() { classLoader } + @Memoized + ClassLoader getClassLoader() { getClassLoader0() } - Session setClassLoader( ClassLoader loader ) { - this.classLoader = loader - return this + @PackageScope + ClassLoader getClassLoader0() { + // extend the class-loader if required + final gcl = new GroovyClassLoader() + final libraries = ConfigHelper.resolveClassPaths(getLibDir()) + + for( Path lib : libraries ) { + def path = lib.complete() + log.debug "Adding to the classpath library: ${path}" + gcl.addClasspath(path.toString()) + } + + return gcl } Barrier getBarrier() { monitorsBarrier } @@ -588,6 +644,14 @@ class Session implements ISession { return libDir } + Map getConfigEnv() { + if( !config.env ) + return Collections.emptyMap() + if( config.env instanceof Map ) + return new LinkedHashMap((Map)config.env) + throw new IllegalStateException("Not a valid config env object: $config.env") + } + @Memoized Manifest getManifest() { if( !config.manifest ) @@ -717,7 +781,7 @@ class Session implements ISession { log.info "Execution cancelled -- Finishing pending tasks before exit" cancelled = true notifyError(handler) - dispatcher.signal() + executorFactory.signalExecutors() processesBarrier.forceTermination() allOperators *. terminate() } @@ -739,7 +803,7 @@ class Session implements ISession { log.debug(status) // force termination notifyError(null) - dispatcher.signal() + executorFactory.signalExecutors() processesBarrier.forceTermination() monitorsBarrier.forceTermination() operatorsForceTermination() @@ -935,15 +999,18 @@ class Session implements ISession { } } - void notifyTaskCached( TaskHandler handler ) { - // -- save a record in the cache index - cache.cacheTaskAsync(handler) + final trace = handler.getTraceRecord() + // save a record in the cache index only the when the trace record is available + // otherwise it means that the event is trigger by a `stored dir` driven task + if( trace ) { + cache.cacheTaskAsync(handler) + } for( int i=0; i(System.getenv()) } + + @CompileDynamic + def fetchContainers() { + + def result = [:] + if( config.process instanceof Map ) { + + /* + * look for `container` definition at process level + */ + config.process.each { String name, value -> + if( name.startsWith('$') && value instanceof Map && value.container ) { + result[name] = resolveClosure(value.container) + } + } + + /* + * default container definition + */ + def container = config.process.container + if( container ) { + if( result ) { + result['default'] = resolveClosure(container) + } + else { + result = resolveClosure(container) + } + } + + } + + return result + } + + /** + * Resolve dynamically defined attributes to the actual value + * + * @param val A process container definition either a plain string or a closure + * @return The actual container value + */ + protected String resolveClosure( val ) { + if( val instanceof Closure ) { + try { + return val.cloneWith(binding).call() + } + catch( Exception e ) { + log.debug "Unable to resolve dynamic `container` directive -- cause: ${e.message ?: e}" + return "(dynamic resolved)" + } + } + + return String.valueOf(val) + } + + /** * Defines the number of tasks the executor will handle in a parallel manner * diff --git a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy index 836a5fb777..5a667c85da 100644 --- a/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy @@ -19,6 +19,7 @@ package nextflow.ast import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.script.BaseScript +import nextflow.script.IncludeDef import nextflow.script.TaskBody import nextflow.script.TaskClosure import nextflow.script.TokenEnvCall @@ -31,11 +32,12 @@ import nextflow.script.TokenVar import org.codehaus.groovy.ast.ASTNode import org.codehaus.groovy.ast.ClassCodeVisitorSupport import org.codehaus.groovy.ast.ClassNode -import org.codehaus.groovy.ast.ConstructorNode +import org.codehaus.groovy.ast.MethodNode import org.codehaus.groovy.ast.Parameter import org.codehaus.groovy.ast.VariableScope import org.codehaus.groovy.ast.expr.ArgumentListExpression import org.codehaus.groovy.ast.expr.BinaryExpression +import org.codehaus.groovy.ast.expr.CastExpression import org.codehaus.groovy.ast.expr.ClosureExpression import org.codehaus.groovy.ast.expr.ConstantExpression import org.codehaus.groovy.ast.expr.ConstructorCallExpression @@ -59,6 +61,7 @@ import org.codehaus.groovy.syntax.SyntaxException import org.codehaus.groovy.syntax.Types import org.codehaus.groovy.transform.ASTTransformation import org.codehaus.groovy.transform.GroovyASTTransformation +import static org.codehaus.groovy.ast.tools.GeneralUtils.constX /** * Implement some syntax sugars of Nextflow DSL scripting. * @@ -70,11 +73,21 @@ import org.codehaus.groovy.transform.GroovyASTTransformation @GroovyASTTransformation(phase = CompilePhase.CONVERSION) class NextflowDSLImpl implements ASTTransformation { + static private Set RESERVED_NAMES + + static { + // method names implicitly defined by the groovy script SHELL + RESERVED_NAMES = ['main','run','runScript'] as Set + // existing method cannot be used for custom script definition + for( def method : BaseScript.getMethods() ) { + RESERVED_NAMES.add(method.name) + } + + } + @Override void visit(ASTNode[] astNodes, SourceUnit unit) { - createVisitor(unit).visitClass((ClassNode)astNodes[1]) - } /* @@ -95,74 +108,43 @@ class NextflowDSLImpl implements ASTTransformation { private Set processNames = [] - protected SourceUnit getSourceUnit() { unit } + private Set workflowNames = [] - DslCodeVisitor(SourceUnit unit) { - this.unit = unit - } + private Set functionNames = [] - /** - * Creates a statement that invokes the {@link nextflow.script.BaseScript#init(java.util.List)} method - * used to initialize the script with metadata collected during script parsing - * @return The method invocation statement - */ - protected Statement invokeBaseScriptInit() { - final names = new ListExpression() - processNames.each { names.addExpression(new ConstantExpression(it.toString())) } + private int anonymousWorkflow - // the method list argument - final args = new ArgumentListExpression() - args.addExpression(names) + protected SourceUnit getSourceUnit() { unit } - final call = new MethodCallExpression(new VariableExpression('this'), 'init', args) - final stm = new ExpressionStatement(call) - return stm - } - /** - * Add to constructor a method call to inject parsed metadata - * @param node - */ - protected void injectMetadata(ClassNode node) { - for( ConstructorNode constructor : node.getDeclaredConstructors() ) { - def code = constructor.getCode() - if( code instanceof BlockStatement ) { - code.addStatement( invokeBaseScriptInit() ) - } - else if( code instanceof ExpressionStatement ) { - def expr = code - def block = new BlockStatement() - block.addStatement(expr) - block.addStatement( invokeBaseScriptInit() ) - constructor.setCode(block) - } - else - throw new IllegalStateException("Invalid constructor expression: $code") - } + DslCodeVisitor(SourceUnit unit) { + this.unit = unit } @Override - protected void visitObjectInitializerStatements(ClassNode node) { - if( node.getSuperClass().getName() == BaseScript.getName() ) { - injectMetadata(node) + void visitMethod(MethodNode node) { + if( node.public && !node.static && !node.synthetic && !node.metaDataMap?.'org.codehaus.groovy.ast.MethodNode.isScriptBody') { + if( !isIllegalName(node.name, node)) + functionNames.add(node.name) } - super.visitObjectInitializerStatements(node) + super.visitMethod(node) } + @Override void visitMethodCallExpression(MethodCallExpression methodCall) { // pre-condition to be verified to apply the transformation - Boolean preCondition = methodCall.objectExpression?.getText() == 'this' + final preCondition = methodCall.objectExpression?.getText() == 'this' + final methodName = methodCall.getMethodAsString() /* * intercept the *process* method in order to transform the script closure */ - if( preCondition && methodCall.getMethodAsString() == 'process' ) { + if( methodName == 'process' && preCondition ) { // clear block label currentLabel = null - // keep track of 'process' method (it may be removed) - currentTaskName = methodCall.getMethodAsString() + currentTaskName = methodName try { convertProcessDef(methodCall,sourceUnit) super.visitMethodCallExpression(methodCall) @@ -171,6 +153,10 @@ class NextflowDSLImpl implements ASTTransformation { currentTaskName = null } } + else if( methodName == 'workflow' && preCondition ) { + convertWorkflowDef(methodCall,sourceUnit) + super.visitMethodCallExpression(methodCall) + } // just apply the default behavior else { @@ -179,6 +165,167 @@ class NextflowDSLImpl implements ASTTransformation { } + @Override + void visitExpressionStatement(ExpressionStatement stm) { + if( stm.text.startsWith('this.include(') && stm.getExpression() instanceof MethodCallExpression ) { + final methodCall = (MethodCallExpression)stm.getExpression() + convertIncludeDef(methodCall) + // this is necessary to invoke the `load` method on the include definition + final loadCall = new MethodCallExpression(methodCall, 'load', new ArgumentListExpression()) + stm.setExpression(loadCall) + } + super.visitExpressionStatement(stm) + } + + protected void convertIncludeDef(MethodCallExpression call) { + if( call.methodAsString=='include' && call.arguments instanceof ArgumentListExpression ) { + final allArgs = (ArgumentListExpression)call.arguments + if( allArgs.size() != 1 ) { + syntaxError(call, "Not a valid include definition -- it must specify the module path") + return + } + + final arg = allArgs[0] + final newArgs = new ArgumentListExpression() + if( arg instanceof ConstantExpression ) { + newArgs.addExpression( newObj(IncludeDef, arg) ) + } + else if( arg instanceof VariableExpression ) { + // the name of the component i.e. process, workflow, etc to import + final component = arg.getName() + // wrap the name in a `TokenVar` type + final token = newObj(TokenVar, new ConstantExpression(component)) + // create a new `IncludeDef` object + newArgs.addExpression(newObj(IncludeDef, token)) + } + else if( arg instanceof CastExpression && arg.getExpression() instanceof VariableExpression) { + def cast = (CastExpression)arg + // the name of the component i.e. process, workflow, etc to import + final component = (cast.expression as VariableExpression).getName() + // wrap the name in a `TokenVar` type + final token = newObj(TokenVar, new ConstantExpression(component)) + // the alias to give it + final alias = constX(cast.type.name) + newArgs.addExpression( newObj(IncludeDef, token, alias) ) + } + else { + syntaxError(call, "Not a valid include definition -- it must specify the module path as a string") + return + } + call.setArguments(newArgs) + } + else if( call.objectExpression instanceof MethodCallExpression ) { + convertIncludeDef((MethodCallExpression)call.objectExpression) + } + } + + /* + * this method transforms the DSL definition + * + * workflow foo (ch1, ch2) { + * code + * } + * + * into a method invocation as + * + * workflow('foo', ['ch1':ch1, 'ch2':ch2], { -> code }) + * + */ + protected void convertWorkflowDef(MethodCallExpression methodCall, SourceUnit unit) { + log.trace "Convert 'workflow' ${methodCall.arguments}" + + assert methodCall.arguments instanceof ArgumentListExpression + def args = (ArgumentListExpression)methodCall.arguments + def len = args.size() + + // anonymous workflow definition + if( len == 1 && args[0] instanceof ClosureExpression ) { + if( anonymousWorkflow++ > 0 ) { + unit.addError( new SyntaxException("Duplicate entry workflow definition", methodCall.lineNumber, methodCall.columnNumber+8)) + return + } + + def newArgs = new ArgumentListExpression() + def body = (ClosureExpression)args[0] + newArgs.addExpression( makeWorkflowBodyWrapper(body) ) + methodCall.setArguments( newArgs ) + return + } + + // extract the first argument which has to be a method-call expression + // the name of this method represent the *workflow* name + if( len != 1 || !args[0].class.isAssignableFrom(MethodCallExpression) ) { + log.debug "Missing name in workflow definition at line: ${methodCall.lineNumber}" + unit.addError( new SyntaxException("Workflow definition syntax error -- A string identifier must be provided after the `workflow` keyword", methodCall.lineNumber, methodCall.columnNumber+8)) + return + } + + final nested = args[0] as MethodCallExpression + final name = nested.getMethodAsString() + // check the process name is not defined yet + if( isIllegalName(name, methodCall) ) { + return + } + workflowNames.add(name) + + // the nested method arguments are the arguments to be passed + // to the process definition, plus adding the process *name* + // as an extra item in the arguments list + args = (ArgumentListExpression)nested.getArguments() + len = args.size() + log.trace "Workflow name: $name with args: $args" + + // make sure to add the 'name' after the map item + // (which represent the named parameter attributes) + def newArgs = new ArgumentListExpression() + + // add the workflow body def + if( len == 0 || !(args[len-1] instanceof ClosureExpression)) { + syntaxError(methodCall, "Invalid workflow definition -- Missing definition block") + return + } + + final body = (ClosureExpression)args[len-1] + newArgs.addExpression( makeWorkflowBodyWrapper(body) ) + newArgs.addExpression( constX(name) ) + newArgs.addExpression( makeArgsList(args) ) + + // set the new list as the new arguments + methodCall.setArguments( newArgs ) + } + + protected Expression makeWorkflowBodyWrapper( ClosureExpression closure ) { + // make a copy to clear closure implicit `it` parameter + def copy = new ClosureExpression(null, closure.code) + def buffer = new StringBuilder() + def block = (BlockStatement) closure.code + for( Statement stm : block.getStatements() ) + readSource(stm, buffer, unit) + makeScriptWrapper(copy, buffer.toString(), 'workflow', unit) + } + + protected void syntaxError(ASTNode node, String message) { + int line = node.lineNumber + int coln = node.columnNumber + unit.addError( new SyntaxException(message,line,coln)) + } + + protected ListExpression makeArgsList(ArgumentListExpression args) { + def result = new ArrayList() + for( int i=0; i statements, StringBuilder source, BlockStatement parent ) { @@ -828,6 +975,18 @@ class NextflowDSLImpl implements ASTTransformation { return false } + protected boolean isIllegalName(String name, ASTNode node) { + if( name in RESERVED_NAMES ) { + unit.addError( new SyntaxException("Identifier `$name` is reserved for internal use", node.lineNumber, node.columnNumber+8) ) + return true + } + if( name in functionNames || name in workflowNames || name in processNames ) { + unit.addError( new SyntaxException("Identifier `$name` is already used by another definition", node.lineNumber, node.columnNumber+8) ) + return true + } + return false + } + /** * This method handle the process definition, so that it transform the user entered syntax * process myName ( named: args, .. ) { code .. } @@ -839,7 +998,7 @@ class NextflowDSLImpl implements ASTTransformation { * @param unit */ protected void convertProcessDef( MethodCallExpression methodCall, SourceUnit unit ) { - log.trace "Converts 'process' ${methodCall.arguments} " + log.trace "Converts 'process' ${methodCall.arguments}" assert methodCall.arguments instanceof ArgumentListExpression def list = (methodCall.arguments as ArgumentListExpression).getExpressions() @@ -848,17 +1007,17 @@ class NextflowDSLImpl implements ASTTransformation { // the name of this method represent the *process* name if( list.size() != 1 || !list[0].class.isAssignableFrom(MethodCallExpression) ) { log.debug "Missing name in process definition at line: ${methodCall.lineNumber}" - unit.addError( new SyntaxException("Process definition syntax error -- You must provide a string identifier after the `process` keyword", methodCall.lineNumber, methodCall.columnNumber+7)) + unit.addError( new SyntaxException("Process definition syntax error -- A string identifier must be provided after the `process` keyword", methodCall.lineNumber, methodCall.columnNumber+7)) return } def nested = list[0] as MethodCallExpression def name = nested.getMethodAsString() // check the process name is not defined yet - if( !processNames.add(name) ) { - unit.addError( new SyntaxException("Process `$name` is already define", methodCall.lineNumber, methodCall.columnNumber+8) ) + if( isIllegalName(name, methodCall) ) { return } + processNames.add(name) // the nested method arguments are the arguments to be passed // to the process definition, plus adding the process *name* @@ -912,6 +1071,8 @@ class NextflowDSLImpl implements ASTTransformation { private boolean declaration + private int deep + VariableVisitor( SourceUnit unit ) { this.sourceUnit = unit } @@ -928,6 +1089,12 @@ class NextflowDSLImpl implements ASTTransformation { return target instanceof VariableExpression } + @Override + void visitClosureExpression(ClosureExpression expression) { + if( deep++ == 0 ) + super.visitClosureExpression(expression) + } + @Override void visitDeclarationExpression(DeclarationExpression expr) { declaration = true diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy index 9cdd5e7ded..46e23619af 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy @@ -227,7 +227,8 @@ class CmdRun extends CmdBase implements HubOptions { // -- create a new runner instance final runner = new ScriptRunner(config) runner.script = scriptFile - runner.profile = profile + runner.session.profile = profile + runner.session.commandLine = launcher.cliString runner.session.ansiLog = launcher.options.ansiLog if( this.test ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy index 69b9fe6437..b8cb4cb471 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy @@ -25,7 +25,6 @@ import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.util.logging.Slf4j -import nextflow.Nextflow import nextflow.cloud.aws.AmazonCloudDriver import nextflow.exception.AbortOperationException import nextflow.executor.Executor @@ -51,22 +50,22 @@ class AwsBatchExecutor extends Executor { * Proxy to throttle AWS batch client requests */ @PackageScope - private static AwsBatchProxy client + private AwsBatchProxy client /** * executor service to throttle service requests */ - private static ThrottlingExecutor submitter + private ThrottlingExecutor submitter /** * Executor service to throttle cancel requests */ - private static ThrottlingExecutor reaper + private ThrottlingExecutor reaper /** * A S3 path where executable scripts need to be uploaded */ - private static Path remoteBinDir = null + private Path remoteBinDir = null /** * @return {@code true} to signal containers are managed directly the AWS Batch service @@ -84,7 +83,7 @@ class AwsBatchExecutor extends Executor { * Initialise the AWS batch executor. */ @Override - void register() { + protected void register() { super.register() /* diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy index a5eb2c2a5c..866813ea2e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/dag/DAG.groovy @@ -16,13 +16,18 @@ package nextflow.dag +import groovy.transform.MapConstructor import groovy.transform.PackageScope import groovy.transform.ToString import groovy.util.logging.Slf4j -import groovyx.gpars.dataflow.DataflowChannel +import groovyx.gpars.dataflow.DataflowBroadcast +import groovyx.gpars.dataflow.DataflowQueue +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression import groovyx.gpars.dataflow.operator.DataflowProcessor -import nextflow.Session +import nextflow.NF +import nextflow.extension.ChannelFactory import nextflow.extension.DataflowHelper import nextflow.processor.TaskProcessor import nextflow.script.DefaultInParam @@ -60,11 +65,6 @@ class DAG { */ private List vertices = new ArrayList<>(50) - /** - * The {@link Session} to which this DAG is bound - */ - private Session session - @PackageScope List getVertices() { vertices } @@ -94,7 +94,7 @@ class DAG { * @param inputs The operator input(s). It can be either a single channel or a list of channels. * @param outputs The operator output(s). It can be either a single channel, a list of channels or {@code null} if the operator has no output. */ - public void addOperatorNode( String label, inputs, outputs, List operators=null ) { + void addOperatorNode( String label, inputs, outputs, List operators=null ) { assert label assert inputs addVertex(Type.OPERATOR, label, normalizeChannels(inputs), normalizeChannels(outputs), operators ) @@ -125,11 +125,11 @@ class DAG { final vertex = createVertex( type, label, extra ) - inbounds?.each { ChannelHandler channel -> + for( ChannelHandler channel : inbounds ) { inbound( vertex, channel ) } - outbounds?.each { ChannelHandler channel -> + for( ChannelHandler channel : outbounds ) { outbound( vertex, channel ) } } @@ -173,7 +173,7 @@ class DAG { } // handle the special case for dataflow variable // this kind of channel can be used more than one time as an input - else if( entering.channel instanceof DataflowExpression ) { + else if( isForkable(entering.channel) ) { if( !edge.from ) { edge.from = new Vertex(Type.ORIGIN); int p = vertices.indexOf(edge.to) @@ -191,6 +191,14 @@ class DAG { } } + private boolean isForkable(obj) { + if( obj instanceof DataflowExpression ) + return true + if( obj instanceof DataflowBroadcast ) + return true + return obj instanceof DataflowQueue && ChannelFactory.isBridge(obj) + } + private void outbound( Vertex vertex, ChannelHandler leaving) { // look for an existing edge for the given dataflow channel @@ -214,7 +222,7 @@ class DAG { inputs .findAll { !( it instanceof DefaultInParam) } - .collect { InParam p -> new ChannelHandler(channel: (DataflowChannel)p.inChannel, label: inputName0(p)) } + .collect { InParam p -> new ChannelHandler(channel: p.rawChannel, label: inputName0(p)) } } @@ -230,7 +238,7 @@ class DAG { outputs.each { OutParam p -> if( p instanceof DefaultOutParam ) return p.outChannels.each { - result << new ChannelHandler(channel: (DataflowChannel)it, label: p instanceof SetOutParam ? null : p.name) + result << new ChannelHandler(channel: it, label: p instanceof SetOutParam ? null : p.name) } } @@ -241,11 +249,11 @@ class DAG { if( entry == null ) { Collections.emptyList() } - else if( entry instanceof DataflowChannel ) { + else if( entry instanceof DataflowReadChannel || entry instanceof DataflowWriteChannel ) { [ new ChannelHandler(channel: entry) ] } else if( entry instanceof Collection || entry instanceof Object[] ) { - entry.collect { new ChannelHandler(channel: (DataflowChannel)it) } + entry.collect { new ChannelHandler(channel: it) } } else { throw new IllegalArgumentException("Not a valid channel type: [${entry.class.name}]") @@ -253,7 +261,7 @@ class DAG { } @PackageScope - Edge findEdge( DataflowChannel channel ) { + Edge findEdge( channel ) { edges.find { edge -> edge.channel.is(channel) } } @@ -283,16 +291,20 @@ class DAG { } @PackageScope - void resolveEdgeNames(Map map) { - edges.each { Edge edge -> - def name = resolveChannelName(map, edge.channel) - if( name ) edge.label = name + void resolveEdgeNames() { + for( Edge edge : edges ) { + final name = lookupVariable(edge.channel) + if( name ) + edge.label = name } } + @PackageScope String lookupVariable(obj) { + NF.lookupVariable(obj) + } @PackageScope - String resolveChannelName( Map map, DataflowChannel channel ) { + String resolveChannelName( Map map, channel ) { def entry = map.find { k,v -> v.is channel } return entry ? entry.key : null } @@ -300,15 +312,12 @@ class DAG { @PackageScope String getChannelName( ChannelHandler handler ) { def result = handler.label - result ?: (session ? resolveChannelName( session.getBinding().getVariables(), handler.channel ) : null ) + result ?: NF.lookupVariable(handler.channel) } void normalize() { normalizeMissingVertices() - if( session ) - resolveEdgeNames(session.getBinding().getVariables()) - else - log.debug "Missing session object -- Cannot normalize edge names" + resolveEdgeNames() } /** @@ -406,12 +415,13 @@ class DAG { */ @PackageScope @ToString(includeNames = true, includes = 'label,from,to', includePackage=false) + @MapConstructor class Edge { /** - * The {@link groovyx.gpars.dataflow.DataflowChannel} that originated this graph edge + * The Dataflow channel that originated this graph edge */ - DataflowChannel channel + Object channel /** * The vertex *from* where the edge starts @@ -439,7 +449,7 @@ class DAG { /** * The {@link groovyx.gpars.dataflow.DataflowChannel} that originated this graph edge */ - DataflowChannel channel + Object channel /** * The edge label diff --git a/modules/nextflow/src/main/groovy/nextflow/dag/NodeMarker.groovy b/modules/nextflow/src/main/groovy/nextflow/dag/NodeMarker.groovy index 8609d758bb..2619b962b6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/dag/NodeMarker.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/dag/NodeMarker.groovy @@ -30,7 +30,7 @@ import nextflow.script.OutputsList @CompileStatic class NodeMarker { - static private List operators = [] + static private List operators = new ArrayList<>(10) static void appendOperator( DataflowProcessor p ) { operators.add(p) @@ -58,8 +58,10 @@ class NodeMarker { * @param outputs The operator output(s). It can be either a single channel, a list of channels or {@code null} if the operator has no output. */ static void addOperatorNode( String name, inputs, outputs ) { - if( session && session.dag && !session.aborted ) - session.dag.addOperatorNode(name, inputs, outputs, operators) + if( session && session.dag && !session.aborted ) { + session.dag.addOperatorNode(name, inputs, outputs, new ArrayList(operators)) + operators.clear() + } } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateModuleIncludeException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateModuleIncludeException.groovy new file mode 100644 index 0000000000..07917a5532 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/exception/DuplicateModuleIncludeException.groovy @@ -0,0 +1,13 @@ +package nextflow.exception + +import groovy.transform.InheritConstructors + +/** + * Exception thrown when is included in a script more than + * a module component with the same name + * + * @author Paolo Di Tommaso + */ +@InheritConstructors +class DuplicateModuleIncludeException extends ProcessException { +} diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/IllegalInvocationException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/IllegalInvocationException.groovy new file mode 100644 index 0000000000..bbf33ed3b1 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/exception/IllegalInvocationException.groovy @@ -0,0 +1,28 @@ +package nextflow.exception + +import nextflow.script.ComponentDef +import nextflow.script.ProcessDef +import nextflow.script.WorkflowDef + +/** + * Exception thrown when a module component is invoked + * in a wrong context eg when invoking a process outside a workflow scope + * + * @author Paolo Di Tommaso + */ +class IllegalInvocationException extends ProcessException { + + IllegalInvocationException(ComponentDef component) { + super(message(component)) + } + + static private String message(ComponentDef component) { + if( component instanceof WorkflowDef ) + return "Workflow '$component.name' can only be invoked from workflow context" + + if( component instanceof ProcessDef ) + return "Process '$component.name' can only be invoked from a workflow context" + + return "Invalid invocation context: $component.name" + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/exception/MissingModuleComponentException.groovy b/modules/nextflow/src/main/groovy/nextflow/exception/MissingModuleComponentException.groovy new file mode 100644 index 0000000000..369986c080 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/exception/MissingModuleComponentException.groovy @@ -0,0 +1,25 @@ +package nextflow.exception + +import groovy.transform.PackageScope +import nextflow.script.ScriptMeta + +/** + * Exception thrown when a module component cannot be found + * + * @author Paolo Di Tommaso + */ +class MissingModuleComponentException extends ProcessException { + + MissingModuleComponentException(ScriptMeta meta, String name ) { + super(message(meta, name)) + } + + @PackageScope + static String message(ScriptMeta meta, String name) { + def result = "Cannot find a component with name '$name' in module: $meta.scriptPath" + def names = meta.getDefinitions().collect { it.name } + def matches = names.closest(name) + result += "\n\nDid you mean one of these?\n" + matches.collect { " $it"}.join('\n') + return result + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/AbstractGridExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/AbstractGridExecutor.groovy index dc83469e62..0c56b9e8ea 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/AbstractGridExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/AbstractGridExecutor.groovy @@ -46,8 +46,9 @@ abstract class AbstractGridExecutor extends Executor { /** * Initialize the executor class */ - void init() { - super.init() + @Override + protected void register () { + super.register() queueInterval = session.getQueueStatInterval(name) log.debug "Creating executor '$name' > queue-stat-interval: ${queueInterval}" } @@ -56,14 +57,14 @@ abstract class AbstractGridExecutor extends Executor { * Create a a queue holder for this executor * @return */ - def TaskMonitor createTaskMonitor() { + TaskMonitor createTaskMonitor() { return TaskPollingMonitor.create(session, name, 100, Duration.of('5 sec')) } /* * Prepare and launch the task in the underlying execution platform */ - def GridTaskHandler createTaskHandler(TaskRun task) { + GridTaskHandler createTaskHandler(TaskRun task) { assert task assert task.workDir diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/Executor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/Executor.groovy index 511fb1b44f..213959a1cb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/Executor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/Executor.groovy @@ -17,6 +17,7 @@ package nextflow.executor import java.nio.file.Path +import java.util.concurrent.CountDownLatch import groovy.transform.PackageScope import groovy.util.logging.Slf4j @@ -51,36 +52,55 @@ abstract class Executor { */ private TaskMonitor monitor - private static final Map registerFlag = [:] - - /** - * Template method executed once for executor class - */ - void register() { } + TaskMonitor getMonitor() { monitor } protected String getDisplayName() { name } /** * Allows to post-initialize the executor */ - void init() { - log.debug "Initializing executor: $name" + final void init() { + log.info "[warm up] executor > ${getDisplayName()}" + monitor = createTaskMonitor() + monitor.start() + register() + } + + protected void register() { } - // -- skip if already assigned, this is only for testing purpose - if( monitor ) - return + void signal() { + monitor.signal() + } + + /** + * Submit the specified task for execution to the underlying system + * and add it to the queue of tasks to be monitored. + * + * @param task A {@code TaskRun} instance + */ + final void submit( TaskRun task, boolean awaitTermination ) { + log.trace "Scheduling process: ${task}" - // -- get the reference to the monitor class for this executor - monitor = session.dispatcher.getOrCreateMonitor(this.class) { - log.info "[warm up] executor > ${getDisplayName()}" - createTaskMonitor() + if( session.isTerminated() ) { + new IllegalStateException("Session terminated - Cannot add process to execution queue: ${task}") } - // call the register template method - if( !registerFlag[this.class] ) { - log.debug "Invoke register for executor: ${name}" - register() - registerFlag[this.class] = true + final handler = createTaskHandler(task) + + // set a count down latch if the execution is blocking + if( awaitTermination ) + handler.latch = new CountDownLatch(1) + + /* + * Add the task to the queue for processing + * Note: queue is implemented as a fixed size blocking queue, when + * there's not space *put* operation will block until, some other tasks finish + */ + monitor.schedule(handler) + if( handler && handler.latch ) { + log.trace "Process ${task} > blocking" + handler.latch.await() + log.trace "Process ${task} > complete" } } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/ExecutorFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/ExecutorFactory.groovy new file mode 100644 index 0000000000..ef216150ab --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/executor/ExecutorFactory.groovy @@ -0,0 +1,203 @@ +/* + * Copyright 2013-2018, Centre for Genomic Regulation (CRG) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.executor + +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic +import groovy.transform.PackageScope +import groovy.util.logging.Slf4j +import nextflow.Session +import nextflow.cloud.aws.batch.AwsBatchExecutor +import nextflow.k8s.K8sExecutor +import nextflow.script.ProcessConfig +import nextflow.script.ScriptType +import nextflow.script.TaskBody +import nextflow.util.ServiceDiscover +import nextflow.util.ServiceName +/** + * Helper class to create {@link Executor} objects + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class ExecutorFactory { + + static public String DEFAULT_EXECUTOR = System.getenv('NXF_EXECUTOR') ?: 'local' + + /* + * Map the executor class to its 'friendly' name + */ + final static Map> BUILT_IN_EXECUTORS = [ + 'nope': NopeExecutor, + 'local': LocalExecutor, + 'sge': SgeExecutor, + 'oge': SgeExecutor, + 'uge': SgeExecutor, + 'lsf': LsfExecutor, + 'pbs': PbsExecutor, + 'pbspro': PbsProExecutor, + 'slurm': SlurmExecutor, + 'crg': CrgExecutor, + 'bsc': LsfExecutor, + 'condor': CondorExecutor, + 'k8s': K8sExecutor, + 'nqsii': NqsiiExecutor, + 'awsbatch': AwsBatchExecutor + ] + + @PackageScope Map> executorsMap + + private Map,? extends Executor> executors = new HashMap<>() + + ExecutorFactory() { + executorsMap = new HashMap(20) + // add built-in executors + executorsMap.putAll(BUILT_IN_EXECUTORS) + // discover non-core executors + for( Class clazz : ServiceDiscover.load(Executor) ) { + log.trace "Discovered executor class: ${clazz.name}" + executorsMap.put(findNameByClass(clazz), clazz) + } + } + + /** + * Extract the executor name by using the annotation {@code ServiceName} or fallback to simple classname + * if the annotation is not provided + * + * @param clazz + * @return + */ + static String findNameByClass( Class clazz ) { + def annotation = clazz.getAnnotation(ServiceName) + if( annotation ) + return annotation.value() + + def name = clazz.getSimpleName().toLowerCase() + if( name.endsWith('executor') ) { + name = name.subSequence(0, name.size()-'executor'.length()) + } + + return name + } + + protected Class getExecutorClass(String executorName) { + log.debug ">> processorType: '$executorName'" + if( !executorName ) + return LocalExecutor + + def clazz = executorsMap[executorName.toLowerCase()] + if( !clazz ) + throw new IllegalArgumentException("Unknown executor name: $executorName") + + if( clazz instanceof Class ) + return clazz + + if( !(clazz instanceof String ) ) + throw new IllegalArgumentException("Not a valid executor class object: $clazz") + + // if the className is empty (because the 'processorType' does not map to any class, fallback to the 'processorType' itself) + if( !clazz ) { + clazz = executorName + } + + log.debug "Loading executor class: ${clazz}" + try { + Thread.currentThread().getContextClassLoader().loadClass(clazz as String) as Class + } + catch( Exception e ) { + throw new IllegalArgumentException("Cannot find a valid class for specified executor: '${executorName}'") + } + + } + + + protected boolean isTypeSupported( ScriptType type, executor ) { + + if( executor instanceof Executor ) { + executor = executor.class + } + + if( executor instanceof Class ) { + def annotation = executor.getAnnotation(SupportedScriptTypes) + if( !annotation ) + throw new IllegalArgumentException("Specified argument is not a valid executor class: $executor -- Missing 'SupportedScriptTypes' annotation") + + return type in annotation.value() + } + + throw new IllegalArgumentException("Specified argument is not a valid executor class: $executor") + } + + Executor getExecutor(String processName, ProcessConfig processConfig, TaskBody script, Session session ) { + // -- load the executor to be used + def name = getExecutorName(processConfig,session) ?: DEFAULT_EXECUTOR + def clazz = getExecutorClass(name) + + if( !isTypeSupported(script.type, clazz) ) { + log.warn "Process '$processName' cannot be executed by '$name' executor -- Using 'local' executor instead" + name = 'local' + clazz = LocalExecutor.class + } + + // this code is not supposed to be executed parallel + def result = executors.get(clazz) + if( result ) + return result + + result = createExecutor(clazz, name, session) + executors.put(clazz, result) + return result + } + + protected Executor createExecutor( Class clazz, String name, Session session) { + def result = clazz.newInstance() + result.session = session + result.name = name + result.init() + return result + } + + /** + * Find out the 'executor' to be used in the process definition or in teh session configuration object + * + * @param taskConfig + */ + @CompileDynamic + protected String getExecutorName(ProcessConfig taskConfig, Session session) { + // create the processor object + def result = taskConfig.executor?.toString() + + if( !result ) { + if( session.config.executor instanceof String ) { + result = session.config.executor + } + else if( session.config.executor?.name instanceof String ) { + result = session.config.executor.name + } + } + + log.debug "<< taskConfig executor: $result" + return result + } + + void signalExecutors() { + for( Executor exec : executors.values() ) + exec.signal() + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/StoredTaskHandler.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/StoredTaskHandler.groovy new file mode 100644 index 0000000000..3bd1887121 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/executor/StoredTaskHandler.groovy @@ -0,0 +1,51 @@ +package nextflow.executor + +import nextflow.processor.TaskHandler +import nextflow.processor.TaskRun +import nextflow.trace.TraceRecord + +/** + * Implements a {@link TaskHandler} instance for nextflow stored task ie. + * tasks whose execution is skipped due the use of the `storeDir` directive. + * + * @author Paolo Di Tommaso + */ +class StoredTaskHandler extends TaskHandler { + + StoredTaskHandler(TaskRun task) { + super(task) + } + + @Override + boolean checkIfRunning() { + return false + } + + @Override + boolean checkIfCompleted() { + return true + } + + @Override + void kill() { + throw new UnsupportedOperationException() + } + + @Override + void submit() { + throw new UnsupportedOperationException() + } + + @Override + String getStatusString() { + "STORED" + } + + /** + * @return Stored tasks are not supposed to have a trace record, therefore returns {@code null} + */ + @Override + TraceRecord getTraceRecord() { + return null + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy index ab07d8d877..d2da7e4eb7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy @@ -19,8 +19,8 @@ package nextflow.extension import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression import groovyx.gpars.dataflow.operator.DataflowEventAdapter import groovyx.gpars.dataflow.operator.DataflowProcessor @@ -52,7 +52,7 @@ class BufferOp { private DataflowReadChannel source - private DataflowQueue target + private DataflowWriteChannel target private Object openCondition @@ -90,8 +90,8 @@ class BufferOp { return this } - DataflowQueue apply() { - target = new DataflowQueue() + DataflowWriteChannel apply() { + target = ChannelFactory.create() if( params?.skip ) this.skip = params.skip as int @@ -149,7 +149,7 @@ class BufferOp { } @CompileDynamic - static private void buffer0(DataflowReadChannel source, DataflowQueue target, Closure startingCriteria, Closure closeCriteria, boolean remainder ) { + static private void buffer0(DataflowReadChannel source, DataflowWriteChannel target, Closure startingCriteria, Closure closeCriteria, boolean remainder ) { assert closeCriteria // the list holding temporary collected elements @@ -178,7 +178,7 @@ class BufferOp { @Override boolean onException(DataflowProcessor processor, Throwable e) { - DataflowExtensions.log.error("@unknown", e) + OperatorEx.log.error("@unknown", e) session.abort(e) return true } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CaptureProperties.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CaptureProperties.groovy index 25889dc31b..e2e0a838a2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CaptureProperties.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CaptureProperties.groovy @@ -25,14 +25,16 @@ import groovy.transform.CompileStatic @CompileStatic class CaptureProperties { - Set names = [] + List names = new ArrayList<>(10) - def propertyMissing(String name) { + Object propertyMissing(String name) { + if( names.contains(name) ) + throw new IllegalArgumentException("Duplicate channel definition: $name") names.add(name) return null } - static Set capture(Closure holder) { + static List capture(Closure holder) { final recorder = new CaptureProperties() holder.setResolveStrategy( Closure.DELEGATE_ONLY ) holder.delegate = recorder diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ChannelEx.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ChannelEx.groovy new file mode 100644 index 0000000000..c6ae1ddd13 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ChannelEx.groovy @@ -0,0 +1,167 @@ +/* + * Copyright 2013-2019, Centre for Genomic Regulation (CRG) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.extension + +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import groovyx.gpars.agent.Agent +import groovyx.gpars.dataflow.DataflowQueue +import groovyx.gpars.dataflow.DataflowWriteChannel +import nextflow.Channel +import nextflow.NF +import nextflow.dag.NodeMarker +import nextflow.script.ChainableDef +import nextflow.script.ChannelArrayList +import nextflow.script.CompositeDef +import nextflow.script.ExecutionStack +/** + * Implements dataflow channel extension methods + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class ChannelEx { + + /** + * Assign the {@code source} channel to a global variable with the name specified by the closure. + * For example: + * 
+     *     Channel.from( ... )
+     *            .map { ... }
+     *            .set { newChannelName }
+     *
+ * + * @param DataflowReadChannel + * @param holder A closure that must define a single variable expression + */ + static void set(DataflowWriteChannel source, Closure holder) { + final name = CaptureProperties.capture(holder) + if( !name ) + throw new IllegalArgumentException("Missing name to which set the channel variable") + + if( name.size()>1 ) + throw new IllegalArgumentException("Operation `set` does not allow more than one target name") + + NF.binding.setVariable(name[0], source) + } + + static void set(ChannelArrayList source, Closure holder) { + final names = CaptureProperties.capture(holder) + if( names.size() > source.size() ) + throw new IllegalArgumentException("Operation `set` expects ${names.size()} channels but only ${source.size()} are provided") + + for( int i=0; i + * Add the {@code update} method to an {@code Agent} so that it call implicitly + * the {@code Agent#updateValue} method + */ + @CompileDynamic + static void update(Agent self, Closure message ) { + assert message != null + + self.send { + message.call(it) + updateValue(it) + } + + } + + static private void checkContext(String method, Object operand) { + if( !NF.isDsl2() ) + throw new MissingMethodException(method, operand.getClass()) + + if( !ExecutionStack.withinWorkflow() ) + throw new IllegalArgumentException("Process invocation are only allowed within a workflow context") + } + + static Object or(DataflowWriteChannel left, ChainableDef right ) { + checkContext('or', left) + return right.invoke_o(left) + } + + static Object or( DataflowWriteChannel left, OpCall operator ) { + checkContext('or', left) + operator.setSource(left).call() + } + + static CompositeDef and(ChainableDef left, ChainableDef right) { + checkContext('and', left) + return new CompositeDef().add(left).add(right) + } + + static CompositeDef and(CompositeDef left, ChainableDef right) { + checkContext('and', left) + left.add(right) + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ChannelFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ChannelFactory.groovy new file mode 100644 index 0000000000..0ed10b9690 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ChannelFactory.groovy @@ -0,0 +1,116 @@ +package nextflow.extension + +import groovy.transform.CompileStatic +import groovy.transform.PackageScope +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowBroadcast +import groovyx.gpars.dataflow.DataflowQueue +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowVariable +import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.expression.DataflowExpression +import groovyx.gpars.dataflow.stream.DataflowStreamReadAdapter +import groovyx.gpars.dataflow.stream.DataflowStreamWriteAdapter +import nextflow.Channel +import nextflow.NF +/** + * Helper class to create dataflow objects + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class ChannelFactory { + + static private Map bridges = new HashMap<>(10) + + static DataflowReadChannel getReadChannel(channel) { + if (channel instanceof DataflowQueue) + return getRead1(channel) + + if (channel instanceof DataflowBroadcast) + return getRead2(channel) + + if (channel instanceof DataflowReadChannel) + return channel + + throw new IllegalArgumentException("Illegal channel source type: ${channel?.getClass()?.getName()}") + } + + static synchronized private DataflowReadChannel getRead1(DataflowQueue queue) { + if( !NF.isDsl2() ) + return queue + + def broadcast = bridges.get(queue) + if( broadcast == null ) { + broadcast = new DataflowBroadcast() + bridges.put(queue, broadcast) + } + return broadcast.createReadChannel() + } + + static private DataflowReadChannel getRead2(DataflowBroadcast channel) { + if( !NF.isDsl2() ) + throw new IllegalStateException("Broadcast channel are only allowed in a workflow definition scope") + channel.createReadChannel() + } + + static synchronized boolean isBridge(DataflowQueue queue) { + bridges.get(queue) != null + } + + static void broadcast() { + // connect all dataflow queue variables to associated broadcast channel + for( DataflowQueue queue : bridges.keySet() ) { + log.debug "Bridging dataflow queue=$queue" + def broadcast = bridges.get(queue) + queue.into(broadcast) + } + } + + static void init() { bridges.clear() } + + @PackageScope + static DataflowWriteChannel close0(DataflowWriteChannel source) { + if( source instanceof DataflowExpression ) { + if( !source.isBound() ) + source.bind(Channel.STOP) + } + else { + source.bind(Channel.STOP) + } + return source + } + + static DataflowWriteChannel createBy(DataflowReadChannel channel) { + create( channel instanceof DataflowExpression ) + } + + static DataflowWriteChannel create(boolean value=false) { + if( value ) + return new DataflowVariable() + + if( NF.isDsl2() ) + return new DataflowBroadcast() + + return new DataflowQueue() + } + + static boolean isChannel(obj) { + obj instanceof DataflowReadChannel || obj instanceof DataflowWriteChannel + } + + static boolean isChannelQueue(obj) { + obj instanceof DataflowQueue || obj instanceof DataflowStreamReadAdapter || obj instanceof DataflowStreamWriteAdapter + } + + static boolean allScalar(List args) { + for( def el : args ) { + if( isChannelQueue(el) ) { + return false + } + } + return true + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ChoiceOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ChoiceOp.groovy index 03bc9297c6..02c50c8d73 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ChoiceOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ChoiceOp.groovy @@ -27,7 +27,7 @@ import nextflow.Channel import nextflow.Global import nextflow.Session /** - * Implements the logic for {@link DataflowExtensions#choice} operator(s) + * Implements the logic for {@link OperatorEx#choice} operator(s) * * @author Paolo Di Tommaso */ diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy index 8284cb03b1..28a2f8d846 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy @@ -15,13 +15,12 @@ */ package nextflow.extension -import static nextflow.util.CheckHelper.checkParams import java.nio.file.Path import groovy.util.logging.Slf4j -import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Channel import nextflow.Global import nextflow.file.FileCollector @@ -29,11 +28,10 @@ import nextflow.file.FileHelper import nextflow.file.SimpleFileCollector import nextflow.file.SortFileCollector import nextflow.util.CacheHelper - import static nextflow.util.CacheHelper.HashMode - +import static nextflow.util.CheckHelper.checkParams /** - * Implements the body of {@link DataflowExtensions#collectFile(groovyx.gpars.dataflow.DataflowReadChannel)} operator + * Implements the body of {@link OperatorEx#collectFile(groovyx.gpars.dataflow.DataflowReadChannel)} operator * * @author Paolo Di Tommaso */ @@ -57,7 +55,7 @@ class CollectFileOp { private final Map params - private DataflowQueue result + private DataflowWriteChannel result private DataflowReadChannel channel @@ -75,7 +73,7 @@ class CollectFileOp { this.params = params this.channel = channel this.closure = closure - this.result = new DataflowQueue() + this.result = ChannelFactory.create() createFileCollector() defineStoreDirAndFileName() @@ -267,7 +265,7 @@ class CollectFileOp { } - DataflowQueue apply() { + DataflowWriteChannel apply() { DataflowHelper.subscribeImpl( channel, [onNext: this.&processItem, onComplete: this.&emitItems] ) return result } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy index 365bc43870..946607dcb0 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy @@ -24,7 +24,7 @@ import groovyx.gpars.dataflow.DataflowVariable import nextflow.Channel import nextflow.util.ArrayBag /** - * Implements {@link DataflowExtensions#collect(groovyx.gpars.dataflow.DataflowReadChannel)} operator + * Implements {@link OperatorEx#collect(groovyx.gpars.dataflow.DataflowReadChannel)} operator * * @author Paolo Di Tommaso */ @@ -73,8 +73,6 @@ class CollectOp { } private List normalise(List list) { - - if( !sort ) return list diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy index 73dc850983..d020bb13f1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy @@ -15,24 +15,21 @@ */ package nextflow.extension + import java.util.concurrent.atomic.AtomicInteger import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.util.logging.Slf4j -import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel -import groovyx.gpars.dataflow.expression.DataflowExpression import nextflow.Channel import nextflow.Nextflow - import static nextflow.extension.DataflowHelper.addToList import static nextflow.extension.DataflowHelper.split - /** - * Implements the {@link DataflowExtensions#spread(groovyx.gpars.dataflow.DataflowReadChannel, java.lang.Object)} operator + * Implements the {@link OperatorEx#spread(groovyx.gpars.dataflow.DataflowReadChannel, java.lang.Object)} operator * * @author Paolo Di Tommaso */ @@ -46,7 +43,7 @@ class CombineOp { private DataflowReadChannel rightChannel - private DataflowQueue target + private DataflowWriteChannel target private Map leftValues = [:] @@ -63,8 +60,7 @@ class CombineOp { leftChannel = left switch(right) { - case DataflowQueue: - case DataflowExpression: + case DataflowReadChannel: rightChannel = (DataflowReadChannel)right; break @@ -151,9 +147,9 @@ class CombineOp { throw new IllegalArgumentException("Not a valid spread operator index: $index") } - public DataflowReadChannel apply() { + DataflowWriteChannel apply() { - target = new DataflowQueue() + target = ChannelFactory.create() if( rightChannel ) { final stopCount = new AtomicInteger(2) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy index bf2fbdf4f4..651d5fb49a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy @@ -17,13 +17,11 @@ package nextflow.extension import groovy.transform.CompileStatic -import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Channel - /** - * Implements the {@link DataflowExtensions#concat} operator + * Implements the {@link OperatorEx#concat} operator * * @author Paolo Di Tommaso */ @@ -43,8 +41,8 @@ class ConcatOp { } - DataflowQueue apply() { - final result = new DataflowQueue() + DataflowWriteChannel apply() { + final result = ChannelFactory.create() final allChannels = [source] allChannels.addAll(target) @@ -57,12 +55,13 @@ class ConcatOp { def current = channels[index++] def next = index < channels.size() ? channels[index] : null - DataflowHelper.subscribeImpl(current, [ - onNext: { result.bind(it) }, - onComplete: { - if(next) append(result, channels, index) - else result.bind(Channel.STOP) - } - ]) + def events = new HashMap(2) + events.onNext = { result.bind(it) } + events.onComplete = { + if(next) append(result, channels, index) + else result.bind(Channel.STOP) + } + + DataflowHelper.subscribeImpl(current, events) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CrossOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CrossOp.groovy index b4758b0e26..7b0707f0c3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CrossOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CrossOp.groovy @@ -20,13 +20,11 @@ import java.util.concurrent.atomic.AtomicInteger import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Channel - /** - * Implements the {@link DataflowExtensions#cross} operator logic + * Implements the {@link OperatorEx#cross} operator logic * * @author Paolo Di Tommaso */ @@ -38,7 +36,7 @@ class CrossOp { private DataflowReadChannel target - private Closure mapper = DataflowExtensions.DEFAULT_MAPPING_CLOSURE + private Closure mapper = OperatorEx.DEFAULT_MAPPING_CLOSURE CrossOp(DataflowReadChannel source, DataflowReadChannel target) { assert source @@ -49,13 +47,13 @@ class CrossOp { } CrossOp setMapper( Closure mapper ) { - this.mapper = mapper ?: DataflowExtensions.DEFAULT_MAPPING_CLOSURE + this.mapper = mapper ?: OperatorEx.DEFAULT_MAPPING_CLOSURE return this } - DataflowQueue apply() { + DataflowWriteChannel apply() { - def result = new DataflowQueue() + final result = ChannelFactory.create() Map> state = [:] final count = 2 @@ -67,36 +65,36 @@ class CrossOp { return result } - static private final Map crossHandlers( Map> buffer, int size, int index, DataflowWriteChannel target, Closure mapper, AtomicInteger stopCount ) { + static private final Map crossHandlers( Map> buffer, int size, int index, DataflowWriteChannel target, Closure mapper, AtomicInteger stopCount ) { - [ - onNext: { - synchronized (buffer) { // phaseImpl is NOT thread safe, synchronize it ! - while( true ) { - def entries = PhaseOp.phaseImpl(buffer, size, index, it, mapper, true) - log.trace "Cross #${target.hashCode()} ($index) > item: $it; entries: $entries " + def result = new HashMap(2) - if( entries ) { - target.bind(entries) - // when it is invoked on the 'left' operator channel - // try to invoke it one more time to consume value eventually produced and accumulated by the 'right' channel - if( index == 0 ) - continue - } - break - } + result.onNext = { + synchronized (buffer) { // phaseImpl is NOT thread safe, synchronize it ! + while( true ) { + def entries = PhaseOp.phaseImpl(buffer, size, index, it, mapper, true) + log.trace "Cross #${target.hashCode()} ($index) > item: $it; entries: $entries " - }}, + if( entries ) { + target.bind(entries) + // when it is invoked on the 'left' operator channel + // try to invoke it one more time to consume value eventually produced and accumulated by the 'right' channel + if( index == 0 ) + continue + } + break + } - onComplete: { - log.trace "Cross #${target.hashCode()} ($index) > Complete" - if( stopCount.decrementAndGet()==0) { - log.trace "Cross #${target.hashCode()} ($index) > STOP" - target << Channel.STOP - }} + }} - ] + result.onComplete = { + log.trace "Cross #${target.hashCode()} ($index) > Complete" + if( stopCount.decrementAndGet()==0) { + log.trace "Cross #${target.hashCode()} ($index) > STOP" + target << Channel.STOP + }} + return result } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index 78b945131f..3cc62f5895 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -19,6 +19,7 @@ package nextflow.extension import groovy.transform.PackageScope import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.Dataflow +import groovyx.gpars.dataflow.DataflowChannel import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowVariable @@ -43,6 +44,29 @@ class DataflowHelper { private static Session getSession() { Global.getSession() as Session } + /** + * Create a dataflow object by the type of the specified source argument + * + * @param source + * @return + */ + @Deprecated + static DataflowChannel newChannelBy(DataflowReadChannel source) { + + switch( source ) { + case DataflowExpression: + return new DataflowVariable() + + case DataflowQueue: + return new DataflowQueue() + + default: + throw new IllegalArgumentException() + } + + } + + /** * Check if a {@code DataflowProcessor} is active * @@ -67,7 +91,7 @@ class DataflowHelper { static DEF_ERROR_LISTENER = new DataflowEventAdapter() { @Override boolean onException(final DataflowProcessor processor, final Throwable e) { - DataflowExtensions.log.error("@unknown", e) + OperatorEx.log.error("@unknown", e) session.abort(e) return true; } @@ -88,7 +112,7 @@ class DataflowHelper { @Override boolean onException(final DataflowProcessor processor, final Throwable e) { - DataflowExtensions.log.error("@unknown", e) + DataflowHelper.log.error("@unknown", e) session.abort(e) return true } @@ -219,7 +243,7 @@ class DataflowHelper { * @param closure * @return */ - static final DataflowProcessor subscribeImpl(final DataflowReadChannel source, final Map events ) { + static final DataflowProcessor subscribeImpl(final DataflowReadChannel source, final Map events ) { checkSubscribeHandlers(events) def error = false @@ -233,7 +257,7 @@ class DataflowHelper { events.onComplete.call(processor) } catch( Exception e ) { - DataflowExtensions.log.error("@unknown", e) + OperatorEx.log.error("@unknown", e) session.abort(e) } } @@ -269,13 +293,13 @@ class DataflowHelper { } - static DataflowProcessor chainImpl(final DataflowReadChannel source, final DataflowReadChannel target, final Map params, final Closure closure) { + static DataflowProcessor chainImpl(final DataflowReadChannel source, final DataflowWriteChannel target, final Map params, final Closure closure) { final Map parameters = new HashMap(params) parameters.put("inputs", asList(source)) parameters.put("outputs", asList(target)) - newOperator(parameters, new ChainWithClosure(closure)) + newOperator(parameters, new ChainWithClosure(closure)) } /** @@ -286,7 +310,7 @@ class DataflowHelper { * @param closure * @return */ - static DataflowProcessor reduceImpl(final DataflowReadChannel channel, final DataflowVariable result, def seed, final Closure closure) { + static DataflowProcessor reduceImpl(final DataflowReadChannel channel, final DataflowVariable result, def seed, final Closure closure) { // the *accumulator* value def accum = seed @@ -296,7 +320,7 @@ class DataflowHelper { /* * call the passed closure each time */ - public void afterRun(final DataflowProcessor processor, final List messages) { + void afterRun(final DataflowProcessor processor, final List messages) { final item = messages.get(0) final value = accum == null ? item : closure.call(accum, item) @@ -314,18 +338,18 @@ class DataflowHelper { /* * when terminates bind the result value */ - public void afterStop(final DataflowProcessor processor) { + void afterStop(final DataflowProcessor processor) { result.bind(accum) } - public boolean onException(final DataflowProcessor processor, final Throwable e) { + boolean onException(final DataflowProcessor processor, final Throwable e) { log.error("@unknown", e) session.abort(e) return true; } } - chainImpl(channel, new DataflowQueue(), [listeners: [listener]], {true}) + chainImpl(channel, ChannelFactory.create(), [listeners: [listener]], {true}) } @@ -365,4 +389,10 @@ class DataflowHelper { } } + static Map eventsMap(Closure onNext, Closure onComplete) { + def result = new HashMap(2) + result.put('onNext', onNext) + result.put('onComplete', onComplete) + return result + } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy index 95fa37deb0..6b03677775 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy @@ -16,15 +16,14 @@ package nextflow.extension -import org.codehaus.groovy.runtime.InvokerHelper -import static nextflow.util.CheckHelper.checkParams - import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Global import nextflow.Session - +import org.codehaus.groovy.runtime.InvokerHelper +import static nextflow.util.CheckHelper.checkParams /** * Implements channel `dump` operator. It prints the content of a channel * only when the `-dump-channels` command line option is specified otherwise @@ -38,6 +37,8 @@ class DumpOp { static final private Map PARAMS_DUMP = [tag: String] + private Session session = (Global.session as Session) + private DataflowReadChannel source private Closure renderer @@ -46,12 +47,17 @@ class DumpOp { protected String tag - DumpOp(DataflowReadChannel source, Map opts, Closure renderer = null) { + DumpOp(Map opts, Closure renderer) { checkParams('dump', opts, PARAMS_DUMP) this.source = source this.tag = opts.tag this.renderer = renderer - this.dumpNames = (Global.session as Session).getDumpChannels() + this.dumpNames = session.getDumpChannels() + } + + DumpOp setSource( DataflowWriteChannel source ) { + this.source = ChannelFactory.getReadChannel(source) + return this } /** Only for testing -- do not use */ @@ -68,13 +74,15 @@ class DumpOp { .find { matcher ==~ /$it/} } - DataflowReadChannel apply() { + DataflowWriteChannel apply() { if( !isEnabled() ) { - return source + if( source instanceof DataflowWriteChannel ) + return (DataflowWriteChannel)source + throw new IllegalArgumentException("Illegal dump operator source channel") } - final target = DataflowExtensions.newChannelBy(source) + final target = ChannelFactory.createBy(source) final events = new HashMap(2) events.onNext = { def marker = 'DUMP' @@ -83,7 +91,7 @@ class DumpOp { target.bind(it) } - events.onComplete = { DataflowExtensions.close(target) } + events.onComplete = { ChannelFactory.close0(target) } DataflowHelper.subscribeImpl(source, events) return target diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy index 7aec89c369..84fe94da36 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy @@ -17,14 +17,14 @@ package nextflow.extension import groovy.util.logging.Slf4j -import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Channel import nextflow.util.ArrayBag import nextflow.util.CacheHelper import nextflow.util.CheckHelper /** - * Implements {@link DataflowExtensions#groupTuple} operator logic + * Implements {@link OperatorEx#groupTuple} operator logic * * @author Paolo Di Tommaso */ @@ -45,7 +45,7 @@ class GroupTupleOp { private List indices - private DataflowQueue target + private DataflowWriteChannel target private DataflowReadChannel channel @@ -60,7 +60,6 @@ class GroupTupleOp { CheckHelper.checkParams('groupTuple', params, GROUP_TUPLE_PARAMS) channel = source - target = new DataflowQueue() indices = getGroupTupleIndices(params) size = params?.size ?: 0 remainder = params?.remainder ?: false @@ -69,6 +68,10 @@ class GroupTupleOp { defineComparator() } + GroupTupleOp setTarget(DataflowWriteChannel target) { + this.target = target + return this + } static private List getGroupTupleIndices( Map params ) { @@ -204,7 +207,10 @@ class GroupTupleOp { * * @return The resulting channel */ - DataflowQueue apply() { + DataflowWriteChannel apply() { + + if( target == null ) + target = ChannelFactory.create() /* * apply the logic the the source channel diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy index 3be89df0e0..153b7071a2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy @@ -28,9 +28,11 @@ import groovyx.gpars.dataflow.operator.DataflowEventAdapter import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import nextflow.Global +import nextflow.NF import nextflow.Session +import static nextflow.extension.DataflowHelper.newChannelBy /** - * Implements the {@link DataflowExtensions#into} operators logic + * Implements the {@link OperatorEx#into} operators logic * * @author Paolo Di Tommaso */ @@ -70,7 +72,6 @@ class IntoOp { assert holder final names = CaptureProperties.capture(holder) - final binding = Global.session.binding if( !names ) throw new IllegalArgumentException("Missing target channel names in `into` operator") if( names.size() == 1 ) @@ -78,9 +79,9 @@ class IntoOp { def targets = [] names.each { identifier -> - def channel = DataflowExtensions.newChannelBy(source) + def channel = newChannelBy(source) targets.add(channel) - binding.setVariable(identifier, channel) + NF.binding.setVariable(identifier, channel) } this.source = source diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy index 514ad90ea4..99ac372d14 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy @@ -15,21 +15,19 @@ */ package nextflow.extension + import java.util.concurrent.atomic.AtomicInteger import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.util.logging.Slf4j -import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Channel import nextflow.util.CheckHelper - import static nextflow.extension.DataflowHelper.addToList - /** - * Implements {@link DataflowExtensions#join} operator logic + * Implements {@link OperatorEx#join} operator logic * * @author Paolo Di Tommaso */ @@ -66,10 +64,10 @@ class JoinOp { return [value as int] } - DataflowQueue apply() { + DataflowWriteChannel apply() { // the resulting channel - final result = new DataflowQueue() + final result = ChannelFactory.create() // the following buffer maintains the state of collected items as a map of maps. // The first map associates the joining key with the collected values // The inner map associates the channel index with the actual values received on that channel diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy index b3d785dbd3..9fb1b27911 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy @@ -16,13 +16,14 @@ package nextflow.extension -import groovyx.gpars.dataflow.DataflowChannel + import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel /** - * Implements {@link DataflowExtensions#map(groovyx.gpars.dataflow.DataflowReadChannel, groovy.lang.Closure)} operator + * Implements {@link OperatorEx#map(groovyx.gpars.dataflow.DataflowReadChannel, groovy.lang.Closure)} operator * * @author Paolo Di Tommaso */ @@ -32,15 +33,24 @@ class MapOp { private Closure mapper + private DataflowWriteChannel target + MapOp( final DataflowReadChannel source, final Closure mapper ) { this.source = source this.mapper = mapper } - DataflowChannel apply() { + MapOp setTarget( DataflowWriteChannel target ) { + this.target = target + return this + } + + DataflowWriteChannel apply() { + + if( target == null ) + target = ChannelFactory.createBy(source) final stopOnFirst = source instanceof DataflowExpression - final target = DataflowExtensions.newChannelBy(source) DataflowHelper.newOperator(source, target) { it -> def result = mapper.call(it) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OpCall.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OpCall.groovy new file mode 100644 index 0000000000..d0e21c5332 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OpCall.groovy @@ -0,0 +1,306 @@ +package nextflow.extension + +import java.lang.reflect.Method +import java.lang.reflect.ParameterizedType +import java.util.concurrent.Callable + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowBroadcast +import groovyx.gpars.dataflow.DataflowQueue +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel +import nextflow.dag.NodeMarker +import org.codehaus.groovy.runtime.InvokerHelper +/** + * Represents an nextflow operation invocation + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class OpCall implements Callable { + + final static private List SPECIAL_NAMES = ["choice","merge","separate"] + + static ThreadLocal current = new ThreadLocal<>() + + private OperatorEx owner + private String methodName + private Method method + private Object source + private Object[] args + private Set inputs = new HashSet(5) + private Set outputs = new HashSet<>(5) + + static OpCall create(String methodName, Object args) { + new OpCall(methodName, InvokerHelper.asArray(args)) + } + + static OpCall create(String method) { + new OpCall(method, InvokerHelper.asArray(null)) + } + + OpCall(OperatorEx owner, Object source, String method, Object[] args ) { + assert owner + assert method + this.owner = owner + this.source = source + this.methodName = method + this.args = args + } + + OpCall(String method, Object[] args ) { + assert method + this.owner = OperatorEx.instance + this.methodName = method + this.args = args + } + + OpCall setSource(DataflowWriteChannel channel) { + this.source = channel + return this + } + + @Override + Object call() throws Exception { + if( source==null ) + throw new IllegalStateException("Missing operator source channel") + current.set(this) + try { + return invoke() + } + finally { + current.remove() + } + } + + Set getInputs() { inputs } + + Set getOutputs() { outputs } + + String getMethodName() { methodName } + + Object[] getArgs() { args } + + private T read0(source){ + if( source instanceof DataflowBroadcast ) + return (T)ChannelFactory.getReadChannel(source) + + if( source instanceof DataflowQueue ) + return (T)ChannelFactory.getReadChannel(source) + + else + return (T)source + } + + private Object[] read1(Object[] args) { + if( methodName != 'separate' && methodName != 'choice' ) { + Object[] params = new Object[args.length] + for( int i=0; i> outputs, final Closure> code) + // can be invoked as: + // queue.separate( x, y, z ) { ... } + + Object[] params = new Object[3] + params[0] = channel + params[1] = toListOfChannel(args) + params[2] = args[args.length - 1] + + return invoke1(methodName, params) + } + else { + // create the invocation parameters array + Object[] params = new Object[args.length + 1] + params[0] = channel + for (int i = 0; i < args.length; i++) { + params[i + 1] = args[i]; + } + + return invoke1(methodName, params); + } + } + + protected Method getMethod0(String methodName, Object[] args) { + def meta = owner.metaClass.getMetaMethod(methodName, args) + if( meta == null ) + throw new MissingMethodException(methodName, owner.getClass()) + method = owner.getClass().getMethod(methodName, meta.getNativeParameterTypes()) + } + + protected Object invoke1(String methodName, Object[] args) { + method = getMethod0(methodName, args) + checkDeprecation(method) + return owner.metaClass.invokeMethod(owner, methodName, args) + } + + protected void checkDeprecation(Method method) { + if( method.getAnnotation(Deprecated) ) { + log.warn "Operator `$methodName` is deprecated -- it will be removed in a future release" + } + } + + @CompileStatic + private static boolean checkOpenArrayDataflowMethod(List validNames, String methodName, Object[] args) { + if( !validNames.contains(methodName) ) + return false + if( args == null || args.length<2 ) + return false + if( !(args[args.length-1] instanceof Closure) ) + return false + for( int i=0; i toListOfChannel( Object[] args ) { + List result = new ArrayList<>(args.length-1); + for( int i=0; i - * Add the {@code update} method to an {@code Agent} so that it call implicitly - * the {@code Agent#updateValue} method - * - */ - static void update( Agent self, Closure message ) { - assert message != null + final public static Set OPERATOR_NAMES - self.send { - message.call(it) - updateValue(it) - } + final static MetaClass meta = OperatorEx.getMetaClass() + static { + OPERATOR_NAMES = getDeclaredExtensionMethods0() + log.trace "Dataflow extension methods: ${OPERATOR_NAMES.sort().join(',')}" } + @CompileStatic + static private Set getDeclaredExtensionMethods0() { + def result = new HashSet(30) + def methods = OperatorEx.class.getDeclaredMethods() + for( def handle : methods ) { + if( !Modifier.isPublic(handle.getModifiers()) ) continue + if( Modifier.isStatic(handle.getModifiers()) ) continue + def params=handle.getParameterTypes() + if( params.length>0 && isReadChannel(params[0]) ) + result.add(handle.name) + } + return result + } - /** - * Create a dataflow object by the type of the specified source argument - * - * @param source - * @return - */ - @PackageScope - static DataflowChannel newChannelBy(DataflowReadChannel source) { - - switch( source ) { - case DataflowExpression: - return new DataflowVariable() - - case DataflowQueue: - return new DataflowQueue() + @CompileStatic + static boolean isReadChannel(Class clazz) { + DataflowReadChannel.class.isAssignableFrom(clazz) + } - default: - throw new IllegalArgumentException() + @CompileStatic + boolean isExtension(Object instance, String name) { + if( instance instanceof DataflowReadChannel || instance instanceof DataflowBroadcast ) { + return OPERATOR_NAMES.contains(name) } + return false + } + + @CompileStatic + Object invokeOperator(Object channel, String method, Object[] args) { + new OpCall(this,channel,method,args).call() } /** @@ -113,9 +116,8 @@ class DataflowExtensions { * @param closure * @return */ - static final DataflowReadChannel subscribe(final DataflowReadChannel source, final Closure closure) { + DataflowReadChannel subscribe(final DataflowReadChannel source, final Closure closure) { subscribeImpl( source, [onNext: closure] ) - NodeMarker.addOperatorNode('subscribe', source, null) return source } @@ -126,9 +128,8 @@ class DataflowExtensions { * @param closure * @return */ - static final DataflowReadChannel subscribe(final DataflowReadChannel source, final Map events ) { + DataflowReadChannel subscribe(final DataflowReadChannel source, final Map events ) { subscribeImpl(source, events) - NodeMarker.addOperatorNode('subscribe', source, null) return source } @@ -139,16 +140,12 @@ class DataflowExtensions { * @param closure * @return */ - static DataflowReadChannel chain(final DataflowReadChannel source, final Closure closure) { - - final DataflowReadChannel target = newChannelBy(source) - newOperator(source, target, new ChainWithClosure(closure)) - NodeMarker.addOperatorNode('chain', source, target) - - return target; + DataflowWriteChannel chain(final DataflowReadChannel source, final Closure closure) { + final target = ChannelFactory.createBy(source) + newOperator(source, target, new ChainWithClosure(closure)) + return target } - /** * Chain operator, this is a synonym of {@code DataflowReadChannel.chainWith} * @@ -156,12 +153,9 @@ class DataflowExtensions { * @param closure * @return */ - static DataflowReadChannel chain(final DataflowReadChannel source, final Map params, final Closure closure) { - - final DataflowReadChannel target = newChannelBy(source) + DataflowWriteChannel chain(final DataflowReadChannel source, final Map params, final Closure closure) { + final target = ChannelFactory.createBy(source) chainImpl(source, target, params, closure) - NodeMarker.addOperatorNode('chain', source, target) - return target; } @@ -172,17 +166,12 @@ class DataflowExtensions { * @param closure * @return */ - static final DataflowReadChannel map(final DataflowReadChannel source, final Closure closure) { + DataflowWriteChannel map(final DataflowReadChannel source, final Closure closure) { assert source != null assert closure - - def target = new MapOp(source, closure).apply() - - NodeMarker.addOperatorNode('map', source, target) - return target; + return new MapOp(source, closure).apply() } - /** * Transform the items emitted by a channel by applying a function to each of them and then flattens the results of that function. * @@ -190,12 +179,11 @@ class DataflowExtensions { * @param closure The closure mapping the values emitted by the source channel * @return The channel emitting the mapped values */ - static final DataflowReadChannel flatMap(final DataflowReadChannel source, final Closure closure=null) { + DataflowWriteChannel flatMap(final DataflowReadChannel source, final Closure closure=null) { assert source != null - final target = new DataflowQueue() - - def listener = stopErrorListener(source,target) + final target = ChannelFactory.create() + final listener = stopErrorListener(source,target) newOperator(source, target, listener) { item -> @@ -228,7 +216,6 @@ class DataflowExtensions { } } - NodeMarker.addOperatorNode('flatMap', source, target) return target } @@ -245,17 +232,15 @@ class DataflowExtensions { * @param closure * @return */ - static final DataflowReadChannel reduce(final DataflowReadChannel source, final Closure closure) { + DataflowWriteChannel reduce(final DataflowReadChannel source, final Closure closure) { if( source instanceof DataflowExpression ) throw new IllegalArgumentException('Operator `reduce` cannot be applied to a value channel') final target = new DataflowVariable() reduceImpl( source, target, null, closure ) - NodeMarker.addOperatorNode('reduce', source, target) return target } - /** * * The reduce( ) operator applies a function of your choosing to the first item emitted by a source channel, @@ -270,31 +255,27 @@ class DataflowExtensions { * @param closure * @return */ - static final DataflowReadChannel reduce(final DataflowReadChannel source, V seed, final Closure closure) { + DataflowWriteChannel reduce(final DataflowReadChannel source, Object seed, final Closure closure) { if( source instanceof DataflowExpression ) throw new IllegalArgumentException('Operator `reduce` cannot be applied to a value channel') final target = new DataflowVariable() reduceImpl( source, target, seed, closure ) - NodeMarker.addOperatorNode('reduce', source, target) return target } - static final DataflowReadChannel collectFile( final DataflowReadChannel source, final Closure closure = null ) { + DataflowWriteChannel collectFile( final DataflowReadChannel source, final Closure closure = null ) { final result = new CollectFileOp(source, null, closure).apply() - NodeMarker.addOperatorNode('collectFile', source, result) return result } - static final DataflowReadChannel collectFile( final DataflowReadChannel source, Map params, final Closure closure = null ) { + DataflowWriteChannel collectFile( final DataflowReadChannel source, Map params, final Closure closure = null ) { def result = new CollectFileOp(source, params, closure).apply() - NodeMarker.addOperatorNode('collectFile', source, result) return result } - static final DataflowReadChannel groupTuple( final DataflowReadChannel source, final Map params ) { + DataflowWriteChannel groupTuple( final DataflowReadChannel source, final Map params=null ) { def result = new GroupTupleOp(params, source).apply() - NodeMarker.addOperatorNode('groupTuple', source, result) return result } @@ -314,10 +295,10 @@ class DataflowExtensions { * @param criteria * @return */ - static final DataflowReadChannel filter(final DataflowReadChannel source, final Object criteria) { + DataflowWriteChannel filter(final DataflowReadChannel source, final Object criteria) { def discriminator = new BooleanReturningMethodInvoker("isCase"); - def target = newChannelBy(source) + def target = ChannelFactory.createBy(source) if( source instanceof DataflowExpression ) { source.whenBound { def result = it instanceof ControlMessage ? false : discriminator.invoke(criteria, (Object)it) @@ -331,12 +312,11 @@ class DataflowExtensions { }) } - NodeMarker.addOperatorNode('filter', source, target) return target } - static DataflowReadChannel filter(DataflowReadChannel source, final Closure closure) { - def target = newChannelBy(source) + DataflowWriteChannel filter(DataflowReadChannel source, final Closure closure) { + def target = ChannelFactory.createBy(source) if( source instanceof DataflowExpression ) { source.whenBound { def result = it instanceof ControlMessage ? false : DefaultTypeTransformation.castToBoolean(closure.call(it)) @@ -350,12 +330,11 @@ class DataflowExtensions { }) } - NodeMarker.addOperatorNode('filter', source, target) return target } - static DataflowReadChannel until(DataflowReadChannel source, final Closure closure) { - def target = newChannelBy(source) + DataflowWriteChannel until(DataflowReadChannel source, final Closure closure) { + def target = ChannelFactory.createBy(source) newOperator(source, target, { final result = DefaultTypeTransformation.castToBoolean(closure.call(it)) final proc = ((DataflowProcessor) getDelegate()) @@ -368,7 +347,7 @@ class DataflowExtensions { proc.bindOutput(it) } }) - NodeMarker.addOperatorNode('until', source, target) + return target } @@ -381,7 +360,7 @@ class DataflowExtensions { * @param source * @return */ - static final DataflowReadChannel unique(final DataflowReadChannel source) { + DataflowWriteChannel unique(final DataflowReadChannel source) { unique(source) { it } } @@ -394,14 +373,14 @@ class DataflowExtensions { * @param comparator * @return */ - static final DataflowReadChannel unique(final DataflowReadChannel source, Closure comparator ) { + DataflowWriteChannel unique(final DataflowReadChannel source, Closure comparator ) { def history = [:] - def target = newChannelBy(source) + def target = ChannelFactory.createBy(source) // when the operator stop clear the history map def events = new DataflowEventAdapter() { - public void afterStop(final DataflowProcessor processor) { + void afterStop(final DataflowProcessor processor) { history.clear() history = null } @@ -416,12 +395,11 @@ class DataflowExtensions { history.put(key,true) return it } - } as Closure + } as Closure // filter removing all duplicates chainImpl(source, target, [listeners: [events]], filter ) - NodeMarker.addOperatorNode('unique', source, target) return target } @@ -433,7 +411,7 @@ class DataflowExtensions { * * @return */ - static final DataflowReadChannel distinct( final DataflowReadChannel source ) { + DataflowWriteChannel distinct( final DataflowReadChannel source ) { distinct(source) {it} } @@ -444,11 +422,11 @@ class DataflowExtensions { * * @return */ - static final DataflowReadChannel distinct( final DataflowReadChannel source, Closure comparator ) { + DataflowWriteChannel distinct( final DataflowReadChannel source, Closure comparator ) { def previous = null - final DataflowReadChannel target = newChannelBy(source) - Closure filter = { it -> + final target = ChannelFactory.createBy(source) + Closure filter = { it -> def key = comparator.call(it) if( key == previous ) { @@ -460,7 +438,6 @@ class DataflowExtensions { chainImpl(source, target, [:], filter) - NodeMarker.addOperatorNode('distinct', source, target) return target } @@ -473,18 +450,17 @@ class DataflowExtensions { * @param source * @return */ - static final DataflowReadChannel first( DataflowReadChannel source ) { + DataflowWriteChannel first( DataflowReadChannel source ) { if( source instanceof DataflowExpression ) { def msg = "The operator `first` is useless when applied to a value channel which returns a single value by definition" - def name = session?.binding?.getVariableName(source) + def name = NF.lookupVariable(source) if( name ) msg += " -- check channel `$name`" log.warn msg } - def target = new DataflowVariable() + def target = new DataflowVariable() source.whenBound { target.bind(it) } - NodeMarker.addOperatorNode('first', source, target) return target } @@ -497,7 +473,7 @@ class DataflowExtensions { * @param source * @return */ - static final DataflowReadChannel first( final DataflowReadChannel source, Object criteria ) { + DataflowWriteChannel first( final DataflowReadChannel source, Object criteria ) { def target = new DataflowVariable() def discriminator = new BooleanReturningMethodInvoker("isCase"); @@ -517,7 +493,6 @@ class DataflowExtensions { } } - NodeMarker.addOperatorNode('first', source, target) return target } @@ -531,12 +506,12 @@ class DataflowExtensions { * @param n The number of items to be taken. The value {@code -1} has a special semantic for all * @return The resulting channel emitting the taken values */ - static final DataflowReadChannel take( final DataflowReadChannel source, int n ) { + DataflowWriteChannel take( final DataflowReadChannel source, int n ) { if( source instanceof DataflowExpression ) throw new IllegalArgumentException("Operator `take` cannot be applied to a value channel") def count = 0 - final target = new DataflowQueue() + final target = ChannelFactory.create() if( n==0 ) { target.bind(Channel.STOP) @@ -545,15 +520,15 @@ class DataflowExtensions { final listener = new DataflowEventAdapter() { @Override - public void afterRun(final DataflowProcessor processor, final List messages) { + void afterRun(final DataflowProcessor processor, final List messages) { if( ++count >= n ) { processor.bindOutput( Channel.STOP ) processor.terminate() } } - public boolean onException(final DataflowProcessor processor, final Throwable e) { - DataflowExtensions.log.error("@unknown", e) + boolean onException(final DataflowProcessor processor, final Throwable e) { + OperatorEx.log.error("@unknown", e) session.abort(e) return true; } @@ -565,7 +540,6 @@ class DataflowExtensions { listeners: (n > 0 ? [listener] : []), new ChainWithClosure(new CopyChannelsClosure())) - NodeMarker.addOperatorNode('take', source, target) return target } @@ -575,22 +549,20 @@ class DataflowExtensions { * @param source The source channel * @return A {@code DataflowVariable} emitting the `last` item in the channel */ - static final DataflowReadChannel last( final DataflowReadChannel source ) { + DataflowWriteChannel last( final DataflowReadChannel source ) { def target = new DataflowVariable() - def V last = null + def last = null subscribeImpl( source, [onNext: { last = it }, onComplete: { target.bind(last) }] ) - NodeMarker.addOperatorNode('last', source, target) return target } - static final DataflowReadChannel collect(final DataflowReadChannel source, Closure action=null) { + DataflowWriteChannel collect(final DataflowReadChannel source, Closure action=null) { collect(source,Collections.emptyMap(),action) } - static final DataflowReadChannel collect(final DataflowReadChannel source, Map opts, Closure action=null) { + DataflowWriteChannel collect(final DataflowReadChannel source, Map opts, Closure action=null) { final target = new CollectOp(source,action,opts).apply() - NodeMarker.addOperatorNode('collect', source, target) return target } @@ -601,21 +573,19 @@ class DataflowExtensions { * @param source The channel to be converted * @return A list holding all the items send over the channel */ - static final DataflowReadChannel toList(final DataflowReadChannel source) { + DataflowWriteChannel toList(final DataflowReadChannel source) { final target = ToListOp.apply(source) - NodeMarker.addOperatorNode('toList', source, target) return target } /** - * Convert a {@code DataflowQueue} alias *channel* to a Java {@code List} sorting its content + * Convert a {@code DataflowReadChannel} alias *channel* to a Java {@code List} sorting its content * * @param source The channel to be converted * @return A list holding all the items send over the channel */ - static final DataflowReadChannel toSortedList(final DataflowReadChannel source, Closure closure = null) { + DataflowWriteChannel toSortedList(final DataflowReadChannel source, Closure closure = null) { final target = new ToListOp(source, closure ?: true).apply() - NodeMarker.addOperatorNode('toSortedList', source, target) return target as DataflowVariable } @@ -626,9 +596,8 @@ class DataflowExtensions { * @param value * @return */ - static final DataflowReadChannel count(final DataflowReadChannel source ) { + DataflowWriteChannel count(final DataflowReadChannel source ) { final target = count0(source, null) - NodeMarker.addOperatorNode('count', source, target) return target } @@ -639,9 +608,8 @@ class DataflowExtensions { * @param criteria * @return */ - static final DataflowReadChannel count(final DataflowReadChannel source, final Object criteria ) { + DataflowWriteChannel count(final DataflowReadChannel source, final Object criteria ) { final target = count0(source, criteria) - NodeMarker.addOperatorNode('count', source, target) return target } @@ -670,7 +638,8 @@ class DataflowExtensions { * @param source The source channel * @return A {@code DataflowVariable} returning the a {@code Map} containing the counting values for each key */ - static final DataflowReadChannel countBy(final DataflowReadChannel source ) { + @Deprecated + DataflowWriteChannel countBy(final DataflowReadChannel source ) { countBy(source, { it }) } @@ -681,7 +650,8 @@ class DataflowExtensions { * @param criteria * @return */ - static final DataflowReadChannel countBy(final DataflowReadChannel source, final Closure criteria ) { + @Deprecated + DataflowWriteChannel countBy(final DataflowReadChannel source, final Closure criteria ) { final target = new DataflowVariable() @@ -692,7 +662,6 @@ class DataflowExtensions { return map } - NodeMarker.addOperatorNode('countBy', source, target) return target } @@ -702,10 +671,9 @@ class DataflowExtensions { * @param source The source channel * @return A {@code DataflowVariable} returning the minimum value */ - static final DataflowReadChannel min(final DataflowReadChannel source) { + DataflowWriteChannel min(final DataflowReadChannel source) { final target = new DataflowVariable() reduceImpl(source, target, null) { min, val -> val DataflowReadChannel min(final DataflowReadChannel source, Closure comparator) { + DataflowWriteChannel min(final DataflowReadChannel source, Closure comparator) { def action if( comparator.getMaximumNumberOfParameters() == 1 ) { - action = (Closure){ min, item -> comparator.call(item) < comparator.call(min) ? item : min } + action = (Closure){ min, item -> comparator.call(item) < comparator.call(min) ? item : min } } else if( comparator.getMaximumNumberOfParameters() == 2 ) { - action = (Closure){ a, b -> comparator.call(a,b) < 0 ? a : b } + action = (Closure){ a, b -> comparator.call(a,b) < 0 ? a : b } } final target = new DataflowVariable() reduceImpl(source, target, null, action) - NodeMarker.addOperatorNode('min', source, target) return target } @@ -742,10 +709,9 @@ class DataflowExtensions { * @param comparator The a {@code Comparator} object * @return A {@code DataflowVariable} returning the minimum value */ - static final DataflowReadChannel min(final DataflowQueue source, Comparator comparator) { + DataflowWriteChannel min(final DataflowReadChannel source, Comparator comparator) { final target = new DataflowVariable() reduceImpl(source, target, null) { a, b -> comparator.compare(a,b)<0 ? a : b } - NodeMarker.addOperatorNode('min', source, target) return target } @@ -755,10 +721,9 @@ class DataflowExtensions { * @param source The source channel * @return A {@code DataflowVariable} emitting the maximum value */ - static final DataflowReadChannel max(final DataflowQueue source) { + DataflowWriteChannel max(final DataflowReadChannel source) { final target = new DataflowVariable() reduceImpl(source,target, null) { max, val -> val>max ? val : max } - NodeMarker.addOperatorNode('max', source, target) return target } @@ -772,14 +737,14 @@ class DataflowExtensions { * parameter and return a Comparable (typically an Integer) which is then used for further comparison * @return A {@code DataflowVariable} emitting the maximum value */ - static final DataflowReadChannel max(final DataflowQueue source, Closure comparator) { + DataflowWriteChannel max(final DataflowReadChannel source, Closure comparator) { def action if( comparator.getMaximumNumberOfParameters() == 1 ) { - action = (Closure){ max, item -> comparator.call(item) > comparator.call(max) ? item : max } + action = (Closure){ max, item -> comparator.call(item) > comparator.call(max) ? item : max } } else if( comparator.getMaximumNumberOfParameters() == 2 ) { - action = (Closure){ a, b -> comparator.call(a,b)>0 ? a : b } + action = (Closure){ a, b -> comparator.call(a,b)>0 ? a : b } } else { throw new IllegalArgumentException("Comparator closure can accept at most 2 arguments") @@ -787,7 +752,6 @@ class DataflowExtensions { final target = new DataflowVariable() reduceImpl(source, target, null, action) - NodeMarker.addOperatorNode('max', source, target) return target } @@ -798,10 +762,9 @@ class DataflowExtensions { * @param comparator A {@code Comparator} object * @return A {@code DataflowVariable} emitting the maximum value */ - static final DataflowVariable max(final DataflowQueue source, Comparator comparator) { + DataflowVariable max(final DataflowReadChannel source, Comparator comparator) { final target = new DataflowVariable() reduceImpl(source, target, null) { a, b -> comparator.compare(a,b)>0 ? a : b } - NodeMarker.addOperatorNode('max', source, target) return target } @@ -812,22 +775,20 @@ class DataflowExtensions { * @param closure A closure that given an entry returns the value to sum * @return A {@code DataflowVariable} emitting the final sum value */ - static final DataflowReadChannel sum(final DataflowQueue source, Closure closure = null) { + DataflowWriteChannel sum(final DataflowReadChannel source, Closure closure = null) { def target = new DataflowVariable() def aggregate = new Aggregate(name: 'sum', action: closure) subscribeImpl(source, [onNext: aggregate.&process, onComplete: { target.bind( aggregate.result ) }]) - NodeMarker.addOperatorNode('sum', source, target) return target } - static final DataflowReadChannel mean(final DataflowQueue source, Closure closure = null) { + DataflowWriteChannel mean(final DataflowReadChannel source, Closure closure = null) { def target = new DataflowVariable() def aggregate = new Aggregate(name: 'mean', action: closure, mean: true) subscribeImpl(source, [onNext: aggregate.&process, onComplete: { target.bind( aggregate.result ) }]) - NodeMarker.addOperatorNode('mean', source, target) return target } @@ -878,8 +839,8 @@ class DataflowExtensions { * @param mapper * @return */ - static final DataflowReadChannel groupBy(final DataflowReadChannel source, final params = null ) { - log.warn "Operator `groupBy` is deprecated and it will be removed in a future release" + @Deprecated + DataflowWriteChannel groupBy(final DataflowReadChannel source, final params = null ) { int index = 0 Closure mapper = DEFAULT_MAPPING_CLOSURE @@ -903,7 +864,6 @@ class DataflowExtensions { return map } - NodeMarker.addOperatorNode('groupBy', source, target) return target } @@ -916,8 +876,8 @@ class DataflowExtensions { * @param targets The routing map i.e. a {@code Map} associating each key to the target channel * @param mapper A optional mapping function that given an entry return its key */ - static final void route( final DataflowReadChannel source, Map targets, Closure mapper = DEFAULT_MAPPING_CLOSURE ) { - log.warn "Operator `route` is deprecated -- It will be removed in a future release" + @Deprecated + void route( final DataflowReadChannel source, Map targets, Closure mapper = DEFAULT_MAPPING_CLOSURE ) { DataflowHelper.subscribeImpl(source, [ @@ -938,23 +898,22 @@ class DataflowExtensions { ] ) - NodeMarker.addOperatorNode('route', source, targets.values()) } - static final DataflowReadChannel route( final DataflowReadChannel source, final Closure mapper = DEFAULT_MAPPING_CLOSURE ) { + @Deprecated + DataflowWriteChannel route( final DataflowReadChannel source, final Closure mapper = DEFAULT_MAPPING_CLOSURE ) { assert !(source instanceof DataflowExpression) - log.warn "Operator `route` is deprecated and it will be removed in a future release" def allChannels = new ConcurrentHashMap() - DataflowQueue target = new DataflowQueue() + def target = ChannelFactory.create() - DataflowHelper.subscribeImpl(source, + subscribeImpl(source, [ onNext: { value -> def key = mapper ? mapper.call(value) : value def channel = allChannels.get(key) if( channel == null ) { - channel = new DataflowQueue() + channel = ChannelFactory.create() allChannels[key] = channel // emit the key - channel pair target << [ key, channel ] @@ -971,22 +930,32 @@ class DataflowExtensions { ] ) - NodeMarker.addOperatorNode('route', source, target) return target } - static final DataflowReadChannel spread( final DataflowReadChannel source, Object other ) { + DataflowWriteChannel spread( final DataflowReadChannel source, Object other ) { - final target = new DataflowQueue() + final target = ChannelFactory.create() def inputs switch(other) { - case DataflowQueue: inputs = ToListOp.apply((DataflowQueue)other); break - case DataflowExpression: inputs = other; break - case Collection: inputs = Channel.value(other); break - case (Object[]): inputs = Channel.value(other as List); break - default: throw new IllegalArgumentException("Not a valid argument for 'spread' operator [${other?.class?.simpleName}]: ${other} -- Use a Collection or a channel instead. ") + case DataflowExpression: + inputs = other + break + case DataflowReadChannel: + inputs = ToListOp.apply((DataflowReadChannel)other); + break + case Collection: + inputs = Channel.value(other) + OpCall.current.get().inputs.add(inputs) + break + case (Object[]): + inputs = Channel.value(other as List) + OpCall.current.get().inputs.add(inputs) + break + default: + throw new IllegalArgumentException("Not a valid argument for 'spread' operator [${other?.class?.simpleName}]: ${other} -- Use a Collection or a channel instead. ") } final stopOnFirst = source instanceof DataflowExpression @@ -999,8 +968,8 @@ class DataflowExtensions { } @Override - public boolean onException(final DataflowProcessor processor, final Throwable e) { - DataflowExtensions.log.error("@unknown", e) + boolean onException(final DataflowProcessor processor, final Throwable e) { + OperatorEx.log.error("@unknown", e) session.abort(e) return true; } @@ -1020,43 +989,38 @@ class DataflowExtensions { .each{ Collection it -> proc.bindOutput(it.flatten()) } } - def sources = new ArrayList(2) - sources.add ( source ) - sources.add ( other instanceof DataflowChannel ? other : inputs ) - NodeMarker.addOperatorNode('spread', sources, target) return target } - static final DataflowReadChannel combine( DataflowReadChannel left, Object right ) { + DataflowWriteChannel combine( DataflowReadChannel left, Object right ) { combine(left, null, right) } - static final DataflowReadChannel combine( DataflowReadChannel left, Map params, Object right ) { + DataflowWriteChannel combine( DataflowReadChannel left, Map params, Object right ) { checkParams('combine', params, [flat:Boolean, by: [List,Integer]]) final op = new CombineOp(left,right) final sources = op.inputs if( params?.by != null ) op.pivot = params.by final target = op.apply() - NodeMarker.addOperatorNode('combine', sources, target) return target } - static final DataflowReadChannel flatten( final DataflowReadChannel source ) { + DataflowWriteChannel flatten( final DataflowReadChannel source ) { final listeners = [] - final target = new DataflowQueue() + final target = ChannelFactory.create() if( source instanceof DataflowExpression ) { listeners << new DataflowEventAdapter() { @Override - public void afterRun(final DataflowProcessor processor, final List messages) { + void afterRun(final DataflowProcessor processor, final List messages) { processor.bindOutput( Channel.STOP ) processor.terminate() } - public boolean onException(final DataflowProcessor processor, final Throwable e) { - DataflowExtensions.log.error("@unknown", e) + boolean onException(final DataflowProcessor processor, final Throwable e) { + OperatorEx.log.error("@unknown", e) session.abort(e) return true; } @@ -1084,7 +1048,6 @@ class DataflowExtensions { } } - NodeMarker.addOperatorNode('flatten', source, target) return target } @@ -1096,17 +1059,16 @@ class DataflowExtensions { * @param closingCriteria A condition that has to be verified to close * @return A newly created dataflow queue which emitted the gathered values as bundles */ - static final DataflowReadChannel buffer( final DataflowReadChannel source, Map params=null, Object closingCriteria ) { + DataflowWriteChannel buffer( final DataflowReadChannel source, Map params=null, Object closingCriteria ) { def target = new BufferOp(source) .setParams(params) .setCloseCriteria(closingCriteria) .apply() - NodeMarker.addOperatorNode('buffer', source, target) return target } - static final DataflowReadChannel buffer( final DataflowReadChannel source, Object startingCriteria, Object closingCriteria ) { + DataflowWriteChannel buffer( final DataflowReadChannel source, Object startingCriteria, Object closingCriteria ) { assert startingCriteria != null assert closingCriteria != null @@ -1115,23 +1077,21 @@ class DataflowExtensions { .setCloseCriteria(closingCriteria) .apply() - NodeMarker.addOperatorNode('buffer', source, target) return target } - static final DataflowReadChannel buffer( DataflowReadChannel source, Map params ) { + DataflowWriteChannel buffer( DataflowReadChannel source, Map params ) { checkParams( 'buffer', params, 'size','skip','remainder' ) def target = new BufferOp(source) .setParams(params) .apply() - NodeMarker.addOperatorNode('buffer', source, target) return target } - static final DataflowReadChannel collate( DataflowReadChannel source, int size, boolean keepRemainder = true ) { + DataflowWriteChannel collate( DataflowReadChannel source, int size, boolean keepRemainder = true ) { if( size <= 0 ) { throw new IllegalArgumentException("Illegal argument 'size' for operator 'collate' -- it must be greater than zero: $size") } @@ -1140,11 +1100,10 @@ class DataflowExtensions { .setParams( size: size, remainder: keepRemainder ) .apply() - NodeMarker.addOperatorNode('collate', source, target) return target } - static final DataflowReadChannel collate( DataflowReadChannel source, int size, int step, boolean keepRemainder = true ) { + DataflowWriteChannel collate( DataflowReadChannel source, int size, int step, boolean keepRemainder = true ) { if( size <= 0 ) { throw new IllegalArgumentException("Illegal argument 'size' for operator 'collate' -- it must be greater than zero: $size") } @@ -1154,7 +1113,7 @@ class DataflowExtensions { } // the result queue - final target = new DataflowQueue(); + final target = ChannelFactory.create() // the list holding temporary collected elements List> allBuffers = [] @@ -1174,7 +1133,7 @@ class DataflowExtensions { @Override boolean onException(DataflowProcessor processor, Throwable e) { - DataflowExtensions.log.error("@unknown", e) + OperatorEx.log.error("@unknown", e) session.abort(e) return true } @@ -1200,7 +1159,6 @@ class DataflowExtensions { } - NodeMarker.addOperatorNode('collate', source, target) return target } @@ -1212,39 +1170,38 @@ class DataflowExtensions { * @param others * @return */ - static final DataflowReadChannel mix( DataflowReadChannel source, DataflowReadChannel... others ) { + DataflowWriteChannel mix( DataflowReadChannel source, DataflowReadChannel[] others ) { assert others.size()>0 - def target = new DataflowQueue() + def target = ChannelFactory.create() def count = new AtomicInteger( others.size()+1 ) def handlers = [ onNext: { target << it }, onComplete: { if(count.decrementAndGet()==0) { target << Channel.STOP } } ] - DataflowHelper.subscribeImpl(source, handlers) - others.each{ DataflowHelper.subscribeImpl(it, handlers) } + subscribeImpl(source, handlers) + for( def it : others ) { + subscribeImpl(it, handlers) + } def allSources = [source] allSources.addAll(others) - NodeMarker.addOperatorNode('mix', allSources, target) return target } - static final DataflowReadChannel join( DataflowReadChannel left, right ) { + DataflowWriteChannel join( DataflowReadChannel left, right ) { if( right==null ) throw new IllegalArgumentException("Operator `join` argument cannot be null") if( !(right instanceof DataflowReadChannel) ) throw new IllegalArgumentException("Invalid operator `join` argument [${right.getClass().getName()}] -- it must be a channel type") def target = new JoinOp(left,right) .apply() - NodeMarker.addOperatorNode('join', [left, right], target) return target } - static final DataflowReadChannel join( DataflowReadChannel left, Map opts, right ) { + DataflowWriteChannel join( DataflowReadChannel left, Map opts, right ) { if( right==null ) throw new IllegalArgumentException("Operator `join` argument cannot be null") if( !(right instanceof DataflowReadChannel) ) throw new IllegalArgumentException("Invalid operator `join` argument [${right.getClass().getName()}] -- it must be a channel type") def target = new JoinOp(left,right,opts) .apply() - NodeMarker.addOperatorNode('join', [left, right], target) return target } @@ -1257,24 +1214,22 @@ class DataflowExtensions { * @param mapper * @return */ - static final DataflowReadChannel phase( DataflowReadChannel source, Map opts, DataflowReadChannel other, Closure mapper = null ) { + DataflowWriteChannel phase( DataflowReadChannel source, Map opts, DataflowReadChannel other, Closure mapper = null ) { def target = new PhaseOp(source,other) .setMapper(mapper) .setOpts(opts) .apply() - NodeMarker.addOperatorNode('phase', [source, other], target) return target } - static final DataflowReadChannel phase( DataflowReadChannel source, DataflowReadChannel other, Closure mapper = null ) { + DataflowWriteChannel phase( DataflowReadChannel source, DataflowReadChannel other, Closure mapper = null ) { def target = new PhaseOp(source,other) .setMapper(mapper) .apply() - NodeMarker.addOperatorNode('phase', [source, other], target) return target } @@ -1291,7 +1246,8 @@ class DataflowExtensions { * @return */ - static Closure DEFAULT_MAPPING_CLOSURE = { obj, int index=0 -> + @PackageScope + static public Closure DEFAULT_MAPPING_CLOSURE = { obj, int index=0 -> switch( obj ) { @@ -1328,13 +1284,12 @@ class DataflowExtensions { } - static DataflowReadChannel cross( DataflowReadChannel source, DataflowReadChannel other, Closure mapper = null ) { + DataflowWriteChannel cross( DataflowReadChannel source, DataflowReadChannel other, Closure mapper = null ) { def target = new CrossOp(source, other) .setMapper(mapper) .apply() - NodeMarker.addOperatorNode('cross', [source,other], target) return target } @@ -1346,14 +1301,13 @@ class DataflowExtensions { * @param others * @return */ - static final DataflowWriteChannel concat( DataflowReadChannel source, DataflowReadChannel... others ) { + DataflowWriteChannel concat( DataflowReadChannel source, DataflowReadChannel... others ) { def target = new ConcatOp(source, others).apply() def allSources = [source] if(others) allSources.addAll(others) - NodeMarker.addOperatorNode('concat', allSources, target) return target } @@ -1366,42 +1320,41 @@ class DataflowExtensions { * @param source The source channel * @param outputs An open array of target channels */ - static void separate( DataflowReadChannel source, final DataflowWriteChannel... outputs ) { + @Deprecated + void separate( DataflowReadChannel source, final DataflowWriteChannel... outputs ) { new SeparateOp(source, outputs as List).apply() - NodeMarker.addOperatorNode('separate', source, outputs) } - static void separate(final DataflowReadChannel source, final List> outputs) { + @Deprecated + void separate(final DataflowReadChannel source, final List outputs) { new SeparateOp(source, outputs).apply() - NodeMarker.addOperatorNode('separate', source, outputs) } - static void separate(final DataflowReadChannel source, final List> outputs, final Closure> code) { + @Deprecated + void separate(final DataflowReadChannel source, final List outputs, final Closure code) { new SeparateOp(source, outputs, code).apply() - NodeMarker.addOperatorNode('separate', source, outputs) } - static final List separate( final DataflowReadChannel source, int n ) { + @Deprecated + List separate( final DataflowReadChannel source, int n ) { def outputs = new SeparateOp(source, n).apply() - NodeMarker.addOperatorNode('separate', source, outputs) return outputs } - static final List separate( final DataflowReadChannel source, int n, Closure mapper ) { + @Deprecated + List separate( final DataflowReadChannel source, int n, Closure mapper ) { def outputs = new SeparateOp(source, n, mapper).apply() - NodeMarker.addOperatorNode('separate', source, outputs) return outputs } - - static final void into( DataflowReadChannel source, final DataflowWriteChannel... targets ) { + @Deprecated + void into( DataflowReadChannel source, final DataflowWriteChannel... targets ) { new IntoOp(source, targets as List).apply() - NodeMarker.addOperatorNode('into', source, targets) } - static final List into( final DataflowReadChannel source, int n ) { + @Deprecated + List into( final DataflowReadChannel source, int n ) { def outputs = new IntoOp(source,n).apply().getOutputs() - NodeMarker.addOperatorNode('into', source, outputs) return outputs } @@ -1419,13 +1372,13 @@ class DataflowExtensions { * @param source The source dataflow channel which items are copied into newly created dataflow variables. * @param holder A closure that defines one or more variable names into which source items are copied. */ - static void into( DataflowReadChannel source, Closure holder ) { + @Deprecated + void into( DataflowReadChannel source, Closure holder ) { def outputs = new IntoOp(source,holder).apply().getOutputs() - NodeMarker.addOperatorNode('into', source, outputs) + OpCall.current.get().outputs.addAll(outputs) } - /** * Implements a tap that create implicitly a new dataflow variable in the global script context. * For example: @@ -1440,41 +1393,17 @@ class DataflowExtensions { * @param holder The closure defining the new variable name * @return The tap resulting dataflow channel */ - static DataflowReadChannel tap( final DataflowReadChannel source, final Closure holder ) { + DataflowWriteChannel tap( final DataflowReadChannel source, final Closure holder ) { def tap = new TapOp(source, holder).apply() - NodeMarker.addOperatorNode('tap', source, tap.outputs) - return (DataflowReadChannel)tap.result + OpCall.current.get().outputs.addAll( tap.outputs ) + return tap.result } - static DataflowReadChannel tap( final DataflowReadChannel source, final DataflowWriteChannel target ) { + DataflowWriteChannel tap( final DataflowReadChannel source, final DataflowWriteChannel target ) { def tap = new TapOp(source, target).apply() - NodeMarker.addOperatorNode('tap', source, tap.outputs) - return (DataflowReadChannel)tap.result + return tap.result } - /** - * Assign the {@code source} channel to a global variable with the name specified by the closure. - * For example: - * 
-     *     Channel.from( ... )
-     *            .map { ... }
-     *            .set { newChannelName }
-     *
- * - * @param DataflowReadChannel - * @param holder A closure that must define a single variable expression - */ - static void set( DataflowReadChannel source, Closure holder ) { - final name = CaptureProperties.capture(holder) - if( !name ) - throw new IllegalArgumentException("Missing name to which set the channel variable") - - if( name.size()>1 ) - throw new IllegalArgumentException("Operation `set` does not allow more than one target name") - - final binding = Global.session.binding - binding.setVariable(name[0], source) - } /** * Empty the specified value only if the source channel to which is applied is empty i.e. do not emit @@ -1484,10 +1413,10 @@ class DataflowExtensions { * @param value The value to emit when the source channel is empty. If a closure is used the the value returned by its invocation is used. * @return The resulting channel emitting the source items or the default value when the channel is empty */ - static DataflowReadChannel ifEmpty( DataflowReadChannel source, value ) { + DataflowWriteChannel ifEmpty( DataflowReadChannel source, value ) { boolean empty = true - final result = newChannelBy(source) + final result = ChannelFactory.createBy(source) final singleton = result instanceof DataflowExpression final next = { result.bind(it); empty=false } final complete = { @@ -1499,7 +1428,6 @@ class DataflowExtensions { subscribeImpl(source, [onNext: next, onComplete: complete]) - NodeMarker.addOperatorNode('ifEmpty', source, result) return result } @@ -1508,10 +1436,9 @@ class DataflowExtensions { * @param source * @param closure */ - static void print(final DataflowReadChannel source, Closure closure = null) { + void print(final DataflowReadChannel source, Closure closure = null) { final print0 = { def obj = closure ? closure.call(it) : it; session.printConsole(obj?.toString(),false) } subscribeImpl(source, [onNext: print0]) - NodeMarker.addOperatorNode('print', source, null) } /** @@ -1519,10 +1446,9 @@ class DataflowExtensions { * @param source * @param closure */ - static void println(final DataflowReadChannel source, Closure closure = null) { + void println(final DataflowReadChannel source, Closure closure = null) { final print0 = { def obj = closure ? closure.call(it) : it; session.printConsole(obj?.toString(),true) } subscribeImpl(source, [onNext: print0]) - NodeMarker.addOperatorNode('println', source, null) } @@ -1535,109 +1461,63 @@ class DataflowExtensions { * @param closure * @return */ - static final DataflowReadChannel view(final DataflowReadChannel source, Map opts, Closure closure = null) { + DataflowWriteChannel view(final DataflowReadChannel source, Map opts, Closure closure = null) { assert source != null checkParams('view', opts, PARAMS_VIEW) final newLine = opts.newLine != false - final target = newChannelBy(source); - - final apply = [ + final target = ChannelFactory.createBy(source); + final apply = new HashMap(2) - onNext: - { - final obj = closure != null ? closure.call(it) : it - session.printConsole(obj?.toString(), newLine) - target.bind(it) - }, + apply.onNext = { + final obj = closure != null ? closure.call(it) : it + session.printConsole(obj?.toString(), newLine) + target.bind(it) + } - onComplete: { - target.close() - } - ] + apply. onComplete = { ChannelFactory.close0(target) } subscribeImpl(source,apply) - - NodeMarker.addOperatorNode('view', source, target) - return target; - - } - - static final DataflowReadChannel view(final DataflowReadChannel source, Closure closure = null) { - view(source, [:], closure) - } - - /** - * Creates a channel emitting the entries in the collection to which is applied - * @param values - * @return - */ - static DataflowQueue channel(Collection values) { - def target = new DataflowQueue() - def itr = values.iterator() - while( itr.hasNext() ) target.bind(itr.next()) - target.bind(Channel.STOP) - NodeMarker.addSourceNode('channel',target) return target } - @Deprecated - static DataflowBroadcast broadcast( DataflowReadChannel source ) { - log.warn("Operator `broadcast` is deprecated -- It will be removed in a future release") - def result = new DataflowBroadcast() - source.into(result) - return result - } - - /** - * Close a dataflow queue channel binding a {@link Channel#STOP} item - * - * @param source The source dataflow channel to be closed. - */ - static close( DataflowReadChannel source ) { - if( source instanceof DataflowQueue ) { - source.bind(Channel.STOP) - } - else if( source instanceof DataflowExpression ) { - if( !source.isBound() ) - source.bind(Channel.STOP) - } - else { - log.warn "Operator `close` cannot be applied to channels of type: ${source?.class?.simpleName}" - } - return source + DataflowWriteChannel view(final DataflowReadChannel source, Closure closure = null) { + view(source, Collections.emptyMap(), closure) } - static void choice(final DataflowReadChannel source, final List> outputs, final Closure code) { + void choice(final DataflowReadChannel source, final List outputs, final Closure code) { new ChoiceOp(source,outputs,code).apply() - NodeMarker.addOperatorNode('choice', source, outputs) } - static DataflowReadChannel merge(final DataflowReadChannel source, final DataflowReadChannel other, final Closure closure=null) { - final result = newChannelBy(source) + // NO DAG + @Deprecated + DataflowWriteChannel merge(final DataflowReadChannel source, final DataflowReadChannel other, final Closure closure=null) { + final result = ChannelFactory.createBy(source) final inputs = [source, other] final action = closure ? new ChainWithClosure<>(closure) : new DefaultMergeClosure(inputs.size()) final listener = stopErrorListener(source,result) final params = createOpParams(inputs, result, listener) newOperator(params, action) - NodeMarker.addOperatorNode('merge', inputs, result) return result; } - static DataflowReadChannel merge(final DataflowReadChannel source, final DataflowReadChannel... others) { - final result = newChannelBy(source) + // NO DAG + @Deprecated + DataflowWriteChannel merge(final DataflowReadChannel source, final DataflowReadChannel... others) { + final result = ChannelFactory.createBy(source) final List inputs = new ArrayList(1 + others.size()) inputs.add(source) inputs.addAll(others) final listener = stopErrorListener(source,result) final params = createOpParams(inputs, result, listener) newOperator(params, new DefaultMergeClosure(1 + others.size())) - NodeMarker.addOperatorNode('merge', inputs, result) return result; } - static DataflowReadChannel merge(final DataflowReadChannel source, final List others, final Closure closure=null) { - final result = newChannelBy(source) + // NO DAG + @Deprecated + DataflowWriteChannel merge(final DataflowReadChannel source, final List others, final Closure closure=null) { + final result = ChannelFactory.createBy(source) final List inputs = new ArrayList(1 + others.size()) final action = closure ? new ChainWithClosure<>(closure) : new DefaultMergeClosure(1 + others.size()) inputs.add(source) @@ -1645,130 +1525,96 @@ class DataflowExtensions { final listener = stopErrorListener(source,result) final params = createOpParams(inputs, result, listener) newOperator(params, action) - NodeMarker.addOperatorNode('merge', inputs, result) return result; } - static DataflowReadChannel randomSample(final DataflowReadChannel source, int n, Long seed = null) { + DataflowWriteChannel randomSample(DataflowReadChannel source, int n, Long seed = null) { if( source instanceof DataflowExpression ) throw new IllegalArgumentException("Operator `randomSample` cannot be applied to a value channel") final result = new RandomSampleOp(source,n, seed).apply() - NodeMarker.addOperatorNode('randomSample', source, result) - return result; + return result } - static DataflowReadChannel toInteger(final DataflowReadChannel source) { - final DataflowReadChannel target = newChannelBy(source) - newOperator(source, target, new ChainWithClosure({ it -> it as Integer })) - NodeMarker.addOperatorNode('toInteger', source, target) + DataflowWriteChannel toInteger(final DataflowReadChannel source) { + final target = ChannelFactory.createBy(source) + newOperator(source, target, new ChainWithClosure({ it -> it as Integer })) return target; } - static DataflowReadChannel toLong(final DataflowReadChannel source) { - final DataflowReadChannel target = newChannelBy(source) - newOperator(source, target, new ChainWithClosure({ it -> it as Long })) - NodeMarker.addOperatorNode('toLong', source, target) + DataflowWriteChannel toLong(final DataflowReadChannel source) { + final target = ChannelFactory.createBy(source) + newOperator(source, target, new ChainWithClosure({ it -> it as Long })) return target; } - static DataflowReadChannel toFloat(final DataflowReadChannel source) { - final DataflowReadChannel target = newChannelBy(source) - newOperator(source, target, new ChainWithClosure({ it -> it as Float })) - NodeMarker.addOperatorNode('toFloat', source, target) + DataflowWriteChannel toFloat(final DataflowReadChannel source) { + final target = ChannelFactory.createBy(source) + newOperator(source, target, new ChainWithClosure({ it -> it as Float })) return target; } - static DataflowReadChannel toDouble(final DataflowReadChannel source) { - final DataflowReadChannel target = newChannelBy(source) - newOperator(source, target, new ChainWithClosure({ it -> it as Double })) - NodeMarker.addOperatorNode('toDouble', source, target) + DataflowWriteChannel toDouble(final DataflowReadChannel source) { + final target = ChannelFactory.createBy(source) + newOperator(source, target, new ChainWithClosure({ it -> it as Double })) return target; } - static final DataflowReadChannel transpose( final DataflowReadChannel source, final Map params=null ) { + DataflowWriteChannel transpose( final DataflowReadChannel source, final Map params=null ) { def result = new TransposeOp(source,params).apply() - NodeMarker.addOperatorNode('transpose', source, result) return result } - static final DataflowReadChannel dump(final DataflowReadChannel source, Closure closure = null) { - dump(source, Collections.emptyMap(), closure) - } - - static final DataflowReadChannel dump(final DataflowReadChannel source, Map opts, Closure closure = null) { - def op = new DumpOp(source, opts, closure) - if( op.isEnabled() ) { - def target = op.apply() - NodeMarker.addOperatorNode('dump', source, target) - return target; - } - else { - return source - } - } - - - static DataflowReadChannel splitText(DataflowReadChannel source, Map opts=null) { + DataflowWriteChannel splitText(DataflowReadChannel source, Map opts=null) { final result = new SplitOp( source, 'splitText', opts ).apply() - NodeMarker.addOperatorNode('splitText', source, result) return result } - static DataflowReadChannel splitText(DataflowReadChannel source, Map opts=null, Closure action) { + DataflowWriteChannel splitText(DataflowReadChannel source, Map opts=null, Closure action) { if( opts==null && action ) { opts = new HashMap<>(5) } opts.put('each', action) final result = new SplitOp( source, 'splitText', opts ).apply() - NodeMarker.addOperatorNode('splitText', source, result) return result } - static DataflowReadChannel splitCsv(DataflowReadChannel source, Map opts=null) { + DataflowWriteChannel splitCsv(DataflowReadChannel source, Map opts=null) { final result = new SplitOp( source, 'splitCsv', opts ).apply() - NodeMarker.addOperatorNode('splitCsv', source, result) return result } - static DataflowReadChannel splitFasta(DataflowReadChannel source, Map opts=null) { + DataflowWriteChannel splitFasta(DataflowReadChannel source, Map opts=null) { final result = new SplitOp( source, 'splitFasta', opts ).apply() - NodeMarker.addOperatorNode('splitFasta', source, result) return result } - static DataflowReadChannel splitFastq(DataflowReadChannel source, Map opts=null) { + DataflowWriteChannel splitFastq(DataflowReadChannel source, Map opts=null) { final result = new SplitOp( source, 'splitFastq', opts ).apply() - NodeMarker.addOperatorNode('splitFastq', source, result) return result } - static DataflowReadChannel countLines(DataflowReadChannel source, Map opts=null) { + DataflowWriteChannel countLines(DataflowReadChannel source, Map opts=null) { final splitter = new TextSplitter() final result = countOverChannel( source, splitter, opts ) - NodeMarker.addOperatorNode('countLines', source, result) return result } - static DataflowReadChannel countFasta(DataflowReadChannel source, Map opts=null) { + DataflowWriteChannel countFasta(DataflowReadChannel source, Map opts=null) { final splitter = new FastaSplitter() final result = countOverChannel( source, splitter, opts ) - NodeMarker.addOperatorNode('countFasta', source, result) return result } - static DataflowReadChannel countFastq(DataflowReadChannel source, Map opts=null) { + DataflowWriteChannel countFastq(DataflowReadChannel source, Map opts=null) { final splitter = new FastqSplitter() final result = countOverChannel( source, splitter, opts ) - NodeMarker.addOperatorNode('countFastq', source, result) return result } - - static DataflowReadChannel countText(DataflowReadChannel source) { - log.warn "Method `countText` has been deprecated -- Use `countLines` instead" + @Deprecated + DataflowWriteChannel countText(DataflowReadChannel source) { countLines(source) } - } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/PhaseOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/PhaseOp.groovy index b4cf74b721..58bae143e2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/PhaseOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/PhaseOp.groovy @@ -19,14 +19,13 @@ import java.util.concurrent.atomic.AtomicInteger import groovy.transform.CompileStatic import groovy.transform.PackageScope -import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Channel import nextflow.util.CheckHelper /** - * Implements {@link DataflowExtensions#phase} operator logic + * Implements {@link OperatorEx#phase} operator logic * * @author Paolo Di Tommaso */ @@ -41,7 +40,7 @@ class PhaseOp { private DataflowReadChannel target - private Closure mapper = DataflowExtensions.DEFAULT_MAPPING_CLOSURE + private Closure mapper = OperatorEx.DEFAULT_MAPPING_CLOSURE PhaseOp( DataflowReadChannel source, DataflowReadChannel target ) { this.source = source @@ -56,14 +55,14 @@ class PhaseOp { } PhaseOp setMapper( Closure mapper ) { - this.mapper = mapper ?: DataflowExtensions.DEFAULT_MAPPING_CLOSURE + this.mapper = mapper ?: OperatorEx.DEFAULT_MAPPING_CLOSURE return this } - DataflowQueue apply() { + DataflowWriteChannel apply() { - def result = new DataflowQueue() - def state = [:] + def result = ChannelFactory.create() + def state = new LinkedHashMap() final count = 2 final stopCount = new AtomicInteger(count) @@ -84,10 +83,10 @@ class PhaseOp { * @param mapper A closure mapping a value to its key * @return A map with {@code OnNext} and {@code onComplete} methods entries */ - static private final Map phaseHandler( Map> buffer, int size, int index, DataflowWriteChannel target, Closure mapper, AtomicInteger stopCount, boolean remainder ) { + static private final Map phaseHandler( Map> buffer, int size, int index, DataflowWriteChannel target, Closure mapper, AtomicInteger stopCount, boolean remainder ) { - [ - onNext: { + DataflowHelper.eventsMap( + { synchronized (buffer) { def entries = phaseImpl(buffer, size, index, it, mapper, false) if( entries ) { @@ -95,14 +94,14 @@ class PhaseOp { } }}, - onComplete: { + { if( stopCount.decrementAndGet()==0) { if( remainder ) phaseRemainder(buffer,size, target) target << Channel.STOP }} - ] + ) } @@ -167,7 +166,7 @@ class PhaseOp { Iterator> itr = channels.iterator() while( itr.hasNext() ) { - def entry = itr.next() + def entry = (Map.Entry)itr.next() def list = entry.getValue() result << list[0] diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy index 4fb4b765ea..2b3dc00d46 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy @@ -17,10 +17,13 @@ package nextflow.extension import groovy.transform.CompileStatic -import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Channel +import static DataflowHelper.eventsMap +import static DataflowHelper.subscribeImpl + /** * Implements Reservoir sampling of channel content * @@ -33,7 +36,7 @@ class RandomSampleOp { private DataflowReadChannel source - private DataflowQueue result + private DataflowWriteChannel result private int N @@ -75,9 +78,9 @@ class RandomSampleOp { result.bind(Channel.STOP) } - DataflowQueue apply() { - result = new DataflowQueue() - DataflowHelper.subscribeImpl(source, [onNext: this.&sampling, onComplete: this.&emit]) + DataflowWriteChannel apply() { + result = ChannelFactory.create() + subscribeImpl(source, eventsMap(this.&sampling, this.&emit)) return result } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/SeparateOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/SeparateOp.groovy index aad4573b59..fd0a6a9ee9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/SeparateOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/SeparateOp.groovy @@ -25,7 +25,7 @@ import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.SeparationClosure /** - * Implements the {@link DataflowExtensions#separate} operator logic + * Implements the {@link OperatorEx#separate} operator logic * * @author Paolo Di Tommaso */ diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy index a4afd4f31f..ff0a56f566 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy @@ -15,10 +15,10 @@ */ package nextflow.extension + import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.util.logging.Slf4j -import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Channel @@ -103,7 +103,7 @@ class SplitOp { // turn off channel auto-close params.autoClose = false - if( params.into && !(params.into instanceof DataflowQueue) ) + if( params.into && !(ChannelFactory.isChannelQueue(params.into)) ) throw new IllegalArgumentException('Parameter `into` must reference a channel object') } @@ -141,7 +141,7 @@ class SplitOp { } // -- now merge the result - def output = new DataflowQueue() + def output = ChannelFactory.create() applyMergingOperator(splitted, output, indexes) return output } @@ -152,7 +152,7 @@ class SplitOp { protected DataflowWriteChannel splitSingleEntry(DataflowReadChannel origin, Map params) { // -- get the output channel - final output = getOrCreateDataflowQueue(params) + final output = getOrCreateWriteChannel(params) // -- the output channel is passed to the splitter by using the `into` parameter params.into = output @@ -198,14 +198,14 @@ class SplitOp { } @PackageScope - DataflowWriteChannel getOrCreateDataflowQueue(Map params) { + DataflowWriteChannel getOrCreateWriteChannel(Map params) { def result // create a new DataflowChannel that will receive the splitter entries - if( params.into instanceof DataflowQueue ) { - result = (DataflowQueue)params.into + if( params.into instanceof DataflowWriteChannel ) { + result = (DataflowWriteChannel)params.into } else { - result = new DataflowQueue<>() + result = ChannelFactory.create() } return result diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy index f3e5f4b5a3..42bc9f5e43 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy @@ -15,15 +15,17 @@ */ package nextflow.extension + import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.ChainWithClosure import groovyx.gpars.dataflow.operator.CopyChannelsClosure -import nextflow.Global +import nextflow.NF +import static nextflow.extension.DataflowHelper.newOperator /** - * Implements the {@link DataflowExtensions#tap} operator + * Implements the {@link OperatorEx#tap} operator * * @author Paolo Di Tommaso */ @@ -57,7 +59,7 @@ class TapOp { assert holder != null this.source = source - this.result = DataflowExtensions.newChannelBy(source) + this.result = ChannelFactory.createBy(source) this.outputs = [result] // -- set the target variable in the script binding context @@ -65,9 +67,9 @@ class TapOp { if( !names ) throw new IllegalArgumentException("Missing target channel on `tap` operator") - final binding = Global.session.binding + final binding = NF.binding names.each { item -> - def channel = DataflowExtensions.newChannelBy(source) + def channel = ChannelFactory.createBy(source) if( binding.hasVariable(item) ) log.warn "A variable named '${item}' already exists in script global context -- Consider renaming it " @@ -91,7 +93,7 @@ class TapOp { } this.source = source - this.result = DataflowExtensions.newChannelBy(source) + this.result = ChannelFactory.createBy(source) this.outputs = [result, target] } @@ -105,9 +107,8 @@ class TapOp { * @return An instance of {@link TapOp} itself */ TapOp apply() { - DataflowHelper.newOperator([source], outputs, new ChainWithClosure(new CopyChannelsClosure())); + newOperator([source], outputs, new ChainWithClosure(new CopyChannelsClosure())); return this } - } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy index edb978fe04..17f5569e35 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy @@ -22,7 +22,7 @@ import groovyx.gpars.dataflow.DataflowVariable import groovyx.gpars.dataflow.expression.DataflowExpression /** - * Implements {@link DataflowExtensions#toList(groovyx.gpars.dataflow.DataflowReadChannel)} operator + * Implements {@link OperatorEx#toList(groovyx.gpars.dataflow.DataflowReadChannel)} operator * * @author Paolo Di Tommaso */ diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy index 39170e11a5..38cc577c4b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy @@ -15,9 +15,9 @@ */ package nextflow.extension + import groovy.transform.CompileStatic import groovy.transform.PackageScope -import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Channel @@ -43,7 +43,7 @@ class TransposeOp { TransposeOp(DataflowReadChannel source, Map params=null) { CheckHelper.checkParams('transpose', params, TRANSPOSE_PARAMS) this.source = source - this.target = new DataflowQueue() + this.target = ChannelFactory.create() this.cols = parseByParam(params?.by) this.remainder = params?.remainder as Boolean } @@ -63,9 +63,9 @@ class TransposeOp { return [value as int] } - DataflowQueue apply() { - DataflowHelper.subscribeImpl(source, [onNext: this.&transpose, onComplete: this.&done]) - return (DataflowQueue)target + DataflowWriteChannel apply() { + DataflowHelper.subscribeImpl(source, DataflowHelper.eventsMap(this.&transpose, this.&done)) + return target } protected void transpose(item) { diff --git a/modules/nextflow/src/main/groovy/nextflow/file/PathVisitor.groovy b/modules/nextflow/src/main/groovy/nextflow/file/PathVisitor.groovy index 79e59004b5..32cd6dfb13 100644 --- a/modules/nextflow/src/main/groovy/nextflow/file/PathVisitor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/file/PathVisitor.groovy @@ -27,9 +27,10 @@ import java.util.regex.Pattern import groovy.transform.CompileStatic import groovy.transform.Memoized import groovy.transform.PackageScope -import groovyx.gpars.dataflow.DataflowQueue +import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Global import nextflow.Session +import nextflow.extension.ChannelFactory import nextflow.util.CustomThreadFactory import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -46,7 +47,7 @@ class PathVisitor { private static Logger log = LoggerFactory.getLogger(PathVisitor) - DataflowQueue target + DataflowWriteChannel target boolean closeChannelOnComplete = true @@ -56,12 +57,12 @@ class PathVisitor { boolean forcePattern - DataflowQueue apply(Object filePattern) { + DataflowWriteChannel apply(Object filePattern) { if( opts == null ) opts = [:] if( !target ) - target = new DataflowQueue() + target = ChannelFactory.create() if( filePattern instanceof Pattern ) applyRegexPattern0(filePattern) diff --git a/modules/nextflow/src/main/groovy/nextflow/k8s/K8sExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/k8s/K8sExecutor.groovy index 74d269dcd4..88a110fbe5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/k8s/K8sExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/k8s/K8sExecutor.groovy @@ -41,7 +41,7 @@ class K8sExecutor extends Executor { /** * The Kubernetes HTTP client */ - static private K8sClient client + private K8sClient client @PackageScope K8sClient getClient() { client @@ -59,7 +59,8 @@ class K8sExecutor extends Executor { /** * Initialise the executor setting-up the kubernetes client configuration */ - void register() { + @Override + protected void register() { super.register() final config = k8sConfig.getClient() client = new K8sClient(config) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/ProcessFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/ProcessFactory.groovy deleted file mode 100755 index 6e4d047aa0..0000000000 --- a/modules/nextflow/src/main/groovy/nextflow/processor/ProcessFactory.groovy +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Copyright 2013-2019, Centre for Genomic Regulation (CRG) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.processor - -import groovy.util.logging.Slf4j -import nextflow.Session -import nextflow.cloud.aws.batch.AwsBatchExecutor -import nextflow.executor.CondorExecutor -import nextflow.executor.CrgExecutor -import nextflow.executor.Executor -import nextflow.executor.LocalExecutor -import nextflow.executor.LsfExecutor -import nextflow.executor.NopeExecutor -import nextflow.executor.NqsiiExecutor -import nextflow.executor.PbsExecutor -import nextflow.executor.PbsProExecutor -import nextflow.executor.SgeExecutor -import nextflow.executor.SlurmExecutor -import nextflow.executor.SupportedScriptTypes -import nextflow.k8s.K8sExecutor -import nextflow.script.BaseScript -import nextflow.script.ScriptType -import nextflow.script.TaskBody -import nextflow.util.ServiceDiscover -import nextflow.util.ServiceName -/** - * Factory class for {@TaskProcessor} instances - * - * @author Paolo Di Tommaso - */ -@Slf4j -class ProcessFactory { - - static public String DEFAULT_EXECUTOR = System.getenv('NXF_EXECUTOR') ?: 'local' - - /* - * Map the executor class to its 'friendly' name - */ - final protected Map executorsMap = [ - 'nope': NopeExecutor, - 'local': LocalExecutor, - 'sge': SgeExecutor, - 'oge': SgeExecutor, - 'uge': SgeExecutor, - 'lsf': LsfExecutor, - 'pbs': PbsExecutor, - 'pbspro': PbsProExecutor, - 'slurm': SlurmExecutor, - 'crg': CrgExecutor, - 'bsc': LsfExecutor, - 'condor': CondorExecutor, - 'k8s': K8sExecutor, - 'nqsii': NqsiiExecutor, - 'awsbatch': AwsBatchExecutor - ] - - private final Session session - - private final Map config - - private BaseScript owner - - /* only for test -- do not use */ - protected ProcessFactory() { - - } - - ProcessFactory( BaseScript ownerScript, Session session, Map config = null ) { - this.owner = ownerScript - this.session = session - this.config = config != null ? config : session.config - - // discover non-core executors - for( Class clazz : ServiceDiscover.load(Executor) ) { - log.trace "Discovered executor class: ${clazz.name}" - executorsMap.put(findNameByClass(clazz), clazz) - } - } - - /** - * Create a new task processor and initialise with the given parameters - * - * @param name The processor name - * @param executor The executor object - * @param session The session object - * @param script The owner script - * @param config The process configuration - * @param taskBody The process task body - * @return An instance of {@link TaskProcessor} - */ - protected TaskProcessor newTaskProcessor( String name, Executor executor, Session session, BaseScript script, ProcessConfig config, TaskBody taskBody ) { - new TaskProcessor(name, executor, session, script, config, taskBody) - } - - /** - * Extract the executor name by using the annotation {@code ServiceName} or fallback to simple classname - * if the annotation is not provided - * - * @param clazz - * @return - */ - static String findNameByClass( Class clazz ) { - def annotation = clazz.getAnnotation(ServiceName) - if( annotation ) - return annotation.value() - - def name = clazz.getSimpleName().toLowerCase() - if( name.endsWith('executor') ) { - name = name.subSequence(0, name.size()-'executor'.length()) - } - - return name - } - - protected Class loadExecutorClass(String executorName) { - log.debug ">> processorType: '$executorName'" - if( !executorName ) - return LocalExecutor - - def clazz = executorsMap[executorName.toLowerCase()] - if( !clazz ) - throw new IllegalArgumentException("Unknown executor name: $executorName") - - if( clazz instanceof Class ) - return clazz - - if( !(clazz instanceof String ) ) - throw new IllegalArgumentException("Not a valid executor class object: $clazz") - - // if the className is empty (because the 'processorType' does not map to any class, fallback to the 'processorType' itself) - if( !clazz ) { - clazz = executorName - } - - log.debug "Loading executor class: ${clazz}" - try { - Thread.currentThread().getContextClassLoader().loadClass(clazz as String) as Class - } - catch( Exception e ) { - throw new IllegalArgumentException("Cannot find a valid class for specified executor: '${executorName}'") - } - - } - - - protected boolean isTypeSupported( ScriptType type, executor ) { - - if( executor instanceof Executor ) { - executor = executor.class - } - - if( executor instanceof Class ) { - def annotation = executor.getAnnotation(SupportedScriptTypes) - if( !annotation ) - throw new IllegalArgumentException("Specified argument is not a valid executor class: $executor -- Missing 'SupportedScriptTypes' annotation") - - return type in annotation.value() - } - - throw new IllegalArgumentException("Specified argument is not a valid executor class: $executor") - } - - /** - * Create a task processor - * - * @param name - * The name of the process as defined in the script - * @param body - * The process declarations provided by the user - * @param options - * A map representing the named parameter specified after the process name eg: - * `process foo(bar: 'x') { }` - * (not used) - * @return - * The {@code Processor} instance - */ - TaskProcessor createProcessor( String name, Closure body, Map options = null ) { - assert body - assert config.process instanceof Map - - /* - * check if exists 'attributes' defined in the 'process' scope for this process, e.g. - * - * process.$name.attribute1 = xxx - * process.$name.attribute2 = yyy - * - * NOTE: THIS HAS BEEN DEPRECATED AND WILL BE REMOVED - */ - Map legacySettings = null - if( config.process['$'+name] instanceof Map ) { - legacySettings = (Map)config.process['$'+name] - log.warn "Process configuration syntax \$processName has been deprecated -- Replace `process.\$$name = ` with a process selector" - } - - // -- the config object - final processConfig = new ProcessConfig(owner).setProcessName(name) - - // Invoke the code block which will return the script closure to the executed. - // As side effect will set all the property declarations in the 'taskConfig' object. - processConfig.throwExceptionOnMissingProperty(true) - final copy = (Closure)body.clone() - copy.setResolveStrategy(Closure.DELEGATE_FIRST) - copy.setDelegate(processConfig) - final script = copy.call() as TaskBody - processConfig.throwExceptionOnMissingProperty(false) - if ( !script ) - throw new IllegalArgumentException("Missing script in the specified process block -- make sure it terminates with the script string to be executed") - - // -- Apply the directives defined in the config object using the`withLabel:` syntax - final processLabels = processConfig.getLabels() ?: [''] - for( String lbl : processLabels ) { - processConfig.applyConfigForLabel(config.process as Map, "withLabel:", lbl) - } - - // -- apply setting for name - processConfig.applyConfigForLabel(config.process as Map, "withName:", name) - - // -- Apply process specific setting defined using `process.$name` syntax - // NOTE: this is deprecated and will be removed - if( legacySettings ) { - processConfig.applyConfigSettings(legacySettings) - } - - // -- Apply defaults - processConfig.applyConfigDefaults( config.process as Map ) - - // -- check for conflicting settings - if( processConfig.scratch && processConfig.stageInMode == 'rellink' ) { - log.warn("Directives `scratch` and `stageInMode=rellink` conflict each other -- Enforcing default stageInMode for process `$name`") - processConfig.remove('stageInMode') - } - - // -- load the executor to be used - def execName = getExecutorName(processConfig) ?: DEFAULT_EXECUTOR - def execClass = loadExecutorClass(execName) - - if( !isTypeSupported(script.type, execClass) ) { - log.warn "Process '$name' cannot be executed by '$execName' executor -- Using 'local' executor instead" - execName = 'local' - execClass = LocalExecutor.class - } - - def execObj = execClass.newInstance() - // -- inject the task configuration into the executor instance - execObj.session = session - execObj.name = execName - execObj.init() - - // -- create processor class - newTaskProcessor( name, execObj, session, owner, processConfig, script ) - } - - - /** - * Find out the 'executor' to be used in the process definition or in teh session configuration object - * - * @param taskConfig - */ - private getExecutorName(ProcessConfig taskConfig) { - // create the processor object - def result = taskConfig.executor?.toString() - - if( !result ) { - if( session.config.executor instanceof String ) { - result = session.config.executor - } - else if( session.config.executor?.name instanceof String ) { - result = session.config.executor.name - } - } - - log.debug "<< taskConfig executor: $result" - return result - } -} diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskContext.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskContext.groovy index be47e069f0..8b510ca67e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskContext.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskContext.groovy @@ -15,6 +15,7 @@ */ package nextflow.processor + import java.nio.file.Path import java.nio.file.Paths import java.util.concurrent.atomic.AtomicBoolean @@ -27,6 +28,7 @@ import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Global import nextflow.exception.ProcessException +import nextflow.script.ScriptBinding import nextflow.util.KryoHelper /** * Map used to delegate variable resolution to script scope @@ -102,7 +104,7 @@ class TaskContext implements Map, Cloneable { /** * @return The inner map holding the process variables */ - public Map getHolder() { holder } + Map getHolder() { holder } private void setHolder( Map holder ) { this.holder = holder @@ -111,14 +113,14 @@ class TaskContext implements Map, Cloneable { /** * @return The script instance to which this map reference i.e. the main script object */ - public Script getScript() { script } + Script getScript() { script } /** * @return * The set of variable and properties referenced in the user script. * NOTE: it includes properties in the form {@code object.propertyName} */ - public Set getVariableNames() { variableNames } + Set getVariableNames() { variableNames } @Override String toString() { @@ -126,7 +128,7 @@ class TaskContext implements Map, Cloneable { } @Override - public Object get(Object property) { + Object get(Object property) { assert property if( holder.containsKey(property) ) { @@ -140,6 +142,7 @@ class TaskContext implements Map, Cloneable { throw new MissingPropertyException("Unknown variable '$property' -- Make sure it is not misspelt and defined somewhere in the script before using it", property as String, null) } + @Override Object invokeMethod(String name, Object args) { if( name == 'template' ) template(args) @@ -147,16 +150,18 @@ class TaskContext implements Map, Cloneable { script.invokeMethod(name, args) } - public getProperty( String name ) { + @Override + def getProperty( String name ) { get((String)name) } - public void setProperty( String name, def value ) { + @Override + void setProperty( String name, def value ) { put(name, value) } @Override - public put(String property, Object newValue) { + put(String property, Object newValue) { if( property == 'task' && !(newValue instanceof TaskConfig ) && !overrideWarnShown.getAndSet(true) ) { log.warn "Process $name overrides reserved variable `task`" @@ -166,7 +171,7 @@ class TaskContext implements Map, Cloneable { } - def byte[] serialize() { + byte[] serialize() { try { def map = holder if( map.get(TaskProcessor.TASK_CONTEXT_PROPERTY_NAME) instanceof TaskConfig ) { @@ -201,7 +206,7 @@ class TaskContext implements Map, Cloneable { /** * Serialize the {@code DelegateMap} instance to a byte array */ - def byte[] dehydrate() { + byte[] dehydrate() { def kryo = KryoHelper.kryo() def buffer = new ByteArrayOutputStream(5*1024) def out = new Output(buffer) @@ -212,21 +217,21 @@ class TaskContext implements Map, Cloneable { kryo.writeObject(out, script.class) // -- only the binding values for which there's an entry in the holder map - final copy = new Binding() + final copy = new HashMap(20) variableNames.each { String it -> // name can be a property, in this case use the root object def p = it.indexOf('.') def var = ( p == -1 ? it : it.substring(0,p) ) checkAndSet(var, copy) } - log.trace "Delegate for $name > binding copy: ${copy.getVariables()}" + log.trace "Delegate for $name > binding copy: ${copy}" kryo.writeObject(out, copy) out.flush() return buffer.toByteArray() } - private void checkAndSet( String name, Binding target ) { + private void checkAndSet( String name, Map target ) { final binding = this.script.getBinding() if( !binding.hasVariable(name) ) @@ -237,7 +242,7 @@ class TaskContext implements Map, Cloneable { return if( val instanceof Path || val instanceof Serializable ) { - target.setVariable(name, val) + target.put(name, val) } } @@ -257,7 +262,7 @@ class TaskContext implements Map, Cloneable { assert binary final kryo = KryoHelper.kryo() - def ClassLoader prev = null + ClassLoader prev = null if( loader ) { prev = kryo.getClassLoader() kryo.setClassLoader(loader) @@ -268,10 +273,9 @@ class TaskContext implements Map, Cloneable { def name = input.readString() Map holder = (Map)kryo.readClassAndObject(input) Class