aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorArun Isaac2024-10-02 00:55:32 +0100
committerArun Isaac2024-10-02 02:08:58 +0100
commit707774f68f11f82cf2756bae406938b380c35d8b (patch)
treeadfe7a5070c55d4617ea4c436a2fbd283f9ade5a
parentcc72d725cb20496e64dbc58bf86477142b8378b7 (diff)
downloadravanan-707774f68f11f82cf2756bae406938b380c35d8b.tar.gz
ravanan-707774f68f11f82cf2756bae406938b380c35d8b.tar.lz
ravanan-707774f68f11f82cf2756bae406938b380c35d8b.zip
command-line-tool: Allow per-batch system requirements support.
* ravanan/command-line-tool.scm (command-line-tool-supported-requirements): New public function. (check-requirements): Accept batch-system and supported-requirements-for-batch-system arguments. * ravanan/command-line-tool.scm (run-command-line-tool): Pass batch-system to build-command-line-tool-script. (build-command-line-tool-script): Accept batch-system argument. Update calls to check-requirements. * ravanan/workflow.scm (%workflow-only-requirements): New variable. (%supported-requirements): Use %workflow-only-requirements. (supported-requirements): New function. (workflow-class->propnet): Accept batch-system argument. Update calls to check-requirements. (workflow-scheduler)[schedule]: Pass batch-system to workflow-class->propnet.
-rw-r--r--ravanan/command-line-tool.scm55
-rw-r--r--ravanan/workflow.scm29
2 files changed, 64 insertions, 20 deletions
diff --git a/ravanan/command-line-tool.scm b/ravanan/command-line-tool.scm
index 1e32767..6a48a20 100644
--- a/ravanan/command-line-tool.scm
+++ b/ravanan/command-line-tool.scm
@@ -48,6 +48,7 @@
check-requirements
inherit-requirements
%command-line-tool-supported-requirements
+ command-line-tool-supported-requirements
script->store-stdout-file
script->store-stderr-file
capture-command-line-tool-output
@@ -70,6 +71,9 @@
"InlineJavascriptRequirement"
"InitialWorkDirRequirement"))
+(define (command-line-tool-supported-requirements batch-system)
+ %command-line-tool-supported-requirements)
+
;; node executable for evaluating javascript on worker nodes
(define %worker-node
(file-append node "/bin/node"))
@@ -99,18 +103,34 @@
(load-listing? output-binding-load-listing?)
(output-eval output-binding-output-eval))
-(define* (check-requirements requirements supported-requirements
+(define* (check-requirements requirements
+ batch-system
+ supported-requirements-for-batch-system
+ supported-requirements
#:optional hint?)
- "Error out if any of @var{requirements} are not in
-@var{supported-requirements}. If @var{hint?} is @code{#t}, only print a warning."
+ "Error out if any of @var{requirements} are not supported by @var{batch-system}.
+If @var{hint?} is @code{#t}, only print a warning.
+@var{supported-requirements-for-batch-system} is a function that when passed a
+batch system returns the requirements supported by it.
+@var{supported-requirements} is the list of requirements supported by at least
+one batch system."
(vector-for-each (lambda (requirement)
(let ((class (assoc-ref requirement "class")))
- (unless (member class supported-requirements)
+ (unless (member class
+ (supported-requirements-for-batch-system batch-system))
(if hint?
- (warning "Ignoring ~a hint; it is not supported"
- class)
- (user-error "Requirement ~a not supported"
- class)))))
+ (if (member class supported-requirements)
+ (warning "Ignoring ~a hint; it is not supported for batch system ~a"
+ class
+ batch-system)
+ (warning "Ignoring ~a hint; it is not supported"
+ class))
+ (if (member class supported-requirements)
+ (user-error "Requirement ~a not supported for batch system ~a"
+ class
+ batch-system)
+ (user-error "Requirement ~a not supported"
+ class))))))
requirements))
(define (inherit-requirements requirements supplementary-requirements)
@@ -384,7 +404,8 @@ path."
;; TODO: Write to the store atomically.
(let* ((script
(build-command-line-tool-script name manifest cwl inputs
- scratch store guix-daemon-socket))
+ scratch store batch-system
+ guix-daemon-socket))
(store-files-directory (script->store-files-directory script store))
(store-data-file (script->store-data-file script store))
(stdout-file (script->store-stdout-file script store))
@@ -548,10 +569,11 @@ order of keys."
(else tree)))
(define (build-command-line-tool-script name manifest cwl inputs
- scratch store guix-daemon-socket)
+ scratch store batch-system
+ guix-daemon-socket)
"Build and return script to run @code{CommandLineTool} class workflow @var{cwl}
named @var{name} with @var{inputs} using tools from Guix manifest
-@var{manifest}.
+@var{manifest} and on @var{batch-system}.
@var{scratch}, @var{store} and @var{guix-daemon-socket} are the same as in
@code{run-workflow} from @code{(ravanan workflow)}."
@@ -685,9 +707,16 @@ named @var{name} with @var{inputs} using tools from Guix manifest
(newline out)))))
(maybe-let* ((requirements (maybe-assoc-ref (just cwl) "requirements")))
- (check-requirements requirements %command-line-tool-supported-requirements))
+ (check-requirements requirements
+ batch-system
+ command-line-tool-supported-requirements
+ %command-line-tool-supported-requirements))
(maybe-let* ((hints (maybe-assoc-ref (just cwl) "hints")))
- (check-requirements hints %command-line-tool-supported-requirements #t))
+ (check-requirements hints
+ batch-system
+ command-line-tool-supported-requirements
+ %command-line-tool-supported-requirements
+ #t))
;; Copy input files and update corresponding input objects.
(build-gexp-script name
(let* ((requirements (inherit-requirements (or (assoc-ref cwl "requirements")
diff --git a/ravanan/workflow.scm b/ravanan/workflow.scm
index 92aec96..af720a8 100644
--- a/ravanan/workflow.scm
+++ b/ravanan/workflow.scm
@@ -38,10 +38,17 @@
#:use-module (ravanan work vectors)
#:export (run-workflow))
+(define %workflow-only-requirements
+ (list "ScatterFeatureRequirement"
+ "SubworkflowFeatureRequirement"))
+
(define %supported-requirements
- (cons* "ScatterFeatureRequirement"
- "SubworkflowFeatureRequirement"
- %command-line-tool-supported-requirements))
+ (append %workflow-only-requirements
+ %command-line-tool-supported-requirements))
+
+(define (supported-requirements batch-system)
+ (append %workflow-only-requirements
+ (command-line-tool-supported-requirements batch-system)))
;; In batch systems that require it, poll job completion status every
;; 5 seconds.
@@ -188,9 +195,10 @@ propagator."
(assoc-ref output "id")))
(assoc-ref cwl "outputs"))))
-(define* (workflow-class->propnet name cwl scheduler)
- "Return a propagator network scheduled using @var{scheduler} for @var{cwl}, a
-@code{Workflow} class workflow with @var{name}."
+(define* (workflow-class->propnet name cwl scheduler batch-system)
+ "Return a propagator network scheduled using @var{scheduler} on
+@var{batch-system} for @var{cwl}, a @code{Workflow} class workflow with
+@var{name}."
(define (normalize-scatter-method scatter-method)
(assoc-ref* '(("dotproduct" . dot-product)
("nested_crossproduct" . nested-cross-product)
@@ -233,9 +241,13 @@ propagator."
(maybe-let* ((requirements (maybe-assoc-ref (just cwl) "requirements")))
(check-requirements requirements
+ batch-system
+ supported-requirements
%supported-requirements))
(maybe-let* ((hints (maybe-assoc-ref (just cwl) "hints")))
(check-requirements hints
+ batch-system
+ supported-requirements
%supported-requirements
#t))
(propnet (vector-map->list step->propagator
@@ -296,7 +308,10 @@ job state object. @var{proc} may either be a @code{<propnet>} object or a
((string=? class "ExpressionTool")
(error "Workflow class not implemented yet" class))
((string=? class "Workflow")
- (workflow-state (schedule-propnet (workflow-class->propnet name cwl scheduler)
+ (workflow-state (schedule-propnet (workflow-class->propnet name
+ cwl
+ scheduler
+ batch-system)
inputs)
(assoc-ref* cwl "outputs"))))))