diff options
| -rw-r--r-- | .guix/ccwl-distribution.scm | 8 | ||||
| -rw-r--r-- | .guix/ccwl-package.scm | 73 | ||||
| -rw-r--r-- | .guix/ccwl-website.scm | 74 | ||||
| -rw-r--r-- | HACKING.md | 27 | ||||
| -rw-r--r-- | Makefile | 10 | ||||
| -rw-r--r-- | NEWS.org | 4 | ||||
| -rw-r--r-- | README.org | 11 | ||||
| -rw-r--r-- | build-aux/build-home-page.el | 5 | ||||
| -rw-r--r-- | build-aux/test-driver.scm | 92 | ||||
| -rw-r--r-- | ccwl/ccwl.scm | 119 | ||||
| -rw-r--r-- | ccwl/conditions.scm | 4 | ||||
| -rw-r--r-- | ccwl/cwl.scm | 101 | ||||
| -rw-r--r-- | ccwl/utils.scm | 28 | ||||
| -rw-r--r-- | doc/ccwl.skb | 10 | ||||
| -rw-r--r-- | doc/prefix-arguments.scm | 5 | ||||
| -rw-r--r-- | doc/unseparated-prefix-arguments.scm | 2 | ||||
| -rwxr-xr-x | scripts/ccwl | 101 | ||||
| -rw-r--r-- | tests/ccwl.scm | 32 | ||||
| -rw-r--r-- | tests/cwl.scm | 16 | ||||
| -rw-r--r-- | tests/ui.scm | 5 | ||||
| -rw-r--r-- | tests/utils.scm | 14 | ||||
| -rw-r--r-- | website/releases/ccwl-0.5.0.tar.lz | bin | 0 -> 47416 bytes | |||
| -rw-r--r-- | website/releases/ccwl-0.5.0.tar.lz.asc | 11 |
23 files changed, 446 insertions, 306 deletions
diff --git a/.guix/ccwl-distribution.scm b/.guix/ccwl-distribution.scm index 2b97f94..0598a00 100644 --- a/.guix/ccwl-distribution.scm +++ b/.guix/ccwl-distribution.scm @@ -1,5 +1,5 @@ ;;; ccwl --- Concise Common Workflow Language -;;; Copyright © 2024 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2024, 2026 Arun Isaac <arunisaac@systemreboot.net> ;;; ;;; This file is part of ccwl. ;;; @@ -54,14 +54,14 @@ "GUILE_LOAD_COMPILED_PATH" (list (string-append "/lib/guile/" (target-guile-effective-version) "/site-ccache")) (list #$development-profile)) - (invoke "git" "clone" #$ccwl-git-repo (getcwd)) + (invoke "git" "clone" (string-append "file://" #$ccwl-git-repo) (getcwd)) (invoke "sh" "configure") (invoke "make" "dist") (match (scandir (getcwd) (cut string-suffix? ".tar.lz" <>)) ((tarball) - (install-file tarball #$output))))))) + (copy-file tarball #$output))))))) (define-public ccwl-distribution - (computed-file "ccwl-distribution" ccwl-distribution-gexp)) + (computed-file "ccwl.tar.lz" ccwl-distribution-gexp)) ccwl-distribution diff --git a/.guix/ccwl-package.scm b/.guix/ccwl-package.scm index a70e3e0..2a73dbd 100644 --- a/.guix/ccwl-package.scm +++ b/.guix/ccwl-package.scm @@ -1,5 +1,5 @@ ;;; ccwl --- Concise Common Workflow Language -;;; Copyright © 2021, 2023–2024 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2021, 2023–2026 Arun Isaac <arunisaac@systemreboot.net> ;;; ;;; This file is part of ccwl. ;;; @@ -18,15 +18,27 @@ (define-module (ccwl-package) #:use-module ((gnu packages bioinformatics) #:prefix guix:) - #:use-module ((gnu packages emacs) #:select (emacs-minimal)) - #:use-module ((gnu packages fonts) #:select (font-charter font-fira-code)) - #:use-module ((guix build-system guile) #:select (%guile-build-system-modules)) + #:use-module ((gnu packages skribilo) + #:select (skribilo) #:prefix guix:) + #:use-module (guix download) #:use-module (guix gexp) #:use-module (guix git-download) #:use-module (guix packages) - #:use-module (guix profiles) #:use-module (guix utils)) +(define skribilo + (package + (inherit guix:skribilo) + (name "skribilo") + (version "0.10.0") + (source (origin + (method url-fetch) + (uri (string-append "mirror://savannah/skribilo/skribilo-" + version ".tar.gz")) + (sha256 + (base32 + "03pm2a9a5k0wkj10ywh6xi8flawm8sd396k4698gvvbc2zp4izwc")))))) + (define-public ccwl (package (inherit guix:ccwl) @@ -34,52 +46,9 @@ "ccwl-checkout" #:recursive? #t #:select? (or (git-predicate (dirname (current-source-directory))) - (const #t)))))) - -(define ccwl-website-gexp - (let ((development-profile - (profile - (content (package->development-manifest ccwl)) - (allow-collisions? #t)))) - (with-imported-modules %guile-build-system-modules - #~(begin - (use-modules (guix build guile-build-system) - (guix build utils)) - - (set-path-environment-variable - "PATH" (list "/bin") (list #$development-profile #$emacs-minimal)) - (set-path-environment-variable - "LIBRARY_PATH" (list "/lib") (list #$development-profile)) - (set-path-environment-variable - "GUILE_LOAD_PATH" - (list (string-append "/share/guile/site/" - (target-guile-effective-version))) - (list #$development-profile)) - (set-path-environment-variable - "GUILE_LOAD_COMPILED_PATH" - (list (string-append "/lib/guile/" (target-guile-effective-version) "/site-ccache")) - (list #$development-profile)) - (copy-recursively #$(package-source ccwl) - (getcwd)) - ;; Emacs modifies README.org presumably for the contained - ;; org dynamic block. So, grant write permissions. - (chmod "README.org" #o644) - (for-each patch-shebang - (list "pre-inst-env" - "build-aux/generate-cwl-output.sh" - "scripts/ccwl")) - (substitute* "Makefile" - (("\\$\\(GUIX_ENVIRONMENT\\)") - #$(profile - (content (packages->manifest (list font-charter font-fira-code)))))) - (invoke "sh" "configure") - (invoke "make" - "--jobs" (number->string (parallel-job-count))) - (invoke "make" "website" - "--jobs" (number->string (parallel-job-count))) - (copy-recursively "website" #$output))))) - -(define-public ccwl-website - (computed-file "ccwl-website" ccwl-website-gexp)) + (const #t)))) + (native-inputs + (modify-inputs (package-native-inputs guix:ccwl) + (replace "skribilo" skribilo))))) ccwl diff --git a/.guix/ccwl-website.scm b/.guix/ccwl-website.scm new file mode 100644 index 0000000..754a0d5 --- /dev/null +++ b/.guix/ccwl-website.scm @@ -0,0 +1,74 @@ +;;; ccwl --- Concise Common Workflow Language +;;; Copyright © 2025 Arun Isaac <arunisaac@systemreboot.net> +;;; +;;; This file is part of ccwl. +;;; +;;; ccwl is free software: you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation, either version 3 of the License, or +;;; (at your option) any later version. +;;; +;;; ccwl is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with ccwl. If not, see <https://www.gnu.org/licenses/>. + +(define-module (ccwl-website) + #:use-module ((ccwl-package) #:select (ccwl)) + #:use-module ((gnu packages emacs) #:select (emacs-minimal)) + #:use-module ((gnu packages fonts) #:select (font-charter font-fira-code)) + #:use-module ((guix build-system guile) #:select (%guile-build-system-modules)) + #:use-module (guix gexp) + #:use-module (guix packages) + #:use-module (guix profiles)) + +(define ccwl-website-gexp + (let ((development-profile + (profile + (content (package->development-manifest ccwl)) + (allow-collisions? #t)))) + (with-imported-modules %guile-build-system-modules + #~(begin + (use-modules (guix build guile-build-system) + (guix build utils)) + + (set-path-environment-variable + "PATH" (list "/bin") (list #$development-profile #$emacs-minimal)) + (set-path-environment-variable + "LIBRARY_PATH" (list "/lib") (list #$development-profile)) + (set-path-environment-variable + "GUILE_LOAD_PATH" + (list (string-append "/share/guile/site/" + (target-guile-effective-version))) + (list #$development-profile)) + (set-path-environment-variable + "GUILE_LOAD_COMPILED_PATH" + (list (string-append "/lib/guile/" (target-guile-effective-version) "/site-ccache")) + (list #$development-profile)) + (copy-recursively #$(package-source ccwl) + (getcwd)) + ;; Emacs modifies README.org presumably for the contained + ;; org dynamic block. So, grant write permissions. + (chmod "README.org" #o644) + (for-each patch-shebang + (list "pre-inst-env" + "build-aux/generate-cwl-output.sh" + "scripts/ccwl")) + (substitute* "Makefile" + (("\\$\\(GUIX_ENVIRONMENT\\)") + #$(profile + (content (packages->manifest (list font-charter font-fira-code)))))) + (invoke "sh" "configure") + (invoke "make" + "--jobs" (number->string (parallel-job-count))) + (invoke "make" "website" + "--jobs" (number->string (parallel-job-count))) + (copy-recursively "website" #$output))))) + +(define-public ccwl-website + (computed-file "ccwl-website" ccwl-website-gexp)) + +ccwl-website diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 0000000..bb822b3 --- /dev/null +++ b/HACKING.md @@ -0,0 +1,27 @@ +# Set up development environment + +Drop into a development environment using `guix shell`. +``` +guix shell -Df guix.scm +``` + +# Make a release +## Tag a release +Tag a release `vx.x.x` putting news into the tag message. +## Create a release tarball, test it, and sign it +``` +cp $(guix build -L .guix -f .guix/ccwl-distribution.scm) ccwl-x.x.x.tar.lz +guix build --with-source=ccwl=ccwl-x.x.x.tar.lz -f guix.scm +make distsign +``` +## Build guix pack, docker and singularity images +``` +guix pack --with-source=ccwl=ccwl-x.x.x.tar.lz --file=guix.scm +guix pack -f docker -S /bin=bin --with-source=ccwl=ccwl-x.x.x.tar.lz --file=guix.scm +guix pack -f squashfs --with-source=ccwl=ccwl-x.x.x.tar.lz bash --file=guix.scm +``` +## Publish release tarball +Add release tarball and signature to website. Publish release tarball, guix pack, docker and singularity images to GitHub. +## Update Guix package +## Publicize +Publicize on the ccwl@systemreboot.net and guix-science@gnu.org mailing lists, and on the [CWL Discourse forum](https://cwl.discourse.group/). diff --git a/Makefile b/Makefile index f4487d9..5a9b07f 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # ccwl --- Concise Common Workflow Language -# Copyright © 2022, 2024 Arun Isaac <arunisaac@systemreboot.net> +# Copyright © 2022, 2024–2025 Arun Isaac <arunisaac@systemreboot.net> # # This file is part of ccwl. # @@ -24,12 +24,12 @@ GIT = git GPG = gpg GUILD = guild GUILE = guile +GUILE_RUN64 = guile-run64 LZIP = lzip SKRIBILO = skribilo FIND_DEPENDENCIES = build-aux/find-dependencies.scm GENERATE_CWL_OUTPUT = build-aux/generate-cwl-output.sh -TEST_DRIVER = build-aux/test-driver.scm top_level_module_dir = $(project) sources = $(wildcard $(top_level_module_dir)/*.scm) @@ -46,7 +46,7 @@ fonts = $(addprefix $(GUIX_ENVIRONMENT)/share/fonts/web/, charter_regular.woff2 distribute_files = $(sources) $(scripts) $(tests) $(test_data) \ $(doc_sources) doc/skribilo.scm $(doc_data) $(DOC_SCM) $(DOC_OTHER) \ pre-inst-env guix.scm Makefile configure configure.scm \ - $(FIND_DEPENDENCIES) $(GENERATE_CWL_OUTPUT) $(TEST_DRIVER) \ + $(FIND_DEPENDENCIES) $(GENERATE_CWL_OUTPUT) \ COPYING NEWS.org README.org scmdir = $(datarootdir)/guile/site/$(guile_effective_version)/$(top_level_module_dir) @@ -63,8 +63,8 @@ all: $(objects) # Run tests -check: $(tests) $(TEST_DRIVER) - $(GUILE) --no-auto-compile -L . $(TEST_DRIVER) $(tests) +check: + ./pre-inst-env $(GUILE_RUN64) $(tests) # Build documentation diff --git a/NEWS.org b/NEWS.org index c351602..02fe372 100644 --- a/NEWS.org +++ b/NEWS.org @@ -1,6 +1,8 @@ #+TITLE: ccwl NEWS – History of user-visible changes -Copyright © 2021, 2024, 2025 Arun Isaac <arunisaac@systemreboot.net> +Copyright © 2021, 2024–2026 Arun Isaac <arunisaac@systemreboot.net> + +Changes in 0.5.0 and later are in the git tag messages. * Changes in 0.4.0 (since 0.3.0) ** Documentation diff --git a/README.org b/README.org index 3d8e31a..3084234 100644 --- a/README.org +++ b/README.org @@ -36,14 +36,14 @@ Download the Docker image from [[https://github.com/arunisaac/ccwl/releases][the it. Then, run ccwl with command-line arguments of your choice. #+BEGIN_SRC shell $ docker load -i ccwl-docker.tar.gz - $ docker run ccwl [ARGS...] + $ docker run ccwl ccwl [ARGS...] #+END_SRC When passing files into the Docker container, remember to share the filesystem. The following command is one way to share the current directory with the container. #+BEGIN_SRC shell - $ docker run -v "$PWD:$PWD" -w "$PWD" ccwl compile foo.scm + $ docker run -v "$PWD:$PWD" -w "$PWD" ccwl ccwl compile foo.scm #+END_SRC ** Using Guix @@ -80,8 +80,11 @@ introduction to ccwl. * Contributing Feedback, suggestions, feature requests, bug reports and pull requests -are all welcome. Unclear and unspecific error messages are considered -a bug. Do report them! +are all welcome. Please write to [[mailto:ccwl@systemreboot.net][ccwl@systemreboot.net]]. You may also +browse the [[https://lists.systemreboot.net/ccwl][archives]] of previous discussions. Unclear and unspecific +error messages are considered a bug. Do report them! + +We do not accept issues or pull requests on GitHub. Thank you! To hack on ccwl, you can use GNU Guix to quickly drop into a development environment by running diff --git a/build-aux/build-home-page.el b/build-aux/build-home-page.el index fd76a9e..15867bd 100644 --- a/build-aux/build-home-page.el +++ b/build-aux/build-home-page.el @@ -1,5 +1,5 @@ ;;; ccwl --- Concise Common Workflow Language -;;; Copyright © 2021, 2022, 2024, 2025 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2021, 2022, 2024–2026 Arun Isaac <arunisaac@systemreboot.net> ;;; ;;; This file is part of ccwl. ;;; @@ -41,7 +41,8 @@ Download release tarballs. ") - (dolist (release '(("2025-01-28" "0.4.0") + (dolist (release '(("2026-01-13" "0.5.0") + ("2025-01-28" "0.4.0") ("2024-01-26" "0.3.0") ("2021-11-05" "0.2.0") ("2021-07-06" "0.1.0"))) diff --git a/build-aux/test-driver.scm b/build-aux/test-driver.scm deleted file mode 100644 index adac481..0000000 --- a/build-aux/test-driver.scm +++ /dev/null @@ -1,92 +0,0 @@ -;;; ccwl --- Concise Common Workflow Language -;;; Copyright © 2021, 2022 Arun Isaac <arunisaac@systemreboot.net> -;;; -;;; This file is part of ccwl. -;;; -;;; ccwl is free software: you can redistribute it and/or modify it -;;; under the terms of the GNU General Public License as published by -;;; the Free Software Foundation, either version 3 of the License, or -;;; (at your option) any later version. -;;; -;;; ccwl is distributed in the hope that it will be useful, but -;;; WITHOUT ANY WARRANTY; without even the implied warranty of -;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -;;; General Public License for more details. -;;; -;;; You should have received a copy of the GNU General Public License -;;; along with ccwl. If not, see <https://www.gnu.org/licenses/>. - -;;; Commentary: - -;; This is a better test driver for Guile's (srfi srfi-64). -;; -;; TODO: Improve Guile's test driver so this module won't be -;; necessary. - -;;; Code: - -(use-modules (ice-9 format) - (ice-9 match) - (srfi srfi-26) - (srfi srfi-64) - (term ansi-color)) - -(define (color color-symbol str color?) - (if color? - (colorize-string str color-symbol) - str)) - -(define red (cut color 'RED <> <>)) -(define green (cut color 'GREEN <> <>)) -(define magenta (cut color 'MAGENTA <> <>)) - -(define (my-gnu-runner color?) - (let ((runner (test-runner-null))) - (test-runner-on-group-begin! runner - (lambda (runner suite-name count) - (format #t (magenta "%%%% ~a~%" color?) suite-name))) - (test-runner-on-group-end! runner - (lambda _ - (newline))) - (test-runner-on-test-end! runner - (lambda (runner) - (let ((name (test-runner-test-name runner)) - (result (string-upcase - (symbol->string (test-result-kind runner)))) - (result-alist (test-result-alist runner))) - (format #t "~a ~a~%" - (case (test-result-kind runner) - ((pass) (green result color?)) - (else (red result color?))) - name) - ;; If test did not pass, print details. - (unless (eq? (test-result-kind runner) 'pass) - (format (current-error-port) - "~a:~a~%expected: ~s~%actual: ~s~%" - (assq-ref result-alist 'source-file) - (assq-ref result-alist 'source-line) - (match (assq-ref result-alist 'source-form) - (('test-assert _ ...) #t) - (_ (assq-ref result-alist 'expected-value))) - (assq-ref result-alist 'actual-value)))))) - runner)) - -(match (command-line) - ((_ test-files ...) - (let ((runner (my-gnu-runner #t))) - (test-with-runner runner - (for-each load-from-path test-files) - (display (magenta "SUMMARY" #t)) - (newline) - (format #t "PASS: ~a -FAIL: ~a -XPASS: ~a -XFAIL: ~a -SKIP: ~a -" - (test-runner-pass-count runner) - (test-runner-fail-count runner) - (test-runner-xpass-count runner) - (test-runner-xfail-count runner) - (test-runner-skip-count runner)) - (exit (zero? (test-runner-fail-count runner))))))) diff --git a/ccwl/ccwl.scm b/ccwl/ccwl.scm index 3fc5364..075ae0c 100644 --- a/ccwl/ccwl.scm +++ b/ccwl/ccwl.scm @@ -1,5 +1,5 @@ ;;; ccwl --- Concise Common Workflow Language -;;; Copyright © 2021–2024 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2021–2026 Arun Isaac <arunisaac@systemreboot.net> ;;; ;;; This file is part of ccwl. ;;; @@ -72,6 +72,7 @@ input-default input-position input-prefix + input-separate? input-separator input-stage? input-other @@ -93,7 +94,7 @@ unspecified-default?)) (define-immutable-record-type <input> - (make-input id type label default position prefix stage? other) + (make-input id type label default position prefix separate? separator stage? other) input? (id input-id) (type input-type) @@ -101,6 +102,7 @@ (default input-default set-input-default) (position input-position set-input-position) (prefix input-prefix set-input-prefix) + (separate? input-separate? set-input-separate?) (separator input-separator set-input-separator) (stage? input-stage?) (other input-other)) @@ -206,7 +208,8 @@ compared using @code{equal?}." #,(if (unspecified-default? default) #'(make-unspecified-default) default) - #,position #,prefix #,stage? '#,other))) + #,position #,prefix #f #f + #,stage? '#,other))) #'(id args ...)))) (id (identifier? #'id) (input #'(id))) (_ (error "Invalid input:" (syntax->datum input-spec))))) @@ -279,7 +282,7 @@ compared using @code{equal?}." (define-immutable-record-type <js-expression> (make-js-expression inputs expression outputs requirements other) js-expression? - (inputs js-expression-inputs) + (inputs js-expression-inputs set-js-expression-inputs) (expression js-expression-expression) (outputs js-expression-outputs) (requirements js-expression-requirements) @@ -289,14 +292,14 @@ compared using @code{equal?}." (make-cwl-workflow file inputs outputs) cwl-workflow? (file cwl-workflow-file) - (inputs cwl-workflow-inputs) + (inputs cwl-workflow-inputs set-cwl-workflow-inputs) (outputs cwl-workflow-outputs)) (define-immutable-record-type <workflow> (make-workflow steps inputs outputs other) workflow? (steps workflow-steps) - (inputs workflow-inputs) + (inputs workflow-inputs set-workflow-inputs) (outputs workflow-outputs) (other workflow-other)) @@ -350,13 +353,13 @@ return @code{#f}." (input (identifier? #'input) (syntax->datum #'input)) ;; prefixed input - ((_ input) (identifier? #'input) + ((_ input _ ...) (identifier? #'input) (syntax->datum #'input)) ;; array input specifier ((array input _ ...) (identifier? #'input) (syntax->datum #'input)) ;; prefixed array input specifier - ((_ (array input _ ...)) (identifier? #'input) + ((_ (array input _ ...) _ ...) (identifier? #'input) (syntax->datum #'input)) (_ #f)))) (and run-arg-input @@ -367,7 +370,27 @@ return @code{#f}." "Return the prefix specified in @var{run-arg} syntax. If not a prefixed input, return #f." (syntax-case run-arg () - ((prefix _) #'prefix) + ((prefix _ ...) (string? (syntax->datum #'prefix)) + #'prefix) + (_ #f))) + +(define (validate-separate? separate?) + "Validate @var{separate?} and raise an exception if it is not valid." + (unless (boolean? separate?) + (raise-exception + (condition (ccwl-violation separate?) + (formatted-message "Invalid #:separate? flag ~a. #:separate? flag must be a boolean." + (syntax->datum separate?)))))) + +(define (run-arg-separate? run-arg) + "Return the separate? specified in @var{run-arg} syntax. If not a +prefixed input, return #f." + (syntax-case run-arg (array) + ((prefix _ args ...) (string? (syntax->datum #'prefix)) + (apply (syntax-lambda** (#:key (separate? #'#t)) + (validate-separate? (syntax->datum separate?)) + separate?) + #'(args ...))) (_ #f))) (define (run-arg-separator run-arg) @@ -385,7 +408,7 @@ input, return #f." separator)) #'(args ...))) ;; prefixed array input specifier - ((_ (array input args ...)) + ((_ (array input args ...) _ ...) (run-arg-separator #'(array input args ...))) (_ #f))) @@ -415,14 +438,20 @@ identifiers defined in the commands." (syntax->run-arg #'input)) ;; Flatten prefixed string arguments. They have no ;; special meaning. - ((prefix string-arg) (and (string? (syntax->datum #'prefix)) - (string? (syntax->datum #'string-arg))) - (list #'prefix #'string-arg)) + ((prefix string-arg args ...) (and (string? (syntax->datum #'prefix)) + (string? (syntax->datum #'string-arg))) + (apply (syntax-lambda** (#:key (separate? #'#t)) + (validate-separate? (syntax->datum separate?)) + (if (syntax->datum separate?) + (list #'prefix #'string-arg) + (list #`#,(string-append (syntax->datum #'prefix) + (syntax->datum #'string-arg))))) + #'(args ...))) ;; Recurse on prefixed inputs. - ((prefix input) (string? (syntax->datum #'prefix)) + ((prefix input _ ...) (string? (syntax->datum #'prefix)) (syntax->run-arg #'input)) ;; Prefixes that are not strings - ((prefix _) + ((prefix _ ...) (raise-exception (condition (ccwl-violation #'prefix) (formatted-message "Invalid prefix ~a. Prefixes must be strings." @@ -482,18 +511,21 @@ identifiers defined in the commands." (let* ((id (input-spec-id input-spec)) (run-arg (find-run-arg id run))) #`(set-input-separator - (set-input-prefix - (set-input-position - #,(input input-spec) - ;; `run-args' returns inputs as quoted symbols. - ;; So, we add quote. - #,(list-index (match-lambda - (`(quote ,input) - (eq? input id)) - (_ #f)) - (syntax->datum flattened-args))) + (set-input-separate? + (set-input-prefix + (set-input-position + #,(input input-spec) + ;; `run-args' returns inputs as quoted symbols. + ;; So, we add quote. + #,(list-index (match-lambda + (`(quote ,input) + (eq? input id)) + (_ #f)) + (syntax->datum flattened-args))) + #,(and run-arg + (run-arg-prefix run-arg))) #,(and run-arg - (run-arg-prefix run-arg))) + (run-arg-separate? run-arg))) #,(and run-arg (run-arg-separator run-arg))))) inputs)) @@ -600,7 +632,7 @@ identifiers defined in the commands." ((id . type) (with-syntax ((id (datum->syntax #f id)) (type (datum->syntax #f type))) - #`(make-input 'id 'type #f #f #f #f #f '())))) + #`(make-input 'id 'type #f #f #f #f #f #f #f '())))) (parameters->id+type (assoc-ref yaml "inputs")))) (list #,@(map (match-lambda ((id . type) @@ -612,7 +644,7 @@ identifiers defined in the commands." (define (function-inputs function) "Return the list of inputs accepted by @var{function}---a @code{<command>}, @code{<js-expression>}, @code{<cwl-workflow>} or -@code{<workflow> object." +@code{<workflow>} object." ((cond ((command? function) command-inputs) ((js-expression? function) js-expression-inputs) @@ -621,6 +653,21 @@ identifiers defined in the commands." (else (error "Unrecognized ccwl function" function))) function)) +(define (set-function-inputs function inputs) + "Set inputs of @var{function} to @var{inputs}---a +@code{<command>}, @code{<js-expression>}, @code{<cwl-workflow>} or +@code{<workflow> object. @code{set-function-inputs} is purely +functional. It returns a copy of @var{function}. @var{function} is not +mutated." + ((cond + ((command? function) set-command-inputs) + ((js-expression? function) set-js-expression-inputs) + ((cwl-workflow? function) set-cwl-workflow-inputs) + ((workflow? function) set-workflow-inputs) + (else (error "Unrecognized ccwl function" function))) + function + inputs)) + (define (function-input-keys function) "Return the list of input keys accepted by FUNCTION, a <command>, <js-expression>, <cwl-workflow> or <workflow> object." @@ -647,11 +694,11 @@ identifiers defined in the commands." ;; Global input/output (symbol->string (key-cwl-id key)))) -(define (apply-partially command partial-arguments) - "Return a new command that is a partial application of -@var{partial-arguments} to @var{command}. @var{partial-arguments} is +(define (apply-partially function partial-arguments) + "Return a new function that is a partial application of +@var{partial-arguments} to @var{function}. @var{partial-arguments} is an association list mapping keyword arguments to their values." - (set-command-inputs command + (set-function-inputs function (map (lambda (input) (set-input-default input (or (any (match-lambda @@ -661,7 +708,7 @@ an association list mapping keyword arguments to their values." value))) partial-arguments) (input-default input)))) - (command-inputs command)))) + (function-inputs function)))) (define (function-object x) "Return the ccwl function object (a <command>, <js-expression>, @@ -711,7 +758,7 @@ represented by <step> objects." (list))) ;; tee ((tee expressions ...) - (let ((key-lists step-lists (mapn (cut collect-steps <> input-keys) + (let ((key-lists step-lists (map2 (cut collect-steps <> input-keys) #'(expressions ...)))) (values ;; Global workflow input keys may be duplicated across the @@ -858,7 +905,9 @@ represented by <step> objects." input-keys))))) (pairify (syntax->datum #'(args ...)))))))) ;; If literal values are provided as arguments, partially - ;; apply those literal values to the command and recurse. + ;; apply those literal values to the function object + ;; (command, workflow, js-expression, cwl-workflow) and + ;; recurse. (_ (collect-steps #`(((module-ref (resolve-module '(ccwl ccwl)) 'apply-partially) diff --git a/ccwl/conditions.scm b/ccwl/conditions.scm index c6523cb..abbd404 100644 --- a/ccwl/conditions.scm +++ b/ccwl/conditions.scm @@ -1,5 +1,5 @@ ;;; ccwl --- Concise Common Workflow Language -;;; Copyright © 2022 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2022, 2025 Arun Isaac <arunisaac@systemreboot.net> ;;; ;;; This file is part of ccwl. ;;; @@ -63,7 +63,7 @@ (arguments formatted-message-arguments)) (define (formatted-message format-string . arguments) - "Return &ccwl-message condition for FORMAT-STRING with ARGUMENTS." + "Return &formatted-message condition for FORMAT-STRING with ARGUMENTS." (make-formatted-message (apply format format-string arguments) format-string arguments)) diff --git a/ccwl/cwl.scm b/ccwl/cwl.scm index baef519..f954f08 100644 --- a/ccwl/cwl.scm +++ b/ccwl/cwl.scm @@ -1,5 +1,5 @@ ;;; ccwl --- Concise Common Workflow Language -;;; Copyright © 2021, 2023–2024 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2021, 2023–2025 Arun Isaac <arunisaac@systemreboot.net> ;;; ;;; This file is part of ccwl. ;;; @@ -51,16 +51,34 @@ specification." (scm->yaml (workflow->cwl-scm workflow) port)) -(define (filter-alist alist) - "Filter ALIST removing entries with #f as the value. If the -resulting association list is empty, return #f. Else, return that -association list." - (match (filter (match-lambda - ((_ . #f) #f) - (_ #t)) - alist) - (() #f) - (result result))) +(define (vector-filter-map proc vec) + "Map @var{proc} over @var{vec} and return a vector of the results that +are not @code{#f}." + (list->vector (filter-map proc (vector->list vec)))) + +(define (prune-tree tree) + "Prune JSON-like @var{tree} removing dictionary keys without a mapping." + (match tree + ;; Array + (#(elements ...) + (vector-filter-map (lambda (element) + (match (prune-tree element) + (() #f) + (pruned-element pruned-element))) + tree)) + ;; Dictionary + ((pairs ...) + (filter-map (match-lambda + ;; Key with a mapping + ((key . value) + (match (prune-tree value) + (() #f) + (pruned-value (cons key pruned-value)))) + ;; Key without a mapping + ((key) #f)) + pairs)) + ;; Atom + (atom atom))) (define* (workflow->cwl-scm workflow) "Render WORKFLOW, a <workflow> object, into a CWL tree." @@ -114,17 +132,17 @@ association list." "Render @var{output}, a @code{<output>} object, into a CWL tree. If @var{workflow?} is @code{#t}, this is a workflow output." `(,(output-id output) - ,@(or (filter-alist - `((type . ,(type->cwl (output-type output))) - ;; outputBinding is relevant only to commands, and - ;; outputSource is relevant only to workflows. - ,@(if workflow? - `((outputSource . ,(match (output-source output) - ((? string? source) source) - ((? input? input) (input-id input))))) - `((outputBinding . ,(output-binding output)))))) - '()) - ,@(output-other output))) + ,@(prune-tree + `((type . ,(type->cwl (output-type output))) + ;; outputBinding is relevant only to commands, and + ;; outputSource is relevant only to workflows. + ,@(if workflow? + `((outputSource . ,(match (output-source output) + ((? string? source) source) + ((? input? input) (input-id input))))) + `((outputBinding . ,(or (output-binding output) + '())))) + ,@(output-other output))))) (define (command->cwl command port) "Render @var{command}, a @code{<command>} object, to @var{port} as a @@ -135,21 +153,30 @@ CWL YAML specification." (define (input->cwl-scm input) "Render @var{input}, a @code{<input>} object, into a CWL tree." `(,(input-id input) - (type . ,(type->cwl (input-type input))) - ,@(or (filter-alist - `((label . ,(input-label input)) - (default . ,(and (not (unspecified-default? (input-default input))) - (input-default input))) - ;; inputBinding is only relevant to commands, not - ;; workflows. But, the input position and prefix are not set - ;; for worklow inputs and therefore this sub-expression has - ;; no effect. So, leave this be. - (inputBinding . ,(filter-alist - `((position . ,(input-position input)) - (prefix . ,(input-prefix input)) - (itemSeparator . ,(input-separator input))))))) - '()) - ,@(input-other input))) + ,@(prune-tree + `((type . ,(type->cwl (input-type input))) + (default . ,(if (unspecified-default? (input-default input)) + '() + (input-default input))) + (label . ,(or (input-label input) + '())) + ;; inputBinding is only relevant to commands, not workflows. + ;; But, the input position and prefix are not set for worklow + ;; inputs and therefore this sub-expression has no effect. + ;; So, leave this be. + (inputBinding + (position . ,(or (input-position input) + '())) + (prefix . ,(or (input-prefix input) + '())) + ;; separate? has a meaningful value only with prefix. + (separate . ,(if (input-prefix input) + (and (input-separate? input) + '()) + '())) + (itemSeparator . ,(or (input-separator input) + '()))) + ,@(input-other input))))) (define (staging-requirements inputs) "Return @samp{InitialWorkDirRequirement} to stage any @var{inputs} that diff --git a/ccwl/utils.scm b/ccwl/utils.scm index 3c18efd..2abc404 100644 --- a/ccwl/utils.scm +++ b/ccwl/utils.scm @@ -1,5 +1,5 @@ ;;; ccwl --- Concise Common Workflow Language -;;; Copyright © 2021, 2022, 2023 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2021, 2022, 2023, 2025 Arun Isaac <arunisaac@systemreboot.net> ;;; ;;; This file is part of ccwl. ;;; @@ -39,6 +39,7 @@ lambda** syntax-lambda** mapn + map2 foldn filter-mapi)) @@ -311,22 +312,31 @@ element. For example, lst (iota (length lst)))) -(define (mapn proc lst) +(define (mapn proc lst n) "Map the procedure PROC over list LST and return a list containing -the results. PROC can return multiple values, in which case, an equal -number of lists are returned. For example, +the results. PROC must return N values, in which case, N lists are +returned. For example, (mapn (lambda (n) (values (expt n 2) (expt n 3))) - (iota 5)) + (iota 5) + 2) => (0 1 4 9 16) => (0 1 8 27 64)" (apply values - (apply zip - (map (lambda (x) - (call-with-values (cut proc x) list)) - lst)))) + (match lst + ;; With an empty list, we cannot know the number of values + ;; proc would return. Hence this special case. + (() (make-list n '())) + (_ + (apply zip + (map (lambda (x) + (call-with-values (cut proc x) list)) + lst)))))) + +(define map2 + (cut mapn <> <> 2)) (define (foldn proc lst . inits) "Apply PROC to the elements of LST to build a result, and return diff --git a/doc/ccwl.skb b/doc/ccwl.skb index 9001f20..7096711 100644 --- a/doc/ccwl.skb +++ b/doc/ccwl.skb @@ -1,5 +1,5 @@ ;;; ccwl --- Concise Common Workflow Language -;;; Copyright © 2021, 2023–2024 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2021, 2023–2025 Arun Isaac <arunisaac@systemreboot.net> ;;; ;;; This file is part of ccwl. ;;; @@ -363,6 +363,14 @@ prefix. For example, in the following example, we associate the input ,(code "output_filename") to the prefix ,(code "-o"). Notice the parentheses around ,(code "-o output_filename").] (scheme-source "doc/prefix-arguments.scm"))) + (section :title [Unseparated prefix arguments] + :ident "section-unseparated-prefix-arguments" + (p [Some programs don't like it when you separate arguments from +their prefixes. You can specify this using the ,(code [#:separate?]) +flag.] + (scheme-source "doc/unseparated-prefix-arguments.scm") + [This is executed as ,(samp [gcc foo.c -ofoo]), rather than +as ,(samp [gcc foo.c -o foo]).])) (section :title [Array types] :ident "section-array-types" (p [ccwl supports array types using the following syntax.] diff --git a/doc/prefix-arguments.scm b/doc/prefix-arguments.scm index ec95aeb..bb54620 100644 --- a/doc/prefix-arguments.scm +++ b/doc/prefix-arguments.scm @@ -1,5 +1,2 @@ (command #:inputs (source #:type File) (output_filename #:type string) - #:run "gcc" source ("-o" output_filename) - #:outputs (executable - #:type File - #:binding ((glob . "$(inputs.output_filename)")))) + #:run "gcc" source ("-o" output_filename)) diff --git a/doc/unseparated-prefix-arguments.scm b/doc/unseparated-prefix-arguments.scm new file mode 100644 index 0000000..9e1e767 --- /dev/null +++ b/doc/unseparated-prefix-arguments.scm @@ -0,0 +1,2 @@ +(command #:inputs (source #:type File) (output_filename #:type string) + #:run "gcc" source ("-o" output_filename #:separate? #f)) diff --git a/scripts/ccwl b/scripts/ccwl index 984d22a..fbd549d 100755 --- a/scripts/ccwl +++ b/scripts/ccwl @@ -3,7 +3,7 @@ exec guile --no-auto-compile -e main -s "$0" "$@" !# ;;; ccwl --- Concise Common Workflow Language -;;; Copyright © 2021–2024 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2021–2025 Arun Isaac <arunisaac@systemreboot.net> ;;; ;;; This file is part of ccwl. ;;; @@ -28,6 +28,7 @@ exec guile --no-auto-compile -e main -s "$0" "$@" (use-modules (rnrs conditions) (rnrs exceptions) + (srfi srfi-26) (srfi srfi-28) (srfi srfi-37) (ice-9 match) @@ -49,6 +50,52 @@ exec guile --no-auto-compile -e main -s "$0" "$@" (lambda (opt name arg result) (acons 'help #t result)))) +(define %options + (list (option (list #\t "to") #t #f + (lambda (opt name arg result) + (let ((supported (list "cwl" "dot"))) + (unless (member arg supported) + (raise-exception + (formatted-message "Invalid target ~a argument ~s. Supported targets are ~a." + (if (char? name) + (string #\- name) + (string-append "--" name)) + arg + (string-join supported ", "))))) + (acons 'to (string->symbol arg) + result))) + (option (list #\o "output") #t #f + (lambda (opt name arg result) + (acons 'output-file arg result))) + %help-option)) + +(define (ccwl-compile source to port) + "Compile @var{source} file to @var{to} format writing output to +@var{port}. @var{to} is either @code{'cwl} or @code{'dot}." + ;; We don't need to compile ccwl files. Loading is sufficient for + ;; our purposes. Besides, compiling would fail since the workflow + ;; macro cannot access command definitions. + (set! %load-should-auto-compile #f) + ((case to + ((cwl) function->cwl) + ((dot) function->dot)) + (let ((result (guard (exception + ;; Handle syntax violation exceptions by + ;; reporting them and exiting. + ((ccwl-violation? exception) + (report-ccwl-violation exception) + (exit #f))) + (load (canonicalize-path source) + read-syntax)))) + (if (or (command? result) + (js-expression? result) + (workflow? result)) + result + (raise-exception + (formatted-message "Last expression in file ~a returns none of workflow, command or js-expression" + source)))) + port)) + (define (main args) (with-exception-handler (lambda (condition) @@ -75,30 +122,17 @@ Thank you! (match args ((program "compile" args ...) (let* ((args (args-fold args - (list (option (list #\t "to") #t #f - (lambda (opt name arg result) - (let ((supported (list "cwl" "dot"))) - (unless (member arg supported) - (scm-error 'misc-error - #f - "Invalid target ~A argument ~S. Supported targets are ~A." - (list (if (char? name) - (string #\- name) - (string-append "--" name)) - arg - (string-join supported ", ")) - #f))) - (acons 'to arg result))) - %help-option) + %options invalid-option (lambda (arg result) (acons 'source-file arg result)) - '((to . "cwl"))))) + '((to . cwl))))) (when (or (assq 'help args) (not (assq-ref args 'source-file))) (display (format "Usage: ~a compile [OPTIONS] SOURCE-FILE Compile SOURCE-FILE. + -o, --output=FILE write compiled output to file -t, --to=TARGET compile SOURCE-FILE to TARGET language; Supported targets are cwl (default) and dot. @@ -106,30 +140,15 @@ Compile SOURCE-FILE. program) (current-error-port)) (exit (assq 'help args))) - ;; We don't need to compile ccwl files. Loading is sufficient - ;; for our purposes. Besides, compiling would fail since the - ;; workflow macro cannot access command definitions. - (set! %load-should-auto-compile #f) - (let ((to (assq-ref args 'to))) - ((cond - ((string=? to "cwl") function->cwl) - ((string=? to "dot") function->dot)) - (guard (exception - ;; Handle syntax violation exceptions by reporting - ;; them and exiting. - ((ccwl-violation? exception) - (report-ccwl-violation exception) - (exit #f))) - (let ((result (load (canonicalize-path (assq-ref args 'source-file)) - read-syntax))) - (if (or (command? result) - (js-expression? result) - (workflow? result)) - result - (raise-exception - (condition (formatted-message "Last expression in file ~a returns none of workflow, command or js-expression" - (assq-ref args 'source-file))))))) - (current-output-port))))) + (if (assq-ref args 'output-file) + (call-with-output-file (assq-ref args 'output-file) + (cut ccwl-compile + (assq-ref args 'source-file) + (assq-ref args 'to) + <>)) + (ccwl-compile (assq-ref args 'source-file) + (assq-ref args 'to) + (current-output-port))))) ((program args ...) (let ((args (args-fold args (list %help-option) diff --git a/tests/ccwl.scm b/tests/ccwl.scm index 2d755ad..85ec322 100644 --- a/tests/ccwl.scm +++ b/tests/ccwl.scm @@ -1,5 +1,5 @@ ;;; ccwl --- Concise Common Workflow Language -;;; Copyright © 2021–2024 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2021–2026 Arun Isaac <arunisaac@systemreboot.net> ;;; ;;; This file is part of ccwl. ;;; @@ -20,6 +20,7 @@ (srfi srfi-1) (srfi srfi-64) (srfi srfi-71) + (ice-9 match) (ccwl ccwl) (ccwl conditions)) @@ -60,17 +61,14 @@ (test-begin "ccwl") -(test-assert "stdin input should not have inputBinding" - (not (assoc-ref - (assoc-ref - (assoc-ref - ((@@ (ccwl cwl) command->cwl-scm) - (command #:inputs (file #:type File) - #:run "wc" "-c" - #:stdin file)) - 'inputs) - 'file) - 'inputBinding))) +(test-equal "stdin input should not have inputBinding" + '((file (type . File))) + (assoc-ref + ((@@ (ccwl cwl) command->cwl-scm) + (command #:inputs (file #:type File) + #:run "wc" "-c" + #:stdin file)) + 'inputs)) (test-equal "read all forms of inputs and outputs from a CWL workflow" '(((spam string)) @@ -189,6 +187,16 @@ (workflow () (print-int #:number 42))) +;; TODO: Define this in the lexical scope of the test that requires +;; it. +(define print-workflow + (workflow ((message #:type string)) + (print #:message message))) + +(test-assert "allow literals as arguments to workflows" + (workflow () + (print-workflow #:message "foo"))) + (test-condition "step supplied with an unknown key must raise a &ccwl-violation condition" ccwl-violation? (macroexpand diff --git a/tests/cwl.scm b/tests/cwl.scm index ba619ab..ae8fa6b 100644 --- a/tests/cwl.scm +++ b/tests/cwl.scm @@ -1,5 +1,5 @@ ;;; ccwl --- Concise Common Workflow Language -;;; Copyright © 2023 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2023, 2025 Arun Isaac <arunisaac@systemreboot.net> ;;; ;;; This file is part of ccwl. ;;; @@ -18,6 +18,12 @@ (use-modules (srfi srfi-64)) +(define make-input + (@@ (ccwl ccwl) make-input)) + +(define input->cwl-scm + (@@ (ccwl cwl) input->cwl-scm)) + (define type->cwl (@@ (ccwl cwl) type->cwl)) @@ -41,4 +47,12 @@ (items . File)))) (type->cwl (make-array-type (make-array-type 'File)))) +(test-equal "Serialize #f defaults in input values" + '("foo" + (type . boolean) + (default . #f) + (label . "foo")) + (input->cwl-scm + (make-input "foo" 'boolean "foo" #f #f #f #t #f #f '()))) + (test-end "cwl") diff --git a/tests/ui.scm b/tests/ui.scm index a5741c0..115a20e 100644 --- a/tests/ui.scm +++ b/tests/ui.scm @@ -1,5 +1,5 @@ ;;; ccwl --- Concise Common Workflow Language -;;; Copyright © 2023 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2023, 2025 Arun Isaac <arunisaac@systemreboot.net> ;;; ;;; This file is part of ccwl. ;;; @@ -16,7 +16,8 @@ ;;; You should have received a copy of the GNU General Public License ;;; along with ccwl. If not, see <https://www.gnu.org/licenses/>. -(use-modules (srfi srfi-64) +(use-modules (srfi srfi-26) + (srfi srfi-64) (term ansi-color) (ccwl ui) (ccwl conditions)) diff --git a/tests/utils.scm b/tests/utils.scm index 50c3396..d786d02 100644 --- a/tests/utils.scm +++ b/tests/utils.scm @@ -1,5 +1,5 @@ ;;; ccwl --- Concise Common Workflow Language -;;; Copyright © 2021, 2022, 2023 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2021, 2022, 2023, 2025 Arun Isaac <arunisaac@systemreboot.net> ;;; ;;; This file is part of ccwl. ;;; @@ -194,7 +194,17 @@ (let ((squares cubes (mapn (lambda (n) (values (expt n 2) (expt n 3))) - (iota 5)))) + (iota 5) + 2))) + (list squares cubes))) + +(test-equal "mapn on an empty list" + '(() ()) + (let ((squares cubes (mapn (lambda (n) + (values (expt n 2) + (expt n 3))) + '() + 2))) (list squares cubes))) (test-equal "foldn" diff --git a/website/releases/ccwl-0.5.0.tar.lz b/website/releases/ccwl-0.5.0.tar.lz new file mode 100644 index 0000000..d8b622a --- /dev/null +++ b/website/releases/ccwl-0.5.0.tar.lz Binary files differdiff --git a/website/releases/ccwl-0.5.0.tar.lz.asc b/website/releases/ccwl-0.5.0.tar.lz.asc new file mode 100644 index 0000000..d99264d --- /dev/null +++ b/website/releases/ccwl-0.5.0.tar.lz.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP SIGNATURE----- + +iQEzBAABCAAdFiEEf3MDQ/Lwnzx3v3nTLiXui2GAK7MFAmllmDoACgkQLiXui2GA +K7MseQgAnkLn1tTRRzv9q02vmnLBLhNf/BN1yHFJnbxzRRzezJK4n3eGGw0fWq8j +vlwlX+uWsGEPZ4Ru7jMfLRlunT4SnJYQTBgZy5rLYyFNRs6ltCaz6DL9JQbS0Hqo +5AY6MWPWSXim1YnKK/cAhIxcL1Lzq65ka5aaeqhTEi8L0/wgjWZHsMLG07dsejsm +dcPwyKCwjyWT7+ypyo61axfiuImlrrfNH0pwRNhhThIXLBCCPffdYkUnsakNg0aY +l3nRKUpbUWMXXMMDABthAFLG1ny6Aw346Irk03Nc/JYsODEiaYTO5+2fdLiipPpY +CiwR7ftRkqXdoR6BH2kwhUCmlgsmeA== +=BrLX +-----END PGP SIGNATURE----- |