From 4618a271193fa5cd5cb1e10d6f8acee9c6e8132a Mon Sep 17 00:00:00 2001 From: Michael R. Crusoe Date: Mon, 4 May 2020 18:37:52 +0200 Subject: add pangenome browser prep --- .gitmodules | 3 ++ .../pangenome-generate/pangenome-generate.cwl | 44 ++++++++++++++++++++++ workflows/tools | 1 + 3 files changed, 48 insertions(+) create mode 100644 .gitmodules create mode 160000 workflows/tools diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..c7d7f99 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "workflows/tools"] + path = workflows/tools + url = https://github.com/common-workflow-library/bio-cwl-tools.git diff --git a/workflows/pangenome-generate/pangenome-generate.cwl b/workflows/pangenome-generate/pangenome-generate.cwl index 6794e2d..05e3511 100644 --- a/workflows/pangenome-generate/pangenome-generate.cwl +++ b/workflows/pangenome-generate/pangenome-generate.cwl @@ -1,10 +1,22 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.1 class: Workflow +requirements: + ScatterFeatureRequirement: {} + StepInputExpressionRequirement: {} inputs: inputReads: File[] metadata: File[] metadataSchema: File subjects: string[] + bin_widths: + type: int[] + default: [ 1, 4, 16, 64, 256, 1000, 4000, 16000] + doc: width of each bin in basepairs along the graph vector + cells_per_file: + type: int + default: 100 + doc: Cells per file on component_segmentation outputs: odgiGraph: type: File @@ -24,6 +36,12 @@ outputs: mergedMetadata: type: File outputSource: mergeMetadata/merged + indexed_paths: + type: File + outputSource: index_paths/indexed_paths + colinear_components: + type: File[] + outputSource: segment_components/colinear_components steps: relabel: in: @@ -66,3 +84,29 @@ steps: originalLabels: relabel/originalLabels out: [merged] run: merge-metadata.cwl + bin_paths: + run: ../tools/odgi/odgi_bin.cwl + in: + sparse_graph_index: buildGraph/odgiGraph + bin_width: bin_widths + scatter: bin_width + out: [ bins, pangenome_sequence ] + index_paths: + label: Create path index + run : ../tools/odgi/odgi_pathindex.cwl + in: + sparse_graph_index: buildGraph/odgiGraph + out: [ indexed_paths ] + segment_components: + label: Run component segmentation + run: ../tools/graph-genome-segmentation/component_segmentation.cwl + in: + bins: bin_paths/bins + cells_per_file: cells_per_file + pangenome_sequence: + source: bin_paths/pangenome_sequence + valueFrom: $(self[0]) + # the bin_paths step is scattered over the bin_width array, but always using the same sparse_graph_index + # the pangenome_sequence that is extracted is exactly the same for the same sparse_graph_index + # regardless of bin_width, so we take the first pangenome_sequence as input for this step + out: [ colinear_components ] diff --git a/workflows/tools b/workflows/tools new file mode 160000 index 0000000..45c2d6d --- /dev/null +++ b/workflows/tools @@ -0,0 +1 @@ +Subproject commit 45c2d6dab199cb931b28c54863924ff3cf9079c1 -- cgit v1.2.3 From 6599feb6a6f8b488bd2d12fcf532c3b8a917a17f Mon Sep 17 00:00:00 2001 From: Michael R. Crusoe Date: Tue, 5 May 2020 13:11:02 +0200 Subject: move some tools into the shared repo --- workflows/pangenome-generate/minimap2.cwl | 23 ------------- workflows/pangenome-generate/odgi-viz.cwl | 25 -------------- workflows/pangenome-generate/odgi_to_rdf.cwl | 9 ++--- .../pangenome-generate/pangenome-generate.cwl | 39 ++++++++++++++-------- workflows/pangenome-generate/seqkit-rmdup.cwl | 37 -------------------- workflows/tools | 2 +- 6 files changed, 30 insertions(+), 105 deletions(-) delete mode 100644 workflows/pangenome-generate/minimap2.cwl delete mode 100644 workflows/pangenome-generate/odgi-viz.cwl delete mode 100644 workflows/pangenome-generate/seqkit-rmdup.cwl diff --git a/workflows/pangenome-generate/minimap2.cwl b/workflows/pangenome-generate/minimap2.cwl deleted file mode 100644 index bf8eb4c..0000000 --- a/workflows/pangenome-generate/minimap2.cwl +++ /dev/null @@ -1,23 +0,0 @@ -cwlVersion: v1.1 -class: CommandLineTool -inputs: - readsFA: File -outputs: - readsPAF: stdout -requirements: - InlineJavascriptRequirement: {} -hints: - DockerRequirement: - dockerPull: "quay.io/biocontainers/minimap2:2.17--h8b12597_1" - ResourceRequirement: - coresMin: 8 - coresMax: 32 - ramMin: $(15 * 1024) - outdirMin: $(Math.ceil(inputs.readsFA.size/(1024*1024*1024) + 20)) -stdout: $(inputs.readsFA.nameroot).paf -baseCommand: minimap2 -arguments: [-cx, asm20, - -w, "1", - -t, $(runtime.cores), - $(inputs.readsFA), - $(inputs.readsFA)] diff --git a/workflows/pangenome-generate/odgi-viz.cwl b/workflows/pangenome-generate/odgi-viz.cwl deleted file mode 100644 index d440fcb..0000000 --- a/workflows/pangenome-generate/odgi-viz.cwl +++ /dev/null @@ -1,25 +0,0 @@ -cwlVersion: v1.1 -class: CommandLineTool -inputs: - inputODGI: File -outputs: - odgiPNG: - type: File - outputBinding: - glob: $(inputs.inputODGI.nameroot).png -requirements: - InlineJavascriptRequirement: {} -hints: - DockerRequirement: - dockerPull: "quay.io/biocontainers/odgi:v0.3--py37h8b12597_0" - ResourceRequirement: - coresMin: 4 - ramMin: $(7 * 1024) - outdirMin: 1 -baseCommand: [odgi, viz] -arguments: [-i, $(inputs.inputODGI), - -o, $(inputs.inputODGI.nameroot).png, - -x, "50000", - -y, "500", - -R, - -P, "4"] diff --git a/workflows/pangenome-generate/odgi_to_rdf.cwl b/workflows/pangenome-generate/odgi_to_rdf.cwl index 31ef0c5..96579a3 100644 --- a/workflows/pangenome-generate/odgi_to_rdf.cwl +++ b/workflows/pangenome-generate/odgi_to_rdf.cwl @@ -10,10 +10,8 @@ requirements: ResourceRequirement: ramMin: $((2 * 1024) + 1) inputs: - - id: odgi - type: File - - id: output_name - type: string? + odgi: File + output_name: string? stdout: $(inputs.output_name || inputs.odgi.nameroot+'.ttl.xz') @@ -23,5 +21,4 @@ arguments: xz, --stdout] outputs: - - id: rdf - type: stdout + rdf: stdout diff --git a/workflows/pangenome-generate/pangenome-generate.cwl b/workflows/pangenome-generate/pangenome-generate.cwl index 05e3511..51ea6b4 100644 --- a/workflows/pangenome-generate/pangenome-generate.cwl +++ b/workflows/pangenome-generate/pangenome-generate.cwl @@ -23,7 +23,7 @@ outputs: outputSource: buildGraph/odgiGraph odgiPNG: type: File - outputSource: vizGraph/odgiPNG + outputSource: vizGraph/graph_image seqwishGFA: type: File outputSource: induceGraph/seqwishGFA @@ -32,7 +32,7 @@ outputs: outputSource: odgi2rdf/rdf readsMergeDedup: type: File - outputSource: dedup/readsMergeDedup + outputSource: dedup/reads_dedup mergedMetadata: type: File outputSource: mergeMetadata/merged @@ -50,17 +50,21 @@ steps: out: [relabeledSeqs, originalLabels] run: relabel-seqs.cwl dedup: - in: {readsFA: relabel/relabeledSeqs} - out: [readsMergeDedup, dups] - run: seqkit-rmdup.cwl + in: {reads: relabel/relabeledSeqs} + out: [reads_dedup, dups] + run: ../tools/seqkit/seqkit_rmdup.cwl overlapReads: - in: {readsFA: dedup/readsMergeDedup} - out: [readsPAF] - run: minimap2.cwl + in: + target: dedup/reads_dedup + query: dedup/reads_dedup + outputCIGAR: + default: true + out: [alignments] + run: ../tools/minimap2/minimap2_paf.cwl induceGraph: in: - readsFA: dedup/readsMergeDedup - readsPAF: overlapReads/readsPAF + readsFA: dedup/reads_dedup + readsPAF: overlapReads/alignments out: [seqwishGFA] run: seqwish.cwl buildGraph: @@ -68,9 +72,18 @@ steps: out: [odgiGraph] run: odgi-build.cwl vizGraph: - in: {inputODGI: buildGraph/odgiGraph} - out: [odgiPNG] - run: odgi-viz.cwl + in: + sparse_graph_index: buildGraph/odgiGraph + width: + default: 50000 + height: + default: 500 + path_per_row: + default: true + path_height: + default: 4 + out: [graph_image] + run: ../tools/odgi/odgi_viz.cwl odgi2rdf: in: {odgi: buildGraph/odgiGraph} out: [rdf] diff --git a/workflows/pangenome-generate/seqkit-rmdup.cwl b/workflows/pangenome-generate/seqkit-rmdup.cwl deleted file mode 100644 index 071fa66..0000000 --- a/workflows/pangenome-generate/seqkit-rmdup.cwl +++ /dev/null @@ -1,37 +0,0 @@ -cwlVersion: v1.1 -class: CommandLineTool -inputs: - readsFA: File -outputs: - readsMergeDedup: - type: File - outputBinding: - glob: readsMergeDedup.fasta - dups: - type: File? - outputBinding: - glob: dups.txt -requirements: - InlineJavascriptRequirement: {} -hints: - DockerRequirement: - dockerPull: "quay.io/biocontainers/seqkit:0.7.1--0" - ResourceRequirement: - coresMin: 8 - coresMax: 32 - ramMin: $(7 * 1024) - outdirMin: | - ${ - var sum = 0; - for (var i = 0; i < inputs.readsFA.length; i++) { - sum += inputs.readsFA[i].size; - } - return (sum/(1024*1024*1024)+1) + 20; - } -baseCommand: seqkit -arguments: [rmdup, - --by-seq, - --ignore-case, - --dup-num-file, dups.txt, - -o, readsMergeDedup.fasta, - $(inputs.readsFA)] diff --git a/workflows/tools b/workflows/tools index 45c2d6d..2d8523e 160000 --- a/workflows/tools +++ b/workflows/tools @@ -1 +1 @@ -Subproject commit 45c2d6dab199cb931b28c54863924ff3cf9079c1 +Subproject commit 2d8523e9a86a9504e77d21f8a878d4f8ef2dac42 -- cgit v1.2.3 From ca7df27a6349e7b7009e1d1d5fec372106a072b1 Mon Sep 17 00:00:00 2001 From: Michael R. Crusoe Date: Tue, 5 May 2020 18:24:56 +0200 Subject: preserve the directory layout of segmentation.py --- workflows/pangenome-generate/pangenome-generate.cwl | 2 +- workflows/tools | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/pangenome-generate/pangenome-generate.cwl b/workflows/pangenome-generate/pangenome-generate.cwl index 51ea6b4..1d49904 100644 --- a/workflows/pangenome-generate/pangenome-generate.cwl +++ b/workflows/pangenome-generate/pangenome-generate.cwl @@ -40,7 +40,7 @@ outputs: type: File outputSource: index_paths/indexed_paths colinear_components: - type: File[] + type: Directory outputSource: segment_components/colinear_components steps: relabel: diff --git a/workflows/tools b/workflows/tools index 2d8523e..659e174 160000 --- a/workflows/tools +++ b/workflows/tools @@ -1 +1 @@ -Subproject commit 2d8523e9a86a9504e77d21f8a878d4f8ef2dac42 +Subproject commit 659e174d0d42ed6b9afd79d9e6f68e225c526d1e -- cgit v1.2.3