Work around CWL content size limit by chunking

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

1 files changed, 16 insertions, 6 deletions

diff --git a/workflows/pangenome-generate/relabel-seqs.py b/workflows/pangenome-generate/relabel-seqs.py
index 1188ceb..970540f 100644
--- a/workflows/pangenome-generate/relabel-seqs.py
+++ b/workflows/pangenome-generate/relabel-seqs.py
@@ -1,5 +1,15 @@
-reads = $(inputs.readsFA)
-subjects = $(inputs.subjects)
+import os
+import json
+
+reads = []
+b = 1
+while os.path.exists("block%i" % b):
+    with open("block%i" % b) as f:
+        reads.extend(json.load(f))
+    b += 1
+
+with open("subjects") as f:
+    subjects = json.load(f)
 
 relabeled_fasta = open("relabeledSeqs.fasta", "wt")
 original_labels = open("originalLabels.ttl", "wt")
@@ -7,12 +17,12 @@ original_labels = open("originalLabels.ttl", "wt")
 for i, r in enumerate(reads):
     with open(r["path"], "rt") as fa:
         label = fa.readline()
-        original_labels.write("<%s> <http://biohackathon.org/bh20-seq-schema/original_fasta_label> \\"%s\\" .\\n" % (subjects[i], label[1:].strip().replace('"', '\\\\"')))
-        relabeled_fasta.write(">"+subjects[i]+"\\n")
+        original_labels.write("<%s> <http://biohackathon.org/bh20-seq-schema/original_fasta_label> \"%s\" .\n" % (subjects[i], label[1:].strip().replace('"', '\\"')))
+        relabeled_fasta.write(">"+subjects[i]+"\n")
         data = fa.read(8096)
         while data:
             relabeled_fasta.write(data)
-            endswithnewline = data.endswith("\\n")
+            endswithnewline = data.endswith("\n")
             data = fa.read(8096)
         if not endswithnewline:
-            relabeled_fasta.write("\\n")
+            relabeled_fasta.write("\n")