Workaround CWL limit by chunking file list

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

1 files changed, 23 insertions, 10 deletions

diff --git a/workflows/pangenome-generate/merge-metadata.py b/workflows/pangenome-generate/merge-metadata.py
index bfec781..65d08a6 100644
--- a/workflows/pangenome-generate/merge-metadata.py
+++ b/workflows/pangenome-generate/merge-metadata.py
@@ -2,12 +2,27 @@ import re
 import schema_salad.schema
 import schema_salad.jsonld_context
 import json
+import sys
+import os
+import logging
+
+metadataSchema = sys.argv[1]
+originalLabels = sys.argv[2]
+dups = None
+if len(sys.argv) == 4:
+    dups = sys.argv[3]
+
+def readitems(stem):
+    items = []
+    b = 1
+    while os.path.exists("%s%i" % (stem, b)):
+        with open("%s%i" % (stem, b)) as f:
+            items.extend(json.load(f))
+        b += 1
+    return items
 
-metadataSchema = '$(inputs.metadataSchema.path)'
-metadata = $(inputs.metadata)
-subjects = $(inputs.subjects)
-dups = json.loads('''$(inputs.dups)''')
-originalLabels = $(inputs.originalLabels)
+metadata = readitems("block")
+subjects = readitems("subs")
 
 (document_loader,
  avsc_names,
@@ -20,17 +35,15 @@ for i, m in enumerate(metadata):
     g = schema_salad.jsonld_context.makerdf(subjects[i], doc, document_loader.ctx)
     print(g.serialize(format="ntriples").decode("utf-8"))
 
-import logging
-
 if dups:
-    sameseqs = open(dups["path"], "rt")
+    sameseqs = open(dups, "rt")
     for d in sameseqs:
         logging.warn(d)
-        g = re.match(r"\\d+\\t(.*)", d)
+        g = re.match(r"\d+\t(.*)", d)
         logging.warn("%s", g.group(1))
         sp = g.group(1).split(",")
         for n in sp[1:]:
             print("<%s> <http://biohackathon.org/bh20-seq-schema/has_duplicate_sequence> <%s> ." % (n.strip(), sp[0].strip()))
 
-orig = open(originalLabels["path"], "rt")
+orig = open(originalLabels, "rt")
 print(orig.read())