Merge branch 'master' of https://github.com/arvados/bh20-seq-resource

5 files changed, 110 insertions, 28 deletions

diff --git a/workflows/pangenome-generate/merge-metadata.cwl b/workflows/pangenome-generate/merge-metadata.cwl
index fcefe32..4d9c808 100644
--- a/workflows/pangenome-generate/merge-metadata.cwl
+++ b/workflows/pangenome-generate/merge-metadata.cwl
@@ -5,16 +5,48 @@ hints:
     dockerPull: commonworkflowlanguage/cwltool_module
 inputs:
   metadata: File[]
-  metadataSchema: File
   subjects: string[]
-  dups: File?
-  originalLabels: File
+  metadataSchema:
+    type: File
+    inputBinding: {position: 2}
+  originalLabels:
+    type: File
+    inputBinding: {position: 3}
+  dups:
+    type: File?
+    inputBinding: {position: 4}
+  script:
+    type: File
+    inputBinding: {position: 1}
+    default: {class: File, location: merge-metadata.py}
 outputs:
   merged: stdout
 stdout: mergedmetadata.ttl
 requirements:
+  InlineJavascriptRequirement: {}
   InitialWorkDirRequirement:
-    listing:
-      - entry: {$include: merge-metadata.py}
-        entryname: merge-metadata.py
-baseCommand: [python3, merge-metadata.py]
+    listing: |
+          ${
+          var i = 0;
+          var b = 1;
+          var out = [];
+          for (; i < inputs.metadata.length; i++) {
+            var block = [];
+            var sub = [];
+            for (; i < (b*150) && i < inputs.metadata.length; i++) {
+              block.push(inputs.metadata[i]);
+              sub.push(inputs.subjects[i]);
+            }
+            out.push({
+              entryname: "block"+b,
+              entry: JSON.stringify(block)
+            });
+            out.push({
+              entryname: "subs"+b,
+              entry: JSON.stringify(sub)
+            });
+            b++;
+          }
+          return out;
+          }
+baseCommand: python
diff --git a/workflows/pangenome-generate/merge-metadata.py b/workflows/pangenome-generate/merge-metadata.py
index bfec781..65d08a6 100644
--- a/workflows/pangenome-generate/merge-metadata.py
+++ b/workflows/pangenome-generate/merge-metadata.py
@@ -2,12 +2,27 @@ import re
 import schema_salad.schema
 import schema_salad.jsonld_context
 import json
+import sys
+import os
+import logging
+
+metadataSchema = sys.argv[1]
+originalLabels = sys.argv[2]
+dups = None
+if len(sys.argv) == 4:
+    dups = sys.argv[3]
+
+def readitems(stem):
+    items = []
+    b = 1
+    while os.path.exists("%s%i" % (stem, b)):
+        with open("%s%i" % (stem, b)) as f:
+            items.extend(json.load(f))
+        b += 1
+    return items
 
-metadataSchema = '$(inputs.metadataSchema.path)'
-metadata = $(inputs.metadata)
-subjects = $(inputs.subjects)
-dups = json.loads('''$(inputs.dups)''')
-originalLabels = $(inputs.originalLabels)
+metadata = readitems("block")
+subjects = readitems("subs")
 
 (document_loader,
  avsc_names,
@@ -20,17 +35,15 @@ for i, m in enumerate(metadata):
     g = schema_salad.jsonld_context.makerdf(subjects[i], doc, document_loader.ctx)
     print(g.serialize(format="ntriples").decode("utf-8"))
 
-import logging
-
 if dups:
-    sameseqs = open(dups["path"], "rt")
+    sameseqs = open(dups, "rt")
     for d in sameseqs:
         logging.warn(d)
-        g = re.match(r"\\d+\\t(.*)", d)
+        g = re.match(r"\d+\t(.*)", d)
         logging.warn("%s", g.group(1))
         sp = g.group(1).split(",")
         for n in sp[1:]:
             print("<%s> <http://biohackathon.org/bh20-seq-schema/has_duplicate_sequence> <%s> ." % (n.strip(), sp[0].strip()))
 
-orig = open(originalLabels["path"], "rt")
+orig = open(originalLabels, "rt")
 print(orig.read())
diff --git a/workflows/pangenome-generate/minimap2.cwl b/workflows/pangenome-generate/minimap2.cwl
index bf19ef7..42d1dce 100644
--- a/workflows/pangenome-generate/minimap2.cwl
+++ b/workflows/pangenome-generate/minimap2.cwl
@@ -12,7 +12,7 @@ hints:
   ResourceRequirement:
     coresMin: 8
     coresMax: 32
-    ramMin: $(7 * 1024)
+    ramMin: $(9 * 1024)
     outdirMin: $(Math.ceil(inputs.readsFA.size/(1024*1024*1024) + 20))
 stdout: $(inputs.readsFA.nameroot).paf
 baseCommand: minimap2
diff --git a/workflows/pangenome-generate/relabel-seqs.cwl b/workflows/pangenome-generate/relabel-seqs.cwl
index 2b780d4..c1f17a4 100644
--- a/workflows/pangenome-generate/relabel-seqs.cwl
+++ b/workflows/pangenome-generate/relabel-seqs.cwl
@@ -3,6 +3,10 @@ class: CommandLineTool
 inputs:
   readsFA: File[]
   subjects: string[]
+  script:
+    type: File
+    default: {class: File, location: relabel-seqs.py}
+    inputBinding: {}
 outputs:
   relabeledSeqs:
     type: File
@@ -15,11 +19,32 @@ outputs:
 requirements:
   InlineJavascriptRequirement: {}
   InitialWorkDirRequirement:
-    listing:
-      - entry: {$include: relabel-seqs.py}
-        entryname: relabel-seqs.py
+    listing: |
+          ${
+          var i = 0;
+          var b = 1;
+          var out = [];
+          for (; i < inputs.readsFA.length; i++) {
+            var block = [];
+            var sub = [];
+            for (; i < (b*150) && i < inputs.readsFA.length; i++) {
+              block.push(inputs.readsFA[i]);
+              sub.push(inputs.subjects[i]);
+            }
+            out.push({
+              entryname: "block"+b,
+              entry: JSON.stringify(block)
+            });
+            out.push({
+              entryname: "subs"+b,
+              entry: JSON.stringify(sub)
+            });
+            b++;
+          }
+          return out;
+          }
 hints:
   DockerRequirement:
     dockerPull: commonworkflowlanguage/cwltool_module
 stdout:
-baseCommand: [python, relabel-seqs.py]
+baseCommand: [python]
diff --git a/workflows/pangenome-generate/relabel-seqs.py b/workflows/pangenome-generate/relabel-seqs.py
index 1188ceb..6b022a0 100644
--- a/workflows/pangenome-generate/relabel-seqs.py
+++ b/workflows/pangenome-generate/relabel-seqs.py
@@ -1,5 +1,17 @@
-reads = $(inputs.readsFA)
-subjects = $(inputs.subjects)
+import os
+import json
+
+def readitems(stem):
+    items = []
+    b = 1
+    while os.path.exists("%s%i" % (stem, b)):
+        with open("%s%i" % (stem, b)) as f:
+            items.extend(json.load(f))
+        b += 1
+    return items
+
+reads = readitems("block")
+subjects = readitems("subs")
 
 relabeled_fasta = open("relabeledSeqs.fasta", "wt")
 original_labels = open("originalLabels.ttl", "wt")
@@ -7,12 +19,12 @@ original_labels = open("originalLabels.ttl", "wt")
 for i, r in enumerate(reads):
     with open(r["path"], "rt") as fa:
         label = fa.readline()
-        original_labels.write("<%s> <http://biohackathon.org/bh20-seq-schema/original_fasta_label> \\"%s\\" .\\n" % (subjects[i], label[1:].strip().replace('"', '\\\\"')))
-        relabeled_fasta.write(">"+subjects[i]+"\\n")
+        original_labels.write("<%s> <http://biohackathon.org/bh20-seq-schema/original_fasta_label> \"%s\" .\n" % (subjects[i], label[1:].strip().replace('"', '\\"')))
+        relabeled_fasta.write(">"+subjects[i]+"\n")
         data = fa.read(8096)
         while data:
             relabeled_fasta.write(data)
-            endswithnewline = data.endswith("\\n")
+            endswithnewline = data.endswith("\n")
             data = fa.read(8096)
         if not endswithnewline:
-            relabeled_fasta.write("\\n")
+            relabeled_fasta.write("\n")