Merge pull request #105 from arvados/fasta-subset-from-query

Extract subset of the all-sequences fasta by running a sparql query.

Also includes some tweaks to the permalinks.

9 files changed, 67 insertions, 10 deletions

diff --git a/bh20seqanalyzer/main.py b/bh20seqanalyzer/main.py
index 5f00080..0906958 100644
--- a/bh20seqanalyzer/main.py
+++ b/bh20seqanalyzer/main.py
@@ -215,7 +215,7 @@ class SeqAnalyzer:
         most_recent_analysis = self.api.groups().list(filters=[['owner_uuid', '=', self.pangenome_analysis_project]],
                                                       order="created_at desc").execute()
         for m in most_recent_analysis["items"]:
-            wf = self.get_workflow_output_from_project(m["uuid"], "arv-main.cwl")
+            wf = self.get_workflow_output_from_project(m["uuid"], "collect-seqs.cwl")
             if wf is None:
                 continue
             src = self.api.collections().get(uuid=wf["output_uuid"]).execute()
diff --git a/bh20simplewebuploader/main.py b/bh20simplewebuploader/main.py
index 73503b4..51048a4 100644
--- a/bh20simplewebuploader/main.py
+++ b/bh20simplewebuploader/main.py
@@ -675,13 +675,27 @@ sparqlURL='http://sparql.genenetwork.org/sparql/'
 @app.route('/resource/<id>')
 def resource(id):
     """Get a COVID19 resource using identifier"""
+
     query=f"""
 PREFIX pubseq: <http://biohackathon.org/bh20-seq-schema#MainSchema/>
 PREFIX sio: <http://semanticscience.org/resource/>
 select distinct ?sample ?geoname ?date ?source ?geo ?sampletype ?institute ?sequenceuri
+where {{
 {{
    ?sample sio:SIO_000115 "{id}" .
    ?sequenceuri pubseq:sample ?sample .
+}}
+union
+{{
+   <http://collections.lugli.arvadosapi.com/c={id}/sequence.fasta> pubseq:sample ?sample .
+   ?sequenceuri pubseq:sample ?sample .
+}}
+union
+{{
+   <http://covid19.genenetwork.org/resource/{id}> pubseq:sample ?sample .
+   ?sequenceuri pubseq:sample ?sample .
+}}
+
    ?sample <http://purl.obolibrary.org/obo/GAZ_00000448> ?geo .
    ?geo rdfs:label ?geoname .
    ?sample <http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl#C25164> ?date .
@@ -699,8 +713,9 @@ select distinct ?sample ?geoname ?date ?source ?geo ?sampletype ?institute ?sequ
     logging.info("^^^^^^^^^^^^^^^^^^^^^^^^^^^^")
     # return jsonify({'sequences': int(result[0]["num"]["value"])})
     sequenceuri=sample['sequenceuri']['value']
-    collectionuri=sequenceuri.split('sequence.fasta')[0]
-    metauri=collectionuri+'metadata.yaml'
+    m = re.match(r"http://collections.lugli.arvadosapi.com/c=([^/]*)/sequence.fasta|http://covid19.genenetwork.org/resource/(.*)", sequenceuri)
+    fastauri = "http://collections.lugli.arvadosapi.com/c=%s/sequence.fasta" % m.group(1)
+    metauri = "http://collections.lugli.arvadosapi.com/c=%s/metadata.yaml" % m.group(1)
     locationuri=sample['geo']['value']
     location=sample['geoname']['value']
     date=sample['date']['value']
@@ -715,7 +730,18 @@ select distinct ?sample ?geoname ?date ?source ?geo ?sampletype ?institute ?sequ
     institute=''
     if 'institute' in sample:
         institute=sample['institute']['value']
-    return render_template('permalink.html',id=id,menu='',uri=f"http://covid19.genenetwork.org/resource/{id}",sequenceuri=sequenceuri,locationuri=locationuri,location=location,date=date,source=source,sampletype=sampletype,institute=institute,collectionuri=collectionuri,metauri=metauri)
+    return render_template('permalink.html',
+                           id=id,
+                           menu='',
+                           uri=f"http://covid19.genenetwork.org/resource/{id}",
+                           sequenceuri=fastauri,
+                           locationuri=locationuri,
+                           location=location,
+                           date=date,
+                           source=source,
+                           sampletype=sampletype,
+                           institute=institute,
+                           metauri=metauri)
 
 # http://covid19.genenetwork.org/location?label=http://www.wikidata.org/entity/Q114
 # http://localhost:5067/location?label=http://www.wikidata.org/entity/Q114
diff --git a/bh20simplewebuploader/templates/resource.html b/bh20simplewebuploader/templates/resource.html
index fc52f13..4c50fb9 100644
--- a/bh20simplewebuploader/templates/resource.html
+++ b/bh20simplewebuploader/templates/resource.html
@@ -10,8 +10,8 @@
 	<p><a href="https://workbench.lugli.arvadosapi.com/projects/lugli-j7d0g-5ct8p1i1wrgyjvp#Data_collections">All sequences project</a></p>
 	<p><a href="https://workbench.lugli.arvadosapi.com/collections/lugli-4zz18-z513nlpqm03hpca/relabeledSeqs.sorted_by_quality_and_len.fasta">All sequences (FASTA) relabled and deduplicated</a></p>
 	<p><a href="https://workbench.lugli.arvadosapi.com/collections/lugli-4zz18-z513nlpqm03hpca/mergedmetadata.ttl">Metadata (RDF) for all sequences</a></p>
-	<p><a href="https://workbench.lugli.arvadosapi.com/collections/lugli-4zz18-z513nlpqm03hpca/relabeledSeqs.sorted_by_quality_and_len.g6.gfa">All sequences in Graphical Fragment Assembly (GFA)</a> - <a href="https://github.com/GFA-spec/GFA-spec">More about GFA</a></p>
-	<p><a href="https://workbench.lugli.arvadosapi.com/collections/lugli-4zz18-z513nlpqm03hpca/relabeledSeqs.sorted_by_quality_and_len.g6.unchop.sorted.odgi">All sequences in Optimized Dynamic Genome/Graph Implementation (ODGI)</a> - <a href="https://github.com/vgteam/odgi">More about ODGI</a></p>
+	<!-- <p><a href="https://workbench.lugli.arvadosapi.com/collections/lugli-4zz18-z513nlpqm03hpca/relabeledSeqs.sorted_by_quality_and_len.g6.gfa">All sequences in Graphical Fragment Assembly (GFA)</a> - <a href="https://github.com/GFA-spec/GFA-spec">More about GFA</a></p> -->
+	<!-- <p><a href="https://workbench.lugli.arvadosapi.com/collections/lugli-4zz18-z513nlpqm03hpca/relabeledSeqs.sorted_by_quality_and_len.g6.unchop.sorted.odgi">All sequences in Optimized Dynamic Genome/Graph Implementation (ODGI)</a> - <a href="https://github.com/vgteam/odgi">More about ODGI</a></p> -->
 	<!-- <p><a href="https://workbench.lugli.arvadosapi.com/collections/lugli-4zz18-z513nlpqm03hpca/relabeledSeqs_dedup_relabeledSeqs_dedup.ttl.xz">All sequences in RDF using spodgi</a> - <a href="https://github.com/pangenome/spodgi">More about spodgi</a></p> -->
 
 
diff --git a/workflows/pangenome-generate/arv-main.cwl b/workflows/pangenome-generate/arv-main.cwl
index dae47e6..1d71ee3 100644
--- a/workflows/pangenome-generate/arv-main.cwl
+++ b/workflows/pangenome-generate/arv-main.cwl
@@ -36,7 +36,7 @@ steps:
     run: collect-seqs.cwl
     in:
       src_project: src_project
-      schema: metadataSchema
+      metadataSchema: metadataSchema
       exclude: exclude
     out: [relabeledSeqs, mergedMetadata]
   pangenome-generate:
diff --git a/workflows/pangenome-generate/collect-seqs.cwl b/workflows/pangenome-generate/collect-seqs.cwl
index 3511df1..635108f 100644
--- a/workflows/pangenome-generate/collect-seqs.cwl
+++ b/workflows/pangenome-generate/collect-seqs.cwl
@@ -25,7 +25,7 @@ inputs:
   src_project:
     type: string
     inputBinding: {position: 2}
-  schema:
+  metadataSchema:
     type: File
     inputBinding: {position: 3}
   exclude:
diff --git a/workflows/pangenome-generate/collect-seqs.py b/workflows/pangenome-generate/collect-seqs.py
index af4a0dc..1a0807c 100644
--- a/workflows/pangenome-generate/collect-seqs.py
+++ b/workflows/pangenome-generate/collect-seqs.py
@@ -38,7 +38,7 @@ for item in validated:
     pdh = item["portable_data_hash"]
     with arvados.collection.CollectionReader(pdh, api_client=api, keep_client=keepclient) as col:
         with col.open("sequence.fasta", "rt") as fa:
-            subject = "http://collections.lugli.arvadosapi.com/c=%s/sequence.fasta" % pdh
+            subject = "http://covid19.genenetwork.org/resource/%s" % pdh
             label = fa.readline().strip()
             merged_metadata.write("<%s> <http://biohackathon.org/bh20-seq-schema/original_fasta_label> \"%s\" .\n" % (subject, label[1:].replace('"', '\\"')))
             skip = (subject in blacklist or label[1:] in blacklist)
diff --git a/workflows/pangenome-generate/from_sparql.cwl b/workflows/pangenome-generate/from_sparql.cwl
new file mode 100644
index 0000000..5bc0792
--- /dev/null
+++ b/workflows/pangenome-generate/from_sparql.cwl
@@ -0,0 +1,23 @@
+cwlVersion: v1.1
+class: CommandLineTool
+requirements:
+  DockerRequirement:
+    dockerFile: |
+      FROM debian:10
+      RUN apt-get update && apt-get -yq --no-install-recommends install samtools python3-rdflib
+    dockerImageId: rdflib-and-samtools
+inputs:
+  script:
+    type: File
+    default:
+      class: File
+      location: from_sparql.py
+  metadata: File
+  fasta:
+    type: File
+    secondaryFiles: [.fai]
+  query: string
+stdout: selected.fasta
+outputs:
+  selected: stdout
+arguments: [python3, $(inputs.script), $(inputs.metadata), $(inputs.fasta), $(inputs.query)]
diff --git a/workflows/pangenome-generate/from_sparql.py b/workflows/pangenome-generate/from_sparql.py
new file mode 100644
index 0000000..4610cad
--- /dev/null
+++ b/workflows/pangenome-generate/from_sparql.py
@@ -0,0 +1,8 @@
+from rdflib import Graph
+import sys
+import subprocess
+g = Graph()
+g.parse(sys.argv[1], format="nt")
+res = g.query(sys.argv[3])
+for r in res:
+    subprocess.run(["samtools", "faidx", sys.argv[2], r[0]])
diff --git a/workflows/update-workflows.sh b/workflows/update-workflows.sh
index 3b69a58..5182ec4 100755
--- a/workflows/update-workflows.sh
+++ b/workflows/update-workflows.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
 arvados-cwl-runner --project-uuid=lugli-j7d0g-5hswinmpyho8dju --update-workflow=lugli-7fd4e-2zp9q4jo5xpif9y fastq2fasta/fastq2fasta.cwl
-arvados-cwl-runner --project-uuid=lugli-j7d0g-5hswinmpyho8dju --update-workflow=lugli-7fd4e-mqfu9y3ofnpnho1 pangenome-generate/arv-main.cwl
+arvados-cwl-runner --project-uuid=lugli-j7d0g-5hswinmpyho8dju --update-workflow=lugli-7fd4e-mqfu9y3ofnpnho1 pangenome-generate/collect-seqs.cwl