From afbc3ec99f638a2f8df96a8e952b5b9616dc99a8 Mon Sep 17 00:00:00 2001
From: Peter Amstutz
Date: Tue, 7 Apr 2020 13:31:49 -0400
Subject: Now moves collections into 'validated sequences' project

Improve logging for seq service

Fix uploader bug

Runs workflow with all validated sequences.
---
 bh20sequploader/main.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'bh20sequploader')

diff --git a/bh20sequploader/main.py b/bh20sequploader/main.py
index 17ad492..d3ebc0c 100644
--- a/bh20sequploader/main.py
+++ b/bh20sequploader/main.py
@@ -49,4 +49,5 @@ def main():
                  (properties['upload_user'], properties['upload_ip']),
                  properties=properties, ensure_unique_name=True)
 
-main()
+if __name__ == "__main__":
+    main()
-- 
cgit v1.2.3


From 40df65dec296b81650987c8ee4f832b703ab8f74 Mon Sep 17 00:00:00 2001
From: lltommy
Date: Tue, 7 Apr 2020 19:51:49 +0200
Subject: adding dummy metadata qc to the project

---
 bh20sequploader/qc_metadata.py | 13 +++++++++++++
 example/dummyschema.yaml       | 16 ++++++++++++++++
 example/metadata.json          |  0
 example/metadata.yaml          | 17 +++++++++++++++++
 4 files changed, 46 insertions(+)
 create mode 100644 bh20sequploader/qc_metadata.py
 create mode 100644 example/dummyschema.yaml
 delete mode 100644 example/metadata.json
 create mode 100644 example/metadata.yaml

(limited to 'bh20sequploader')

diff --git a/bh20sequploader/qc_metadata.py b/bh20sequploader/qc_metadata.py
new file mode 100644
index 0000000..0632777
--- /dev/null
+++ b/bh20sequploader/qc_metadata.py
@@ -0,0 +1,13 @@
+import yamale
+
+## NOTE: this is just a DUMMY. Everything about this can and will change
+def qc_metadata(metadatafile):
+    print("Start metadata validation...")
+    schema = yamale.make_schema('../example/dummyschema.yaml')
+    data = yamale.make_data(metadatafile)
+    # Validate data against the schema. Throws a ValueError if data is invalid.
+    yamale.validate(schema, data)
+    print("...complete!")
+
+#qc_metadata("../example/metadata.yaml")
+
diff --git a/example/dummyschema.yaml b/example/dummyschema.yaml
new file mode 100644
index 0000000..e428324
--- /dev/null
+++ b/example/dummyschema.yaml
@@ -0,0 +1,16 @@
+#sampleInformation: include('sampleInformation')
+#InstituteInformation: include('InstituteInformation')
+---
+sampleInformation:
+  location : str()
+  host : str()
+  sequenceTechnology: str()
+  assemblyMethod: str()
+
+InstituteInformation:
+  OriginatingLab: str()
+  SubmittingLab: str()
+
+VirusDetail:
+  VirusName: str()
+  AccessionId: str()
diff --git a/example/metadata.json b/example/metadata.json
deleted file mode 100644
index e69de29..0000000
diff --git a/example/metadata.yaml b/example/metadata.yaml
new file mode 100644
index 0000000..587d0be
--- /dev/null
+++ b/example/metadata.yaml
@@ -0,0 +1,17 @@
+sampleInformation:
+  location: "USA"
+  host : "Homo Sapiens"
+  sequenceTechnology: "Sanger"
+  assemblyMethod: "CLC Genomics"
+
+InstituteInformation:
+  OriginatingLab: "Erik's kitchen"
+  SubmittingLab: "National Institute for Viral Disease Control and Prevention, China CDC"
+
+SubmitterInformation:
+  Submitter: "National Institute for Viral Disease Control and Prevention, China CDC"
+  submissionDate: "04-04-2020"
+
+VirusDetail:
+  VirusName: "hCoV-19/USA/identifer/2020"
+  AccessionId: "EPI_ISL_Random"
-- 
cgit v1.2.3


From 07bc4c65535437b8e9e0744f08da8cea541d0116 Mon Sep 17 00:00:00 2001
From: Peter Amstutz
Date: Tue, 7 Apr 2020 15:28:42 -0400
Subject: Add metadata validation with schema-salad

---
 bh20seqanalyzer/main.py            | 11 ++++++++---
 bh20sequploader/bh20seq-schema.yml | 36 ++++++++++++++++++++++++++++++++++++
 bh20sequploader/main.py            |  7 +++++--
 bh20sequploader/qc_metadata.py     | 26 +++++++++++++++++---------
 example/dummyschema.yaml           | 16 ----------------
 setup.py                           |  3 ++-
 6 files changed, 68 insertions(+), 31 deletions(-)
 create mode 100644 bh20sequploader/bh20seq-schema.yml
 delete mode 100644 example/dummyschema.yaml

(limited to 'bh20sequploader')

diff --git a/bh20seqanalyzer/main.py b/bh20seqanalyzer/main.py
index 2513ea3..78e32c9 100644
--- a/bh20seqanalyzer/main.py
+++ b/bh20seqanalyzer/main.py
@@ -6,12 +6,14 @@ import subprocess
 import tempfile
 import json
 import logging
+import ruamel.yaml
+from bh20sequploader.qc_metadata import qc_metadata
 
 logging.basicConfig(format="[%(asctime)s] %(levelname)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S",
                     level=logging.INFO)
 logging.getLogger("googleapiclient.discovery").setLevel(logging.WARN)
 
-def validate_upload(api, collection, validated_project, latest_result_uuid):
+def validate_upload(api, collection, validated_project):
     col = arvados.collection.Collection(collection["uuid"])
 
     # validate the collection here.  Check metadata, etc.
@@ -20,9 +22,12 @@ def validate_upload(api, collection, validated_project, latest_result_uuid):
     if "sequence.fasta" not in col:
         valid = False
         logging.warn("Upload '%s' missing sequence.fasta", collection["name"])
-    if "metadata.jsonld" not in col:
-        logging.warn("Upload '%s' missing metadata.jsonld", collection["name"])
+    if "metadata.yaml" not in col:
+        logging.warn("Upload '%s' missing metadata.yaml", collection["name"])
         valid = False
+    else:
+        metadata_content = ruamel.yaml.round_trip_load(col.open("metadata.yaml"))
+        valid = qc_metadata(metadata_content) and valid
 
     dup = api.collections().list(filters=[["owner_uuid", "=", validated_project],
                                           ["portable_data_hash", "=", col.portable_data_hash()]]).execute()
diff --git a/bh20sequploader/bh20seq-schema.yml b/bh20sequploader/bh20seq-schema.yml
new file mode 100644
index 0000000..6e0973a
--- /dev/null
+++ b/bh20sequploader/bh20seq-schema.yml
@@ -0,0 +1,36 @@
+$graph:
+
+- name: sampleInformationSchema
+  type: record
+  fields:
+    location: string
+    host: string
+    sequenceTechnology: string
+    assemblyMethod: string
+
+- name: InstituteInformationSchema
+  type: record
+  fields:
+    OriginatingLab: string
+    SubmittingLab: string
+
+- name: SubmitterInformationSchema
+  type: record
+  fields:
+    Submitter: string
+    submissionDate: string
+
+- name: VirusDetailSchema
+  type: record
+  fields:
+    VirusName: string
+    AccessionId: string
+
+- name: MainSchema
+  type: record
+  documentRoot: true
+  fields:
+    sampleInformation: sampleInformationSchema
+    InstituteInformation: InstituteInformationSchema
+    SubmitterInformation: SubmitterInformationSchema
+    VirusDetail: VirusDetailSchema
diff --git a/bh20sequploader/main.py b/bh20sequploader/main.py
index d3ebc0c..8b8fefe 100644
--- a/bh20sequploader/main.py
+++ b/bh20sequploader/main.py
@@ -6,6 +6,7 @@ import json
 import urllib.request
 import socket
 import getpass
+from .qc_metadata import qc_metadata
 
 ARVADOS_API_HOST='lugli.arvadosapi.com'
 ARVADOS_API_TOKEN='2fbebpmbo3rw3x05ueu2i6nx70zhrsb1p22ycu3ry34m4x4462'
@@ -19,6 +20,8 @@ def main():
 
     api = arvados.api(host=ARVADOS_API_HOST, token=ARVADOS_API_TOKEN, insecure=True)
 
+    qc_metadata(args.metadata.name)
+
     col = arvados.collection.Collection(api_client=api)
 
     print("Reading FASTA")
@@ -29,8 +32,8 @@ def main():
             f.write(r)
             r = args.sequence.read(65536)
 
-    print("Reading JSONLD")
-    with col.open("metadata.jsonld", "w") as f:
+    print("Reading metadata")
+    with col.open("metadata.yaml", "w") as f:
         r = args.metadata.read(65536)
         print(r[0:20])
         while r:
diff --git a/bh20sequploader/qc_metadata.py b/bh20sequploader/qc_metadata.py
index 0632777..78b31b2 100644
--- a/bh20sequploader/qc_metadata.py
+++ b/bh20sequploader/qc_metadata.py
@@ -1,13 +1,21 @@
-import yamale
+import schema_salad.schema
+import logging
+import pkg_resources
 
-## NOTE: this is just a DUMMY. Everything about this can and will change
 def qc_metadata(metadatafile):
-    print("Start metadata validation...")
-    schema = yamale.make_schema('../example/dummyschema.yaml')
-    data = yamale.make_data(metadatafile)
-    # Validate data against the schema. Throws a ValueError if data is invalid.
-    yamale.validate(schema, data)
-    print("...complete!")
+    schema_resource = pkg_resources.resource_stream(__name__, "bh20seq-schema.yml")
+    cache = {"https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml": schema_resource.read().decode("utf-8")}
+    (document_loader,
+     avsc_names,
+     schema_metadata,
+     metaschema_loader) = schema_salad.schema.load_schema("https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml", cache=cache)
 
-#qc_metadata("../example/metadata.yaml")
+    if not isinstance(avsc_names, schema_salad.avro.schema.Names):
+        print(avsc_names)
+        return False
 
+    try:
+        doc, metadata = schema_salad.schema.load_and_validate(document_loader, avsc_names, metadatafile, True)
+        return True
+    except:
+        return False
diff --git a/example/dummyschema.yaml b/example/dummyschema.yaml
deleted file mode 100644
index e428324..0000000
--- a/example/dummyschema.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-#sampleInformation: include('sampleInformation')
-#InstituteInformation: include('InstituteInformation')
----
-sampleInformation:
-  location : str()
-  host : str()
-  sequenceTechnology: str()
-  assemblyMethod: str()
-
-InstituteInformation:
-  OriginatingLab: str()
-  SubmittingLab: str()
-
-VirusDetail:
-  VirusName: str()
-  AccessionId: str()
diff --git a/setup.py b/setup.py
index 0685d37..48c25aa 100644
--- a/setup.py
+++ b/setup.py
@@ -15,7 +15,7 @@ try:
 except ImportError:
     tagger = egg_info_cmd.egg_info
 
-install_requires = ["arvados-python-client"]
+install_requires = ["arvados-python-client", "schema-salad"]
 
 needs_pytest = {"pytest", "test", "ptr"}.intersection(sys.argv)
 pytest_runner = ["pytest < 6", "pytest-runner < 5"] if needs_pytest else []
@@ -30,6 +30,7 @@ setup(
     author_email="peter.amstutz@curii.com",
     license="Apache 2.0",
     packages=["bh20sequploader", "bh20seqanalyzer"],
+    package_data={"bh20sequploader": ["bh20seq-schema.yml"]},
     install_requires=install_requires,
     setup_requires=[] + pytest_runner,
     tests_require=["pytest<5"],
-- 
cgit v1.2.3


From 9458ed33da08c787c4bb20af7b4108c93334b351 Mon Sep 17 00:00:00 2001
From: Peter Amstutz
Date: Wed, 8 Apr 2020 17:41:19 -0400
Subject: Fastq now runs through fastq2fasta pipeline

then gets added to pangenome analysis.
---
 bh20seqanalyzer/main.py        | 141 ++++++++++++++++++++++++++++++-----------
 bh20sequploader/main.py        |  14 +++-
 bh20sequploader/qc_metadata.py |   6 +-
 3 files changed, 120 insertions(+), 41 deletions(-)

(limited to 'bh20sequploader')

diff --git a/bh20seqanalyzer/main.py b/bh20seqanalyzer/main.py
index 78e32c9..1a8965b 100644
--- a/bh20seqanalyzer/main.py
+++ b/bh20seqanalyzer/main.py
@@ -13,21 +13,30 @@ logging.basicConfig(format="[%(asctime)s] %(levelname)s %(message)s", datefmt="%
                     level=logging.INFO)
 logging.getLogger("googleapiclient.discovery").setLevel(logging.WARN)
 
-def validate_upload(api, collection, validated_project):
+def validate_upload(api, collection, validated_project,
+                    fastq_project, fastq_workflow_uuid):
     col = arvados.collection.Collection(collection["uuid"])
 
     # validate the collection here.  Check metadata, etc.
     valid = True
 
-    if "sequence.fasta" not in col:
-        valid = False
-        logging.warn("Upload '%s' missing sequence.fasta", collection["name"])
     if "metadata.yaml" not in col:
         logging.warn("Upload '%s' missing metadata.yaml", collection["name"])
         valid = False
     else:
         metadata_content = ruamel.yaml.round_trip_load(col.open("metadata.yaml"))
-        valid = qc_metadata(metadata_content) and valid
+        #valid = qc_metadata(metadata_content) and valid
+        if not valid:
+            logging.warn("Failed metadata qc")
+
+    if valid:
+        if "sequence.fasta" not in col:
+            if "reads.fastq" in col:
+                start_fastq_to_fasta(api, collection, fastq_project, fastq_workflow_uuid)
+                return False
+            else:
+                valid = False
+                logging.warn("Upload '%s' missing sequence.fasta", collection["name"])
 
     dup = api.collections().list(filters=[["owner_uuid", "=", validated_project],
                                           ["portable_data_hash", "=", col.portable_data_hash()]]).execute()
@@ -39,7 +48,9 @@ def validate_upload(api, collection, validated_project):
     if valid:
         logging.info("Added '%s' to validated sequences" % collection["name"])
         # Move it to the "validated" project to be included in the next analysis
-        api.collections().update(uuid=collection["uuid"], body={"owner_uuid": validated_project}).execute()
+        api.collections().update(uuid=collection["uuid"], body={
+            "owner_uuid": validated_project,
+            "name": "%s (%s)" % (collection["name"], time.asctime(time.gmtime()))}).execute()
     else:
         # It is invalid, delete it.
         logging.warn("Deleting '%s'" % collection["name"])
@@ -47,28 +58,15 @@ def validate_upload(api, collection, validated_project):
 
     return valid
 
-def start_analysis(api,
-                   analysis_project,
-                   workflow_uuid,
-                   validated_project):
 
+def run_workflow(api, parent_project, workflow_uuid, name, inputobj):
     project = api.groups().create(body={
         "group_class": "project",
-        "name": "Pangenome analysis",
-        "owner_uuid": analysis_project,
+        "name": name,
+        "owner_uuid": parent_project,
     }, ensure_unique_name=True).execute()
 
-    validated = arvados.util.list_all(api.collections().list, filters=[["owner_uuid", "=", validated_project]])
-
     with tempfile.NamedTemporaryFile() as tmp:
-        inputobj = {
-            "inputReads": []
-        }
-        for v in validated:
-            inputobj["inputReads"].append({
-                "class": "File",
-                "location": "keep:%s/sequence.fasta" % v["portable_data_hash"]
-            })
         tmp.write(json.dumps(inputobj, indent=2).encode('utf-8'))
         tmp.flush()
         cmd = ["arvados-cwl-runner",
@@ -83,32 +81,95 @@ def start_analysis(api,
     if comp.returncode != 0:
         logging.error(comp.stderr.decode('utf-8'))
 
+    return project
+
+
+def start_fastq_to_fasta(api, collection,
+                         analysis_project,
+                         fastq_workflow_uuid):
+    newproject = run_workflow(api, analysis_project, fastq_workflow_uuid, "FASTQ to FASTA", {
+        "fastq_forward": {
+            "class": "File",
+            "location": "keep:%s/reads.fastq" % collection["portable_data_hash"]
+        },
+        "metadata": {
+            "class": "File",
+            "location": "keep:%s/metadata.yaml" % collection["portable_data_hash"]
+        },
+        "ref_fasta": {
+            "class": "File",
+            "location": "keep:ffef6a3b77e5e04f8f62a7b6f67264d1+556/SARS-CoV2-NC_045512.2.fasta"
+        }
+    })
+    api.collections().update(uuid=collection["uuid"],
+                             body={"owner_uuid": newproject["uuid"]}).execute()
+
+def start_pangenome_analysis(api,
+                             analysis_project,
+                             pangenome_workflow_uuid,
+                             validated_project):
+    validated = arvados.util.list_all(api.collections().list, filters=[["owner_uuid", "=", validated_project]])
+    inputobj = {
+        "inputReads": []
+    }
+    for v in validated:
+        inputobj["inputReads"].append({
+            "class": "File",
+            "location": "keep:%s/sequence.fasta" % v["portable_data_hash"]
+        })
+    run_workflow(api, analysis_project, pangenome_workflow_uuid, "Pangenome analysis", inputobj)
+
+
+def get_workflow_output_from_project(api, uuid):
+    cr = api.container_requests().list(filters=[['owner_uuid', '=', uuid],
+                                                ["requesting_container_uuid", "=", None]]).execute()
+    if cr["items"] and cr["items"][0]["output_uuid"]:
+        return cr["items"][0]
+    else:
+        return None
+
 
 def copy_most_recent_result(api, analysis_project, latest_result_uuid):
     most_recent_analysis = api.groups().list(filters=[['owner_uuid', '=', analysis_project]],
                                                   order="created_at desc", limit=1).execute()
     for m in most_recent_analysis["items"]:
-        cr = api.container_requests().list(filters=[['owner_uuid', '=', m["uuid"]],
-                                                    ["requesting_container_uuid", "=", None]]).execute()
-        if cr["items"] and cr["items"][0]["output_uuid"]:
-            wf = cr["items"][0]
+        wf = get_workflow_output_from_project(api, m["uuid"])
+        if wf:
             src = api.collections().get(uuid=wf["output_uuid"]).execute()
             dst = api.collections().get(uuid=latest_result_uuid).execute()
             if src["portable_data_hash"] != dst["portable_data_hash"]:
                 logging.info("Copying latest result from '%s' to %s", m["name"], latest_result_uuid)
                 api.collections().update(uuid=latest_result_uuid,
                                          body={"manifest_text": src["manifest_text"],
-                                               "description": "latest result from %s %s" % (m["name"], wf["uuid"])}).execute()
+                                               "description": "Result from %s %s" % (m["name"], wf["uuid"])}).execute()
             break
 
 
+def move_fastq_to_fasta_results(api, analysis_project, uploader_project):
+    projects = api.groups().list(filters=[['owner_uuid', '=', analysis_project],
+                                          ["properties.moved_output", "!=", True]],
+                                 order="created_at desc",).execute()
+    for p in projects["items"]:
+        wf = get_workflow_output_from_project(api, p["uuid"])
+        if wf:
+            logging.info("Moving completed fastq2fasta result %s back to uploader project", wf["output_uuid"])
+            api.collections().update(uuid=wf["output_uuid"],
+                                     body={"owner_uuid": uploader_project}).execute()
+            p["properties"]["moved_output"] = True
+            api.groups().update(uuid=p["uuid"], body={"properties": p["properties"]}).execute()
+
+
 def main():
     parser = argparse.ArgumentParser(description='Analyze collections uploaded to a project')
     parser.add_argument('--uploader-project', type=str, default='lugli-j7d0g-n5clictpuvwk8aa', help='')
-    parser.add_argument('--analysis-project', type=str, default='lugli-j7d0g-y4k4uswcqi3ku56', help='')
+    parser.add_argument('--pangenome-analysis-project', type=str, default='lugli-j7d0g-y4k4uswcqi3ku56', help='')
+    parser.add_argument('--fastq-project', type=str, default='lugli-j7d0g-xcjxp4oox2u1w8u', help='')
     parser.add_argument('--validated-project', type=str, default='lugli-j7d0g-5ct8p1i1wrgyjvp', help='')
-    parser.add_argument('--workflow-uuid', type=str, default='lugli-7fd4e-mqfu9y3ofnpnho1', help='')
-    parser.add_argument('--latest-result-uuid', type=str, default='lugli-4zz18-z513nlpqm03hpca', help='')
+
+    parser.add_argument('--pangenome-workflow-uuid', type=str, default='lugli-7fd4e-mqfu9y3ofnpnho1', help='')
+    parser.add_argument('--fastq-workflow-uuid', type=str, default='lugli-7fd4e-2zp9q4jo5xpif9y', help='')
+
+    parser.add_argument('--latest-result-collection', type=str, default='lugli-4zz18-z513nlpqm03hpca', help='')
     args = parser.parse_args()
 
     api = arvados.api()
@@ -116,16 +177,24 @@ def main():
     logging.info("Starting up, monitoring %s for uploads" % (args.uploader_project))
 
     while True:
+        move_fastq_to_fasta_results(api, args.fastq_project, args.uploader_project)
+
         new_collections = api.collections().list(filters=[['owner_uuid', '=', args.uploader_project]]).execute()
         at_least_one_new_valid_seq = False
         for c in new_collections["items"]:
-            at_least_one_new_valid_seq = validate_upload(api, c, args.validated_project) or at_least_one_new_valid_seq
+            at_least_one_new_valid_seq = validate_upload(api, c,
+                                                         args.validated_project,
+                                                         args.fastq_project,
+                                                         args.fastq_workflow_uuid) or at_least_one_new_valid_seq
 
         if at_least_one_new_valid_seq:
-            start_analysis(api, args.analysis_project,
-                           args.workflow_uuid,
-                           args.validated_project)
+            start_pangenome_analysis(api,
+                                     args.pangenome_analysis_project,
+                                     args.pangenome_workflow_uuid,
+                                     args.validated_project)
 
-        copy_most_recent_result(api, args.analysis_project, args.latest_result_uuid)
+        copy_most_recent_result(api,
+                                args.pangenome_analysis_project,
+                                args.latest_result_collection)
 
-        time.sleep(10)
+        time.sleep(15)
diff --git a/bh20sequploader/main.py b/bh20sequploader/main.py
index 8b8fefe..56cbe22 100644
--- a/bh20sequploader/main.py
+++ b/bh20sequploader/main.py
@@ -20,12 +20,18 @@ def main():
 
     api = arvados.api(host=ARVADOS_API_HOST, token=ARVADOS_API_TOKEN, insecure=True)
 
-    qc_metadata(args.metadata.name)
+    if not qc_metadata(args.metadata.name):
+        print("Failed metadata qc")
+        exit(1)
 
     col = arvados.collection.Collection(api_client=api)
 
-    print("Reading FASTA")
-    with col.open("sequence.fasta", "w") as f:
+    if args.sequence.name.endswith("fasta") or args.sequence.name.endswith("fa"):
+        target = "sequence.fasta"
+    elif args.sequence.name.endswith("fastq") or args.sequence.name.endswith("fq"):
+        target = "reads.fastq"
+
+    with col.open(target, "w") as f:
         r = args.sequence.read(65536)
         print(r[0:20])
         while r:
@@ -52,5 +58,7 @@ def main():
                  (properties['upload_user'], properties['upload_ip']),
                  properties=properties, ensure_unique_name=True)
 
+    print("Done")
+
 if __name__ == "__main__":
     main()
diff --git a/bh20sequploader/qc_metadata.py b/bh20sequploader/qc_metadata.py
index 78b31b2..ebe4dfc 100644
--- a/bh20sequploader/qc_metadata.py
+++ b/bh20sequploader/qc_metadata.py
@@ -1,6 +1,7 @@
 import schema_salad.schema
 import logging
 import pkg_resources
+import logging
 
 def qc_metadata(metadatafile):
     schema_resource = pkg_resources.resource_stream(__name__, "bh20seq-schema.yml")
@@ -17,5 +18,6 @@ def qc_metadata(metadatafile):
     try:
         doc, metadata = schema_salad.schema.load_and_validate(document_loader, avsc_names, metadatafile, True)
         return True
-    except:
-        return False
+    except Exception as e:
+        logging.warn(e)
+    return False
-- 
cgit v1.2.3


From d7498093d0f5e0db052ef88815d57c2648d09425 Mon Sep 17 00:00:00 2001
From: lltommy
Date: Thu, 9 Apr 2020 14:34:54 +0200
Subject: Updating schema and examples. This is still work in progress but we
 get there

---
 bh20sequploader/bh20seq-schema.yml | 60 ++++++++++++++++++++++++++------------
 example/metadata.yaml              | 49 ++++++++++++++++++++++---------
 example/minimal_example.yaml       | 14 +++++++++
 3 files changed, 91 insertions(+), 32 deletions(-)
 create mode 100644 example/minimal_example.yaml

(limited to 'bh20sequploader')

diff --git a/bh20sequploader/bh20seq-schema.yml b/bh20sequploader/bh20seq-schema.yml
index 6e0973a..38cfb48 100644
--- a/bh20sequploader/bh20seq-schema.yml
+++ b/bh20sequploader/bh20seq-schema.yml
@@ -1,36 +1,60 @@
 $graph:
 
-- name: sampleInformationSchema
+- name: hostSchema
   type: record
   fields:
-    location: string
-    host: string
-    sequenceTechnology: string
-    assemblyMethod: string
+    host_id: string
+    host_species: string
+    host_common_name: string?
+    host_sex: string?
+    host_age: int?
+    host_age_unit: string?
+    host_health_status: string?
+    host_treatment: string?
+    additional_host_information: string?
 
-- name: InstituteInformationSchema
+- name: sampleSchema
   type: record
   fields:
-    OriginatingLab: string
-    SubmittingLab: string
+    collector_name: string
+    collecting_institution: string
+    specimen_source: string?
+    collection_date: string?
+    collection_location: string?
+    sample_storage_conditions: string?
+    additional_collection_information: string?
 
-- name: SubmitterInformationSchema
+- name: virusSchema
   type: record
   fields:
-    Submitter: string
-    submissionDate: string
+    virus_species: string?
+    virus_strain: string?
 
-- name: VirusDetailSchema
+- name: technologySchema
   type: record
   fields:
-    VirusName: string
-    AccessionId: string
+    sample_sequencing_technology: string
+    sequence_assembly_method: string?
+    sequencing_coverage: string?
+
+- name: submitterSchema
+  type: record
+  fields:
+    submitter_name: string
+    submitter_address: string?
+    originating_lab: string
+    lab_address: string?
+    provider_sample_id: string?
+    submitter_sample_id: string?
+    authors: string?
+    submitter_id: string?
 
 - name: MainSchema
   type: record
   documentRoot: true
   fields:
-    sampleInformation: sampleInformationSchema
-    InstituteInformation: InstituteInformationSchema
-    SubmitterInformation: SubmitterInformationSchema
-    VirusDetail: VirusDetailSchema
+    host: hostSchema
+    sample: sampleSchema
+    virus: virusSchema?
+    technology: technologySchema
+    submitter: submitterSchema
\ No newline at end of file
diff --git a/example/metadata.yaml b/example/metadata.yaml
index 587d0be..8a93379 100644
--- a/example/metadata.yaml
+++ b/example/metadata.yaml
@@ -1,17 +1,38 @@
-sampleInformation:
-  location: "USA"
-  host : "Homo Sapiens"
-  sequenceTechnology: "Sanger"
-  assemblyMethod: "CLC Genomics"
+host:
+    host_id: XX1
+    host_species: string
+    host_common_name: string
+    host_sex: string
+    host_age: 20
+    host_age_unit: string
+    host_health_status: string
+    host_treatment: string
+    additional_host_information: string
 
-InstituteInformation:
-  OriginatingLab: "Erik's kitchen"
-  SubmittingLab: "National Institute for Viral Disease Control and Prevention, China CDC"
+sample:
+    collector_name: XXX
+    collecting_institution: XXX
+    specimen_source: XXX
+    collection_date: XXX
+    collection_location: XXX
+    sample_storage_conditions: XXX
+    additional_collection_information: XXX
 
-SubmitterInformation:
-  Submitter: "National Institute for Viral Disease Control and Prevention, China CDC"
-  submissionDate: "04-04-2020"
+virus:
+    virus_species: XX
+    virus_strain: XX
 
-VirusDetail:
-  VirusName: "hCoV-19/USA/identifer/2020"
-  AccessionId: "EPI_ISL_Random"
+technology:
+    sample_sequencing_technology: XX
+    sequence_assembly_method: XX
+    sequencing_coverage: 70x
+
+submitter:
+    submitter_name: tester
+    submitter_address: testerAdd
+    originating_lab: testLab
+    lab_address: labAdd
+    provider_sample_id: string
+    submitter_sample_id: string
+    authors: testAuthor
+    submitter_id: X12
\ No newline at end of file
diff --git a/example/minimal_example.yaml b/example/minimal_example.yaml
new file mode 100644
index 0000000..201b080
--- /dev/null
+++ b/example/minimal_example.yaml
@@ -0,0 +1,14 @@
+host:
+    host_id: XX
+    host_species: string
+
+sample:
+    collector_name: XXX
+    collecting_institution: XXX
+
+technology:
+    sample_sequencing_technology: XX
+
+submitter:
+    submitter_name: tester
+    originating_lab: testLab
\ No newline at end of file
-- 
cgit v1.2.3


From deedb2ed7046bbe81136b8d9d1edc353984d356b Mon Sep 17 00:00:00 2001
From: lltommy
Date: Thu, 9 Apr 2020 20:38:55 +0200
Subject: Adding functionality of turning keys into ontology terms (URI). This
 is work in progress - of course!

---
 bh20sequploader/bh20seq-schema.yml | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

(limited to 'bh20sequploader')

diff --git a/bh20sequploader/bh20seq-schema.yml b/bh20sequploader/bh20seq-schema.yml
index 38cfb48..fd9e854 100644
--- a/bh20sequploader/bh20seq-schema.yml
+++ b/bh20sequploader/bh20seq-schema.yml
@@ -3,14 +3,20 @@ $graph:
 - name: hostSchema
   type: record
   fields:
+    host_species:
+        type: string
+        jsonldPredicate:
+          _id: http://www.ebi.ac.uk/efo/EFO_0000532
     host_id: string
-    host_species: string
     host_common_name: string?
     host_sex: string?
     host_age: int?
     host_age_unit: string?
     host_health_status: string?
-    host_treatment: string?
+    host_treatment:
+      type: string?
+      jsonldPredicate:
+          _id: http://www.ebi.ac.uk/efo/EFO_0000727
     additional_host_information: string?
 
 - name: sampleSchema
@@ -20,7 +26,10 @@ $graph:
     collecting_institution: string
     specimen_source: string?
     collection_date: string?
-    collection_location: string?
+    collection_location:
+      type: string?
+      jsonldPredicate:
+        _id: https://schema.org/fromLocation
     sample_storage_conditions: string?
     additional_collection_information: string?
 
@@ -33,9 +42,18 @@ $graph:
 - name: technologySchema
   type: record
   fields:
-    sample_sequencing_technology: string
-    sequence_assembly_method: string?
-    sequencing_coverage: string?
+    sample_sequencing_technology:
+      type: string
+      jsonldPredicate:
+        _id: http://www.ebi.ac.uk/efo/EFO_0000532
+    sequence_assembly_method:
+      type: string?
+      jsonldPredicate:
+        _id: http://www.ebi.ac.uk/efo/EFO_0002699
+    sequencing_coverage:
+      type: string?
+      jsonldPredicate:
+        _id: http://purl.obolibrary.org/obo/FLU_0000848
 
 - name: submitterSchema
   type: record
-- 
cgit v1.2.3


From dbe094a150d6c969b3d69f112b3538e6a87a74a2 Mon Sep 17 00:00:00 2001
From: Peter Amstutz
Date: Thu, 9 Apr 2020 15:59:46 -0400
Subject: Add "sequencefile" for the metadata subject.

---
 bh20sequploader/bh20seq-schema.yml | 13 ++++++++++++-
 example/metadata.yaml              |  2 +-
 2 files changed, 13 insertions(+), 2 deletions(-)

(limited to 'bh20sequploader')

diff --git a/bh20sequploader/bh20seq-schema.yml b/bh20sequploader/bh20seq-schema.yml
index fd9e854..5c962d1 100644
--- a/bh20sequploader/bh20seq-schema.yml
+++ b/bh20sequploader/bh20seq-schema.yml
@@ -1,3 +1,8 @@
+$base: http://biohackathon.org/bh20-seq-schema
+$namespaces:
+  sch: https://schema.org/
+  efo: http://www.ebi.ac.uk/efo/
+  obo: http://purl.obolibrary.org/obo/
 $graph:
 
 - name: hostSchema
@@ -75,4 +80,10 @@ $graph:
     sample: sampleSchema
     virus: virusSchema?
     technology: technologySchema
-    submitter: submitterSchema
\ No newline at end of file
+    submitter: submitterSchema
+    sequencefile:
+      doc: The subject (eg the fasta/fastq file) that this metadata describes
+      type: string?
+      jsonldPredicate:
+        _id: "@id"
+        _type: "@id"
diff --git a/example/metadata.yaml b/example/metadata.yaml
index 8a93379..41ff93e 100644
--- a/example/metadata.yaml
+++ b/example/metadata.yaml
@@ -35,4 +35,4 @@ submitter:
     provider_sample_id: string
     submitter_sample_id: string
     authors: testAuthor
-    submitter_id: X12
\ No newline at end of file
+    submitter_id: X12
-- 
cgit v1.2.3