Add metadata validation with schema-salad

6 files changed, 68 insertions, 31 deletions

diff --git a/bh20seqanalyzer/main.py b/bh20seqanalyzer/main.py
index 2513ea3..78e32c9 100644
--- a/bh20seqanalyzer/main.py
+++ b/bh20seqanalyzer/main.py
@@ -6,12 +6,14 @@ import subprocess
 import tempfile
 import json
 import logging
+import ruamel.yaml
+from bh20sequploader.qc_metadata import qc_metadata
 
 logging.basicConfig(format="[%(asctime)s] %(levelname)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S",
                     level=logging.INFO)
 logging.getLogger("googleapiclient.discovery").setLevel(logging.WARN)
 
-def validate_upload(api, collection, validated_project, latest_result_uuid):
+def validate_upload(api, collection, validated_project):
     col = arvados.collection.Collection(collection["uuid"])
 
     # validate the collection here.  Check metadata, etc.
@@ -20,9 +22,12 @@ def validate_upload(api, collection, validated_project, latest_result_uuid):
     if "sequence.fasta" not in col:
         valid = False
         logging.warn("Upload '%s' missing sequence.fasta", collection["name"])
-    if "metadata.jsonld" not in col:
-        logging.warn("Upload '%s' missing metadata.jsonld", collection["name"])
+    if "metadata.yaml" not in col:
+        logging.warn("Upload '%s' missing metadata.yaml", collection["name"])
         valid = False
+    else:
+        metadata_content = ruamel.yaml.round_trip_load(col.open("metadata.yaml"))
+        valid = qc_metadata(metadata_content) and valid
 
     dup = api.collections().list(filters=[["owner_uuid", "=", validated_project],
                                           ["portable_data_hash", "=", col.portable_data_hash()]]).execute()
diff --git a/bh20sequploader/bh20seq-schema.yml b/bh20sequploader/bh20seq-schema.yml
new file mode 100644
index 0000000..6e0973a
--- /dev/null
+++ b/bh20sequploader/bh20seq-schema.yml
@@ -0,0 +1,36 @@
+$graph:
+
+- name: sampleInformationSchema
+  type: record
+  fields:
+    location: string
+    host: string
+    sequenceTechnology: string
+    assemblyMethod: string
+
+- name: InstituteInformationSchema
+  type: record
+  fields:
+    OriginatingLab: string
+    SubmittingLab: string
+
+- name: SubmitterInformationSchema
+  type: record
+  fields:
+    Submitter: string
+    submissionDate: string
+
+- name: VirusDetailSchema
+  type: record
+  fields:
+    VirusName: string
+    AccessionId: string
+
+- name: MainSchema
+  type: record
+  documentRoot: true
+  fields:
+    sampleInformation: sampleInformationSchema
+    InstituteInformation: InstituteInformationSchema
+    SubmitterInformation: SubmitterInformationSchema
+    VirusDetail: VirusDetailSchema
diff --git a/bh20sequploader/main.py b/bh20sequploader/main.py
index d3ebc0c..8b8fefe 100644
--- a/bh20sequploader/main.py
+++ b/bh20sequploader/main.py
@@ -6,6 +6,7 @@ import json
 import urllib.request
 import socket
 import getpass
+from .qc_metadata import qc_metadata
 
 ARVADOS_API_HOST='lugli.arvadosapi.com'
 ARVADOS_API_TOKEN='2fbebpmbo3rw3x05ueu2i6nx70zhrsb1p22ycu3ry34m4x4462'
@@ -19,6 +20,8 @@ def main():
 
     api = arvados.api(host=ARVADOS_API_HOST, token=ARVADOS_API_TOKEN, insecure=True)
 
+    qc_metadata(args.metadata.name)
+
     col = arvados.collection.Collection(api_client=api)
 
     print("Reading FASTA")
@@ -29,8 +32,8 @@ def main():
             f.write(r)
             r = args.sequence.read(65536)
 
-    print("Reading JSONLD")
-    with col.open("metadata.jsonld", "w") as f:
+    print("Reading metadata")
+    with col.open("metadata.yaml", "w") as f:
         r = args.metadata.read(65536)
         print(r[0:20])
         while r:
diff --git a/bh20sequploader/qc_metadata.py b/bh20sequploader/qc_metadata.py
index 0632777..78b31b2 100644
--- a/bh20sequploader/qc_metadata.py
+++ b/bh20sequploader/qc_metadata.py
@@ -1,13 +1,21 @@
-import yamale
+import schema_salad.schema
+import logging
+import pkg_resources
 
-## NOTE: this is just a DUMMY. Everything about this can and will change
 def qc_metadata(metadatafile):
-    print("Start metadata validation...")
-    schema = yamale.make_schema('../example/dummyschema.yaml')
-    data = yamale.make_data(metadatafile)
-    # Validate data against the schema. Throws a ValueError if data is invalid.
-    yamale.validate(schema, data)
-    print("...complete!")
+    schema_resource = pkg_resources.resource_stream(__name__, "bh20seq-schema.yml")
+    cache = {"https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml": schema_resource.read().decode("utf-8")}
+    (document_loader,
+     avsc_names,
+     schema_metadata,
+     metaschema_loader) = schema_salad.schema.load_schema("https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml", cache=cache)
 
-#qc_metadata("../example/metadata.yaml")
+    if not isinstance(avsc_names, schema_salad.avro.schema.Names):
+        print(avsc_names)
+        return False
 
+    try:
+        doc, metadata = schema_salad.schema.load_and_validate(document_loader, avsc_names, metadatafile, True)
+        return True
+    except:
+        return False
diff --git a/example/dummyschema.yaml b/example/dummyschema.yaml
deleted file mode 100644
index e428324..0000000
--- a/example/dummyschema.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-#sampleInformation: include('sampleInformation')
-#InstituteInformation: include('InstituteInformation')
----
-sampleInformation:
-  location : str()
-  host : str()
-  sequenceTechnology: str()
-  assemblyMethod: str()
-
-InstituteInformation:
-  OriginatingLab: str()
-  SubmittingLab: str()
-
-VirusDetail:
-  VirusName: str()
-  AccessionId: str()
diff --git a/setup.py b/setup.py
index 0685d37..48c25aa 100644
--- a/setup.py
+++ b/setup.py
@@ -15,7 +15,7 @@ try:
 except ImportError:
     tagger = egg_info_cmd.egg_info
 
-install_requires = ["arvados-python-client"]
+install_requires = ["arvados-python-client", "schema-salad"]
 
 needs_pytest = {"pytest", "test", "ptr"}.intersection(sys.argv)
 pytest_runner = ["pytest < 6", "pytest-runner < 5"] if needs_pytest else []
@@ -30,6 +30,7 @@ setup(
     author_email="peter.amstutz@curii.com",
     license="Apache 2.0",
     packages=["bh20sequploader", "bh20seqanalyzer"],
+    package_data={"bh20sequploader": ["bh20seq-schema.yml"]},
     install_requires=install_requires,
     setup_requires=[] + pytest_runner,
     tests_require=["pytest<5"],