aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPeter Amstutz2020-04-20 12:50:03 -0400
committerPeter Amstutz2020-04-20 12:50:03 -0400
commit1219eaf496c899f3043b90e30eb956f0f363bfb3 (patch)
treea0735d5df88e608c5461d5b3cb9a3c6bb2f0f32e
parenta2d19fa7b34170abab3759cdff14d8b052178a8a (diff)
downloadbh20-seq-resource-1219eaf496c899f3043b90e30eb956f0f363bfb3.tar.gz
bh20-seq-resource-1219eaf496c899f3043b90e30eb956f0f363bfb3.tar.lz
bh20-seq-resource-1219eaf496c899f3043b90e30eb956f0f363bfb3.zip
Add ShEx validation
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>
-rw-r--r--bh20sequploader/bh20seq-shex.rdf (renamed from bh20sequploader/validation_shape.rdf)2
-rw-r--r--bh20sequploader/main.py11
-rw-r--r--bh20sequploader/qc_metadata.py13
-rw-r--r--example/metadata.yaml2
-rw-r--r--setup.py7
5 files changed, 29 insertions, 6 deletions
diff --git a/bh20sequploader/validation_shape.rdf b/bh20sequploader/bh20seq-shex.rdf
index dd34fa9..8d3f5fc 100644
--- a/bh20sequploader/validation_shape.rdf
+++ b/bh20sequploader/bh20seq-shex.rdf
@@ -1,4 +1,4 @@
-PREFIX : <http://whatever/>
+PREFIX : <https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-shex.rdf#>
PREFIX MainSchema: <http://biohackathon.org/bh20-seq-schema#MainSchema/>
PREFIX hostSchema: <http://biohackathon.org/bh20-seq-schema#hostSchema/>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
diff --git a/bh20sequploader/main.py b/bh20sequploader/main.py
index e0a6a9a..49d012d 100644
--- a/bh20sequploader/main.py
+++ b/bh20sequploader/main.py
@@ -21,16 +21,25 @@ def main():
parser = argparse.ArgumentParser(description='Upload SARS-CoV-19 sequences for analysis')
parser.add_argument('sequence', type=argparse.FileType('r'), help='sequence FASTA/FASTQ')
parser.add_argument('metadata', type=argparse.FileType('r'), help='sequence metadata json')
+ parser.add_argument("--validate", action="store_true", help="Dry run, validate only")
args = parser.parse_args()
api = arvados.api(host=ARVADOS_API_HOST, token=ARVADOS_API_TOKEN, insecure=True)
- target = qc_fasta(args.sequence)
+ try:
+ target = qc_fasta(args.sequence)
+ except ValueError as e:
+ print(e)
+ exit(1)
if not qc_metadata(args.metadata.name):
print("Failed metadata qc")
exit(1)
+ if args.validate:
+ print("Valid")
+ exit(0)
+
col = arvados.collection.Collection(api_client=api)
with col.open(target, "w") as f:
diff --git a/bh20sequploader/qc_metadata.py b/bh20sequploader/qc_metadata.py
index e477f21..fbfd286 100644
--- a/bh20sequploader/qc_metadata.py
+++ b/bh20sequploader/qc_metadata.py
@@ -1,9 +1,12 @@
import schema_salad.schema
import schema_salad.ref_resolver
+import schema_salad.jsonld_context
import logging
import pkg_resources
import logging
import traceback
+from rdflib import Graph, Namespace
+from pyshex.evaluate import evaluate
def qc_metadata(metadatafile):
schema_resource = pkg_resources.resource_stream(__name__, "bh20seq-schema.yml")
@@ -13,13 +16,21 @@ def qc_metadata(metadatafile):
schema_metadata,
metaschema_loader) = schema_salad.schema.load_schema("https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml", cache=cache)
+ shex = pkg_resources.resource_stream(__name__, "bh20seq-shex.rdf").read().decode("utf-8")
+
if not isinstance(avsc_names, schema_salad.avro.schema.Names):
print(avsc_names)
return False
try:
doc, metadata = schema_salad.schema.load_and_validate(document_loader, avsc_names, metadatafile, True)
- return True
+ g = schema_salad.jsonld_context.makerdf("workflow", doc, document_loader.ctx)
+ rslt, reason = evaluate(g, shex, doc["id"], "https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-shex.rdf#submissionShape")
+
+ if not rslt:
+ print(reason)
+
+ return rslt
except Exception as e:
traceback.print_exc()
logging.warn(e)
diff --git a/example/metadata.yaml b/example/metadata.yaml
index 8bbf980..1e83400 100644
--- a/example/metadata.yaml
+++ b/example/metadata.yaml
@@ -17,7 +17,7 @@ sample:
collecting_institution: Institute that was responsible of sampeling
specimen_source: http://purl.obolibrary.org/obo/NCIT_C155831
specimen_source2: http://purl.obolibrary.org/obo/NCIT_C155835
- collection_date: 2020-01-01
+ collection_date: "2020-01-01"
collection_location: http://www.wikidata.org/entity/Q148
sample_storage_conditions: XXX
additional_collection_information: XXX
diff --git a/setup.py b/setup.py
index 0e91274..4ab6329 100644
--- a/setup.py
+++ b/setup.py
@@ -15,7 +15,7 @@ try:
except ImportError:
tagger = egg_info_cmd.egg_info
-install_requires = ["arvados-python-client", "schema-salad", "python-magic"]
+install_requires = ["arvados-python-client", "schema-salad", "python-magic", "pyshex"]
web_requires = ["flask", "pyyaml"]
needs_pytest = {"pytest", "test", "ptr"}.intersection(sys.argv)
@@ -31,7 +31,10 @@ setup(
author_email="peter.amstutz@curii.com",
license="Apache 2.0",
packages=["bh20sequploader", "bh20seqanalyzer", "bh20simplewebuploader"],
- package_data={"bh20sequploader": ["bh20seq-schema.yml", "bh20seq-options.yml", "validation/formats"],
+ package_data={"bh20sequploader": ["bh20seq-schema.yml",
+ "bh20seq-options.yml",
+ "bh20seq-shex.rdf",
+ "validation/formats"],
},
install_requires=install_requires,
extras_require={