aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--bh20seqanalyzer/main.py31
-rw-r--r--bh20sequploader/bh20seq-schema.yml36
-rw-r--r--bh20sequploader/main.py7
-rw-r--r--bh20sequploader/qc_metadata.py26
-rw-r--r--example/dummyschema.yaml16
-rw-r--r--setup.py3
6 files changed, 89 insertions, 30 deletions
diff --git a/bh20seqanalyzer/main.py b/bh20seqanalyzer/main.py
index dae8eca..78e32c9 100644
--- a/bh20seqanalyzer/main.py
+++ b/bh20seqanalyzer/main.py
@@ -6,6 +6,8 @@ import subprocess
import tempfile
import json
import logging
+import ruamel.yaml
+from bh20sequploader.qc_metadata import qc_metadata
logging.basicConfig(format="[%(asctime)s] %(levelname)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S",
level=logging.INFO)
@@ -20,9 +22,12 @@ def validate_upload(api, collection, validated_project):
if "sequence.fasta" not in col:
valid = False
logging.warn("Upload '%s' missing sequence.fasta", collection["name"])
- if "metadata.jsonld" not in col:
- logging.warn("Upload '%s' missing metadata.jsonld", collection["name"])
+ if "metadata.yaml" not in col:
+ logging.warn("Upload '%s' missing metadata.yaml", collection["name"])
valid = False
+ else:
+ metadata_content = ruamel.yaml.round_trip_load(col.open("metadata.yaml"))
+ valid = qc_metadata(metadata_content) and valid
dup = api.collections().list(filters=[["owner_uuid", "=", validated_project],
["portable_data_hash", "=", col.portable_data_hash()]]).execute()
@@ -79,12 +84,31 @@ def start_analysis(api,
logging.error(comp.stderr.decode('utf-8'))
+def copy_most_recent_result(api, analysis_project, latest_result_uuid):
+ most_recent_analysis = api.groups().list(filters=[['owner_uuid', '=', analysis_project]],
+ order="created_at desc", limit=1).execute()
+ for m in most_recent_analysis["items"]:
+ cr = api.container_requests().list(filters=[['owner_uuid', '=', m["uuid"]],
+ ["requesting_container_uuid", "=", None]]).execute()
+ if cr["items"] and cr["items"][0]["output_uuid"]:
+ wf = cr["items"][0]
+ src = api.collections().get(uuid=wf["output_uuid"]).execute()
+ dst = api.collections().get(uuid=latest_result_uuid).execute()
+ if src["portable_data_hash"] != dst["portable_data_hash"]:
+ logging.info("Copying latest result from '%s' to %s", m["name"], latest_result_uuid)
+ api.collections().update(uuid=latest_result_uuid,
+ body={"manifest_text": src["manifest_text"],
+ "description": "latest result from %s %s" % (m["name"], wf["uuid"])}).execute()
+ break
+
+
def main():
parser = argparse.ArgumentParser(description='Analyze collections uploaded to a project')
parser.add_argument('--uploader-project', type=str, default='lugli-j7d0g-n5clictpuvwk8aa', help='')
parser.add_argument('--analysis-project', type=str, default='lugli-j7d0g-y4k4uswcqi3ku56', help='')
parser.add_argument('--validated-project', type=str, default='lugli-j7d0g-5ct8p1i1wrgyjvp', help='')
parser.add_argument('--workflow-uuid', type=str, default='lugli-7fd4e-mqfu9y3ofnpnho1', help='')
+ parser.add_argument('--latest-result-uuid', type=str, default='lugli-4zz18-z513nlpqm03hpca', help='')
args = parser.parse_args()
api = arvados.api()
@@ -101,4 +125,7 @@ def main():
start_analysis(api, args.analysis_project,
args.workflow_uuid,
args.validated_project)
+
+ copy_most_recent_result(api, args.analysis_project, args.latest_result_uuid)
+
time.sleep(10)
diff --git a/bh20sequploader/bh20seq-schema.yml b/bh20sequploader/bh20seq-schema.yml
new file mode 100644
index 0000000..6e0973a
--- /dev/null
+++ b/bh20sequploader/bh20seq-schema.yml
@@ -0,0 +1,36 @@
+$graph:
+
+- name: sampleInformationSchema
+ type: record
+ fields:
+ location: string
+ host: string
+ sequenceTechnology: string
+ assemblyMethod: string
+
+- name: InstituteInformationSchema
+ type: record
+ fields:
+ OriginatingLab: string
+ SubmittingLab: string
+
+- name: SubmitterInformationSchema
+ type: record
+ fields:
+ Submitter: string
+ submissionDate: string
+
+- name: VirusDetailSchema
+ type: record
+ fields:
+ VirusName: string
+ AccessionId: string
+
+- name: MainSchema
+ type: record
+ documentRoot: true
+ fields:
+ sampleInformation: sampleInformationSchema
+ InstituteInformation: InstituteInformationSchema
+ SubmitterInformation: SubmitterInformationSchema
+ VirusDetail: VirusDetailSchema
diff --git a/bh20sequploader/main.py b/bh20sequploader/main.py
index d3ebc0c..8b8fefe 100644
--- a/bh20sequploader/main.py
+++ b/bh20sequploader/main.py
@@ -6,6 +6,7 @@ import json
import urllib.request
import socket
import getpass
+from .qc_metadata import qc_metadata
ARVADOS_API_HOST='lugli.arvadosapi.com'
ARVADOS_API_TOKEN='2fbebpmbo3rw3x05ueu2i6nx70zhrsb1p22ycu3ry34m4x4462'
@@ -19,6 +20,8 @@ def main():
api = arvados.api(host=ARVADOS_API_HOST, token=ARVADOS_API_TOKEN, insecure=True)
+ qc_metadata(args.metadata.name)
+
col = arvados.collection.Collection(api_client=api)
print("Reading FASTA")
@@ -29,8 +32,8 @@ def main():
f.write(r)
r = args.sequence.read(65536)
- print("Reading JSONLD")
- with col.open("metadata.jsonld", "w") as f:
+ print("Reading metadata")
+ with col.open("metadata.yaml", "w") as f:
r = args.metadata.read(65536)
print(r[0:20])
while r:
diff --git a/bh20sequploader/qc_metadata.py b/bh20sequploader/qc_metadata.py
index 0632777..78b31b2 100644
--- a/bh20sequploader/qc_metadata.py
+++ b/bh20sequploader/qc_metadata.py
@@ -1,13 +1,21 @@
-import yamale
+import schema_salad.schema
+import logging
+import pkg_resources
-## NOTE: this is just a DUMMY. Everything about this can and will change
def qc_metadata(metadatafile):
- print("Start metadata validation...")
- schema = yamale.make_schema('../example/dummyschema.yaml')
- data = yamale.make_data(metadatafile)
- # Validate data against the schema. Throws a ValueError if data is invalid.
- yamale.validate(schema, data)
- print("...complete!")
+ schema_resource = pkg_resources.resource_stream(__name__, "bh20seq-schema.yml")
+ cache = {"https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml": schema_resource.read().decode("utf-8")}
+ (document_loader,
+ avsc_names,
+ schema_metadata,
+ metaschema_loader) = schema_salad.schema.load_schema("https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml", cache=cache)
-#qc_metadata("../example/metadata.yaml")
+ if not isinstance(avsc_names, schema_salad.avro.schema.Names):
+ print(avsc_names)
+ return False
+ try:
+ doc, metadata = schema_salad.schema.load_and_validate(document_loader, avsc_names, metadatafile, True)
+ return True
+ except:
+ return False
diff --git a/example/dummyschema.yaml b/example/dummyschema.yaml
deleted file mode 100644
index e428324..0000000
--- a/example/dummyschema.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-#sampleInformation: include('sampleInformation')
-#InstituteInformation: include('InstituteInformation')
----
-sampleInformation:
- location : str()
- host : str()
- sequenceTechnology: str()
- assemblyMethod: str()
-
-InstituteInformation:
- OriginatingLab: str()
- SubmittingLab: str()
-
-VirusDetail:
- VirusName: str()
- AccessionId: str()
diff --git a/setup.py b/setup.py
index 0685d37..48c25aa 100644
--- a/setup.py
+++ b/setup.py
@@ -15,7 +15,7 @@ try:
except ImportError:
tagger = egg_info_cmd.egg_info
-install_requires = ["arvados-python-client"]
+install_requires = ["arvados-python-client", "schema-salad"]
needs_pytest = {"pytest", "test", "ptr"}.intersection(sys.argv)
pytest_runner = ["pytest < 6", "pytest-runner < 5"] if needs_pytest else []
@@ -30,6 +30,7 @@ setup(
author_email="peter.amstutz@curii.com",
license="Apache 2.0",
packages=["bh20sequploader", "bh20seqanalyzer"],
+ package_data={"bh20sequploader": ["bh20seq-schema.yml"]},
install_requires=install_requires,
setup_requires=[] + pytest_runner,
tests_require=["pytest<5"],