From 46e28c1d5da1c242fe49e5e5d6fdcae7dc544070 Mon Sep 17 00:00:00 2001
From: Peter Amstutz
Date: Wed, 8 Jul 2020 15:48:44 -0400
Subject: Hack in a valid-uri check
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>
---
bh20sequploader/qc_metadata.py | 4 ++++
1 file changed, 4 insertions(+)
(limited to 'bh20sequploader')
diff --git a/bh20sequploader/qc_metadata.py b/bh20sequploader/qc_metadata.py
index 2b57991..3e1e471 100644
--- a/bh20sequploader/qc_metadata.py
+++ b/bh20sequploader/qc_metadata.py
@@ -27,6 +27,10 @@ def qc_metadata(metadatafile):
g = schema_salad.jsonld_context.makerdf("workflow", doc, document_loader.ctx)
rslt, reason = evaluate(g, shex, doc["id"], "https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-shex.rdf#submissionShape")
+ # As part of QC make sure serialization works too, this will raise
+ # an exception if there are invalid URIs.
+ g.serialize(format="ntriples")
+
if not rslt:
raise Exception(reason)
--
cgit v1.2.3
From ae8991618927f81343038f235c3e3f76932f80ea Mon Sep 17 00:00:00 2001
From: Peter Amstutz
Date: Wed, 8 Jul 2020 16:17:09 -0400
Subject: Avoid redundant loading of schema
---
bh20sequploader/qc_metadata.py | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
(limited to 'bh20sequploader')
diff --git a/bh20sequploader/qc_metadata.py b/bh20sequploader/qc_metadata.py
index 3e1e471..0a0c780 100644
--- a/bh20sequploader/qc_metadata.py
+++ b/bh20sequploader/qc_metadata.py
@@ -8,15 +8,20 @@ import traceback
from rdflib import Graph, Namespace
from pyshex.evaluate import evaluate
+metadata_schema = None
def qc_metadata(metadatafile):
+ global metadata_schema
log = logging.getLogger(__name__ )
- schema_resource = pkg_resources.resource_stream(__name__, "bh20seq-schema.yml")
- cache = {"https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml": schema_resource.read().decode("utf-8")}
- (document_loader,
- avsc_names,
- schema_metadata,
- metaschema_loader) = schema_salad.schema.load_schema("https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml", cache=cache)
+ if metadata_schema is None:
+ schema_resource = pkg_resources.resource_stream(__name__, "bh20seq-schema.yml")
+ cache = {"https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml": schema_resource.read().decode("utf-8")}
+ metadata_schema = schema_salad.schema.load_schema("https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml", cache=cache)
+
+ (document_loader,
+ avsc_names,
+ schema_metadata,
+ metaschema_loader) = metadata_schema
shex = pkg_resources.resource_stream(__name__, "bh20seq-shex.rdf").read().decode("utf-8")
--
cgit v1.2.3
From bfcebbd349687ebe2d1d47185db9fda38803b674 Mon Sep 17 00:00:00 2001
From: Peter Amstutz
Date: Wed, 8 Jul 2020 20:51:14 +0000
Subject: Uploader checks for duplicates
---
Dockerfile | 2 +-
bh20sequploader/main.py | 14 ++++++++++++--
2 files changed, 13 insertions(+), 3 deletions(-)
(limited to 'bh20sequploader')
diff --git a/Dockerfile b/Dockerfile
index 43fa8f2..1901ac7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,7 +13,7 @@ ADD bh20seqanalyzer /app/bh20simplewebuploader
ADD bh20sequploader /app/bh20sequploader
ADD bh20simplewebuploader /app/bh20simplewebuploader
-RUN pip3 install -e .
+RUN pip3 install -e .[web]
ENV PORT 8080
CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:8080", "bh20simplewebuploader.main:app"]
diff --git a/bh20sequploader/main.py b/bh20sequploader/main.py
index fd0278d..56e0150 100644
--- a/bh20sequploader/main.py
+++ b/bh20sequploader/main.py
@@ -19,8 +19,10 @@ log = logging.getLogger(__name__ )
log.debug("Entering sequence uploader")
ARVADOS_API_HOST='lugli.arvadosapi.com'
-ARVADOS_API_TOKEN='2fbebpmbo3rw3x05ueu2i6nx70zhrsb1p22ycu3ry34m4x4462'
+UPLOADER_API_TOKEN='2fbebpmbo3rw3x05ueu2i6nx70zhrsb1p22ycu3ry34m4x4462'
+ANONYMOUS_API_TOKEN='5o42qdxpxp5cj15jqjf7vnxx5xduhm4ret703suuoa3ivfglfh'
UPLOAD_PROJECT='lugli-j7d0g-n5clictpuvwk8aa'
+VALIDATED_PROJECT='lugli-j7d0g-n5clictpuvwk8aa'
def qc_stuff(metadata, sequence_p1, sequence_p2, do_qc=True):
failed = False
@@ -69,7 +71,7 @@ def main():
parser.add_argument("--skip-qc", action="store_true", help="Skip local qc check")
args = parser.parse_args()
- api = arvados.api(host=ARVADOS_API_HOST, token=ARVADOS_API_TOKEN, insecure=True)
+ api = arvados.api(host=ARVADOS_API_HOST, token=UPLOADER_API_TOKEN, insecure=True)
target = qc_stuff(args.metadata, args.sequence_p1, args.sequence_p2, not args.skip_qc)
seqlabel = target[0][1]
@@ -106,6 +108,14 @@ def main():
"upload_user": "%s@%s" % (username, socket.gethostname())
}
+ api2 = arvados.api(host=ARVADOS_API_HOST, token=ANONYMOUS_API_TOKEN, insecure=True)
+ dup = api2.collections().list(filters=[["owner_uuid", "in", [VALIDATED_PROJECT, UPLOAD_PROJECT]],
+ ["portable_data_hash", "=", col.portable_data_hash()]]).execute()
+ if dup["items"]:
+ # This exact collection has been uploaded before.
+ print("Duplicate of %s" % ([d["uuid"] for d in dup["items"]]))
+ exit(1)
+
col.save_new(owner_uuid=UPLOAD_PROJECT, name="%s uploaded by %s from %s" %
(seqlabel, properties['upload_user'], properties['upload_ip']),
properties=properties, ensure_unique_name=True)
--
cgit v1.2.3
From e821857e7a9403739f321feb7418d33d6bd8b2c7 Mon Sep 17 00:00:00 2001
From: Peter Amstutz
Date: Wed, 8 Jul 2020 17:10:10 -0400
Subject: Give uploader an option to trust local validation (for batch imports)
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>
---
bh20sequploader/main.py | 17 ++++++++++++++---
bh20sequploader/qc_metadata.py | 10 +++++-----
2 files changed, 19 insertions(+), 8 deletions(-)
(limited to 'bh20sequploader')
diff --git a/bh20sequploader/main.py b/bh20sequploader/main.py
index 56e0150..f744a8c 100644
--- a/bh20sequploader/main.py
+++ b/bh20sequploader/main.py
@@ -22,7 +22,7 @@ ARVADOS_API_HOST='lugli.arvadosapi.com'
UPLOADER_API_TOKEN='2fbebpmbo3rw3x05ueu2i6nx70zhrsb1p22ycu3ry34m4x4462'
ANONYMOUS_API_TOKEN='5o42qdxpxp5cj15jqjf7vnxx5xduhm4ret703suuoa3ivfglfh'
UPLOAD_PROJECT='lugli-j7d0g-n5clictpuvwk8aa'
-VALIDATED_PROJECT='lugli-j7d0g-n5clictpuvwk8aa'
+VALIDATED_PROJECT='lugli-j7d0g-5ct8p1i1wrgyjvp'
def qc_stuff(metadata, sequence_p1, sequence_p2, do_qc=True):
failed = False
@@ -69,9 +69,14 @@ def main():
parser.add_argument('sequence_p2', type=argparse.FileType('rb'), default=None, nargs='?', help='sequence FASTQ pair')
parser.add_argument("--validate", action="store_true", help="Dry run, validate only")
parser.add_argument("--skip-qc", action="store_true", help="Skip local qc check")
+ parser.add_argument("--trusted", action="store_true", help="Trust local validation and add directly to validated project")
args = parser.parse_args()
- api = arvados.api(host=ARVADOS_API_HOST, token=UPLOADER_API_TOKEN, insecure=True)
+ if args.trusted:
+ # Use credentials from environment
+ api = arvados.api()
+ else:
+ api = arvados.api(host=ARVADOS_API_HOST, token=UPLOADER_API_TOKEN, insecure=True)
target = qc_stuff(args.metadata, args.sequence_p1, args.sequence_p2, not args.skip_qc)
seqlabel = target[0][1]
@@ -116,7 +121,13 @@ def main():
print("Duplicate of %s" % ([d["uuid"] for d in dup["items"]]))
exit(1)
- col.save_new(owner_uuid=UPLOAD_PROJECT, name="%s uploaded by %s from %s" %
+ if args.trusted:
+ properties["status"] = "validated"
+ owner_uuid = VALIDATED_PROJECT
+ else:
+ owner_uuid = UPLOAD_PROJECT
+
+ col.save_new(owner_uuid=owner_uuid, name="%s uploaded by %s from %s" %
(seqlabel, properties['upload_user'], properties['upload_ip']),
properties=properties, ensure_unique_name=True)
diff --git a/bh20sequploader/qc_metadata.py b/bh20sequploader/qc_metadata.py
index 0a0c780..27657b1 100644
--- a/bh20sequploader/qc_metadata.py
+++ b/bh20sequploader/qc_metadata.py
@@ -16,12 +16,12 @@ def qc_metadata(metadatafile):
if metadata_schema is None:
schema_resource = pkg_resources.resource_stream(__name__, "bh20seq-schema.yml")
cache = {"https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml": schema_resource.read().decode("utf-8")}
- metadata_schema = schema_salad.schema.load_schema("https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml", cache=cache)
+ metadata_schema = schema_salad.schema.load_schema("https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml", cache=cache)
- (document_loader,
- avsc_names,
- schema_metadata,
- metaschema_loader) = metadata_schema
+ (document_loader,
+ avsc_names,
+ schema_metadata,
+ metaschema_loader) = metadata_schema
shex = pkg_resources.resource_stream(__name__, "bh20seq-shex.rdf").read().decode("utf-8")
--
cgit v1.2.3