diff options
author | Peter Amstutz | 2020-07-16 16:41:33 -0400 |
---|---|---|
committer | GitHub | 2020-07-16 16:41:33 -0400 |
commit | eb3bdab1109959deb5b11fd74310832bdaa50899 (patch) | |
tree | a79ace1119a78401c9383b0e107da4f9c1734ba1 /bh20simplewebuploader | |
parent | 8d995b271f20d15b2b6a7845ade22c396a383da3 (diff) | |
parent | d49f6b5e11a41a51cb257bbafdcba410544f8486 (diff) | |
download | bh20-seq-resource-eb3bdab1109959deb5b11fd74310832bdaa50899.tar.gz bh20-seq-resource-eb3bdab1109959deb5b11fd74310832bdaa50899.tar.lz bh20-seq-resource-eb3bdab1109959deb5b11fd74310832bdaa50899.zip |
Merge pull request #98 from arvados/analysis-refactor
Analysis refactor
Diffstat (limited to 'bh20simplewebuploader')
-rw-r--r-- | bh20simplewebuploader/main.py | 222 | ||||
-rw-r--r-- | bh20simplewebuploader/static/main.css | 17 | ||||
-rw-r--r-- | bh20simplewebuploader/templates/error.html | 2 | ||||
-rw-r--r-- | bh20simplewebuploader/templates/footer.html | 5 | ||||
-rw-r--r-- | bh20simplewebuploader/templates/status.html | 3 | ||||
-rw-r--r-- | bh20simplewebuploader/templates/success.html | 2 | ||||
-rw-r--r-- | bh20simplewebuploader/templates/validated.html | 17 |
7 files changed, 188 insertions, 80 deletions
diff --git a/bh20simplewebuploader/main.py b/bh20simplewebuploader/main.py index 8089883..7dd07fe 100644 --- a/bh20simplewebuploader/main.py +++ b/bh20simplewebuploader/main.py @@ -8,7 +8,7 @@ import os import sys import re import string -import yaml +import ruamel.yaml as yaml import pkg_resources from flask import Flask, request, redirect, send_file, send_from_directory, render_template, jsonify import os.path @@ -16,6 +16,9 @@ import requests import io import arvados from markupsafe import Markup +from schema_salad.sourceline import add_lc_filename +from schema_salad.schema import shortname +from typing import MutableSequence, MutableMapping ARVADOS_API = 'lugli.arvadosapi.com' ANONYMOUS_TOKEN = '5o42qdxpxp5cj15jqjf7vnxx5xduhm4ret703suuoa3ivfglfh' @@ -47,6 +50,8 @@ def type_to_heading(type_name): Turn a type name like "sampleSchema" from the metadata schema into a human-readable heading. """ + type_name = shortname(type_name) + print(type_name,file=sys.stderr) # Remove camel case decamel = re.sub('([A-Z])', r' \1', type_name) @@ -78,7 +83,7 @@ def is_iri(string): return string.startswith('http') -def generate_form(schema, options): +def generate_form(components, options): """ Linearize the schema into a list of dicts. @@ -101,9 +106,6 @@ def generate_form(schema, options): IRI. """ - # Get the list of form components, one of which is the root - components = schema.get('$graph', []) - # Find the root root_name = None # And also index components by type name @@ -131,55 +133,54 @@ def generate_form(schema, options): # First make a heading, if we aren't the very root of the form yield {'heading': type_to_heading(type_name)} - for field_name, field_type in by_name.get(type_name, {}).get('fields', {}).items(): + for field in by_name.get(type_name, {}).get('fields', []): + field_name = shortname(field["name"]) + field_type = field["type"] # For each field ref_iri = None docstring = None - if not isinstance(field_type, str): - # If the type isn't a string - - # It may have documentation - docstring = field_type.get('doc', None) - - # See if it has a more info/what goes here URL - predicate = field_type.get('jsonldPredicate', {}) - # Predicate may be a URL, a dict with a URL in _id, maybe a - # dict with a URL in _type, or a dict with _id and _type but no - # URLs anywhere. Some of these may not technically be allowed - # by the format, but if they occur, we might as well try to - # handle them. - if isinstance(predicate, str): - if is_iri(predicate): - ref_iri = predicate - else: - # Assume it's a dict. Look at the fields we know about. - for field in ['_id', 'type']: - field_value = predicate.get(field, None) - if isinstance(field_value, str) and is_iri(field_value) and ref_iri is None: - # Take the first URL-looking thing we find - ref_iri = field_value - break - - # Now overwrite the field type with the actual type string - field_type = field_type.get('type', '') - - # Decide if the field is optional (type ends in ?) optional = False - if field_type.endswith('?'): - # It's optional - optional = True - # Drop the ? - field_type = field_type[:-1] - - # Decide if the field is a list (type ends in []) is_list = False - if field_type.endswith('[]'): - # It's a list - is_list = True - # Reduce to the normal type - field_type = field_type[:-2] + + # It may have documentation + docstring = field.get('doc', None) + + # See if it has a more info/what goes here URL + predicate = field.get('jsonldPredicate', {}) + # Predicate may be a URL, a dict with a URL in _id, maybe a + # dict with a URL in _type, or a dict with _id and _type but no + # URLs anywhere. Some of these may not technically be allowed + # by the format, but if they occur, we might as well try to + # handle them. + if isinstance(predicate, str): + if is_iri(predicate): + ref_iri = predicate + else: + # Assume it's a dict. Look at the fields we know about. + for field in ['_id', 'type']: + field_value = predicate.get(field, None) + if isinstance(field_value, str) and is_iri(field_value) and ref_iri is None: + # Take the first URL-looking thing we find + ref_iri = field_value + break + + if isinstance(field_type, MutableSequence): + if field_type[0] == "null" and len(field_type) == 2: + optional = True + field_type = field_type[1] + else: + raise Exception("Can't handle it") + + if isinstance(field_type, MutableMapping): + if field_type["type"] == "array": + # Now replace the field type with the actual type string + is_list = True + field_type = field_type.get('items', '') + else: + field_type = field_type.get('type', '') + pass if field_type in by_name: # This is a subrecord. We need to recurse @@ -227,15 +228,24 @@ def generate_form(schema, options): return list(walk_fields(root_name)) -# At startup, we need to load the metadata schema from the uploader module, so we can make a form for it -if os.path.isfile("bh20sequploader/bh20seq-schema.yml"): - METADATA_SCHEMA = yaml.safe_load(open("bh20sequploader/bh20seq-schema.yml","r").read()) - METADATA_OPTION_DEFINITIONS = yaml.safe_load(open("bh20sequploader/bh20seq-options.yml","r").read()) -else: - METADATA_SCHEMA = yaml.safe_load(pkg_resources.resource_stream("bh20sequploader", "bh20seq-schema.yml")) - METADATA_OPTION_DEFINITIONS = yaml.safe_load(pkg_resources.resource_stream("bh20sequploader", "bh20seq-options.yml")) -# print(METADATA_SCHEMA,file=sys.stderr) -FORM_ITEMS = generate_form(METADATA_SCHEMA, METADATA_OPTION_DEFINITIONS) +import schema_salad.schema +def load_schema_generate_form(): + # At startup, we need to load the metadata schema from the uploader module, so we can make a form for it + if os.path.isfile("bh20sequploader/bh20seq-schema.yml"): + METADATA_SCHEMA = yaml.round_trip_load(open("bh20sequploader/bh20seq-schema.yml","r").read()) + METADATA_OPTION_DEFINITIONS = yaml.safe_load(open("bh20sequploader/bh20seq-options.yml","r").read()) + else: + METADATA_SCHEMA = yaml.round_trip_load(pkg_resources.resource_stream("bh20sequploader", "bh20seq-schema.yml")) + METADATA_OPTION_DEFINITIONS = yaml.safe_load(pkg_resources.resource_stream("bh20sequploader", "bh20seq-options.yml")) + + METADATA_SCHEMA["name"] = "bh20seq-schema.yml" + add_lc_filename(METADATA_SCHEMA, "bh20seq-schema.yml") + metaschema_names, _metaschema_doc, metaschema_loader = schema_salad.schema.get_metaschema() + schema_doc, schema_metadata = metaschema_loader.resolve_ref(METADATA_SCHEMA, "") + + return generate_form(schema_doc, METADATA_OPTION_DEFINITIONS) + +FORM_ITEMS = load_schema_generate_form() @app.route('/') def send_home(): @@ -435,12 +445,12 @@ def receive_files(): if result.returncode != 0: # It didn't work. Complain. - error_message="Uploader returned value {} and said:".format(result.returncode) + str(result.stderr.decode('utf-8')) + error_message="Uploader returned value {} and said:\n".format(result.returncode) + str(result.stderr.decode('utf-8')) print(error_message, file=sys.stderr) return (render_template('error.html', error_message=error_message), 403) else: # It worked. Say so. - return render_template('success.html', log=result.stdout.decode('utf-8', errors='replace')) + return render_template('success.html', log=result.stderr.decode('utf-8', errors='replace')) finally: shutil.rmtree(dest_dir) @@ -479,10 +489,13 @@ def pending_table(output, items): for r in items: if r["status"] != "pending": continue - output.write("<tr>") - output.write("<td><a href='https://workbench.lugli.arvadosapi.com/collections/%s'>%s</a></td>" % (r["uuid"], r["uuid"])) - output.write("<td>%s</td>" % Markup.escape(r["sequence_label"])) - output.write("</tr>") + try: + output.write("<tr>") + output.write("<td><a href='https://workbench.lugli.arvadosapi.com/collections/%s'>%s</a></td>" % (r["uuid"], r["uuid"])) + output.write("<td>%s</td>" % Markup.escape(r.get("sequence_label"))) + output.write("</tr>") + except: + pass output.write( """ </table> @@ -497,18 +510,69 @@ def rejected_table(output, items): <th>Errors</th></tr> """) for r in items: - if r["status"] != "rejected": - continue + try: + if r["status"] != "rejected": + continue + output.write("<tr>") + output.write("<td><a href='https://workbench.lugli.arvadosapi.com/collections/%s'>%s</a></td>" % (r["uuid"], r["uuid"])) + output.write("<td>%s</td>" % Markup.escape(r.get("sequence_label"))) + output.write("<td><pre>%s</pre></td>" % Markup.escape("\n".join(r.get("errors", [])))) + output.write("</tr>") + except: + pass + output.write( +""" +</table> +""") + +def workflows_table(output, items): + output.write( +""" +<table> +<tr> +<th>Name</th> +<th>Sample id</th> +<th>Started</th> +<th>Container request</th> +</tr> +""") + for r in items: output.write("<tr>") - output.write("<td><a href='https://workbench.lugli.arvadosapi.com/collections/%s'>%s</a></td>" % (r["uuid"], r["uuid"])) - output.write("<td>%s</td>" % Markup.escape(r["sequence_label"])) - output.write("<td><pre>%s</pre></td>" % Markup.escape("\n".join(r.get("errors", [])))) + try: + sid = r["mounts"]["/var/lib/cwl/cwl.input.json"]["content"]["sample_id"] + output.write("<td>%s</td>" % Markup.escape(r["name"])) + output.write("<td>%s</td>" % Markup.escape(sid)) + output.write("<td>%s</td>" % Markup.escape(r["created_at"])) + output.write("<td><a href='https://workbench.lugli.arvadosapi.com/container_requests/%s'>%s</a></td>" % (r["uuid"], r["uuid"])) + except: + pass output.write("</tr>") output.write( """ </table> """) +def validated_table(output, items): + output.write( +""" +<table> +<tr> +<th>Collection</th> +<th>Sequence label</th> +</tr> +""") + for r in items: + try: + output.write("<tr>") + output.write("<td><a href='https://workbench.lugli.arvadosapi.com/collections/%s'>%s</a></td>" % (r["uuid"], r["uuid"])) + output.write("<td>%s</td>" % Markup.escape(r["properties"].get("sequence_label"))) + output.write("</tr>") + except: + pass + output.write( +""" +</table> +""") @app.route('/status') def status_page(): @@ -529,22 +593,39 @@ def status_page(): prop["uuid"] = p["uuid"] status[prop["status"]] = status.get(prop["status"], 0) + 1 + workflows = arvados.util.list_all(api.container_requests().list, + filters=[["name", "in", ["fastq2fasta.cwl"]], ["state", "=", "Committed"]], + order="created_at asc") + output = io.StringIO() validated = api.collections().list(filters=[["owner_uuid", "=", VALIDATED_PROJECT]], limit=1).execute() status["passed"] = validated["items_available"] - for s in (("passed", "/download"), ("pending", "#pending"), ("rejected", "#rejected")): + for s in (("passed", "/validated"), ("pending", "#pending"), ("rejected", "#rejected")): output.write("<p><a href='%s'>%s sequences QC %s</a></p>" % (s[1], status.get(s[0], 0), s[0])) - output.write("<a id='pending'><h1>Pending</h1>") + output.write("<p><a href='%s'>%s analysis workflows running</a></p>" % ('#workflows', len(workflows))) + + output.write("<a id='pending'><h1>Pending</h1></a>") pending_table(output, out) - output.write("<a id='rejected'><h1>Rejected</h1>") + output.write("<a id='rejected'><h1>Rejected</h1></a>") rejected_table(output, out) + output.write("<a id='workflows'><h1>Running Workflows</h1></a>") + workflows_table(output, workflows) + return render_template('status.html', table=Markup(output.getvalue()), menu='STATUS') +@app.route('/validated') +def validated_page(): + api = arvados.api(host=ARVADOS_API, token=ANONYMOUS_TOKEN, insecure=True) + output = io.StringIO() + validated = arvados.util.list_all(api.collections().list, filters=[["owner_uuid", "=", VALIDATED_PROJECT]]) + validated_table(output, validated) + return render_template('validated.html', table=Markup(output.getvalue()), menu='STATUS') + @app.route('/demo') def demo_page(): return render_template('demo.html',menu='DEMO') @@ -569,7 +650,6 @@ def map_page(): return render_template('map.html',menu='DEMO') - ## Dynamic API functions starting here ## This is quick and dirty for now, just to get something out and demonstrate the queries ## Feel free to rename the functions/endpoints, feel free to process result so we get nicer JSON diff --git a/bh20simplewebuploader/static/main.css b/bh20simplewebuploader/static/main.css index 47fb408..b28ee9c 100644 --- a/bh20simplewebuploader/static/main.css +++ b/bh20simplewebuploader/static/main.css @@ -178,7 +178,7 @@ span.dropt:hover {text-decoration: none; background: #ffffff; z-index: 6; } .about { display: grid; - grid-template-columns: repeat(2, 1fr); + grid-template-columns: 1fr 1fr; grid-auto-flow: row; } @@ -229,7 +229,7 @@ a { #metadata_fill_form { column-count: 4; margin-top: 0.5em; - column-width: 250px; + column-width: 15em; } .record, .record .field-group, .record .field-group .field { @@ -238,6 +238,8 @@ a { -webkit-column-break-inside: avoid; /* Chrome, Safari, Opera */ page-break-inside: avoid; /* Firefox */ break-inside: avoid; + display: block; + width: 90%; } .record { @@ -258,6 +260,10 @@ a { width: max-content; } +.control { + width: 100%; +} + .filter-options { width: 100%; } @@ -304,9 +310,10 @@ footer { } .sponsors img { - width: 80%; - display:block; - margin:auto; + width: auto; + display: block; + margin: auto; + height: 4em; } .loader { diff --git a/bh20simplewebuploader/templates/error.html b/bh20simplewebuploader/templates/error.html index b1d9402..fc08aed 100644 --- a/bh20simplewebuploader/templates/error.html +++ b/bh20simplewebuploader/templates/error.html @@ -15,7 +15,7 @@ </pre> </p> <p> - <a href="/">Click here to try again.</a> + <a href="/upload">Click here to try again.</a> </p> <hr> </body> diff --git a/bh20simplewebuploader/templates/footer.html b/bh20simplewebuploader/templates/footer.html index 37a6b64..5a1f3c9 100644 --- a/bh20simplewebuploader/templates/footer.html +++ b/bh20simplewebuploader/templates/footer.html @@ -21,12 +21,15 @@ <img src="static/image/covid19biohackathon.png"></a> </div> <div class="sponsorimg"> - <a href="https://www.commonwl.org/"><img src="static/image/CWL.png"></a> + <a href="https://www.curii.com/"><img src="static/image/curii.logo.ai.svg"></a> </div> <div class="sponsorimg"> <a href="https://arvados.org/"><img src="static/image/arvados-logo.png"></a> </div> <div class="sponsorimg"> + <a href="https://www.commonwl.org/"><img src="static/image/CWL.png"></a> + </div> + <div class="sponsorimg"> <a href="https://uthsc.edu/"><img src="static/image/UTHSC-primary-stacked-logo-4c.png"></a> </div> <div class="sponsorimg"> diff --git a/bh20simplewebuploader/templates/status.html b/bh20simplewebuploader/templates/status.html index a1cf28f..e89437e 100644 --- a/bh20simplewebuploader/templates/status.html +++ b/bh20simplewebuploader/templates/status.html @@ -7,7 +7,8 @@ <h1>Sequence upload processing status</h1> - <div class="status"> + <div class="status"> + {{ table }} </div> diff --git a/bh20simplewebuploader/templates/success.html b/bh20simplewebuploader/templates/success.html index 9f0987c..c2302fa 100644 --- a/bh20simplewebuploader/templates/success.html +++ b/bh20simplewebuploader/templates/success.html @@ -9,7 +9,7 @@ <h1>Upload Successful</h1> <hr> <p> - Your files have been uploaded. They should soon appear as output of the <a href="/download">Public SARS-CoV-2 Sequence Resource</a>. + Your files have been uploaded. You can track their <a href="/status">QC status</a>, once validated they will be part of the <a href="/download">Public SARS-CoV-2 Sequence Resource</a>. </p> <p> The upload log was: diff --git a/bh20simplewebuploader/templates/validated.html b/bh20simplewebuploader/templates/validated.html new file mode 100644 index 0000000..cee94bd --- /dev/null +++ b/bh20simplewebuploader/templates/validated.html @@ -0,0 +1,17 @@ +<!DOCTYPE html> +<html> + {% include 'header.html' %} + <body> + {% include 'banner.html' %} + {% include 'menu.html' %} + + <h1>Validated sequences</h1> + + <div class="status"> + {{ table }} + </div> + +{% include 'footer.html' %} + + </body> +</html> |