aboutsummaryrefslogtreecommitdiff
path: root/bh20simplewebuploader
diff options
context:
space:
mode:
authorPeter Amstutz2020-07-16 16:41:33 -0400
committerGitHub2020-07-16 16:41:33 -0400
commiteb3bdab1109959deb5b11fd74310832bdaa50899 (patch)
treea79ace1119a78401c9383b0e107da4f9c1734ba1 /bh20simplewebuploader
parent8d995b271f20d15b2b6a7845ade22c396a383da3 (diff)
parentd49f6b5e11a41a51cb257bbafdcba410544f8486 (diff)
downloadbh20-seq-resource-eb3bdab1109959deb5b11fd74310832bdaa50899.tar.gz
bh20-seq-resource-eb3bdab1109959deb5b11fd74310832bdaa50899.tar.lz
bh20-seq-resource-eb3bdab1109959deb5b11fd74310832bdaa50899.zip
Merge pull request #98 from arvados/analysis-refactor
Analysis refactor
Diffstat (limited to 'bh20simplewebuploader')
-rw-r--r--bh20simplewebuploader/main.py222
-rw-r--r--bh20simplewebuploader/static/main.css17
-rw-r--r--bh20simplewebuploader/templates/error.html2
-rw-r--r--bh20simplewebuploader/templates/footer.html5
-rw-r--r--bh20simplewebuploader/templates/status.html3
-rw-r--r--bh20simplewebuploader/templates/success.html2
-rw-r--r--bh20simplewebuploader/templates/validated.html17
7 files changed, 188 insertions, 80 deletions
diff --git a/bh20simplewebuploader/main.py b/bh20simplewebuploader/main.py
index 8089883..7dd07fe 100644
--- a/bh20simplewebuploader/main.py
+++ b/bh20simplewebuploader/main.py
@@ -8,7 +8,7 @@ import os
import sys
import re
import string
-import yaml
+import ruamel.yaml as yaml
import pkg_resources
from flask import Flask, request, redirect, send_file, send_from_directory, render_template, jsonify
import os.path
@@ -16,6 +16,9 @@ import requests
import io
import arvados
from markupsafe import Markup
+from schema_salad.sourceline import add_lc_filename
+from schema_salad.schema import shortname
+from typing import MutableSequence, MutableMapping
ARVADOS_API = 'lugli.arvadosapi.com'
ANONYMOUS_TOKEN = '5o42qdxpxp5cj15jqjf7vnxx5xduhm4ret703suuoa3ivfglfh'
@@ -47,6 +50,8 @@ def type_to_heading(type_name):
Turn a type name like "sampleSchema" from the metadata schema into a human-readable heading.
"""
+ type_name = shortname(type_name)
+
print(type_name,file=sys.stderr)
# Remove camel case
decamel = re.sub('([A-Z])', r' \1', type_name)
@@ -78,7 +83,7 @@ def is_iri(string):
return string.startswith('http')
-def generate_form(schema, options):
+def generate_form(components, options):
"""
Linearize the schema into a list of dicts.
@@ -101,9 +106,6 @@ def generate_form(schema, options):
IRI.
"""
- # Get the list of form components, one of which is the root
- components = schema.get('$graph', [])
-
# Find the root
root_name = None
# And also index components by type name
@@ -131,55 +133,54 @@ def generate_form(schema, options):
# First make a heading, if we aren't the very root of the form
yield {'heading': type_to_heading(type_name)}
- for field_name, field_type in by_name.get(type_name, {}).get('fields', {}).items():
+ for field in by_name.get(type_name, {}).get('fields', []):
+ field_name = shortname(field["name"])
+ field_type = field["type"]
# For each field
ref_iri = None
docstring = None
- if not isinstance(field_type, str):
- # If the type isn't a string
-
- # It may have documentation
- docstring = field_type.get('doc', None)
-
- # See if it has a more info/what goes here URL
- predicate = field_type.get('jsonldPredicate', {})
- # Predicate may be a URL, a dict with a URL in _id, maybe a
- # dict with a URL in _type, or a dict with _id and _type but no
- # URLs anywhere. Some of these may not technically be allowed
- # by the format, but if they occur, we might as well try to
- # handle them.
- if isinstance(predicate, str):
- if is_iri(predicate):
- ref_iri = predicate
- else:
- # Assume it's a dict. Look at the fields we know about.
- for field in ['_id', 'type']:
- field_value = predicate.get(field, None)
- if isinstance(field_value, str) and is_iri(field_value) and ref_iri is None:
- # Take the first URL-looking thing we find
- ref_iri = field_value
- break
-
- # Now overwrite the field type with the actual type string
- field_type = field_type.get('type', '')
-
- # Decide if the field is optional (type ends in ?)
optional = False
- if field_type.endswith('?'):
- # It's optional
- optional = True
- # Drop the ?
- field_type = field_type[:-1]
-
- # Decide if the field is a list (type ends in [])
is_list = False
- if field_type.endswith('[]'):
- # It's a list
- is_list = True
- # Reduce to the normal type
- field_type = field_type[:-2]
+
+ # It may have documentation
+ docstring = field.get('doc', None)
+
+ # See if it has a more info/what goes here URL
+ predicate = field.get('jsonldPredicate', {})
+ # Predicate may be a URL, a dict with a URL in _id, maybe a
+ # dict with a URL in _type, or a dict with _id and _type but no
+ # URLs anywhere. Some of these may not technically be allowed
+ # by the format, but if they occur, we might as well try to
+ # handle them.
+ if isinstance(predicate, str):
+ if is_iri(predicate):
+ ref_iri = predicate
+ else:
+ # Assume it's a dict. Look at the fields we know about.
+ for field in ['_id', 'type']:
+ field_value = predicate.get(field, None)
+ if isinstance(field_value, str) and is_iri(field_value) and ref_iri is None:
+ # Take the first URL-looking thing we find
+ ref_iri = field_value
+ break
+
+ if isinstance(field_type, MutableSequence):
+ if field_type[0] == "null" and len(field_type) == 2:
+ optional = True
+ field_type = field_type[1]
+ else:
+ raise Exception("Can't handle it")
+
+ if isinstance(field_type, MutableMapping):
+ if field_type["type"] == "array":
+ # Now replace the field type with the actual type string
+ is_list = True
+ field_type = field_type.get('items', '')
+ else:
+ field_type = field_type.get('type', '')
+ pass
if field_type in by_name:
# This is a subrecord. We need to recurse
@@ -227,15 +228,24 @@ def generate_form(schema, options):
return list(walk_fields(root_name))
-# At startup, we need to load the metadata schema from the uploader module, so we can make a form for it
-if os.path.isfile("bh20sequploader/bh20seq-schema.yml"):
- METADATA_SCHEMA = yaml.safe_load(open("bh20sequploader/bh20seq-schema.yml","r").read())
- METADATA_OPTION_DEFINITIONS = yaml.safe_load(open("bh20sequploader/bh20seq-options.yml","r").read())
-else:
- METADATA_SCHEMA = yaml.safe_load(pkg_resources.resource_stream("bh20sequploader", "bh20seq-schema.yml"))
- METADATA_OPTION_DEFINITIONS = yaml.safe_load(pkg_resources.resource_stream("bh20sequploader", "bh20seq-options.yml"))
-# print(METADATA_SCHEMA,file=sys.stderr)
-FORM_ITEMS = generate_form(METADATA_SCHEMA, METADATA_OPTION_DEFINITIONS)
+import schema_salad.schema
+def load_schema_generate_form():
+ # At startup, we need to load the metadata schema from the uploader module, so we can make a form for it
+ if os.path.isfile("bh20sequploader/bh20seq-schema.yml"):
+ METADATA_SCHEMA = yaml.round_trip_load(open("bh20sequploader/bh20seq-schema.yml","r").read())
+ METADATA_OPTION_DEFINITIONS = yaml.safe_load(open("bh20sequploader/bh20seq-options.yml","r").read())
+ else:
+ METADATA_SCHEMA = yaml.round_trip_load(pkg_resources.resource_stream("bh20sequploader", "bh20seq-schema.yml"))
+ METADATA_OPTION_DEFINITIONS = yaml.safe_load(pkg_resources.resource_stream("bh20sequploader", "bh20seq-options.yml"))
+
+ METADATA_SCHEMA["name"] = "bh20seq-schema.yml"
+ add_lc_filename(METADATA_SCHEMA, "bh20seq-schema.yml")
+ metaschema_names, _metaschema_doc, metaschema_loader = schema_salad.schema.get_metaschema()
+ schema_doc, schema_metadata = metaschema_loader.resolve_ref(METADATA_SCHEMA, "")
+
+ return generate_form(schema_doc, METADATA_OPTION_DEFINITIONS)
+
+FORM_ITEMS = load_schema_generate_form()
@app.route('/')
def send_home():
@@ -435,12 +445,12 @@ def receive_files():
if result.returncode != 0:
# It didn't work. Complain.
- error_message="Uploader returned value {} and said:".format(result.returncode) + str(result.stderr.decode('utf-8'))
+ error_message="Uploader returned value {} and said:\n".format(result.returncode) + str(result.stderr.decode('utf-8'))
print(error_message, file=sys.stderr)
return (render_template('error.html', error_message=error_message), 403)
else:
# It worked. Say so.
- return render_template('success.html', log=result.stdout.decode('utf-8', errors='replace'))
+ return render_template('success.html', log=result.stderr.decode('utf-8', errors='replace'))
finally:
shutil.rmtree(dest_dir)
@@ -479,10 +489,13 @@ def pending_table(output, items):
for r in items:
if r["status"] != "pending":
continue
- output.write("<tr>")
- output.write("<td><a href='https://workbench.lugli.arvadosapi.com/collections/%s'>%s</a></td>" % (r["uuid"], r["uuid"]))
- output.write("<td>%s</td>" % Markup.escape(r["sequence_label"]))
- output.write("</tr>")
+ try:
+ output.write("<tr>")
+ output.write("<td><a href='https://workbench.lugli.arvadosapi.com/collections/%s'>%s</a></td>" % (r["uuid"], r["uuid"]))
+ output.write("<td>%s</td>" % Markup.escape(r.get("sequence_label")))
+ output.write("</tr>")
+ except:
+ pass
output.write(
"""
</table>
@@ -497,18 +510,69 @@ def rejected_table(output, items):
<th>Errors</th></tr>
""")
for r in items:
- if r["status"] != "rejected":
- continue
+ try:
+ if r["status"] != "rejected":
+ continue
+ output.write("<tr>")
+ output.write("<td><a href='https://workbench.lugli.arvadosapi.com/collections/%s'>%s</a></td>" % (r["uuid"], r["uuid"]))
+ output.write("<td>%s</td>" % Markup.escape(r.get("sequence_label")))
+ output.write("<td><pre>%s</pre></td>" % Markup.escape("\n".join(r.get("errors", []))))
+ output.write("</tr>")
+ except:
+ pass
+ output.write(
+"""
+</table>
+""")
+
+def workflows_table(output, items):
+ output.write(
+"""
+<table>
+<tr>
+<th>Name</th>
+<th>Sample id</th>
+<th>Started</th>
+<th>Container request</th>
+</tr>
+""")
+ for r in items:
output.write("<tr>")
- output.write("<td><a href='https://workbench.lugli.arvadosapi.com/collections/%s'>%s</a></td>" % (r["uuid"], r["uuid"]))
- output.write("<td>%s</td>" % Markup.escape(r["sequence_label"]))
- output.write("<td><pre>%s</pre></td>" % Markup.escape("\n".join(r.get("errors", []))))
+ try:
+ sid = r["mounts"]["/var/lib/cwl/cwl.input.json"]["content"]["sample_id"]
+ output.write("<td>%s</td>" % Markup.escape(r["name"]))
+ output.write("<td>%s</td>" % Markup.escape(sid))
+ output.write("<td>%s</td>" % Markup.escape(r["created_at"]))
+ output.write("<td><a href='https://workbench.lugli.arvadosapi.com/container_requests/%s'>%s</a></td>" % (r["uuid"], r["uuid"]))
+ except:
+ pass
output.write("</tr>")
output.write(
"""
</table>
""")
+def validated_table(output, items):
+ output.write(
+"""
+<table>
+<tr>
+<th>Collection</th>
+<th>Sequence label</th>
+</tr>
+""")
+ for r in items:
+ try:
+ output.write("<tr>")
+ output.write("<td><a href='https://workbench.lugli.arvadosapi.com/collections/%s'>%s</a></td>" % (r["uuid"], r["uuid"]))
+ output.write("<td>%s</td>" % Markup.escape(r["properties"].get("sequence_label")))
+ output.write("</tr>")
+ except:
+ pass
+ output.write(
+"""
+</table>
+""")
@app.route('/status')
def status_page():
@@ -529,22 +593,39 @@ def status_page():
prop["uuid"] = p["uuid"]
status[prop["status"]] = status.get(prop["status"], 0) + 1
+ workflows = arvados.util.list_all(api.container_requests().list,
+ filters=[["name", "in", ["fastq2fasta.cwl"]], ["state", "=", "Committed"]],
+ order="created_at asc")
+
output = io.StringIO()
validated = api.collections().list(filters=[["owner_uuid", "=", VALIDATED_PROJECT]], limit=1).execute()
status["passed"] = validated["items_available"]
- for s in (("passed", "/download"), ("pending", "#pending"), ("rejected", "#rejected")):
+ for s in (("passed", "/validated"), ("pending", "#pending"), ("rejected", "#rejected")):
output.write("<p><a href='%s'>%s sequences QC %s</a></p>" % (s[1], status.get(s[0], 0), s[0]))
- output.write("<a id='pending'><h1>Pending</h1>")
+ output.write("<p><a href='%s'>%s analysis workflows running</a></p>" % ('#workflows', len(workflows)))
+
+ output.write("<a id='pending'><h1>Pending</h1></a>")
pending_table(output, out)
- output.write("<a id='rejected'><h1>Rejected</h1>")
+ output.write("<a id='rejected'><h1>Rejected</h1></a>")
rejected_table(output, out)
+ output.write("<a id='workflows'><h1>Running Workflows</h1></a>")
+ workflows_table(output, workflows)
+
return render_template('status.html', table=Markup(output.getvalue()), menu='STATUS')
+@app.route('/validated')
+def validated_page():
+ api = arvados.api(host=ARVADOS_API, token=ANONYMOUS_TOKEN, insecure=True)
+ output = io.StringIO()
+ validated = arvados.util.list_all(api.collections().list, filters=[["owner_uuid", "=", VALIDATED_PROJECT]])
+ validated_table(output, validated)
+ return render_template('validated.html', table=Markup(output.getvalue()), menu='STATUS')
+
@app.route('/demo')
def demo_page():
return render_template('demo.html',menu='DEMO')
@@ -569,7 +650,6 @@ def map_page():
return render_template('map.html',menu='DEMO')
-
## Dynamic API functions starting here
## This is quick and dirty for now, just to get something out and demonstrate the queries
## Feel free to rename the functions/endpoints, feel free to process result so we get nicer JSON
diff --git a/bh20simplewebuploader/static/main.css b/bh20simplewebuploader/static/main.css
index 47fb408..b28ee9c 100644
--- a/bh20simplewebuploader/static/main.css
+++ b/bh20simplewebuploader/static/main.css
@@ -178,7 +178,7 @@ span.dropt:hover {text-decoration: none; background: #ffffff; z-index: 6; }
.about {
display: grid;
- grid-template-columns: repeat(2, 1fr);
+ grid-template-columns: 1fr 1fr;
grid-auto-flow: row;
}
@@ -229,7 +229,7 @@ a {
#metadata_fill_form {
column-count: 4;
margin-top: 0.5em;
- column-width: 250px;
+ column-width: 15em;
}
.record, .record .field-group, .record .field-group .field {
@@ -238,6 +238,8 @@ a {
-webkit-column-break-inside: avoid; /* Chrome, Safari, Opera */
page-break-inside: avoid; /* Firefox */
break-inside: avoid;
+ display: block;
+ width: 90%;
}
.record {
@@ -258,6 +260,10 @@ a {
width: max-content;
}
+.control {
+ width: 100%;
+}
+
.filter-options {
width: 100%;
}
@@ -304,9 +310,10 @@ footer {
}
.sponsors img {
- width: 80%;
- display:block;
- margin:auto;
+ width: auto;
+ display: block;
+ margin: auto;
+ height: 4em;
}
.loader {
diff --git a/bh20simplewebuploader/templates/error.html b/bh20simplewebuploader/templates/error.html
index b1d9402..fc08aed 100644
--- a/bh20simplewebuploader/templates/error.html
+++ b/bh20simplewebuploader/templates/error.html
@@ -15,7 +15,7 @@
</pre>
</p>
<p>
- <a href="/">Click here to try again.</a>
+ <a href="/upload">Click here to try again.</a>
</p>
<hr>
</body>
diff --git a/bh20simplewebuploader/templates/footer.html b/bh20simplewebuploader/templates/footer.html
index 37a6b64..5a1f3c9 100644
--- a/bh20simplewebuploader/templates/footer.html
+++ b/bh20simplewebuploader/templates/footer.html
@@ -21,12 +21,15 @@
<img src="static/image/covid19biohackathon.png"></a>
</div>
<div class="sponsorimg">
- <a href="https://www.commonwl.org/"><img src="static/image/CWL.png"></a>
+ <a href="https://www.curii.com/"><img src="static/image/curii.logo.ai.svg"></a>
</div>
<div class="sponsorimg">
<a href="https://arvados.org/"><img src="static/image/arvados-logo.png"></a>
</div>
<div class="sponsorimg">
+ <a href="https://www.commonwl.org/"><img src="static/image/CWL.png"></a>
+ </div>
+ <div class="sponsorimg">
<a href="https://uthsc.edu/"><img src="static/image/UTHSC-primary-stacked-logo-4c.png"></a>
</div>
<div class="sponsorimg">
diff --git a/bh20simplewebuploader/templates/status.html b/bh20simplewebuploader/templates/status.html
index a1cf28f..e89437e 100644
--- a/bh20simplewebuploader/templates/status.html
+++ b/bh20simplewebuploader/templates/status.html
@@ -7,7 +7,8 @@
<h1>Sequence upload processing status</h1>
- <div class="status">
+ <div class="status">
+
{{ table }}
</div>
diff --git a/bh20simplewebuploader/templates/success.html b/bh20simplewebuploader/templates/success.html
index 9f0987c..c2302fa 100644
--- a/bh20simplewebuploader/templates/success.html
+++ b/bh20simplewebuploader/templates/success.html
@@ -9,7 +9,7 @@
<h1>Upload Successful</h1>
<hr>
<p>
- Your files have been uploaded. They should soon appear as output of the <a href="/download">Public SARS-CoV-2 Sequence Resource</a>.
+ Your files have been uploaded. You can track their <a href="/status">QC status</a>, once validated they will be part of the <a href="/download">Public SARS-CoV-2 Sequence Resource</a>.
</p>
<p>
The upload log was:
diff --git a/bh20simplewebuploader/templates/validated.html b/bh20simplewebuploader/templates/validated.html
new file mode 100644
index 0000000..cee94bd
--- /dev/null
+++ b/bh20simplewebuploader/templates/validated.html
@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+ {% include 'header.html' %}
+ <body>
+ {% include 'banner.html' %}
+ {% include 'menu.html' %}
+
+ <h1>Validated sequences</h1>
+
+ <div class="status">
+ {{ table }}
+ </div>
+
+{% include 'footer.html' %}
+
+ </body>
+</html>