Merge pull request #92 from arvados/upload-download-status

Split upload tab.  Add upload status tab.  Also a bunch of QC and uploader improvements.

12 files changed, 296 insertions, 95 deletions

diff --git a/Dockerfile b/Dockerfile
index 43fa8f2..1901ac7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,7 +13,7 @@ ADD bh20seqanalyzer /app/bh20simplewebuploader
 ADD bh20sequploader /app/bh20sequploader
 ADD bh20simplewebuploader /app/bh20simplewebuploader
 
-RUN pip3 install -e .
+RUN pip3 install -e .[web]
 
 ENV PORT 8080
 CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:8080", "bh20simplewebuploader.main:app"]
diff --git a/bh20seqanalyzer/main.py b/bh20seqanalyzer/main.py
index 1746587..0b52e6b 100644
--- a/bh20seqanalyzer/main.py
+++ b/bh20seqanalyzer/main.py
@@ -17,10 +17,11 @@ logging.basicConfig(format="[%(asctime)s] %(levelname)s %(message)s", datefmt="%
 logging.getLogger("googleapiclient.discovery").setLevel(logging.WARN)
 
 def validate_upload(api, collection, validated_project,
-                    fastq_project, fastq_workflow_uuid):
+                    fastq_project, fastq_workflow_uuid,
+                    revalidate):
     col = arvados.collection.Collection(collection["uuid"])
 
-    if collection.get("status") in ("validated", "rejected"):
+    if not revalidate and collection["properties"].get("status") in ("validated", "rejected"):
         return False
 
     # validate the collection here.  Check metadata, etc.
@@ -28,11 +29,12 @@ def validate_upload(api, collection, validated_project,
 
     errors = []
 
-    dup = api.collections().list(filters=[["owner_uuid", "=", validated_project],
-                                          ["portable_data_hash", "=", col.portable_data_hash()]]).execute()
-    if dup["items"]:
-        # This exact collection has been uploaded before.
-        errors.append("Duplicate of %s" % ([d["uuid"] for d in dup["items"]]))
+    if collection["owner_uuid"] != validated_project:
+        dup = api.collections().list(filters=[["owner_uuid", "=", validated_project],
+                                              ["portable_data_hash", "=", col.portable_data_hash()]]).execute()
+        if dup["items"]:
+            # This exact collection has been uploaded before.
+            errors.append("Duplicate of %s" % ([d["uuid"] for d in dup["items"]]))
 
     if not errors:
         if "metadata.yaml" not in col:
@@ -70,12 +72,15 @@ def validate_upload(api, collection, validated_project,
 
 
     if not errors:
-        logging.info("Added '%s' to validated sequences" % collection["name"])
         # Move it to the "validated" project to be included in the next analysis
+        if "errors" in collection["properties"]:
+            del collection["properties"]["errors"]
         collection["properties"]["status"] = "validated"
         api.collections().update(uuid=collection["uuid"], body={
             "owner_uuid": validated_project,
-            "name": "%s (%s)" % (collection["name"], time.asctime(time.gmtime()))}).execute()
+            "name": "%s (%s)" % (collection["name"], time.asctime(time.gmtime())),
+            "properties": collection["properties"]}).execute()
+        logging.info("Added '%s' to validated sequences" % collection["name"])
         return True
     else:
         # It is invalid
@@ -155,7 +160,9 @@ def start_pangenome_analysis(api,
                              validated_project,
                              schema_ref,
                              exclude_list):
-    validated = arvados.util.list_all(api.collections().list, filters=[["owner_uuid", "=", validated_project]])
+    validated = arvados.util.list_all(api.collections().list, filters=[
+        ["owner_uuid", "=", validated_project],
+        ["properties.status", "=", "validated"]])
     inputobj = {
         "inputReads": [],
         "metadata": [],
@@ -187,14 +194,15 @@ def get_workflow_output_from_project(api, uuid):
     cr = api.container_requests().list(filters=[['owner_uuid', '=', uuid],
                                                 ["requesting_container_uuid", "=", None]]).execute()
     if cr["items"] and cr["items"][0]["output_uuid"]:
-        return cr["items"][0]
-    else:
-        return None
+        container = api.containers().get(uuid=cr["items"][0]["container_uuid"]).execute()
+        if container["state"] == "Complete" and container["exit_code"] == 0:
+            return cr["items"][0]
+    return None
 
 
 def copy_most_recent_result(api, analysis_project, latest_result_uuid):
     most_recent_analysis = api.groups().list(filters=[['owner_uuid', '=', analysis_project]],
-                                                  order="created_at desc", limit=1).execute()
+                                                  order="created_at desc").execute()
     for m in most_recent_analysis["items"]:
         wf = get_workflow_output_from_project(api, m["uuid"])
         if wf:
@@ -220,6 +228,7 @@ def move_fastq_to_fasta_results(api, analysis_project, uploader_project):
                                      body={"owner_uuid": uploader_project}).execute()
             p["properties"]["moved_output"] = True
             api.groups().update(uuid=p["uuid"], body={"properties": p["properties"]}).execute()
+            break
 
 
 def upload_schema(api, workflow_def_project):
@@ -297,6 +306,7 @@ def main():
     parser.add_argument('--no-start-analysis', action="store_true")
     parser.add_argument('--once', action="store_true")
     parser.add_argument('--print-status', type=str, default=None)
+    parser.add_argument('--revalidate', action="store_true", default=None)
     args = parser.parse_args()
 
     api = arvados.api()
@@ -330,7 +340,8 @@ def main():
             at_least_one_new_valid_seq = validate_upload(api, c,
                                                          args.validated_project,
                                                          args.fastq_project,
-                                                         args.fastq_workflow_uuid) or at_least_one_new_valid_seq
+                                                         args.fastq_workflow_uuid,
+                                                         args.revalidate) or at_least_one_new_valid_seq
 
         if at_least_one_new_valid_seq and not args.no_start_analysis:
             start_pangenome_analysis(api,
diff --git a/bh20sequploader/main.py b/bh20sequploader/main.py
index fd0278d..f744a8c 100644
--- a/bh20sequploader/main.py
+++ b/bh20sequploader/main.py
@@ -19,8 +19,10 @@ log = logging.getLogger(__name__ )
 log.debug("Entering sequence uploader")
 
 ARVADOS_API_HOST='lugli.arvadosapi.com'
-ARVADOS_API_TOKEN='2fbebpmbo3rw3x05ueu2i6nx70zhrsb1p22ycu3ry34m4x4462'
+UPLOADER_API_TOKEN='2fbebpmbo3rw3x05ueu2i6nx70zhrsb1p22ycu3ry34m4x4462'
+ANONYMOUS_API_TOKEN='5o42qdxpxp5cj15jqjf7vnxx5xduhm4ret703suuoa3ivfglfh'
 UPLOAD_PROJECT='lugli-j7d0g-n5clictpuvwk8aa'
+VALIDATED_PROJECT='lugli-j7d0g-5ct8p1i1wrgyjvp'
 
 def qc_stuff(metadata, sequence_p1, sequence_p2, do_qc=True):
     failed = False
@@ -67,9 +69,14 @@ def main():
     parser.add_argument('sequence_p2', type=argparse.FileType('rb'), default=None, nargs='?', help='sequence FASTQ pair')
     parser.add_argument("--validate", action="store_true", help="Dry run, validate only")
     parser.add_argument("--skip-qc", action="store_true", help="Skip local qc check")
+    parser.add_argument("--trusted", action="store_true", help="Trust local validation and add directly to validated project")
     args = parser.parse_args()
 
-    api = arvados.api(host=ARVADOS_API_HOST, token=ARVADOS_API_TOKEN, insecure=True)
+    if args.trusted:
+        # Use credentials from environment
+        api = arvados.api()
+    else:
+        api = arvados.api(host=ARVADOS_API_HOST, token=UPLOADER_API_TOKEN, insecure=True)
 
     target = qc_stuff(args.metadata, args.sequence_p1, args.sequence_p2, not args.skip_qc)
     seqlabel = target[0][1]
@@ -106,7 +113,21 @@ def main():
         "upload_user": "%s@%s" % (username, socket.gethostname())
     }
 
-    col.save_new(owner_uuid=UPLOAD_PROJECT, name="%s uploaded by %s from %s" %
+    api2 = arvados.api(host=ARVADOS_API_HOST, token=ANONYMOUS_API_TOKEN, insecure=True)
+    dup = api2.collections().list(filters=[["owner_uuid", "in", [VALIDATED_PROJECT, UPLOAD_PROJECT]],
+                                           ["portable_data_hash", "=", col.portable_data_hash()]]).execute()
+    if dup["items"]:
+        # This exact collection has been uploaded before.
+        print("Duplicate of %s" % ([d["uuid"] for d in dup["items"]]))
+        exit(1)
+
+    if args.trusted:
+        properties["status"] = "validated"
+        owner_uuid = VALIDATED_PROJECT
+    else:
+        owner_uuid = UPLOAD_PROJECT
+
+    col.save_new(owner_uuid=owner_uuid, name="%s uploaded by %s from %s" %
                  (seqlabel, properties['upload_user'], properties['upload_ip']),
                  properties=properties, ensure_unique_name=True)
 
diff --git a/bh20sequploader/qc_metadata.py b/bh20sequploader/qc_metadata.py
index 2b57991..27657b1 100644
--- a/bh20sequploader/qc_metadata.py
+++ b/bh20sequploader/qc_metadata.py
@@ -8,15 +8,20 @@ import traceback
 from rdflib import Graph, Namespace
 from pyshex.evaluate import evaluate
 
+metadata_schema = None
 
 def qc_metadata(metadatafile):
+    global metadata_schema
     log = logging.getLogger(__name__ )
-    schema_resource = pkg_resources.resource_stream(__name__, "bh20seq-schema.yml")
-    cache = {"https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml": schema_resource.read().decode("utf-8")}
+    if metadata_schema is None:
+        schema_resource = pkg_resources.resource_stream(__name__, "bh20seq-schema.yml")
+        cache = {"https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml": schema_resource.read().decode("utf-8")}
+        metadata_schema = schema_salad.schema.load_schema("https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml", cache=cache)
+
     (document_loader,
      avsc_names,
      schema_metadata,
-     metaschema_loader) = schema_salad.schema.load_schema("https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-schema.yml", cache=cache)
+     metaschema_loader) = metadata_schema
 
     shex = pkg_resources.resource_stream(__name__, "bh20seq-shex.rdf").read().decode("utf-8")
 
@@ -27,6 +32,10 @@ def qc_metadata(metadatafile):
     g = schema_salad.jsonld_context.makerdf("workflow", doc, document_loader.ctx)
     rslt, reason = evaluate(g, shex, doc["id"], "https://raw.githubusercontent.com/arvados/bh20-seq-resource/master/bh20sequploader/bh20seq-shex.rdf#submissionShape")
 
+    # As part of QC make sure serialization works too, this will raise
+    # an exception if there are invalid URIs.
+    g.serialize(format="ntriples")
+
     if not rslt:
         raise Exception(reason)
 
diff --git a/bh20simplewebuploader/main.py b/bh20simplewebuploader/main.py
index 3100dfd..9132453 100644
--- a/bh20simplewebuploader/main.py
+++ b/bh20simplewebuploader/main.py
@@ -13,12 +13,20 @@ import pkg_resources
 from flask import Flask, request, redirect, send_file, send_from_directory, render_template, jsonify
 import os.path
 import requests
+import io
+import arvados
+from markupsafe import Markup
+
+ARVADOS_API = 'lugli.arvadosapi.com'
+ANONYMOUS_TOKEN = '5o42qdxpxp5cj15jqjf7vnxx5xduhm4ret703suuoa3ivfglfh'
+UPLOADER_PROJECT = 'lugli-j7d0g-n5clictpuvwk8aa'
+VALIDATED_PROJECT = 'lugli-j7d0g-5ct8p1i1wrgyjvp'
 
 logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger(__name__ )
 log.debug("Entering web uploader")
 
-if not os.path.isfile('bh20sequploader/mainx.py'):
+if not os.path.isfile('bh20sequploader/main.py'):
     print("WARNING: run FLASK from the root of the source repository!", file=sys.stderr)
 
 app = Flask(__name__, static_url_path='/static', static_folder='static')
@@ -224,12 +232,21 @@ METADATA_OPTION_DEFINITIONS = yaml.safe_load(pkg_resources.resource_stream("bh20
 FORM_ITEMS = generate_form(METADATA_SCHEMA, METADATA_OPTION_DEFINITIONS)
 
 @app.route('/')
+def send_home():
+    """
+    Send the front page.
+    """
+
+    return render_template('home.html', menu='HOME')
+
+
+@app.route('/upload')
 def send_form():
     """
     Send the file upload form/front page.
     """
 
-    return render_template('form.html', fields=FORM_ITEMS, menu='HOME')
+    return render_template('form.html', fields=FORM_ITEMS, menu='UPLOAD')
 
 class FileTooBigError(RuntimeError):
     """
@@ -405,7 +422,7 @@ def receive_files():
 
         # Try and upload files to Arvados using the sequence uploader CLI
 
-        cmd = ['python3','bh20sequploader/main.py', fasta_dest, metadata_dest]
+        cmd = ['python3','bh20sequploader/main.py', metadata_dest, fasta_dest]
         print(" ".join(cmd),file=sys.stderr)
         result = subprocess.run(cmd,
             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -439,7 +456,83 @@ def get_html_body(fn):
 @app.route('/download')
 def download_page():
     buf = get_html_body('doc/web/download.html')
-    return render_template('about.html',menu='DOWNLOAD',embed=buf)
+    return render_template('resource.html',menu='DOWNLOAD',embed=buf)
+
+def pending_table(output, items):
+    output.write(
+"""
+<table>
+<tr><th>Collection</th>
+<th>Sequence label</th></tr>
+""")
+    for r in items:
+        if r["status"] != "pending":
+            continue
+        output.write("<tr>")
+        output.write("<td><a href='https://workbench.lugli.arvadosapi.com/collections/%s'>%s</a></td>" % (r["uuid"], r["uuid"]))
+        output.write("<td>%s</td>" % Markup.escape(r["sequence_label"]))
+        output.write("</tr>")
+    output.write(
+"""
+</table>
+""")
+
+def rejected_table(output, items):
+    output.write(
+"""
+<table>
+<tr><th>Collection</th>
+<th>Sequence label</th>
+<th>Errors</th></tr>
+""")
+    for r in items:
+        if r["status"] != "rejected":
+            continue
+        output.write("<tr>")
+        output.write("<td><a href='https://workbench.lugli.arvadosapi.com/collections/%s'>%s</a></td>" % (r["uuid"], r["uuid"]))
+        output.write("<td>%s</td>" % Markup.escape(r["sequence_label"]))
+        output.write("<td><pre>%s</pre></td>" % Markup.escape("\n".join(r.get("errors", []))))
+        output.write("</tr>")
+    output.write(
+"""
+</table>
+""")
+
+
+@app.route('/status')
+def status_page():
+    """
+    Processing status
+    """
+
+    api = arvados.api(host=ARVADOS_API, token=ANONYMOUS_TOKEN)
+    pending = arvados.util.list_all(api.collections().list, filters=[["owner_uuid", "=", UPLOADER_PROJECT]])
+    out = []
+    status = {}
+    for p in pending:
+        prop = p["properties"]
+        out.append(prop)
+        if "status" not in prop:
+            prop["status"] = "pending"
+        prop["created_at"] = p["created_at"]
+        prop["uuid"] = p["uuid"]
+        status[prop["status"]] = status.get(prop["status"], 0) + 1
+
+    output = io.StringIO()
+
+    validated = api.collections().list(filters=[["owner_uuid", "=", VALIDATED_PROJECT]], limit=1).execute()
+    status["passed"] = validated["items_available"]
+
+    for s in (("passed", "/download"), ("pending", "#pending"), ("rejected", "#rejected")):
+        output.write("<p><a href='%s'>%s sequences QC %s</a></p>" % (s[1], status.get(s[0], 0), s[0]))
+
+    output.write("<a id='pending'><h1>Pending</h1>")
+    pending_table(output, out)
+
+    output.write("<a id='rejected'><h1>Rejected</h1>")
+    rejected_table(output, out)
+
+    return render_template('status.html', table=Markup(output.getvalue()), menu='STATUS')
 
 @app.route('/demo')
 def demo_page():
@@ -474,20 +567,10 @@ baseURL='http://sparql.genenetwork.org/sparql/'
 
 @app.route('/api/getCount', methods=['GET'])
 def getCount():
-    query="""
-PREFIX pubseq: <http://biohackathon.org/bh20-seq-schema#MainSchema/>
-select (COUNT(distinct ?dataset) as ?num)
-{
-   ?dataset pubseq:submitter ?id .
-   ?id ?p ?submitter
-}
-"""
-    payload = {'query': query, 'format': 'json'}
-    r = requests.get(baseURL, params=payload)
-    result = r.json()['results']['bindings']
-    # [{'num': {'type': 'typed-literal', 'datatype': 'http://www.w3.org/2001/XMLSchema#integer', 'value': '1352'}}]
-    # print(result, file=sys.stderr)
-    return jsonify({'sequences': int(result[0]["num"]["value"])})
+    api = arvados.api(host=ARVADOS_API, token=ANONYMOUS_TOKEN)
+    c = api.collections().list(filters=[["owner_uuid", "=", VALIDATED_PROJECT]], limit=1).execute()
+
+    return jsonify({'sequences': c["items_available"]})
 
 @app.route('/api/getAllaccessions', methods=['GET'])
 def getAllaccessions():
diff --git a/bh20simplewebuploader/static/main.css b/bh20simplewebuploader/static/main.css
index 5a9f231..b9b27f4 100644
--- a/bh20simplewebuploader/static/main.css
+++ b/bh20simplewebuploader/static/main.css
@@ -168,11 +168,11 @@ span.dropt:hover {text-decoration: none; background: #ffffff; z-index: 6; }
     grid-template-rows: auto;
     row-gap:5px;
     grid-template-areas:
-        "a a b b"
-            "a a c c"
-            "a a d d"
-            "e e e e"
-            "f f f f";
+        "b b a a"
+	"b b c c"
+        "b b d d"
+        "e e e e"
+        "f f f f";
     grid-auto-flow: column;
 }
 
@@ -361,3 +361,19 @@ footer {
 .blog-table-body {
     display: table-row-group;
 }
+
+div.status {
+    margin: 1em;
+}
+
+.status table {
+    display: table;
+    width: 100%;
+}
+
+.status td, th {
+    padding-left: 1em;
+    padding-right: 1em;
+    vertical-align: top;
+    border-bottom: 1px solid #ddd;
+}
diff --git a/bh20simplewebuploader/templates/footer.html b/bh20simplewebuploader/templates/footer.html
index 9326b1e..a1dd4fd 100644
--- a/bh20simplewebuploader/templates/footer.html
+++ b/bh20simplewebuploader/templates/footer.html
@@ -41,3 +41,17 @@
   </div>
 </section>
 <script type="text/javascript" src="/static/main.js"></script>
+
+<script type="text/javascript">
+  document.addEventListener("DOMContentLoaded", function(){
+      var count = fetch("/api/getCount")
+          .then((resp) => resp.json())
+          .then(function (data) {
+              count = data["sequences"];
+              console.log(count);
+              span = document.getElementById("Counter");
+              txt = document.createTextNode(count);
+              span.appendChild(txt);
+      });
+  });
+</script>
diff --git a/bh20simplewebuploader/templates/form.html b/bh20simplewebuploader/templates/form.html
index 0ad2080..b9b3776 100644
--- a/bh20simplewebuploader/templates/form.html
+++ b/bh20simplewebuploader/templates/form.html
@@ -7,43 +7,6 @@
 
         <section>
             <form action="/submit" method="POST" enctype="multipart/form-data" id="main_form" class="grid-container">
-                <div class="intro">
-                    <p>
-                        Make your sequence
-                        data <a href="https://en.wikipedia.org/wiki/FAIR_data">FAIR</a>. Upload
-                        your SARS-CoV-2 sequence (FASTA or FASTQ
-                        formats) with metadata (JSONLD) to
-                        the <a href="/about">public sequence
-                        resource</a>. The upload will trigger a
-                        recompute with all available sequences into a
-                        Pangenome available for
-                        <a href="/download">download</a>!
-                    </p>
-                    <p>
-                        Your uploaded sequence will automatically be
-                        processed and incorporated into the public
-                        pangenome with metadata using worklows from
-                        the High Performance Open Biology Lab
-                        defined <a href="https://github.com/hpobio-lab/viral-analysis/tree/master/cwl/pangenome-generate">here</a>. All
-                        data is published under
-                        a <a href="https://creativecommons.org/licenses/by/4.0/">Creative
-                        Commons license</a> You can take the published
-                        (GFA/RDF/FASTA) data and store it in a triple
-                        store for further processing.  Clinical
-                        data can be stored
-                        securely
-                        at <a href="https://redcap-covid19.elixir-luxembourg.org/redcap/">REDCap</a>.
-                    </p>
-                    <p>
-                      Note that form fields contain
-                      web <a href="https://en.wikipedia.org/wiki/Web_Ontology_Language">ontology
-                      URI's</a>
-                      for <a href="https://en.wikipedia.org/wiki/Wikipedia:Disambiguation">disambiguation</a>
-                      and machine readable metadata. For examples of
-                      use, see the <a href="/blog">BLOG</a>.
-                    </p>
-                </div>
-
                 <div class="fasta-file-select">
                     <h2><svg class="bi bi-cloud-upload" width="1.2em" height="1.2em" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
                         <path d="M4.887 6.2l-.964-.165A2.5 2.5 0 103.5 11H6v1H3.5a3.5 3.5 0 11.59-6.95 5.002 5.002 0 119.804 1.98A2.501 2.501 0 0113.5 12H10v-1h3.5a1.5 1.5 0 00.237-2.981L12.7 7.854l.216-1.028a4 4 0 10-7.843-1.587l-.185.96z"/>
@@ -160,18 +123,6 @@
 
   setMode()
 
-  document.addEventListener("DOMContentLoaded", function(){
-      var count = fetch("/api/getCount")
-          .then((resp) => resp.json())
-          .then(function (data) {
-              count = data["sequences"];
-              console.log(count);
-              span = document.getElementById("Counter");
-              txt = document.createTextNode(count);
-              span.appendChild(txt);
-      });
-});
-
 </script>
 
    </body>
diff --git a/bh20simplewebuploader/templates/home.html b/bh20simplewebuploader/templates/home.html
new file mode 100644
index 0000000..b90a18d
--- /dev/null
+++ b/bh20simplewebuploader/templates/home.html
@@ -0,0 +1,50 @@
+<!DOCTYPE html>
+<html>
+  {% include 'header.html' %}
+    <body>
+      {% include 'banner.html' %}
+      {% include 'menu.html' %}
+
+      <section>
+                <div class="intro">
+                    <p>
+                        Make your sequence
+                        data <a href="https://en.wikipedia.org/wiki/FAIR_data">FAIR</a>. Upload
+                        your SARS-CoV-2 sequence (FASTA or FASTQ
+                        formats) with metadata (JSONLD) to
+                        the <a href="/about">public sequence
+                        resource</a>. The upload will trigger a
+                        recompute with all available sequences into a
+                        Pangenome available for
+                        <a href="/download">download</a>!
+                    </p>
+                    <p>
+                        Your uploaded sequence will automatically be
+                        processed and incorporated into the public
+                        pangenome with metadata using worklows from
+                        the High Performance Open Biology Lab
+                        defined <a href="https://github.com/hpobio-lab/viral-analysis/tree/master/cwl/pangenome-generate">here</a>. All
+                        data is published under
+                        a <a href="https://creativecommons.org/licenses/by/4.0/">Creative
+                        Commons license</a> You can take the published
+                        (GFA/RDF/FASTA) data and store it in a triple
+                        store for further processing.  Clinical
+                        data can be stored
+                        securely
+                        at <a href="https://redcap-covid19.elixir-luxembourg.org/redcap/">REDCap</a>.
+                    </p>
+                    <p>
+                      Note that form fields contain
+                      web <a href="https://en.wikipedia.org/wiki/Web_Ontology_Language">ontology
+                      URI's</a>
+                      for <a href="https://en.wikipedia.org/wiki/Wikipedia:Disambiguation">disambiguation</a>
+                      and machine readable metadata. For examples of
+                      use, see the <a href="/blog">BLOG</a>.
+                    </p>
+                </div>
+        </section>
+
+{% include 'footer.html' %}
+
+   </body>
+</html>
diff --git a/bh20simplewebuploader/templates/menu.html b/bh20simplewebuploader/templates/menu.html
index 6f97e19..0f6003f 100644
--- a/bh20simplewebuploader/templates/menu.html
+++ b/bh20simplewebuploader/templates/menu.html
@@ -1,7 +1,9 @@
 <section class="menu">
   <div class="topnav" id="myTopnav">
-    <a href="/" class="{{ 'active' if menu=='HOME' }}">COVID-19</a>
+    <a href="/" class="{{ 'active' if menu=='HOME' }}">PUBSEQ</a>
     <a href="/download" class="{{ 'active' if menu=='DOWNLOAD' }}">DOWNLOAD</a>
+    <a href="/upload" class="{{ 'active' if menu=='UPLOAD' }}">UPLOAD</a>
+    <a href="/status" class="{{ 'active' if menu=='STATUS' }}">STATUS</a>
     <a href="/demo" class="{{ 'active' if menu=='DEMO' }}">DEMO</a>
     <a href="/blog" class="{{ 'active' if menu=='BLOG' }}">BLOG</a>
     <a href="/about" class="{{ 'active' if menu=='ABOUT' }}">ABOUT</a>
diff --git a/bh20simplewebuploader/templates/resource.html b/bh20simplewebuploader/templates/resource.html
new file mode 100644
index 0000000..91b6c20
--- /dev/null
+++ b/bh20simplewebuploader/templates/resource.html
@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+  {% include 'header.html' %}
+    <body>
+      {% include 'banner.html' %}
+      {% include 'menu.html' %}
+
+      <div class="status">
+	<p><img src="https://workbench.lugli.arvadosapi.com/collections/lugli-4zz18-z513nlpqm03hpca/relabeledSeqs_dedup_relabeledSeqs_dedup.png" height="300px"></p>
+	<p><a href="https://workbench.lugli.arvadosapi.com/projects/lugli-j7d0g-5ct8p1i1wrgyjvp#Data_collections">All sequences project</a></p>
+	<p><a href="https://workbench.lugli.arvadosapi.com/collections/lugli-4zz18-z513nlpqm03hpca/relabeledSeqs_dedup.fasta">All sequences (FASTA) relabled and deduplicated</a></p>
+	<p><a href="https://workbench.lugli.arvadosapi.com/collections/lugli-4zz18-z513nlpqm03hpca/mergedmetadata.ttl">Metadata (RDF) for all sequences</a></p>
+	<p><a href="https://workbench.lugli.arvadosapi.com/collections/lugli-4zz18-z513nlpqm03hpca/relabeledSeqs_dedup_relabeledSeqs_dedup.gfa">All sequences in Graphical Fragment Assembly (GFA)</a> - <a href="https://github.com/GFA-spec/GFA-spec">More about GFA</a></p>
+	<p><a href="https://workbench.lugli.arvadosapi.com/collections/lugli-4zz18-z513nlpqm03hpca/relabeledSeqs_dedup_relabeledSeqs_dedup.gfa">All sequences in Optimized Dynamic Genome/Graph Implementation (ODGI)</a> - <a href="https://github.com/vgteam/odgi">More about ODGI</a></p>
+	<p><a href="https://workbench.lugli.arvadosapi.com/collections/lugli-4zz18-z513nlpqm03hpca/relabeledSeqs_dedup_relabeledSeqs_dedup.ttl.xz">All sequences in RDF using spodgi</a> - <a href="https://github.com/pangenome/spodgi">More about spodgi</a></p>
+
+
+	<p><a href="http://sparql.genenetwork.org/sparql/">SPARQL endpoint</a> - <a href="http://sparql.genenetwork.org/sparql/?default-graph-uri=&query=SELECT+DISTINCT+%3Ffasta+%3Fvalue+WHERE+%7B%3Ffasta+%3Fx%5B+%3Chttp%3A%2F%2Fedamontology.org%2Fdata_2091%3E+%3Fvalue+%5D%7D%0D%0A&format=text%2Fhtml&timeout=0&debug=on&run=+Run+Query+">Sample query for accessions</a>
+
+	{{ embed|safe }}
+
+	</div>
+
+{% include 'footer.html' %}
+
+   </body>
+</html>
diff --git a/bh20simplewebuploader/templates/status.html b/bh20simplewebuploader/templates/status.html
new file mode 100644
index 0000000..a1cf28f
--- /dev/null
+++ b/bh20simplewebuploader/templates/status.html
@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+  {% include 'header.html' %}
+    <body>
+      {% include 'banner.html' %}
+      {% include 'menu.html' %}
+
+      <h1>Sequence upload processing status</h1>
+
+        <div class="status">
+	  {{ table }}
+        </div>
+
+{% include 'footer.html' %}
+
+   </body>
+</html>