From 1f470de17eba27d4406bb2840d5301cab3ccffda Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 5 Nov 2020 11:31:50 +0000 Subject: Using 'DOCS' instead of 'BLOG' --- bh20simplewebuploader/templates/blog.html | 5 ++--- bh20simplewebuploader/templates/home.html | 2 +- bh20simplewebuploader/templates/menu.html | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) (limited to 'bh20simplewebuploader') diff --git a/bh20simplewebuploader/templates/blog.html b/bh20simplewebuploader/templates/blog.html index 88ce4c3..fa08f5b 100644 --- a/bh20simplewebuploader/templates/blog.html +++ b/bh20simplewebuploader/templates/blog.html @@ -9,12 +9,11 @@ {{ embed|safe }}
-

Other blog entries

+

Other documents

{% else %} - {% include 'blurb.html' %} -

BLOG Entries:

+

Documents:

{% endif %}
diff --git a/bh20simplewebuploader/templates/home.html b/bh20simplewebuploader/templates/home.html index 77d2fb6..5986d72 100644 --- a/bh20simplewebuploader/templates/home.html +++ b/bh20simplewebuploader/templates/home.html @@ -73,7 +73,7 @@ URI's for disambiguation and machine readable metadata. For examples of - use, see the BLOG. + use, see the docs.

diff --git a/bh20simplewebuploader/templates/menu.html b/bh20simplewebuploader/templates/menu.html index 5d5fdc8..8e6ef52 100644 --- a/bh20simplewebuploader/templates/menu.html +++ b/bh20simplewebuploader/templates/menu.html @@ -6,7 +6,7 @@ STATUS DEMO EXPORT - BLOG + DOCS ABOUT CONTACT -- cgit v1.2.3 From b5143c79de268b844f3a6a63d92c6389b047f35e Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 9 Nov 2020 16:55:33 -0500 Subject: Make it so "pangenome analysis" only runs collect-seqs. Will ensure that metadata is kept up to date. GFA isn't being generated. Will introduce new workflow that uses from_sparql to analyze a subset. Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- bh20seqanalyzer/main.py | 2 +- bh20simplewebuploader/templates/resource.html | 4 ++-- workflows/update-workflows.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'bh20simplewebuploader') diff --git a/bh20seqanalyzer/main.py b/bh20seqanalyzer/main.py index 5f00080..0906958 100644 --- a/bh20seqanalyzer/main.py +++ b/bh20seqanalyzer/main.py @@ -215,7 +215,7 @@ class SeqAnalyzer: most_recent_analysis = self.api.groups().list(filters=[['owner_uuid', '=', self.pangenome_analysis_project]], order="created_at desc").execute() for m in most_recent_analysis["items"]: - wf = self.get_workflow_output_from_project(m["uuid"], "arv-main.cwl") + wf = self.get_workflow_output_from_project(m["uuid"], "collect-seqs.cwl") if wf is None: continue src = self.api.collections().get(uuid=wf["output_uuid"]).execute() diff --git a/bh20simplewebuploader/templates/resource.html b/bh20simplewebuploader/templates/resource.html index fc52f13..4c50fb9 100644 --- a/bh20simplewebuploader/templates/resource.html +++ b/bh20simplewebuploader/templates/resource.html @@ -10,8 +10,8 @@

All sequences project

All sequences (FASTA) relabled and deduplicated

Metadata (RDF) for all sequences

-

All sequences in Graphical Fragment Assembly (GFA) - More about GFA

-

All sequences in Optimized Dynamic Genome/Graph Implementation (ODGI) - More about ODGI

+ + diff --git a/workflows/update-workflows.sh b/workflows/update-workflows.sh index 3b69a58..5182ec4 100755 --- a/workflows/update-workflows.sh +++ b/workflows/update-workflows.sh @@ -1,3 +1,3 @@ #!/bin/sh arvados-cwl-runner --project-uuid=lugli-j7d0g-5hswinmpyho8dju --update-workflow=lugli-7fd4e-2zp9q4jo5xpif9y fastq2fasta/fastq2fasta.cwl -arvados-cwl-runner --project-uuid=lugli-j7d0g-5hswinmpyho8dju --update-workflow=lugli-7fd4e-mqfu9y3ofnpnho1 pangenome-generate/arv-main.cwl +arvados-cwl-runner --project-uuid=lugli-j7d0g-5hswinmpyho8dju --update-workflow=lugli-7fd4e-mqfu9y3ofnpnho1 pangenome-generate/collect-seqs.cwl -- cgit v1.2.3 From 1082b907d816f5da52aba6233073737632d0242f Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 9 Nov 2020 17:20:27 -0500 Subject: Make resource link work for both portable data hashes and sample id Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- bh20simplewebuploader/main.py | 14 ++++++++++++++ workflows/pangenome-generate/collect-seqs.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) (limited to 'bh20simplewebuploader') diff --git a/bh20simplewebuploader/main.py b/bh20simplewebuploader/main.py index 73503b4..405544c 100644 --- a/bh20simplewebuploader/main.py +++ b/bh20simplewebuploader/main.py @@ -675,13 +675,27 @@ sparqlURL='http://sparql.genenetwork.org/sparql/' @app.route('/resource/') def resource(id): """Get a COVID19 resource using identifier""" + query=f""" PREFIX pubseq: PREFIX sio: select distinct ?sample ?geoname ?date ?source ?geo ?sampletype ?institute ?sequenceuri +where {{ {{ ?sample sio:SIO_000115 "{id}" . ?sequenceuri pubseq:sample ?sample . +}} +union +{{ + pubseq:sample ?sample . + ?sequenceuri pubseq:sample ?sample . +}} +union +{{ + pubseq:sample ?sample . + ?sequenceuri pubseq:sample ?sample . +}} + ?sample ?geo . ?geo rdfs:label ?geoname . ?sample ?date . diff --git a/workflows/pangenome-generate/collect-seqs.py b/workflows/pangenome-generate/collect-seqs.py index af4a0dc..1a0807c 100644 --- a/workflows/pangenome-generate/collect-seqs.py +++ b/workflows/pangenome-generate/collect-seqs.py @@ -38,7 +38,7 @@ for item in validated: pdh = item["portable_data_hash"] with arvados.collection.CollectionReader(pdh, api_client=api, keep_client=keepclient) as col: with col.open("sequence.fasta", "rt") as fa: - subject = "http://collections.lugli.arvadosapi.com/c=%s/sequence.fasta" % pdh + subject = "http://covid19.genenetwork.org/resource/%s" % pdh label = fa.readline().strip() merged_metadata.write("<%s> \"%s\" .\n" % (subject, label[1:].replace('"', '\\"'))) skip = (subject in blacklist or label[1:] in blacklist) -- cgit v1.2.3 From 98a80bd64d8a495b8fddffdef6e07e5a3fbea1e3 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 9 Nov 2020 17:30:16 -0500 Subject: Extract PDH from result and construct URIs from that Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- bh20simplewebuploader/main.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'bh20simplewebuploader') diff --git a/bh20simplewebuploader/main.py b/bh20simplewebuploader/main.py index 405544c..51048a4 100644 --- a/bh20simplewebuploader/main.py +++ b/bh20simplewebuploader/main.py @@ -713,8 +713,9 @@ union logging.info("^^^^^^^^^^^^^^^^^^^^^^^^^^^^") # return jsonify({'sequences': int(result[0]["num"]["value"])}) sequenceuri=sample['sequenceuri']['value'] - collectionuri=sequenceuri.split('sequence.fasta')[0] - metauri=collectionuri+'metadata.yaml' + m = re.match(r"http://collections.lugli.arvadosapi.com/c=([^/]*)/sequence.fasta|http://covid19.genenetwork.org/resource/(.*)", sequenceuri) + fastauri = "http://collections.lugli.arvadosapi.com/c=%s/sequence.fasta" % m.group(1) + metauri = "http://collections.lugli.arvadosapi.com/c=%s/metadata.yaml" % m.group(1) locationuri=sample['geo']['value'] location=sample['geoname']['value'] date=sample['date']['value'] @@ -729,7 +730,18 @@ union institute='' if 'institute' in sample: institute=sample['institute']['value'] - return render_template('permalink.html',id=id,menu='',uri=f"http://covid19.genenetwork.org/resource/{id}",sequenceuri=sequenceuri,locationuri=locationuri,location=location,date=date,source=source,sampletype=sampletype,institute=institute,collectionuri=collectionuri,metauri=metauri) + return render_template('permalink.html', + id=id, + menu='', + uri=f"http://covid19.genenetwork.org/resource/{id}", + sequenceuri=fastauri, + locationuri=locationuri, + location=location, + date=date, + source=source, + sampletype=sampletype, + institute=institute, + metauri=metauri) # http://covid19.genenetwork.org/location?label=http://www.wikidata.org/entity/Q114 # http://localhost:5067/location?label=http://www.wikidata.org/entity/Q114 -- cgit v1.2.3 From 2e2e2280f59b783759b7b70d410fb7a5c59206d5 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 10 Nov 2020 03:42:12 +0300 Subject: Add tweet feed * bh20simplewebuploader/main.py (send_home): Fetch tweets from Redis. * bh20simplewebuploader/static/main.css: Add styles for tweets * bh20simplewebuploader/templates/home.html: Add tweet section in frontpage. --- bh20simplewebuploader/main.py | 27 +++++++++++++++++++++++++-- bh20simplewebuploader/static/main.css | 27 +++++++++++++++++++++++++++ bh20simplewebuploader/templates/home.html | 27 +++++++++++++++++++++++---- 3 files changed, 75 insertions(+), 6 deletions(-) (limited to 'bh20simplewebuploader') diff --git a/bh20simplewebuploader/main.py b/bh20simplewebuploader/main.py index 73503b4..d3b9591 100644 --- a/bh20simplewebuploader/main.py +++ b/bh20simplewebuploader/main.py @@ -7,6 +7,7 @@ import logging import os import sys import re +import redis import string import ruamel.yaml as yaml import pkg_resources @@ -252,8 +253,19 @@ def send_home(): """ Send the front page. """ - - return render_template('home.html', menu='HOME', load_map=True) + redis_client = redis.Redis(host=os.environ.get('HOST', 'localhost'), + port=os.environ.get('PORT', 6379), + db=os.environ.get('REDIS_DB', 0)) + tweets = [] + for tweet_id in redis_client.zrevrange('gn2-tweet-score:', + 0, -1): + tweets.append( + {k.decode("utf-8"): v.decode("utf-8") for k, v in + redis_client.hgetall(tweet_id).items()} + ) + return render_template('home.html', menu='HOME', + tweets=tweets, + load_map=True) @app.route('/upload') @@ -652,6 +664,17 @@ def blog_page(): buf = get_html_body('doc/blog/'+blog_content+'.html',"https://github.com/arvados/bh20-seq-resource/blob/master/doc/blog/"+blog_content+".org") return render_template('blog.html',menu='BLOG',embed=buf,blog=blog_content) +@app.route('/feed', methods=['GET']) +def feed(): + redis_client = redis.Redis(host=os.environ.get('HOST', 'localhost'), + port=os.environ.get('PORT', 6379), + db=os.environ.get('REDIS_DB', 0)) + tweets = [redis_client.hgetall(tweet_id) + for tweet_id in redis_client.zrevrange('bh20-tweet-score:', + 0, -1)] + return render_template('feed.html', + menu='FEED', + tweets=tweets) @app.route('/about') def about_page(): diff --git a/bh20simplewebuploader/static/main.css b/bh20simplewebuploader/static/main.css index bc4f705..76a1755 100644 --- a/bh20simplewebuploader/static/main.css +++ b/bh20simplewebuploader/static/main.css @@ -506,3 +506,30 @@ div.status { font-size: 16px; box-shadow: 0 8px 16px 0 rgba(0,0,0,0.2), 0 6px 20px 0 rgba(0,0,0,0.19); } + +.flex-container { + display: flex; + flex-direction: row; + flex-wrap: wrap; + justify-content: space-evenly; +} + +#twitter-feed { + background-color: #f5f8fa; + max-height: 440px; + max-width: 500px; + overflow-y: auto; + padding: 20px; + margin-left: auto; + margin-right: auto; +} + + +#twitter-feed ul { + list-style-type: none; +} + +#twitter-feed ul li { + border-bottom: 2px solid white; + padding-bottom: 20px; +} diff --git a/bh20simplewebuploader/templates/home.html b/bh20simplewebuploader/templates/home.html index 77d2fb6..516fb33 100644 --- a/bh20simplewebuploader/templates/home.html +++ b/bh20simplewebuploader/templates/home.html @@ -22,9 +22,27 @@ or REST API. For more information see the FAQ!.

-
-
-
+ +
+
+
+
+
+
    + {% for tweet in tweets|sort(reverse=true, attribute="timeposted")%} +
  • +

    + {{ tweet.tweet|urlize(40, target="_blank") }}
    + by {{ tweet.author }} +

    + + {{ tweet.timeposted }} + +
    • + {% endfor %} +
+
+

@@ -76,7 +94,8 @@ use, see the BLOG.

- + + {% include 'footer.html' %} -- cgit v1.2.3 From d42e6bc3fb58a087ba64427709034e0adf090524 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 10 Nov 2020 15:40:36 +0300 Subject: Fetch tweets from the correct redis queue * bh20simplewebuploader/main.py (send_home): Replace "gn2-tweet-score" with "bh20-tweet-score". --- bh20simplewebuploader/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bh20simplewebuploader') diff --git a/bh20simplewebuploader/main.py b/bh20simplewebuploader/main.py index d3b9591..9d27879 100644 --- a/bh20simplewebuploader/main.py +++ b/bh20simplewebuploader/main.py @@ -257,7 +257,7 @@ def send_home(): port=os.environ.get('PORT', 6379), db=os.environ.get('REDIS_DB', 0)) tweets = [] - for tweet_id in redis_client.zrevrange('gn2-tweet-score:', + for tweet_id in redis_client.zrevrange('bh20-tweet-score:', 0, -1): tweets.append( {k.decode("utf-8"): v.decode("utf-8") for k, v in -- cgit v1.2.3 From 203293363640e7f45ceaff09fb1a83f8c0f1496a Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 10 Nov 2020 15:42:00 +0300 Subject: Delete unused endpoint * bh20simplewebuploader/main.py (feed): Remove it. --- bh20simplewebuploader/main.py | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'bh20simplewebuploader') diff --git a/bh20simplewebuploader/main.py b/bh20simplewebuploader/main.py index 9d27879..510df1c 100644 --- a/bh20simplewebuploader/main.py +++ b/bh20simplewebuploader/main.py @@ -664,17 +664,6 @@ def blog_page(): buf = get_html_body('doc/blog/'+blog_content+'.html',"https://github.com/arvados/bh20-seq-resource/blob/master/doc/blog/"+blog_content+".org") return render_template('blog.html',menu='BLOG',embed=buf,blog=blog_content) -@app.route('/feed', methods=['GET']) -def feed(): - redis_client = redis.Redis(host=os.environ.get('HOST', 'localhost'), - port=os.environ.get('PORT', 6379), - db=os.environ.get('REDIS_DB', 0)) - tweets = [redis_client.hgetall(tweet_id) - for tweet_id in redis_client.zrevrange('bh20-tweet-score:', - 0, -1)] - return render_template('feed.html', - menu='FEED', - tweets=tweets) @app.route('/about') def about_page(): -- cgit v1.2.3 From d53120b6106e301c7f5b58bdb4dc89f85dbe2ad0 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Tue, 10 Nov 2020 12:59:30 +0000 Subject: Demo links to permalink for countries --- bh20simplewebuploader/static/main.js | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'bh20simplewebuploader') diff --git a/bh20simplewebuploader/static/main.js b/bh20simplewebuploader/static/main.js index 89bc603..6cc0d9f 100644 --- a/bh20simplewebuploader/static/main.js +++ b/bh20simplewebuploader/static/main.js @@ -68,6 +68,12 @@ function fetchHTMLTable(apiEndPoint) { htmlString="" for (var i=0; i" } -- cgit v1.2.3 From c01188ec20936462357b317f81567aadc64c8f33 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Tue, 10 Nov 2020 11:52:37 -0500 Subject: Use arvados uuids for RDF subjects. Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- bh20simplewebuploader/main.py | 5 +++++ workflows/pangenome-generate/collect-seqs.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'bh20simplewebuploader') diff --git a/bh20simplewebuploader/main.py b/bh20simplewebuploader/main.py index 51048a4..bcdf8d8 100644 --- a/bh20simplewebuploader/main.py +++ b/bh20simplewebuploader/main.py @@ -695,6 +695,11 @@ union pubseq:sample ?sample . ?sequenceuri pubseq:sample ?sample . }} +union +{{ + ?sequenceuri "{id}" . + ?sequenceuri pubseq:sample ?sample . +}} ?sample ?geo . ?geo rdfs:label ?geoname . diff --git a/workflows/pangenome-generate/collect-seqs.py b/workflows/pangenome-generate/collect-seqs.py index 1a0807c..225a61f 100644 --- a/workflows/pangenome-generate/collect-seqs.py +++ b/workflows/pangenome-generate/collect-seqs.py @@ -36,11 +36,14 @@ if len(sys.argv) > 3: for item in validated: pdh = item["portable_data_hash"] + uuid = item["uuid"] with arvados.collection.CollectionReader(pdh, api_client=api, keep_client=keepclient) as col: with col.open("sequence.fasta", "rt") as fa: - subject = "http://covid19.genenetwork.org/resource/%s" % pdh + subject = "http://covid19.genenetwork.org/resource/%s" % uuid label = fa.readline().strip() merged_metadata.write("<%s> \"%s\" .\n" % (subject, label[1:].replace('"', '\\"'))) + merged_metadata.write("<%s> \"%s\" .\n" % (subject, pdh)) + merged_metadata.write("<%s> \"%s\" .\n" % (subject, item["version"])) skip = (subject in blacklist or label[1:] in blacklist) if skip: merged_metadata.write("<%s> \"true\"^^ .\n" % subject) -- cgit v1.2.3
"+label+""+data[i]["count"]+"