diff options
| -rw-r--r-- | README.md | 2 | ||||
| -rw-r--r-- | bh20simplewebuploader/main.py | 16 | ||||
| -rw-r--r-- | bh20simplewebuploader/static/main.css | 27 | ||||
| -rw-r--r-- | bh20simplewebuploader/static/main.js | 6 | ||||
| -rw-r--r-- | bh20simplewebuploader/templates/home.html | 27 | ||||
| -rw-r--r-- | doc/INSTALL.md | 6 | ||||
| -rw-r--r-- | doc/blog/using-covid-19-pubseq-part2.html | 127 | ||||
| -rw-r--r-- | doc/blog/using-covid-19-pubseq-part2.org | 21 | ||||
| -rwxr-xr-x | scripts/update_virtuoso/check_for_updates.py | 52 | ||||
| -rw-r--r-- | setup.py | 2 | ||||
| -rw-r--r-- | test/rest-api.org | 2 |
11 files changed, 198 insertions, 90 deletions
@@ -40,7 +40,7 @@ Note that you will need to repeat the `. venv/bin/activate` step from this direc Install from PyPi: ```sh -pip3 bh20-seq-uploader +pip3 install bh20-seq-uploader ``` Install from git: diff --git a/bh20simplewebuploader/main.py b/bh20simplewebuploader/main.py index bcdf8d8..1d7154c 100644 --- a/bh20simplewebuploader/main.py +++ b/bh20simplewebuploader/main.py @@ -7,6 +7,7 @@ import logging import os import sys import re +import redis import string import ruamel.yaml as yaml import pkg_resources @@ -252,8 +253,19 @@ def send_home(): """ Send the front page. """ - - return render_template('home.html', menu='HOME', load_map=True) + redis_client = redis.Redis(host=os.environ.get('HOST', 'localhost'), + port=os.environ.get('PORT', 6379), + db=os.environ.get('REDIS_DB', 0)) + tweets = [] + for tweet_id in redis_client.zrevrange('bh20-tweet-score:', + 0, -1): + tweets.append( + {k.decode("utf-8"): v.decode("utf-8") for k, v in + redis_client.hgetall(tweet_id).items()} + ) + return render_template('home.html', menu='HOME', + tweets=tweets, + load_map=True) @app.route('/upload') diff --git a/bh20simplewebuploader/static/main.css b/bh20simplewebuploader/static/main.css index bc4f705..76a1755 100644 --- a/bh20simplewebuploader/static/main.css +++ b/bh20simplewebuploader/static/main.css @@ -506,3 +506,30 @@ div.status { font-size: 16px; box-shadow: 0 8px 16px 0 rgba(0,0,0,0.2), 0 6px 20px 0 rgba(0,0,0,0.19); } + +.flex-container { + display: flex; + flex-direction: row; + flex-wrap: wrap; + justify-content: space-evenly; +} + +#twitter-feed { + background-color: #f5f8fa; + max-height: 440px; + max-width: 500px; + overflow-y: auto; + padding: 20px; + margin-left: auto; + margin-right: auto; +} + + +#twitter-feed ul { + list-style-type: none; +} + +#twitter-feed ul li { + border-bottom: 2px solid white; + padding-bottom: 20px; +} diff --git a/bh20simplewebuploader/static/main.js b/bh20simplewebuploader/static/main.js index 89bc603..6cc0d9f 100644 --- a/bh20simplewebuploader/static/main.js +++ b/bh20simplewebuploader/static/main.js @@ -68,6 +68,12 @@ function fetchHTMLTable(apiEndPoint) { htmlString="<table>" for (var i=0; i<data.length;i++) { let url = data[i]["key"]; + continents = ["Q538", "Q48", "Q49", "Q18", "Q15", "Q27611" ]; + node = url.split("/").pop(); + console.log(continents.includes(node)); + if (url.includes("wikidata") && !continents.includes(node)) { + url = "http://covid19.genenetwork.org/location?label="+url ; + } let label = data[i]["label"]; htmlString=htmlString+"<tr><td><a href=\""+url+"\">"+label+"</a></td><td>"+data[i]["count"]+"<td></tr>" } diff --git a/bh20simplewebuploader/templates/home.html b/bh20simplewebuploader/templates/home.html index 5986d72..be948f6 100644 --- a/bh20simplewebuploader/templates/home.html +++ b/bh20simplewebuploader/templates/home.html @@ -22,9 +22,27 @@ or <a href="/apidoc">REST API</a>. For more information see the <a href="/about">FAQ!</a>. </p> - <section id="map_view" class="map"> - <div id="mapid"></div> - </section> + + <section class="flex-container"> + <div id="map_view" class="map"> + <div id="mapid"></div> + </div> + <div id="twitter-feed"> + <ul> + {% for tweet in tweets|sort(reverse=true, attribute="timeposted")%} + <li> + <p class="tweet"> + {{ tweet.tweet|urlize(40, target="_blank") }} <br/> + by {{ tweet.author }} + </p> + <small class="timeposted"> + {{ tweet.timeposted }} + </small> + </li> + {% endfor %} + </ul> + </div> + </section> <a href="https://projectredcap.org/"><img class="img-right" src="static/image/REDCap.png" /></a> <p> @@ -76,7 +94,8 @@ use, see the <a href="/blog">docs</a>. </p> </div> - </section> + </section> + {% include 'footer.html' %} diff --git a/doc/INSTALL.md b/doc/INSTALL.md index 367b452..0367c63 100644 --- a/doc/INSTALL.md +++ b/doc/INSTALL.md @@ -107,3 +107,9 @@ And run from the data dir guix environment --ad-hoc virtuoso-ose -- virtuoso-t -f Visit http://localhost:8890/sparql + +To update the turtle files do + + guix environment -C guix --ad-hoc python python-requests raptor2 curl --network -- python3 ./scripts/update_virtuoso/check_for_updates.py cache.txt dba dba + +where dba is the default password. diff --git a/doc/blog/using-covid-19-pubseq-part2.html b/doc/blog/using-covid-19-pubseq-part2.html index 567980d..eff6fcd 100644 --- a/doc/blog/using-covid-19-pubseq-part2.html +++ b/doc/blog/using-covid-19-pubseq-part2.html @@ -3,7 +3,7 @@ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"> <head> -<!-- 2020-08-26 Wed 05:01 --> +<!-- 2020-11-10 Tue 05:08 --> <meta http-equiv="Content-Type" content="text/html;charset=utf-8" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <title>COVID-19 PubSeq - Arvados</title> @@ -40,7 +40,7 @@ } pre.src { position: relative; - overflow: visible; + overflow: auto; padding-top: 1.2em; } pre.src:before { @@ -195,50 +195,26 @@ </style> <link rel="Blog stylesheet" type="text/css" href="blog.css" /> <script type="text/javascript"> -/* -@licstart The following is the entire license notice for the -JavaScript code in this tag. - -Copyright (C) 2012-2020 Free Software Foundation, Inc. - -The JavaScript code in this tag is free software: you can -redistribute it and/or modify it under the terms of the GNU -General Public License (GNU GPL) as published by the Free Software -Foundation, either version 3 of the License, or (at your option) -any later version. The code is distributed WITHOUT ANY WARRANTY; -without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU GPL for more details. - -As additional permission under GNU GPL version 3 section 7, you -may distribute non-source (e.g., minimized or compacted) forms of -that code without the copy of the GNU GPL normally required by -section 4, provided you include this license notice and a URL -through which recipients can access the Corresponding Source. - - -@licend The above is the entire license notice -for the JavaScript code in this tag. -*/ +// @license magnet:?xt=urn:btih:e95b018ef3580986a04669f1b5879592219e2a7a&dn=public-domain.txt Public Domain <!--/*--><![CDATA[/*><!--*/ - function CodeHighlightOn(elem, id) - { - var target = document.getElementById(id); - if(null != target) { - elem.cacheClassElem = elem.className; - elem.cacheClassTarget = target.className; - target.className = "code-highlighted"; - elem.className = "code-highlighted"; - } - } - function CodeHighlightOff(elem, id) - { - var target = document.getElementById(id); - if(elem.cacheClassElem) - elem.className = elem.cacheClassElem; - if(elem.cacheClassTarget) - target.className = elem.cacheClassTarget; - } -/*]]>*///--> + function CodeHighlightOn(elem, id) + { + var target = document.getElementById(id); + if(null != target) { + elem.classList.add("code-highlighted"); + target.classList.add("code-highlighted"); + } + } + function CodeHighlightOff(elem, id) + { + var target = document.getElementById(id); + if(null != target) { + elem.classList.remove("code-highlighted"); + target.classList.remove("code-highlighted"); + } + } + /*]]>*///--> +// @license-end </script> </head> <body> @@ -252,18 +228,18 @@ for the JavaScript code in this tag. <h2>Table of Contents</h2> <div id="text-table-of-contents"> <ul> -<li><a href="#org6501d83">1. The Arvados Web Server</a></li> -<li><a href="#orgcb7854f">2. The Arvados file interface</a></li> -<li><a href="#orgc8c3ccd">3. The PubSeq Arvados shell</a></li> -<li><a href="#org028c1b4">4. Wiring up CWL</a></li> -<li><a href="#org7cdc8cc">5. Using the Arvados API</a></li> -<li><a href="#org5961211">6. Troubleshooting</a></li> +<li><a href="#org10ef830">1. The Arvados Web Server</a></li> +<li><a href="#orgb6a7a42">2. The Arvados file interface</a></li> +<li><a href="#org0c7b94e">3. The PubSeq Arvados shell</a></li> +<li><a href="#org756005d">4. Wiring up CWL</a></li> +<li><a href="#orgf30b46f">5. Using the Arvados API</a></li> +<li><a href="#org3af3122">6. Troubleshooting</a></li> </ul> </div> </div> -<div id="outline-container-org6501d83" class="outline-2"> -<h2 id="org6501d83"><span class="section-number-2">1</span> The Arvados Web Server</h2> +<div id="outline-container-org10ef830" class="outline-2"> +<h2 id="org10ef830"><span class="section-number-2">1</span> The Arvados Web Server</h2> <div class="outline-text-2" id="text-1"> <p> We are using Arvados to run common workflow language (CWL) pipelines. @@ -283,8 +259,8 @@ workflows and the output of analysis pipelines (here CWL workflows). </div> -<div id="outline-container-orgcb7854f" class="outline-2"> -<h2 id="orgcb7854f"><span class="section-number-2">2</span> The Arvados file interface</h2> +<div id="outline-container-orgb6a7a42" class="outline-2"> +<h2 id="orgb6a7a42"><span class="section-number-2">2</span> The Arvados file interface</h2> <div class="outline-text-2" id="text-2"> <p> Arvados has the web server, but it also has a REST API and associated @@ -361,8 +337,8 @@ arv-get 2be6af7b4741f2a5c5f8ff2bc6152d73+1955623+Ab9ad65d7fe958a053b3a57d545839d </div> </div> -<div id="outline-container-orgc8c3ccd" class="outline-2"> -<h2 id="orgc8c3ccd"><span class="section-number-2">3</span> The PubSeq Arvados shell</h2> +<div id="outline-container-org0c7b94e" class="outline-2"> +<h2 id="org0c7b94e"><span class="section-number-2">3</span> The PubSeq Arvados shell</h2> <div class="outline-text-2" id="text-3"> <p> When you login to Arvados (you can request permission from us) it is @@ -414,11 +390,34 @@ the git repo and starts a new run calling into /data/pubseq/bh20-seq-resource/venv3/bin/bh20-seq-analyzer which is essentially <a href="https://github.com/arvados/bh20-seq-resource/blob/2baa88b766ec540bd34b96599014dd16e393af39/bh20seqanalyzer/main.py#L354">monitoring</a> for uploads. </p> + +<p> +On <code>run --help</code> +</p> + +<pre class="example" id="org93c3a8a"> +optional arguments: + -h, --help show this help message and exit + --uploader-project UPLOADER_PROJECT + --pangenome-analysis-project PANGENOME_ANALYSIS_PROJECT + --fastq-project FASTQ_PROJECT + --validated-project VALIDATED_PROJECT + --workflow-def-project WORKFLOW_DEF_PROJECT + --pangenome-workflow-uuid PANGENOME_WORKFLOW_UUID + --fastq-workflow-uuid FASTQ_WORKFLOW_UUID + --exclude-list EXCLUDE_LIST + --latest-result-collection LATEST_RESULT_COLLECTION + --kickoff + --no-start-analysis + --once + --print-status PRINT_STATUS + --revalidate +</pre> </div> </div> -<div id="outline-container-org028c1b4" class="outline-2"> -<h2 id="org028c1b4"><span class="section-number-2">4</span> Wiring up CWL</h2> +<div id="outline-container-org756005d" class="outline-2"> +<h2 id="org756005d"><span class="section-number-2">4</span> Wiring up CWL</h2> <div class="outline-text-2" id="text-4"> <p> In above script <code>bh20-seq-analyzer</code> you can see that the <a href="https://www.commonwl.org/">Common @@ -459,8 +458,8 @@ For more see <a href="https://hpc.guix.info/blog/2019/01/creating-a-reproducible </div> </div> -<div id="outline-container-org7cdc8cc" class="outline-2"> -<h2 id="org7cdc8cc"><span class="section-number-2">5</span> Using the Arvados API</h2> +<div id="outline-container-orgf30b46f" class="outline-2"> +<h2 id="orgf30b46f"><span class="section-number-2">5</span> Using the Arvados API</h2> <div class="outline-text-2" id="text-5"> <p> Arvados provides a rich API for accessing internals of the Cloud @@ -476,8 +475,8 @@ get a list of <a href="https://github.com/arvados/bh20-seq-resource/blob/2baa88b </div> </div> -<div id="outline-container-org5961211" class="outline-2"> -<h2 id="org5961211"><span class="section-number-2">6</span> Troubleshooting</h2> +<div id="outline-container-org3af3122" class="outline-2"> +<h2 id="org3af3122"><span class="section-number-2">6</span> Troubleshooting</h2> <div class="outline-text-2" id="text-6"> <p> When workflows have errors we should check the logs in Arvados. @@ -494,7 +493,7 @@ see what parts failed. </div> </div> <div id="postamble" class="status"> -<hr><small>Created by <a href="http://thebird.nl/">Pjotr Prins</a> (pjotr.public768 at thebird 'dot' nl) using Emacs org-mode and a healthy dose of Lisp!<br />Modified 2020-08-26 Wed 05:01</small>. +<hr><small>Created by <a href="http://thebird.nl/">Pjotr Prins</a> (pjotr.public768 at thebird 'dot' nl) using Emacs org-mode and a healthy dose of Lisp!<br />Modified 2020-11-09 Mon 01:20</small>. </div> </body> </html> diff --git a/doc/blog/using-covid-19-pubseq-part2.org b/doc/blog/using-covid-19-pubseq-part2.org index 4b827f5..d7816ba 100644 --- a/doc/blog/using-covid-19-pubseq-part2.org +++ b/doc/blog/using-covid-19-pubseq-part2.org @@ -96,6 +96,27 @@ the git repo and starts a new run calling into /data/pubseq/bh20-seq-resource/venv3/bin/bh20-seq-analyzer which is essentially [[https://github.com/arvados/bh20-seq-resource/blob/2baa88b766ec540bd34b96599014dd16e393af39/bh20seqanalyzer/main.py#L354][monitoring]] for uploads. +On ~run --help~ + +#+begin_example +optional arguments: + -h, --help show this help message and exit + --uploader-project UPLOADER_PROJECT + --pangenome-analysis-project PANGENOME_ANALYSIS_PROJECT + --fastq-project FASTQ_PROJECT + --validated-project VALIDATED_PROJECT + --workflow-def-project WORKFLOW_DEF_PROJECT + --pangenome-workflow-uuid PANGENOME_WORKFLOW_UUID + --fastq-workflow-uuid FASTQ_WORKFLOW_UUID + --exclude-list EXCLUDE_LIST + --latest-result-collection LATEST_RESULT_COLLECTION + --kickoff + --no-start-analysis + --once + --print-status PRINT_STATUS + --revalidate +#+end_example + * Wiring up CWL In above script ~bh20-seq-analyzer~ you can see that the [[https://www.commonwl.org/][Common diff --git a/scripts/update_virtuoso/check_for_updates.py b/scripts/update_virtuoso/check_for_updates.py index 8761c8a..939a575 100755 --- a/scripts/update_virtuoso/check_for_updates.py +++ b/scripts/update_virtuoso/check_for_updates.py @@ -4,7 +4,7 @@ # # You can run this in a Guix container with # -# ~/opt/guix/bin/guix environment -C guix --ad-hoc python python-requests curl --network -- python3 ./scripts/update_virtuoso/check_for_updates.py cache.txt dba dba +# ~/opt/guix/bin/guix environment -C guix --ad-hoc raptor2 python python-requests curl --network -- python3 ./scripts/update_virtuoso/check_for_updates.py cache.txt dba dba # # Note you'll need to run from the root dir. Remove the ./cache.txt file if you want to force an update. # @@ -19,6 +19,11 @@ fn = sys.argv[1] user = sys.argv[2] pwd = sys.argv[3] +no_cache = False +if fn == "--no-cache": + no_cache = True + print("Skipping cache check and download of metadata.ttl") + scriptdir = os.path.dirname(os.path.realpath(__file__)) print(scriptdir) basedir = os.path.dirname(os.path.dirname(scriptdir)) @@ -29,6 +34,15 @@ def upload(fn): # print("DELETE "+fn) # cmd = ("curl --digest --user dba:%s --verbose --url -G http://sparql.genenetwork.org/sparql-graph-crud-auth --data-urlencode graph=http://covid-19.genenetwork.org/graph -X DELETE" % pwd).split(" ") + print("VALIDATE "+fn) + cmd = f"rapper -i turtle {fn}" + print(cmd) + p = subprocess.Popen(cmd.split(" "),stdout=subprocess.PIPE,stderr=subprocess.PIPE) + out, err = p.communicate() + if p.returncode != 0: + print(out,err) + assert(p.returncode == 0) + print("UPLOADING "+fn) cmd = ("curl -X PUT --digest -u dba:%s -H Content-Type:text/turtle -T %s -G http://sparql.genenetwork.org/sparql-graph-crud-auth --data-urlencode graph=http://covid-19.genenetwork.org/graph/%s" % (pwd, fn, os.path.basename(fn)) ) print(cmd) @@ -39,24 +53,26 @@ def upload(fn): url = 'https://download.lugli.arvadosapi.com/c=lugli-4zz18-z513nlpqm03hpca/_/mergedmetadata.ttl' # --- Fetch headers from TTL file on Arvados +# curl --head https://download.lugli.arvadosapi.com/c=lugli-4zz18-z513nlpqm03hpca/_/mergedmetadata.ttl r = requests.head(url) print(r.headers) -print(r.headers['Last-Modified']) +if not no_cache: + print(r.headers['Last-Modified']) -# --- Convert/validate time stamp -# ValueError: time data 'Tue, 21 Apr 2020 23:47:43 GMT' does not match format '%a %b %d %H:%M:%S %Y' -last_modified_str = r.headers['Last-Modified'] -t_stamp = time.strptime(last_modified_str,"%a, %d %b %Y %H:%M:%S %Z" ) -print(t_stamp) + # --- Convert/validate time stamp + # ValueError: time data 'Tue, 21 Apr 2020 23:47:43 GMT' does not match format '%a %b %d %H:%M:%S %Y' + last_modified_str = r.headers['Last-Modified'] + t_stamp = time.strptime(last_modified_str,"%a, %d %b %Y %H:%M:%S %Z" ) + print(t_stamp) -# OK, it works, now check last stored value +# OK, it works, now check last stored value in the cache stamp = None if os.path.isfile(fn): file = open(fn,"r") stamp = file.read() file.close -if stamp != last_modified_str: +if no_cache or stamp != last_modified_str: print("Delete graphs") for graph in ["labels.ttl", "metadata.ttl", "countries.ttl"]: cmd = ("curl --digest -u dba:%s --verbose --url http://127.0.0.1:8890/sparql-graph-crud-auth?graph=http://covid-19.genenetwork.org/graph/%s -X DELETE" % (pwd, graph)) @@ -69,14 +85,16 @@ if stamp != last_modified_str: upload(basedir+"/semantic_enrichment/labels.ttl") upload(basedir+"/semantic_enrichment/countries.ttl") - print("Fetch metadata TTL") - r = requests.get(url) - assert(r.status_code == 200) - with open("metadata.ttl", "w") as f: - f.write(r.text) - f.close + if not no_cache: + print("Fetch metadata TTL") + r = requests.get(url) + assert(r.status_code == 200) + with open("metadata.ttl", "w") as f: + f.write(r.text) + f.close upload("metadata.ttl") - with open(fn,"w") as f: - f.write(last_modified_str) + if not no_cache: + with open(fn,"w") as f: + f.write(last_modified_str) else: print("Metadata is up to date") @@ -17,7 +17,7 @@ except ImportError: install_requires = ["arvados-python-client", "schema-salad", "python-magic", "pyshex", "pyshexc==0.7.0", "py-dateutil"] -web_requires = ["flask", "pyyaml"] +web_requires = ["flask", "pyyaml", "redis"] needs_pytest = {"pytest", "test", "ptr"}.intersection(sys.argv) pytest_runner = ["pytest < 6", "pytest-runner < 5"] if needs_pytest else [] diff --git a/test/rest-api.org b/test/rest-api.org index 945631a..66639c3 100644 --- a/test/rest-api.org +++ b/test/rest-api.org @@ -169,4 +169,4 @@ To skip confirmations you may also want to set : (setq org-confirm-babel-evaluate nil) -To see output of the inpreter open then *Python* buffer. +To see output of the interpreter open then *Python* buffer. |