about summary refs log tree commit diff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--README.md2
-rw-r--r--bh20simplewebuploader/main.py16
-rw-r--r--bh20simplewebuploader/static/main.css27
-rw-r--r--bh20simplewebuploader/static/main.js6
-rw-r--r--bh20simplewebuploader/templates/home.html27
-rw-r--r--doc/INSTALL.md6
-rw-r--r--doc/blog/using-covid-19-pubseq-part2.html127
-rw-r--r--doc/blog/using-covid-19-pubseq-part2.org21
-rwxr-xr-xscripts/update_virtuoso/check_for_updates.py52
-rw-r--r--setup.py2
-rw-r--r--test/rest-api.org2
11 files changed, 198 insertions, 90 deletions
diff --git a/README.md b/README.md
index 03e4297..3815bf4 100644
--- a/README.md
+++ b/README.md
@@ -40,7 +40,7 @@ Note that you will need to repeat the `. venv/bin/activate` step from this direc
 Install from PyPi:
 
 ```sh
-pip3 bh20-seq-uploader
+pip3 install bh20-seq-uploader
 ```
 
 Install from git:
diff --git a/bh20simplewebuploader/main.py b/bh20simplewebuploader/main.py
index bcdf8d8..1d7154c 100644
--- a/bh20simplewebuploader/main.py
+++ b/bh20simplewebuploader/main.py
@@ -7,6 +7,7 @@ import logging
 import os
 import sys
 import re
+import redis
 import string
 import ruamel.yaml as yaml
 import pkg_resources
@@ -252,8 +253,19 @@ def send_home():
     """
     Send the front page.
     """
-
-    return render_template('home.html', menu='HOME', load_map=True)
+    redis_client = redis.Redis(host=os.environ.get('HOST', 'localhost'),
+                               port=os.environ.get('PORT', 6379),
+                               db=os.environ.get('REDIS_DB', 0))
+    tweets = []
+    for tweet_id in redis_client.zrevrange('bh20-tweet-score:',
+                                           0, -1):
+        tweets.append(
+            {k.decode("utf-8"): v.decode("utf-8") for k, v in
+             redis_client.hgetall(tweet_id).items()}
+        )
+    return render_template('home.html', menu='HOME',
+                           tweets=tweets,
+                           load_map=True)
 
 
 @app.route('/upload')
diff --git a/bh20simplewebuploader/static/main.css b/bh20simplewebuploader/static/main.css
index bc4f705..76a1755 100644
--- a/bh20simplewebuploader/static/main.css
+++ b/bh20simplewebuploader/static/main.css
@@ -506,3 +506,30 @@ div.status {
     font-size: 16px;
     box-shadow: 0 8px 16px 0 rgba(0,0,0,0.2), 0 6px 20px 0 rgba(0,0,0,0.19);
 }
+
+.flex-container {
+    display: flex;
+    flex-direction: row;
+    flex-wrap: wrap;
+    justify-content: space-evenly;
+}
+
+#twitter-feed {
+    background-color: #f5f8fa;
+    max-height: 440px;
+    max-width: 500px;
+    overflow-y: auto;
+    padding: 20px;
+    margin-left: auto;
+    margin-right: auto;
+}
+
+
+#twitter-feed ul {
+    list-style-type: none;
+}
+
+#twitter-feed ul li {
+    border-bottom: 2px solid white;
+    padding-bottom: 20px;
+}
diff --git a/bh20simplewebuploader/static/main.js b/bh20simplewebuploader/static/main.js
index 89bc603..6cc0d9f 100644
--- a/bh20simplewebuploader/static/main.js
+++ b/bh20simplewebuploader/static/main.js
@@ -68,6 +68,12 @@ function fetchHTMLTable(apiEndPoint) {
             htmlString="<table>"
             for (var i=0; i<data.length;i++) {
                 let url = data[i]["key"];
+                continents = ["Q538", "Q48", "Q49", "Q18", "Q15", "Q27611" ];
+                node = url.split("/").pop();
+                console.log(continents.includes(node));
+                if (url.includes("wikidata") && !continents.includes(node)) {
+                    url = "http://covid19.genenetwork.org/location?label="+url ;
+                }
                 let label = data[i]["label"];
                 htmlString=htmlString+"<tr><td><a href=\""+url+"\">"+label+"</a></td><td>"+data[i]["count"]+"<td></tr>"
             }
diff --git a/bh20simplewebuploader/templates/home.html b/bh20simplewebuploader/templates/home.html
index 5986d72..be948f6 100644
--- a/bh20simplewebuploader/templates/home.html
+++ b/bh20simplewebuploader/templates/home.html
@@ -22,9 +22,27 @@
                     or <a href="/apidoc">REST API</a>. For more
                     information see the <a href="/about">FAQ!</a>.
                   </p>
-      <section id="map_view" class="map">
-        <div id="mapid"></div>
-      </section>
+
+                  <section class="flex-container">
+                      <div id="map_view" class="map">
+                          <div id="mapid"></div>
+                      </div>
+                      <div id="twitter-feed">
+                          <ul>
+                              {% for tweet in tweets|sort(reverse=true, attribute="timeposted")%}
+                              <li>
+                                  <p class="tweet">
+                                      {{ tweet.tweet|urlize(40, target="_blank") }} <br/>
+                                      by {{ tweet.author }}
+                                  </p>
+                                  <small class="timeposted">
+                                      {{ tweet.timeposted }}
+                                  </small>
+                              </li>
+                              {% endfor %}
+                          </ul>
+                      </div>
+                  </section>
 
                   <a href="https://projectredcap.org/"><img class="img-right" src="static/image/REDCap.png" /></a>
                   <p>
@@ -76,7 +94,8 @@
                     use, see the <a href="/blog">docs</a>.
                   </p>
                 </div>
-        </section>
+              </section>
+
 
       {% include 'footer.html' %}
 
diff --git a/doc/INSTALL.md b/doc/INSTALL.md
index 367b452..0367c63 100644
--- a/doc/INSTALL.md
+++ b/doc/INSTALL.md
@@ -107,3 +107,9 @@ And run from the data dir
     guix environment --ad-hoc virtuoso-ose -- virtuoso-t -f
 
 Visit http://localhost:8890/sparql
+
+To update the turtle files do
+
+    guix environment -C guix --ad-hoc python python-requests raptor2 curl --network -- python3 ./scripts/update_virtuoso/check_for_updates.py cache.txt dba dba
+
+where dba is the default password.
diff --git a/doc/blog/using-covid-19-pubseq-part2.html b/doc/blog/using-covid-19-pubseq-part2.html
index 567980d..eff6fcd 100644
--- a/doc/blog/using-covid-19-pubseq-part2.html
+++ b/doc/blog/using-covid-19-pubseq-part2.html
@@ -3,7 +3,7 @@
 "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
 <head>
-<!-- 2020-08-26 Wed 05:01 -->
+<!-- 2020-11-10 Tue 05:08 -->
 <meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
 <meta name="viewport" content="width=device-width, initial-scale=1" />
 <title>COVID-19 PubSeq - Arvados</title>
@@ -40,7 +40,7 @@
   }
   pre.src {
     position: relative;
-    overflow: visible;
+    overflow: auto;
     padding-top: 1.2em;
   }
   pre.src:before {
@@ -195,50 +195,26 @@
 </style>
 <link rel="Blog stylesheet" type="text/css" href="blog.css" />
 <script type="text/javascript">
-/*
-@licstart  The following is the entire license notice for the
-JavaScript code in this tag.
-
-Copyright (C) 2012-2020 Free Software Foundation, Inc.
-
-The JavaScript code in this tag is free software: you can
-redistribute it and/or modify it under the terms of the GNU
-General Public License (GNU GPL) as published by the Free Software
-Foundation, either version 3 of the License, or (at your option)
-any later version.  The code is distributed WITHOUT ANY WARRANTY;
-without even the implied warranty of MERCHANTABILITY or FITNESS
-FOR A PARTICULAR PURPOSE.  See the GNU GPL for more details.
-
-As additional permission under GNU GPL version 3 section 7, you
-may distribute non-source (e.g., minimized or compacted) forms of
-that code without the copy of the GNU GPL normally required by
-section 4, provided you include this license notice and a URL
-through which recipients can access the Corresponding Source.
-
-
-@licend  The above is the entire license notice
-for the JavaScript code in this tag.
-*/
+// @license magnet:?xt=urn:btih:e95b018ef3580986a04669f1b5879592219e2a7a&dn=public-domain.txt Public Domain
 <!--/*--><![CDATA[/*><!--*/
- function CodeHighlightOn(elem, id)
- {
-   var target = document.getElementById(id);
-   if(null != target) {
-     elem.cacheClassElem = elem.className;
-     elem.cacheClassTarget = target.className;
-     target.className = "code-highlighted";
-     elem.className   = "code-highlighted";
-   }
- }
- function CodeHighlightOff(elem, id)
- {
-   var target = document.getElementById(id);
-   if(elem.cacheClassElem)
-     elem.className = elem.cacheClassElem;
-   if(elem.cacheClassTarget)
-     target.className = elem.cacheClassTarget;
- }
-/*]]>*///-->
+     function CodeHighlightOn(elem, id)
+     {
+       var target = document.getElementById(id);
+       if(null != target) {
+         elem.classList.add("code-highlighted");
+         target.classList.add("code-highlighted");
+       }
+     }
+     function CodeHighlightOff(elem, id)
+     {
+       var target = document.getElementById(id);
+       if(null != target) {
+         elem.classList.remove("code-highlighted");
+         target.classList.remove("code-highlighted");
+       }
+     }
+    /*]]>*///-->
+// @license-end
 </script>
 </head>
 <body>
@@ -252,18 +228,18 @@ for the JavaScript code in this tag.
 <h2>Table of Contents</h2>
 <div id="text-table-of-contents">
 <ul>
-<li><a href="#org6501d83">1. The Arvados Web Server</a></li>
-<li><a href="#orgcb7854f">2. The Arvados file interface</a></li>
-<li><a href="#orgc8c3ccd">3. The PubSeq Arvados shell</a></li>
-<li><a href="#org028c1b4">4. Wiring up CWL</a></li>
-<li><a href="#org7cdc8cc">5. Using the Arvados API</a></li>
-<li><a href="#org5961211">6. Troubleshooting</a></li>
+<li><a href="#org10ef830">1. The Arvados Web Server</a></li>
+<li><a href="#orgb6a7a42">2. The Arvados file interface</a></li>
+<li><a href="#org0c7b94e">3. The PubSeq Arvados shell</a></li>
+<li><a href="#org756005d">4. Wiring up CWL</a></li>
+<li><a href="#orgf30b46f">5. Using the Arvados API</a></li>
+<li><a href="#org3af3122">6. Troubleshooting</a></li>
 </ul>
 </div>
 </div>
 
-<div id="outline-container-org6501d83" class="outline-2">
-<h2 id="org6501d83"><span class="section-number-2">1</span> The Arvados Web Server</h2>
+<div id="outline-container-org10ef830" class="outline-2">
+<h2 id="org10ef830"><span class="section-number-2">1</span> The Arvados Web Server</h2>
 <div class="outline-text-2" id="text-1">
 <p>
 We are using Arvados to run common workflow language (CWL) pipelines.
@@ -283,8 +259,8 @@ workflows and the output of analysis pipelines (here CWL workflows).
 </div>
 
 
-<div id="outline-container-orgcb7854f" class="outline-2">
-<h2 id="orgcb7854f"><span class="section-number-2">2</span> The Arvados file interface</h2>
+<div id="outline-container-orgb6a7a42" class="outline-2">
+<h2 id="orgb6a7a42"><span class="section-number-2">2</span> The Arvados file interface</h2>
 <div class="outline-text-2" id="text-2">
 <p>
 Arvados has the web server, but it also has a REST API and associated
@@ -361,8 +337,8 @@ arv-get 2be6af7b4741f2a5c5f8ff2bc6152d73+1955623+Ab9ad65d7fe958a053b3a57d545839d
 </div>
 </div>
 
-<div id="outline-container-orgc8c3ccd" class="outline-2">
-<h2 id="orgc8c3ccd"><span class="section-number-2">3</span> The PubSeq Arvados shell</h2>
+<div id="outline-container-org0c7b94e" class="outline-2">
+<h2 id="org0c7b94e"><span class="section-number-2">3</span> The PubSeq Arvados shell</h2>
 <div class="outline-text-2" id="text-3">
 <p>
 When you login to Arvados (you can request permission from us) it is
@@ -414,11 +390,34 @@ the git repo and starts a new run calling into
 /data/pubseq/bh20-seq-resource/venv3/bin/bh20-seq-analyzer which is
 essentially <a href="https://github.com/arvados/bh20-seq-resource/blob/2baa88b766ec540bd34b96599014dd16e393af39/bh20seqanalyzer/main.py#L354">monitoring</a> for uploads.
 </p>
+
+<p>
+On <code>run --help</code>
+</p>
+
+<pre class="example" id="org93c3a8a">
+optional arguments:
+  -h, --help            show this help message and exit
+  --uploader-project UPLOADER_PROJECT
+  --pangenome-analysis-project PANGENOME_ANALYSIS_PROJECT
+  --fastq-project FASTQ_PROJECT
+  --validated-project VALIDATED_PROJECT
+  --workflow-def-project WORKFLOW_DEF_PROJECT
+  --pangenome-workflow-uuid PANGENOME_WORKFLOW_UUID
+  --fastq-workflow-uuid FASTQ_WORKFLOW_UUID
+  --exclude-list EXCLUDE_LIST
+  --latest-result-collection LATEST_RESULT_COLLECTION
+  --kickoff
+  --no-start-analysis
+  --once
+  --print-status PRINT_STATUS
+  --revalidate
+</pre>
 </div>
 </div>
 
-<div id="outline-container-org028c1b4" class="outline-2">
-<h2 id="org028c1b4"><span class="section-number-2">4</span> Wiring up CWL</h2>
+<div id="outline-container-org756005d" class="outline-2">
+<h2 id="org756005d"><span class="section-number-2">4</span> Wiring up CWL</h2>
 <div class="outline-text-2" id="text-4">
 <p>
 In above script <code>bh20-seq-analyzer</code> you can see that the <a href="https://www.commonwl.org/">Common
@@ -459,8 +458,8 @@ For more see <a href="https://hpc.guix.info/blog/2019/01/creating-a-reproducible
 </div>
 </div>
 
-<div id="outline-container-org7cdc8cc" class="outline-2">
-<h2 id="org7cdc8cc"><span class="section-number-2">5</span> Using the Arvados API</h2>
+<div id="outline-container-orgf30b46f" class="outline-2">
+<h2 id="orgf30b46f"><span class="section-number-2">5</span> Using the Arvados API</h2>
 <div class="outline-text-2" id="text-5">
 <p>
 Arvados provides a rich API for accessing internals of the Cloud
@@ -476,8 +475,8 @@ get a list of <a href="https://github.com/arvados/bh20-seq-resource/blob/2baa88b
 </div>
 </div>
 
-<div id="outline-container-org5961211" class="outline-2">
-<h2 id="org5961211"><span class="section-number-2">6</span> Troubleshooting</h2>
+<div id="outline-container-org3af3122" class="outline-2">
+<h2 id="org3af3122"><span class="section-number-2">6</span> Troubleshooting</h2>
 <div class="outline-text-2" id="text-6">
 <p>
 When workflows have errors we should check the logs in Arvados.
@@ -494,7 +493,7 @@ see what parts failed.
 </div>
 </div>
 <div id="postamble" class="status">
-<hr><small>Created by <a href="http://thebird.nl/">Pjotr Prins</a> (pjotr.public768 at thebird 'dot' nl) using Emacs org-mode and a healthy dose of Lisp!<br />Modified 2020-08-26 Wed 05:01</small>.
+<hr><small>Created by <a href="http://thebird.nl/">Pjotr Prins</a> (pjotr.public768 at thebird 'dot' nl) using Emacs org-mode and a healthy dose of Lisp!<br />Modified 2020-11-09 Mon 01:20</small>.
 </div>
 </body>
 </html>
diff --git a/doc/blog/using-covid-19-pubseq-part2.org b/doc/blog/using-covid-19-pubseq-part2.org
index 4b827f5..d7816ba 100644
--- a/doc/blog/using-covid-19-pubseq-part2.org
+++ b/doc/blog/using-covid-19-pubseq-part2.org
@@ -96,6 +96,27 @@ the git repo and starts a new run calling into
 /data/pubseq/bh20-seq-resource/venv3/bin/bh20-seq-analyzer which is
 essentially [[https://github.com/arvados/bh20-seq-resource/blob/2baa88b766ec540bd34b96599014dd16e393af39/bh20seqanalyzer/main.py#L354][monitoring]] for uploads.
 
+On ~run --help~
+
+#+begin_example
+optional arguments:
+  -h, --help            show this help message and exit
+  --uploader-project UPLOADER_PROJECT
+  --pangenome-analysis-project PANGENOME_ANALYSIS_PROJECT
+  --fastq-project FASTQ_PROJECT
+  --validated-project VALIDATED_PROJECT
+  --workflow-def-project WORKFLOW_DEF_PROJECT
+  --pangenome-workflow-uuid PANGENOME_WORKFLOW_UUID
+  --fastq-workflow-uuid FASTQ_WORKFLOW_UUID
+  --exclude-list EXCLUDE_LIST
+  --latest-result-collection LATEST_RESULT_COLLECTION
+  --kickoff
+  --no-start-analysis
+  --once
+  --print-status PRINT_STATUS
+  --revalidate
+#+end_example
+
 * Wiring up CWL
 
 In above script ~bh20-seq-analyzer~ you can see that the [[https://www.commonwl.org/][Common
diff --git a/scripts/update_virtuoso/check_for_updates.py b/scripts/update_virtuoso/check_for_updates.py
index 8761c8a..939a575 100755
--- a/scripts/update_virtuoso/check_for_updates.py
+++ b/scripts/update_virtuoso/check_for_updates.py
@@ -4,7 +4,7 @@
 #
 # You can run this in a Guix container with
 #
-#    ~/opt/guix/bin/guix environment -C guix --ad-hoc python python-requests curl --network -- python3 ./scripts/update_virtuoso/check_for_updates.py cache.txt dba dba
+#    ~/opt/guix/bin/guix environment -C guix --ad-hoc raptor2 python python-requests curl --network -- python3 ./scripts/update_virtuoso/check_for_updates.py cache.txt dba dba
 #
 # Note you'll need to run from the root dir. Remove the ./cache.txt file if you want to force an update.
 #
@@ -19,6 +19,11 @@ fn = sys.argv[1]
 user = sys.argv[2]
 pwd = sys.argv[3]
 
+no_cache = False
+if fn == "--no-cache":
+  no_cache = True
+  print("Skipping cache check and download of metadata.ttl")
+
 scriptdir = os.path.dirname(os.path.realpath(__file__))
 print(scriptdir)
 basedir = os.path.dirname(os.path.dirname(scriptdir))
@@ -29,6 +34,15 @@ def upload(fn):
     # print("DELETE "+fn)
     # cmd = ("curl --digest --user dba:%s --verbose --url -G http://sparql.genenetwork.org/sparql-graph-crud-auth --data-urlencode graph=http://covid-19.genenetwork.org/graph -X DELETE" % pwd).split(" ")
 
+    print("VALIDATE "+fn)
+    cmd = f"rapper -i turtle {fn}"
+    print(cmd)
+    p = subprocess.Popen(cmd.split(" "),stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+    out, err = p.communicate()
+    if p.returncode != 0:
+      print(out,err)
+    assert(p.returncode == 0)
+
     print("UPLOADING "+fn)
     cmd = ("curl -X PUT --digest -u dba:%s -H Content-Type:text/turtle -T %s -G http://sparql.genenetwork.org/sparql-graph-crud-auth --data-urlencode graph=http://covid-19.genenetwork.org/graph/%s" % (pwd, fn, os.path.basename(fn)) )
     print(cmd)
@@ -39,24 +53,26 @@ def upload(fn):
 
 url = 'https://download.lugli.arvadosapi.com/c=lugli-4zz18-z513nlpqm03hpca/_/mergedmetadata.ttl'
 # --- Fetch headers from TTL file on Arvados
+#  curl --head https://download.lugli.arvadosapi.com/c=lugli-4zz18-z513nlpqm03hpca/_/mergedmetadata.ttl
 r = requests.head(url)
 print(r.headers)
-print(r.headers['Last-Modified'])
+if not no_cache:
+  print(r.headers['Last-Modified'])
 
-# --- Convert/validate time stamp
-# ValueError: time data 'Tue, 21 Apr 2020 23:47:43 GMT' does not match format '%a %b %d %H:%M:%S %Y'
-last_modified_str = r.headers['Last-Modified']
-t_stamp = time.strptime(last_modified_str,"%a, %d %b %Y %H:%M:%S %Z" )
-print(t_stamp)
+  # --- Convert/validate time stamp
+  # ValueError: time data 'Tue, 21 Apr 2020 23:47:43 GMT' does not match format '%a %b %d %H:%M:%S %Y'
+  last_modified_str = r.headers['Last-Modified']
+  t_stamp = time.strptime(last_modified_str,"%a, %d %b %Y %H:%M:%S %Z" )
+  print(t_stamp)
 
-# OK, it works, now check last stored value
+# OK, it works, now check last stored value in the cache
 stamp = None
 if os.path.isfile(fn):
     file = open(fn,"r")
     stamp = file.read()
     file.close
 
-if stamp != last_modified_str:
+if no_cache or stamp != last_modified_str:
     print("Delete graphs")
     for graph in ["labels.ttl", "metadata.ttl", "countries.ttl"]:
         cmd = ("curl --digest -u dba:%s --verbose --url http://127.0.0.1:8890/sparql-graph-crud-auth?graph=http://covid-19.genenetwork.org/graph/%s -X DELETE" % (pwd, graph))
@@ -69,14 +85,16 @@ if stamp != last_modified_str:
     upload(basedir+"/semantic_enrichment/labels.ttl")
     upload(basedir+"/semantic_enrichment/countries.ttl")
 
-    print("Fetch metadata TTL")
-    r = requests.get(url)
-    assert(r.status_code == 200)
-    with open("metadata.ttl", "w") as f:
-        f.write(r.text)
-        f.close
+    if not no_cache:
+        print("Fetch metadata TTL")
+        r = requests.get(url)
+        assert(r.status_code == 200)
+        with open("metadata.ttl", "w") as f:
+            f.write(r.text)
+            f.close
     upload("metadata.ttl")
-    with open(fn,"w") as f:
-        f.write(last_modified_str)
+    if not no_cache:
+        with open(fn,"w") as f:
+            f.write(last_modified_str)
 else:
     print("Metadata is up to date")
diff --git a/setup.py b/setup.py
index 1a2a8f6..ec15b59 100644
--- a/setup.py
+++ b/setup.py
@@ -17,7 +17,7 @@ except ImportError:
 
 install_requires = ["arvados-python-client", "schema-salad",
                     "python-magic", "pyshex", "pyshexc==0.7.0", "py-dateutil"]
-web_requires = ["flask", "pyyaml"]
+web_requires = ["flask", "pyyaml", "redis"]
 
 needs_pytest = {"pytest", "test", "ptr"}.intersection(sys.argv)
 pytest_runner = ["pytest < 6", "pytest-runner < 5"] if needs_pytest else []
diff --git a/test/rest-api.org b/test/rest-api.org
index 945631a..66639c3 100644
--- a/test/rest-api.org
+++ b/test/rest-api.org
@@ -169,4 +169,4 @@ To skip confirmations you may also want to set
 
 : (setq org-confirm-babel-evaluate nil)
 
-To see output of the inpreter open then *Python* buffer.
+To see output of the interpreter open then *Python* buffer.
generated by cgit-pink 1.4.1 (git 2.36.1) at 2025-08-21 07:29:03 +0000