Add script that updates Virtuoso - run as a CRON job

2 files changed, 61 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
index 8e8c01b..925698c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
 *.py~
 build/
+cache.txt
+metadata.ttl
diff --git a/scripts/update_virtuoso/check_for_updates.py b/scripts/update_virtuoso/check_for_updates.py
new file mode 100755
index 0000000..f3b8a86
--- /dev/null
+++ b/scripts/update_virtuoso/check_for_updates.py
@@ -0,0 +1,59 @@
+#! /usr/bin/env python3
+#
+# Check for updates on Arvados, pull the TTL and
+# push into Virtuoso
+#
+# You can run this in a Guix container with
+#
+#  ~/opt/guix/bin/guix environment -C guix --ad-hoc python python-requests curl --network -- python3 ./scripts/update_virtuoso/check_for_updates.py cache.txt dba dba
+
+import requests
+import time
+
+url = 'https://download.lugli.arvadosapi.com/c=lugli-4zz18-z513nlpqm03hpca/_/mergedmetadata.ttl'
+# --- Fetch headers from TTL file on Arvados
+r = requests.head(url)
+print(r.headers)
+
+print(r.headers['Last-Modified'])
+
+# --- Convert/validate time stamp
+# ValueError: time data 'Tue, 21 Apr 2020 23:47:43 GMT' does not match format '%a %b %d %H:%M:%S %Y'
+last_modified_str = r.headers['Last-Modified']
+t_stamp = time.strptime(last_modified_str,"%a, %d %b %Y %H:%M:%S %Z" )
+print(t_stamp)
+
+# OK, it works, now check last stored value
+import sys
+assert(len(sys.argv)==4)
+fn = sys.argv[1]
+user = sys.argv[2]
+pwd = sys.argv[3]
+
+import os.path
+stamp = None
+if os.path.isfile(fn):
+    file = open(fn,"r")
+    stamp = file.read()
+    file.close
+
+import subprocess
+if stamp != last_modified_str:
+    print("Fetch metadata TTL")
+    r = requests.get(url)
+    assert(r.status_code == 200)
+    with open("metadata.ttl", "w") as f:
+        f.write(r.text)
+        f.close
+    # Now push into Virtuoso using CURL
+    # cmd = "curl -X PUT --digest -u dba:dba -H Content-Type:text/turtle -T metadata.ttl -G http://localhost:8890/sparql-graph-crud-auth --data-urlencode graph=http://covid-19.genenetwork.org/graph".split(" ")
+    print("Push metadata TTL")
+    cmd = ("curl -X PUT --digest -u dba:%s -H Content-Type:text/turtle -T metadata.ttl -G http://sparql.genenetwork.org/sparql-graph-crud-auth --data-urlencode graph=http://covid-19.genenetwork.org/graph" % pwd ).split(" ")
+    print(cmd)
+    p = subprocess.Popen(cmd)
+    output = p.communicate()[0]
+    print(output)
+    assert(p.returncode == 0)
+
+    with open(fn,"w") as f:
+        f.write(last_modified_str)