aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.gitignore2
-rwxr-xr-xscripts/update_virtuoso/check_for_updates.py59
2 files changed, 61 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
index 8e8c01b..925698c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
*.py~
build/
+cache.txt
+metadata.ttl
diff --git a/scripts/update_virtuoso/check_for_updates.py b/scripts/update_virtuoso/check_for_updates.py
new file mode 100755
index 0000000..f3b8a86
--- /dev/null
+++ b/scripts/update_virtuoso/check_for_updates.py
@@ -0,0 +1,59 @@
+#! /usr/bin/env python3
+#
+# Check for updates on Arvados, pull the TTL and
+# push into Virtuoso
+#
+# You can run this in a Guix container with
+#
+# ~/opt/guix/bin/guix environment -C guix --ad-hoc python python-requests curl --network -- python3 ./scripts/update_virtuoso/check_for_updates.py cache.txt dba dba
+
+import requests
+import time
+
+url = 'https://download.lugli.arvadosapi.com/c=lugli-4zz18-z513nlpqm03hpca/_/mergedmetadata.ttl'
+# --- Fetch headers from TTL file on Arvados
+r = requests.head(url)
+print(r.headers)
+
+print(r.headers['Last-Modified'])
+
+# --- Convert/validate time stamp
+# ValueError: time data 'Tue, 21 Apr 2020 23:47:43 GMT' does not match format '%a %b %d %H:%M:%S %Y'
+last_modified_str = r.headers['Last-Modified']
+t_stamp = time.strptime(last_modified_str,"%a, %d %b %Y %H:%M:%S %Z" )
+print(t_stamp)
+
+# OK, it works, now check last stored value
+import sys
+assert(len(sys.argv)==4)
+fn = sys.argv[1]
+user = sys.argv[2]
+pwd = sys.argv[3]
+
+import os.path
+stamp = None
+if os.path.isfile(fn):
+ file = open(fn,"r")
+ stamp = file.read()
+ file.close
+
+import subprocess
+if stamp != last_modified_str:
+ print("Fetch metadata TTL")
+ r = requests.get(url)
+ assert(r.status_code == 200)
+ with open("metadata.ttl", "w") as f:
+ f.write(r.text)
+ f.close
+ # Now push into Virtuoso using CURL
+ # cmd = "curl -X PUT --digest -u dba:dba -H Content-Type:text/turtle -T metadata.ttl -G http://localhost:8890/sparql-graph-crud-auth --data-urlencode graph=http://covid-19.genenetwork.org/graph".split(" ")
+ print("Push metadata TTL")
+ cmd = ("curl -X PUT --digest -u dba:%s -H Content-Type:text/turtle -T metadata.ttl -G http://sparql.genenetwork.org/sparql-graph-crud-auth --data-urlencode graph=http://covid-19.genenetwork.org/graph" % pwd ).split(" ")
+ print(cmd)
+ p = subprocess.Popen(cmd)
+ output = p.communicate()[0]
+ print(output)
+ assert(p.returncode == 0)
+
+ with open(fn,"w") as f:
+ f.write(last_modified_str)