From ebd9226e5a561840f6665a76ca50d63274d9d3e5 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 24 Apr 2020 13:55:19 -0500 Subject: Add script that updates Virtuoso - run as a CRON job --- scripts/update_virtuoso/check_for_updates.py | 59 ++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100755 scripts/update_virtuoso/check_for_updates.py (limited to 'scripts') diff --git a/scripts/update_virtuoso/check_for_updates.py b/scripts/update_virtuoso/check_for_updates.py new file mode 100755 index 0000000..f3b8a86 --- /dev/null +++ b/scripts/update_virtuoso/check_for_updates.py @@ -0,0 +1,59 @@ +#! /usr/bin/env python3 +# +# Check for updates on Arvados, pull the TTL and +# push into Virtuoso +# +# You can run this in a Guix container with +# +# ~/opt/guix/bin/guix environment -C guix --ad-hoc python python-requests curl --network -- python3 ./scripts/update_virtuoso/check_for_updates.py cache.txt dba dba + +import requests +import time + +url = 'https://download.lugli.arvadosapi.com/c=lugli-4zz18-z513nlpqm03hpca/_/mergedmetadata.ttl' +# --- Fetch headers from TTL file on Arvados +r = requests.head(url) +print(r.headers) + +print(r.headers['Last-Modified']) + +# --- Convert/validate time stamp +# ValueError: time data 'Tue, 21 Apr 2020 23:47:43 GMT' does not match format '%a %b %d %H:%M:%S %Y' +last_modified_str = r.headers['Last-Modified'] +t_stamp = time.strptime(last_modified_str,"%a, %d %b %Y %H:%M:%S %Z" ) +print(t_stamp) + +# OK, it works, now check last stored value +import sys +assert(len(sys.argv)==4) +fn = sys.argv[1] +user = sys.argv[2] +pwd = sys.argv[3] + +import os.path +stamp = None +if os.path.isfile(fn): + file = open(fn,"r") + stamp = file.read() + file.close + +import subprocess +if stamp != last_modified_str: + print("Fetch metadata TTL") + r = requests.get(url) + assert(r.status_code == 200) + with open("metadata.ttl", "w") as f: + f.write(r.text) + f.close + # Now push into Virtuoso using CURL + # cmd = "curl -X PUT --digest -u dba:dba -H Content-Type:text/turtle -T metadata.ttl -G http://localhost:8890/sparql-graph-crud-auth --data-urlencode graph=http://covid-19.genenetwork.org/graph".split(" ") + print("Push metadata TTL") + cmd = ("curl -X PUT --digest -u dba:%s -H Content-Type:text/turtle -T metadata.ttl -G http://sparql.genenetwork.org/sparql-graph-crud-auth --data-urlencode graph=http://covid-19.genenetwork.org/graph" % pwd ).split(" ") + print(cmd) + p = subprocess.Popen(cmd) + output = p.communicate()[0] + print(output) + assert(p.returncode == 0) + + with open(fn,"w") as f: + f.write(last_modified_str) -- cgit v1.2.3