From c31835f787f3ae36e26bad0a1803f8557f8084e7 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 6 Jan 2021 02:33:35 -0600
Subject: Pubseq fetch: sometimes a request times out. So repeat with
 intervals.

---
 workflows/tools/pubseq-fetch-data.py | 33 +++++++++++++++++++--------------
 1 file changed, 19 insertions(+), 14 deletions(-)

(limited to 'workflows')

diff --git a/workflows/tools/pubseq-fetch-data.py b/workflows/tools/pubseq-fetch-data.py
index 2119fdf..ef4edde 100755
--- a/workflows/tools/pubseq-fetch-data.py
+++ b/workflows/tools/pubseq-fetch-data.py
@@ -5,6 +5,7 @@ import json
 import os
 import requests
 import sys
+import time
 
 parser = argparse.ArgumentParser(description="""
 
@@ -33,18 +34,22 @@ for id in ids:
     print(id)
     jsonfn = dir+"/"+id+".json"
     if not os.path.exists(jsonfn):
+        count = 0
         r = requests.get(f"http://covid19.genenetwork.org/api/sample/{id}.json")
-        if r:
-            m_url = r.json()[0]['metadata']
-            mr = requests.get(m_url)
-            with open(dir+"/"+id+".json","w") as outf:
-                outf.write(mr.text)
-            if args.fasta:
-                fastafn = dir+"/"+id+".fa"
-                if os.path.exists(fastafn): continue
-                fa_url = r.json()[0]['fasta']
-                fr = requests.get(fa_url)
-                with open(fastafn,"w") as outf:
-                    outf.write(fr.text)
-        else:
-            raise Exception(f"Can not find record for {id}")
+        while not r:
+            count += 1
+            if count>10: raise Exception(f"Can not find record for {id}")
+            time.sleep(15)
+            r = requests.get(f"http://covid19.genenetwork.org/api/sample/{id}.json")
+        m_url = r.json()[0]['metadata']
+        mr = requests.get(m_url)
+        with open(dir+"/"+id+".json","w") as outf:
+            outf.write(mr.text)
+        if args.fasta:
+            fastafn = dir+"/"+id+".fa"
+            if os.path.exists(fastafn): continue
+            fa_url = r.json()[0]['fasta']
+            fr = requests.get(fa_url)
+            with open(fastafn,"w") as outf:
+                outf.write(fr.text)
+
-- 
cgit v1.2.3