From 1718ff1bf4611e05d2fb952240369e27fe1504bd Mon Sep 17 00:00:00 2001
From: Arun Isaac
Date: Sat, 4 Jan 2025 01:34:35 +0000
Subject: Implement downloader in python too.

Passing cookies to wget on the command line is a security risk. On a
shared machine, other users can see your full command line. Passing it
in using the --load-cookies option is too tedious—the file format
required is archaic and hard to replicate by hand. So, we simply
implement the downloader in python too. In any case, this makes for a
more cohesive user experience.
---
 globus-weblinks | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

(limited to 'globus-weblinks')

diff --git a/globus-weblinks b/globus-weblinks
index 74fd6f0..92f598d 100755
--- a/globus-weblinks
+++ b/globus-weblinks
@@ -1,9 +1,12 @@
 #! /usr/bin/env python3
 
 import argparse
-from pathlib import PurePath
+from pathlib import Path, PurePath
+import json
+import requests
 import sys
 import globus_sdk
+from urllib.parse import urlparse
 
 # This is the tutorial client ID from
 # https://globus-sdk-python.readthedocs.io/en/stable/tutorial.html.
@@ -31,12 +34,31 @@ def find_files(transfer_client, endpoint_id, path=PurePath("/")):
         else:
             yield path / file["name"]
 
+def download_file(url, cookies):
+    filepath = Path(urlparse(url).path).relative_to("/")
+    filepath.parent.mkdir(parents=True, exist_ok=True)
+    with open(filepath, "wb") as f:
+        for chunk in (requests.get(url, cookies=cookies, stream=True)
+                      .iter_content(chunk_size=1024*1024)):
+            f.write(chunk)
+
 parser = argparse.ArgumentParser(description="Get web links for Globus collection")
 parser.add_argument("endpoint_id", metavar="endpoint-id", help="Endpoint ID of collection")
+parser.add_argument("cookies", help="JSON file with cookies from Globus web app")
 args = parser.parse_args()
 
 transfer_client = globus_sdk.TransferClient(
     authorizer=globus_sdk.AccessTokenAuthorizer(get_transfer_token()))
 endpoint = transfer_client.get_endpoint(args.endpoint_id)
-for path in find_files(transfer_client, args.endpoint_id):
-    print(endpoint["https_server"] + str(path))
+urls = [endpoint["https_server"] + str(path)
+        for path in find_files(transfer_client, args.endpoint_id)]
+total = len(urls)
+print(f"Found {total} files")
+
+with open(args.cookies) as f:
+    cookies = json.load(f)
+
+for i, url in enumerate(urls, 1):
+    print(f"{i}/{total}: Downloading {url}")
+    download_file(url, cookies)
+print("Download complete!")
-- 
cgit v1.2.3