Skip to content

Commit

Permalink
fix issue with global var
Browse files Browse the repository at this point in the history
  • Loading branch information
cuuupid committed Feb 10, 2025
1 parent e198a34 commit 528b8ff
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions pget.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,16 +64,18 @@ def make_manifest(manifest_filename: str = 'manifest.pget'):
# Step 4: Copy files to cache
if CACHE_URI.startswith('s3://'):
cp_command = ['aws', 's3', 'cp']
bucket = CACHE_URI
elif CACHE_URI.startswith('r2://'):
cp_command = ['aws', 's3', 'cp', '--endpoint-url', 'https://3309f63723c6de8a36dab1a22068e3aa.r2.cloudflarestorage.com']
CACHE_URI = "s3://" + CACHE_URI[5:]
bucket = "s3://" + CACHE_URI[5:]
elif CACHE_URI.startswith('gs://'):
cp_command = ['gcloud', 'storage', 'cp']
bucket = CACHE_URI
else:
raise ValueError("Invalid CACHE_URI. Must start with 's3://' or 'gs://'")

for filepath, _ in tqdm(large_files, desc="Copying files to cache"):
dest_path = os.path.join(CACHE_URI, filepath.lstrip('./'))
dest_path = os.path.join(bucket, filepath.lstrip('./'))
subprocess.run(cp_command + [filepath, dest_path], check=True)

# Step 5: Generate manifest file
Expand Down

0 comments on commit 528b8ff

Please sign in to comment.