From a201fe48c42f82c03a5a63f0744f32eeaaf5934a Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 13 Mar 2023 10:25:01 -0700 Subject: [PATCH 1/3] Refacto conda backup --- s3_management/backup_conda.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/s3_management/backup_conda.py b/s3_management/backup_conda.py index a75c23407..16f9115a6 100644 --- a/s3_management/backup_conda.py +++ b/s3_management/backup_conda.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 -# Downloads domain library packages from channel +# Downloads domain pytorch andlibrary packages from channel # And backs them up to S3 # Do not use unless you know what you are doing +# Usage: python backup_conda.py --version 1.6.0 import conda.api import boto3 @@ -9,6 +10,7 @@ import urllib import os import hashlib +import argparse S3 = boto3.resource('s3') BUCKET = S3.Bucket('pytorch-backup') @@ -21,13 +23,17 @@ def compute_md5(path:str) -> str: def download_conda_package(package:str, version:Optional[str] = None, depends:Optional[str] = None, channel:Optional[str] = None) -> List[str]: + packages = conda.api.SubdirData.query_all(package, channels = [channel] if channel is not None else None, subdirs = _known_subdirs) rc = [] + + # print(f"Packages found {packages}") for pkg in packages: if version is not None and pkg.version != version: continue if depends is not None and depends not in pkg.depends: continue + print(f"Downloading {pkg.url}...") os.makedirs(pkg.subdir, exist_ok = True) fname = f"{pkg.subdir}/{pkg.fn}" @@ -48,8 +54,19 @@ def upload_to_s3(prefix: str, fnames: List[str]) -> None: print(fname) - if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--version", + help="PyTorch Version to backup", + type=str, + required = True + ) + options = parser.parse_args() + rc = download_conda_package("pytorch", channel = "pytorch", version = options.version) + upload_to_s3(f"v{options.version}/conda", rc) + for libname in ["torchvision", "torchaudio", "torchtext"]: - rc = download_conda_package(libname, channel = "pytorch", depends = "pytorch 1.9.0") - upload_to_s3("v1.9.0-rc4/conda", rc) + print(f"processing {libname}") + rc = download_conda_package(libname, channel = "pytorch", depends = f"pytorch {options.version}") + upload_to_s3f(f"v{options.version}/conda", rc) From 6eabd39936daed27aa7e649526478ec9607dd499 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 13 Mar 2023 10:30:01 -0700 Subject: [PATCH 2/3] Fix space --- s3_management/backup_conda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3_management/backup_conda.py b/s3_management/backup_conda.py index 16f9115a6..d6a0a076b 100644 --- a/s3_management/backup_conda.py +++ b/s3_management/backup_conda.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Downloads domain pytorch andlibrary packages from channel +# Downloads domain pytorch and library packages from channel # And backs them up to S3 # Do not use unless you know what you are doing # Usage: python backup_conda.py --version 1.6.0 From 9abc2ed16196a3d0a0cda82643cf4fbf9c476a3f Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 13 Mar 2023 10:31:07 -0700 Subject: [PATCH 3/3] Minor style --- s3_management/backup_conda.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/s3_management/backup_conda.py b/s3_management/backup_conda.py index d6a0a076b..f239bbea1 100644 --- a/s3_management/backup_conda.py +++ b/s3_management/backup_conda.py @@ -23,11 +23,9 @@ def compute_md5(path:str) -> str: def download_conda_package(package:str, version:Optional[str] = None, depends:Optional[str] = None, channel:Optional[str] = None) -> List[str]: - packages = conda.api.SubdirData.query_all(package, channels = [channel] if channel is not None else None, subdirs = _known_subdirs) rc = [] - # print(f"Packages found {packages}") for pkg in packages: if version is not None and pkg.version != version: continue @@ -54,6 +52,7 @@ def upload_to_s3(prefix: str, fnames: List[str]) -> None: print(fname) + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument(