diff --git a/torchbenchmark/__init__.py b/torchbenchmark/__init__.py index b07c121a48..164bfa3cc3 100644 --- a/torchbenchmark/__init__.py +++ b/torchbenchmark/__init__.py @@ -59,7 +59,7 @@ def _install_deps(model_path: str, verbose: bool = True) -> Tuple[bool, Any]: [sys.executable, install_file], ] run_env = os.environ.copy() - run_env["PYTHONPATH"] = this_dir.parent + run_env["PYTHONPATH"] = Path(this_dir.parent).as_posix() run_kwargs = { "cwd": model_path, "check": True, @@ -67,7 +67,7 @@ def _install_deps(model_path: str, verbose: bool = True) -> Tuple[bool, Any]: } output_buffer = None - _, stdout_fpath = tempfile.mkstemp() + fd, stdout_fpath = tempfile.mkstemp() try: output_buffer = io.FileIO(stdout_fpath, mode="w") @@ -89,7 +89,9 @@ def _install_deps(model_path: str, verbose: bool = True) -> Tuple[bool, Any]: except Exception as e: return (False, e, io.FileIO(stdout_fpath, mode="r").read().decode()) finally: + output_buffer.close() del output_buffer + os.close(fd) os.remove(stdout_fpath) return (True, None, None) diff --git a/torchbenchmark/models/fastNLP_Bert/cmrc2018_simulator.py b/torchbenchmark/models/fastNLP_Bert/cmrc2018_simulator.py index 62ef9ef44a..a34b72a550 100644 --- a/torchbenchmark/models/fastNLP_Bert/cmrc2018_simulator.py +++ b/torchbenchmark/models/fastNLP_Bert/cmrc2018_simulator.py @@ -108,7 +108,7 @@ def _generate_train(batch_size): def _generate_vocab(): never_split = ["[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]"] VOCAB_SET.update(never_split) - with open(CMRC2018_VOCAB_SIM, "w") as vf: + with open(CMRC2018_VOCAB_SIM, "w", encoding='utf-8') as vf: vf.write("\n".join(list(VOCAB_SET))) def _copy_bert_config(): diff --git a/torchbenchmark/models/sam/install.py b/torchbenchmark/models/sam/install.py index 0646de166e..7c94fa7bae 100644 --- a/torchbenchmark/models/sam/install.py +++ b/torchbenchmark/models/sam/install.py @@ -1,15 +1,30 @@ import os import subprocess import sys +import requests + +def download(uri): + directory = '.data' + filename = os.path.basename(uri) # Extracts the filename from the URI + filepath = os.path.join(directory, filename) + os.makedirs(directory, exist_ok=True) + response = requests.get(uri, stream=True) + if response.status_code == 200: + with open(filepath, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): # Define the chunk size to be used + f.write(chunk) + else: + print(f'Failed to download file with status code {response.status_code}') + def pip_install_requirements(): subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) def download_checkpoint(): - subprocess.check_call(['wget', '-P', '.data', 'https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth']) + download('https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth') def download_data(): - subprocess.check_call(['wget', '-P', '.data', 'https://github.com/facebookresearch/segment-anything/raw/main/notebooks/images/truck.jpg']) + download('https://github.com/facebookresearch/segment-anything/raw/main/notebooks/images/truck.jpg') if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/models/sam_fast/install.py b/torchbenchmark/models/sam_fast/install.py index 0646de166e..7c94fa7bae 100644 --- a/torchbenchmark/models/sam_fast/install.py +++ b/torchbenchmark/models/sam_fast/install.py @@ -1,15 +1,30 @@ import os import subprocess import sys +import requests + +def download(uri): + directory = '.data' + filename = os.path.basename(uri) # Extracts the filename from the URI + filepath = os.path.join(directory, filename) + os.makedirs(directory, exist_ok=True) + response = requests.get(uri, stream=True) + if response.status_code == 200: + with open(filepath, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): # Define the chunk size to be used + f.write(chunk) + else: + print(f'Failed to download file with status code {response.status_code}') + def pip_install_requirements(): subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) def download_checkpoint(): - subprocess.check_call(['wget', '-P', '.data', 'https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth']) + download('https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth') def download_data(): - subprocess.check_call(['wget', '-P', '.data', 'https://github.com/facebookresearch/segment-anything/raw/main/notebooks/images/truck.jpg']) + download('https://github.com/facebookresearch/segment-anything/raw/main/notebooks/images/truck.jpg') if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/models/torch_multimodal_clip/install.py b/torchbenchmark/models/torch_multimodal_clip/install.py index 647fb28094..0723509634 100644 --- a/torchbenchmark/models/torch_multimodal_clip/install.py +++ b/torchbenchmark/models/torch_multimodal_clip/install.py @@ -1,13 +1,24 @@ import os import subprocess import sys +import requests + +def download(output_filename, uri): + # Download the file with streaming to handle large files + response = requests.get(uri, stream=True) + if response.status_code == 200: + with open(output_filename, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): # Define the chunk size to be used + f.write(chunk) + else: + print(f'Failed to download file with status code {response.status_code}') def pip_install_requirements(): subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) def download_data(data_folder): # CC-0 image from wikipedia page on pizza so legal to use - subprocess.check_call(['wget', '-O', os.path.join(data_folder, 'pizza.jpg'), 'https://upload.wikimedia.org/wikipedia/commons/thumb/9/91/Pizza-3007395.jpg/2880px-Pizza-3007395.jpg']) + download(os.path.join(data_folder, 'pizza.jpg'), 'https://upload.wikimedia.org/wikipedia/commons/thumb/9/91/Pizza-3007395.jpg/2880px-Pizza-3007395.jpg') if __name__ == '__main__': pip_install_requirements()