From 9250f66d63720b182b1002cd6d8554894e58502e Mon Sep 17 00:00:00 2001 From: Wenyu Date: Mon, 5 Feb 2024 16:19:58 +0800 Subject: [PATCH] remove _wget (#61356) (#61569) * remove _wget * remove _wget * remove wget test --- python/paddle/hapi/hub.py | 1 - python/paddle/utils/download.py | 38 +------------------------------ test/legacy_test/test_download.py | 15 +----------- 3 files changed, 2 insertions(+), 52 deletions(-) diff --git a/python/paddle/hapi/hub.py b/python/paddle/hapi/hub.py index c39fa57ad5681..f25640804fdbc 100644 --- a/python/paddle/hapi/hub.py +++ b/python/paddle/hapi/hub.py @@ -117,7 +117,6 @@ def _get_cache_or_reload(repo, force_reload, verbose=True, source='github'): hub_dir, check_exist=not force_reload, decompress=False, - method=('wget' if source == 'gitee' else 'get'), ) shutil.move(fpath, cached_file) diff --git a/python/paddle/utils/download.py b/python/paddle/utils/download.py index 30f02a20b969b..e0cd17662d7f1 100644 --- a/python/paddle/utils/download.py +++ b/python/paddle/utils/download.py @@ -15,14 +15,11 @@ import hashlib import os import os.path as osp -import shlex import shutil -import subprocess import sys import tarfile import time import zipfile -from urllib.parse import urlparse import httpx @@ -198,40 +195,7 @@ def _get_download(url, fullname): return False -def _wget_download(url: str, fullname: str): - try: - assert urlparse(url).scheme in ( - 'http', - 'https', - ), 'Only support https and http url' - # using wget to download url - tmp_fullname = shlex.quote(fullname + "_tmp") - url = shlex.quote(url) - # –user-agent - command = f'wget -O {tmp_fullname} -t {DOWNLOAD_RETRY_LIMIT} {url}' - subprc = subprocess.Popen( - command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - _ = subprc.communicate() - - if subprc.returncode != 0: - raise RuntimeError( - f'{command} failed. Please make sure `wget` is installed or {url} exists' - ) - - shutil.move(tmp_fullname, fullname) - - except Exception as e: # requests.exceptions.ConnectionError - logger.info(f"Downloading {url} failed with exception {str(e)}") - return False - - return fullname - - -_download_methods = { - 'get': _get_download, - 'wget': _wget_download, -} +_download_methods = {'get': _get_download} def _download(url, path, md5sum=None, method='get'): diff --git a/test/legacy_test/test_download.py b/test/legacy_test/test_download.py index 742c4b2a65190..da25a3021a31e 100644 --- a/test/legacy_test/test_download.py +++ b/test/legacy_test/test_download.py @@ -120,14 +120,6 @@ def test_retry_exception( './test', ) - def test_wget_download_error( - self, - ): - with self.assertRaises(RuntimeError): - from paddle.utils.download import _download - - _download('www.baidu', './test', method='wget') - def test_download_methods( self, ): @@ -136,14 +128,9 @@ def test_download_methods( "https://paddle-hapi.bj.bcebos.com/unittest/files.zip", ] - import sys - from paddle.utils.download import _download - if sys.platform == 'linux': - methods = ['wget', 'get'] - else: - methods = ['get'] + methods = ['get'] for url in urls: for method in methods: