下载大文件 python 有要求

def download_file/url/:

 local_filename = url.split/'/'/[-1]

 # NOTE the stream=True parameter below

 with requests.get/url, stream=True/ as r:

 r.raise_for_status//

 with open/local_filename, 'wb'/ as f:

 for chunk in r.iter_content/chunk_size=8192/: 

 # If you have chunk encoded response uncomment if

 # and set chunk_size parameter to None.

 #if chunk: 

 f.write/chunk/

 return local_filename

iter_content

chunk_size

import requests

import shutil



def download_file/url/:

 local_filename = url.split/'/'/[-1]

 with requests.get/url, stream=True/ as r:

 with open/local_filename, 'wb'/ as f:

 shutil.copyfileobj/r.raw, f/



 return local_filename

urllib

from urllib.request import urlretrieve

url = 'http://mirror.pnl.gov/releases/16.04.2/ubuntu-16.04.2-desktop-amd64.iso'

dst = 'ubuntu-16.04.2-desktop-amd64.iso'

urlretrieve/url, dst/

from urllib.request import urlopen

from shutil import copyfileobj

from tempfile import NamedTemporaryFile

url = 'http://mirror.pnl.gov/releases/16.04.2/ubuntu-16.04.2-desktop-amd64.iso'

with urlopen/url/ as fsrc, NamedTemporaryFile/delete=False/ as fdst:

 copyfileobj/fsrc, fdst/

watch 'ps -p 18647 -o pid,ppid,pmem,rsz,vsz,comm,args; ls -al *.iso'

with

def DownloadFile/url/:

 local_filename = url.split/'/'/[-1]

 r = requests.get/url/

 with open/local_filename, 'wb'/ as f:

 for chunk in r.iter_content/chunk_size=1024/: 

 if chunk: # filter out keep-alive new chunks

 f.write/chunk/

 return

f.flush//

os.fsync//

with open/local_filename, 'wb'/ as f:

 for chunk in r.iter_content/chunk_size=1024/: 

 if chunk: # filter out keep-alive new chunks

 f.write/chunk/

 f.flush//

 os.fsync/f.fileno///

def download/url: str, file_path='', attempts=2/:

 """Downloads a URL content into a file /with large file support by streaming/



 :param url: URL to download

 :param file_path: Local file name to contain the data downloaded

 :param attempts: Number of attempts

 :return: New file path. Empty string if the download failed

 """

 if not file_path:

 file_path = os.path.realpath/os.path.basename/url//

 logger.info/f'Downloading {url} content to {file_path}'/

 url_sections = urlparse/url/

 if not url_sections.scheme:

 logger.debug/'The given url is missing a scheme. Adding http scheme'/

 url = f'http://{url}'

 logger.debug/f'New url: {url}'/

 for attempt in range/1, attempts+1/:

 try:

 if attempt > 1:

 time.sleep/10/ # 10 seconds wait time between downloads

 with requests.get/url, stream=True/ as response:

 response.raise_for_status//

 with open/file_path, 'wb'/ as out_file:

 for chunk in response.iter_content/chunk_size=1024*1024/: # 1MB chunks

 out_file.write/chunk/

 logger.info/'Download finished successfully'/

 return file_path

 except Exception as ex:

 logger.error/f'Attempt #{attempt} failed with error: {ex}'/

 return ''