1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
|
# Copyright (c) 2016 Weitian LI <liweitianux@live.com>
# MIT license
"""
Utilities to download files
"""
import os
import subprocess
import logging
from .hashutil import calc_md5
logger = logging.getLogger(__name__)
def _check_filesize(filepath, size):
"""
Check whether the file size equals the given size.
Parameters
----------
filepath : str
Path to the input file
size : int
Expected/true size (in bytes) of the file
Returns
-------
valid : bool
``True`` if the on-disk file size equals the given size,
otherwise ``False``.
"""
size_ondisk = os.path.getsize(filepath)
return size_ondisk == size
def _check_md5(filepath, md5):
"""
Check whether the file MD5 digest matches the given MD5.
Parameters
----------
filepath : str
Path to the input file
md5 : str
Expected/true MD5 digest of the file
Returns
-------
valid : bool
``True`` if the on-disk file MD5 digest matches the given MD5,
otherwise ``False``.
"""
md5_ondisk = calc_md5(filepath)
return md5_ondisk == md5
def download_file(url, outfile=None, size=None, md5=None, clobber=False):
"""
Download file using "wget" and validate the file size and MD5 digest.
If the expected MD5 digest is provided, and the on-disk file has the
same MD5 digest, then the download is skipped.
If the output file already exists but with unmatched MD5, then
re-download it if ``clobber=True``, otherwise, an ``IOError`` raised.
Parameters
----------
url : str
The URL from where to download the file.
outfile : str, optional
The path and filename for the downloaded file.
If not provided, then use the basename of the URL.
size : int, optional
Expected/true size (in bytes) of the file
If provided, then check the file size after download.
md5 : str, optional
Expected/true MD5 digest of the file
If provided, then check the MD5 digest after download.
clobber : bool, optional
Whether to overwrite the existing file?
Raises
------
IOError :
Output file with unmatched MD5 digest already exists
while ``clobber=False``.
"""
if outfile is None:
outfile = os.path.basename(url)
# Check whether can skip the download
if os.path.exists(outfile):
if (md5 is not None) and _check_md5(outfile, md5):
logger.info("Skip already downloaded file: {0}".format(outfile))
return
elif clobber:
os.remove(outfile)
logger.info("Removed wrong existing file: {0}".format(outfile))
else:
raise IOError("Exists wrong output file: {0}".format(outfile))
#
cmd = ["wget", "-O", outfile, url]
logger.info("CMD: {0}".format(" ".join(cmd)))
subprocess.check_call(cmd)
if (size is not None) and (not _check_filesize(outfile, size)):
raise ValueError("Downloaded file has wrong size")
if (md5 is not None) and (not _check_md5(outfile, md5)):
raise ValueError("Downloaded file has unmatched MD5 digest")
logger.info("Downloaded and validated file: {0}".format(outfile))
|
