1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
|
#!/usr/bin/env python3
#
# Copyright (c) 2016-2017 Weitian LI <weitian@aaronly.me>
# MIT license
"""
Download the data (e.g., template maps) for simulations according to the
manifest file.
"""
import os
import sys
import argparse
import logging
import json
from fg21sim.share import CONFIGS
from fg21sim.utils import setup_logging
from fg21sim.utils.download import download_file
def main():
parser = argparse.ArgumentParser(
description="Download data according to a manifest file")
parser.add_argument("manifest", help="data manifest file")
parser.add_argument("outdir", nargs="?", default=".",
help="output directory to save the downloaded data " +
"(default: .)")
parser.add_argument("-C", "--clobber", action="store_true",
help="overwrite the existing output file")
args = parser.parse_args()
tool = os.path.basename(sys.argv[0])
setup_logging(dict_config=CONFIGS.logging)
logger = logging.getLogger(tool)
logger.info("COMMAND: {0}".format(" ".join(sys.argv)))
if not os.path.exists(args.outdir):
os.mkdir(args.outdir)
logger.info("Created output directory: {0}".format(args.outdir))
manifest = json.load(open(args.manifest))
logger.info("Loaded data manifest from file: {0}".format(args.manifest))
nfile = len(manifest)
for i, fdata in enumerate(manifest):
filename = fdata["filename"]
outfile = os.path.join(args.outdir, filename)
size = fdata["size"]
md5 = fdata["md5"]
url = fdata["url"]
logger.info("[#{0}/{1}] Downloading file: {2} ...".format(
(i+1), nfile, filename))
download_file(url, outfile=outfile, size=size, md5=md5,
clobber=args.clobber)
logger.info("Finished downloading data!")
if __name__ == "__main__":
main()
|
