1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
|
#!/usr/bin/env python3
#
# Copyright (c) 2016-2017 Weitian LI <weitian@aaronly.me>
# MIT license
"""
Download the required data (e.g., template maps) for simulations.
"""
import os
import sys
import argparse
import logging
import json
from fg21sim.configs import CONFIGS
from fg21sim.utils import setup_logging
from fg21sim.utils.download import download_file
def main():
parser = argparse.ArgumentParser(
description="Retrieve required input templates and catalogs")
parser.add_argument("manifest", help="data manifest file")
parser.add_argument("outdir", nargs="?", default=".",
help="output directory to save the downloaded data " +
"(default: .)")
parser.add_argument("-C", "--clobber", action="store_true",
help="overwrite the existing output file")
args = parser.parse_args()
tool = os.path.basename(sys.argv[0])
setup_logging(dict_config=CONFIGS.logging)
logger = logging.getLogger(tool)
logger.info("COMMAND: {0}".format(" ".join(sys.argv)))
if not os.path.exists(args.outdir):
os.mkdir(args.outdir)
logger.info("Created output directory: {0}".format(args.outdir))
manifest = json.load(open(args.manifest))
logger.info("Loaded data manifest from file: {0}".format(args.manifest))
nfile = len(manifest)
for i, fdata in enumerate(manifest):
filename = fdata["filename"]
outfile = os.path.join(args.outdir, filename)
size = fdata["size"]
md5 = fdata["md5"]
url = fdata["url"]
logger.info("[#{0}/{1}] Downloading file: {2} ...".format(
(i+1), nfile, filename))
download_file(url, outfile=outfile, size=size, md5=md5,
clobber=args.clobber)
logger.info("Finished downloading data!")
if __name__ == "__main__":
main()
|
