aboutsummaryrefslogtreecommitdiffstats
path: root/bin/fg21sim-download-data
blob: d4d03b433cb7a2f66ccb29b41f5d1943dbf77137 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
#!/usr/bin/env python3
#
# Copyright (c) 2016 Weitian LI <liweitianux@live.com>
# MIT license

"""
Download the required data (e.g., template maps) for simulations.
"""

import os
import sys
import argparse
import logging
import json

from fg21sim.configs import configs
from fg21sim.utils import setup_logging
from fg21sim.utils.download import download_file


def main():
    parser = argparse.ArgumentParser(
        description="Retrieve required input templates and catalogs")
    parser.add_argument("manifest", help="data manifest file")
    parser.add_argument("outdir", nargs="?", default=".",
                        help="output directory to save the downloaded data " +
                        "(default: .)")
    parser.add_argument("-C", "--clobber", action="store_true",
                        help="overwrite the existing output file")
    args = parser.parse_args()

    tool = os.path.basename(sys.argv[0])
    setup_logging(dict_config=configs.logging)
    logger = logging.getLogger(tool)
    logger.info("COMMAND: {0}".format(" ".join(sys.argv)))

    if not os.path.exists(args.outdir):
        os.mkdir(args.outdir)
        logger.info("Created output directory: {0}".format(args.outdir))

    manifest = json.load(open(args.manifest))
    logger.info("Loaded data manifest from file: {0}".format(args.manifest))
    nfile = len(manifest)
    for i, fdata in enumerate(manifest):
        filename = fdata["filename"]
        outfile = os.path.join(args.outdir, filename)
        size = fdata["size"]
        md5 = fdata["md5"]
        url = fdata["url"]
        logger.info("[#{0}/{1}] Downloading file: {2} ...".format(
            (i+1), nfile, filename))
        download_file(url, outfile=outfile, size=size, md5=md5,
                      clobber=args.clobber)
    logger.info("Finished downloading data!")


if __name__ == "__main__":
    main()