aboutsummaryrefslogtreecommitdiffstats
path: root/bin/fg21sim-download-snr-data
blob: a6ace8c7205e2c424873c230d774c6cf9f90991e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
#!/usr/bin/env python3
#
# Copyright (c) 2016-2017 Weitian LI <weitian@aaronly.me>
# MIT license

"""
Retrieve the Galactic SNRs catalog data by parsing the web by /D. A. Green/:

    http://www.mrao.cam.ac.uk/surveys/snrs/
    http://www.mrao.cam.ac.uk/surveys/snrs/snrs.data.html

and save the data in CSV format.
"""


import os
import sys
import argparse
import logging
import csv

from fg21sim.configs import CONFIGS
from fg21sim.utils import setup_logging
from fg21sim.data import retrieve_snr_data_green, SNRDataGreen


# Web page to the Galactic SNRs catalog summary
SNR_DATA_URL = "http://www.mrao.cam.ac.uk/surveys/snrs/snrs.data.html"


def main():
    outfile_default = "GalacticSNRs_Green2014.csv"

    parser = argparse.ArgumentParser(
        description="Retrieve Galactic SNRs catalog data")
    parser.add_argument("outfile", nargs="?", default=outfile_default,
                        help="output CSV file to save the catalog data " +
                        "(default: %s)" % outfile_default)
    parser.add_argument("-U", "--url", default=SNR_DATA_URL,
                        help="URL to Green's SNRs catalog summary page " +
                        "or a local HTML file (default: %s)" % SNR_DATA_URL)
    parser.add_argument("-C", "--clobber", action="store_true",
                        help="overwrite the existing output file")
    parser.add_argument("-l", "--log", dest="loglevel", default=None,
                        choices=["DEBUG", "INFO", "WARNING",
                                 "ERROR", "CRITICAL"],
                        help="set the log level")
    parser.add_argument("-L", "--logfile", default=None,
                        help="filename where to save the log messages")
    parser.add_argument("-Q", "--quiet", action="store_true",
                        help="be quiet so do not log messages to screen")
    args = parser.parse_args()

    log_stream = "" if args.quiet else None
    setup_logging(dict_config=CONFIGS.logging,
                  level=args.loglevel,
                  stream=log_stream,
                  logfile=args.logfile)
    tool = os.path.basename(sys.argv[0])
    logger = logging.getLogger(tool)
    logger.info("COMMAND: {0}".format(" ".join(sys.argv)))

    if os.path.exists(args.outfile) and (not args.clobber):
        raise IOError("output file already exists: %s" % args.outfile)

    snrdata_str = retrieve_snr_data_green(args.url)
    snrdata = SNRDataGreen(snrdata_str[0])
    header = list(snrdata.data_flat.keys())
    with open(args.outfile, "w") as csvfile:
        csvwriter = csv.writer(csvfile)
        csvwriter.writerow(header)
        for i, dstr in enumerate(snrdata_str):
            logger.info("Parse data string #{0:03d}: '{1}'".format(i+1, dstr))
            snrdata = SNRDataGreen(dstr)
            values = list(snrdata.data_flat.values())
            csvwriter.writerow(values)
    logger.info("Galactic SNRs catalog data write to: %s" % args.outfile)


if __name__ == "__main__":
    main()