diff options
| -rwxr-xr-x | bin/fg21sim-download-data | 10 | ||||
| -rwxr-xr-x | bin/healpix2hpx | 30 | ||||
| -rwxr-xr-x | bin/hpx2healpix | 30 | ||||
| -rw-r--r-- | fg21sim/configs/00-general.conf.spec | 34 | ||||
| -rw-r--r-- | fg21sim/configs/manager.py | 44 | ||||
| -rw-r--r-- | fg21sim/utils/__init__.py | 1 | ||||
| -rw-r--r-- | fg21sim/utils/healpix.py | 30 | ||||
| -rw-r--r-- | fg21sim/utils/logging.py | 85 |
8 files changed, 246 insertions, 18 deletions
diff --git a/bin/fg21sim-download-data b/bin/fg21sim-download-data new file mode 100755 index 0000000..db432f0 --- /dev/null +++ b/bin/fg21sim-download-data @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2016 Weitian LI <liweitianux@live.com> +# MIT license + +""" +Download the required data (e.g., template maps) for simulations. +""" + +raise NotImplementedError("TODO") diff --git a/bin/healpix2hpx b/bin/healpix2hpx index 9887bc6..f90c637 100755 --- a/bin/healpix2hpx +++ b/bin/healpix2hpx @@ -11,12 +11,14 @@ Reorganize the sky map in HEALPix table format into image in HPX projection. import os import sys import argparse +import logging import numpy as np from astropy.io import fits import fg21sim -from fg21sim.utils import healpix2hpx +from fg21sim.configs import configs +from fg21sim.utils import healpix2hpx, setup_logging def main(): @@ -28,16 +30,36 @@ def main(): help="overwrite the existing output file") parser.add_argument("-F", "--float", action="store_true", help="use float (single precision) instead of double") + parser.add_argument("-l", "--log", dest="loglevel", default=None, + help="log level (valid values: " + "DEBUG, INFO, WARNING, ERROR, CRITICAL)") + parser.add_argument("-L", "--logfile", default=None, + help="filename where to save the log messages") + parser.add_argument("-Q", "--quiet", action="store_true", + help="be quiet so do not log messages to screen") args = parser.parse_args() + if args.quiet: + log_stream = "" + else: + log_stream = None + tool = os.path.basename(sys.argv[0]) + pkgname = fg21sim.__pkgname__ + + setup_logging(dict_config=configs.logging, + level=args.loglevel, + stream=log_stream, + logfile=args.logfile) + logger = logging.getLogger(tool) + logger.info("COMMAND: {0}".format(" ".join(sys.argv))) + history = [ "TOOL: {0}".format(tool), "PARAM: {0}".format(" ".join(sys.argv[1:])), ] comments = [ - 'Tool "{0}" is part of the "{1}" package'.format(tool, - fg21sim.__title__), + 'Tool "{0}" is part of the "{1}" package'.format(tool, pkgname), 'distributed under {0} license.'.format(fg21sim.__license__), 'See also {0}'.format(fg21sim.__url__) ] @@ -46,9 +68,11 @@ def main(): append_history=history, append_comment=comments) if args.float: + logger.info("HPX FITS images: use single-precision float numbers") hpx_data = hpx_data.astype(np.float32) hdu = fits.PrimaryHDU(data=hpx_data, header=hpx_header) hdu.writeto(args.outfile, clobber=args.clobber, checksum=True) + logger.info("HPX FITS images write to: %s" % args.outfile) if __name__ == "__main__": diff --git a/bin/hpx2healpix b/bin/hpx2healpix index 6f83426..c9a916c 100755 --- a/bin/hpx2healpix +++ b/bin/hpx2healpix @@ -11,12 +11,14 @@ i.e., the reverse of `healpix2hpx.py`. import os import sys import argparse +import logging import numpy as np from astropy.io import fits import fg21sim -from fg21sim.utils import hpx2healpix +from fg21sim.configs import configs +from fg21sim.utils import hpx2healpix, setup_logging # Reference: @@ -39,16 +41,36 @@ def main(): help="overwrite the existing output file") parser.add_argument("-F", "--float", action="store_true", help="use float (single precision) instead of double") + parser.add_argument("-l", "--log", dest="loglevel", default=None, + help="log level (valid values: " + "DEBUG, INFO, WARNING, ERROR, CRITICAL)") + parser.add_argument("-L", "--logfile", default=None, + help="filename where to save the log messages") + parser.add_argument("-Q", "--quiet", action="store_true", + help="be quiet so do not log messages to screen") args = parser.parse_args() + if args.quiet: + log_stream = "" + else: + log_stream = None + tool = os.path.basename(sys.argv[0]) + pkgname = fg21sim.__pkgname__ + + setup_logging(dict_config=configs.logging, + level=args.loglevel, + stream=log_stream, + logfile=args.logfile) + logger = logging.getLogger(tool) + logger.info("COMMAND: {0}".format(" ".join(sys.argv))) + history = [ "TOOL: {0}".format(tool), "PARAM: {0}".format(" ".join(sys.argv[1:])), ] comments = [ - 'Tool "{0}" is part of the "{1}" package'.format(tool, - fg21sim.__title__), + 'Tool "{0}" is part of the "{1}" package'.format(tool, pkgname), 'distributed under {0} license.'.format(fg21sim.__license__), 'See also {0}'.format(fg21sim.__url__) ] @@ -57,12 +79,14 @@ def main(): append_history=history, append_comment=comments) if args.float: + logger.info("HEALPix data: use single-precision float numbers") hp_data = hp_data.astype(np.float32) hdu = fits.BinTableHDU.from_columns([ fits.Column(name="I", array=hp_data, format=FITS_COLUMN_FORMATS.get(hp_data.dtype)) ], header=hp_header) hdu.writeto(args.outfile, clobber=args.clobber, checksum=True) + logger.info("HEALPix data write to FITS file: %s" % args.outfile) if __name__ == "__main__": diff --git a/fg21sim/configs/00-general.conf.spec b/fg21sim/configs/00-general.conf.spec new file mode 100644 index 0000000..c1a0eb3 --- /dev/null +++ b/fg21sim/configs/00-general.conf.spec @@ -0,0 +1,34 @@ +# Configurations for "fg21sim" +# -*- mode: conf -*- +# +# Syntax: `ConfigObj`, https://github.com/DiffSK/configobj +# +# This file contains the general configurations, which control the general +# behaviors, or will be used in other configuration sections. + +[logging] +# DEBUG: Detailed information, typically of interest only when diagnosing +# problems. +# INFO: Confirmation that things are working as expected. +# WARNING: An dinciation that something unexpected happended, or indicative +# of some problem in the near future (e.g., "disk space low"). +# The software is still working as expected. +# ERROR: Due to a more serious problem, the software has not been able to +# perform some function. +# CRITICAL: A serious error, indicating that the program itself may be unable +# to continue running. +level = option("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", default="INFO") + +# Set the format of displayed messages +format = string(default="%(asctime)s [%(levelname)s] %(name)s: %(message)s") + +# Set the date/time format in messages (default: ISO8601) +datefmt = string(default="%Y-%m-%dT%H:%M:%S") + +# Set the logging filename (will create a `FileHandler`) +filename = string(default="") +# Set the mode to open the above logging file +filemode = option("w", "a", default="a") + +# Set the stream used to initialize the `StreamHandler` +stream = option("stderr", "stdout", "", default="stderr") diff --git a/fg21sim/configs/manager.py b/fg21sim/configs/manager.py index 933a63d..f5465a9 100644 --- a/fg21sim/configs/manager.py +++ b/fg21sim/configs/manager.py @@ -10,13 +10,19 @@ Configuration manager. """ import os +import sys from glob import glob -from errors import ConfigError +import logging from configobj import ConfigObj, ConfigObjError, flatten_errors from validate import Validator +class ConfigError(Exception): + """Could not parse user configurations""" + pass + + CONFIGS_PATH = os.path.dirname(__file__) @@ -30,21 +36,23 @@ class ConfigManager: spec = "\n".join([open(f).read() for f in configs_spec]).split("\n") self._configspec = ConfigObj(spec, interpolation=False, list_values=False, _inspec=True) - self._validator = Validator() - configs_default = ConfigObj(configspec=self._configspec) + configs_default = ConfigObj(interpolation=False, + configspec=self._configspec) self._config = self.validate(configs_default) if configs: for config in configs: self.read_config(config) def read_config(self, config): - newconfig = ConfigObj(config, configspec=self._configspec) + newconfig = ConfigObj(config, interpolation=False, + configspec=self._configspec) newconfig = self.validate(newconfig) self._config.merge(newconfig) def validate(self, config): + validator = Validator() try: - results = config.validate(self._validator, preserve_errors=True) + results = config.validate(validator, preserve_errors=True) except ConfigObjError as e: raise ConfigError(e.message) if not results: @@ -72,3 +80,29 @@ class ConfigManager: def set(self, key, value): self._config[key] = value + + @property + def logging(self): + """Get and prepare the logging configurations for + `logging.basicConfig()` + """ + from logging import FileHandler, StreamHandler + conf = self.get("logging") + # logging handlers + handlers = [] + stream = conf["stream"] + if stream: + handlers.append(StreamHandler(getattr(sys, stream))) + logfile = conf["filename"] + filemode = conf["filemode"] + if logfile: + handlers.append(FileHandler(logfile, mode=filemode)) + # + logconf = { + "level": getattr(logging, conf["level"]), + "format": conf["format"], + "datefmt": conf["datefmt"], + "filemode": filemode, + "handlers": handlers, + } + return logconf diff --git a/fg21sim/utils/__init__.py b/fg21sim/utils/__init__.py index 3548b3f..650ab37 100644 --- a/fg21sim/utils/__init__.py +++ b/fg21sim/utils/__init__.py @@ -2,3 +2,4 @@ # MIT license from .healpix import healpix2hpx, hpx2healpix +from .logging import setup_logging diff --git a/fg21sim/utils/healpix.py b/fg21sim/utils/healpix.py index af0df59..acd85ef 100644 --- a/fg21sim/utils/healpix.py +++ b/fg21sim/utils/healpix.py @@ -33,7 +33,7 @@ from astropy.io import fits from .. import logging -logger = logging.getLogger() +logger = logging.getLogger(__name__) def healpix2hpx(data, header=None, append_history=None, append_comment=None): @@ -61,11 +61,16 @@ def healpix2hpx(data, header=None, append_history=None, append_comment=None): hp_data, hp_header = hp.read_map(data, nest=False, h=True, verbose=False) hp_header = fits.header.Header(hp_header) + logger.info("Read HEALPix data from file or HDU") else: hp_data, hp_header = np.asarray(data), fits.header.Header(header) + logger.info("Read HEALPix data from array and header") + logger.info("HEALPix index ordering: %s" % hp_header["ORDERING"]) if hp_header["ORDERING"] != "RING": raise ValueError("only 'RING' ordering currently supported") - nside = hp.npix2nside(len(hp_data)) + npix = len(hp_data) + nside = hp.npix2nside(npix) + logger.info("HEALPix data: Npix=%d, Nside=%d" % (npix, nside)) if nside != hp_header["NSIDE"]: raise ValueError("HEALPix data Nside does not match the header") hp_data = np.concatenate([hp_data, [np.nan]]) @@ -103,20 +108,24 @@ def hpx2healpix(data, header=None, append_history=None, append_comment=None): if isinstance(data, str): hpx_hdu = fits.open(data)[0] hpx_data, hpx_header = hpx_hdu.data, hpx_hdu.header + logger.info("Read HPX image from file") elif isinstance(data, fits.PrimaryHDU): hpx_data, hpx_header = data.data, data.header + logger.info("Read HPX image from HDU") else: hpx_data, hpx_header = np.asarray(data), fits.header.Header(header) + logger.info("Read HPX image from array and header") + logger.info("HPX coordinate system: ({0}, {1})".format( + hpx_header["CTYPE1"], hpx_header["CTYPE2"])) if ((hpx_header["CTYPE1"], hpx_header["CTYPE2"]) != ("GLON-HPX", "GLAT-HPX")): - logger.debug("Input coordinate system: ({0}, {1})".format( - hpx_header["CTYPE1"], hpx_header["CTYPE2"])) raise ValueError("only Galactic 'HPX' projection currently supported") # calculate Nside nside = round(hpx_header["NAXIS1"] / 5) nside2 = round(90 / np.sqrt(2) / hpx_header["CDELT2"]) if nside != nside2: raise ValueError("Cannot determine the Nside value") + logger.info("Determined HEALPix Nside=%d" % nside) # npix = hp.nside2npix(nside) hpx_idx = _calc_hpx_indices(nside).flatten() @@ -150,7 +159,7 @@ def _calc_hpx_indices(nside): ---- * The indices are 0-based; * Currently only HEALPix RING ordering supported; - * The null/empty elements in the HPX projection are filled with '-1'. + * The null/empty elements in the HPX projection are filled with "-1". """ # number of horizontal/vertical facet nfacet = 5 @@ -166,6 +175,7 @@ def _calc_hpx_indices(nside): # shape = (nfacet*nside, nfacet*nside) indices = -np.ones(shape).astype(np.int) + logger.info("HPX indices matrix shape: {0}".format(shape)) # # Loop vertically facet-by-facet for jfacet in range(nfacet): @@ -285,6 +295,7 @@ def _make_hpx_header(header, append_history=None, append_comment=None): "[deg] Galactic latitude at the reference point") header["PV2_1"] = (4, "HPX H parameter (longitude)") header["PV2_2"] = (3, "HPX K parameter (latitude)") + logger.info("Made HPX FITS header") # header["DATE"] = ( datetime.now(timezone.utc).astimezone().isoformat(), @@ -301,9 +312,11 @@ def _make_hpx_header(header, append_history=None, append_comment=None): header.add_comment(comment) # if append_history is not None: + logger.info("HPX FITS header: append history") for history in append_history: header.add_history(history) - if append_history is not None: + if append_comment is not None: + logger.info("HPX FITS header: append comments") for comment in append_comment: header.add_comment(comment) return header @@ -323,6 +336,7 @@ def _make_healpix_header(header, nside, header["NPIX"] = (npix, "Total number of pixels") header["FIRSTPIX"] = (0, "First pixel # (0 based)") header["LASTPIX"] = (npix-1, "Last pixel # (0 based)") + logger.info("Made HEALPix FITS header") # header["DATE"] = ( datetime.now(timezone.utc).astimezone().isoformat(), @@ -330,9 +344,11 @@ def _make_healpix_header(header, nside, ) # if append_history is not None: + logger.info("HEALPix FITS header: append history") for history in append_history: header.add_history(history) - if append_history is not None: + if append_comment is not None: + logger.info("HEALPix FITS header: append comments") for comment in append_comment: header.add_comment(comment) return header diff --git a/fg21sim/utils/logging.py b/fg21sim/utils/logging.py new file mode 100644 index 0000000..0abf4d3 --- /dev/null +++ b/fg21sim/utils/logging.py @@ -0,0 +1,85 @@ +# Copyright (c) 2016 Weitian LI <liweitianux@live.com> +# MIT license + +""" +Logging utilities. +""" + +import sys +import logging +from logging import FileHandler, StreamHandler + + +def setup_logging(dict_config=None, level=None, stream=None, logfile=None): + """Setup the logging. + This will override the logging configurations in the config file + if specified (e.g., by command line arguments). + + Parameters + ---------- + dict_config : dict + Dict of logging configurations specified in the config file. + level : string; + Override the existing log level + stream : string; "stderr", "stdout", or "" + This controls where the log messages go to. + If not None, then override the old ``StreamHandler`` settings; + if ``stream=""``, then disable the ``StreamHandler``. + logfile : string + Specify the file where the log messages go to. + If ``logfile=""``, then disable the ``FileHandler``. + + NOTE + ---- + If the logging already has ``StreamHandler`` or ``FileHandler`` + configured, then the old handler will be *replaced* (i.e., remove + the old one, then add the new one). + """ + # default file open mode for logging to file + filemode = "a" + # + if dict_config: + # XXX: + # "basicConfig()" does NOT accept paramter "filemode" if the + # corresponding parameter "filename" NOT specified. + filemode = dict_config.pop("filemode", filemode) + logging.basicConfig(**dict_config) + # + root_logger = logging.getLogger() + # + if level is not None: + level_int = getattr(logging, level.upper(), None) + if not isinstance(level_int, int): + raise ValueError("invalid log level: %s" % level) + root_logger.setLevel(level_int) + # + if stream is None: + pass + elif stream in ["", "stderr", "stdout"]: + for handler in root_logger.handlers: + if isinstance(handler, StreamHandler): + # remove old ``StreamHandler`` + root_logger.removeHandler(handler) + if stream == "": + # disable ``StreamHandler`` + pass + else: + # add new ``StreamHandler`` + handler = StreamHandler(getattr(sys, stream)) + root_logger.addHandler(handler) + else: + raise ValueError("invalid stream: %s" % stream) + # + if logfile is not None: + for handler in root_logger.handlers: + if isinstance(handler, FileHandler): + filemode = handler.mode + # remove old ``FileHandler`` + root_logger.removeHandler(handler) + if logfile == "": + # disable ``FileHandler`` + pass + else: + # add new ``FileHandler`` + handler = FileHandler(logfile, mode=filemode) + root_logger.addHandler(handler) |