diff options
| -rw-r--r-- | acispy/manifest.py | 429 | ||||
| -rw-r--r-- | acispy/results.py | 33 | ||||
| -rwxr-xr-x | scripts/manifest.py | 413 | ||||
| -rwxr-xr-x | scripts/results.py | 27 |
4 files changed, 468 insertions, 434 deletions
diff --git a/acispy/manifest.py b/acispy/manifest.py new file mode 100644 index 0000000..ae013ec --- /dev/null +++ b/acispy/manifest.py @@ -0,0 +1,429 @@ +# Copyright (c) 2017 Weitian LI <liweitianux@live.com> +# MIT license +# +# Weitian LI +# 2017-02-11 + +""" +Manage the observation manifest in YAML format. + +NOTE +---- +Use `ruamel.yaml`_ instead of `PyYAML`_ to preserve the comments +and other structures in the YAML file. + +.. _`ruamel.yaml`: https://bitbucket.org/ruamel/yaml +.. _`PyYAML`: http://pyyaml.org/ +""" + +import os +from collections import OrderedDict +import argparse + +import ruamel.yaml + + +class Manifest: + """ + Manage the observational products manifest. + """ + def __init__(self, filepath): + self.filepath = os.path.abspath(filepath) + self.manifest = ruamel.yaml.load( + open(filepath), Loader=ruamel.yaml.RoundTripLoader) + if self.manifest is None: + self.manifest = ruamel.yaml.comments.CommentedMap() + + def dump(self): + return ruamel.yaml.dump(self.manifest, + Dumper=ruamel.yaml.RoundTripDumper) + + def save(self): + with open(self.filepath, "w") as f: + f.write(self.dump()) + + def show(self): + print(self.dump()) + + def get(self, key): + """ + Get the value of the specified item in the manifest. + + Parameters + ---------- + key : str + The key of the item to be requested. + + Raises + ------ + KeyError : + If the specified item doesn't exist. + """ + if key in self.manifest: + return self.manifest[key] + else: + raise KeyError("manifest doesn't have item: '%s'" % key) + + def gets(self, keys, default=None, splitlist=False): + """ + Get the value of the specified item in the manifest. + + TODO: splitlist + + Parameters + ---------- + keys : list[str] + A list of keys specifying the items to be requested. + default : optional + The default value to return if the item not exists. + splitlist : bool, optional + Split the item value if it is a list, making it is easier + to export as CSV format. + + Returns + ------- + data : `~OrderedDict` + Ordered dictionary containing the requested items. + + Returns + ------- + """ + data = OrderedDict([ + (key, self.manifest.get(key, default)) for key in keys + ]) + if splitlist: + ds = OrderedDict() + for k, v in data.items(): + if isinstance(v, list): + for i, vi in enumerate(v): + ki = "{0}[{1}]".format(k, i) + ds[ki] = vi + else: + ds[k] = v + data = ds + return data + + def getpath(self, key, relative=False): + """ + Get the absolute path to the specified item by joining + with the location of this manifest file. + """ + value = self.get(key) + cwd = os.getcwd() + if isinstance(value, list): + path = [os.path.join(os.path.dirname(self.filepath), f) + for f in value] + if relative: + path = [os.path.relpath(p, start=cwd) for p in path] + else: + path = os.path.join(os.path.dirname(self.filepath), value) + if relative: + path = os.path.relpath(path, start=cwd) + return path + + def set(self, key, value): + """ + Set the value of the specified item in the manifest. + (Will add a new item or update an existing item.) + """ + self.manifest[key] = self.parse_value(value) + self.save() + + def setpath(self, key, path): + """ + Get the relative path of the given file w.r.t. this manifest file, + and set it to be the value of the specified item. + (Will add a new item or update an existing item.) + """ + dirname = os.path.dirname(self.filepath) + if isinstance(path, list): + abspath = [os.path.abspath(p) for p in path] + relpath = [os.path.relpath(p, start=dirname) for p in abspath] + else: + abspath = os.path.abspath(path) + relpath = os.path.relpath(abspath, start=dirname) + self.manifest[key] = relpath + self.save() + + def add(self, key, value): + """ + Add the specified new item in the manifest. + + If the specified item already exists, raise a ``KeyError``. + """ + if key in self.manifest: + raise KeyError("manifest already has item: '%s'" % key) + else: + self.set(key, value) + + def update(self, key, value): + """ + Update the specified existing item in the manifest. + + If the specified item doesn't exist, raise a ``KeyError``. + """ + if key in self.manifest: + self.set(key, value) + else: + raise KeyError("manifest doesn't have item: '%s'" % key) + + def delete(self, key): + """ + Delete the specified item from the manifest. + """ + del self.manifest[key] + self.save() + + @staticmethod + def parse_value(values): + """ + Try to parse the given (list of) value(s) from string to + integer or float. + """ + if not isinstance(values, list): + values = [values] + # + parsed_values = [] + for value in values: + try: + v = int(value) + except ValueError: + try: + v = float(value) + except ValueError: + # string/boolean + if value.lower() in ["true", "yes"]: + v = True + elif value.lower() in ["false", "no"]: + v = False + else: + v = value # string + parsed_values.append(v) + # + if len(parsed_values) == 1: + return parsed_values[0] + else: + return parsed_values + + +def find_manifest(filename="manifest.yaml", startdir=os.getcwd()): + """ + Find the specified manifest file in current directory and + the upper-level directories. + + Parameters + ---------- + filename : str, optional + Filename of the manifest file (default: ``manifest.yaml``) + + Returns + ------- + filepath : str + Absolute path to the manifest file if found. + + Raises + ------ + FileNotFoundError : + Cannot found the specified manifest + """ + dirname = startdir + filepath = os.path.join(dirname, filename) + while dirname != "/": + if os.path.exists(filepath): + return filepath + # go upper by one level + dirname = os.path.dirname(dirname) + filepath = os.path.join(dirname, filename) + # not found + raise FileNotFoundError("cannot found manifest file: %s" % filename) + + +def get_manifest(filename="manifest.yaml"): + """ + Find the manifest file and return the Manifest instance of it. + + Parameters + ---------- + filename : str, optional + Filename of the manifest file (default: ``manifest.yaml``) + + Returns + ------- + manifest : `~Manifest` + Manifest instance of the found manifest file. + """ + return Manifest(find_manifest(filename)) + + +# COMMAND-LINE INTERFACE ---------------------------------------------------- + +def cmd_show(args, manifest): + """ + Default sub-command "show": Show manifest contents. + """ + manifest.show() + + +def cmd_get(args, manifest): + """ + Sub-command "get": Get the value of an item in the manifest. + """ + if not args.brief: + print("%s:" % args.key, end=" ") + value = manifest.get(args.key) + if isinstance(value, list): + if args.field: + print(value[args.field-1]) + else: + print(args.separator.join(value)) + else: + print(value) + + +def cmd_getpath(args, manifest): + """ + Sub-command "getpath": Get the absolute path to the specified file + item in the manifest. + """ + if not args.brief: + print("%s:" % args.key, end=" ") + path = manifest.getpath(args.key, relative=args.relative) + if isinstance(path, list): + print(args.separator.join(path)) + else: + print(path) + + +def cmd_set(args, manifest): + """ + Sub-command "set": Set the value of an item in the manifest. + (Will add a new item or update an existing item.) + """ + manifest.set(args.key, args.value) + if not args.brief: + print("Set item '{0}': {1}".format(args.key, manifest.get(args.key))) + + +def cmd_setpath(args, manifest): + """ + Sub-command "setpath": Set the specified file item in the manifest + to be the relative path of the given file w.r.t. the manifest file. + """ + manifest.setpath(args.key, args.value) + if not args.brief: + print("Set file item '{0}': {1}".format(args.key, + manifest.get(args.key))) + + +def cmd_add(args, manifest): + """ + Sub-command "add": Add a new item to the manifest. + """ + manifest.add(args.key, args.value) + if not args.brief: + print("Added item '{0}': {1}".format(args.key, manifest.get(args.key))) + + +def cmd_update(args, manifest): + """ + Sub-command "update": Update the value of an existing item in the + manifest. + """ + value_old = manifest.get(args.key) + manifest.update(args.key, args.value) + if not args.brief: + print("Updated item '{0}': {1} -> {2}".format( + args.key, value_old, manifest.get(args.key))) + + +def cmd_delete(args, manifest): + """ + Sub-command "delete": Delete an item from the manifest. + """ + manifest.delete(args.key) + if not args.brief: + print("Deleted item: %s" % args.key) + + +def main(description="Manage the observation manifest (YAML format)", + default_file="manifest.yaml"): + parser = argparse.ArgumentParser(description=description) + parser.add_argument("-F", "--file", dest="file", default=default_file, + help="Manifest file (default: %s)" % default_file) + parser.add_argument("-b", "--brief", dest="brief", + action="store_true", help="Be brief") + parser.add_argument("-C", "--directory", dest="directory", default=".", + help="From where to find the manifest file") + parser.add_argument("-s", "--separator", dest="separator", default=" ", + help="Separator to join output list values " + + "(default: whitespace)") + subparsers = parser.add_subparsers(dest="cmd_name", + title="sub-commands", + help="additional help") + # sub-command: show + parser_show = subparsers.add_parser("show", help="Show manifest contents") + parser_show.set_defaults(func=cmd_show) + # sub-command: get + parser_get = subparsers.add_parser("get", help="Get an item from manifest") + parser_get.add_argument("-f", "--field", dest="field", type=int, + help="which field to get (default: all fields)") + parser_get.add_argument("key", help="key of the item") + parser_get.set_defaults(func=cmd_get) + # sub-command: getpath + parser_getpath = subparsers.add_parser( + "getpath", help="Get the path to a file item from manifest") + parser_getpath.add_argument("-r", "--relative", dest="relative", + action="store_true", + help="Return relative path w.r.t. current " + + "working directory instead of absolute path") + parser_getpath.add_argument("key", help="key of the file item") + parser_getpath.set_defaults(func=cmd_getpath) + # sub-command: set + parser_set = subparsers.add_parser( + "set", help="Set (add/update) an item in manifest") + parser_set.add_argument("key", help="key of the item") + parser_set.add_argument("value", nargs="+", + help="value of the item") + parser_set.set_defaults(func=cmd_set) + # sub-command: setpath + parser_setpath = subparsers.add_parser( + "setpath", help="Set a file item using relative path of given files") + parser_setpath.add_argument("key", help="key of the file item") + parser_setpath.add_argument("value", nargs="+", + help="paths to the files") + parser_setpath.set_defaults(func=cmd_setpath) + # sub-command: add + parser_add = subparsers.add_parser( + "add", help="Add a new item to manifest") + parser_add.add_argument("key", help="key of the item") + parser_add.add_argument("value", nargs="+", + help="value of the item") + parser_add.set_defaults(func=cmd_add) + # sub-command: update + parser_update = subparsers.add_parser( + "update", help="Update an existing item in manifest") + parser_update.add_argument("key", help="key of the item") + parser_update.add_argument("value", nargs="+", + help="new value of the item") + parser_update.set_defaults(func=cmd_update) + # sub-command: delete + parser_delete = subparsers.add_parser( + "delete", help="Delete item from manifest") + parser_delete.add_argument("key", help="key of the item") + parser_delete.set_defaults(func=cmd_delete) + # + args = parser.parse_args() + + if os.path.exists(args.file): + manifest_file = args.file + else: + manifest_file = find_manifest( + args.file, startdir=os.path.abspath(args.directory)) + + manifest = Manifest(manifest_file) + + if args.cmd_name: + # Dispatch sub-commands to call its specified function + args.func(args, manifest) + else: + cmd_show(None, manifest) diff --git a/acispy/results.py b/acispy/results.py new file mode 100644 index 0000000..5d454a6 --- /dev/null +++ b/acispy/results.py @@ -0,0 +1,33 @@ +# Copyright (c) 2017 Weitian LI <liweitianux@live.com> +# MIT license +# +# Weitian LI +# 2017-02-11 + +""" +Manage the analysis results in YAML format. +""" + +from . import manifest + + +def get_results(filename="results.yaml"): + """ + Find the results file and return the Manifest instance of it. + + Parameters + ---------- + filename : str, optional + Filename of the results file (default: ``results.yaml``) + + Returns + ------- + results : `~Manifest` + Manifest instance (i.e., results) of the found results file. + """ + return manifest.get_manifest(filename) + + +def main(description="Manage the analysis results (YAML format)", + default_file="results.yaml"): + manifest.main(description=description, default_file=default_file) diff --git a/scripts/manifest.py b/scripts/manifest.py index 4f932d0..df0361f 100755 --- a/scripts/manifest.py +++ b/scripts/manifest.py @@ -18,416 +18,9 @@ and other structures in the YAML file. .. _`PyYAML`: http://pyyaml.org/ """ -import os -import argparse -from collections import OrderedDict - -import ruamel.yaml - - -class Manifest: - """ - Manage the observational products manifest. - """ - def __init__(self, filepath): - self.filepath = os.path.abspath(filepath) - self.manifest = ruamel.yaml.load( - open(filepath), Loader=ruamel.yaml.RoundTripLoader) - if self.manifest is None: - self.manifest = ruamel.yaml.comments.CommentedMap() - - def dump(self): - return ruamel.yaml.dump(self.manifest, - Dumper=ruamel.yaml.RoundTripDumper) - - def save(self): - with open(self.filepath, "w") as f: - f.write(self.dump()) - - def show(self): - print(self.dump()) - - def get(self, key): - """ - Get the value of the specified item in the manifest. - - Parameters - ---------- - key : str - The key of the item to be requested. - - Raises - ------ - KeyError : - If the specified item doesn't exist. - """ - if key in self.manifest: - return self.manifest[key] - else: - raise KeyError("manifest doesn't have item: '%s'" % key) - - def gets(self, keys, default=None, splitlist=False): - """ - Get the value of the specified item in the manifest. - - TODO: splitlist - - Parameters - ---------- - keys : list[str] - A list of keys specifying the items to be requested. - default : optional - The default value to return if the item not exists. - splitlist : bool, optional - Split the item value if it is a list, making it is easier - to export as CSV format. - - Returns - ------- - data : `~OrderedDict` - Ordered dictionary containing the requested items. - - Returns - ------- - """ - data = OrderedDict([ - (key, self.manifest.get(key, default)) for key in keys - ]) - if splitlist: - ds = OrderedDict() - for k, v in data.items(): - if isinstance(v, list): - for i, vi in enumerate(v): - ki = "{0}[{1}]".format(k, i) - ds[ki] = vi - else: - ds[k] = v - data = ds - return data - - def getpath(self, key, relative=False): - """ - Get the absolute path to the specified item by joining - with the location of this manifest file. - """ - value = self.get(key) - cwd = os.getcwd() - if isinstance(value, list): - path = [os.path.join(os.path.dirname(self.filepath), f) - for f in value] - if relative: - path = [os.path.relpath(p, start=cwd) for p in path] - else: - path = os.path.join(os.path.dirname(self.filepath), value) - if relative: - path = os.path.relpath(path, start=cwd) - return path - - def set(self, key, value): - """ - Set the value of the specified item in the manifest. - (Will add a new item or update an existing item.) - """ - self.manifest[key] = self.parse_value(value) - self.save() - - def setpath(self, key, path): - """ - Get the relative path of the given file w.r.t. this manifest file, - and set it to be the value of the specified item. - (Will add a new item or update an existing item.) - """ - dirname = os.path.dirname(self.filepath) - if isinstance(path, list): - abspath = [os.path.abspath(p) for p in path] - relpath = [os.path.relpath(p, start=dirname) for p in abspath] - else: - abspath = os.path.abspath(path) - relpath = os.path.relpath(abspath, start=dirname) - self.manifest[key] = relpath - self.save() - - def add(self, key, value): - """ - Add the specified new item in the manifest. - - If the specified item already exists, raise a ``KeyError``. - """ - if key in self.manifest: - raise KeyError("manifest already has item: '%s'" % key) - else: - self.set(key, value) - - def update(self, key, value): - """ - Update the specified existing item in the manifest. - - If the specified item doesn't exist, raise a ``KeyError``. - """ - if key in self.manifest: - self.set(key, value) - else: - raise KeyError("manifest doesn't have item: '%s'" % key) - - def delete(self, key): - """ - Delete the specified item from the manifest. - """ - del self.manifest[key] - self.save() - - @staticmethod - def parse_value(values): - """ - Try to parse the given (list of) value(s) from string to - integer or float. - """ - if not isinstance(values, list): - values = [values] - # - parsed_values = [] - for value in values: - try: - v = int(value) - except ValueError: - try: - v = float(value) - except ValueError: - # string/boolean - if value.lower() in ["true", "yes"]: - v = True - elif value.lower() in ["false", "no"]: - v = False - else: - v = value # string - parsed_values.append(v) - # - if len(parsed_values) == 1: - return parsed_values[0] - else: - return parsed_values - - -def find_manifest(filename="manifest.yaml", startdir=os.getcwd()): - """ - Find the specified manifest file in current directory and - the upper-level directories. - - Parameters - ---------- - filename : str, optional - Filename of the manifest file (default: ``manifest.yaml``) - - Returns - ------- - filepath : str - Absolute path to the manifest file if found. - - Raises - ------ - FileNotFoundError : - Cannot found the specified manifest - """ - dirname = startdir - filepath = os.path.join(dirname, filename) - while dirname != "/": - if os.path.exists(filepath): - return filepath - # go upper by one level - dirname = os.path.dirname(dirname) - filepath = os.path.join(dirname, filename) - # not found - raise FileNotFoundError("cannot found manifest file: %s" % filename) - - -def get_manifest(filename="manifest.yaml"): - """ - Find the manifest file and return the Manifest instance of it. - - Parameters - ---------- - filename : str, optional - Filename of the manifest file (default: ``manifest.yaml``) - - Returns - ------- - manifest : `~Manifest` - Manifest instance of the found manifest file. - """ - return Manifest(find_manifest(filename)) - - -def cmd_show(args, manifest): - """ - Default sub-command "show": Show manifest contents. - """ - manifest.show() - - -def cmd_get(args, manifest): - """ - Sub-command "get": Get the value of an item in the manifest. - """ - if not args.brief: - print("%s:" % args.key, end=" ") - value = manifest.get(args.key) - if isinstance(value, list): - if args.field: - print(value[args.field-1]) - else: - print(args.separator.join(value)) - else: - print(value) - - -def cmd_getpath(args, manifest): - """ - Sub-command "getpath": Get the absolute path to the specified file - item in the manifest. - """ - if not args.brief: - print("%s:" % args.key, end=" ") - path = manifest.getpath(args.key, relative=args.relative) - if isinstance(path, list): - print(args.separator.join(path)) - else: - print(path) - - -def cmd_set(args, manifest): - """ - Sub-command "set": Set the value of an item in the manifest. - (Will add a new item or update an existing item.) - """ - manifest.set(args.key, args.value) - if not args.brief: - print("Set item '{0}': {1}".format(args.key, manifest.get(args.key))) - - -def cmd_setpath(args, manifest): - """ - Sub-command "setpath": Set the specified file item in the manifest - to be the relative path of the given file w.r.t. the manifest file. - """ - manifest.setpath(args.key, args.value) - if not args.brief: - print("Set file item '{0}': {1}".format(args.key, - manifest.get(args.key))) - - -def cmd_add(args, manifest): - """ - Sub-command "add": Add a new item to the manifest. - """ - manifest.add(args.key, args.value) - if not args.brief: - print("Added item '{0}': {1}".format(args.key, manifest.get(args.key))) - - -def cmd_update(args, manifest): - """ - Sub-command "update": Update the value of an existing item in the - manifest. - """ - value_old = manifest.get(args.key) - manifest.update(args.key, args.value) - if not args.brief: - print("Updated item '{0}': {1} -> {2}".format( - args.key, value_old, manifest.get(args.key))) - - -def cmd_delete(args, manifest): - """ - Sub-command "delete": Delete an item from the manifest. - """ - manifest.delete(args.key) - if not args.brief: - print("Deleted item: %s" % args.key) - - -def main(description="Manage the observation manifest (YAML format)", - default_file="manifest.yaml"): - parser = argparse.ArgumentParser(description=description) - parser.add_argument("-F", "--file", dest="file", default=default_file, - help="Manifest file (default: %s)" % default_file) - parser.add_argument("-b", "--brief", dest="brief", - action="store_true", help="Be brief") - parser.add_argument("-C", "--directory", dest="directory", default=".", - help="From where to find the manifest file") - parser.add_argument("-s", "--separator", dest="separator", default=" ", - help="Separator to join output list values " + - "(default: whitespace)") - subparsers = parser.add_subparsers(dest="cmd_name", - title="sub-commands", - help="additional help") - # sub-command: show - parser_show = subparsers.add_parser("show", help="Show manifest contents") - parser_show.set_defaults(func=cmd_show) - # sub-command: get - parser_get = subparsers.add_parser("get", help="Get an item from manifest") - parser_get.add_argument("-f", "--field", dest="field", type=int, - help="which field to get (default: all fields)") - parser_get.add_argument("key", help="key of the item") - parser_get.set_defaults(func=cmd_get) - # sub-command: getpath - parser_getpath = subparsers.add_parser( - "getpath", help="Get the path to a file item from manifest") - parser_getpath.add_argument("-r", "--relative", dest="relative", - action="store_true", - help="Return relative path w.r.t. current " + - "working directory instead of absolute path") - parser_getpath.add_argument("key", help="key of the file item") - parser_getpath.set_defaults(func=cmd_getpath) - # sub-command: set - parser_set = subparsers.add_parser( - "set", help="Set (add/update) an item in manifest") - parser_set.add_argument("key", help="key of the item") - parser_set.add_argument("value", nargs="+", - help="value of the item") - parser_set.set_defaults(func=cmd_set) - # sub-command: setpath - parser_setpath = subparsers.add_parser( - "setpath", help="Set a file item using relative path of given files") - parser_setpath.add_argument("key", help="key of the file item") - parser_setpath.add_argument("value", nargs="+", - help="paths to the files") - parser_setpath.set_defaults(func=cmd_setpath) - # sub-command: add - parser_add = subparsers.add_parser( - "add", help="Add a new item to manifest") - parser_add.add_argument("key", help="key of the item") - parser_add.add_argument("value", nargs="+", - help="value of the item") - parser_add.set_defaults(func=cmd_add) - # sub-command: update - parser_update = subparsers.add_parser( - "update", help="Update an existing item in manifest") - parser_update.add_argument("key", help="key of the item") - parser_update.add_argument("value", nargs="+", - help="new value of the item") - parser_update.set_defaults(func=cmd_update) - # sub-command: delete - parser_delete = subparsers.add_parser( - "delete", help="Delete item from manifest") - parser_delete.add_argument("key", help="key of the item") - parser_delete.set_defaults(func=cmd_delete) - # - args = parser.parse_args() - - if os.path.exists(args.file): - manifest_file = args.file - else: - manifest_file = find_manifest( - args.file, startdir=os.path.abspath(args.directory)) - - manifest = Manifest(manifest_file) - - if args.cmd_name: - # Dispatch sub-commands to call its specified function - args.func(args, manifest) - else: - cmd_show(None, manifest) +from context import acispy +from acispy import manifest if __name__ == "__main__": - main() + manifest.main() diff --git a/scripts/results.py b/scripts/results.py index 1bf504e..c3c2379 100755 --- a/scripts/results.py +++ b/scripts/results.py @@ -10,30 +10,9 @@ Manage the analysis results in YAML format. """ -import manifest - - -def get_results(filename="results.yaml"): - """ - Find the results file and return the Manifest instance of it. - - Parameters - ---------- - filename : str, optional - Filename of the results file (default: ``results.yaml``) - - Returns - ------- - results : `~Manifest` - Manifest instance (i.e., results) of the found results file. - """ - return manifest.get_manifest(filename) - - -def main(description="Manage the analysis results (YAML format)", - default_file="results.yaml"): - manifest.main(description=description, default_file=default_file) +from context import acispy +from acispy import results if __name__ == "__main__": - main() + results.main() |