<commit_before>#! /usr/bin/env python

"""
This project is about developing a dynamic CMD class based on cmd.CMD.
We assume the following directory structure::

  ./shell.py
  ./plugins/foo.py
  ./plugins/bar.py
  ./plugins/activate.py

We have provided examples of the classes in this document

foo and bar contain some classes that include he usual do_ methods. It
also includes an activate method that is called wih the acivation
module, so you can control its behavior upon startup.

To specify the plugins please use::

  plugins = ["foo", "bar","activate"]

Now you can set a name for your class::

  name = "CmCLI"

The rest is pretty simple::

  (cmd, plugin_objects) = DynamicCmd(name, plugins)
  cmd.activate(plugin_objects)
  cmd.cmdloop()

The activate method is not called automatically as to give more
flexibility during startup.

Here are the sample classes::

   class bar:

       def activate_bar(self):
           print "... activate bar"

       def do_that(self, arg):
           print "THAT", arg

   class foo:

       def do_this(self, arg):
           print "THIS", arg
           self.activate_status()

       def activate_foo(self):
           print "... activate foo"

   class activate:

       active = False

       def do_on(self, arg):
           self.active = True
           self.activate_status()

       def do_off(self, arg):
           self.active = False
           self.activate_status()

       def activate_status(self):
           print "Active:", self.active

       def activate(self, plugins):
           d = dir(self)
           result = []
           for key in d:
               if key.startswith("activate_"):
                   result.append(key)
           print result
           for key in result:
               print "> %s" % key.replace("_"," ")
               exec("self.%s()" % key)
"""
import traceback
import pkg_resources  # part of setuptools
import sys
import cmd
import readline
import glob
import os
import getopt
import textwrap
from docopt import docopt
import inspect
from pprint import pprint
from compiler.ast import flatten

echo = False

#
# SETTING UP A LOGGER
#

import logging
log = logging.getLogger('cmd3')
log.setLevel(logging.DEBUG)
formatter = logging.Formatter('CMD3: [%(levelname)s] %(message)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
log.addHandler(handler)

#
# DYNAMIC COMMAND MANAGEMENT
#
#
# Gregor von Laszewski
#
# code insired from cyberaide and cogkit, while trying to develop a
# dynamic CMD that loads from plugin directory
#

def DynamicCmd(name, plugins):

    exec('class %s(cmd.Cmd):\n    prompt="cm> "' % name)
    plugin_objects = []
    for plugin in plugins:
        classprefix = plugin['class']
        plugin_list = plugin['plugins']
        plugin_objects = plugin_objects + \
            load_plugins(classprefix, plugin_list)

    cmd = make_cmd_class(name, *plugin_objects)()
    return (cmd, plugin_objects)

def make_cmd_class(name, *bases):
    return type(cmd.Cmd)(name, bases + (cmd.Cmd,), {})

def get_plugins(dir):
    # not just plugin_*.py
    plugins = []
    list = glob.glob(dir + "/*.py")
    for p in list:
        p = p.replace(dir + "/", "").replace(".py", "")
        if not p.startswith('_'):
            plugins.append(p)
    #log.info("Loading Plugins from {0}".format(dir))
    #log.info("   {0}".format(str(plugins)))
    return plugins

def load_plugins(classprefix, list):
    # classprefix "cmd3.plugins."
    plugins = []
    object = {}
    #log.info(str(list))
    for plugin in list:
        try:
            object[plugin] = __import__(
                classprefix + "." + plugin, globals(), locals(), [plugin], -1)
            exec("cls = object['%s'].%s" % (plugin, plugin))
            plugins.append(cls)
        except:
            if echo:
                print "No module found", plugin, classprefix
    return plugins

#
# DECORATOR: COMMAND
#

def command(func):
    classname = inspect.getouterframes(inspect.currentframe())[1][3]
    name = func.__name__
    help_name = name.replace("do_", "help_")
    doc = textwrap.dedent(func.__doc__)

    def new(instance, args):
                # instance.new.__doc__ = doc
        try:
            arguments = docopt(doc, help=True, argv=args)
            func(instance, args, arguments)
        except SystemExit:
            if not args in ('-h', '--help'):
                print "Error: Wrong Format"
            print doc
    new.__doc__ = doc
    return new

#
# MAIN
#

def create_file(filename):
    expanded_filename = os.path.expanduser(os.path.expandvars(filename))
    if not os.path.exists(expanded_filename):
        open(expanded_filename, "a").close()

def create_dir(dir_path):
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)

def main():
    """cm.

    Usage:
      cm [-v] [--file=SCRIPT] [--interactive] [--quiet] [COMMAND ...]
      cm [-v] [-f SCRIPT] [-i] [-q] [COMMAND ...]

    Arguments:
      COMMAND                  A command to be executed

    Options:
      --file=SCRIPT -f SCRIPT  Executes the scipt
      --interactive -i         After start keep the shell interactive, otherwise quit
      --quiet       -q         Surpress some of the informational messages.
    """

    #    __version__ = pkg_resources.require("cmd3")[0].version
    #arguments = docopt(main.__doc__, help=True, version=__version__)

    arguments = docopt(main.__doc__, help=True)

    script_file = arguments['--file']
    interactive = arguments['--interactive']
    echo = arguments['-v']

    def get_plugins_from_dir(dir_path, classbase):
        """dir_path/classbase/plugins"""

        if dir_path == "sys":
            dir_path = os.path.abspath(
                os.path.join(os.path.dirname(__file__), 'plugins'))
            dir_plugins = get_plugins(dir_path)
            return {"dir": dir_path, "plugins": dir_plugins, "class": classbase}

        if dir_path == ".":
            dir_path = os.path.expanduser(
                os.path.expandvars(os.path.join(os.getcwd(), 'plugins')))
            dir_plugins = get_plugins(dir_path)
            return {"dir": dir_path, "plugins": dir_plugins, "class": classbase}
        else:

            dir_path = os.path.expanduser(os.path.expandvars(dir_path))
            prefix = "{0}/{1}".format(dir_path, classbase)

            user_path = "{0}/plugins".format(prefix)

            create_dir(user_path)
            create_file("{0}/__init__.py".format(prefix))
            create_file("{0}/plugins/__init__.py".format(prefix))
            sys.path.append(os.path.expanduser(dir_path))
            dir_plugins = get_plugins(user_path)
            return {"dir": dir_path, "plugins": dir_plugins, "class": classbase}

    plugins = []
    plugins.append(dict(get_plugins_from_dir("sys", "cmd3")))
    plugins.append(dict(get_plugins_from_dir("~/.futuregrid", "cmd3local")))
    #plugins.append(dict(get_plugins_from_dir (".", "dot")))

    for plugin in plugins:
        sys.path.append(os.path.expanduser(plugin['dir']))
    sys.path.append("../..")
    sys.path.append(".")
    sys.path.append("..")

    for plugin in plugins:
        plugin['class'] = plugin['class'] + ".plugins"

    #pprint(plugins)
    #pprint(sys.path)

    # sys.exit()
    name = "CmCli"

    #
    # not yet quite what i want, but falling back to a flatt array
    #

    (cmd, plugin_objects) = DynamicCmd(name, plugins)

    cmd.version()
    cmd.set_verbose(echo)
    cmd.activate()
    cmd.set_verbose(echo)
    cmd.do_exec(script_file)

    if len(arguments['COMMAND']) > 0:
        try:
            user_cmd = " ".join(arguments['COMMAND'])
            if echo:
                print ">", user_cmd
            cmd.onecmd(user_cmd)
        except Exception, e:
            print "'%s' is not recognized" % user_cmd
            print e
            print traceback.format_exc()
    elif not script_file or interactive:
        cmd.cmdloop()

def is_subcmd(opts, args):
    """if sys.argv[1:] does not match any getopt options,
    we simply assume it is a sub command to execute onecmd()"""

    if not opts and args:
        return True
    return False

if __name__ == "__main__":
    main()

"""
bash$ echo "help" | shell.py > output.txt

stupid example: shell.py metric analyze -y 2013

/trunk/cog-shell/src/cogkit/Shell/CoGCli.py?revision=3648&view=markup

386def runCLI(filename=None, silent=False, interactive=False):
387    if filename == None:
388        cli = CogShell(silent)
389        cli.cmdloop()
390    else:
391        cli = CogShell(silent=True)
392        cli.do_exec(filename)
393        if interactive:
394            cli.cmdloop()
395

69    cogkit.Shell.CoGCli.runCLI(script_file, quiet, interactive)
"""
<commit_msg>added comments<commit_after>#! /usr/bin/env python

"""
This project is about developing a dynamic CMD class based on cmd.CMD.
We assume the following directory structure::

  ./shell.py
  ./plugins/foo.py
  ./plugins/bar.py
  ./plugins/activate.py

We have provided examples of the classes in this document

foo and bar contain some classes that include he usual do_ methods. It
also includes an activate method that is called wih the acivation
module, so you can control its behavior upon startup.

To specify the plugins please use::

  plugins = ["foo", "bar","activate"]

Now you can set a name for your class::

  name = "CmCLI"

The rest is pretty simple::

  (cmd, plugin_objects) = DynamicCmd(name, plugins)
  cmd.activate(plugin_objects)
  cmd.cmdloop()

The activate method is not called automatically as to give more
flexibility during startup.

Here are the sample classes::

   class bar:

       def activate_bar(self):
           print "... activate bar"

       def do_that(self, arg):
           print "THAT", arg

   class foo:

       def do_this(self, arg):
           print "THIS", arg
           self.activate_status()

       def activate_foo(self):
           print "... activate foo"

   class activate:

       active = False

       def do_on(self, arg):
           self.active = True
           self.activate_status()

       def do_off(self, arg):
           self.active = False
           self.activate_status()

       def activate_status(self):
           print "Active:", self.active

       def activate(self, plugins):
           d = dir(self)
           result = []
           for key in d:
               if key.startswith("activate_"):
                   result.append(key)
           print result
           for key in result:
               print "> %s" % key.replace("_"," ")
               exec("self.%s()" % key)
"""
from compiler.ast import flatten
from docopt import docopt
from pprint import pprint
import cmd
import getopt
import glob
import inspect
import logging
import os
import pkg_resources # part of setuptools
import readline
import sys
import textwrap
import traceback

echo = False

#
# SETTING UP A LOGGER
#

log = logging.getLogger('cmd3')
log.setLevel(logging.DEBUG)
formatter = logging.Formatter('CMD3: [%(levelname)s] %(message)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
log.addHandler(handler)

#
# DYNAMIC COMMAND MANAGEMENT
#
#
# Gregor von Laszewski
#
# code insired from cyberaide and cogkit, while trying to develop a
# dynamic CMD that loads from plugin directory
#

def DynamicCmd(name, plugins):
    '''
    Returns a cmd with the added plugins,

    :param name: TODO:
    :param plugins: list of plugins
    '''
    exec('class %s(cmd.Cmd):\n    prompt="cm> "' % name)
    plugin_objects = []
    for plugin in plugins:
        classprefix = plugin['class']
        plugin_list = plugin['plugins']
        plugin_objects = plugin_objects + \
            load_plugins(classprefix, plugin_list)

    cmd = make_cmd_class(name, *plugin_objects)()
    return (cmd, plugin_objects)

def make_cmd_class(name, *bases):
    return type(cmd.Cmd)(name, bases + (cmd.Cmd,), {})

def get_plugins(dir):
    '''
    returns the list of plugins from the specified directory
    :param dir: directory that contains the plugins. Files starting with _ will be ignored.
    '''
    # not just plugin_*.py
    plugins = []
    list = glob.glob(dir + "/*.py")
    for p in list:
        p = p.replace(dir + "/", "").replace(".py", "")
        if not p.startswith('_'):
            plugins.append(p)
    #log.info("Loading Plugins from {0}".format(dir))
    #log.info("   {0}".format(str(plugins)))
    return plugins

def load_plugins(classprefix, list):
    '''
    loads the plugins specified in the list
    :param classprefix: the class prefix
    :param list: the list of plugins
    '''
    # classprefix "cmd3.plugins."
    plugins = []
    object = {}
    #log.info(str(list))
    for plugin in list:
        try:
            object[plugin] = __import__(
                classprefix + "." + plugin, globals(), locals(), [plugin], -1)
            exec("cls = object['%s'].%s" % (plugin, plugin))
            plugins.append(cls)
        except:
            if echo:
                print "No module found", plugin, classprefix
    return plugins

#
# DECORATOR: COMMAND
#

def command(func):
    '''
    A decorator to create a function with docopt arguments. It also generates a help function

    @command
    def do_myfunc(self, args):
        """ docopts text """
        pass

    will create

    def do_myfunc(self, args, arguments):
        """ docopts text """
        ...

    def help_myfunc(self, args, arguments):
        ... prints the docopt text ...

    :param func: the function for the decorator
    '''
    classname = inspect.getouterframes(inspect.currentframe())[1][3]
    name = func.__name__
    help_name = name.replace("do_", "help_")
    doc = textwrap.dedent(func.__doc__)

    def new(instance, args):
                # instance.new.__doc__ = doc
        try:
            arguments = docopt(doc, help=True, argv=args)
            func(instance, args, arguments)
        except SystemExit:
            if not args in ('-h', '--help'):
                print "Error: Wrong Format"
            print doc
    new.__doc__ = doc
    return new

#
# MAIN
#

def create_file(filename):
    '''
    Creates a new file if the file name does not exists
    :param filename: the name of the file
    '''

    expanded_filename = os.path.expanduser(os.path.expandvars(filename))
    if not os.path.exists(expanded_filename):
        open(expanded_filename, "a").close()

def create_dir(dir_path):
    '''
    creates a director at the given path
    :param dir_path: the directory path
    '''
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)

def main():
    """cm.

    Usage:
      cm [-v] [--file=SCRIPT] [--interactive] [--quiet] [COMMAND ...]
      cm [-v] [-f SCRIPT] [-i] [-q] [COMMAND ...]

    Arguments:
      COMMAND                  A command to be executed

    Options:
      --file=SCRIPT -f SCRIPT  Executes the scipt
      --interactive -i         After start keep the shell interactive, otherwise quit
      --quiet       -q         Surpress some of the informational messages.
    """

    #    __version__ = pkg_resources.require("cmd3")[0].version
    #arguments = docopt(main.__doc__, help=True, version=__version__)

    arguments = docopt(main.__doc__, help=True)

    script_file = arguments['--file']
    interactive = arguments['--interactive']
    echo = arguments['-v']

    def get_plugins_from_dir(dir_path, classbase):
        """dir_path/classbase/plugins"""

        if dir_path == "sys":
            dir_path = os.path.abspath(
                os.path.join(os.path.dirname(__file__), 'plugins'))
            dir_plugins = get_plugins(dir_path)
            return {"dir": dir_path, "plugins": dir_plugins, "class": classbase}

        if dir_path == ".":
            dir_path = os.path.expanduser(
                os.path.expandvars(os.path.join(os.getcwd(), 'plugins')))
            dir_plugins = get_plugins(dir_path)
            return {"dir": dir_path, "plugins": dir_plugins, "class": classbase}
        else:

            dir_path = os.path.expanduser(os.path.expandvars(dir_path))
            prefix = "{0}/{1}".format(dir_path, classbase)

            user_path = "{0}/plugins".format(prefix)

            create_dir(user_path)
            create_file("{0}/__init__.py".format(prefix))
            create_file("{0}/plugins/__init__.py".format(prefix))
            sys.path.append(os.path.expanduser(dir_path))
            dir_plugins = get_plugins(user_path)
            return {"dir": dir_path, "plugins": dir_plugins, "class": classbase}

    plugins = []
    plugins.append(dict(get_plugins_from_dir("sys", "cmd3")))
    plugins.append(dict(get_plugins_from_dir("~/.futuregrid", "cmd3local")))
    #plugins.append(dict(get_plugins_from_dir (".", "dot")))

    for plugin in plugins:
        sys.path.append(os.path.expanduser(plugin['dir']))
    sys.path.append("../..")
    sys.path.append(".")
    sys.path.append("..")

    for plugin in plugins:
        plugin['class'] = plugin['class'] + ".plugins"

    #pprint(plugins)
    #pprint(sys.path)

    # sys.exit()
    name = "CmCli"

    #
    # not yet quite what i want, but falling back to a flatt array
    #

    (cmd, plugin_objects) = DynamicCmd(name, plugins)

    cmd.version()
    cmd.set_verbose(echo)
    cmd.activate()
    cmd.set_verbose(echo)
    cmd.do_exec(script_file)

    if len(arguments['COMMAND']) > 0:
        try:
            user_cmd = " ".join(arguments['COMMAND'])
            if echo:
                print ">", user_cmd
            cmd.onecmd(user_cmd)
        except Exception, e:
            print "'%s' is not recognized" % user_cmd
            print e
            print traceback.format_exc()
    elif not script_file or interactive:
        cmd.cmdloop()

def is_subcmd(opts, args):
    """if sys.argv[1:] does not match any getopt options,
    we simply assume it is a sub command to execute onecmd()"""

    if not opts and args:
        return True
    return False

if __name__ == "__main__":
    main()

"""
bash$ echo "help" | shell.py > output.txt

stupid example: shell.py metric analyze -y 2013

/trunk/cog-shell/src/cogkit/Shell/CoGCli.py?revision=3648&view=markup

386def runCLI(filename=None, silent=False, interactive=False):
387    if filename == None:
388        cli = CogShell(silent)
389        cli.cmdloop()
390    else:
391        cli = CogShell(silent=True)
392        cli.do_exec(filename)
393        if interactive:
394            cli.cmdloop()
395

69    cogkit.Shell.CoGCli.runCLI(script_file, quiet, interactive)
"""
<|endoftext|>


<commit_before>"""
Codes for creating and manipulating gate filters. New functions: use of trained
Gaussian Mixture Models to remove noise and clutter from CPOL data before 2009.

@title: filtering
@author: Valentin Louf <valentin.louf@monash.edu>
@institutions: Monash University and the Australian Bureau of Meteorology
@date: 20/11/2017
@modification: 09/03/2020

.. autosummary::
    :toctree: generated/

    texture
    get_clustering
    get_gatefilter_GMM
    do_gatefilter_cpol
    do_gatefilter
"""
# Libraries
import os
import gzip
import pickle

import pyart
import netCDF4
import numpy as np
import pandas as pd

def texture(data):
    """
    Compute the texture of data.
    Compute the texture of the data by comparing values with a 3x3 neighborhood
    (based on :cite:`Gourley2007`). NaN values in the original array have
    NaN textures. (Wradlib function)

    Parameters:
    ==========
    data : :class:`numpy:numpy.ndarray`
        multi-dimensional array with shape (..., number of beams, number
        of range bins)

    Returns:
    =======
    texture : :class:`numpy:numpy.ndarray`
        array of textures with the same shape as data
    """
    x1 = np.roll(data, 1, -2)  # center:2
    x2 = np.roll(data, 1, -1)  # 4
    x3 = np.roll(data, -1, -2)  # 8
    x4 = np.roll(data, -1, -1)  # 6
    x5 = np.roll(x1, 1, -1)  # 1
    x6 = np.roll(x4, 1, -2)  # 3
    x7 = np.roll(x3, -1, -1)  # 9
    x8 = np.roll(x2, -1, -2)  # 7

    # at least one NaN would give a sum of NaN
    xa = np.array([x1, x2, x3, x4, x5, x6, x7, x8])

    # get count of valid neighboring pixels
    xa_valid = np.ones(np.shape(xa))
    xa_valid[np.isnan(xa)] = 0
    # count number of valid neighbors
    xa_valid_count = np.sum(xa_valid, axis=0)

    num = np.zeros(data.shape)
    for xarr in xa:
        diff = data - xarr
        # difference of NaNs will be converted to zero
        # (to not affect the summation)
        diff[np.isnan(diff)] = 0
        # only those with valid values are considered in the summation
        num += diff ** 2

    # reinforce that NaN values should have NaN textures
    num[np.isnan(data)] = np.nan

    return np.sqrt(num / xa_valid_count)

def get_clustering(radar, vel_name='VEL', phidp_name='PHIDP', zdr_name='ZDR'):
    '''
    get_clustering
    '''
    with gzip.GzipFile('GM_model_CPOL.pkl.gz', 'r') as gzid:
        gmm = pickle.load(gzid)

    df_orig = pd.DataFrame({'VEL': texture(radar.fields[vel_name]['data']).flatten(),
                            'PHIDP': texture(radar.fields[phidp_name]['data']).flatten(),
                            'ZDR': texture(radar.fields[zdr_name]['data']).flatten()})

    df = df_orig.dropna()
    pos_droped = df_orig.dropna().index
    clusters = gmm.predict(df)

    r = radar.range['data']
    time = radar.time['data']
    R, _ = np.meshgrid(r, time)

    clus = np.zeros_like(R.flatten())
    clus[pos_droped] = clusters + 1
    cluster = clus.reshape(R.shape)

    return cluster

def get_gatefilter_GMM(radar, refl_name='DBZ', vel_name='VEL', phidp_name='PHIDP', zdr_name='ZDR'):
    """
    Filtering function adapted to CPOL before 2009 using ML Gaussian Mixture
    Model. Function does 4 things:
    1) Cutoff of the reflectivities below the noise level.
    2) GMM using the texture of velocity, phidp and zdr.
    3) Filtering using 1) and 2) results.
    4) Removing temporary fields from the radar object.

    Parameters:
    ===========
    radar:
        Py-ART radar structure.
    refl_name: str
        Reflectivity field name.
    vel_name: str
        Velocity field name.
    phidp_name: str
        Name of the PHIDP field.
    zdr_name: str
        Name of the differential_reflectivity field.

    Returns:
    ========
    gf: GateFilter
        Gate filter (excluding all bad data).
    """
    # 1) Cutoff
    r = radar.range['data']
    azi = radar.azimuth['data']
    R, _ = np.meshgrid(r, azi)
    cutoff = 10 * np.log10(4e-5 * R)

    refl = radar.fields[refl_name]['data'].copy()
    refl[refl < cutoff] = np.NaN
    radar.add_field_like(refl_name, 'NPOS', refl, replace_existing=True)

    # 2) GMM clustering (indpdt from cutoff)
    cluster = get_clustering(radar, vel_name=vel_name, phidp_name=phidp_name, zdr_name=zdr_name)
    radar.add_field_like(refl_name, 'CLUS', cluster, replace_existing=True)

    pos = (cluster == 1) & (radar.fields[refl_name]['data'] < 20)
    radar.add_field_like(refl_name, 'TPOS', pos, replace_existing=True)

    # 3) Using GMM and Cutoff to filter.
    gf = pyart.filters.GateFilter(radar)
    gf.exclude_equal('CLUS', 5)
    gf.exclude_equal('CLUS', 2)
    gf.exclude_equal('TPOS', 1)
    gf.exclude_invalid('NPOS')
    gf = pyart.correct.despeckle_field(radar, 'DBZ', gatefilter=gf)

    # 4)  Removing temp files.
    for k in ['TPOS', 'NPOS', 'CLUS']:
        try:
            radar.pop(k)
        except KeyError:
            continue

    return gf

def do_gatefilter_cpol(radar,
                       refl_name='DBZ',
                       phidp_name="PHIDP",
                       rhohv_name='RHOHV_CORR',
                       zdr_name="ZDR",
                       snr_name='SNR',
                       vel_name='VEL'):
    """
    Filtering function adapted to CPOL.

    Parameters:
    ===========
        radar:
            Py-ART radar structure.
        refl_name: str
            Reflectivity field name.
        rhohv_name: str
            Cross correlation ratio field name.
        ncp_name: str
            Name of the normalized_coherent_power field.
        zdr_name: str
            Name of the differential_reflectivity field.

    Returns:
    ========
        gf_despeckeld: GateFilter
            Gate filter (excluding all bad data).
    """
    radar_start_date = netCDF4.num2date(radar.time['data'][0], radar.time['units'].replace("since", "since "))

    if radar_start_date.year < 2009:
        return get_gatefilter_GMM(radar,
                                  refl_name=refl_name,
                                  vel_name=vel_name,
                                  phidp_name=phidp_name,
                                  zdr_name=zdr_name)

    r = radar.range['data']
    azi = radar.azimuth['data']
    R, _ = np.meshgrid(r, azi)
    refl = radar.fields[refl_name]['data'].copy()
    fcut = 10 * np.log10(4e-5 * R)
    refl[refl < fcut] = np.NaN
    radar.add_field_like(refl_name, 'NDBZ', refl)

    gf = pyart.filters.GateFilter(radar)
    gf.exclude_invalid('NDBZ')
    gf.exclude_below(snr_name, 9)

    gf.exclude_outside(zdr_name, -3.0, 7.0)
    gf.exclude_outside(refl_name, -20.0, 80.0)

    # dphi = texture(radar.fields[phidp_name]['data'])
    # radar.add_field_like(phidp_name, 'PHITXT', dphi)
    # gf.exclude_above('PHITXT', 20)
    if radar_start_date.year > 2007:
        gf.exclude_below(rhohv_name, 0.5)

    # Remove rings in march 1999.
    if radar_start_date.year == 1999 and radar_start_date.month == 3:
        radar.add_field_like(refl_name, 'RRR', R)
        gf.exclude_above('RRR', 140e3)
        radar.fields.pop('RRR')

    gf_despeckeld = pyart.correct.despeckle_field(radar, refl_name, gatefilter=gf)

    # Remove tmp fields.
    try:
        radar.fields.pop('NDBZ')
        # radar.fields.pop('PHITXT')
    except Exception:
        pass

    return gf_despeckeld

def do_gatefilter(radar, refl_name='DBZ', phidp_name="PHIDP", rhohv_name='RHOHV_CORR', zdr_name="ZDR", snr_name='SNR'):
    """
    Basic filtering function for dual-polarisation data.

    Parameters:
    ===========
        radar:
            Py-ART radar structure.
        refl_name: str
            Reflectivity field name.
        rhohv_name: str
            Cross correlation ratio field name.
        ncp_name: str
            Name of the normalized_coherent_power field.
        zdr_name: str
            Name of the differential_reflectivity field.

    Returns:
    ========
        gf_despeckeld: GateFilter
            Gate filter (excluding all bad data).
    """
    # Initialize gatefilter
    gf = pyart.filters.GateFilter(radar)

    # Remove obviously wrong data.
    gf.exclude_outside(zdr_name, -6.0, 7.0)
    gf.exclude_outside(refl_name, -20.0, 80.0)

    # Compute texture of PHIDP and remove noise.
    dphi = texture(radar.fields[phidp_name]['data'])
    radar.add_field_like(phidp_name, 'PHITXT', dphi)
    gf.exclude_above('PHITXT', 20)
    gf.exclude_below(rhohv_name, 0.6)

    # Despeckle
    gf_despeckeld = pyart.correct.despeckle_field(radar, refl_name, gatefilter=gf)

    try:
        # Remove PHIDP texture
        radar.fields.pop('PHITXT')
    except Exception:
        pass

    return gf_despeckeld
<commit_msg>ADDED Comment<commit_after>"""
Codes for creating and manipulating gate filters. New functions: use of trained
Gaussian Mixture Models to remove noise and clutter from CPOL data before 2009.

@title: filtering
@author: Valentin Louf <valentin.louf@monash.edu>
@institutions: Monash University and the Australian Bureau of Meteorology
@date: 20/11/2017
@modification: 09/03/2020

.. autosummary::
    :toctree: generated/

    texture
    get_clustering
    get_gatefilter_GMM
    do_gatefilter_cpol
    do_gatefilter
"""
# Libraries
import os
import gzip
import pickle

import pyart
import netCDF4
import numpy as np
import pandas as pd

def texture(data):
    """
    Compute the texture of data.
    Compute the texture of the data by comparing values with a 3x3 neighborhood
    (based on :cite:`Gourley2007`). NaN values in the original array have
    NaN textures. (Wradlib function)

    Parameters:
    ==========
    data : :class:`numpy:numpy.ndarray`
        multi-dimensional array with shape (..., number of beams, number
        of range bins)

    Returns:
    =======
    texture : :class:`numpy:numpy.ndarray`
        array of textures with the same shape as data
    """
    x1 = np.roll(data, 1, -2)  # center:2
    x2 = np.roll(data, 1, -1)  # 4
    x3 = np.roll(data, -1, -2)  # 8
    x4 = np.roll(data, -1, -1)  # 6
    x5 = np.roll(x1, 1, -1)  # 1
    x6 = np.roll(x4, 1, -2)  # 3
    x7 = np.roll(x3, -1, -1)  # 9
    x8 = np.roll(x2, -1, -2)  # 7

    # at least one NaN would give a sum of NaN
    xa = np.array([x1, x2, x3, x4, x5, x6, x7, x8])

    # get count of valid neighboring pixels
    xa_valid = np.ones(np.shape(xa))
    xa_valid[np.isnan(xa)] = 0
    # count number of valid neighbors
    xa_valid_count = np.sum(xa_valid, axis=0)

    num = np.zeros(data.shape)
    for xarr in xa:
        diff = data - xarr
        # difference of NaNs will be converted to zero
        # (to not affect the summation)
        diff[np.isnan(diff)] = 0
        # only those with valid values are considered in the summation
        num += diff ** 2

    # reinforce that NaN values should have NaN textures
    num[np.isnan(data)] = np.nan

    return np.sqrt(num / xa_valid_count)

def get_clustering(radar, vel_name='VEL', phidp_name='PHIDP', zdr_name='ZDR'):
    '''
    Create cluster using a trained Gaussian Mixture Model (I use scikit-learn)
    to cluster the radar data. Cluster 5 is clutter and 2 is noise. Cluster 1
    correponds to a high gradient on PHIDP (folding), so it may corresponds to
    either real data that fold or noise. A threshold on reflectivity should be
    used on cluster 1.

    Parameters:
    ===========
    radar:
        Py-ART radar structure.
    vel_name: str
        Velocity field name.
    phidp_name: str
        Name of the PHIDP field.
    zdr_name: str
        Name of the differential_reflectivity field.

    Returns:
    ========
    cluster: ndarray
        Data ID using GMM (5: clutter, 2: noise, and 1: high-phidp gradient).
    '''
    with gzip.GzipFile('GM_model_CPOL.pkl.gz', 'r') as gzid:
        gmm = pickle.load(gzid)

    df_orig = pd.DataFrame({'VEL': texture(radar.fields[vel_name]['data']).flatten(),
                            'PHIDP': texture(radar.fields[phidp_name]['data']).flatten(),
                            'ZDR': texture(radar.fields[zdr_name]['data']).flatten()})

    df = df_orig.dropna()
    pos_droped = df_orig.dropna().index
    clusters = gmm.predict(df)

    r = radar.range['data']
    time = radar.time['data']
    R, _ = np.meshgrid(r, time)

    clus = np.zeros_like(R.flatten())
    clus[pos_droped] = clusters + 1
    cluster = clus.reshape(R.shape)

    return cluster

def get_gatefilter_GMM(radar, refl_name='DBZ', vel_name='VEL', phidp_name='PHIDP', zdr_name='ZDR'):
    """
    Filtering function adapted to CPOL before 2009 using ML Gaussian Mixture
    Model. Function does 4 things:
    1) Cutoff of the reflectivities below the noise level.
    2) GMM using the texture of velocity, phidp and zdr.
    3) Filtering using 1) and 2) results.
    4) Removing temporary fields from the radar object.

    Parameters:
    ===========
    radar:
        Py-ART radar structure.
    refl_name: str
        Reflectivity field name.
    vel_name: str
        Velocity field name.
    phidp_name: str
        Name of the PHIDP field.
    zdr_name: str
        Name of the differential_reflectivity field.

    Returns:
    ========
    gf: GateFilter
        Gate filter (excluding all bad data).
    """
    # 1) Cutoff
    r = radar.range['data']
    azi = radar.azimuth['data']
    R, _ = np.meshgrid(r, azi)
    cutoff = 10 * np.log10(4e-5 * R)

    refl = radar.fields[refl_name]['data'].copy()
    refl[refl < cutoff] = np.NaN
    radar.add_field_like(refl_name, 'NPOS', refl, replace_existing=True)

    # 2) GMM clustering (indpdt from cutoff)
    cluster = get_clustering(radar, vel_name=vel_name, phidp_name=phidp_name, zdr_name=zdr_name)
    radar.add_field_like(refl_name, 'CLUS', cluster, replace_existing=True)

    pos = (cluster == 1) & (radar.fields[refl_name]['data'] < 20)
    radar.add_field_like(refl_name, 'TPOS', pos, replace_existing=True)

    # 3) Using GMM and Cutoff to filter.
    gf = pyart.filters.GateFilter(radar)
    gf.exclude_equal('CLUS', 5)
    gf.exclude_equal('CLUS', 2)
    gf.exclude_equal('TPOS', 1)
    gf.exclude_invalid('NPOS')
    gf = pyart.correct.despeckle_field(radar, 'DBZ', gatefilter=gf)

    # 4)  Removing temp files.
    for k in ['TPOS', 'NPOS', 'CLUS']:
        try:
            radar.pop(k)
        except KeyError:
            continue

    return gf

def do_gatefilter_cpol(radar,
                       refl_name='DBZ',
                       phidp_name="PHIDP",
                       rhohv_name='RHOHV_CORR',
                       zdr_name="ZDR",
                       snr_name='SNR',
                       vel_name='VEL'):
    """
    Filtering function adapted to CPOL.

    Parameters:
    ===========
        radar:
            Py-ART radar structure.
        refl_name: str
            Reflectivity field name.
        rhohv_name: str
            Cross correlation ratio field name.
        ncp_name: str
            Name of the normalized_coherent_power field.
        zdr_name: str
            Name of the differential_reflectivity field.

    Returns:
    ========
        gf_despeckeld: GateFilter
            Gate filter (excluding all bad data).
    """
    radar_start_date = netCDF4.num2date(radar.time['data'][0], radar.time['units'].replace("since", "since "))

    if radar_start_date.year < 2009:
        return get_gatefilter_GMM(radar,
                                  refl_name=refl_name,
                                  vel_name=vel_name,
                                  phidp_name=phidp_name,
                                  zdr_name=zdr_name)

    r = radar.range['data']
    azi = radar.azimuth['data']
    R, _ = np.meshgrid(r, azi)
    refl = radar.fields[refl_name]['data'].copy()
    fcut = 10 * np.log10(4e-5 * R)
    refl[refl < fcut] = np.NaN
    radar.add_field_like(refl_name, 'NDBZ', refl)

    gf = pyart.filters.GateFilter(radar)
    gf.exclude_invalid('NDBZ')
    gf.exclude_below(snr_name, 9)

    gf.exclude_outside(zdr_name, -3.0, 7.0)
    gf.exclude_outside(refl_name, -20.0, 80.0)

    # dphi = texture(radar.fields[phidp_name]['data'])
    # radar.add_field_like(phidp_name, 'PHITXT', dphi)
    # gf.exclude_above('PHITXT', 20)
    if radar_start_date.year > 2007:
        gf.exclude_below(rhohv_name, 0.5)

    # Remove rings in march 1999.
    if radar_start_date.year == 1999 and radar_start_date.month == 3:
        radar.add_field_like(refl_name, 'RRR', R)
        gf.exclude_above('RRR', 140e3)
        radar.fields.pop('RRR')

    gf_despeckeld = pyart.correct.despeckle_field(radar, refl_name, gatefilter=gf)

    # Remove tmp fields.
    try:
        radar.fields.pop('NDBZ')
        # radar.fields.pop('PHITXT')
    except Exception:
        pass

    return gf_despeckeld

def do_gatefilter(radar, refl_name='DBZ', phidp_name="PHIDP", rhohv_name='RHOHV_CORR', zdr_name="ZDR", snr_name='SNR'):
    """
    Basic filtering function for dual-polarisation data.

    Parameters:
    ===========
        radar:
            Py-ART radar structure.
        refl_name: str
            Reflectivity field name.
        rhohv_name: str
            Cross correlation ratio field name.
        ncp_name: str
            Name of the normalized_coherent_power field.
        zdr_name: str
            Name of the differential_reflectivity field.

    Returns:
    ========
        gf_despeckeld: GateFilter
            Gate filter (excluding all bad data).
    """
    # Initialize gatefilter
    gf = pyart.filters.GateFilter(radar)

    # Remove obviously wrong data.
    gf.exclude_outside(zdr_name, -6.0, 7.0)
    gf.exclude_outside(refl_name, -20.0, 80.0)

    # Compute texture of PHIDP and remove noise.
    dphi = texture(radar.fields[phidp_name]['data'])
    radar.add_field_like(phidp_name, 'PHITXT', dphi)
    gf.exclude_above('PHITXT', 20)
    gf.exclude_below(rhohv_name, 0.6)

    # Despeckle
    gf_despeckeld = pyart.correct.despeckle_field(radar, refl_name, gatefilter=gf)

    try:
        # Remove PHIDP texture
        radar.fields.pop('PHITXT')
    except Exception:
        pass

    return gf_despeckeld
<|endoftext|>


<commit_before>#! /usr/bin/env python

"""
This project is about developing a dynamic CMD class based on cmd.CMD.
We assume the following directory structure::

  ./shell.py
  ./plugins/foo.py
  ./plugins/bar.py
  ./plugins/activate.py

We have provided examples of the classes in this document

foo and bar contain some classes that include he usual do_ methods. It
also includes an activate method that is called wih the acivation
module, so you can control its behavior upon startup.

To specify the plugins please use::

  plugins = ["foo", "bar","activate"]

Now you can set a name for your class::

  name = "CmCLI"

The rest is pretty simple::

  (cmd, plugin_objects) = DynamicCmd(name, plugins)
  cmd.activate(plugin_objects)
  cmd.cmdloop()

The activate method is not called automatically as to give more
flexibility during startup.

Here are the sample classes::

   class bar:

       def activate_bar(self):
           print "... activate bar"

       def do_that(self, arg):
           print "THAT", arg

   class foo:

       def do_this(self, arg):
           print "THIS", arg
           self.activate_status()

       def activate_foo(self):
           print "... activate foo"

   class activate:

       active = False

       def do_on(self, arg):
           self.active = True
           self.activate_status()

       def do_off(self, arg):
           self.active = False
           self.activate_status()

       def activate_status(self):
           print "Active:", self.active

       def activate(self, plugins):
           d = dir(self)
           result = []
           for key in d:
               if key.startswith("activate_"):
                   result.append(key)
           print result
           for key in result:
               print "> %s" % key.replace("_"," ")
               exec("self.%s()" % key)
"""
from __future__ import print_function
import textwrap
from compiler.ast import flatten
from docopt import docopt
from pprint import pprint
import cmd
import getopt
import glob
import inspect
import logging
import os
import pkg_resources  # part of setuptools
import readline
import sys
import textwrap
import traceback

#echo = False
echo = False

#
# SETTING UP A LOGGER
#

log = logging.getLogger('cmd3')
log.setLevel(logging.DEBUG)
formatter = logging.Formatter('CMD3: [%(levelname)s] %(message)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
log.addHandler(handler)

#
# DYNAMIC COMMAND MANAGEMENT
#
#
# Gregor von Laszewski
#
# code insired from cyberaide and cogkit, while trying to develop a
# dynamic CMD that loads from plugin directory
#

def get_version(self):
    # import pkg_resources  # part of setuptools
    # self.__version__ = pkg_resources.require("cmd3")[0].version
    import cmd3
    self.__version__ = cmd3.__version__
    return cmd3.__version__

def DynamicCmd(name, plugins):
    '''
    Returns a cmd with the added plugins,

    :param name: TODO:
    :param plugins: list of plugins
    '''
    exec('class %s(cmd.Cmd):\n    prompt="cm> "' % name)
    plugin_objects = []
    for plugin in plugins:
        classprefix = plugin['class']
        plugin_list = plugin['plugins']
        plugin_objects = plugin_objects + \
            load_plugins(classprefix, plugin_list)

    cmd = make_cmd_class(name, *plugin_objects)()
    return (cmd, plugin_objects)

def make_cmd_class(name, *bases):
    return type(cmd.Cmd)(name, bases + (cmd.Cmd,), {})

def get_plugins(dir):
    '''
    returns the list of plugins from the specified directory
    :param dir: directory that contains the plugins. Files starting with _ will be ignored.
    '''
    # not just plugin_*.py
    plugins = []
    list = glob.glob(dir + "/*.py")
    for p in list:
        p = p.replace(dir + "/", "").replace(".py", "")
        if not p.startswith('_'):
            plugins.append(p)
    # log.info("Loading Plugins from {0}".format(dir))
    # log.info("   {0}".format(str(plugins)))
    return plugins

def load_plugins(classprefix, list):
    '''
    loads the plugins specified in the list
    :param classprefix: the class prefix
    :param list: the list of plugins
    '''
    # classprefix "cmd3.plugins."
    plugins = []
    object = {}
    # log.info(str(list))
    for plugin in list:
        try:
            object[plugin] = __import__(
                classprefix + "." + plugin, globals(), locals(), [plugin], -1)
            exec("cls = object['%s'].%s" % (plugin, plugin))
            plugins.append(cls)
        except Exception, e:
            #if echo:
            print("ERROR: loading module ", plugin, classprefix)
            print(70 * "=")
            print(e)
            print(70 * "=")
            print(traceback.format_exc())
            print(70 * "-")
            print(sys.exc_info()[0])
            print(70 * "-")

    return plugins

#
# DECORATOR: COMMAND
#

def command(func):
    '''
    A decorator to create a function with docopt arguments. It also generates a help function

    @command
    def do_myfunc(self, args):
        """ docopts text """
        pass

    will create

    def do_myfunc(self, args, arguments):
        """ docopts text """
        ...

    def help_myfunc(self, args, arguments):
        ... prints the docopt text ...

    :param func: the function for the decorator
    '''
    classname = inspect.getouterframes(inspect.currentframe())[1][3]
    name = func.__name__
    help_name = name.replace("do_", "help_")
    doc = textwrap.dedent(func.__doc__)

    def new(instance, args):
                # instance.new.__doc__ = doc
        try:
            arguments = docopt(doc, help=True, argv=args)
            func(instance, args, arguments)
        except SystemExit:
            if not args in ('-h', '--help'):
                print("Error: Wrong Format")
            print(doc)
    new.__doc__ = doc
    return new
#
# DECORATOR: COMMAND
#

def function_command(main_func):
    def _function_command(func):
        '''
        A decorator to create a function with docopt arguments. It also generates a help function

        @command
        def do_myfunc(self, args):
            """ docopts text """
            pass

        will create

        def do_myfunc(self, args, arguments):
            """ docopts text """
            ...

        def help_myfunc(self, args, arguments):
            ... prints the docopt text ...

        :param func: the function for the decorator
        '''
        classname = inspect.getouterframes(inspect.currentframe())[1][3]
        name = func.__name__
        help_name = name.replace("do_", "help_")
        doc = textwrap.dedent(main_func.__doc__)

        def new(instance, args):
                    # instance.new.__doc__ = doc
            try:
                arguments = docopt(doc, help=True, argv=args)
                func(instance, args, arguments)
                #func.__doc__ = doc
            except SystemExit:
                if not args in ('-h', '--help'):
                    print("Error: Wrong Format")
                print(doc)
        new.__doc__ = doc
        return new
    return _function_command

#
# MAIN
#

def create_file(filename):
    '''
    Creates a new file if the file name does not exists
    :param filename: the name of the file
    '''

    expanded_filename = os.path.expanduser(os.path.expandvars(filename))
    if not os.path.exists(expanded_filename):
        open(expanded_filename, "a").close()

def create_dir(dir_path):
    '''
    creates a director at the given path
    :param dir_path: the directory path
    '''
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)

def get_plugins_from_dir(dir_path, classbase):
    """dir_path/classbase/plugins"""

    if dir_path == "sys":
        dir_path = os.path.abspath(
            os.path.join(os.path.dirname(__file__), 'plugins'))
        dir_plugins = get_plugins(dir_path)
        return {"dir": dir_path, "plugins": dir_plugins, "class": classbase}

    if dir_path == ".":
        dir_path = os.path.expanduser(
            os.path.expandvars(os.path.join(os.getcwd(), 'plugins')))
        dir_plugins = get_plugins(dir_path)
        return {"dir": dir_path, "plugins": dir_plugins, "class": classbase}
    else:

        dir_path = os.path.expanduser(os.path.expandvars(dir_path))
        prefix = "{0}/{1}".format(dir_path, classbase)

        user_path = "{0}/plugins".format(prefix)

        create_dir(user_path)
        create_file("{0}/__init__.py".format(prefix))
        create_file("{0}/plugins/__init__.py".format(prefix))
        sys.path.append(os.path.expanduser(dir_path))
        dir_plugins = get_plugins(user_path)
        # pprint({"dir": dir_path, "plugins": dir_plugins, "class": classbase})
        return {"dir": dir_path, "plugins": dir_plugins, "class": classbase}

def get_plugins_from_module(name):
    cmd3_module = __import__(name)
    location = os.path.dirname(cmd3_module.__file__)
    package_location = os.path.dirname(location)
    class_name = os.path.basename(location)
    return dict(get_plugins_from_dir(package_location, class_name))

class plugin_manager:

    def __init__():
        plugins = []

    def add(self, plugin):
        plugins.append(plugin)

def main():
    """cm.

    Usage:
      cm [-q] help
      cm [-v] [-b] [--file=SCRIPT] [-i] [COMMAND ...]

    Arguments:
      COMMAND                  A command to be executed

    Options:
      --file=SCRIPT  -f  SCRIPT  Executes the script
      -i                 After start keep the shell interactive,
                         otherwise quit [default: False]
      -b                 surpress the printing of the banner [default: False]
    """

    #    __version__ = pkg_resources.require("cmd3")[0].version
    # arguments = docopt(main.__doc__, help=True, version=__version__)

    arguments = docopt(main.__doc__, help=True)

    # fixing the help parameter parsing
    if arguments['help']:
        arguments['COMMAND'] = ['help']
        arguments['help'] = 'False'

    script_file = arguments['--file']
    interactive = arguments['-i']
    echo = arguments['-v']
    if echo:
        print(arguments)

    plugins = []

    plugins.append(dict(get_plugins_from_dir("sys", "cmd3")))
    # plugins.append(dict(get_plugins_from_dir("~/.cloudmesh", "cmd3local")))

    modules = ['cloudmesh_cmd3.plugins',
               'cloudmesh_docker.plugins']
               'cloudmesh_cluster.plugins']
    for module_name in modules:
        # print "INSTALL", module_name
        try:
            plugins.append(dict(get_plugins_from_module(module_name)))
        except:
            #print "WARNING: could not find", module_name
            pass

    #sys.exit()
    #plugins.append(dict(get_plugins_from_dir("~/.cloudmesh", "cmd3local")))

    # plugins.append(dict(get_plugins_from_dir (".", "dot")))

    for plugin in plugins:
        sys.path.append(os.path.expanduser(plugin['dir']))
    sys.path.append("../..")
    sys.path.append(".")
    sys.path.append("..")

    for plugin in plugins:
        plugin['class'] = plugin['class'] + ".plugins"

    # pprint(plugins)
    # pprint(sys.path)

    # sys.exit()
    name = "CmCli"

    #
    # not yet quite what i want, but falling back to a flatt array
    #

    (cmd, plugin_objects) = DynamicCmd(name, plugins)

    cmd.get_version()
    cmd.set_verbose(echo)
    cmd.activate()
    cmd.set_verbose(echo)

    if arguments['-b']:
        cmd.set_banner("")
    if script_file is not None:
        cmd.do_exec(script_file)

    if len(arguments['COMMAND']) > 0:
        try:
            user_cmd = " ".join(arguments['COMMAND'])
            if echo:
                print(">", user_cmd)
            cmd.onecmd(user_cmd)
        except Exception, e:
            print()
            print("ERROR: executing command '{0}'".format(user_cmd))
            print()
            print(e)
            print(traceback.format_exc())
        if interactive:
            cmd.cmdloop()

    elif not script_file or interactive:
        cmd.cmdloop()

def is_subcmd(opts, args):
    """if sys.argv[1:] does not match any getopt options,
    we simply assume it is a sub command to execute onecmd()"""

    if not opts and args:
        return True
    return False

def cmd3_call(f):
    """ calls a function defined with cmd3 arguments the function must be
        defined as f(arguments)"""

    arguments = docopt(f.__doc__)
    f(arguments)

if __name__ == "__main__":
    main()

"""
bash$ echo "help" | shell.py > output.txt

stupid example: shell.py metric analyze -y 2013

/trunk/cog-shell/src/cogkit/Shell/CoGCli.py?revision=3648&view=markup

386def runCLI(filename=None, silent=False, interactive=False):
387    if filename == None:
388        cli = CogShell(silent)
389        cli.cmdloop()
390    else:
391        cli = CogShell(silent=True)
392        cli.do_exec(filename)
393        if interactive:
394            cli.cmdloop()
395

69    cogkit.Shell.CoGCli.runCLI(script_file, quiet, interactive)
"""
<commit_msg>missing ,<commit_after>#! /usr/bin/env python

"""
This project is about developing a dynamic CMD class based on cmd.CMD.
We assume the following directory structure::

  ./shell.py
  ./plugins/foo.py
  ./plugins/bar.py
  ./plugins/activate.py

We have provided examples of the classes in this document

foo and bar contain some classes that include he usual do_ methods. It
also includes an activate method that is called wih the acivation
module, so you can control its behavior upon startup.

To specify the plugins please use::

  plugins = ["foo", "bar","activate"]

Now you can set a name for your class::

  name = "CmCLI"

The rest is pretty simple::

  (cmd, plugin_objects) = DynamicCmd(name, plugins)
  cmd.activate(plugin_objects)
  cmd.cmdloop()

The activate method is not called automatically as to give more
flexibility during startup.

Here are the sample classes::

   class bar:

       def activate_bar(self):
           print "... activate bar"

       def do_that(self, arg):
           print "THAT", arg

   class foo:

       def do_this(self, arg):
           print "THIS", arg
           self.activate_status()

       def activate_foo(self):
           print "... activate foo"

   class activate:

       active = False

       def do_on(self, arg):
           self.active = True
           self.activate_status()

       def do_off(self, arg):
           self.active = False
           self.activate_status()

       def activate_status(self):
           print "Active:", self.active

       def activate(self, plugins):
           d = dir(self)
           result = []
           for key in d:
               if key.startswith("activate_"):
                   result.append(key)
           print result
           for key in result:
               print "> %s" % key.replace("_"," ")
               exec("self.%s()" % key)
"""
from __future__ import print_function
import textwrap
from compiler.ast import flatten
from docopt import docopt
from pprint import pprint
import cmd
import getopt
import glob
import inspect
import logging
import os
import pkg_resources  # part of setuptools
import readline
import sys
import textwrap
import traceback

#echo = False
echo = False

#
# SETTING UP A LOGGER
#

log = logging.getLogger('cmd3')
log.setLevel(logging.DEBUG)
formatter = logging.Formatter('CMD3: [%(levelname)s] %(message)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
log.addHandler(handler)

#
# DYNAMIC COMMAND MANAGEMENT
#
#
# Gregor von Laszewski
#
# code insired from cyberaide and cogkit, while trying to develop a
# dynamic CMD that loads from plugin directory
#

def get_version(self):
    # import pkg_resources  # part of setuptools
    # self.__version__ = pkg_resources.require("cmd3")[0].version
    import cmd3
    self.__version__ = cmd3.__version__
    return cmd3.__version__

def DynamicCmd(name, plugins):
    '''
    Returns a cmd with the added plugins,

    :param name: TODO:
    :param plugins: list of plugins
    '''
    exec('class %s(cmd.Cmd):\n    prompt="cm> "' % name)
    plugin_objects = []
    for plugin in plugins:
        classprefix = plugin['class']
        plugin_list = plugin['plugins']
        plugin_objects = plugin_objects + \
            load_plugins(classprefix, plugin_list)

    cmd = make_cmd_class(name, *plugin_objects)()
    return (cmd, plugin_objects)

def make_cmd_class(name, *bases):
    return type(cmd.Cmd)(name, bases + (cmd.Cmd,), {})

def get_plugins(dir):
    '''
    returns the list of plugins from the specified directory
    :param dir: directory that contains the plugins. Files starting with _ will be ignored.
    '''
    # not just plugin_*.py
    plugins = []
    list = glob.glob(dir + "/*.py")
    for p in list:
        p = p.replace(dir + "/", "").replace(".py", "")
        if not p.startswith('_'):
            plugins.append(p)
    # log.info("Loading Plugins from {0}".format(dir))
    # log.info("   {0}".format(str(plugins)))
    return plugins

def load_plugins(classprefix, list):
    '''
    loads the plugins specified in the list
    :param classprefix: the class prefix
    :param list: the list of plugins
    '''
    # classprefix "cmd3.plugins."
    plugins = []
    object = {}
    # log.info(str(list))
    for plugin in list:
        try:
            object[plugin] = __import__(
                classprefix + "." + plugin, globals(), locals(), [plugin], -1)
            exec("cls = object['%s'].%s" % (plugin, plugin))
            plugins.append(cls)
        except Exception, e:
            #if echo:
            print("ERROR: loading module ", plugin, classprefix)
            print(70 * "=")
            print(e)
            print(70 * "=")
            print(traceback.format_exc())
            print(70 * "-")
            print(sys.exc_info()[0])
            print(70 * "-")

    return plugins

#
# DECORATOR: COMMAND
#

def command(func):
    '''
    A decorator to create a function with docopt arguments. It also generates a help function

    @command
    def do_myfunc(self, args):
        """ docopts text """
        pass

    will create

    def do_myfunc(self, args, arguments):
        """ docopts text """
        ...

    def help_myfunc(self, args, arguments):
        ... prints the docopt text ...

    :param func: the function for the decorator
    '''
    classname = inspect.getouterframes(inspect.currentframe())[1][3]
    name = func.__name__
    help_name = name.replace("do_", "help_")
    doc = textwrap.dedent(func.__doc__)

    def new(instance, args):
                # instance.new.__doc__ = doc
        try:
            arguments = docopt(doc, help=True, argv=args)
            func(instance, args, arguments)
        except SystemExit:
            if not args in ('-h', '--help'):
                print("Error: Wrong Format")
            print(doc)
    new.__doc__ = doc
    return new
#
# DECORATOR: COMMAND
#

def function_command(main_func):
    def _function_command(func):
        '''
        A decorator to create a function with docopt arguments. It also generates a help function

        @command
        def do_myfunc(self, args):
            """ docopts text """
            pass

        will create

        def do_myfunc(self, args, arguments):
            """ docopts text """
            ...

        def help_myfunc(self, args, arguments):
            ... prints the docopt text ...

        :param func: the function for the decorator
        '''
        classname = inspect.getouterframes(inspect.currentframe())[1][3]
        name = func.__name__
        help_name = name.replace("do_", "help_")
        doc = textwrap.dedent(main_func.__doc__)

        def new(instance, args):
                    # instance.new.__doc__ = doc
            try:
                arguments = docopt(doc, help=True, argv=args)
                func(instance, args, arguments)
                #func.__doc__ = doc
            except SystemExit:
                if not args in ('-h', '--help'):
                    print("Error: Wrong Format")
                print(doc)
        new.__doc__ = doc
        return new
    return _function_command

#
# MAIN
#

def create_file(filename):
    '''
    Creates a new file if the file name does not exists
    :param filename: the name of the file
    '''

    expanded_filename = os.path.expanduser(os.path.expandvars(filename))
    if not os.path.exists(expanded_filename):
        open(expanded_filename, "a").close()

def create_dir(dir_path):
    '''
    creates a director at the given path
    :param dir_path: the directory path
    '''
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)

def get_plugins_from_dir(dir_path, classbase):
    """dir_path/classbase/plugins"""

    if dir_path == "sys":
        dir_path = os.path.abspath(
            os.path.join(os.path.dirname(__file__), 'plugins'))
        dir_plugins = get_plugins(dir_path)
        return {"dir": dir_path, "plugins": dir_plugins, "class": classbase}

    if dir_path == ".":
        dir_path = os.path.expanduser(
            os.path.expandvars(os.path.join(os.getcwd(), 'plugins')))
        dir_plugins = get_plugins(dir_path)
        return {"dir": dir_path, "plugins": dir_plugins, "class": classbase}
    else:

        dir_path = os.path.expanduser(os.path.expandvars(dir_path))
        prefix = "{0}/{1}".format(dir_path, classbase)

        user_path = "{0}/plugins".format(prefix)

        create_dir(user_path)
        create_file("{0}/__init__.py".format(prefix))
        create_file("{0}/plugins/__init__.py".format(prefix))
        sys.path.append(os.path.expanduser(dir_path))
        dir_plugins = get_plugins(user_path)
        # pprint({"dir": dir_path, "plugins": dir_plugins, "class": classbase})
        return {"dir": dir_path, "plugins": dir_plugins, "class": classbase}

def get_plugins_from_module(name):
    cmd3_module = __import__(name)
    location = os.path.dirname(cmd3_module.__file__)
    package_location = os.path.dirname(location)
    class_name = os.path.basename(location)
    return dict(get_plugins_from_dir(package_location, class_name))

class plugin_manager:

    def __init__():
        plugins = []

    def add(self, plugin):
        plugins.append(plugin)

def main():
    """cm.

    Usage:
      cm [-q] help
      cm [-v] [-b] [--file=SCRIPT] [-i] [COMMAND ...]

    Arguments:
      COMMAND                  A command to be executed

    Options:
      --file=SCRIPT  -f  SCRIPT  Executes the script
      -i                 After start keep the shell interactive,
                         otherwise quit [default: False]
      -b                 surpress the printing of the banner [default: False]
    """

    #    __version__ = pkg_resources.require("cmd3")[0].version
    # arguments = docopt(main.__doc__, help=True, version=__version__)

    arguments = docopt(main.__doc__, help=True)

    # fixing the help parameter parsing
    if arguments['help']:
        arguments['COMMAND'] = ['help']
        arguments['help'] = 'False'

    script_file = arguments['--file']
    interactive = arguments['-i']
    echo = arguments['-v']
    if echo:
        print(arguments)

    plugins = []

    plugins.append(dict(get_plugins_from_dir("sys", "cmd3")))
    # plugins.append(dict(get_plugins_from_dir("~/.cloudmesh", "cmd3local")))

    modules = ['cloudmesh_cmd3.plugins',
               'cloudmesh_docker.plugins',
               'cloudmesh_cluster.plugins']
    for module_name in modules:
        # print "INSTALL", module_name
        try:
            plugins.append(dict(get_plugins_from_module(module_name)))
        except:
            #print "WARNING: could not find", module_name
            pass

    #sys.exit()
    #plugins.append(dict(get_plugins_from_dir("~/.cloudmesh", "cmd3local")))

    # plugins.append(dict(get_plugins_from_dir (".", "dot")))

    for plugin in plugins:
        sys.path.append(os.path.expanduser(plugin['dir']))
    sys.path.append("../..")
    sys.path.append(".")
    sys.path.append("..")

    for plugin in plugins:
        plugin['class'] = plugin['class'] + ".plugins"

    # pprint(plugins)
    # pprint(sys.path)

    # sys.exit()
    name = "CmCli"

    #
    # not yet quite what i want, but falling back to a flatt array
    #

    (cmd, plugin_objects) = DynamicCmd(name, plugins)

    cmd.get_version()
    cmd.set_verbose(echo)
    cmd.activate()
    cmd.set_verbose(echo)

    if arguments['-b']:
        cmd.set_banner("")
    if script_file is not None:
        cmd.do_exec(script_file)

    if len(arguments['COMMAND']) > 0:
        try:
            user_cmd = " ".join(arguments['COMMAND'])
            if echo:
                print(">", user_cmd)
            cmd.onecmd(user_cmd)
        except Exception, e:
            print()
            print("ERROR: executing command '{0}'".format(user_cmd))
            print()
            print(e)
            print(traceback.format_exc())
        if interactive:
            cmd.cmdloop()

    elif not script_file or interactive:
        cmd.cmdloop()

def is_subcmd(opts, args):
    """if sys.argv[1:] does not match any getopt options,
    we simply assume it is a sub command to execute onecmd()"""

    if not opts and args:
        return True
    return False

def cmd3_call(f):
    """ calls a function defined with cmd3 arguments the function must be
        defined as f(arguments)"""

    arguments = docopt(f.__doc__)
    f(arguments)

if __name__ == "__main__":
    main()

"""
bash$ echo "help" | shell.py > output.txt

stupid example: shell.py metric analyze -y 2013

/trunk/cog-shell/src/cogkit/Shell/CoGCli.py?revision=3648&view=markup

386def runCLI(filename=None, silent=False, interactive=False):
387    if filename == None:
388        cli = CogShell(silent)
389        cli.cmdloop()
390    else:
391        cli = CogShell(silent=True)
392        cli.do_exec(filename)
393        if interactive:
394            cli.cmdloop()
395

69    cogkit.Shell.CoGCli.runCLI(script_file, quiet, interactive)
"""
<|endoftext|>


<commit_before>import dicom
import dicom.errors
import exifread
import logger
from geopy.geocoders import Nominatim
import skimage
import skimage.io
import skimage.exposure
import skimage.feature
import skimage.color

def convert_exif_dms_to_decimal(dms_array, ref):
    assert isinstance(dms_array, list)
    assert isinstance(ref, basestring)

    decimal_number = ((float(dms_array[0].num) / float(dms_array[0].den)) +
                      (float(dms_array[1].num) / (float(dms_array[1].den) * 60.0)) +
                      (float(dms_array[2].num) / (float(dms_array[2].den) * 3600.0)))
    if ref.upper() == ('S' or 'W'):
        decimal_number *= -1
    return decimal_number

class EXIFReader(object):
    def __init__(self):
        self.geolocator = Nominatim()

    @staticmethod
    def read_exif_tags(filepath):
        with open(filepath, 'rb') as fp:
            data = exifread.process_file(fp)
        return data

    def retrieve_location(self, tags):
        lat = convert_exif_dms_to_decimal(tags['GPS GPSLatitude'].values, tags['GPS GPSLatitudeRef'].values)
        lon = convert_exif_dms_to_decimal(tags['GPS GPSLongitude'].values, tags['GPS GPSLongitudeRef'].values)
        location = self.geolocator.reverse((lat, lon))
        return location

    def retrieve_exif_information_strings(self, filepath):
        tags = self.read_exif_tags(filepath)
        exif_string = exifread.make_string(tags)

        if all(k in tags for k in ('GPS GPSLatitude', 'GPS GPSLatitudeRef',
                                   'GPS GPSLongitude', 'GPS GPSLongitudeRef')):
            address = self.retrieve_location(tags).address
        else:
            address = ''
        return exif_string, address

def read_dicom_file_meta(filename):
    try:
        with open(filename, 'rb') as fp:
            data = dicom.read_file(fp)
        return str(data)
    except dicom.errors.InvalidDicomError as err:
        logger.Logger.warning('DICOM data could not be retrieved: {}'.format(err))
        return ''

def rgb_histogram(filename, normalize=True):

    image = skimage.io.imread(filename)
    imagef = skimage.img_as_float(image)

    r, g, b = imagef[:, :, 0].copy(), imagef[:, :, 1].copy(), imagef[:, :, 2].copy()
    rh, gh, bh, = map(skimage.exposure.histogram, (r.copy(), g.copy(), b.copy()))
    rhs, ghs, bhs = map(lambda x: sum(x[0]*x[1]), (rh, gh, bh))

    if normalize:
        s = sum((rhs, ghs, bhs))
        rhs, ghs, bhs = map(lambda x: x/s, (rhs, ghs, bhs))

    return rhs, ghs, bhs

def is_low_contrast(filename):
    image = skimage.io.imread(filename)
    return skimage.exposure.is_low_contrast(image)

def censure(filename):
    """
    The CENSURE feature detector is a scale-invariant center-surround detector (CENSURE) that claims to outperform
    other detectors and is capable of real-time implementation.
    """

    detector = skimage.feature.CENSURE()
    image = skimage.color.rgb2gray(skimage.io.imread(filename))
    detector.detect(image)
    return detector.keypoints

<commit_msg>close file<commit_after>import dicom
import dicom.errors
import exifread
import logger
from geopy.geocoders import Nominatim
import skimage
import skimage.io
import skimage.exposure
import skimage.feature
import skimage.color

def convert_exif_dms_to_decimal(dms_array, ref):
    assert isinstance(dms_array, list)
    assert isinstance(ref, basestring)

    decimal_number = ((float(dms_array[0].num) / float(dms_array[0].den)) +
                      (float(dms_array[1].num) / (float(dms_array[1].den) * 60.0)) +
                      (float(dms_array[2].num) / (float(dms_array[2].den) * 3600.0)))
    if ref.upper() == ('S' or 'W'):
        decimal_number *= -1
    return decimal_number

class EXIFReader(object):
    def __init__(self):
        self.geolocator = Nominatim()

    @staticmethod
    def read_exif_tags(filepath):
        with open(filepath, 'rb') as fp:
            data = exifread.process_file(fp)
            fp.close()
        return data

    def retrieve_location(self, tags):
        lat = convert_exif_dms_to_decimal(tags['GPS GPSLatitude'].values, tags['GPS GPSLatitudeRef'].values)
        lon = convert_exif_dms_to_decimal(tags['GPS GPSLongitude'].values, tags['GPS GPSLongitudeRef'].values)
        location = self.geolocator.reverse((lat, lon))
        return location

    def retrieve_exif_information_strings(self, filepath):
        tags = self.read_exif_tags(filepath)
        exif_string = exifread.make_string(tags)

        if all(k in tags for k in ('GPS GPSLatitude', 'GPS GPSLatitudeRef',
                                   'GPS GPSLongitude', 'GPS GPSLongitudeRef')):
            address = self.retrieve_location(tags).address
        else:
            address = ''
        return exif_string, address

def read_dicom_file_meta(filename):
    try:
        with open(filename, 'rb') as fp:
            data = dicom.read_file(fp)
        return str(data)
    except dicom.errors.InvalidDicomError as err:
        logger.Logger.warning('DICOM data could not be retrieved: {}'.format(err))
        return ''

def rgb_histogram(filename, normalize=True):

    image = skimage.io.imread(filename)
    imagef = skimage.img_as_float(image)

    r, g, b = imagef[:, :, 0].copy(), imagef[:, :, 1].copy(), imagef[:, :, 2].copy()
    rh, gh, bh, = map(skimage.exposure.histogram, (r.copy(), g.copy(), b.copy()))
    rhs, ghs, bhs = map(lambda x: sum(x[0]*x[1]), (rh, gh, bh))

    if normalize:
        s = sum((rhs, ghs, bhs))
        rhs, ghs, bhs = map(lambda x: x/s, (rhs, ghs, bhs))

    return rhs, ghs, bhs

def is_low_contrast(filename):
    image = skimage.io.imread(filename)
    return skimage.exposure.is_low_contrast(image)

def censure(filename):
    """
    The CENSURE feature detector is a scale-invariant center-surround detector (CENSURE) that claims to outperform
    other detectors and is capable of real-time implementation.
    """

    detector = skimage.feature.CENSURE()
    image = skimage.color.rgb2gray(skimage.io.imread(filename))
    detector.detect(image)
    return detector.keypoints

<|endoftext|>


<commit_before>from pyjamas.ui.Composite import Composite
from pyjamas.ui.CheckBox import CheckBox
from pyjamas.ui.HorizontalPanel import HorizontalPanel
from pyjamas.ui.VerticalPanel import VerticalPanel
from pyjamas.ui.Button import Button
from pyjamas.ui import HasAlignment

class DayCheckBox(CheckBox):

    def __init__(self, owner, caption, day):
        CheckBox.__init__(self, caption)

        self.day = day
        self.addClickListener(owner.dayCheckBoxListener)
        self.setChecked(owner.calendar.getDayIncluded(day))

class DayCheckBoxListener:
    def __init__(self, calendar):
        self.calendar = calendar

    def onClick(self, sender):
        self.calendar.setDayIncluded(sender.day, sender.isChecked())

class DayFilterWidget(Composite):

    def __init__(self, calendar):
        Composite.__init__(self)

        self.calendar = calendar
        self.dayCheckBoxListener = DayCheckBoxListener(calendar)
        self.outer = VerticalPanel()
        self.initWidget(self.outer)
        self.setStyleName("DynaTable-DayFilterWidget")
        self.outer.add(DayCheckBox(self, "Sunday", 0))
        self.outer.add(DayCheckBox(self, "Monday", 1))
        self.outer.add(DayCheckBox(self, "Tuesday", 2))
        self.outer.add(DayCheckBox(self, "Wednesday", 3))
        self.outer.add(DayCheckBox(self, "Thursday", 4))
        self.outer.add(DayCheckBox(self, "Friday", 5))
        self.outer.add(DayCheckBox(self, "Saturday", 6))

        self.buttonAll = Button("All", self)
        self.buttonNone = Button("None", self)

        hp = HorizontalPanel()
        hp.setHorizontalAlignment(HasAlignment.ALIGN_CENTER)
        hp.add(self.buttonAll)
        hp.add(self.buttonNone)

        self.outer.add(hp)
        self.outer.setCellVerticalAlignment(hp, HasAlignment.ALIGN_BOTTOM)
        self.outer.setCellHorizontalAlignment(hp, HasAlignment.ALIGN_CENTER)

    def setAllCheckBoxes(self, checked):
        for widget in self.outer:
            if widget.setChecked:
                widget.setChecked(checked)
                self.dayCheckBoxListener.onClick(widget)

    def onClick(self, sender):
        if self.buttonAll == sender:
            self.setAllCheckBoxes(True)
        elif self.buttonNone == sender:
            self.setAllCheckBoxes(False)

<commit_msg>yippee - attribute failures work!<commit_after>from pyjamas.ui.Composite import Composite
from pyjamas.ui.CheckBox import CheckBox
from pyjamas.ui.HorizontalPanel import HorizontalPanel
from pyjamas.ui.VerticalPanel import VerticalPanel
from pyjamas.ui.Button import Button
from pyjamas.ui import HasAlignment

class DayCheckBox(CheckBox):

    def __init__(self, owner, caption, day):
        CheckBox.__init__(self, caption)

        self.day = day
        self.addClickListener(owner.dayCheckBoxListener)
        self.setChecked(owner.calendar.getDayIncluded(day))

class DayCheckBoxListener:
    def __init__(self, calendar):
        self.calendar = calendar

    def onClick(self, sender):
        self.calendar.setDayIncluded(sender.day, sender.isChecked())

class DayFilterWidget(Composite):

    def __init__(self, calendar):
        Composite.__init__(self)

        self.calendar = calendar
        self.dayCheckBoxListener = DayCheckBoxListener(calendar)
        self.outer = VerticalPanel()
        self.initWidget(self.outer)
        self.setStyleName("DynaTable-DayFilterWidget")
        self.outer.add(DayCheckBox(self, "Sunday", 0))
        self.outer.add(DayCheckBox(self, "Monday", 1))
        self.outer.add(DayCheckBox(self, "Tuesday", 2))
        self.outer.add(DayCheckBox(self, "Wednesday", 3))
        self.outer.add(DayCheckBox(self, "Thursday", 4))
        self.outer.add(DayCheckBox(self, "Friday", 5))
        self.outer.add(DayCheckBox(self, "Saturday", 6))

        self.buttonAll = Button("All", self)
        self.buttonNone = Button("None", self)

        hp = HorizontalPanel()
        hp.setHorizontalAlignment(HasAlignment.ALIGN_CENTER)
        hp.add(self.buttonAll)
        hp.add(self.buttonNone)

        self.outer.add(hp)
        self.outer.setCellVerticalAlignment(hp, HasAlignment.ALIGN_BOTTOM)
        self.outer.setCellHorizontalAlignment(hp, HasAlignment.ALIGN_CENTER)

    def setAllCheckBoxes(self, checked):
        for widget in self.outer:
            if hasattr(widget, "setChecked"):
                widget.setChecked(checked)
                self.dayCheckBoxListener.onClick(widget)

    def onClick(self, sender):
        if self.buttonAll == sender:
            self.setAllCheckBoxes(True)
        elif self.buttonNone == sender:
            self.setAllCheckBoxes(False)

<|endoftext|>


<commit_before>"""Parse command line arguments to webdiff."""
import argparse
import os
import re

import dirdiff
import githubdiff
import github_fetcher
from localfilediff import LocalFileDiff

class UsageError(Exception):
    pass

USAGE = (
'''Usage: webdiff <left_dir> <right_dir>
       webdiff <left_file> <right_file>
       webdiff https://github.com/<owner>/<repo>/pull/<num>

Or run "git webdiff" from a git repository.
''')

# e.g. https://github.com/danvk/dygraphs/pull/292
PULL_REQUEST_RE = re.compile(r'http[s]://(?:www.)?github.com\/([^/]+)/([^/]+)/pull/([0-9]+)(?:/.*)?')
PULL_REQUEST_NUM_RE = re.compile(r'^#([0-9]+)$')

def parse(args):
    """Returns {port, dirs: [], files: [], pr: {owner, repo, number}}."""
    parser = argparse.ArgumentParser(description='Run webdiff.', usage=USAGE)
    parser.add_argument('--port', '-p', type=int, help="Port to run webdiff on.", default=-1)
    parser.add_argument('dirs', type=str, nargs='+',
                        help="Directories to diff, or a github pull request URL.")
    args = parser.parse_args(args=args)

    out = {}
    if args.port != -1:
        out['port'] = args.port

    if len(args.dirs) > 2:
        raise UsageError('You must specify two files/dirs (got %d)' % len(args.dirs))

    if len(args.dirs) == 1:
        # must be a github pull request URL
        owner, repo, num = None, None, None
        m = re.match(PULL_REQUEST_RE, args.dirs[0])
        if m:
            owner, repo, num = m.groups()

        # Or perhaps something simpler like '#292'?
        m = re.match(PULL_REQUEST_NUM_RE, args.dirs[0])
        if m:
            num = int(m.group(1))
            owner, repo, num = github_fetcher.get_pr_repo(num)

        if not owner:
            raise UsageError('You must either specify two files, two '
                             'directories or a github pull request URL/#number')
        out['github'] = {'owner': owner, 'repo': repo, 'num': int(num)}

    else:
        a, b = args.dirs
        for x in (a, b):
            if not os.path.exists(x):
                raise UsageError('"%s" does not exist' % x)

        a_dir = os.path.isdir(a)
        b_dir = os.path.isdir(b)
        if a_dir and not b_dir:
            raise UsageError('"%s" is a directory but "%s" is not' % (a, b))
        if not a_dir and b_dir:
            raise UsageError('"%s" is a directory but "%s" is not' % (b, a))

        if a_dir and b_dir:
            out['dirs'] = (a, b)
        else:
            out['files'] = (a, b)

    return out

# TODO: move into dirdiff?
def _shim_for_file_diff(a_file, b_file):
    '''Sets A_DIR, B_DIR and DIFF to do a one-file diff.'''
    dirname = os.path.dirname
    basename = os.path.basename
    return LocalFileDiff(dirname(a_file), basename(a_file),
                         dirname(b_file), basename(b_file),
                         False)  # probably not a move

def diff_for_args(args):
    '''Returns a list of Diff objects for parsed command line args.'''
    if 'dirs' in args:
        return dirdiff.diff(*args['dirs'])

    if 'files' in args:
        return [_shim_for_file_diff(*args['files'])]

    if 'github' in args:
        gh = args['github']
        return githubdiff.fetch_pull_request(gh['owner'], gh['repo'], gh['num'])
<commit_msg>Fix issue with one file diff shim<commit_after>"""Parse command line arguments to webdiff."""
import argparse
import os
import re

import dirdiff
import githubdiff
import github_fetcher
from localfilediff import LocalFileDiff

class UsageError(Exception):
    pass

USAGE = (
'''Usage: webdiff <left_dir> <right_dir>
       webdiff <left_file> <right_file>
       webdiff https://github.com/<owner>/<repo>/pull/<num>

Or run "git webdiff" from a git repository.
''')

# e.g. https://github.com/danvk/dygraphs/pull/292
PULL_REQUEST_RE = re.compile(r'http[s]://(?:www.)?github.com\/([^/]+)/([^/]+)/pull/([0-9]+)(?:/.*)?')
PULL_REQUEST_NUM_RE = re.compile(r'^#([0-9]+)$')

def parse(args):
    """Returns {port, dirs: [], files: [], pr: {owner, repo, number}}."""
    parser = argparse.ArgumentParser(description='Run webdiff.', usage=USAGE)
    parser.add_argument('--port', '-p', type=int, help="Port to run webdiff on.", default=-1)
    parser.add_argument('dirs', type=str, nargs='+',
                        help="Directories to diff, or a github pull request URL.")
    args = parser.parse_args(args=args)

    out = {}
    if args.port != -1:
        out['port'] = args.port

    if len(args.dirs) > 2:
        raise UsageError('You must specify two files/dirs (got %d)' % len(args.dirs))

    if len(args.dirs) == 1:
        # must be a github pull request URL
        owner, repo, num = None, None, None
        m = re.match(PULL_REQUEST_RE, args.dirs[0])
        if m:
            owner, repo, num = m.groups()

        # Or perhaps something simpler like '#292'?
        m = re.match(PULL_REQUEST_NUM_RE, args.dirs[0])
        if m:
            num = int(m.group(1))
            owner, repo, num = github_fetcher.get_pr_repo(num)

        if not owner:
            raise UsageError('You must either specify two files, two '
                             'directories or a github pull request URL/#number')
        out['github'] = {'owner': owner, 'repo': repo, 'num': int(num)}

    else:
        a, b = args.dirs
        for x in (a, b):
            if not os.path.exists(x):
                raise UsageError('"%s" does not exist' % x)

        a_dir = os.path.isdir(a)
        b_dir = os.path.isdir(b)
        if a_dir and not b_dir:
            raise UsageError('"%s" is a directory but "%s" is not' % (a, b))
        if not a_dir and b_dir:
            raise UsageError('"%s" is a directory but "%s" is not' % (b, a))

        if a_dir and b_dir:
            out['dirs'] = (a, b)
        else:
            out['files'] = (a, b)

    return out

# TODO: move into dirdiff?
def _shim_for_file_diff(a_file, b_file):
    '''Sets A_DIR, B_DIR and DIFF to do a one-file diff.'''
    return LocalFileDiff(os.path.dirname(a_file), a_file,
                         os.path.dirname(b_file), b_file,
                         False)  # probably not a move

def diff_for_args(args):
    '''Returns a list of Diff objects for parsed command line args.'''
    if 'dirs' in args:
        return dirdiff.diff(*args['dirs'])

    if 'files' in args:
        return [_shim_for_file_diff(*args['files'])]

    if 'github' in args:
        gh = args['github']
        return githubdiff.fetch_pull_request(gh['owner'], gh['repo'], gh['num'])
<|endoftext|>


<commit_before># this script starts a new AIMS calculation.  Ethylene, SA2-CASSCF(2/2).
import numpy as np
import pyspawn
import pyspawn.general

# random number seed
seed=28396

# Velocity Verlet classical propagator
clas_prop = "vv"

# adapative 2nd-order Runge-Kutta quantum propagator
qm_prop = "rk2"

# adiabtic NPI quantum Hamiltonian
qm_ham = "adiabatic"

# use TeraChem CASSCF or CASCI to compute potentials
potential = "terachem_cas"

# initial time
t0 = 0.0

# time step
ts = 10.0

# final simulation time
tfinal = 8000.0

# number of dimensions
numdims = 18

# number of electronic states
numstates = 2

# TeraChem job options
tc_options = {
    "method":       'hf',
    "basis":        '6-31g',
    "atoms":        ["C", "C", "H", "H", "H", "H"],
    "charge":       0,
    "spinmult":     1,
    "closed_shell": True,
    "restricted":   True,

    "precision":    "double",
    "threall":      1.0e-20,

    "casscf":        "yes",
    "closed":       7,
    "active":       2,
    "cassinglets":  2,
    "castargetmult": 1,
    "cas_energy_states": [0, 1],
    "cas_energy_mults": [1, 1],
    }

# trajectory parameters
traj_params = {
    # initial time
    "time": t0,
    # time step
    "timestep": ts,
    # final simulation time
    "maxtime": tfinal,
    # coupling threshhold
    "spawnthresh": (0.5 * np.pi) / ts / 20.0,
    # initial electronic state (indexed such that 0 is the ground state)
    "istate": 1,
    # Gaussian widths
    "widths": np.asarray([30.0, 30.0, 30.0,
                        30.0, 30.0, 30.0,
                        6.0, 6.0, 6.0,
                        6.0, 6.0, 6.0,
                        6.0, 6.0, 6.0,
                        6.0, 6.0, 6.0]),
    # atom labels
    "atoms": tc_options["atoms"],
    # nuclear masses (in a.u)
    "masses": np.asarray([21864.0, 21864.0, 21864.0,
                    21864.0, 21864.0, 21864.0,
                    1822.0, 1822.0, 1822.0,
                    1822.0, 1822.0, 1822.0,
                    1822.0, 1822.0, 1822.0,
                    1822.0, 1822.0, 1822.0]),
    # terachem options (above)
    "tc_options": tc_options
    }

sim_params = {
    # initial time
    "quantum_time": traj_params["time"],
    # time step
    "timestep": traj_params["timestep"],
    # final simulation time
    "max_quantum_time": traj_params["maxtime"],
    # initial qm amplitudes
    "qm_amplitudes": np.ones(1,dtype=np.complex128),
    # energy shift used in quantum propagation
    "qm_energy_shift": 77.6,
}

# import routines needed for propagation
exec("pyspawn.import_methods.into_simulation(pyspawn.qm_integrator." + qm_prop + ")")
exec("pyspawn.import_methods.into_simulation(pyspawn.qm_hamiltonian." + qm_ham + ")")
exec("pyspawn.import_methods.into_traj(pyspawn.potential." + potential + ")")
exec("pyspawn.import_methods.into_traj(pyspawn.classical_integrator." + clas_prop + ")")

# check for the existence of files from a past run
pyspawn.general.check_files()

# set up first trajectory
traj1 = pyspawn.traj()
traj1.set_numstates(numstates)
traj1.set_numdims(numdims)
traj1.set_parameters(traj_params)

# sample initial position and momentum from Wigner distribution (requires hessian.hdf5)
traj1.initial_wigner(seed)

# set up simulation
sim = pyspawn.simulation()
sim.add_traj(traj1)
sim.set_parameters(sim_params)

# begin propagation
sim.propagate()

<commit_msg>Changed random seed for example job because this seed gives very complex (challenging) spawning pattern.<commit_after># this script starts a new AIMS calculation.  Ethylene, SA2-CASSCF(2/2).
import numpy as np
import pyspawn
import pyspawn.general

# random number seed
seed=87061

# Velocity Verlet classical propagator
clas_prop = "vv"

# adapative 2nd-order Runge-Kutta quantum propagator
qm_prop = "rk2"

# adiabtic NPI quantum Hamiltonian
qm_ham = "adiabatic"

# use TeraChem CASSCF or CASCI to compute potentials
potential = "terachem_cas"

# initial time
t0 = 0.0

# time step
ts = 10.0

# final simulation time
tfinal = 8000.0

# number of dimensions
numdims = 18

# number of electronic states
numstates = 2

# TeraChem job options
tc_options = {
    "method":       'hf',
    "basis":        '6-31g',
    "atoms":        ["C", "C", "H", "H", "H", "H"],
    "charge":       0,
    "spinmult":     1,
    "closed_shell": True,
    "restricted":   True,

    "precision":    "double",
    "threall":      1.0e-20,

    "casscf":        "yes",
    "closed":       7,
    "active":       2,
    "cassinglets":  2,
    "castargetmult": 1,
    "cas_energy_states": [0, 1],
    "cas_energy_mults": [1, 1],
    }

# trajectory parameters
traj_params = {
    # initial time
    "time": t0,
    # time step
    "timestep": ts,
    # final simulation time
    "maxtime": tfinal,
    # coupling threshhold
    "spawnthresh": (0.5 * np.pi) / ts / 20.0,
    # initial electronic state (indexed such that 0 is the ground state)
    "istate": 1,
    # Gaussian widths
    "widths": np.asarray([30.0, 30.0, 30.0,
                        30.0, 30.0, 30.0,
                        6.0, 6.0, 6.0,
                        6.0, 6.0, 6.0,
                        6.0, 6.0, 6.0,
                        6.0, 6.0, 6.0]),
    # atom labels
    "atoms": tc_options["atoms"],
    # nuclear masses (in a.u)
    "masses": np.asarray([21864.0, 21864.0, 21864.0,
                    21864.0, 21864.0, 21864.0,
                    1822.0, 1822.0, 1822.0,
                    1822.0, 1822.0, 1822.0,
                    1822.0, 1822.0, 1822.0,
                    1822.0, 1822.0, 1822.0]),
    # terachem options (above)
    "tc_options": tc_options
    }

sim_params = {
    # initial time
    "quantum_time": traj_params["time"],
    # time step
    "timestep": traj_params["timestep"],
    # final simulation time
    "max_quantum_time": traj_params["maxtime"],
    # initial qm amplitudes
    "qm_amplitudes": np.ones(1,dtype=np.complex128),
    # energy shift used in quantum propagation
    "qm_energy_shift": 77.6,
}

# import routines needed for propagation
exec("pyspawn.import_methods.into_simulation(pyspawn.qm_integrator." + qm_prop + ")")
exec("pyspawn.import_methods.into_simulation(pyspawn.qm_hamiltonian." + qm_ham + ")")
exec("pyspawn.import_methods.into_traj(pyspawn.potential." + potential + ")")
exec("pyspawn.import_methods.into_traj(pyspawn.classical_integrator." + clas_prop + ")")

# check for the existence of files from a past run
pyspawn.general.check_files()

# set up first trajectory
traj1 = pyspawn.traj()
traj1.set_numstates(numstates)
traj1.set_numdims(numdims)
traj1.set_parameters(traj_params)

# sample initial position and momentum from Wigner distribution (requires hessian.hdf5)
traj1.initial_wigner(seed)

# set up simulation
sim = pyspawn.simulation()
sim.add_traj(traj1)
sim.set_parameters(sim_params)

# begin propagation
sim.propagate()

<|endoftext|>