<commit_before># coding: utf-8
"""
JSON serialization and deserialization utilities.
"""

from __future__ import absolute_import, unicode_literals
import os
import json
import datetime
import six
import inspect

from hashlib import sha1
from collections import OrderedDict, defaultdict

try:
    from importlib import import_module
except ImportError:
    # importlib only available in Python 2.7+, 3.1+
    def importlib(x):
        return x

try:
    from inspect import getfullargspec as getargspec
except ImportError:
    from inspect import getargspec

try:
    import numpy as np
except ImportError:
    np = None

try:
    import bson
except ImportError:
    bson = None

try:
    import ruamel.yaml as yaml
    try:  # Default to using CLoader and CDumper for speed.
        from ruamel.yaml import CLoader as Loader
        from ruamel.yaml import CDumper as Dumper
    except ImportError:
        from ruamel.yaml import Loader
        from ruamel.yaml import Dumper
except ImportError:
    try:
        import yaml
        try:  # Default to using CLoader and CDumper for speed.
            from yaml import CLoader as Loader
            from yaml import CDumper as Dumper
        except ImportError:
            from yaml import Loader
            from yaml import Dumper
    except ImportError:
        yaml = None

__author__ = "Shyue Ping Ong"
__copyright__ = "Copyright 2014, The Materials Virtual Lab"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "ongsp@ucsd.edu"
__date__ = "1/24/14"

def _load_redirect(redirect_file):
    try:
        with open(redirect_file, "rt") as f:
            d = yaml.safe_load(f)
    except IOError:
        # If we can't find the file
        # Just use an empty redirect dict
        return {}

    # Convert the full paths to module/class
    redirect_dict = defaultdict(dict)
    for old_path, new_path in d.items():
        old_class = old_path.split(".")[-1]
        old_module = ".".join(old_path.split(".")[:-1])

        new_class = new_path.split(".")[-1]
        new_module = ".".join(new_path.split(".")[:-1])

        redirect_dict[old_module][old_class] = {
            "@module": new_module,
            "@class": new_class,
        }

    return dict(redirect_dict)

class MSONable(object):
    """
    This is a mix-in base class specifying an API for msonable objects. MSON
    is Monty JSON. Essentially, MSONable objects must implement an as_dict
    method, which must return a json serializable dict and must also support
    no arguments (though optional arguments to finetune the output is ok),
    and a from_dict class method that regenerates the object from the dict
    generated by the as_dict method. The as_dict method should contain the
    "@module" and "@class" keys which will allow the MontyEncoder to
    dynamically deserialize the class. E.g.::

        d["@module"] = self.__class__.__module__
        d["@module"] = self.__class__.__name__

    A default implementation is provided in MSONable, which automatically
    determines if the class already contains self.argname or self._argname
    attributes for every arg. If so, these will be used for serialization in
    the dict format. Similarly, the default from_dict will deserialization
    classes of such form. An example is given below::

        class MSONClass(MSONable):

        def __init__(self, a, b, c, d=1, **kwargs):
            self.a = a
            self.b = b
            self._c = c
            self._d = d
            self.kwargs = kwargs

    For such classes, you merely need to inherit from MSONable and you do not
    need to implement your own as_dict or from_dict protocol.

    New to Monty V2.0.6....
    Classes can be redirected to moved implementations by putting in the old
    fully qualified path and new fully qualified path into .monty.yaml in the
    home folder

    Example:
    old_module.old_class: new_module.new_class
    """

    REDIRECT = _load_redirect(os.path.join(os.path.expanduser("~"),
                                           ".monty.yaml"))

    def as_dict(self):
        """
        A JSON serializable dict representation of an object.
        """
        d = {"@module": self.__class__.__module__,
             "@class": self.__class__.__name__}

        try:
            parent_module = self.__class__.__module__.split('.')[0]
            module_version = import_module(parent_module).__version__
            d["@version"] = u"{}".format(module_version)
        except AttributeError:
            d["@version"] = None

        args = getargspec(self.__class__.__init__).args

        def recursive_as_dict(obj):
            if isinstance(obj, (list, tuple)):
                return [recursive_as_dict(it) for it in obj]
            elif isinstance(obj, dict):
                return {kk: recursive_as_dict(vv) for kk, vv in obj.items()}
            elif hasattr(obj, "as_dict"):
                return obj.as_dict()
            return obj

        for c in args:
            if c != "self":
                try:
                    a = self.__getattribute__(c)
                except AttributeError:
                    try:
                        a = self.__getattribute__("_" + c)
                    except AttributeError:
                        raise NotImplementedError(
                            "Unable to automatically determine as_dict "
                            "format from class. MSONAble requires all "
                            "args to be present as either self.argname or "
                            "self._argname, and kwargs to be present under"
                            "a self.kwargs variable to automatically "
                            "determine the dict format. Alternatively, "
                            "you can implement both as_dict and from_dict.")
                d[c] = recursive_as_dict(a)
        if hasattr(self, "kwargs"):
            d.update(**self.kwargs)
        if hasattr(self, "_kwargs"):
            d.update(**self._kwargs)
        return d

    @classmethod
    def from_dict(cls, d):
        decoded = {k: MontyDecoder().process_decoded(v) for k, v in d.items()
                   if not k.startswith("@")}
        return cls(**decoded)

    def to_json(self):
        """
        Returns a json string representation of the MSONable object.
        """
        return json.dumps(self, cls=MontyEncoder)

    def unsafe_hash(self):
        """
        Returns an hash of the current object. This uses a generic but low
        performance method of converting the object to a dictionary, flattening
        any nested keys, and then performing a hash on the resulting object
        """

        def flatten(obj, seperator="."):
            # Flattens a dictionary

            flat_dict = {}
            for key, value in obj.items():
                if isinstance(value, dict):
                    flat_dict.update(
                        {
                            seperator.join([key, _key]): _value
                            for _key, _value in flatten(value).items()
                        }
                    )
                elif isinstance(value, list):
                    list_dict = {
                        "{}{}{}".format(key, seperator, num): item
                        for num, item in enumerate(value)
                    }
                    flat_dict.update(flatten(list_dict))
                else:
                    flat_dict[key] = value

            return flat_dict

        ordered_keys = sorted(flatten(jsanitize(self.as_dict())).items(),
                              key=lambda x: x[0])
        ordered_keys = [item for item in ordered_keys if "@" not in item[0]]
        return sha1(json.dumps(OrderedDict(ordered_keys)).encode("utf-8"))

class MontyEncoder(json.JSONEncoder):
    """
    A Json Encoder which supports the MSONable API, plus adds support for
    numpy arrays, datetime objects, bson ObjectIds (requires bson).

    Usage::

        # Add it as a *cls* keyword when using json.dump
        json.dumps(object, cls=MontyEncoder)
    """

    def default(self, o):
        """
        Overriding default method for JSON encoding. This method does two
        things: (a) If an object has a to_dict property, return the to_dict
        output. (b) If the @module and @class keys are not in the to_dict,
        add them to the output automatically. If the object has no to_dict
        property, the default Python json encoder default method is called.

        Args:
            o: Python object.

        Return:
            Python dict representation.
        """
        if isinstance(o, datetime.datetime):
            return {"@module": "datetime", "@class": "datetime",
                    "string": o.__str__()}
        if np is not None:
            if isinstance(o, np.ndarray):
                return {"@module": "numpy",
                        "@class": "array",
                        "dtype": o.dtype.__str__(),
                        "data": o.tolist()}
            elif isinstance(o, np.generic):
                return o.item()
        if bson is not None:
            if isinstance(o, bson.objectid.ObjectId):
                return {"@module": "bson.objectid",
                        "@class": "ObjectId",
                        "oid": str(o)}

        try:
            d = o.as_dict()
            if "@module" not in d:
                d["@module"] = u"{}".format(o.__class__.__module__)
            if "@class" not in d:
                d["@class"] = u"{}".format(o.__class__.__name__)
            if "@version" not in d:
                try:
                    parent_module = o.__class__.__module__.split('.')[0]
                    module_version = import_module(parent_module).__version__
                    d["@version"] = u"{}".format(module_version)
                except AttributeError:
                    d["@version"] = None
            return d
        except AttributeError:
            return json.JSONEncoder.default(self, o)

class MontyDecoder(json.JSONDecoder):
    """
    A Json Decoder which supports the MSONable API. By default, the
    decoder attempts to find a module and name associated with a dict. If
    found, the decoder will generate a Pymatgen as a priority.  If that fails,
    the original decoded dictionary from the string is returned. Note that
    nested lists and dicts containing pymatgen object will be decoded correctly
    as well.

    Usage:

        # Add it as a *cls* keyword when using json.load
        json.loads(json_string, cls=MontyDecoder)
    """

    def process_decoded(self, d):
        """
        Recursive method to support decoding dicts and lists containing
        pymatgen objects.
        """
        if isinstance(d, dict):
            if "@module" in d and "@class" in d:
                modname = d["@module"]
                classname = d["@class"]
                if classname in MSONable.REDIRECT.get(modname, {}):
                    modname = MSONable.REDIRECT[modname][classname]["@module"]
                    classname = MSONable.REDIRECT[modname][classname]["@class"]
            else:
                modname = None
                classname = None
            if modname and modname not in ["bson.objectid", "numpy"]:
                if modname == "datetime" and classname == "datetime":
                    try:
                        dt = datetime.datetime.strptime(d["string"],
                                                        "%Y-%m-%d %H:%M:%S.%f")
                    except ValueError:
                        dt = datetime.datetime.strptime(d["string"],
                                                        "%Y-%m-%d %H:%M:%S")
                    return dt

                mod = __import__(modname, globals(), locals(), [classname], 0)
                if hasattr(mod, classname):
                    cls_ = getattr(mod, classname)
                    data = {k: v for k, v in d.items()
                            if not k.startswith("@")}
                    if hasattr(cls_, "from_dict"):
                        return cls_.from_dict(data)
            elif np is not None and modname == "numpy" and classname == \
                    "array":
                return np.array(d["data"], dtype=d["dtype"])

            elif (bson is not None) and modname == "bson.objectid" and \
                    classname == "ObjectId":
                return bson.objectid.ObjectId(d["oid"])

            return {self.process_decoded(k): self.process_decoded(v)
                    for k, v in d.items()}
        elif isinstance(d, list):
            return [self.process_decoded(x) for x in d]

        return d

    def decode(self, s):
        d = json.JSONDecoder.decode(self, s)
        return self.process_decoded(d)

class MSONError(Exception):
    """
    Exception class for serialization errors.
    """
    pass

def jsanitize(obj, strict=False, allow_bson=False):
    """
    This method cleans an input json-like object, either a list or a dict or
    some sequence, nested or otherwise, by converting all non-string
    dictionary keys (such as int and float) to strings, and also recursively
    encodes all objects using Monty's as_dict() protocol.

    Args:
        obj: input json-like object.
        strict (bool): This parameters sets the behavior when jsanitize
            encounters an object it does not understand. If strict is True,
            jsanitize will try to get the as_dict() attribute of the object. If
            no such attribute is found, an attribute error will be thrown. If
            strict is False, jsanitize will simply call str(object) to convert
            the object to a string representation.
        allow_bson (bool): This parameters sets the behavior when jsanitize
            encounters an bson supported type such as objectid and datetime. If
            True, such bson types will be ignored, allowing for proper
            insertion into MongoDb databases.

    Returns:
        Sanitized dict that can be json serialized.
    """
    if allow_bson and (isinstance(obj, (datetime.datetime, bytes)) or
                       (bson is not None and
                        isinstance(obj, bson.objectid.ObjectId))):
        return obj
    if isinstance(obj, (list, tuple)):
        return [jsanitize(i, strict=strict, allow_bson=allow_bson) for i in obj]
    elif np is not None and isinstance(obj, np.ndarray):
        return [jsanitize(i, strict=strict, allow_bson=allow_bson) for i in
                obj.tolist()]
    elif isinstance(obj, dict):
        return {k.__str__(): jsanitize(v, strict=strict, allow_bson=allow_bson)
                for k, v in obj.items()}
    elif isinstance(obj, six.integer_types + tuple([float])):
        return obj
    elif obj is None:
        return None

    else:
        if not strict:
            return obj.__str__()
        else:
            if isinstance(obj, six.string_types):
                return obj.__str__()
            else:
                return jsanitize(obj.as_dict(), strict=strict,
                                 allow_bson=allow_bson)
<commit_msg>Add support for serializing/deserializing Enums<commit_after># coding: utf-8
"""
JSON serialization and deserialization utilities.
"""

from __future__ import absolute_import, unicode_literals
import os
import json
import datetime
import six
import inspect

from hashlib import sha1
from collections import OrderedDict, defaultdict
from enum import Enum

try:
    from importlib import import_module
except ImportError:
    # importlib only available in Python 2.7+, 3.1+
    def importlib(x):
        return x

try:
    from inspect import getfullargspec as getargspec
except ImportError:
    from inspect import getargspec

try:
    import numpy as np
except ImportError:
    np = None

try:
    import bson
except ImportError:
    bson = None

try:
    import ruamel.yaml as yaml
    try:  # Default to using CLoader and CDumper for speed.
        from ruamel.yaml import CLoader as Loader
        from ruamel.yaml import CDumper as Dumper
    except ImportError:
        from ruamel.yaml import Loader
        from ruamel.yaml import Dumper
except ImportError:
    try:
        import yaml
        try:  # Default to using CLoader and CDumper for speed.
            from yaml import CLoader as Loader
            from yaml import CDumper as Dumper
        except ImportError:
            from yaml import Loader
            from yaml import Dumper
    except ImportError:
        yaml = None

__author__ = "Shyue Ping Ong"
__copyright__ = "Copyright 2014, The Materials Virtual Lab"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "ongsp@ucsd.edu"
__date__ = "1/24/14"

def _load_redirect(redirect_file):
    try:
        with open(redirect_file, "rt") as f:
            d = yaml.safe_load(f)
    except IOError:
        # If we can't find the file
        # Just use an empty redirect dict
        return {}

    # Convert the full paths to module/class
    redirect_dict = defaultdict(dict)
    for old_path, new_path in d.items():
        old_class = old_path.split(".")[-1]
        old_module = ".".join(old_path.split(".")[:-1])

        new_class = new_path.split(".")[-1]
        new_module = ".".join(new_path.split(".")[:-1])

        redirect_dict[old_module][old_class] = {
            "@module": new_module,
            "@class": new_class,
        }

    return dict(redirect_dict)

class MSONable(object):
    """
    This is a mix-in base class specifying an API for msonable objects. MSON
    is Monty JSON. Essentially, MSONable objects must implement an as_dict
    method, which must return a json serializable dict and must also support
    no arguments (though optional arguments to finetune the output is ok),
    and a from_dict class method that regenerates the object from the dict
    generated by the as_dict method. The as_dict method should contain the
    "@module" and "@class" keys which will allow the MontyEncoder to
    dynamically deserialize the class. E.g.::

        d["@module"] = self.__class__.__module__
        d["@module"] = self.__class__.__name__

    A default implementation is provided in MSONable, which automatically
    determines if the class already contains self.argname or self._argname
    attributes for every arg. If so, these will be used for serialization in
    the dict format. Similarly, the default from_dict will deserialization
    classes of such form. An example is given below::

        class MSONClass(MSONable):

        def __init__(self, a, b, c, d=1, **kwargs):
            self.a = a
            self.b = b
            self._c = c
            self._d = d
            self.kwargs = kwargs

    For such classes, you merely need to inherit from MSONable and you do not
    need to implement your own as_dict or from_dict protocol.

    New to Monty V2.0.6....
    Classes can be redirected to moved implementations by putting in the old
    fully qualified path and new fully qualified path into .monty.yaml in the
    home folder

    Example:
    old_module.old_class: new_module.new_class
    """

    REDIRECT = _load_redirect(os.path.join(os.path.expanduser("~"),
                                           ".monty.yaml"))

    def as_dict(self):
        """
        A JSON serializable dict representation of an object.
        """
        d = {"@module": self.__class__.__module__,
             "@class": self.__class__.__name__}

        try:
            parent_module = self.__class__.__module__.split('.')[0]
            module_version = import_module(parent_module).__version__
            d["@version"] = u"{}".format(module_version)
        except AttributeError:
            d["@version"] = None

        args = getargspec(self.__class__.__init__).args

        def recursive_as_dict(obj):
            if isinstance(obj, (list, tuple)):
                return [recursive_as_dict(it) for it in obj]
            elif isinstance(obj, dict):
                return {kk: recursive_as_dict(vv) for kk, vv in obj.items()}
            elif hasattr(obj, "as_dict"):
                return obj.as_dict()
            return obj

        for c in args:
            if c != "self":
                try:
                    a = self.__getattribute__(c)
                except AttributeError:
                    try:
                        a = self.__getattribute__("_" + c)
                    except AttributeError:
                        raise NotImplementedError(
                            "Unable to automatically determine as_dict "
                            "format from class. MSONAble requires all "
                            "args to be present as either self.argname or "
                            "self._argname, and kwargs to be present under"
                            "a self.kwargs variable to automatically "
                            "determine the dict format. Alternatively, "
                            "you can implement both as_dict and from_dict.")
                d[c] = recursive_as_dict(a)
        if hasattr(self, "kwargs"):
            d.update(**self.kwargs)
        if hasattr(self, "_kwargs"):
            d.update(**self._kwargs)
        if isinstance(self, Enum):
            d.update({"value": self.value})
        return d

    @classmethod
    def from_dict(cls, d):
        decoded = {k: MontyDecoder().process_decoded(v) for k, v in d.items()
                   if not k.startswith("@")}
        return cls(**decoded)

    def to_json(self):
        """
        Returns a json string representation of the MSONable object.
        """
        return json.dumps(self, cls=MontyEncoder)

    def unsafe_hash(self):
        """
        Returns an hash of the current object. This uses a generic but low
        performance method of converting the object to a dictionary, flattening
        any nested keys, and then performing a hash on the resulting object
        """

        def flatten(obj, seperator="."):
            # Flattens a dictionary

            flat_dict = {}
            for key, value in obj.items():
                if isinstance(value, dict):
                    flat_dict.update(
                        {
                            seperator.join([key, _key]): _value
                            for _key, _value in flatten(value).items()
                        }
                    )
                elif isinstance(value, list):
                    list_dict = {
                        "{}{}{}".format(key, seperator, num): item
                        for num, item in enumerate(value)
                    }
                    flat_dict.update(flatten(list_dict))
                else:
                    flat_dict[key] = value

            return flat_dict

        ordered_keys = sorted(flatten(jsanitize(self.as_dict())).items(),
                              key=lambda x: x[0])
        ordered_keys = [item for item in ordered_keys if "@" not in item[0]]
        return sha1(json.dumps(OrderedDict(ordered_keys)).encode("utf-8"))

class MontyEncoder(json.JSONEncoder):
    """
    A Json Encoder which supports the MSONable API, plus adds support for
    numpy arrays, datetime objects, bson ObjectIds (requires bson).

    Usage::

        # Add it as a *cls* keyword when using json.dump
        json.dumps(object, cls=MontyEncoder)
    """

    def default(self, o):
        """
        Overriding default method for JSON encoding. This method does two
        things: (a) If an object has a to_dict property, return the to_dict
        output. (b) If the @module and @class keys are not in the to_dict,
        add them to the output automatically. If the object has no to_dict
        property, the default Python json encoder default method is called.

        Args:
            o: Python object.

        Return:
            Python dict representation.
        """
        if isinstance(o, datetime.datetime):
            return {"@module": "datetime", "@class": "datetime",
                    "string": o.__str__()}
        if np is not None:
            if isinstance(o, np.ndarray):
                return {"@module": "numpy",
                        "@class": "array",
                        "dtype": o.dtype.__str__(),
                        "data": o.tolist()}
            elif isinstance(o, np.generic):
                return o.item()
        if bson is not None:
            if isinstance(o, bson.objectid.ObjectId):
                return {"@module": "bson.objectid",
                        "@class": "ObjectId",
                        "oid": str(o)}

        try:
            d = o.as_dict()
            if "@module" not in d:
                d["@module"] = u"{}".format(o.__class__.__module__)
            if "@class" not in d:
                d["@class"] = u"{}".format(o.__class__.__name__)
            if "@version" not in d:
                try:
                    parent_module = o.__class__.__module__.split('.')[0]
                    module_version = import_module(parent_module).__version__
                    d["@version"] = u"{}".format(module_version)
                except AttributeError:
                    d["@version"] = None
            return d
        except AttributeError:
            return json.JSONEncoder.default(self, o)

class MontyDecoder(json.JSONDecoder):
    """
    A Json Decoder which supports the MSONable API. By default, the
    decoder attempts to find a module and name associated with a dict. If
    found, the decoder will generate a Pymatgen as a priority.  If that fails,
    the original decoded dictionary from the string is returned. Note that
    nested lists and dicts containing pymatgen object will be decoded correctly
    as well.

    Usage:

        # Add it as a *cls* keyword when using json.load
        json.loads(json_string, cls=MontyDecoder)
    """

    def process_decoded(self, d):
        """
        Recursive method to support decoding dicts and lists containing
        pymatgen objects.
        """
        if isinstance(d, dict):
            if "@module" in d and "@class" in d:
                modname = d["@module"]
                classname = d["@class"]
                if classname in MSONable.REDIRECT.get(modname, {}):
                    modname = MSONable.REDIRECT[modname][classname]["@module"]
                    classname = MSONable.REDIRECT[modname][classname]["@class"]
            else:
                modname = None
                classname = None
            if modname and modname not in ["bson.objectid", "numpy"]:
                if modname == "datetime" and classname == "datetime":
                    try:
                        dt = datetime.datetime.strptime(d["string"],
                                                        "%Y-%m-%d %H:%M:%S.%f")
                    except ValueError:
                        dt = datetime.datetime.strptime(d["string"],
                                                        "%Y-%m-%d %H:%M:%S")
                    return dt

                mod = __import__(modname, globals(), locals(), [classname], 0)
                if hasattr(mod, classname):
                    cls_ = getattr(mod, classname)
                    data = {k: v for k, v in d.items()
                            if not k.startswith("@")}
                    if hasattr(cls_, "from_dict"):
                        return cls_.from_dict(data)
            elif np is not None and modname == "numpy" and classname == \
                    "array":
                return np.array(d["data"], dtype=d["dtype"])

            elif (bson is not None) and modname == "bson.objectid" and \
                    classname == "ObjectId":
                return bson.objectid.ObjectId(d["oid"])

            return {self.process_decoded(k): self.process_decoded(v)
                    for k, v in d.items()}
        elif isinstance(d, list):
            return [self.process_decoded(x) for x in d]

        return d

    def decode(self, s):
        d = json.JSONDecoder.decode(self, s)
        return self.process_decoded(d)

class MSONError(Exception):
    """
    Exception class for serialization errors.
    """
    pass

def jsanitize(obj, strict=False, allow_bson=False):
    """
    This method cleans an input json-like object, either a list or a dict or
    some sequence, nested or otherwise, by converting all non-string
    dictionary keys (such as int and float) to strings, and also recursively
    encodes all objects using Monty's as_dict() protocol.

    Args:
        obj: input json-like object.
        strict (bool): This parameters sets the behavior when jsanitize
            encounters an object it does not understand. If strict is True,
            jsanitize will try to get the as_dict() attribute of the object. If
            no such attribute is found, an attribute error will be thrown. If
            strict is False, jsanitize will simply call str(object) to convert
            the object to a string representation.
        allow_bson (bool): This parameters sets the behavior when jsanitize
            encounters an bson supported type such as objectid and datetime. If
            True, such bson types will be ignored, allowing for proper
            insertion into MongoDb databases.

    Returns:
        Sanitized dict that can be json serialized.
    """
    if allow_bson and (isinstance(obj, (datetime.datetime, bytes)) or
                       (bson is not None and
                        isinstance(obj, bson.objectid.ObjectId))):
        return obj
    if isinstance(obj, (list, tuple)):
        return [jsanitize(i, strict=strict, allow_bson=allow_bson) for i in obj]
    elif np is not None and isinstance(obj, np.ndarray):
        return [jsanitize(i, strict=strict, allow_bson=allow_bson) for i in
                obj.tolist()]
    elif isinstance(obj, dict):
        return {k.__str__(): jsanitize(v, strict=strict, allow_bson=allow_bson)
                for k, v in obj.items()}
    elif isinstance(obj, six.integer_types + tuple([float])):
        return obj
    elif obj is None:
        return None

    else:
        if not strict:
            return obj.__str__()
        else:
            if isinstance(obj, six.string_types):
                return obj.__str__()
            else:
                return jsanitize(obj.as_dict(), strict=strict,
                                 allow_bson=allow_bson)
<|endoftext|>


<commit_before>import autograd.numpy as np
from autograd.extend import primitive, defvjp

from ceviche.utils import circ2eps, grid_coords
from ceviche.primitives import vjp_maker_num

class Param_Base(object):

    def __init__(self):
        pass

    @classmethod
    def get_eps(cls, params, *args):
        raise NotImplementedError("Need to implement a function for computing permittivity from parameters")

""" Topology / Continuous Optimization """

class Param_Topology(Param_Base):

    def __init__(self):
        super().__init__(self)

    @staticmethod
    def _density2eps(mat_density, eps_max):
        return 1 + (eps_max - 1) * mat_density

    @classmethod
    def get_eps(cls, params, eps_background, design_region, eps_max):

        mat_density = params
        eps_inner = cls._density2eps(mat_density, eps_max) * (design_region == 1)
        eps_outer = eps_background * (design_region == 0)

        return eps_inner + eps_outer

""" Shape Optimization """

class Param_Shape(Param_Base):

    def __init__(self):
        super().__init__()

    @staticmethod
    def get_eps(*args):
        raise NotImplementedError("Need to implement a function for computing permittivity from parameters")

    @staticmethod
    def sigmoid(x, strength=.2):
        # used to anti-alias the circle, higher strength means sharper boundary
        return np.exp(x * strength) / (1 + np.exp(x * strength))

class Circle_Shapes(Param_Shape):

    def __init__(self, eps_background, dL):
        self.eps_background = eps_background
        self.dL = dL
        self.xs, self.ys = grid_coords(self.eps_background, self.dL)
        super().__init__()

    def circle(self, xs, ys, x, y, r):
        # defines an anti aliased circle
        dist_from_edge = (xs - x)**2 + (ys - y)**2 - r**2
        return self.sigmoid(-dist_from_edge / self.dL**2)

    def write_circle(self, x, y, r, value):
        # creates an array that, when added to the background epsilon, adds a circle with (x,y,r,value)
        circle_mask = self.circle(self.xs, self.ys, x, y, r)
        new_array = (value - self.eps_background) * circle_mask
        return new_array

    def get_eps(self, xs, ys, rs, values):
        # returns the permittivity array for a bunch of holes at positions (xs, ys) with radii rs and epsilon values (val)
        eps_r = self.eps_background.copy()
        for x, y, r, value in zip(xs, ys, rs, values):
            circle_eps = self.write_circle(x, y, r, value)
            eps_r += circle_eps
        return eps_r

""" Level Set Optimization """

class Param_LevelSet(Param_Base):

    def __init__(self):
        super().__init__(self)

    @classmethod
    def get_eps(cls, params, eps_background, design_region, eps_max):
        raise NotImplementedError("Need to implement a function for computing permittivity from parameters")

if __name__ == '__main__':

    # example of calling one

    shape = (10, 20)
    params = np.random.random(shape)
    eps_background = np.ones(shape)
    design_region = np.zeros(shape)
    design_region[5:,:] = 1
    eps_max = 5

    eps = Param_Topology.get_eps(params, eps_background, design_region, eps_max)

<commit_msg>reset sigmoid strength<commit_after>import autograd.numpy as np
from autograd.extend import primitive, defvjp

from ceviche.utils import circ2eps, grid_coords
from ceviche.primitives import vjp_maker_num

class Param_Base(object):

    def __init__(self):
        pass

    @classmethod
    def get_eps(cls, params, *args):
        raise NotImplementedError("Need to implement a function for computing permittivity from parameters")

""" Topology / Continuous Optimization """

class Param_Topology(Param_Base):

    def __init__(self):
        super().__init__(self)

    @staticmethod
    def _density2eps(mat_density, eps_max):
        return 1 + (eps_max - 1) * mat_density

    @classmethod
    def get_eps(cls, params, eps_background, design_region, eps_max):

        mat_density = params
        eps_inner = cls._density2eps(mat_density, eps_max) * (design_region == 1)
        eps_outer = eps_background * (design_region == 0)

        return eps_inner + eps_outer

""" Shape Optimization """

class Param_Shape(Param_Base):

    def __init__(self):
        super().__init__()

    @staticmethod
    def get_eps(*args):
        raise NotImplementedError("Need to implement a function for computing permittivity from parameters")

    @staticmethod
    def sigmoid(x, strength=.1):
        # used to anti-alias the circle, higher strength means sharper boundary
        return np.exp(x * strength) / (1 + np.exp(x * strength))

class Circle_Shapes(Param_Shape):

    def __init__(self, eps_background, dL):
        self.eps_background = eps_background
        self.dL = dL
        self.xs, self.ys = grid_coords(self.eps_background, self.dL)
        super().__init__()

    def circle(self, xs, ys, x, y, r):
        # defines an anti aliased circle
        dist_from_edge = (xs - x)**2 + (ys - y)**2 - r**2
        return self.sigmoid(-dist_from_edge / self.dL**2)

    def write_circle(self, x, y, r, value):
        # creates an array that, when added to the background epsilon, adds a circle with (x,y,r,value)
        circle_mask = self.circle(self.xs, self.ys, x, y, r)
        new_array = (value - self.eps_background) * circle_mask
        return new_array

    def get_eps(self, xs, ys, rs, values):
        # returns the permittivity array for a bunch of holes at positions (xs, ys) with radii rs and epsilon values (val)
        eps_r = self.eps_background.copy()
        for x, y, r, value in zip(xs, ys, rs, values):
            circle_eps = self.write_circle(x, y, r, value)
            eps_r += circle_eps
        return eps_r

""" Level Set Optimization """

class Param_LevelSet(Param_Base):

    def __init__(self):
        super().__init__(self)

    @classmethod
    def get_eps(cls, params, eps_background, design_region, eps_max):
        raise NotImplementedError("Need to implement a function for computing permittivity from parameters")

if __name__ == '__main__':

    # example of calling one

    shape = (10, 20)
    params = np.random.random(shape)
    eps_background = np.ones(shape)
    design_region = np.zeros(shape)
    design_region[5:,:] = 1
    eps_max = 5

    eps = Param_Topology.get_eps(params, eps_background, design_region, eps_max)

<|endoftext|>


<commit_before>from __future__ import print_function
import multiprocessing
import os

import kombu.five
from celery import Celery
from celery import current_app
from celery.backends.base import DisabledBackend
from celery.task import task
from celery.worker.autoscale import Autoscaler
from django.conf import settings
from djcelery.loaders import DjangoLoader
from datetime import datetime
from time import sleep, time

def no_result_task(*args, **kwargs):
    """
    We use an instance of DatabaseBackend to store results of celery tasks.

    But even with ignore_result=True, celery will still create an entry into
    celery_taskmeta when creating the task because we have the DatabaseBackend
    configured.

    In order to avoid creating an entry into celery_taskmeta, we also need
    to override the task's result backend to be an instance of the DisabledBackend.

    Use this decorator to create tasks for which we don't need to store
    result info.
    """
    kwargs['ignore_result'] = True
    kwargs['backend'] = DisabledBackend(current_app)

    def wrapper(fcn):
        return task(*args, **kwargs)(fcn)

    return wrapper

class TaskInfo(object):

    def __init__(self, _id, name, time_start=None):
        self.id = _id
        self.name = name
        # http://stackoverflow.com/questions/20091505/celery-task-with-a-time-start-attribute-in-1970
        if time_start:
            self.time_start = datetime.fromtimestamp(time() - kombu.five.monotonic() + time_start)
        else:
            self.time_start = None

class InvalidTaskTypeError(Exception):
    pass

def get_active_task_info(tasks):
    result = []
    for task_dict in tasks:
        result.append(TaskInfo(task_dict.get('id'), task_dict.get('name'), task_dict.get('time_start')))
    return result

def get_reserved_task_info(tasks):
    result = []
    for task_dict in tasks:
        result.append(TaskInfo(task_dict.get('id'), task_dict.get('name')))
    return result

def get_scheduled_task_info(tasks):
    result = []
    for task_dict in tasks:
        task_dict = task_dict.get('request', {})
        result.append(TaskInfo(task_dict.get('id'), task_dict.get('name')))
    return result

def _validate_task_state(task_state, allow_active=True, allow_scheduled=True,
        allow_reserved=True, allow_revoked=True):

    allowed_states = []
    if allow_active:
        allowed_states.append('active')

    if allow_scheduled:
        allowed_states.append('scheduled')

    if allow_reserved:
        allowed_states.append('reserved')

    if allow_revoked:
        allowed_states.append('revoked')

    if task_state not in allowed_states:
        raise InvalidTaskTypeError("Task state must be one of: {}".format(allowed_states))

def _get_task_info_fcn(task_state):
    return {
        'active': get_active_task_info,
        'reserved': get_reserved_task_info,
        'scheduled': get_scheduled_task_info,
    }.get(task_state)

def revoke_tasks(task_names, interval=5):
    """
    Constantly polls all workers for any active, reserved, or scheduled
    tasks, and revokes the tasks if they match any of the given task names.

    Note that when reserved or scheduled tasks are revoked, they don't disappear from
    the list of reserved or scheduled tasks immediately. Instead, they will just be
    discarded by the worker when they try to move to active.

    :param task_names: a list of fully qualified task names to revoke
    :param interval: the interval (in seconds) on which to poll the workers for tasks

    Example:
    revoke_tasks(['couchexport.tasks.export_async'])
    """
    app = Celery()
    app.config_from_object(settings)
    task_ids = set()
    while True:
        tasks = []
        inspect = app.control.inspect()
        for task_state in ['active', 'reserved', 'scheduled']:
            result = getattr(inspect, task_state)()
            if not result:
                continue

            for worker, task_dicts in result.iteritems():
                tasks.extend(_get_task_info_fcn(task_state)(task_dicts))

        for task in tasks:
            if task.name in task_names and task.id not in task_ids:
                app.control.revoke(task.id, terminate=True)
                task_ids.add(task.id)
                print(datetime.utcnow(), 'Revoked', task.id, task.name)

        sleep(interval)

def print_tasks(worker, task_state):
    """
    Prints celery tasks that have been received by the given worker that are in the
    given state.

    Examples:
    print_tasks('celery@hqcelery0.internal-va.commcarehq.org_main', 'active')
    print_tasks('celery@hqcelery1.internal-va.commcarehq.org_sms_queue', 'reserved')
    """
    _validate_task_state(task_state)

    app = Celery()
    app.config_from_object(settings)
    inspect = app.control.inspect([worker])
    fcn = getattr(inspect, task_state)
    result = fcn()
    if not result:
        print("Worker does not appear to be online. Check worker name, and that it is running.")
        return

    tasks = result[worker]

    if not tasks:
        print('(none)')
        return

    if task_state == 'revoked':
        for task_id in tasks:
            print(task_id)
        return

    tasks = _get_task_info_fcn(task_state)(tasks)

    for task_info in tasks:
        if task_info.time_start:
            print(task_info.id, task_info.time_start, task_info.name)
        else:
            print(task_info.id, task_info.name)

def get_running_workers(timeout=10):
    app = Celery()
    app.config_from_object(settings)
    result = app.control.ping(timeout=timeout)

    worker_names = []
    for worker_info in result:
        worker_names.extend(worker_info.keys())

    return worker_names

class LoadBasedAutoscaler(Autoscaler):
    def _maybe_scale(self, req=None):
        procs = self.processes
        cur = min(self.qty, self.max_concurrency)

        available_cpus = multiprocessing.cpu_count()
        try:
            load_avgs = os.getloadavg()
            one_min_avg = load_avgs[0]
            normalized_load = one_min_avg / available_cpus
        except OSError:
            # if we can't get the load average, let's just use normal autoscaling
            load_avgs = None
            normalized_load = 0

        if cur > procs and normalized_load < 0.90:
            self.scale_up(cur - procs)
            return True
        elif procs > self.min_concurrency:
            if cur < procs:
                self.scale_down(min(procs - cur, procs - self.min_concurrency))
                return True
            elif normalized_load > 0.90:
                # if load is too high trying scaling down 1 worker at a time.
                # if we're already at minimum concurrency let's just ride it out
                self.scale_down(1)
                return True

class OffPeakLoadBasedAutoscaler(LoadBasedAutoscaler):
    def _is_off_peak(self):
        now = datetime.utcnow().time()
        if settings.OFF_PEAK_TIME:
            time_begin = settings.OFF_PEAK_TIME[0]
            time_end = settings.OFF_PEAK_TIME[1]
            if time_begin < time_end:  # off peak is middle of day
                if time_begin < now < time_end:
                    return True
            else:  # off peak is overnight
                if time_begin < now or now < time_end:
                    return True

        # if this setting isn't set consider us always during peak time
        return False

    def _during_peak_time(self):
        return not self._is_off_peak()

    def _maybe_scale(self, req=None):
        procs = self.processes

        if self._during_peak_time():
            if procs > self.min_concurrency:
                self.scale_down(1)
                return True
            elif procs == self.min_concurrency:
                return False

        return super(OffPeakLoadBasedAutoscaler, self)._maybe_scale(req)

class LoadBasedLoader(DjangoLoader):
    def read_configuration(self):
        ret = super(LoadBasedLoader, self).read_configuration()
        ret['CELERYD_AUTOSCALER'] = 'corehq.util.celery_utils:LoadBasedAutoscaler'
        return ret

class OffPeakLoadBasedLoader(DjangoLoader):
    def read_configuration(self):
        ret = super(OffPeakLoadBasedLoader, self).read_configuration()
        ret['CELERYD_AUTOSCALER'] = 'corehq.util.celery_utils:OffPeakLoadBasedAutoscaler'
        return ret
<commit_msg>Update celery settings.<commit_after>from __future__ import print_function
import multiprocessing
import os

import kombu.five
from celery import Celery
from celery import current_app
from celery.backends.base import DisabledBackend
from celery.task import task
from celery.worker.autoscale import Autoscaler
from django.conf import settings
from djcelery.loaders import DjangoLoader
from datetime import datetime
from time import sleep, time

def no_result_task(*args, **kwargs):
    """
    We use an instance of DatabaseBackend to store results of celery tasks.

    But even with ignore_result=True, celery will still create an entry into
    celery_taskmeta when creating the task because we have the DatabaseBackend
    configured.

    In order to avoid creating an entry into celery_taskmeta, we also need
    to override the task's result backend to be an instance of the DisabledBackend.

    Use this decorator to create tasks for which we don't need to store
    result info.
    """
    kwargs['ignore_result'] = True
    kwargs['backend'] = DisabledBackend(current_app)

    def wrapper(fcn):
        return task(*args, **kwargs)(fcn)

    return wrapper

class TaskInfo(object):

    def __init__(self, _id, name, time_start=None):
        self.id = _id
        self.name = name
        # http://stackoverflow.com/questions/20091505/celery-task-with-a-time-start-attribute-in-1970
        if time_start:
            self.time_start = datetime.fromtimestamp(time() - kombu.five.monotonic() + time_start)
        else:
            self.time_start = None

class InvalidTaskTypeError(Exception):
    pass

def get_active_task_info(tasks):
    result = []
    for task_dict in tasks:
        result.append(TaskInfo(task_dict.get('id'), task_dict.get('name'), task_dict.get('time_start')))
    return result

def get_reserved_task_info(tasks):
    result = []
    for task_dict in tasks:
        result.append(TaskInfo(task_dict.get('id'), task_dict.get('name')))
    return result

def get_scheduled_task_info(tasks):
    result = []
    for task_dict in tasks:
        task_dict = task_dict.get('request', {})
        result.append(TaskInfo(task_dict.get('id'), task_dict.get('name')))
    return result

def _validate_task_state(task_state, allow_active=True, allow_scheduled=True,
        allow_reserved=True, allow_revoked=True):

    allowed_states = []
    if allow_active:
        allowed_states.append('active')

    if allow_scheduled:
        allowed_states.append('scheduled')

    if allow_reserved:
        allowed_states.append('reserved')

    if allow_revoked:
        allowed_states.append('revoked')

    if task_state not in allowed_states:
        raise InvalidTaskTypeError("Task state must be one of: {}".format(allowed_states))

def _get_task_info_fcn(task_state):
    return {
        'active': get_active_task_info,
        'reserved': get_reserved_task_info,
        'scheduled': get_scheduled_task_info,
    }.get(task_state)

def revoke_tasks(task_names, interval=5):
    """
    Constantly polls all workers for any active, reserved, or scheduled
    tasks, and revokes the tasks if they match any of the given task names.

    Note that when reserved or scheduled tasks are revoked, they don't disappear from
    the list of reserved or scheduled tasks immediately. Instead, they will just be
    discarded by the worker when they try to move to active.

    :param task_names: a list of fully qualified task names to revoke
    :param interval: the interval (in seconds) on which to poll the workers for tasks

    Example:
    revoke_tasks(['couchexport.tasks.export_async'])
    """
    app = Celery()
    app.config_from_object(settings)
    task_ids = set()
    while True:
        tasks = []
        inspect = app.control.inspect()
        for task_state in ['active', 'reserved', 'scheduled']:
            result = getattr(inspect, task_state)()
            if not result:
                continue

            for worker, task_dicts in result.iteritems():
                tasks.extend(_get_task_info_fcn(task_state)(task_dicts))

        for task in tasks:
            if task.name in task_names and task.id not in task_ids:
                app.control.revoke(task.id, terminate=True)
                task_ids.add(task.id)
                print(datetime.utcnow(), 'Revoked', task.id, task.name)

        sleep(interval)

def print_tasks(worker, task_state):
    """
    Prints celery tasks that have been received by the given worker that are in the
    given state.

    Examples:
    print_tasks('celery@hqcelery0.internal-va.commcarehq.org_main', 'active')
    print_tasks('celery@hqcelery1.internal-va.commcarehq.org_sms_queue', 'reserved')
    """
    _validate_task_state(task_state)

    app = Celery()
    app.config_from_object(settings)
    inspect = app.control.inspect([worker])
    fcn = getattr(inspect, task_state)
    result = fcn()
    if not result:
        print("Worker does not appear to be online. Check worker name, and that it is running.")
        return

    tasks = result[worker]

    if not tasks:
        print('(none)')
        return

    if task_state == 'revoked':
        for task_id in tasks:
            print(task_id)
        return

    tasks = _get_task_info_fcn(task_state)(tasks)

    for task_info in tasks:
        if task_info.time_start:
            print(task_info.id, task_info.time_start, task_info.name)
        else:
            print(task_info.id, task_info.name)

def get_running_workers(timeout=10):
    app = Celery()
    app.config_from_object(settings)
    result = app.control.ping(timeout=timeout)

    worker_names = []
    for worker_info in result:
        worker_names.extend(worker_info.keys())

    return worker_names

class LoadBasedAutoscaler(Autoscaler):
    def _maybe_scale(self, req=None):
        procs = self.processes
        cur = min(self.qty, self.max_concurrency)

        available_cpus = multiprocessing.cpu_count()
        try:
            load_avgs = os.getloadavg()
            one_min_avg = load_avgs[0]
            normalized_load = one_min_avg / available_cpus
        except OSError:
            # if we can't get the load average, let's just use normal autoscaling
            load_avgs = None
            normalized_load = 0

        if cur > procs and normalized_load < 0.90:
            self.scale_up(cur - procs)
            return True
        elif procs > self.min_concurrency:
            if cur < procs:
                self.scale_down(min(procs - cur, procs - self.min_concurrency))
                return True
            elif normalized_load > 0.90:
                # if load is too high trying scaling down 1 worker at a time.
                # if we're already at minimum concurrency let's just ride it out
                self.scale_down(1)
                return True

class OffPeakLoadBasedAutoscaler(LoadBasedAutoscaler):
    def _is_off_peak(self):
        now = datetime.utcnow().time()
        if settings.OFF_PEAK_TIME:
            time_begin = settings.OFF_PEAK_TIME[0]
            time_end = settings.OFF_PEAK_TIME[1]
            if time_begin < time_end:  # off peak is middle of day
                if time_begin < now < time_end:
                    return True
            else:  # off peak is overnight
                if time_begin < now or now < time_end:
                    return True

        # if this setting isn't set consider us always during peak time
        return False

    def _during_peak_time(self):
        return not self._is_off_peak()

    def _maybe_scale(self, req=None):
        procs = self.processes

        if self._during_peak_time():
            if procs > self.min_concurrency:
                self.scale_down(1)
                return True
            elif procs == self.min_concurrency:
                return False

        return super(OffPeakLoadBasedAutoscaler, self)._maybe_scale(req)

class LoadBasedLoader(DjangoLoader):
    def read_configuration(self):
        ret = super(LoadBasedLoader, self).read_configuration()
        ret['CELERYD_AUTOSCALER'] = 'corehq.util.celery_utils:LoadBasedAutoscaler'
        ret['CELERYD_PREFETCH_MULTIPLIER'] = 1
        ret['AUTOSCALE_KEEPALIVE'] = 120
        return ret

class OffPeakLoadBasedLoader(LoadBasedLoader):
    def read_configuration(self):
        ret = super(OffPeakLoadBasedLoader, self).read_configuration()
        ret['CELERYD_AUTOSCALER'] = 'corehq.util.celery_utils:OffPeakLoadBasedAutoscaler'
        return ret
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import code
import six
import sys
import threading
import websocket
try:
    import readline
except:
    pass

OPCODE_DATA = (websocket.ABNF.OPCODE_TEXT, websocket.ABNF.OPCODE_BINARY)
ENCODING = getattr(sys.stdin, "encoding", "").lower()

class VAction(argparse.Action):
    def __call__(self, parser, args, values, option_string=None):
        if values==None:
            values = "1"
        try:
            values = int(values)
        except ValueError:
            values = values.count("v")+1
        setattr(args, self.dest, values)

def parse_args():
    parser = argparse.ArgumentParser(description="WebSocket Simple Dump Tool")
    parser.add_argument("url", metavar="ws_url",
                        help="websocket url. ex. ws://echo.websocket.org/")
    parser.add_argument("-v", "--verbose", default=0, nargs='?', action=VAction,
                        dest="verbose",
                        help="set verbose mode. If set to 1, show opcode. "
                        "If set to 2, enable to trace  websocket module")
    parser.add_argument("-n", "--nocert", action='store_true',
                        help="Ignore invalid SSL cert")

    return parser.parse_args()

class InteractiveConsole(code.InteractiveConsole):
    def write(self, data):
        sys.stdout.write("\033[2K\033[E")
        # sys.stdout.write("\n")
        sys.stdout.write("\033[34m" + data + "\033[39m")
        sys.stdout.write("\n> ")
        sys.stdout.flush()

    def raw_input(self, prompt):
        if six.PY3:
            line = input(prompt)
        else:
            line = raw_input(prompt)

        if ENCODING and ENCODING != "utf-8" and not isinstance(line, six.text_type):
            line = line.decode(ENCODING).encode("utf-8")
        elif isinstance(line, six.text_type):
            line = line.encode("utf-8")

        return line

def main():
    args = parse_args()
    console = InteractiveConsole()
    if args.verbose > 1:
        websocket.enableTrace(True)
    opts = {}
    if (args.nocert):
        opts = { "cert_reqs": websocket.ssl.CERT_NONE, "check_hostname": False }
    ws = websocket.create_connection(args.url, sslopt=opts)
    print("Press Ctrl+C to quit")

    def recv():
        frame = ws.recv_frame()
        if not frame:
            raise websocket.WebSocketException("Not a valid frame %s" % frame)
        elif frame.opcode in OPCODE_DATA:
            return (frame.opcode, frame.data)
        elif frame.opcode == websocket.ABNF.OPCODE_CLOSE:
            ws.send_close()
            return (frame.opcode, None)
        elif frame.opcode == websocket.ABNF.OPCODE_PING:
            ws.pong("Hi!")
            return frame.opcode, frame.data

        return frame.opcode, frame.data

    def recv_ws():
        while True:
            opcode, data = recv()
            msg = None
            if not args.verbose and opcode in OPCODE_DATA:
                msg = "< %s" % data
            elif args.verbose:
                msg = "< %s: %s" % (websocket.ABNF.OPCODE_MAP.get(opcode), data)

            if msg:
                console.write(msg)

    thread = threading.Thread(target=recv_ws)
    thread.daemon = True
    thread.start()

    while True:
        try:
            message = console.raw_input("> ")
            ws.send(message)
        except KeyboardInterrupt:
            return
        except EOFError:
            return

if __name__ == "__main__":
    try:
        main()
    except Exception as e:
        print(e)
<commit_msg>Changes to wsdump.py. Add command line options and handle closed socket. Additional comamnd line options: * -s proto1 proto2 - set one or more subprotocols * -o origin - set origin * -t "initial text" - allow some text be set from command line<commit_after>#!/usr/bin/env python

import argparse
import code
import six
import sys
import threading
import websocket
try:
    import readline
except:
    pass

OPCODE_DATA = (websocket.ABNF.OPCODE_TEXT, websocket.ABNF.OPCODE_BINARY)
ENCODING = getattr(sys.stdin, "encoding", "").lower()

class VAction(argparse.Action):
    def __call__(self, parser, args, values, option_string=None):
        if values==None:
            values = "1"
        try:
            values = int(values)
        except ValueError:
            values = values.count("v")+1
        setattr(args, self.dest, values)

def parse_args():
    parser = argparse.ArgumentParser(description="WebSocket Simple Dump Tool")
    parser.add_argument("url", metavar="ws_url",
                        help="websocket url. ex. ws://echo.websocket.org/")
    parser.add_argument("-v", "--verbose", default=0, nargs='?', action=VAction,
                        dest="verbose",
                        help="set verbose mode. If set to 1, show opcode. "
                        "If set to 2, enable to trace  websocket module")
    parser.add_argument("-n", "--nocert", action='store_true',
                        help="Ignore invalid SSL cert")
    parser.add_argument("-s", "--subprotocols", nargs='*',
                        help="Set subprotocols")
    parser.add_argument("-o", "--origin",
                        help="Set origin")
    parser.add_argument("-t", "--text",
                        help="Send initial text")

    return parser.parse_args()

class InteractiveConsole(code.InteractiveConsole):
    def write(self, data):
        sys.stdout.write("\033[2K\033[E")
        # sys.stdout.write("\n")
        sys.stdout.write("\033[34m" + data + "\033[39m")
        sys.stdout.write("\n> ")
        sys.stdout.flush()

    def raw_input(self, prompt):
        if six.PY3:
            line = input(prompt)
        else:
            line = raw_input(prompt)

        if ENCODING and ENCODING != "utf-8" and not isinstance(line, six.text_type):
            line = line.decode(ENCODING).encode("utf-8")
        elif isinstance(line, six.text_type):
            line = line.encode("utf-8")

        return line

def main():
    args = parse_args()
    console = InteractiveConsole()
    if args.verbose > 1:
        websocket.enableTrace(True)
    opts = {}
    if (args.origin):
        opts = { "origin": args.origin }
    if (args.subprotocols):
        opts = { "subprotocols": args.subprotocols }
    if (args.nocert):
        opts = { "cert_reqs": websocket.ssl.CERT_NONE, "check_hostname": False }
    ws = websocket.create_connection(args.url, sslopt=opts)
    print("Press Ctrl+C to quit")

    def recv():
        try:
            frame = ws.recv_frame()
        except websocket.WebSocketException:
            return (websocket.ABNF.OPCODE_CLOSE, None)
        if not frame:
            raise websocket.WebSocketException("Not a valid frame %s" % frame)
        elif frame.opcode in OPCODE_DATA:
            return (frame.opcode, frame.data)
        elif frame.opcode == websocket.ABNF.OPCODE_CLOSE:
            ws.send_close()
            return (frame.opcode, None)
        elif frame.opcode == websocket.ABNF.OPCODE_PING:
            ws.pong(frame.data)
            return frame.opcode, frame.data

        return frame.opcode, frame.data

    def recv_ws():
        while True:
            opcode, data = recv()
            msg = None
            if not args.verbose and opcode in OPCODE_DATA:
                msg = "< %s" % data
            elif args.verbose:
                msg = "< %s: %s" % (websocket.ABNF.OPCODE_MAP.get(opcode), data)

            if msg:
                console.write(msg)

            if opcode == websocket.ABNF.OPCODE_CLOSE:
                break

    thread = threading.Thread(target=recv_ws)
    thread.daemon = True
    thread.start()

    if args.text:
        ws.send(args.text)

    while True:
        try:
            message = console.raw_input("> ")
            ws.send(message)
        except KeyboardInterrupt:
            return
        except EOFError:
            return

if __name__ == "__main__":
    try:
        main()
    except Exception as e:
        print(e)
<|endoftext|>


<commit_before>from galah.web import app
from galah.web.auth import account_type_required
from galah.db.models import Assignment, Submission, Archive
from bson.objectid import ObjectId
from bson.errors import InvalidId
from flask import send_file, abort
from flask.ext.login import current_user
import os.path
import subprocess
import tempfile
import datetime

@app.route("/assignments/<assignment_id>/<submission_id>/download.tar.gz")
@account_type_required(("student", "teacher"))
def download_submission(assignment_id, submission_id):
    # Figure out which assignment the user asked for.
    try:
        assignment_id = ObjectId(assignment_id)
        assignment = Assignment.objects.get(id = assignment_id)
    except InvalidId, Assignment.DoesNotExist:
        app.logger.exception("Could not retrieve assignment.")

        abort(404)

    # Figure out which submission the user is trying to download
    try:
        submission_id = ObjectId(submission_id)
        submission = Submission.objects.get(id = submission_id)
    except InvalidId, Submission.DoesNotExist:
        app.logger.exception("Could not retrieve submission.")

        abort(404)

    # Find any expired archives and remove them
    for i in Archive.objects(expires__lt = datetime.datetime.today()):
        if i.file_location:
            app.logger.debug("Erasing old archive at '%s'." % i.file_location)

            try:
                os.remove(i.file_location)
            except OSError:
                app.logger.exception(
                    "Could not remove expired archive at %s." %
                        i.file_location
                )

        i.delete()

    new_archive = Archive(
        requester = current_user.id,
        archive_type = "single_submission"
    )

    archive_file_name = ""
    try:
        # Create the actual archive file.
        # TODO: Create it in galah's /var/ directory
        archive_fd, archive_file_name = tempfile.mkstemp(suffix = ".tar.gz")

        # Close the file because we won't actually do any writing to it,
        # rather tar will.
        os.close(archive_fd)

        # Run tar and do the actual archiving. Will block until it's finished.
        subprocess.check_call(
            [
                "tar", "--dereference", "--create", "--gzip", "--directory",
                os.path.join(app.config["SUBMISSION_DIRECTORY"], str(submission_id)),
                "--file", archive_file_name
            ] + submission.uploaded_filenames
        )

        new_archive.file_location = archive_file_name

        new_archive.expires = \
                datetime.datetime.today() + app.config["ARCHIVE_LIFETIME"]

        new_archive.save(force_insert = True)

        return send_file(archive_file_name)
    except Exception as e:
        app.logger.exception("An error occured while creating an archive.")

        # If we created a temporary archive file we need to delete it.
        new_archive.file_location = None
        if archive_file_name:
            os.remove(archive_file_name)

        new_archive.error_string = str(e)
        new_archive.save(force_insert = True)

        abort(500)<commit_msg>Changed .tar.gz extention to .tgz.<commit_after>from galah.web import app
from galah.web.auth import account_type_required
from galah.db.models import Assignment, Submission, Archive
from bson.objectid import ObjectId
from bson.errors import InvalidId
from flask import send_file, abort
from flask.ext.login import current_user
import os.path
import subprocess
import tempfile
import datetime

@app.route("/assignments/<assignment_id>/<submission_id>/download.tar.gz")
@account_type_required(("student", "teacher"))
def download_submission(assignment_id, submission_id):
    # Figure out which assignment the user asked for.
    try:
        assignment_id = ObjectId(assignment_id)
        assignment = Assignment.objects.get(id = assignment_id)
    except InvalidId, Assignment.DoesNotExist:
        app.logger.exception("Could not retrieve assignment.")

        abort(404)

    # Figure out which submission the user is trying to download
    try:
        submission_id = ObjectId(submission_id)
        submission = Submission.objects.get(id = submission_id)
    except InvalidId, Submission.DoesNotExist:
        app.logger.exception("Could not retrieve submission.")

        abort(404)

    # Find any expired archives and remove them
    for i in Archive.objects(expires__lt = datetime.datetime.today()):
        if i.file_location:
            app.logger.debug("Erasing old archive at '%s'." % i.file_location)

            try:
                os.remove(i.file_location)
            except OSError:
                app.logger.exception(
                    "Could not remove expired archive at %s." %
                        i.file_location
                )

        i.delete()

    new_archive = Archive(
        requester = current_user.id,
        archive_type = "single_submission"
    )

    archive_file_name = ""
    try:
        # Create the actual archive file.
        # TODO: Create it in galah's /var/ directory
        archive_fd, archive_file_name = tempfile.mkstemp(suffix = ".tgz")

        # Close the file because we won't actually do any writing to it,
        # rather tar will.
        os.close(archive_fd)

        # Run tar and do the actual archiving. Will block until it's finished.
        subprocess.check_call(
            [
                "tar", "--dereference", "--create", "--gzip", "--directory",
                os.path.join(app.config["SUBMISSION_DIRECTORY"], str(submission_id)),
                "--file", archive_file_name
            ] + submission.uploaded_filenames
        )

        new_archive.file_location = archive_file_name

        new_archive.expires = \
                datetime.datetime.today() + app.config["ARCHIVE_LIFETIME"]

        new_archive.save(force_insert = True)

        return send_file(archive_file_name)
    except Exception as e:
        app.logger.exception("An error occured while creating an archive.")

        # If we created a temporary archive file we need to delete it.
        new_archive.file_location = None
        if archive_file_name:
            os.remove(archive_file_name)

        new_archive.error_string = str(e)
        new_archive.save(force_insert = True)

        abort(500)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Manage the information in the hosts file
'''
from __future__ import absolute_import

# Import python libs
import os

# Import salt libs
import salt.utils
import salt.utils.odict as odict
from salt.ext.six.moves import range
import salt.ext.six as six

# pylint: disable=C0103
def __get_hosts_filename():
    '''
    Return the path to the appropriate hosts file
    '''
    # TODO: Investigate using  "%SystemRoot%\system32" for this
    if salt.utils.is_windows():
        return 'C:\\Windows\\System32\\drivers\\etc\\hosts'

    return __salt__['config.option']('hosts.file')

def _list_hosts():
    '''
    Return the hosts found in the hosts file in as an OrderedDict
    '''
    count = 0
    hfn = __get_hosts_filename()
    ret = odict.OrderedDict()
    if not os.path.isfile(hfn):
        return ret
    with salt.utils.fopen(hfn) as ifile:
        for line in ifile:
            line = line.strip()
            if not line:
                continue
            if line.startswith('#'):
                ret.setdefault('comment-{0}'.format(count), []).extend(line)
                count += 1
                continue
            if '#' in line:
                line = line[:line.index('#')].strip()
            comps = line.split()
            ip = comps.pop(0)
            ret.setdefault(ip, []).extend(comps)
    return ret

def list_hosts():
    '''
    Return the hosts found in the hosts file in this format::

        {'<ip addr>': ['alias1', 'alias2', ...]}

    CLI Example:

    .. code-block:: bash

        salt '*' hosts.list_hosts
    '''
    # msgpack does not like OrderedDict's
    return dict(_list_hosts())

def get_ip(host):
    '''
    Return the ip associated with the named host

    CLI Example:

    .. code-block:: bash

        salt '*' hosts.get_ip <hostname>
    '''
    hosts = _list_hosts()
    if not hosts:
        return ''
    # Look for the op
    for addr in hosts:
        if host in hosts[addr]:
            return addr
    # ip not found
    return ''

def get_alias(ip):
    '''
    Return the list of aliases associated with an ip

    CLI Example:

    .. code-block:: bash

        salt '*' hosts.get_alias <ip addr>
    '''
    hosts = _list_hosts()
    if ip in hosts:
        return hosts[ip]
    return []

def has_pair(ip, alias):
    '''
    Return true if the alias is set

    CLI Example:

    .. code-block:: bash

        salt '*' hosts.has_pair <ip> <alias>
    '''
    hosts = _list_hosts()
    return ip in hosts and alias in hosts[ip]

def set_host(ip, alias):
    '''
    Set the host entry in the hosts file for the given ip, this will overwrite
    any previous entry for the given ip

    CLI Example:

    .. code-block:: bash

        salt '*' hosts.set_host <ip> <alias>
    '''
    hfn = __get_hosts_filename()
    ovr = False
    if not os.path.isfile(hfn):
        return False
    lines = salt.utils.fopen(hfn).readlines()
    for ind, line in enumerate(lines):
        tmpline = line.strip()
        if not tmpline:
            continue
        if tmpline.startswith('#'):
            continue
        comps = tmpline.split()
        if comps[0] == ip:
            if not ovr:
                lines[ind] = ip + '\t\t' + alias + '\n'
                ovr = True
            else:  # remove other entries
                lines[ind] = ''
    if not ovr:
        # make sure there is a newline
        if lines and not lines[-1].endswith(('\n', '\r')):
            lines[-1] = '{0}\n'.format(lines[-1])
        line = ip + '\t\t' + alias + '\n'
        lines.append(line)
    with salt.utils.fopen(hfn, 'w+') as ofile:
        ofile.writelines(lines)
    return True

def rm_host(ip, alias):
    '''
    Remove a host entry from the hosts file

    CLI Example:

    .. code-block:: bash

        salt '*' hosts.rm_host <ip> <alias>
    '''
    if not has_pair(ip, alias):
        return True
    hfn = __get_hosts_filename()
    lines = salt.utils.fopen(hfn).readlines()
    for ind in range(len(lines)):
        tmpline = lines[ind].strip()
        if not tmpline:
            continue
        if tmpline.startswith('#'):
            continue
        comps = tmpline.split()
        if comps[0] == ip:
            newline = '{0}\t\t'.format(comps[0])
            for existing in comps[1:]:
                if existing == alias:
                    continue
                newline += ' {0}'.format(existing)
            if newline.strip() == ip:
                # No aliases exist for the line, make it empty
                lines[ind] = ''
            else:
                # Only an alias was removed
                lines[ind] = '{0}\n'.format(newline)
    with salt.utils.fopen(hfn, 'w+') as ofile:
        ofile.writelines(lines)
    return True

def add_host(ip, alias):
    '''
    Add a host to an existing entry, if the entry is not in place then create
    it with the given host

    CLI Example:

    .. code-block:: bash

        salt '*' hosts.add_host <ip> <alias>
    '''
    hfn = __get_hosts_filename()
    if not os.path.isfile(hfn):
        return False

    if has_pair(ip, alias):
        return True

    hosts = _list_hosts()
    inserted = False
    for i, h in hosts.items():
        for j in range(len(h)):
            if h[j].startswith('#') and i == ip:
                h.insert(j, alias)
                inserted = True
    if not inserted:
        hosts.setdefault(ip, []).append(alias)
    _write_hosts(hosts)
    return True

def _write_hosts(hosts):
    lines = []
    for ip, aliases in six.iteritems(hosts):
        if ip:
            if ip.startswith('comment'):
                line = ''.join(aliases)
            else:
                line = '{0}\t\t{1}'.format(
                    ip,
                    ' '.join(aliases)
                    )
        lines.append(line)

    hfn = __get_hosts_filename()
    with salt.utils.fopen(hfn, 'w+') as ofile:
        for line in lines:
            if line.strip():
                # /etc/hosts needs to end with EOL so that some utils that read
                # it do not break
                ofile.write('{0}\n'.format(line.strip()))
<commit_msg>Py3 compatibility fixes.<commit_after># -*- coding: utf-8 -*-
'''
Manage the information in the hosts file
'''

# Import python libs
from __future__ import absolute_import
import os

# Import salt libs
import salt.utils
import salt.utils.odict as odict

# Import 3rd-party libs
import salt.ext.six as six
from salt.ext.six.moves import range  # pylint: disable=import-error,no-name-in-module,redefined-builtin

# pylint: disable=C0103
def __get_hosts_filename():
    '''
    Return the path to the appropriate hosts file
    '''
    # TODO: Investigate using  "%SystemRoot%\system32" for this
    if salt.utils.is_windows():
        return 'C:\\Windows\\System32\\drivers\\etc\\hosts'

    return __salt__['config.option']('hosts.file')

def _list_hosts():
    '''
    Return the hosts found in the hosts file in as an OrderedDict
    '''
    count = 0
    hfn = __get_hosts_filename()
    ret = odict.OrderedDict()
    if not os.path.isfile(hfn):
        return ret
    with salt.utils.fopen(hfn) as ifile:
        for line in ifile:
            line = line.strip()
            if not line:
                continue
            if line.startswith('#'):
                ret.setdefault('comment-{0}'.format(count), []).extend(line)
                count += 1
                continue
            if '#' in line:
                line = line[:line.index('#')].strip()
            comps = line.split()
            ip = comps.pop(0)
            ret.setdefault(ip, []).extend(comps)
    return ret

def list_hosts():
    '''
    Return the hosts found in the hosts file in this format::

        {'<ip addr>': ['alias1', 'alias2', ...]}

    CLI Example:

    .. code-block:: bash

        salt '*' hosts.list_hosts
    '''
    # msgpack does not like OrderedDict's
    return dict(_list_hosts())

def get_ip(host):
    '''
    Return the ip associated with the named host

    CLI Example:

    .. code-block:: bash

        salt '*' hosts.get_ip <hostname>
    '''
    hosts = _list_hosts()
    if not hosts:
        return ''
    # Look for the op
    for addr in hosts:
        if host in hosts[addr]:
            return addr
    # ip not found
    return ''

def get_alias(ip):
    '''
    Return the list of aliases associated with an ip

    CLI Example:

    .. code-block:: bash

        salt '*' hosts.get_alias <ip addr>
    '''
    hosts = _list_hosts()
    if ip in hosts:
        return hosts[ip]
    return []

def has_pair(ip, alias):
    '''
    Return true if the alias is set

    CLI Example:

    .. code-block:: bash

        salt '*' hosts.has_pair <ip> <alias>
    '''
    hosts = _list_hosts()
    return ip in hosts and alias in hosts[ip]

def set_host(ip, alias):
    '''
    Set the host entry in the hosts file for the given ip, this will overwrite
    any previous entry for the given ip

    CLI Example:

    .. code-block:: bash

        salt '*' hosts.set_host <ip> <alias>
    '''
    hfn = __get_hosts_filename()
    ovr = False
    if not os.path.isfile(hfn):
        return False
    lines = salt.utils.fopen(hfn).readlines()
    for ind, line in enumerate(lines):
        tmpline = line.strip()
        if not tmpline:
            continue
        if tmpline.startswith('#'):
            continue
        comps = tmpline.split()
        if comps[0] == ip:
            if not ovr:
                lines[ind] = ip + '\t\t' + alias + '\n'
                ovr = True
            else:  # remove other entries
                lines[ind] = ''
    if not ovr:
        # make sure there is a newline
        if lines and not lines[-1].endswith(('\n', '\r')):
            lines[-1] = '{0}\n'.format(lines[-1])
        line = ip + '\t\t' + alias + '\n'
        lines.append(line)
    with salt.utils.fopen(hfn, 'w+') as ofile:
        ofile.writelines(lines)
    return True

def rm_host(ip, alias):
    '''
    Remove a host entry from the hosts file

    CLI Example:

    .. code-block:: bash

        salt '*' hosts.rm_host <ip> <alias>
    '''
    if not has_pair(ip, alias):
        return True
    hfn = __get_hosts_filename()
    lines = salt.utils.fopen(hfn).readlines()
    for ind in range(len(lines)):
        tmpline = lines[ind].strip()
        if not tmpline:
            continue
        if tmpline.startswith('#'):
            continue
        comps = tmpline.split()
        if comps[0] == ip:
            newline = '{0}\t\t'.format(comps[0])
            for existing in comps[1:]:
                if existing == alias:
                    continue
                newline += ' {0}'.format(existing)
            if newline.strip() == ip:
                # No aliases exist for the line, make it empty
                lines[ind] = ''
            else:
                # Only an alias was removed
                lines[ind] = '{0}\n'.format(newline)
    with salt.utils.fopen(hfn, 'w+') as ofile:
        ofile.writelines(lines)
    return True

def add_host(ip, alias):
    '''
    Add a host to an existing entry, if the entry is not in place then create
    it with the given host

    CLI Example:

    .. code-block:: bash

        salt '*' hosts.add_host <ip> <alias>
    '''
    hfn = __get_hosts_filename()
    if not os.path.isfile(hfn):
        return False

    if has_pair(ip, alias):
        return True

    hosts = _list_hosts()
    inserted = False
    for i, h in six.iteritems(hosts):
        for j in range(len(h)):
            if h[j].startswith('#') and i == ip:
                h.insert(j, alias)
                inserted = True
    if not inserted:
        hosts.setdefault(ip, []).append(alias)
    _write_hosts(hosts)
    return True

def _write_hosts(hosts):
    lines = []
    for ip, aliases in six.iteritems(hosts):
        if ip:
            if ip.startswith('comment'):
                line = ''.join(aliases)
            else:
                line = '{0}\t\t{1}'.format(
                    ip,
                    ' '.join(aliases)
                    )
        lines.append(line)

    hfn = __get_hosts_filename()
    with salt.utils.fopen(hfn, 'w+') as ofile:
        for line in lines:
            if line.strip():
                # /etc/hosts needs to end with EOL so that some utils that read
                # it do not break
                ofile.write('{0}\n'.format(line.strip()))
<|endoftext|>


<commit_before># This file is part of morss
#
# Copyright (C) 2013-2020 pictuga <contact@pictuga.com>
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <https://www.gnu.org/licenses/>.

import os.path
import sys

def pkg_path(path=''):
    return os.path.join(os.path.dirname(__file__), path)

data_path_base = None

def data_path(path=''):
    global data_path_base

    if data_path_base is not None:
        return os.path.join(data_path_base, path)

    bases = [
        os.path.join(sys.prefix, 'share/morss/www'),
        pkg_path('../../../share/morss/www'),
        pkg_path('../../../../share/morss/www'),
        pkg_path('../www'),
        pkg_path('../..')
    ]

    for base in bases:
        full_path = os.path.join(base, path)

        if os.path.isfile(full_path):
            data_path_base = os.path.abspath(base)
            return data_path(path)

    else:
        raise IOError()
<commit_msg>util: support more `data_files` location<commit_after># This file is part of morss
#
# Copyright (C) 2013-2020 pictuga <contact@pictuga.com>
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <https://www.gnu.org/licenses/>.

import os.path
import sys

def pkg_path(path=''):
    return os.path.join(os.path.dirname(__file__), path)

data_path_base = None

def data_path(path=''):
    global data_path_base

    if data_path_base is not None:
        return os.path.join(data_path_base, path)

    bases = [
        os.path.join(sys.prefix, 'share/morss/www'), # when installed as root
        pkg_path('../../../share/morss/www'),
        pkg_path('../../../../share/morss/www'),
        pkg_path('../share/morss/www'), # for `pip install --target=dir morss`
        pkg_path('../www'), # when running from source tree
        pkg_path('../..'), # when running on `.cgi` subdir on Apache
    ]

    for base in bases:
        full_path = os.path.join(base, path)

        if os.path.isfile(full_path):
            data_path_base = os.path.abspath(base)
            return data_path(path)

    else:
        raise IOError()
<|endoftext|>


<commit_before>#coding: utf-8
import numpy as np
from scipy import optimize
from matplotlib import pyplot
from sklearn.datasets import load_digits, fetch_mldata
from sklearn.preprocessing import LabelBinarizer

"""
多層パーセプトロンによる手書き文字認識
・バッチ処理
・共役勾配法（Conjugate Gradient Method）によるパラメータ最適化
"""

def displayData(X):
    """
    データからランダムに100サンプル選んで可視化
    画像データは8x8ピクセルを仮定
    """
    # ランダムに100サンプル選ぶ
    sel = np.random.permutation(X.shape[0])
    sel = sel[:100]
    X = X[sel, :]
    for index, data in enumerate(X):
        pyplot.subplot(10, 10, index + 1)
        pyplot.axis('off')
        image = data.reshape((8, 8))
        print image.shape
        pyplot.imshow(image, cmap=pyplot.cm.gray_r,
                     interpolation='nearest')
    pyplot.show()

def randInitializeWeights(L_in, L_out):
    """
    (-epsilon_init, +epsilon_init) の範囲で
    重みをランダムに初期化した重み行列を返す
    """
    # 入力となる層にはバイアス項が入るので+1が必要なので注意
    epsilon_init = 0.12
    W = np.random.rand(L_out, 1 + L_in) * 2 * epsilon_init - epsilon_init
    return W

def sigmoid(z):
    return 1.0 / (1 + np.exp(-z))

def sigmoidGradient(z):
    return sigmoid(z) * (1 - sigmoid(z))

def safe_log(x, minval=0.0000000001):
    return np.log(x.clip(min=minval))

def nnCostFunction(nn_params, *args):
    """コスト関数Jの偏微分
    偏微分を求めるためには各層の誤差が必要なのでbackpropagationも必要
    Jを求めるときにforward propagationはしているので無駄あり"""
    in_size, hid_size, num_labels, X, y, lam = args

    # ニューラルネットの全パラメータを行列形式に復元
    Theta1 = nn_params[0:(in_size + 1) * hid_size].reshape((hid_size, in_size + 1))
    Theta2 = nn_params[(in_size + 1) * hid_size:].reshape((num_labels, hid_size + 1))

    # パラメータの偏微分
    Theta1_grad = np.zeros(Theta1.shape)
    Theta2_grad = np.zeros(Theta2.shape)

    # 訓練データ数
    m = X.shape[0]

    # 訓練データの1列目にバイアス項に対応する1を追加
    X = np.hstack((np.ones((m, 1)), X))

    # 教師ラベルを1-of-K表記に変換
    lb = LabelBinarizer()
    lb.fit(y)
    y = lb.transform(y)

    J = 0
    for i in range(m):
        xi = X[i, :]
        yi = y[i]
        # forward propagation
        a1 = xi
        z2 = np.dot(Theta1, a1)
        a2 = sigmoid(z2)
        a2 = np.hstack((1, a2))
        z3 = np.dot(Theta2, a2)
        a3 = sigmoid(z3)
        J += sum(-yi * safe_log(a3) - (1 - yi) * safe_log(1 - a3))
        # backpropagation
        delta3 = a3 - yi
        delta2 = np.dot(Theta2.T, delta3) * sigmoidGradient(np.hstack((1, z2)))
        delta2 = delta2[1:]  # バイアス項に対応する要素を除外
        # ベクトル x ベクトル = 行列の演算をしなければならないので
        # 縦ベクトルへのreshapeが必要
        # 行数に-1を指定すると自動的に入る
        delta2 = delta2.reshape((-1, 1))
        delta3 = delta3.reshape((-1, 1))
        a1 = a1.reshape((-1, 1))
        a2 = a2.reshape((-1, 1))
        # 正則化ありのときのデルタの演算
        Theta1_grad += np.dot(delta2, a1.T)
        Theta2_grad += np.dot(delta3, a2.T)

    J /= m

    # コスト関数の正則化項
    temp = 0.0;
    for j in range(hid_size):
        for k in range(1, in_size + 1):  # バイアスに対応する重みは加えない
            temp += Theta1[j, k] ** 2
    for j in range(num_labels):
        for k in range(1, hid_size + 1): # バイアスに対応する重みは加えない
            temp += Theta2[j, k] ** 2
    J += lam / (2.0 * m) * temp;

    # 偏微分の正則化項
    Theta1_grad /= m
    Theta1_grad[:, 1:] += (lam / m) * Theta1_grad[:, 1:]
    Theta2_grad /= m
    Theta2_grad[:, 1:] += (lam / m) * Theta2_grad[:, 1:]

    # ベクトルに変換
    grad = np.hstack((np.ravel(Theta1_grad), np.ravel(Theta2_grad)))

    print J

    return J, grad

if __name__ == "__main__":
    in_size = 64
    hid_size = 25
    num_labels = 10

    # 訓練データをロード
    digits = load_digits()
    X = digits.data
    X /= X.max()
    y = digits.target

    # データを可視化
#    displayData(X)

    # パラメータをランダムに初期化
    initial_Theta1 = randInitializeWeights(in_size, hid_size)
    initial_Theta2 = randInitializeWeights(hid_size, num_labels)
    # 行列ではなくベクトルに展開
    initial_nn_params = np.hstack((np.ravel(initial_Theta1), np.ravel(initial_Theta2)))
    print initial_Theta1.shape
    print initial_Theta2.shape
    print initial_nn_params.shape

    # 正則化係数
    lam = 1.0

    # 初期状態のコストを計算
    J, grad = nnCostFunction(initial_nn_params, in_size, hid_size, num_labels, X, y, lam)
    print "initial cost:", J

    # Conjugate Gradientでパラメータ推定
    # NNはコスト関数と偏微分の計算を同じ関数（nnCostFunction）で行うので
    # fmin_cgではなくminimizeを使用する
    # minimize()はscipy 0.11.0以上が必要
    nn_params = optimize.minimize(func=nnCostFunction, x0=initial_nn_params, method="CG",
                                  options={'maxiter':20, 'disp':True},
                                  args=(in_size, hid_size, num_labels, X, y, lam))

    # パラメータを分解
    Theta1 = nn_params[0:(in_size + 1) * hid_size].reshape((hid_size, in_size + 1))
    Theta2 = nn_params[(in_size + 1) * hid_size:].reshape((num_labels, hid_size + 1))

    print X.shape
    print Theta1.shape
    displayData(Theta1[:, 1:])
<commit_msg>jac=Trueを追加<commit_after>#coding: utf-8
import numpy as np
from scipy import optimize
from matplotlib import pyplot
from sklearn.datasets import load_digits, fetch_mldata
from sklearn.preprocessing import LabelBinarizer

"""
多層パーセプトロンによる手書き文字認識
・バッチ処理
・共役勾配法（Conjugate Gradient Method）によるパラメータ最適化
"""

def displayData(X):
    """
    データからランダムに100サンプル選んで可視化
    画像データは8x8ピクセルを仮定
    """
    # ランダムに100サンプル選ぶ
    sel = np.random.permutation(X.shape[0])
    sel = sel[:100]
    X = X[sel, :]
    for index, data in enumerate(X):
        pyplot.subplot(10, 10, index + 1)
        pyplot.axis('off')
        image = data.reshape((8, 8))
        print image.shape
        pyplot.imshow(image, cmap=pyplot.cm.gray_r,
                     interpolation='nearest')
    pyplot.show()

def randInitializeWeights(L_in, L_out):
    """
    (-epsilon_init, +epsilon_init) の範囲で
    重みをランダムに初期化した重み行列を返す
    """
    # 入力となる層にはバイアス項が入るので+1が必要なので注意
    epsilon_init = 0.12
    W = np.random.rand(L_out, 1 + L_in) * 2 * epsilon_init - epsilon_init
    return W

def sigmoid(z):
    return 1.0 / (1 + np.exp(-z))

def sigmoidGradient(z):
    return sigmoid(z) * (1 - sigmoid(z))

def safe_log(x, minval=0.0000000001):
    return np.log(x.clip(min=minval))

def nnCostFunction(nn_params, *args):
    """コスト関数Jの偏微分
    偏微分を求めるためには各層の誤差が必要なのでbackpropagationも必要
    Jを求めるときにforward propagationはしているので無駄あり"""
    in_size, hid_size, num_labels, X, y, lam = args

    # ニューラルネットの全パラメータを行列形式に復元
    Theta1 = nn_params[0:(in_size + 1) * hid_size].reshape((hid_size, in_size + 1))
    Theta2 = nn_params[(in_size + 1) * hid_size:].reshape((num_labels, hid_size + 1))

    # パラメータの偏微分
    Theta1_grad = np.zeros(Theta1.shape)
    Theta2_grad = np.zeros(Theta2.shape)

    # 訓練データ数
    m = X.shape[0]

    # 訓練データの1列目にバイアス項に対応する1を追加
    X = np.hstack((np.ones((m, 1)), X))

    # 教師ラベルを1-of-K表記に変換
    lb = LabelBinarizer()
    lb.fit(y)
    y = lb.transform(y)

    J = 0
    for i in range(m):
        xi = X[i, :]
        yi = y[i]
        # forward propagation
        a1 = xi
        z2 = np.dot(Theta1, a1)
        a2 = sigmoid(z2)
        a2 = np.hstack((1, a2))
        z3 = np.dot(Theta2, a2)
        a3 = sigmoid(z3)
        J += sum(-yi * safe_log(a3) - (1 - yi) * safe_log(1 - a3))
        # backpropagation
        delta3 = a3 - yi
        delta2 = np.dot(Theta2.T, delta3) * sigmoidGradient(np.hstack((1, z2)))
        delta2 = delta2[1:]  # バイアス項に対応する要素を除外
        # ベクトル x ベクトル = 行列の演算をしなければならないので
        # 縦ベクトルへのreshapeが必要
        # 行数に-1を指定すると自動的に入る
        delta2 = delta2.reshape((-1, 1))
        delta3 = delta3.reshape((-1, 1))
        a1 = a1.reshape((-1, 1))
        a2 = a2.reshape((-1, 1))
        # 正則化ありのときのデルタの演算
        Theta1_grad += np.dot(delta2, a1.T)
        Theta2_grad += np.dot(delta3, a2.T)

    J /= m

    # コスト関数の正則化項
    temp = 0.0;
    for j in range(hid_size):
        for k in range(1, in_size + 1):  # バイアスに対応する重みは加えない
            temp += Theta1[j, k] ** 2
    for j in range(num_labels):
        for k in range(1, hid_size + 1): # バイアスに対応する重みは加えない
            temp += Theta2[j, k] ** 2
    J += lam / (2.0 * m) * temp;

    # 偏微分の正則化項
    Theta1_grad /= m
    Theta1_grad[:, 1:] += (lam / m) * Theta1_grad[:, 1:]
    Theta2_grad /= m
    Theta2_grad[:, 1:] += (lam / m) * Theta2_grad[:, 1:]

    # ベクトルに変換
    grad = np.hstack((np.ravel(Theta1_grad), np.ravel(Theta2_grad)))

    print J

    return J, grad

if __name__ == "__main__":
    in_size = 64
    hid_size = 25
    num_labels = 10

    # 訓練データをロード
    digits = load_digits()
    X = digits.data
    X /= X.max()
    y = digits.target

    # データを可視化
#    displayData(X)

    # パラメータをランダムに初期化
    initial_Theta1 = randInitializeWeights(in_size, hid_size)
    initial_Theta2 = randInitializeWeights(hid_size, num_labels)
    # 行列ではなくベクトルに展開
    initial_nn_params = np.hstack((np.ravel(initial_Theta1), np.ravel(initial_Theta2)))
    print initial_Theta1.shape
    print initial_Theta2.shape
    print initial_nn_params.shape

    # 正則化係数
    lam = 1.0

    # 初期状態のコストを計算
    J, grad = nnCostFunction(initial_nn_params, in_size, hid_size, num_labels, X, y, lam)
    print "initial cost:", J

    # Conjugate Gradientでパラメータ推定
    # NNはコスト関数と偏微分の計算を同じ関数（nnCostFunction）で行うので
    # fmin_cgではなくminimizeを使用する
    # minimize()はscipy 0.11.0以上が必要
    nn_params = optimize.minimize(func=nnCostFunction, x0=initial_nn_params, method="CG", jac=True,
                                  options={'maxiter':20, 'disp':True},
                                  args=(in_size, hid_size, num_labels, X, y, lam))

    # パラメータを分解
    Theta1 = nn_params[0:(in_size + 1) * hid_size].reshape((hid_size, in_size + 1))
    Theta2 = nn_params[(in_size + 1) * hid_size:].reshape((num_labels, hid_size + 1))

    print X.shape
    print Theta1.shape
    displayData(Theta1[:, 1:])
<|endoftext|>