<commit_before>from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

import numpy as np
import scipy.sparse as sp
from six import integer_types
import warnings

from . import Utils
from .Tests import checkDerivative
from . import Maps
from . import Props

__all__ = [
    'BaseObjectiveFunction', 'ComboObjectiveFunction', 'L2ObjectiveFunction'
]

class BaseObjectiveFunction(Props.BaseSimPEG):
    """
    Base Objective Function

    Inherit this to build your own objective function. If building a
    regularization, have a look at
    :class:`SimPEG.Regularization.BaseRegularization` as there are additional
    methods and properties tailored to regularization of a model. Similarly,
    for building a data misfit, see :class:`SimPEG.DataMisfit.BaseDataMisfit`.
    """

    counter = None
    debug = False

    mapPair = Maps.IdentityMap  #: Base class of expected maps
    _mapping = None  #: An IdentityMap instance.
    _hasFields = False  #: should we have the option to store fields

    _nP = None  #: number of parameters

    def __init__(self, nP=None, **kwargs):
        if nP is not None:
            self._nP = nP
        Utils.setKwargs(self, **kwargs)

    def __call__(self, x, f=None):
        raise NotImplementedError(
            '__call__ has not been implemented for {} yet'.format(
                self.__class__.__name__
            )
        )

    @property
    def nP(self):
        """
        Number of model parameters expected.
        """
        if self._nP is not None:
            return self._nP
        if getattr(self, 'mapping', None) is not None:
            return self.mapping.nP
        return '*'

    @property
    def _nC_residual(self):
        """
        Shape of the residual
        """
        if getattr(self, 'mapping', None) is not None:
            return self.mapping.shape[0]
        else:
            return self.nP

    @property
    def mapping(self):
        """
        A `SimPEG.Maps` instance
        """
        if getattr(self, '_mapping') is None:
            if getattr(self, '_nP') is not None:
                self._mapping = self.mapPair(nP=self.nP)
            else:
                self._mapping = self.mapPair()
        return self._mapping

    @mapping.setter
    def mapping(self, value):
        assert isinstance(value, self.mapPair), (
            'mapping must be an instance of a {}, not a {}'
        ).format(self.mapPair, value.__class__.__name__)
        self._mapping = value

    @Utils.timeIt
    def __call__(self, x, f=None):
        raise NotImplementedError(
            "The method __call__ has not been implemented for {}".format(
                self.__class__.__name__
            )
        )

    @Utils.timeIt
    def deriv(self, x, **kwargs):
        """
        First derivative of the objective function with respect to the model
        """
        raise NotImplementedError(
            "The method deriv has not been implemented for {}".format(
                self.__class__.__name__
            )
        )

    @Utils.timeIt
    def deriv2(self, x, v=None, **kwargs):
        """
        Second derivative of the objective function with respect to the model
        """
        raise NotImplementedError(
            "The method _deriv2 has not been implemented for {}".format(
                self.__class__.__name__
            )
        )

    def _test_deriv(self, x=None, num=4, plotIt=False, **kwargs):
        print('Testing {0!s} Deriv'.format(self.__class__.__name__))
        if x is None:
            if self.nP == '*':
                x = np.random.randn(np.random.randint(1e2, high=1e3))
            else:
                x = np.random.randn(self.nP)

        return checkDerivative(
            lambda m: [self(m), self.deriv(m)], x, num=num, plotIt=plotIt,
            **kwargs
        )

    def _test_deriv2(self, x=None, num=4, plotIt=False, **kwargs):
        print('Testing {0!s} Deriv2'.format(self.__class__.__name__))
        if x is None:
            if self.nP == '*':
                x = np.random.randn(np.random.randint(1e2, high=1e3))
            else:
                x = np.random.randn(self.nP)

        v = x + 0.01*np.random.rand(len(x))
        return checkDerivative(
            lambda m: [self.deriv(m).dot(v), self.deriv2(m, v=v)],
                x, num=num, expectedOrder=1,
                plotIt=plotIt, **kwargs
        )

    def test(self, x=None, num=4, plotIt=False, **kwargs):
        """
        Run a convergence test on both the first and second derivatives - they
        should be second order!
        """
        deriv = self._test_deriv(x=x, num=num, **kwargs)
        deriv2 = self._test_deriv2(x=x, num=num, plotIt=False, **kwargs)
        return (deriv & deriv2)

    __numpy_ufunc__ = True

    def __add__(self, objfct2):

        if isinstance(objfct2, Utils.Zero):
            return self

        if not isinstance(objfct2, BaseObjectiveFunction):
            raise Exception(
                "Cannot add type {} to an objective function. Only "
                "ObjectiveFunctions can be added together".format(
                    objfct2.__class__.__name__
                )
            )

        if self.__class__.__name__ != 'ComboObjectiveFunction': #not isinstance(self, ComboObjectiveFunction):
            self = 1 * self

        if objfct2.__class__.__name__ != 'ComboObjectiveFunction': #not isinstance(objfct2, ComboObjectiveFunction):
            objfct2 = 1 * objfct2

        objfctlist = self.objfcts + objfct2.objfcts
        multipliers = self.multipliers + objfct2.multipliers

        return ComboObjectiveFunction(
            objfcts=objfctlist, multipliers=multipliers
        )

    def __radd__(self, objfct2):
        return self+objfct2

    def __mul__(self, multiplier):
        return ComboObjectiveFunction([self], [multiplier])

    def __rmul__(self, multiplier):
        return self * multiplier

    def __div__(self, denominator):
        return self.__mul__(1./denominator)

    def __truediv__(self, denominator):
        return self.__mul__(1./denominator)

    def __rdiv__(self, denominator):
        return self.__mul__(1./denominator)

class ComboObjectiveFunction(BaseObjectiveFunction):
    """
    A composite objective function that consists of multiple objective
    functions. Objective functions are stored in a list, and multipliers
    are stored in a parallel list.

    .. code::python

        import SimPEG.ObjectiveFunction
        phi1 = ObjectiveFunction.L2ObjectiveFunction(nP=10)
        phi2 = ObjectiveFunction.L2ObjectiveFunction(nP=10)

        phi = 2*phi1 + 3*phi2

    is equivalent to

        .. code::python

            import SimPEG.ObjectiveFunction
            phi1 = ObjectiveFunction.L2ObjectiveFunction(nP=10)
            phi2 = ObjectiveFunction.L2ObjectiveFunction(nP=10)

            phi = ObjectiveFunction.ComboObjectiveFunction(
                [phi1, phi2], [2, 3]
            )

    """
    _multiplier_types = (float, None, Utils.Zero, np.float64) + integer_types # Directive

    def __init__(self, objfcts=[], multipliers=None, **kwargs):

        if multipliers is None:
            multipliers = len(objfcts)*[1]

        self._nP = '*'

        assert(len(objfcts)==len(multipliers)),(
            "Must have the same number of Objective Functions and Multipliers "
            "not {} and {}".format(len(objfcts),len(multipliers))
            )

        def validate_list(objfctlist, multipliers):
            """
            ensure that the number of parameters expected by each objective
            function is the same, ensure that if multpliers are supplied, that
            list matches the length of the objective function list
            """
            for fct, mult in zip(objfctlist, multipliers):
                assert (
                    isinstance(fct, BaseObjectiveFunction)
                ) , (
                    "Unrecognized objective function type {} in objfcts. "
                    "All entries in objfcts must inherit from "
                    "ObjectiveFunction".format(fct.__class__.__name__)
                )

                assert(type(mult) in self._multiplier_types), (
                    "Objective Functions can only be multiplied by a "
                    "float, or a properties.Float, not a {}, {}".format(
                        type(mult), mult
                    )
                )

                if fct.nP != '*':
                    if self._nP != '*':
                        assert self._nP == fct.nP, (
                            "Objective Functions must all have the same "
                            "nP={}, not {}".format(self.nP, [f.nP for f in objfcts])
                        )
                    else:
                        self._nP = fct.nP

        validate_list(objfcts, multipliers)

        self.objfcts = objfcts
        self._multipliers = multipliers

        super(ComboObjectiveFunction, self).__init__(**kwargs)

    def __len__(self):
        return len(self.multipliers)

    def __getitem__(self, key):
        return self.multipliers[key], self.objfcts[key]

    @property
    def __len__(self):
        return self.objfcts.__len__

    @property
    def multipliers(self):
        return self._multipliers

    @multipliers.setter
    def multipliers(self, value):
        for val in value:
            assert type(val) in self._multiplier_types, (
                'Multiplier must be in type {} not {}'.format(
                    self._multiplier_types, type(val)
                )
            )

        assert len(value) == len(self.objfcts), (
            'the length of multipliers should be the same as the number of'
            ' objective functions ({}), not {}'.format(
                len(self.objfcts, len(value))
            )
        )

        self._multipliers = value

    def __call__(self, m, f=None):

        fct = 0.
        for i, phi in enumerate(self):
            multiplier, objfct = phi
            if multiplier == 0.: # don't evaluate the fct
                continue
            else:
                if f is not None and objfct._hasFields:
                    fct += multiplier * objfct(m, f=f[i])
                else:
                    fct += multiplier * objfct(m)
        return fct

    def deriv(self, m, f=None):
        """
        First derivative of the composite objective function is the sum of the
        derivatives of each objective function in the list, weighted by their
        respective multplier.

        :param numpy.ndarray m: model
        :param SimPEG.Fields f: Fields object (if applicable)
        """
        g = Utils.Zero()
        for i, phi in enumerate(self):
            multiplier, objfct = phi
            if multiplier == 0.: # don't evaluate the fct
                continue
            else:
                if f is not None and objfct._hasFields:
                    g += multiplier * objfct.deriv(m, f=f[i])
                else:
                    g += multiplier * objfct.deriv(m)
        return g

    def deriv2(self, m, v=None, f=None):
        """
        Second derivative of the composite objective function is the sum of the
        second derivatives of each objective function in the list, weighted by
        their respective multplier.

        :param numpy.ndarray m: model
        :param numpy.ndarray v: vector we are multiplying by
        :param SimPEG.Fields f: Fields object (if applicable)
        """
        H = Utils.Zero()
        for i, phi in enumerate(self):
            multiplier, objfct = phi
            if multiplier == 0.: # don't evaluate the fct
                continue
            else:
                if f is not None and objfct._hasFields:
                    objfct_H = objfct.deriv2(m, v, f=f[i])
                else:
                    objfct_H = objfct.deriv2(m, v)
                H = H + multiplier * objfct_H
        return H

    # This assumes all objective functions have a W.
    # The base class currently does not.
    @property
    def W(self):
        """
        W matrix for the full objective function. Includes multiplying by the
        square root of alpha.
        """
        W = []
        for mult, fct in self:
            curW = np.sqrt(mult) * fct.W
            if not isinstance(curW, Utils.Zero):
                W.append(curW)
        return sp.vstack(W)

class L2ObjectiveFunction(BaseObjectiveFunction):
    """
    An L2-Objective Function

    .. math::

        \phi = \frac{1}{2}||\mathbf{W} \mathbf{m}||^2
    """
    def __init__(self, W=None, **kwargs):

        super(L2ObjectiveFunction, self).__init__(**kwargs)
        if W is not None:
            if self.nP == '*':
                self._nP = W.shape[1]
        self._W = W

    @property
    def W(self):
        """
        Weighting matrix. The default if not sepcified is an identity.
        """
        if getattr(self, '_W', None) is None:
            if self._nC_residual != '*':
                self._W = sp.eye(self._nC_residual)
            else:
                self._W = Utils.Identity()
        return self._W

    def __call__(self, m):
        r = self.W * (self.mapping * m)
        return 0.5 * r.dot(r)

    def deriv(self, m):
        """
        First derivative with respect to the model

        :param numpy.ndarray m: model
        """
        return (
            self.mapping.deriv(m).T *
            (self.W.T * (self.W * (self.mapping * m)))
        )

    def deriv2(self, m, v=None):
        """
        Second derivative with respect to the model

        :param numpy.ndarray m: model
        :param numpy.ndarray v: vector to multiply by
        """
        if v is not None:
            return (
                self.mapping.deriv(m).T * (
                    self.W.T * (self.W * (self.mapping.deriv(m) * v))
                )
            )
        W = self.W * self.mapping.deriv(m)
        return self.W.T * self.W
<commit_msg>make sure _multipliers is cleared<commit_after>from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

import numpy as np
import scipy.sparse as sp
from six import integer_types
import warnings

from . import Utils
from .Tests import checkDerivative
from . import Maps
from . import Props

__all__ = [
    'BaseObjectiveFunction', 'ComboObjectiveFunction', 'L2ObjectiveFunction'
]

class BaseObjectiveFunction(Props.BaseSimPEG):
    """
    Base Objective Function

    Inherit this to build your own objective function. If building a
    regularization, have a look at
    :class:`SimPEG.Regularization.BaseRegularization` as there are additional
    methods and properties tailored to regularization of a model. Similarly,
    for building a data misfit, see :class:`SimPEG.DataMisfit.BaseDataMisfit`.
    """

    counter = None
    debug = False

    mapPair = Maps.IdentityMap  #: Base class of expected maps
    _mapping = None  #: An IdentityMap instance.
    _hasFields = False  #: should we have the option to store fields

    _nP = None  #: number of parameters

    def __init__(self, nP=None, **kwargs):
        if nP is not None:
            self._nP = nP
        Utils.setKwargs(self, **kwargs)

    def __call__(self, x, f=None):
        raise NotImplementedError(
            '__call__ has not been implemented for {} yet'.format(
                self.__class__.__name__
            )
        )

    @property
    def nP(self):
        """
        Number of model parameters expected.
        """
        if self._nP is not None:
            return self._nP
        if getattr(self, 'mapping', None) is not None:
            return self.mapping.nP
        return '*'

    @property
    def _nC_residual(self):
        """
        Shape of the residual
        """
        if getattr(self, 'mapping', None) is not None:
            return self.mapping.shape[0]
        else:
            return self.nP

    @property
    def mapping(self):
        """
        A `SimPEG.Maps` instance
        """
        if getattr(self, '_mapping') is None:
            if getattr(self, '_nP') is not None:
                self._mapping = self.mapPair(nP=self.nP)
            else:
                self._mapping = self.mapPair()
        return self._mapping

    @mapping.setter
    def mapping(self, value):
        assert isinstance(value, self.mapPair), (
            'mapping must be an instance of a {}, not a {}'
        ).format(self.mapPair, value.__class__.__name__)
        self._mapping = value

    @Utils.timeIt
    def __call__(self, x, f=None):
        raise NotImplementedError(
            "The method __call__ has not been implemented for {}".format(
                self.__class__.__name__
            )
        )

    @Utils.timeIt
    def deriv(self, x, **kwargs):
        """
        First derivative of the objective function with respect to the model
        """
        raise NotImplementedError(
            "The method deriv has not been implemented for {}".format(
                self.__class__.__name__
            )
        )

    @Utils.timeIt
    def deriv2(self, x, v=None, **kwargs):
        """
        Second derivative of the objective function with respect to the model
        """
        raise NotImplementedError(
            "The method _deriv2 has not been implemented for {}".format(
                self.__class__.__name__
            )
        )

    def _test_deriv(self, x=None, num=4, plotIt=False, **kwargs):
        print('Testing {0!s} Deriv'.format(self.__class__.__name__))
        if x is None:
            if self.nP == '*':
                x = np.random.randn(np.random.randint(1e2, high=1e3))
            else:
                x = np.random.randn(self.nP)

        return checkDerivative(
            lambda m: [self(m), self.deriv(m)], x, num=num, plotIt=plotIt,
            **kwargs
        )

    def _test_deriv2(self, x=None, num=4, plotIt=False, **kwargs):
        print('Testing {0!s} Deriv2'.format(self.__class__.__name__))
        if x is None:
            if self.nP == '*':
                x = np.random.randn(np.random.randint(1e2, high=1e3))
            else:
                x = np.random.randn(self.nP)

        v = x + 0.01*np.random.rand(len(x))
        return checkDerivative(
            lambda m: [self.deriv(m).dot(v), self.deriv2(m, v=v)],
                x, num=num, expectedOrder=1,
                plotIt=plotIt, **kwargs
        )

    def test(self, x=None, num=4, plotIt=False, **kwargs):
        """
        Run a convergence test on both the first and second derivatives - they
        should be second order!
        """
        deriv = self._test_deriv(x=x, num=num, **kwargs)
        deriv2 = self._test_deriv2(x=x, num=num, plotIt=False, **kwargs)
        return (deriv & deriv2)

    __numpy_ufunc__ = True

    def __add__(self, objfct2):

        if isinstance(objfct2, Utils.Zero):
            return self

        if not isinstance(objfct2, BaseObjectiveFunction):
            raise Exception(
                "Cannot add type {} to an objective function. Only "
                "ObjectiveFunctions can be added together".format(
                    objfct2.__class__.__name__
                )
            )

        if self.__class__.__name__ != 'ComboObjectiveFunction': #not isinstance(self, ComboObjectiveFunction):
            self = 1 * self

        if objfct2.__class__.__name__ != 'ComboObjectiveFunction': #not isinstance(objfct2, ComboObjectiveFunction):
            objfct2 = 1 * objfct2

        objfctlist = self.objfcts + objfct2.objfcts
        multipliers = self.multipliers + objfct2.multipliers

        return ComboObjectiveFunction(
            objfcts=objfctlist, multipliers=multipliers
        )

    def __radd__(self, objfct2):
        return self+objfct2

    def __mul__(self, multiplier):
        return ComboObjectiveFunction([self], [multiplier])

    def __rmul__(self, multiplier):
        return self * multiplier

    def __div__(self, denominator):
        return self.__mul__(1./denominator)

    def __truediv__(self, denominator):
        return self.__mul__(1./denominator)

    def __rdiv__(self, denominator):
        return self.__mul__(1./denominator)

class ComboObjectiveFunction(BaseObjectiveFunction):
    """
    A composite objective function that consists of multiple objective
    functions. Objective functions are stored in a list, and multipliers
    are stored in a parallel list.

    .. code::python

        import SimPEG.ObjectiveFunction
        phi1 = ObjectiveFunction.L2ObjectiveFunction(nP=10)
        phi2 = ObjectiveFunction.L2ObjectiveFunction(nP=10)

        phi = 2*phi1 + 3*phi2

    is equivalent to

        .. code::python

            import SimPEG.ObjectiveFunction
            phi1 = ObjectiveFunction.L2ObjectiveFunction(nP=10)
            phi2 = ObjectiveFunction.L2ObjectiveFunction(nP=10)

            phi = ObjectiveFunction.ComboObjectiveFunction(
                [phi1, phi2], [2, 3]
            )

    """
    _multiplier_types = (float, None, Utils.Zero, np.float64) + integer_types # Directive
    _multipliers = None

    def __init__(self, objfcts=[], multipliers=None, **kwargs):

        if multipliers is None:
            multipliers = len(objfcts)*[1]

        self._nP = '*'

        assert(len(objfcts)==len(multipliers)),(
            "Must have the same number of Objective Functions and Multipliers "
            "not {} and {}".format(len(objfcts),len(multipliers))
            )

        def validate_list(objfctlist, multipliers):
            """
            ensure that the number of parameters expected by each objective
            function is the same, ensure that if multpliers are supplied, that
            list matches the length of the objective function list
            """
            for fct, mult in zip(objfctlist, multipliers):
                assert (
                    isinstance(fct, BaseObjectiveFunction)
                ) , (
                    "Unrecognized objective function type {} in objfcts. "
                    "All entries in objfcts must inherit from "
                    "ObjectiveFunction".format(fct.__class__.__name__)
                )

                assert(type(mult) in self._multiplier_types), (
                    "Objective Functions can only be multiplied by a "
                    "float, or a properties.Float, not a {}, {}".format(
                        type(mult), mult
                    )
                )

                if fct.nP != '*':
                    if self._nP != '*':
                        assert self._nP == fct.nP, (
                            "Objective Functions must all have the same "
                            "nP={}, not {}".format(self.nP, [f.nP for f in objfcts])
                        )
                    else:
                        self._nP = fct.nP

        validate_list(objfcts, multipliers)

        self.objfcts = objfcts
        self._multipliers = multipliers

        super(ComboObjectiveFunction, self).__init__(**kwargs)

    def __len__(self):
        return len(self.multipliers)

    def __getitem__(self, key):
        return self.multipliers[key], self.objfcts[key]

    @property
    def __len__(self):
        return self.objfcts.__len__

    @property
    def multipliers(self):
        return self._multipliers

    @multipliers.setter
    def multipliers(self, value):
        for val in value:
            assert type(val) in self._multiplier_types, (
                'Multiplier must be in type {} not {}'.format(
                    self._multiplier_types, type(val)
                )
            )

        assert len(value) == len(self.objfcts), (
            'the length of multipliers should be the same as the number of'
            ' objective functions ({}), not {}'.format(
                len(self.objfcts, len(value))
            )
        )

        self._multipliers = value

    def __call__(self, m, f=None):

        fct = 0.
        for i, phi in enumerate(self):
            multiplier, objfct = phi
            if multiplier == 0.: # don't evaluate the fct
                continue
            else:
                if f is not None and objfct._hasFields:
                    fct += multiplier * objfct(m, f=f[i])
                else:
                    fct += multiplier * objfct(m)
        return fct

    def deriv(self, m, f=None):
        """
        First derivative of the composite objective function is the sum of the
        derivatives of each objective function in the list, weighted by their
        respective multplier.

        :param numpy.ndarray m: model
        :param SimPEG.Fields f: Fields object (if applicable)
        """
        g = Utils.Zero()
        for i, phi in enumerate(self):
            multiplier, objfct = phi
            if multiplier == 0.: # don't evaluate the fct
                continue
            else:
                if f is not None and objfct._hasFields:
                    g += multiplier * objfct.deriv(m, f=f[i])
                else:
                    g += multiplier * objfct.deriv(m)
        return g

    def deriv2(self, m, v=None, f=None):
        """
        Second derivative of the composite objective function is the sum of the
        second derivatives of each objective function in the list, weighted by
        their respective multplier.

        :param numpy.ndarray m: model
        :param numpy.ndarray v: vector we are multiplying by
        :param SimPEG.Fields f: Fields object (if applicable)
        """
        H = Utils.Zero()
        for i, phi in enumerate(self):
            multiplier, objfct = phi
            if multiplier == 0.: # don't evaluate the fct
                continue
            else:
                if f is not None and objfct._hasFields:
                    objfct_H = objfct.deriv2(m, v, f=f[i])
                else:
                    objfct_H = objfct.deriv2(m, v)
                H = H + multiplier * objfct_H
        return H

    # This assumes all objective functions have a W.
    # The base class currently does not.
    @property
    def W(self):
        """
        W matrix for the full objective function. Includes multiplying by the
        square root of alpha.
        """
        W = []
        for mult, fct in self:
            curW = np.sqrt(mult) * fct.W
            if not isinstance(curW, Utils.Zero):
                W.append(curW)
        return sp.vstack(W)

class L2ObjectiveFunction(BaseObjectiveFunction):
    """
    An L2-Objective Function

    .. math::

        \phi = \frac{1}{2}||\mathbf{W} \mathbf{m}||^2
    """
    def __init__(self, W=None, **kwargs):

        super(L2ObjectiveFunction, self).__init__(**kwargs)
        if W is not None:
            if self.nP == '*':
                self._nP = W.shape[1]
        self._W = W

    @property
    def W(self):
        """
        Weighting matrix. The default if not sepcified is an identity.
        """
        if getattr(self, '_W', None) is None:
            if self._nC_residual != '*':
                self._W = sp.eye(self._nC_residual)
            else:
                self._W = Utils.Identity()
        return self._W

    def __call__(self, m):
        r = self.W * (self.mapping * m)
        return 0.5 * r.dot(r)

    def deriv(self, m):
        """
        First derivative with respect to the model

        :param numpy.ndarray m: model
        """
        return (
            self.mapping.deriv(m).T *
            (self.W.T * (self.W * (self.mapping * m)))
        )

    def deriv2(self, m, v=None):
        """
        Second derivative with respect to the model

        :param numpy.ndarray m: model
        :param numpy.ndarray v: vector to multiply by
        """
        if v is not None:
            return (
                self.mapping.deriv(m).T * (
                    self.W.T * (self.W * (self.mapping.deriv(m) * v))
                )
            )
        W = self.W * self.mapping.deriv(m)
        return self.W.T * self.W
<|endoftext|>


<commit_before>"""
Views for Djax.
"""
from djax.content import sync_content, sync_record
from django.http import HttpResponse
import base64
from djax.models import AuthToken
from django.conf import settings

def check_auth(view):
    """
    Decorator to check auth token.
    """
    def wrapper(request,*args,**kwargs):
        if hasattr(settings,'DJAX_DISABLE_AUTH') and settings.DJAX_DISABLE_AUTH:
            print 'auth disabled'
            return view(request,*args,**kwargs)
        else:
            if 'HTTP_AUTHORIZATION' in request.META:
                print 'auth credentials present, checking...'
                basic_flag, auth_string = request.META['HTTP_AUTHORIZATION'].split()
                token = base64.b64decode(auth_string)[:-1] # chop trailing colon
                try:
                    auth_token = AuthToken.objects.get(token=token)
                    print 'auth token found'
                    if auth_token.origin_domain:
                        print 'auth token requires origin domain',auth_token.origin_domain
                        origin_domain = request.META.get('REMOTE_HOST',None)
                        if not origin_domain == auth_token.origin_domain:
                            print 'non-matching origin domain'
                            return HttpResponse('Not Allowed',status=403)

                    print 'auth token is good, running target view'
                    return view(request,*args,**kwargs)
                except AuthToken.DoesNotExist:
                    print 'cannot find auth token'
                    pass

            print 'request is not authorized'
            return HttpResponse('Not Allowed',status=403)

    return wrapper

def phone_home(request,token=None):
    """
    Initiates a content sync as long as there is no existing sync lock.
    """
    sync_content(token)

@check_auth
def sync_record_view(request):
    """
    Syncs a single record.
    """
    content_type = request.GET.get('content_type',None)
    content_key = request.GET.get('content_key',None)
    if content_type and content_key:
        if sync_record(content_type,content_key):
            return HttpResponse('Created %s:%s' % (content_type,content_key),status=201)
        else:
            return HttpResponse('Updated %s:%s' % (content_type,content_key))
    else:
        return HttpResponse('Badly formed request, specify content_type and content_key in request params.',status=409)
<commit_msg>Removing print statements.<commit_after>"""
Views for Djax.
"""
from djax.content import sync_content, sync_record
from django.http import HttpResponse
import base64
from djax.models import AuthToken
from django.conf import settings

def check_auth(view):
    """
    Decorator to check auth token.
    """
    def wrapper(request,*args,**kwargs):
        if hasattr(settings,'DJAX_DISABLE_AUTH') and settings.DJAX_DISABLE_AUTH:
            return view(request,*args,**kwargs)
        else:
            if 'HTTP_AUTHORIZATION' in request.META:
                basic_flag, auth_string = request.META['HTTP_AUTHORIZATION'].split()
                token = base64.b64decode(auth_string)[:-1] # chop trailing colon
                try:
                    auth_token = AuthToken.objects.get(token=token)
                    if auth_token.origin_domain:
                        origin_domain = request.META.get('REMOTE_HOST',None)
                        if not origin_domain == auth_token.origin_domain:
                            return HttpResponse('Not Allowed',status=403)

                    return view(request,*args,**kwargs)
                except AuthToken.DoesNotExist:
                    pass

            return HttpResponse('Not Allowed',status=403)

    return wrapper

def phone_home(request,token=None):
    """
    Initiates a content sync as long as there is no existing sync lock.
    """
    sync_content(token)

@check_auth
def sync_record_view(request):
    """
    Syncs a single record.
    """
    content_type = request.GET.get('content_type',None)
    content_key = request.GET.get('content_key',None)
    if content_type and content_key:
        if sync_record(content_type,content_key):
            return HttpResponse('Created %s:%s' % (content_type,content_key),status=201)
        else:
            return HttpResponse('Updated %s:%s' % (content_type,content_key))
    else:
        return HttpResponse('Badly formed request, specify content_type and content_key in request params.',status=409)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Python daemon implementation from:

http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/
"""
import atexit
import os
import sys
import time
from signal import SIGTERM

class Daemon:
    """ A generic daemon class

    Usage: subclass the Daemon class and override the run() method
    """
    def __init__(self, pidfile, stdin='/dev/null',
                 stdout='/dev/null', stderr='/dev/null'):
        """ Constructor """
        self.stdin = stdin
        self.stdout = stdout
        self.stderr = stderr
        self.pidfile = pidfile

    def daemonize(self):
        """ Do the UNIX double-fork magic

        See Stevens' "Advanced Programming in the UNIX Environment" for
        details (ISBN 0201563177)
        http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16
        """
        try:
            pid = os.fork()
            if pid > 0:
                # exit first parent
                sys.exit(0)
        except OSError as err:
            sys.stderr.write(
                "fork #1 failed: %d (%s)\n" % (err.errno, err.strerror))
            sys.exit(1)

        # decouple from parent environment
        os.chdir("/")
        os.setsid()
        os.umask(0)

        # do second fork
        try:
            pid = os.fork()
            if pid > 0:
                # exit from second parent
                sys.exit(0)
        except OSError as err:
            sys.stderr.write(
                "fork #2 failed: %d (%s)\n" % (err.errno, err.strerror))
            sys.exit(1)

        # redirect standard file descriptors
        sys.stdout.flush()
        sys.stderr.flush()
        si = file(self.stdin, 'r')
        so = file(self.stdout, 'a+')
        se = file(self.stderr, 'a+', 0)
        os.dup2(si.fileno(), sys.stdin.fileno())
        os.dup2(so.fileno(), sys.stdout.fileno())
        os.dup2(se.fileno(), sys.stderr.fileno())

        # write pidfile
        atexit.register(self.delpid)
        pid = str(os.getpid())
        file(self.pidfile, 'w+').write("%s\n" % pid)

    def delpid(self):
        os.remove(self.pidfile)

    def start(self, *args, **kwargs):
        """ Start the daemon """
        # Check for a pidfile to see if the daemon already runs
        try:
            pf = file(self.pidfile, 'r')
            pid = int(pf.read().strip())
            pf.close()
        except IOError:
            pid = None

        if pid:
            message = "pidfile %s already exist. Daemon already running?\n"
            sys.stderr.write(message % self.pidfile)
            sys.exit(1)

        # Start the daemon
        self.daemonize()
        self.run(*args, **kwargs)

    def stop(self):
        """ Stop the daemon """
        # Get the pid from the pidfile
        try:
            pf = file(self.pidfile, 'r')
            pid = int(pf.read().strip())
            pf.close()
        except IOError:
            pid = None

        if not pid:
            sys.stderr.write(
                "pidfile {0} does not exist. Daemon not running?\n".format(
                    self.pidfile))
            return  # not an error in a restart

        # Try killing the daemon process
        try:
            while 1:
                os.kill(pid, SIGTERM)
                time.sleep(0.1)
        except OSError, err:
            err = str(err)
            if err.find("No such process") > 0:
                if os.path.exists(self.pidfile):
                    os.remove(self.pidfile)
            else:
                print str(err)
                sys.exit(1)

    def restart(self, *args, **kwargs):
        """ Restart the daemon """
        self.stop()
        self.start(*args, **kwargs)

    def run(self):
        """
        You should override this method when you subclass Daemon.
        It will be called after the process has been
        daemonized by start() or restart().
        """
<commit_msg>Changed the daemon class to use new class object<commit_after># -*- coding: utf-8 -*-
"""
Python daemon implementation from:

http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/
"""
import atexit
import os
import sys
import time
from signal import SIGTERM

class Daemon(object):
    """ A generic daemon class

    Usage: subclass the Daemon class and override the run() method
    """
    def __init__(self, pidfile, stdin='/dev/null',
                 stdout='/dev/null', stderr='/dev/null'):
        """ Constructor """
        self.stdin = stdin
        self.stdout = stdout
        self.stderr = stderr
        self.pidfile = pidfile

    def daemonize(self):
        """ Do the UNIX double-fork magic

        See Stevens' "Advanced Programming in the UNIX Environment" for
        details (ISBN 0201563177)
        http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16
        """
        try:
            pid = os.fork()
            if pid > 0:
                # exit first parent
                sys.exit(0)
        except OSError as err:
            sys.stderr.write(
                "fork #1 failed: %d (%s)\n" % (err.errno, err.strerror))
            sys.exit(1)

        # decouple from parent environment
        os.chdir("/")
        os.setsid()
        os.umask(0)

        # do second fork
        try:
            pid = os.fork()
            if pid > 0:
                # exit from second parent
                sys.exit(0)
        except OSError as err:
            sys.stderr.write(
                "fork #2 failed: %d (%s)\n" % (err.errno, err.strerror))
            sys.exit(1)

        # redirect standard file descriptors
        sys.stdout.flush()
        sys.stderr.flush()
        si = file(self.stdin, 'r')
        so = file(self.stdout, 'a+')
        se = file(self.stderr, 'a+', 0)
        os.dup2(si.fileno(), sys.stdin.fileno())
        os.dup2(so.fileno(), sys.stdout.fileno())
        os.dup2(se.fileno(), sys.stderr.fileno())

        # write pidfile
        atexit.register(self.delpid)
        pid = str(os.getpid())
        file(self.pidfile, 'w+').write("%s\n" % pid)

    def delpid(self):
        os.remove(self.pidfile)

    def start(self, *args, **kwargs):
        """ Start the daemon """
        # Check for a pidfile to see if the daemon already runs
        try:
            pf = file(self.pidfile, 'r')
            pid = int(pf.read().strip())
            pf.close()
        except IOError:
            pid = None

        if pid:
            message = "pidfile %s already exist. Daemon already running?\n"
            sys.stderr.write(message % self.pidfile)
            sys.exit(1)

        # Start the daemon
        self.daemonize()
        self.run(*args, **kwargs)

    def stop(self):
        """ Stop the daemon """
        # Get the pid from the pidfile
        try:
            pf = file(self.pidfile, 'r')
            pid = int(pf.read().strip())
            pf.close()
        except IOError:
            pid = None

        if not pid:
            sys.stderr.write(
                "pidfile {0} does not exist. Daemon not running?\n".format(
                    self.pidfile))
            return  # not an error in a restart

        # Try killing the daemon process
        try:
            while 1:
                os.kill(pid, SIGTERM)
                time.sleep(0.1)
        except OSError, err:
            err = str(err)
            if err.find("No such process") > 0:
                if os.path.exists(self.pidfile):
                    os.remove(self.pidfile)
            else:
                print str(err)
                sys.exit(1)

    def restart(self, *args, **kwargs):
        """ Restart the daemon """
        self.stop()
        self.start(*args, **kwargs)

    def run(self):
        """
        You should override this method when you subclass Daemon.
        It will be called after the process has been
        daemonized by start() or restart().
        """
<|endoftext|>


<commit_before># file: plotCFL.py
# author: Olivier Mesnard (mesnardo@gwu.edu)
# brief: Plots the instantaneous maximum CFL number.

import argparse
import os
import sys

import numpy
from scipy import signal
from matplotlib import pyplot

sys.path.append('{}/scripts/library'.format(os.environ['SCRIPTS']))
import miscellaneous

def parse_command_line():
  """Parses the command-line."""
  print('[info] parsing command-line ...'),
  # create the parser
  parser = argparse.ArgumentParser(description='Plots the maximum CFL number over time',
                                   formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  # fill the parser with arguments
  parser.add_argument('--directory', dest='directory',
                      type=str, default=os.getcwd(),
                      help='directory of the simulation')
  parser.add_argument('--path', dest='file_path',
                      type=str,
                      help='path of the IcoFOAM logging file')
  parser.add_argument('--average', dest='average_limits',
                      type=float, nargs='+', default=[0.0, float('inf')],
                      help='time limits to compute averaged cfl')
  parser.add_argument('--no-show', dest='show',
                      action='store_false',
                      help='does not display the figure')
  parser.add_argument('--save-name', dest='save_name',
                      type=str, default=None,
                      help='name of the figure of save')
  parser.add_argument('--limits', dest='plot_limits',
                      type=float, nargs='+', default=[None, None, None, None],
                      help='limits of the plot')
  parser.add_argument('--extrema', dest='display_extrema',
                      action='store_true',
                      help='displays the forces extrema')
  parser.add_argument('--order', dest='order',
                      type=int, default=5,
                      help='number of side-points used for comparison to get extrema')
  # default options
  parser.set_defaults(show=True)
  # parse given options file
  parser.add_argument('--options',
                      type=open, action=miscellaneous.ReadOptionsFromFile,
                      help='path of the file with options to parse')
  print('done')
  return parser.parse_args()

def read_maximum_cfl(file_path):
  """Reads the instantaneous maximum CFL number from a given file.

  Parameters
  ----------
  file_path: string
    Path of the logging file containing the instantaneous maximum CFL number.

  Returns
  -------
  time: numpy array of float
    Discrete time values.
  cfl: numpy array of float
    Discrete maximum CFL values
  """
  print('[info] reading file {} ...'.format(file_path)),
  with open(file_path, 'r') as infile:
    time = numpy.array([float(line.split()[-1])
                        for line in infile if line.startswith('Time = ')])
  with open(file_path, 'r') as infile:
    cfl = numpy.array([float(line.split()[-1])
                        for line in infile if line.startswith('Courant Number mean')])
  assert(time.shape == cfl.shape)
  print('done')
  return time, cfl

def get_mean(time, array, limits=[0.0, float('inf')], output=False):
  """Computes the mean force.

  Parameters
  ----------
  time: numpy array of floats
    Discrete time values.
  array: numpy array of floats
    Array to use to compute the mean value.
  limits: list of floats
    Time-limits to compute the mean value; default: [0.0, float('inf')].
  output: bool
    Set 'True' to display the time-averaged value; default: False.

  Returns
  -------
  start, end: floats
    Time-limits used to compute the mean value.
  mean: float
    The mean value.
  """
  print('[info] computing the mean value ...'),
  mask = numpy.where(numpy.logical_and(time >= limits[0],
                                       time <= limits[1]))[0]
  start, end = time[mask[0]], time[mask[-1]]
  mean = numpy.mean(array[mask])
  if output:
    print('\n[info] averaging the maximum CFL '
          'number between {} and {} time-units:'.format(start, end))
    print('\t<max(CFL)> = {}\n'.format(mean))
  print('done')
  return start, end, mean

def get_extrema(array, order=5):
  """Computes masks (i.e. arrays of indices) of the extrema of the force.

  Parameters
  ----------
  array: numpy array
    Array on which to define the extrema.
  order: integer
    Number of neighboring points used to define an extremum; default: 5.

  Returns
  -------
  minima: Numpy array of integers
    Index of minima.
  maxima: Numpy array of integers
    Index of maxima.
  """
  print('[info] computing extrema ...'),
  minima = signal.argrelextrema(array, numpy.less_equal, order=order)[0][:-1]
  maxima = signal.argrelextrema(array, numpy.greater_equal, order=order)[0][:-1]
  # remove indices that are too close
  minima = minima[numpy.append(True, minima[1:]-minima[:-1] > order)]
  maxima = maxima[numpy.append(True, maxima[1:]-maxima[:-1] > order)]
  print('done')
  return minima, maxima

def plot_cfl(time, cfl,
             display_extrema=False, order=5,
             limits=[0.0, float('inf'), 0.0, float('inf')],
             directory=os.getcwd(), save_name=None,
             show=True):
  """Plots the instantaneous maximum CFL number.

  Parameters
  ----------
  time: numpy array of floats
    Discrete time values.
  cfl: numpy array of floats
    Maximum CFL values.
  display_extrema: bool
    Set 'True' to emphasize the extrema of the curves; default: False.
  order: int
    Number of neighbors used on each side to define an extremum; default: 5.
  limits: list of floats
    Limits of the axes [xmin, xmax, ymin, ymax]; default: [0.0, +inf, 0.0, +inf].
  directory: string
    Directory of the simulation; default: $PWD.
  save_name: string
    Name of the .PNG file to save; default: None (does not save).
  show: bool
    Set 'True' to display the figure; default: False.
  """
  print('[info] plotting cfl ...')
  pyplot.style.use('{}/styles/mesnardo.mplstyle'.format(os.environ['SCRIPTS']))
  color = '#386cb0'
  fig, ax = pyplot.subplots(figsize=(8, 6))
  pyplot.grid(True, zorder=0)
  pyplot.xlabel('time')
  pyplot.ylabel('maximum CFL')
  pyplot.plot(time, cfl, color=color, linestyle='-', zorder=10)
  if display_extrema:
    minima, maxima = get_extrema(cfl, order=order)
    pyplot.scatter(time[minima], cfl[minima], c=color, marker='o', zorder=10)
    pyplot.scatter(time[maxima], cfl[maxima], c=color, marker='o', zorder=10)
  pyplot.axis(limits)
  if save_name:
    images_directory = '{}/images'.format(directory)
    print('[info] saving figure in directory {} ...'.format(images_directory))
    if not os.path.isdir(images_directory):
      os.makedirs(images_directory)
    pyplot.savefig('{}/{}.png'.format(images_directory, save_name))
  if show:
    print('[info] displaying figure ...')
    pyplot.show()
  pyplot.close()

def main():
  """Plots the instantaneous maximum CFL number."""
  args = parse_command_line()
  time, cfl = read_maximum_cfl(args.file_path)
  get_mean(time, cfl,  limits=args.average_limits, output=True)
  plot_cfl(time, cfl,
           display_extrema=args.display_extrema, order=args.order,
           limits=args.plot_limits,
           directory=args.directory, save_name=args.save_name,
           show=args.show)

if __name__ == '__main__':
  print('\n[{}] START\n'.format(os.path.basename(__file__)))
  main()
  print('\n[{}] END\n'.format(os.path.basename(__file__)))<commit_msg>Run script from anywhere<commit_after># file: plotCFL.py
# author: Olivier Mesnard (mesnardo@gwu.edu)
# brief: Plots the instantaneous maximum CFL number.

import argparse
import os
import sys

import numpy
from scipy import signal
from matplotlib import pyplot

sys.path.append('{}/scripts/library'.format(os.environ['SCRIPTS']))
import miscellaneous

def parse_command_line():
  """Parses the command-line."""
  print('[info] parsing command-line ...'),
  # create the parser
  parser = argparse.ArgumentParser(description='Plots the maximum CFL number over time',
                                   formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  # fill the parser with arguments
  parser.add_argument('--directory', dest='directory',
                      type=str, default=os.getcwd(),
                      help='directory of the simulation')
  parser.add_argument('--path', dest='file_path',
                      type=str,
                      help='path of the IcoFOAM logging file')
  parser.add_argument('--average', dest='average_limits',
                      type=float, nargs='+', default=[0.0, float('inf')],
                      help='time limits to compute averaged cfl')
  parser.add_argument('--no-show', dest='show',
                      action='store_false',
                      help='does not display the figure')
  parser.add_argument('--save-name', dest='save_name',
                      type=str, default=None,
                      help='name of the figure of save')
  parser.add_argument('--limits', dest='plot_limits',
                      type=float, nargs='+', default=[None, None, None, None],
                      help='limits of the plot')
  parser.add_argument('--extrema', dest='display_extrema',
                      action='store_true',
                      help='displays the forces extrema')
  parser.add_argument('--order', dest='order',
                      type=int, default=5,
                      help='number of side-points used for comparison to get extrema')
  # default options
  parser.set_defaults(show=True)
  # parse given options file
  parser.add_argument('--options',
                      type=open, action=miscellaneous.ReadOptionsFromFile,
                      help='path of the file with options to parse')
  print('done')
  return parser.parse_args()

def read_maximum_cfl(file_path):
  """Reads the instantaneous maximum CFL number from a given file.

  Parameters
  ----------
  file_path: string
    Path of the logging file containing the instantaneous maximum CFL number.

  Returns
  -------
  time: numpy array of float
    Discrete time values.
  cfl: numpy array of float
    Discrete maximum CFL values
  """
  print('[info] reading file {} ...'.format(file_path)),
  with open(file_path, 'r') as infile:
    time = numpy.array([float(line.split()[-1])
                        for line in infile if line.startswith('Time = ')])
  with open(file_path, 'r') as infile:
    cfl = numpy.array([float(line.split()[-1])
                        for line in infile if line.startswith('Courant Number mean')])
  assert(time.shape == cfl.shape)
  print('done')
  return time, cfl

def get_mean(time, array, limits=[0.0, float('inf')], output=False):
  """Computes the mean force.

  Parameters
  ----------
  time: numpy array of floats
    Discrete time values.
  array: numpy array of floats
    Array to use to compute the mean value.
  limits: list of floats
    Time-limits to compute the mean value; default: [0.0, float('inf')].
  output: bool
    Set 'True' to display the time-averaged value; default: False.

  Returns
  -------
  start, end: floats
    Time-limits used to compute the mean value.
  mean: float
    The mean value.
  """
  print('[info] computing the mean value ...'),
  mask = numpy.where(numpy.logical_and(time >= limits[0],
                                       time <= limits[1]))[0]
  start, end = time[mask[0]], time[mask[-1]]
  mean = numpy.mean(array[mask])
  if output:
    print('\n[info] averaging the maximum CFL '
          'number between {} and {} time-units:'.format(start, end))
    print('\t<max(CFL)> = {}\n'.format(mean))
  print('done')
  return start, end, mean

def get_extrema(array, order=5):
  """Computes masks (i.e. arrays of indices) of the extrema of the force.

  Parameters
  ----------
  array: numpy array
    Array on which to define the extrema.
  order: integer
    Number of neighboring points used to define an extremum; default: 5.

  Returns
  -------
  minima: Numpy array of integers
    Index of minima.
  maxima: Numpy array of integers
    Index of maxima.
  """
  print('[info] computing extrema ...'),
  minima = signal.argrelextrema(array, numpy.less_equal, order=order)[0][:-1]
  maxima = signal.argrelextrema(array, numpy.greater_equal, order=order)[0][:-1]
  # remove indices that are too close
  minima = minima[numpy.append(True, minima[1:]-minima[:-1] > order)]
  maxima = maxima[numpy.append(True, maxima[1:]-maxima[:-1] > order)]
  print('done')
  return minima, maxima

def plot_cfl(time, cfl,
             display_extrema=False, order=5,
             limits=[0.0, float('inf'), 0.0, float('inf')],
             directory=os.getcwd(), save_name=None,
             show=True):
  """Plots the instantaneous maximum CFL number.

  Parameters
  ----------
  time: numpy array of floats
    Discrete time values.
  cfl: numpy array of floats
    Maximum CFL values.
  display_extrema: bool
    Set 'True' to emphasize the extrema of the curves; default: False.
  order: int
    Number of neighbors used on each side to define an extremum; default: 5.
  limits: list of floats
    Limits of the axes [xmin, xmax, ymin, ymax]; default: [0.0, +inf, 0.0, +inf].
  directory: string
    Directory of the simulation; default: $PWD.
  save_name: string
    Name of the .PNG file to save; default: None (does not save).
  show: bool
    Set 'True' to display the figure; default: False.
  """
  print('[info] plotting cfl ...')
  pyplot.style.use('{}/styles/mesnardo.mplstyle'.format(os.environ['SCRIPTS']))
  color = '#386cb0'
  fig, ax = pyplot.subplots(figsize=(8, 6))
  pyplot.grid(True, zorder=0)
  pyplot.xlabel('time')
  pyplot.ylabel('maximum CFL')
  pyplot.plot(time, cfl, color=color, linestyle='-', zorder=10)
  if display_extrema:
    minima, maxima = get_extrema(cfl, order=order)
    pyplot.scatter(time[minima], cfl[minima], c=color, marker='o', zorder=10)
    pyplot.scatter(time[maxima], cfl[maxima], c=color, marker='o', zorder=10)
  pyplot.axis(limits)
  if save_name:
    images_directory = '{}/images'.format(directory)
    print('[info] saving figure in directory {} ...'.format(images_directory))
    if not os.path.isdir(images_directory):
      os.makedirs(images_directory)
    pyplot.savefig('{}/{}.png'.format(images_directory, save_name))
  if show:
    print('[info] displaying figure ...')
    pyplot.show()
  pyplot.close()

def main():
  """Plots the instantaneous maximum CFL number."""
  args = parse_command_line()
  time, cfl = read_maximum_cfl('{}/{}'.format(args.directory, args.file_path))
  get_mean(time, cfl,  limits=args.average_limits, output=True)
  plot_cfl(time, cfl,
           display_extrema=args.display_extrema, order=args.order,
           limits=args.plot_limits,
           directory=args.directory, save_name=args.save_name,
           show=args.show)

if __name__ == '__main__':
  print('\n[{}] START\n'.format(os.path.basename(__file__)))
  main()
  print('\n[{}] END\n'.format(os.path.basename(__file__)))<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Python Version: 2.7
# boto Version 2.38 / boto3 version 1.2.3
#
# Create a VPC
#

# Must be the first line
from __future__ import print_function

# https://pythonhosted.org/netaddr
from netaddr import *

import sys
import json
import boto.vpc
import boto.ec2
from boto3.session import Session

class Tag():

  def __init__(self, name, resource, region):
    abbr = '-nul'
    if region == 'us-east-1':      abbr = '-ue1'
    if region == 'eu-west-1':      abbr = '-ew1'
    if region == 'ap-northeast-1': abbr = '-an1'
    if region == 'us-west-1':      abbr = '-uw1'
    if region == 'us-west-2':      abbr = '-uw2'
    if region == 'ap-southeast-1': abbr = '-as1'
    if region == 'ap-southeast-2': abbr = '-as2'
    if region == 'eu-central-1':   abbr = '-ec1'
    if region == 'sa-east-1':      abbr = '-se1'

    self.name = name + resource + abbr

  def tag_resource(self, conn, resource_id):
    conn.create_tags(resource_id, { 'Name' : self.name })

class Template:

  RolePolicy =  {
                  'Version': '2012-10-17',
                  'Statement': [
                    {
                      'Sid': '',
                      'Effect': 'Allow',
                      'Principal': {
                        'Service': 'vpc-flow-logs.amazonaws.com'
                      },
                      'Action': 'sts:AssumeRole'
                    }
                  ]
                }

  LogsPolicy =  {
                  'Version': '2012-10-17',
                  'Statement': [
                    {
                      'Action': [
                        'logs:CreateLogGroup',
                        'logs:CreateLogStream',
                        'logs:DescribeLogGroups',
                        'logs:DescribeLogStreams',
                        'logs:PutLogEvents'
                      ],
                      'Effect': 'Allow',
                      'Resource': '*'
                    }
                  ]
                }

def create_vpc(conn, name, region, cidr):
  """ Create the VPC """

  try:
    vpc = conn.create_vpc(cidr, instance_tenancy='default')
  except boto.exception.EC2ResponseError as e:
    print(e.message)
    exit(1)
  else:
    conn.modify_vpc_attribute(vpc.id, enable_dns_support=True)
    conn.modify_vpc_attribute(vpc.id, enable_dns_hostnames=True)
    t = Tag(name, 'vpc', region); t.tag_resource(conn, vpc.id)

    print("vpc-id: ", vpc.id, "\tname: ", t.name)
    return vpc.id

def create_igw(conn, name, region, vpc_id):
  """ Create and attach an igw """

  try:
    igw = conn.create_internet_gateway()
  except boto.exception.EC2ResponseError as e:
    print(e.message)
    exit(1)
  else:
    conn.attach_internet_gateway(igw.id, vpc_id)
    t = Tag(name, 'igw', region); t.tag_resource(conn, igw.id)

    return igw.id

def subnet_sizes(azs, cidr):
  """
  Calculate subnets sizes

  Possible scenarios:
    a) /25 2AZ  (4 Subnets = /27)
    b) /24 2AZ  (4 Subnets = /26)
    c) /23 2AZ  (4 Subnets = /25)
    d) /22 2AZ  (4 Subnets = /24)
    e) /23 3AZ  (3 Subnets = /27, 3 Subnets = /25)
    f) /22 3AZ  (3 Subnets = /26, 3 Subnets = /24)
  """

  if azs != 2 and azs != 3:
    print("ERROR: Number of AZs should be 2 or 3.")
    exit(1)

  netmasks = ('255.255.252.0', '255.255.254.0', '255.255.255.0', '255.255.255.128')

  ip = IPNetwork(cidr)
  mask = ip.netmask

  if azs == 3:
    if str(mask) not in netmasks[0:2]:
      print("ERROR: Netmask " + str(mask) + " not found.")
      exit(1)

    for n, netmask in enumerate(netmasks):
      if str(mask) == netmask:
        pub_net = list(ip.subnet(n + 24))
        pri_subs = pub_net[1:]
        pub_mask = pub_net[0].netmask

    pub_split = list(ip.subnet(26)) if (str(pub_mask) == '255.255.255.0') else list(ip.subnet(27))
    pub_subs = pub_split[:3]

    subnets = pub_subs + pri_subs

  else:
    if str(mask) not in netmasks:
      print("ERROR: Netmask " + str(mask) + " not found.")
      exit(1)

    for n, netmask in enumerate(netmasks):
      if str(mask) == netmask:
        subnets = list(ip.subnet(n + 24))

  return subnets

def create_sub(conn, name, region, vpc_id, azs, subnets, zones):
  """ Create subnets """

  i = 0; sub_ids = []; tier = 'public'
  for sub in subnets:
    subnet = conn.create_subnet(vpc_id, sub, availability_zone=zones[i])
    t = Tag(name, tier + '-sub', region); t.tag_resource(conn, subnet.id)

    sub_ids.append(subnet.id)
    print("sub-id: ", subnet.id, "\tsize: ", sub, "\tzone: ", zones[i])
    i += 1
    if i == azs: i = 0; tier = 'private'

  return sub_ids

def create_rtb(conn, name, region, vpc_id, azs, sub_ids, igw_id):
  """ Create and associate route-tables """

  i = 0; rtb_ids = []; tier = 'public'
  for sub in sub_ids:
    if i == 0:
      rtb = conn.create_route_table(vpc_id)
      conn.create_route(rtb.id, '0.0.0.0/0', igw_id)
      t = Tag(name, tier + '-rtb', region); t.tag_resource(conn, rtb.id)

      rtb_ids.append(rtb.id)
    conn.associate_route_table(rtb.id, sub)
    i += 1
    if i == azs: i = 0; tier = 'private'

  return rtb_ids

def create_acl(conn, name, region, vpc_id, azs, sub_ids, cidr):
  """ Create and associate network access-lists

      https://blogs.aws.amazon.com/security/post/Tx3NVS2JAL7KWOM/How-to-Help-Prepare-for-DDoS-Attacks-by-Reducing-Your-Attack-Surface
  """

  i = 0; acl_ids = []; tier = 'public'
  for sub in sub_ids:
    if i == 0:
      acl = conn.create_network_acl(vpc_id)
      conn.create_network_acl_entry(acl.id, 100, -1, 'allow',  cidr, egress=False)
      conn.create_network_acl_entry(acl.id, 200, 6,  'allow', '0.0.0.0/0', egress=False, port_range_from=443, port_range_to=443)
      conn.create_network_acl_entry(acl.id, 300, 6,  'allow', '0.0.0.0/0', egress=False, port_range_from=80, port_range_to=80)
      conn.create_network_acl_entry(acl.id, 400, 6,  'allow', '0.0.0.0/0', egress=False, port_range_from=1024, port_range_to=65535)
      conn.create_network_acl_entry(acl.id, 500, 6,  'allow', '0.0.0.0/0', egress=False, port_range_from=22, port_range_to=22)
      conn.create_network_acl_entry(acl.id, 100, -1, 'allow',  cidr, egress=True)
      conn.create_network_acl_entry(acl.id, 200, 6,  'allow', '0.0.0.0/0', egress=True,  port_range_from=443, port_range_to=443)
      conn.create_network_acl_entry(acl.id, 300, 6,  'allow', '0.0.0.0/0', egress=True,  port_range_from=80, port_range_to=80)
      conn.create_network_acl_entry(acl.id, 400, 6,  'allow', '0.0.0.0/0', egress=True,  port_range_from=1024, port_range_to=65535)
      conn.create_network_acl_entry(acl.id, 500, 6,  'allow', '0.0.0.0/0', egress=True,  port_range_from=22, port_range_to=22)
      t = Tag(name, tier + '-acl', region); t.tag_resource(conn, acl.id)

      acl_ids.append(acl.id)
    conn.associate_network_acl(acl.id, sub)
    i += 1
    if i == azs: i = 0; tier = 'private'

  return acl_ids

def create_flows(vpc_id, keyid, secret, region):
  """ Create VPC flow logs """

  session = Session(aws_access_key_id = keyid, aws_secret_access_key = secret, region_name = region)

  iam = session.client('iam')
  logs = session.client('logs')
  ec2 = session.client('ec2')

  # Check for an existing Role with standard name
  try:
    role = iam.get_role(RoleName='flowlogsRole')
  except:
    role = 'None'

  # Create VPC Flows Logs IAM Role
  if role == 'None':
    role = iam.create_role(
      Path = '/',
      RoleName = 'flowlogsRole',
      AssumeRolePolicyDocument = json.dumps(Template.RolePolicy))

    # Create VPC Flow Logs policy
    policy = iam.create_policy(
      Path = '/',
      PolicyName =  'flowlogsPolicy',
      Description = 'Grants access to CloudWatch Logs.',
      PolicyDocument = json.dumps(Template.LogsPolicy))

    role_name = role['Role']['RoleName']
    role_arn = role['Role']['Arn']
    policy_arn = policy['Policy']['Arn']

    # Attach policy to the IAM Role
    attach = iam.attach_role_policy(
      RoleName = role_name,
      PolicyArn = policy_arn)
  else:
    role_arn = role['Role']['Arn']

  logs_name = 'flowlogsGroup' + '-' + vpc_id

  # Create CloudWatch Logs group
  cwlogs = logs.create_log_group(
    logGroupName = logs_name)

  # Enable VPC Flow Logs
  flow_id = ec2.create_flow_logs(
    ResourceIds = [vpc_id],
    ResourceType = 'VPC',
    TrafficType = 'ALL',
    LogGroupName = logs_name,
    DeliverLogsPermissionArn = role_arn)

  return flow_id

def main(azs, region, keyid, secret, cidr, owner, env):
  """
  Do the work

  1.) Setup/validate region and availability-zones
  2.) Create the VPC
  3.) Create and attach an internet-gateway
  4.) Calculate subnet sizes (netaddr)
  5.) Create subnets
  6.) Create and associate route-tables
  7.) Create and associate network access-lists
  8.) Enable VPC Flow Logs
  """

  # Validate the region
  myregion = boto.ec2.get_region(region_name=region)
  if myregion == None:
    print("Unknown region.")
    exit(1)

  # Establish a VPC service connection
  try:
    conn = boto.vpc.VPCConnection(aws_access_key_id=keyid, aws_secret_access_key=secret, region=myregion)
  except boto.exception.EC2ResponseError as e:
    print(e.message)
    exit(1)

  # Grab the availability-zones
  zones = []
  all_zones = conn.get_all_zones()
  for zone in all_zones:
    if zone.state != 'available':
      continue
    zones.append(zone.name)

  subnets = subnet_sizes(azs, cidr)               # Calculate the subnet sizes
  name = owner.lower() + '-' + env.lower() + '-'  # Used for tagging

  vpc_id  = create_vpc(conn, name, region, cidr)
  igw_id  = create_igw(conn, name, region, vpc_id)
  sub_ids = create_sub(conn, name, region, vpc_id, azs, subnets, zones)
  rtb_ids = create_rtb(conn, name, region, vpc_id, azs, sub_ids, igw_id)
  acl_ids = create_acl(conn, name, region, vpc_id, azs, sub_ids)
  flow_id = create_flows(vpc_id, keyid, secret, region)

if __name__ == "__main__":

  main(azs = 3, region = 'us-west-2', keyid = 'XXXXX', secret = 'XXXXX', cidr = '10.64.0.0/23', owner = 'eng', env = 'dev')
<commit_msg>fix acl bug<commit_after>#!/usr/bin/env python
#
# Python Version: 2.7
# boto Version 2.38 / boto3 version 1.2.3
#
# Create a VPC
#

# Must be the first line
from __future__ import print_function

# https://pythonhosted.org/netaddr
from netaddr import *

import sys
import json
import boto.vpc
import boto.ec2
from boto3.session import Session

class Tag():

  def __init__(self, name, resource, region):
    abbr = '-nul'
    if region == 'us-east-1':      abbr = '-ue1'
    if region == 'eu-west-1':      abbr = '-ew1'
    if region == 'ap-northeast-1': abbr = '-an1'
    if region == 'us-west-1':      abbr = '-uw1'
    if region == 'us-west-2':      abbr = '-uw2'
    if region == 'ap-southeast-1': abbr = '-as1'
    if region == 'ap-southeast-2': abbr = '-as2'
    if region == 'eu-central-1':   abbr = '-ec1'
    if region == 'sa-east-1':      abbr = '-se1'

    self.name = name + resource + abbr

  def tag_resource(self, conn, resource_id):
    conn.create_tags(resource_id, { 'Name' : self.name })

class Template:

  RolePolicy =  {
                  'Version': '2012-10-17',
                  'Statement': [
                    {
                      'Sid': '',
                      'Effect': 'Allow',
                      'Principal': {
                        'Service': 'vpc-flow-logs.amazonaws.com'
                      },
                      'Action': 'sts:AssumeRole'
                    }
                  ]
                }

  LogsPolicy =  {
                  'Version': '2012-10-17',
                  'Statement': [
                    {
                      'Action': [
                        'logs:CreateLogGroup',
                        'logs:CreateLogStream',
                        'logs:DescribeLogGroups',
                        'logs:DescribeLogStreams',
                        'logs:PutLogEvents'
                      ],
                      'Effect': 'Allow',
                      'Resource': '*'
                    }
                  ]
                }

def create_vpc(conn, name, region, cidr):
  """ Create the VPC """

  try:
    vpc = conn.create_vpc(cidr, instance_tenancy='default')
  except boto.exception.EC2ResponseError as e:
    print(e.message)
    exit(1)
  else:
    conn.modify_vpc_attribute(vpc.id, enable_dns_support=True)
    conn.modify_vpc_attribute(vpc.id, enable_dns_hostnames=True)
    t = Tag(name, 'vpc', region); t.tag_resource(conn, vpc.id)

    print("vpc-id: ", vpc.id, "\tname: ", t.name)
    return vpc.id

def create_igw(conn, name, region, vpc_id):
  """ Create and attach an igw """

  try:
    igw = conn.create_internet_gateway()
  except boto.exception.EC2ResponseError as e:
    print(e.message)
    exit(1)
  else:
    conn.attach_internet_gateway(igw.id, vpc_id)
    t = Tag(name, 'igw', region); t.tag_resource(conn, igw.id)

    return igw.id

def subnet_sizes(azs, cidr):
  """
  Calculate subnets sizes

  Possible scenarios:
    a) /25 2AZ  (4 Subnets = /27)
    b) /24 2AZ  (4 Subnets = /26)
    c) /23 2AZ  (4 Subnets = /25)
    d) /22 2AZ  (4 Subnets = /24)
    e) /23 3AZ  (3 Subnets = /27, 3 Subnets = /25)
    f) /22 3AZ  (3 Subnets = /26, 3 Subnets = /24)
  """

  if azs != 2 and azs != 3:
    print("ERROR: Number of AZs should be 2 or 3.")
    exit(1)

  netmasks = ('255.255.252.0', '255.255.254.0', '255.255.255.0', '255.255.255.128')

  ip = IPNetwork(cidr)
  mask = ip.netmask

  if azs == 3:
    if str(mask) not in netmasks[0:2]:
      print("ERROR: Netmask " + str(mask) + " not found.")
      exit(1)

    for n, netmask in enumerate(netmasks):
      if str(mask) == netmask:
        pub_net = list(ip.subnet(n + 24))
        pri_subs = pub_net[1:]
        pub_mask = pub_net[0].netmask

    pub_split = list(ip.subnet(26)) if (str(pub_mask) == '255.255.255.0') else list(ip.subnet(27))
    pub_subs = pub_split[:3]

    subnets = pub_subs + pri_subs

  else:
    if str(mask) not in netmasks:
      print("ERROR: Netmask " + str(mask) + " not found.")
      exit(1)

    for n, netmask in enumerate(netmasks):
      if str(mask) == netmask:
        subnets = list(ip.subnet(n + 24))

  return subnets

def create_sub(conn, name, region, vpc_id, azs, subnets, zones):
  """ Create subnets """

  i = 0; sub_ids = []; tier = 'public'
  for sub in subnets:
    subnet = conn.create_subnet(vpc_id, sub, availability_zone=zones[i])
    t = Tag(name, tier + '-sub', region); t.tag_resource(conn, subnet.id)

    sub_ids.append(subnet.id)
    print("sub-id: ", subnet.id, "\tsize: ", sub, "\tzone: ", zones[i])
    i += 1
    if i == azs: i = 0; tier = 'private'

  return sub_ids

def create_rtb(conn, name, region, vpc_id, azs, sub_ids, igw_id):
  """ Create and associate route-tables """

  i = 0; rtb_ids = []; tier = 'public'
  for sub in sub_ids:
    if i == 0:
      rtb = conn.create_route_table(vpc_id)
      conn.create_route(rtb.id, '0.0.0.0/0', igw_id)
      t = Tag(name, tier + '-rtb', region); t.tag_resource(conn, rtb.id)

      rtb_ids.append(rtb.id)
    conn.associate_route_table(rtb.id, sub)
    i += 1
    if i == azs: i = 0; tier = 'private'

  return rtb_ids

def create_acl(conn, name, region, vpc_id, azs, sub_ids, cidr):
  """ Create and associate network access-lists

      https://blogs.aws.amazon.com/security/post/Tx3NVS2JAL7KWOM/How-to-Help-Prepare-for-DDoS-Attacks-by-Reducing-Your-Attack-Surface
  """

  i = 0; acl_ids = []; tier = 'public'
  for sub in sub_ids:
    if i == 0:
      acl = conn.create_network_acl(vpc_id)
      conn.create_network_acl_entry(acl.id, 100, -1, 'allow',  cidr, egress=False)
      conn.create_network_acl_entry(acl.id, 200, 6,  'allow', '0.0.0.0/0', egress=False, port_range_from=443, port_range_to=443)
      conn.create_network_acl_entry(acl.id, 300, 6,  'allow', '0.0.0.0/0', egress=False, port_range_from=80, port_range_to=80)
      conn.create_network_acl_entry(acl.id, 400, 6,  'allow', '0.0.0.0/0', egress=False, port_range_from=1024, port_range_to=65535)
      conn.create_network_acl_entry(acl.id, 500, 6,  'allow', '0.0.0.0/0', egress=False, port_range_from=22, port_range_to=22)
      conn.create_network_acl_entry(acl.id, 100, -1, 'allow',  cidr, egress=True)
      conn.create_network_acl_entry(acl.id, 200, 6,  'allow', '0.0.0.0/0', egress=True,  port_range_from=443, port_range_to=443)
      conn.create_network_acl_entry(acl.id, 300, 6,  'allow', '0.0.0.0/0', egress=True,  port_range_from=80, port_range_to=80)
      conn.create_network_acl_entry(acl.id, 400, 6,  'allow', '0.0.0.0/0', egress=True,  port_range_from=1024, port_range_to=65535)
      conn.create_network_acl_entry(acl.id, 500, 6,  'allow', '0.0.0.0/0', egress=True,  port_range_from=22, port_range_to=22)
      t = Tag(name, tier + '-acl', region); t.tag_resource(conn, acl.id)

      acl_ids.append(acl.id)
    conn.associate_network_acl(acl.id, sub)
    i += 1
    if i == azs: i = 0; tier = 'private'

  return acl_ids

def create_flows(vpc_id, keyid, secret, region):
  """ Create VPC flow logs """

  session = Session(aws_access_key_id = keyid, aws_secret_access_key = secret, region_name = region)

  iam = session.client('iam')
  logs = session.client('logs')
  ec2 = session.client('ec2')

  # Check for an existing Role with standard name
  try:
    role = iam.get_role(RoleName='flowlogsRole')
  except:
    role = 'None'

  # Create VPC Flows Logs IAM Role
  if role == 'None':
    role = iam.create_role(
      Path = '/',
      RoleName = 'flowlogsRole',
      AssumeRolePolicyDocument = json.dumps(Template.RolePolicy))

    # Create VPC Flow Logs policy
    policy = iam.create_policy(
      Path = '/',
      PolicyName =  'flowlogsPolicy',
      Description = 'Grants access to CloudWatch Logs.',
      PolicyDocument = json.dumps(Template.LogsPolicy))

    role_name = role['Role']['RoleName']
    role_arn = role['Role']['Arn']
    policy_arn = policy['Policy']['Arn']

    # Attach policy to the IAM Role
    attach = iam.attach_role_policy(
      RoleName = role_name,
      PolicyArn = policy_arn)
  else:
    role_arn = role['Role']['Arn']

  logs_name = 'flowlogsGroup' + '-' + vpc_id

  # Create CloudWatch Logs group
  cwlogs = logs.create_log_group(
    logGroupName = logs_name)

  # Enable VPC Flow Logs
  flow_id = ec2.create_flow_logs(
    ResourceIds = [vpc_id],
    ResourceType = 'VPC',
    TrafficType = 'ALL',
    LogGroupName = logs_name,
    DeliverLogsPermissionArn = role_arn)

  return flow_id

def main(azs, region, keyid, secret, cidr, owner, env):
  """
  Do the work

  1.) Setup/validate region and availability-zones
  2.) Create the VPC
  3.) Create and attach an internet-gateway
  4.) Calculate subnet sizes (netaddr)
  5.) Create subnets
  6.) Create and associate route-tables
  7.) Create and associate network access-lists
  8.) Enable VPC Flow Logs
  """

  # Validate the region
  myregion = boto.ec2.get_region(region_name=region)
  if myregion == None:
    print("Unknown region.")
    exit(1)

  # Establish a VPC service connection
  try:
    conn = boto.vpc.VPCConnection(aws_access_key_id=keyid, aws_secret_access_key=secret, region=myregion)
  except boto.exception.EC2ResponseError as e:
    print(e.message)
    exit(1)

  # Grab the availability-zones
  zones = []
  all_zones = conn.get_all_zones()
  for zone in all_zones:
    if zone.state != 'available':
      continue
    zones.append(zone.name)

  subnets = subnet_sizes(azs, cidr)               # Calculate the subnet sizes
  name = owner.lower() + '-' + env.lower() + '-'  # Used for tagging

  vpc_id  = create_vpc(conn, name, region, cidr)
  igw_id  = create_igw(conn, name, region, vpc_id)
  sub_ids = create_sub(conn, name, region, vpc_id, azs, subnets, zones)
  rtb_ids = create_rtb(conn, name, region, vpc_id, azs, sub_ids, igw_id)
  acl_ids = create_acl(conn, name, region, vpc_id, azs, sub_ids, cidr)
  flow_id = create_flows(vpc_id, keyid, secret, region)

if __name__ == "__main__":

  main(azs = 3, region = 'us-west-2', keyid = 'XXXXX', secret = 'XXXXX', cidr = '10.64.0.0/23', owner = 'eng', env = 'dev')
<|endoftext|>


<commit_before>#This code is used for converting kernels. The rest of the
#code for this application will have the same extension as
#before

#asks for target file, has to be opencl
opencl_name = input("Whats the OpenCL file name? ")

#checks if the name is indeed a .cl file
splited_name = opencl_name.split(".")

if not((splited_name[1] == "cl")):
    print(opencl_name + " is not a valid name. Exiting... ")
    exit()

#use with open for a more secure method
try:
    with open(opencl_name) as opencl_file:
        opencl_data = opencl_file.read()

#if something wrong happen, exit the code
except:
    print ("Not possible to open the file. Exiting")
    exit()

#if everything works, try to create the cuda file
cuda_name = ".".join([splited_name[0], "cu"])
print(cuda_name)
try:
    cuda_data = open(cuda_name, "w")

except:
    print ("Not possible to create the file")
    exit()

print(opencl_data)
<commit_msg>creating a directory to keep files on their right places<commit_after>#This code is used for converting kernels. The rest of the
#code for this application will have the same extension as
#before

import os
cuda_path = "./CUDA_Files/"

#asks for target file, has to be opencl
opencl_name = input("Whats the OpenCL file name? ")

#checks if the name is indeed a .cl file
splited_name = opencl_name.split(".")

if not((splited_name[1] == "cl")):
    print(opencl_name + " is not a valid name. Exiting... ")
    exit()

#use with open for a more secure method
try:
    with open(opencl_name) as opencl_file:
        opencl_data = opencl_file.read()

#if something wrong happen, exit the code
except:
    print ("Not possible to open the file. Exiting")
    exit()

#if everything works, try to create the cuda file and directory
cuda_name = ".".join([splited_name[0], "cu"])
os.mkdir("CUDA_Files")

try:
    cuda_data = open(cuda_path + cuda_name, "w")

except:
    print ("Not possible to create the file")
    exit()

print(opencl_data)
<|endoftext|>


<commit_before>"""Module for constructing etymological dictionary from www.etymonline.com."""

__author__ = "Zachary Yocum"
__email__  = "zyocum@brandeis.edu"

import io, os, json, re, sys, time
from collections import defaultdict
from operator import itemgetter
from string import *

from bs4 import BeautifulSoup as BS, CData
from bs4.element import Tag
from progressbar import ProgressBar, Percentage, Bar
from language import LANGUAGES
from nltk.stem import WordNetLemmatizer
from nltk.corpus import wordnet

wnl = WordNetLemmatizer()

term = re.compile(
    r"""
    ^                         # start of line
    \s*                       # optional whitespace
    (?P<term>.+?)             # the term
    \s*                       # optional whitespace
    (\((?P<pos>\w+\.)\)?)?    # optional part of speech
    (\(?(?P<index>[0-9]+)\))? # optional term (if term has multiple entries)
    \s*                       # optional whitespace
    $                         # end of line
    """,
    re.VERBOSE
)

prefix = re.compile(
    r"""
    ^                   # start of line
    (?P<prefix>[^-'].+) # prefix text
    [-']                # prefix end
    $                   # end of line
    """,
    re.VERBOSE
)

infix = re.compile(
    r"""
    ^             # start of line
    -             # infix start
    (?P<infix>.+) # infix text
    -             # infix end
    $             # end of line
    """,
    re.VERBOSE
)

suffix = re.compile(
    r"""
    ^                  # start of line
    [-']               # suffix start
    (?P<suffix>.+[^-]) # suffix text
    $                  # end of line
    """,
    re.VERBOSE
)

word = re.compile(
    r"""
    ^                      # start of line
    (?P<word>[^-'].+[^-']) # word text
    $                      # end of line
    """,
    re.VERBOSE
)

AFFIXES = prefix, infix, suffix, word

FIRST, SECOND, PENULTIMATE, ULTIMATE = map(itemgetter, range(2) + range(-2, 0))

def nest():
    """A recursive dictionary."""
    return defaultdict(nest)

def find_files(directory='.', pattern='.*', recursive=True):
    """Search recursively for files matching a pattern"""
    if recursive:
        return (os.path.join(directory, filename)
            for directory, subdirectories, filenames in os.walk(directory)
            for filename in filenames if re.match(pattern, filename))
    else:
        return (os.path.join(directory, filename)
            for filename in os.listdir(directory)
            if re.match(pattern, filename))

class Entry(dict):
    """dict wrapper to represent an instance of an etymology entry."""
    def __init__(self, term, definition):
        super(Entry, self).__init__()
        self.parse(term)
        self.definition = definition.text
        self['definition'] = self.definition
        self.text = self['term']
        self.pos = unicode(self['pos'] or '')
        self.prefix = self.get('prefix')
        self.infix = self.get('infix')
        self.suffix = self.get('suffix')
        self.word = self.get('word')
        self.key = u'/'.join([self.text, self.pos])
        self.languages = search(
            self.definition,
            sorted(LANGUAGES, key=len),
            set()
        )
        self['languages'] = sorted(self.languages)

    def __repr__(self):
        return json.dumps(self, indent=4, ensure_ascii=False)

    def parse(self, dt):
        """Parse the entry and update it."""
        entry = re.match(term, dt.text).groupdict()
        index = entry.pop('index')
        if not index:
            self.index = 0
        else:
            self.index = int(index) - 1
        for affix in AFFIXES:
            match = re.match(affix, entry['term'])
            if match:
                gd = match.groupdict()
                for k, v in gd.iteritems():
                    entry.update({k : v.strip()})
        self.update(entry)

def search(text, languages, results):
    """Find the set of all languages mentioned in text."""
    if not languages:
        return results
    else:
        language = languages.pop()
        match = re.search(language, text)
        if match:
            results.add(language)
            text = u''.join(text.split(language))
        return search(text, languages, results)

def load(filename):
    with io.open(filename, mode='r', encoding='utf-8') as file:
        return json.load(file)

LANGUAGES = load(os.path.join('resources', 'languages.json'))

def load_page(filename):
    """Parse an HTML page and get a list of entries from it."""
    with open(filename, 'r') as file:
        soup = BS(file)
        dl = soup.find('dl')   # list of terms and their definitions
        dts = dl.findAll('dt') # terms
        dds = dl.findAll('dd') # definitions
    entries = zip(dts, dds)
    return entries

def dump(data, filename):
    """Dump an object to file as JSON."""
    with io.open(filename, mode='w', encoding='utf-8') as file:
        json.dump(
            data,
            file,
            encoding='utf-8',
            ensure_ascii=False,
            sort_keys=True,
            indent=4
        )

def etymologies(pages):
    """Build the dictionary from the scraped list of pages."""
    etymologies = defaultdict(list)
    print('Building etymology dictionary...')
    with ProgressBar(maxval=len(pages)) as progress:
        for i, page in enumerate(pages):
            entries = (Entry(*e) for e in load_page(page))
            for entry in entries:
                etymologies[entry.text].insert(entry.index, entry)
            progress.update(i)
    return etymologies

def affixes(etymologies):
    """Generate entries for affixes only."""
    for affix in AFFIXES:
        name = FIRST(affix.groupindex.keys())
        print('building {} dictionary...'.format(name))
        dictionary = defaultdict(list)
        with ProgressBar(maxval=len(etymologies)) as progress:
            for i, key in enumerate(etymologies):
                for definition in etymologies[key]:
                    if definition.has_key(name):
                        dictionary[key].append(definition)
                progress.update(i)
        yield name, dictionary

def etym(query, pos, dictionary):
    """Look up an etymology in the dictionary."""
    wn_pos = wordnet_pos(pos)
    try:
        lemma = wnl.lemmatize(query, wn_pos)
    except KeyError:
        lemma = query
    results = dictionary.get(lemma, [])
    for result in results:
        try:
            wn_pos_tags = map(wordnet_pos, (result['pos'] or '', wn_pos))
            match = re.match(*wn_pos_tags)
        except TypeError:
            #print 'query:', query
            #print 'pos:', pos
            #print 'wn_pos:', wn_pos
            #print 'lemma:', lemma
            #print 'wn_pos_tags:', wn_pos_tags
            match = False
        if match:
            return lemma, [result]
    return lemma, results

def lookup(*args):
    """Returns a structured result from a query term + pos + dictionary."""
    lemma, results = etym(*args)
    languages = nest()
    if not results:
        query, _, dictionary = args
        lemma, results = etym(query, None, dictionary)
    for result in results:
        languages[lemma][unicode(result['pos'])] = result['languages']
    return languages

def wordnet_pos(pos):
    """Munge part-of-speech tags to be compatible with WordNet."""
    pos = (pos or '').strip(punctuation + whitespace).lower()
    if pos.startswith('j') or 'adj'.startswith(pos):
        return wordnet.ADJ
    elif pos.startswith('v'):
        return wordnet.VERB
    elif pos.startswith('n'):
        return wordnet.NOUN
    elif pos.startswith('r') or 'adv'.startswith(pos):
        return wordnet.ADV
    elif pos.startswith('in') or 'prep'.startswith(pos):
        return u'p'
    elif pos.startswith('fw'):
        return u'v'
    else:
        return None

site = os.path.join('resources', 'www.etymonline.com')

etymology_file = os.path.join('resources', 'etymology.json')

def setup():
    """Setup the etymological dictionary and store it as JSON."""
    if not os.path.isfile(etymology_file):
        page = re.compile(r'index.php\?l=\w+&p=\d+&allowed_in_frame=0.html')
        pages = list(find_files(directory=site, pattern=page, recursive=False))
        etymology = etymologies(pages)
        dump(etymology, etymology_file)
        for affix, dictionary in affixes(etymology):
            affix_file = os.path.join('resources', '{}.json'.format(affix))
            if not os.path.isfile(affix_file):
                dump(dictionary, affix_file)
setup()

ETYMOLOGY = load(etymology_file)

def languages(query, pos):
    """Look up a set of source languages for the query term."""
    lemma, results = etym(query, pos, ETYMOLOGY)
    languages = set()
    if not results:
        lemma, results = etym(query, None, ETYMOLOGY)
    for result in results:
        languages.update(set(result['languages']))
    return languages

def main(args):
    args = dict(enumerate(args))
    query = args.get(0, '')
    pos = args.get(1, '')
    print(
        json.dumps(
            {
                'query' : filter(None, [query, pos]),
                'results' : list(languages(query, pos))
            },
            indent=True,
            ensure_ascii=False
        )
    )

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>removed unecessary import<commit_after>"""Module for constructing etymological dictionary from www.etymonline.com."""

__author__ = "Zachary Yocum"
__email__  = "zyocum@brandeis.edu"

import io, os, json, re, sys, time
from collections import defaultdict
from operator import itemgetter
from string import *

from bs4 import BeautifulSoup as BS, CData
from bs4.element import Tag
from language import LANGUAGES
from nltk.stem import WordNetLemmatizer
from nltk.corpus import wordnet

wnl = WordNetLemmatizer()

term = re.compile(
    r"""
    ^                         # start of line
    \s*                       # optional whitespace
    (?P<term>.+?)             # the term
    \s*                       # optional whitespace
    (\((?P<pos>\w+\.)\)?)?    # optional part of speech
    (\(?(?P<index>[0-9]+)\))? # optional term (if term has multiple entries)
    \s*                       # optional whitespace
    $                         # end of line
    """,
    re.VERBOSE
)

prefix = re.compile(
    r"""
    ^                   # start of line
    (?P<prefix>[^-'].+) # prefix text
    [-']                # prefix end
    $                   # end of line
    """,
    re.VERBOSE
)

infix = re.compile(
    r"""
    ^             # start of line
    -             # infix start
    (?P<infix>.+) # infix text
    -             # infix end
    $             # end of line
    """,
    re.VERBOSE
)

suffix = re.compile(
    r"""
    ^                  # start of line
    [-']               # suffix start
    (?P<suffix>.+[^-]) # suffix text
    $                  # end of line
    """,
    re.VERBOSE
)

word = re.compile(
    r"""
    ^                      # start of line
    (?P<word>[^-'].+[^-']) # word text
    $                      # end of line
    """,
    re.VERBOSE
)

AFFIXES = prefix, infix, suffix, word

FIRST, SECOND, PENULTIMATE, ULTIMATE = map(itemgetter, range(2) + range(-2, 0))

def nest():
    """A recursive dictionary."""
    return defaultdict(nest)

def find_files(directory='.', pattern='.*', recursive=True):
    """Search recursively for files matching a pattern"""
    if recursive:
        return (os.path.join(directory, filename)
            for directory, subdirectories, filenames in os.walk(directory)
            for filename in filenames if re.match(pattern, filename))
    else:
        return (os.path.join(directory, filename)
            for filename in os.listdir(directory)
            if re.match(pattern, filename))

class Entry(dict):
    """dict wrapper to represent an instance of an etymology entry."""
    def __init__(self, term, definition):
        super(Entry, self).__init__()
        self.parse(term)
        self.definition = definition.text
        self['definition'] = self.definition
        self.text = self['term']
        self.pos = unicode(self['pos'] or '')
        self.prefix = self.get('prefix')
        self.infix = self.get('infix')
        self.suffix = self.get('suffix')
        self.word = self.get('word')
        self.key = u'/'.join([self.text, self.pos])
        self.languages = search(
            self.definition,
            sorted(LANGUAGES, key=len),
            set()
        )
        self['languages'] = sorted(self.languages)

    def __repr__(self):
        return json.dumps(self, indent=4, ensure_ascii=False)

    def parse(self, dt):
        """Parse the entry and update it."""
        entry = re.match(term, dt.text).groupdict()
        index = entry.pop('index')
        if not index:
            self.index = 0
        else:
            self.index = int(index) - 1
        for affix in AFFIXES:
            match = re.match(affix, entry['term'])
            if match:
                gd = match.groupdict()
                for k, v in gd.iteritems():
                    entry.update({k : v.strip()})
        self.update(entry)

def search(text, languages, results):
    """Find the set of all languages mentioned in text."""
    if not languages:
        return results
    else:
        language = languages.pop()
        match = re.search(language, text)
        if match:
            results.add(language)
            text = u''.join(text.split(language))
        return search(text, languages, results)

def load(filename):
    with io.open(filename, mode='r', encoding='utf-8') as file:
        return json.load(file)

LANGUAGES = load(os.path.join('resources', 'languages.json'))

def load_page(filename):
    """Parse an HTML page and get a list of entries from it."""
    with open(filename, 'r') as file:
        soup = BS(file)
        dl = soup.find('dl')   # list of terms and their definitions
        dts = dl.findAll('dt') # terms
        dds = dl.findAll('dd') # definitions
    entries = zip(dts, dds)
    return entries

def dump(data, filename):
    """Dump an object to file as JSON."""
    with io.open(filename, mode='w', encoding='utf-8') as file:
        json.dump(
            data,
            file,
            encoding='utf-8',
            ensure_ascii=False,
            sort_keys=True,
            indent=4
        )

def etymologies(pages):
    """Build the dictionary from the scraped list of pages."""
    etymologies = defaultdict(list)
    print('Building etymology dictionary...')
    with ProgressBar(maxval=len(pages)) as progress:
        for i, page in enumerate(pages):
            entries = (Entry(*e) for e in load_page(page))
            for entry in entries:
                etymologies[entry.text].insert(entry.index, entry)
            progress.update(i)
    return etymologies

def affixes(etymologies):
    """Generate entries for affixes only."""
    for affix in AFFIXES:
        name = FIRST(affix.groupindex.keys())
        print('building {} dictionary...'.format(name))
        dictionary = defaultdict(list)
        with ProgressBar(maxval=len(etymologies)) as progress:
            for i, key in enumerate(etymologies):
                for definition in etymologies[key]:
                    if definition.has_key(name):
                        dictionary[key].append(definition)
                progress.update(i)
        yield name, dictionary

def etym(query, pos, dictionary):
    """Look up an etymology in the dictionary."""
    wn_pos = wordnet_pos(pos)
    try:
        lemma = wnl.lemmatize(query, wn_pos)
    except KeyError:
        lemma = query
    results = dictionary.get(lemma, [])
    for result in results:
        try:
            wn_pos_tags = map(wordnet_pos, (result['pos'] or '', wn_pos))
            match = re.match(*wn_pos_tags)
        except TypeError:
            #print 'query:', query
            #print 'pos:', pos
            #print 'wn_pos:', wn_pos
            #print 'lemma:', lemma
            #print 'wn_pos_tags:', wn_pos_tags
            match = False
        if match:
            return lemma, [result]
    return lemma, results

def lookup(*args):
    """Returns a structured result from a query term + pos + dictionary."""
    lemma, results = etym(*args)
    languages = nest()
    if not results:
        query, _, dictionary = args
        lemma, results = etym(query, None, dictionary)
    for result in results:
        languages[lemma][unicode(result['pos'])] = result['languages']
    return languages

def wordnet_pos(pos):
    """Munge part-of-speech tags to be compatible with WordNet."""
    pos = (pos or '').strip(punctuation + whitespace).lower()
    if pos.startswith('j') or 'adj'.startswith(pos):
        return wordnet.ADJ
    elif pos.startswith('v'):
        return wordnet.VERB
    elif pos.startswith('n'):
        return wordnet.NOUN
    elif pos.startswith('r') or 'adv'.startswith(pos):
        return wordnet.ADV
    elif pos.startswith('in') or 'prep'.startswith(pos):
        return u'p'
    elif pos.startswith('fw'):
        return u'v'
    else:
        return None

site = os.path.join('resources', 'www.etymonline.com')

etymology_file = os.path.join('resources', 'etymology.json')

def setup():
    """Setup the etymological dictionary and store it as JSON."""
    if not os.path.isfile(etymology_file):
        page = re.compile(r'index.php\?l=\w+&p=\d+&allowed_in_frame=0.html')
        pages = list(find_files(directory=site, pattern=page, recursive=False))
        etymology = etymologies(pages)
        dump(etymology, etymology_file)
        for affix, dictionary in affixes(etymology):
            affix_file = os.path.join('resources', '{}.json'.format(affix))
            if not os.path.isfile(affix_file):
                dump(dictionary, affix_file)
setup()

ETYMOLOGY = load(etymology_file)

def languages(query, pos):
    """Look up a set of source languages for the query term."""
    lemma, results = etym(query, pos, ETYMOLOGY)
    languages = set()
    if not results:
        lemma, results = etym(query, None, ETYMOLOGY)
    for result in results:
        languages.update(set(result['languages']))
    return languages

def main(args):
    args = dict(enumerate(args))
    query = args.get(0, '')
    pos = args.get(1, '')
    print(
        json.dumps(
            {
                'query' : filter(None, [query, pos]),
                'results' : list(languages(query, pos))
            },
            indent=True,
            ensure_ascii=False
        )
    )

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>