<commit_before>from parcels.codegenerator import KernelGenerator, LoopGenerator
from parcels.compiler import get_cache_dir
from parcels.tools.error import ErrorCode, recovery_map as recovery_base_map
from parcels.field import FieldSamplingError, Field, SummedField, NestedField
from parcels.tools.loggers import logger
from parcels.kernels.advection import AdvectionRK4_3D
from os import path, remove
import numpy as np
import numpy.ctypeslib as npct
import time
from ctypes import c_int, c_float, c_double, c_void_p, byref
import _ctypes
from sys import platform
from ast import parse, FunctionDef, Module
import inspect
from copy import deepcopy
import re
from hashlib import md5
import math  # noqa
import random  # noqa

__all__ = ['Kernel']

re_indent = re.compile(r"^(\s+)")

def fix_indentation(string):
    """Fix indentation to allow in-lined kernel definitions"""
    lines = string.split('\n')
    indent = re_indent.match(lines[0])
    if indent:
        lines = [l.replace(indent.groups()[0], '', 1) for l in lines]
    return "\n".join(lines)

class Kernel(object):
    """Kernel object that encapsulates auto-generated code.

    :arg fieldset: FieldSet object providing the field information
    :arg ptype: PType object for the kernel particle

    Note: A Kernel is either created from a compiled <function ...> object
    or the necessary information (funcname, funccode, funcvars) is provided.
    The py_ast argument may be derived from the code string, but for
    concatenation, the merged AST plus the new header definition is required.
    """

    def __init__(self, fieldset, ptype, pyfunc=None, funcname=None,
                 funccode=None, py_ast=None, funcvars=None, c_include=""):
        self.fieldset = fieldset
        self.ptype = ptype
        self._lib = None

        # Derive meta information from pyfunc, if not given
        self.funcname = funcname or pyfunc.__name__
        if pyfunc is AdvectionRK4_3D:
            warning = False
            if isinstance(fieldset.W, Field) and fieldset.W.creation_log != 'from_nemo' and fieldset.W._scaling_factor > 0:
                warning = True
            if type(fieldset.W) in [SummedField, NestedField]:
                for f in fieldset.W:
                    if f.creation_log != 'from_nemo' and f._scaling_factor > 0:
                        warning = True
            if warning:
                logger.warning_once('Note that in AdvectionRK4_3D, vertical velocity is assumed positive towards increasing z.\n'
                                    '         If z increases downward and w is positive upward you can re-orient it downwards by setting fieldset.W.set_scaling_factor(-1.)')
        if funcvars is not None:
            self.funcvars = funcvars
        elif hasattr(pyfunc, '__code__'):
            self.funcvars = list(pyfunc.__code__.co_varnames)
        else:
            self.funcvars = None
        self.funccode = funccode or inspect.getsource(pyfunc.__code__)
        # Parse AST if it is not provided explicitly
        self.py_ast = py_ast or parse(fix_indentation(self.funccode)).body[0]
        if pyfunc is None:
            # Extract user context by inspecting the call stack
            stack = inspect.stack()
            try:
                user_ctx = stack[-1][0].f_globals
                user_ctx['math'] = globals()['math']
                user_ctx['random'] = globals()['random']
                user_ctx['ErrorCode'] = globals()['ErrorCode']
            except:
                logger.warning("Could not access user context when merging kernels")
                user_ctx = globals()
            finally:
                del stack  # Remove cyclic references
            # Compile and generate Python function from AST
            py_mod = Module(body=[self.py_ast])
            exec(compile(py_mod, "<ast>", "exec"), user_ctx)
            self.pyfunc = user_ctx[self.funcname]
        else:
            self.pyfunc = pyfunc
        assert len(inspect.getargspec(self.pyfunc).args) == 3, \
            'Since Parcels v2.0, kernels do only take 3 arguments: particle, fieldset, time !! AND !! Argument order in field interpolation is time, depth, lat, lon.'

        self.name = "%s%s" % (ptype.name, self.funcname)

        # Generate the kernel function and add the outer loop
        if self.ptype.uses_jit:
            kernelgen = KernelGenerator(fieldset, ptype)
            kernel_ccode = kernelgen.generate(deepcopy(self.py_ast),
                                              self.funcvars)
            self.field_args = kernelgen.field_args
            self.vector_field_args = kernelgen.vector_field_args
            fieldset = self.fieldset
            for fname in self.vector_field_args:
                f = getattr(fieldset, fname)
                Wname = f.W.name if f.W else 'not_defined'
                for sF in [f.U.name, f.V.name, Wname]:
                    if sF not in self.field_args:
                        try:
                            self.field_args[sF] = getattr(fieldset, sF)
                        except:
                            continue
            self.const_args = kernelgen.const_args
            loopgen = LoopGenerator(fieldset, ptype)
            if path.isfile(c_include):
                with open(c_include, 'r') as f:
                    c_include_str = f.read()
            else:
                c_include_str = c_include
            self.ccode = loopgen.generate(self.funcname, self.field_args, self.const_args,
                                          kernel_ccode, c_include_str)

            basename = path.join(get_cache_dir(), self._cache_key)
            self.src_file = "%s.c" % basename
            self.lib_file = "%s.%s" % (basename, 'dll' if platform == 'win32' else 'so')
            self.log_file = "%s.log" % basename

    def __del__(self):
        # Clean-up the in-memory dynamic linked libraries.
        # This is not really necessary, as these programs are not that large, but with the new random
        # naming scheme which is required on Windows OS'es to deal with updates to a Parcels' kernel.
        if self._lib is not None:
            _ctypes.FreeLibrary(self._lib._handle) if platform == 'win32' else _ctypes.dlclose(self._lib._handle)
            del self._lib
            self._lib = None
            if path.isfile(self.lib_file):
                [remove(s) for s in [self.src_file, self.lib_file, self.log_file]]

    @property
    def _cache_key(self):
        field_keys = "-".join(["%s:%s" % (name, field.units.__class__.__name__)
                               for name, field in self.field_args.items()])
        key = self.name + self.ptype._cache_key + field_keys + ('TIME:%f' % time.time())
        return md5(key.encode('utf-8')).hexdigest()

    def remove_lib(self):
        # Unload the currently loaded dynamic linked library to be secure
        if self._lib is not None:
            _ctypes.FreeLibrary(self._lib._handle) if platform == 'win32' else _ctypes.dlclose(self._lib._handle)
            del self._lib
            self._lib = None
        # If file already exists, pull new names. This is necessary on a Windows machine, because
        # Python's ctype does not deal in any sort of manner well with dynamic linked libraries on this OS.
        if path.isfile(self.lib_file):
            [remove(s) for s in [self.src_file, self.lib_file, self.log_file]]
            basename = path.join(get_cache_dir(), self._cache_key)
            self.src_file = "%s.c" % basename
            self.lib_file = "%s.%s" % (basename, 'dll' if platform == 'win32' else 'so')
            self.log_file = "%s.log" % basename

    def compile(self, compiler):
        """ Writes kernel code to file and compiles it."""
        with open(self.src_file, 'w') as f:
            f.write(self.ccode)
        compiler.compile(self.src_file, self.lib_file, self.log_file)
        logger.info("Compiled %s ==> %s" % (self.name, self.lib_file))

    def load_lib(self):
        self._lib = npct.load_library(self.lib_file, '.')
        self._function = self._lib.particle_loop

    def execute_jit(self, pset, endtime, dt):
        """Invokes JIT engine to perform the core update loop"""
        if len(pset.particles) > 0:
            assert pset.fieldset.gridset.size == len(pset.particles[0].xi), \
                'FieldSet has different amount of grids than Particle.xi. Have you added Fields after creating the ParticleSet?'
        for g in pset.fieldset.gridset.grids:
            g.cstruct = None  # This force to point newly the grids from Python to C
        # Make a copy of the transposed array to enforce
        # C-contiguous memory layout for JIT mode.
        for f in self.field_args.values():
            if not f.data.flags.c_contiguous:
                f.data = f.data.copy()
        for g in pset.fieldset.gridset.grids:
            if not g.depth.flags.c_contiguous:
                g.depth = g.depth.copy()
            if not g.lon.flags.c_contiguous:
                g.lon = g.lon.copy()
            if not g.lat.flags.c_contiguous:
                g.lat = g.lat.copy()
        fargs = [byref(f.ctypes_struct) for f in self.field_args.values()]
        fargs += [c_float(f) for f in self.const_args.values()]
        particle_data = pset._particle_data.ctypes.data_as(c_void_p)
        self._function(c_int(len(pset)), particle_data,
                       c_double(endtime), c_float(dt), *fargs)

    def execute_python(self, pset, endtime, dt):
        """Performs the core update loop via Python"""
        sign_dt = np.sign(dt)

        # back up variables in case of ErrorCode.Repeat
        p_var_back = {}

        for p in pset.particles:
            ptype = p.getPType()
            # Don't execute particles that aren't started yet
            sign_end_part = np.sign(endtime - p.time)
            if (sign_end_part != sign_dt) and (dt != 0):
                continue

            # Compute min/max dt for first timestep
            dt_pos = min(abs(p.dt), abs(endtime - p.time))
            while dt_pos > 1e-6 or dt == 0:
                for var in ptype.variables:
                    p_var_back[var.name] = getattr(p, var.name)
                try:
                    p.dt = sign_dt * dt_pos
                    res = self.pyfunc(p, pset.fieldset, p.time)
                except FieldSamplingError as fse:
                    res = ErrorCode.ErrorOutOfBounds
                    p.exception = fse
                except Exception as e:
                    res = ErrorCode.Error
                    p.exception = e

                # Update particle state for explicit returns
                if res is not None:
                    p.state = res

                # Handle particle time and time loop
                if res is None or res == ErrorCode.Success:
                    # Update time and repeat
                    p.time += sign_dt * dt_pos
                    dt_pos = min(abs(p.dt), abs(endtime - p.time))
                    if dt == 0:
                        break
                    continue
                elif res == ErrorCode.Repeat:
                    # Try again without time update
                    for var in ptype.variables:
                        if var.name not in ['dt', 'state']:
                            setattr(p, var.name, p_var_back[var.name])
                    dt_pos = min(abs(p.dt), abs(endtime - p.time))
                    break
                else:
                    break  # Failure - stop time loop

    def execute(self, pset, endtime, dt, recovery=None, output_file=None):
        """Execute this Kernel over a ParticleSet for several timesteps"""

        def remove_deleted(pset):
            """Utility to remove all particles that signalled deletion"""
            indices = [i for i, p in enumerate(pset.particles)
                       if p.state in [ErrorCode.Delete]]
            if len(indices) > 0 and output_file is not None:
                output_file.write(pset[indices], endtime, deleted_only=True)
            pset.remove(indices)

        if recovery is None:
            recovery = {}
        recovery_map = recovery_base_map.copy()
        recovery_map.update(recovery)

        # Execute the kernel over the particle set
        if self.ptype.uses_jit:
            self.execute_jit(pset, endtime, dt)
        else:
            self.execute_python(pset, endtime, dt)

        # Remove all particles that signalled deletion
        remove_deleted(pset)

        # Identify particles that threw errors
        error_particles = [p for p in pset.particles
                           if p.state != ErrorCode.Success]
        while len(error_particles) > 0:
            # Apply recovery kernel
            for p in error_particles:
                if p.state == ErrorCode.Repeat:
                    p.state = ErrorCode.Success
                else:
                    recovery_kernel = recovery_map[p.state]
                    p.state = ErrorCode.Success
                    recovery_kernel(p, self.fieldset, p.time)

            # Remove all particles that signalled deletion
            remove_deleted(pset)

            # Execute core loop again to continue interrupted particles
            if self.ptype.uses_jit:
                self.execute_jit(pset, endtime, dt)
            else:
                self.execute_python(pset, endtime, dt)

            error_particles = [p for p in pset.particles
                               if p.state != ErrorCode.Success]

    def merge(self, kernel):
        funcname = self.funcname + kernel.funcname
        func_ast = FunctionDef(name=funcname, args=self.py_ast.args,
                               body=self.py_ast.body + kernel.py_ast.body,
                               decorator_list=[], lineno=1, col_offset=0)
        return Kernel(self.fieldset, self.ptype, pyfunc=None,
                      funcname=funcname, funccode=self.funccode + kernel.funccode,
                      py_ast=func_ast, funcvars=self.funcvars + kernel.funcvars)

    def __add__(self, kernel):
        if not isinstance(kernel, Kernel):
            kernel = Kernel(self.fieldset, self.ptype, pyfunc=kernel)
        return self.merge(kernel)

    def __radd__(self, kernel):
        if not isinstance(kernel, Kernel):
            kernel = Kernel(self.fieldset, self.ptype, pyfunc=kernel)
        return kernel.merge(self)
<commit_msg>bug fix<commit_after>from parcels.codegenerator import KernelGenerator, LoopGenerator
from parcels.compiler import get_cache_dir
from parcels.tools.error import ErrorCode, recovery_map as recovery_base_map
from parcels.field import FieldSamplingError, Field, SummedField, NestedField
from parcels.tools.loggers import logger
from parcels.kernels.advection import AdvectionRK4_3D
from os import path, remove
import numpy as np
import numpy.ctypeslib as npct
import time
from ctypes import c_int, c_float, c_double, c_void_p, byref
import _ctypes
from sys import platform
from ast import parse, FunctionDef, Module
import inspect
from copy import deepcopy
import re
from hashlib import md5
import math  # noqa
import random  # noqa

__all__ = ['Kernel']

re_indent = re.compile(r"^(\s+)")

def fix_indentation(string):
    """Fix indentation to allow in-lined kernel definitions"""
    lines = string.split('\n')
    indent = re_indent.match(lines[0])
    if indent:
        lines = [l.replace(indent.groups()[0], '', 1) for l in lines]
    return "\n".join(lines)

class Kernel(object):
    """Kernel object that encapsulates auto-generated code.

    :arg fieldset: FieldSet object providing the field information
    :arg ptype: PType object for the kernel particle

    Note: A Kernel is either created from a compiled <function ...> object
    or the necessary information (funcname, funccode, funcvars) is provided.
    The py_ast argument may be derived from the code string, but for
    concatenation, the merged AST plus the new header definition is required.
    """

    def __init__(self, fieldset, ptype, pyfunc=None, funcname=None,
                 funccode=None, py_ast=None, funcvars=None, c_include=""):
        self.fieldset = fieldset
        self.ptype = ptype
        self._lib = None

        # Derive meta information from pyfunc, if not given
        self.funcname = funcname or pyfunc.__name__
        if pyfunc is AdvectionRK4_3D:
            warning = False
            if isinstance(fieldset.W, Field) and fieldset.W.creation_log != 'from_nemo' and \
               fieldset.W._scaling_factor is not None and fieldset.W._scaling_factor > 0:
                warning = True
            if type(fieldset.W) in [SummedField, NestedField]:
                for f in fieldset.W:
                    if f.creation_log != 'from_nemo' and f._scaling_factor is not None and f._scaling_factor > 0:
                        warning = True
            if warning:
                logger.warning_once('Note that in AdvectionRK4_3D, vertical velocity is assumed positive towards increasing z.\n'
                                    '         If z increases downward and w is positive upward you can re-orient it downwards by setting fieldset.W.set_scaling_factor(-1.)')
        if funcvars is not None:
            self.funcvars = funcvars
        elif hasattr(pyfunc, '__code__'):
            self.funcvars = list(pyfunc.__code__.co_varnames)
        else:
            self.funcvars = None
        self.funccode = funccode or inspect.getsource(pyfunc.__code__)
        # Parse AST if it is not provided explicitly
        self.py_ast = py_ast or parse(fix_indentation(self.funccode)).body[0]
        if pyfunc is None:
            # Extract user context by inspecting the call stack
            stack = inspect.stack()
            try:
                user_ctx = stack[-1][0].f_globals
                user_ctx['math'] = globals()['math']
                user_ctx['random'] = globals()['random']
                user_ctx['ErrorCode'] = globals()['ErrorCode']
            except:
                logger.warning("Could not access user context when merging kernels")
                user_ctx = globals()
            finally:
                del stack  # Remove cyclic references
            # Compile and generate Python function from AST
            py_mod = Module(body=[self.py_ast])
            exec(compile(py_mod, "<ast>", "exec"), user_ctx)
            self.pyfunc = user_ctx[self.funcname]
        else:
            self.pyfunc = pyfunc
        assert len(inspect.getargspec(self.pyfunc).args) == 3, \
            'Since Parcels v2.0, kernels do only take 3 arguments: particle, fieldset, time !! AND !! Argument order in field interpolation is time, depth, lat, lon.'

        self.name = "%s%s" % (ptype.name, self.funcname)

        # Generate the kernel function and add the outer loop
        if self.ptype.uses_jit:
            kernelgen = KernelGenerator(fieldset, ptype)
            kernel_ccode = kernelgen.generate(deepcopy(self.py_ast),
                                              self.funcvars)
            self.field_args = kernelgen.field_args
            self.vector_field_args = kernelgen.vector_field_args
            fieldset = self.fieldset
            for fname in self.vector_field_args:
                f = getattr(fieldset, fname)
                Wname = f.W.name if f.W else 'not_defined'
                for sF in [f.U.name, f.V.name, Wname]:
                    if sF not in self.field_args:
                        try:
                            self.field_args[sF] = getattr(fieldset, sF)
                        except:
                            continue
            self.const_args = kernelgen.const_args
            loopgen = LoopGenerator(fieldset, ptype)
            if path.isfile(c_include):
                with open(c_include, 'r') as f:
                    c_include_str = f.read()
            else:
                c_include_str = c_include
            self.ccode = loopgen.generate(self.funcname, self.field_args, self.const_args,
                                          kernel_ccode, c_include_str)

            basename = path.join(get_cache_dir(), self._cache_key)
            self.src_file = "%s.c" % basename
            self.lib_file = "%s.%s" % (basename, 'dll' if platform == 'win32' else 'so')
            self.log_file = "%s.log" % basename

    def __del__(self):
        # Clean-up the in-memory dynamic linked libraries.
        # This is not really necessary, as these programs are not that large, but with the new random
        # naming scheme which is required on Windows OS'es to deal with updates to a Parcels' kernel.
        if self._lib is not None:
            _ctypes.FreeLibrary(self._lib._handle) if platform == 'win32' else _ctypes.dlclose(self._lib._handle)
            del self._lib
            self._lib = None
            if path.isfile(self.lib_file):
                [remove(s) for s in [self.src_file, self.lib_file, self.log_file]]

    @property
    def _cache_key(self):
        field_keys = "-".join(["%s:%s" % (name, field.units.__class__.__name__)
                               for name, field in self.field_args.items()])
        key = self.name + self.ptype._cache_key + field_keys + ('TIME:%f' % time.time())
        return md5(key.encode('utf-8')).hexdigest()

    def remove_lib(self):
        # Unload the currently loaded dynamic linked library to be secure
        if self._lib is not None:
            _ctypes.FreeLibrary(self._lib._handle) if platform == 'win32' else _ctypes.dlclose(self._lib._handle)
            del self._lib
            self._lib = None
        # If file already exists, pull new names. This is necessary on a Windows machine, because
        # Python's ctype does not deal in any sort of manner well with dynamic linked libraries on this OS.
        if path.isfile(self.lib_file):
            [remove(s) for s in [self.src_file, self.lib_file, self.log_file]]
            basename = path.join(get_cache_dir(), self._cache_key)
            self.src_file = "%s.c" % basename
            self.lib_file = "%s.%s" % (basename, 'dll' if platform == 'win32' else 'so')
            self.log_file = "%s.log" % basename

    def compile(self, compiler):
        """ Writes kernel code to file and compiles it."""
        with open(self.src_file, 'w') as f:
            f.write(self.ccode)
        compiler.compile(self.src_file, self.lib_file, self.log_file)
        logger.info("Compiled %s ==> %s" % (self.name, self.lib_file))

    def load_lib(self):
        self._lib = npct.load_library(self.lib_file, '.')
        self._function = self._lib.particle_loop

    def execute_jit(self, pset, endtime, dt):
        """Invokes JIT engine to perform the core update loop"""
        if len(pset.particles) > 0:
            assert pset.fieldset.gridset.size == len(pset.particles[0].xi), \
                'FieldSet has different amount of grids than Particle.xi. Have you added Fields after creating the ParticleSet?'
        for g in pset.fieldset.gridset.grids:
            g.cstruct = None  # This force to point newly the grids from Python to C
        # Make a copy of the transposed array to enforce
        # C-contiguous memory layout for JIT mode.
        for f in self.field_args.values():
            if not f.data.flags.c_contiguous:
                f.data = f.data.copy()
        for g in pset.fieldset.gridset.grids:
            if not g.depth.flags.c_contiguous:
                g.depth = g.depth.copy()
            if not g.lon.flags.c_contiguous:
                g.lon = g.lon.copy()
            if not g.lat.flags.c_contiguous:
                g.lat = g.lat.copy()
        fargs = [byref(f.ctypes_struct) for f in self.field_args.values()]
        fargs += [c_float(f) for f in self.const_args.values()]
        particle_data = pset._particle_data.ctypes.data_as(c_void_p)
        self._function(c_int(len(pset)), particle_data,
                       c_double(endtime), c_float(dt), *fargs)

    def execute_python(self, pset, endtime, dt):
        """Performs the core update loop via Python"""
        sign_dt = np.sign(dt)

        # back up variables in case of ErrorCode.Repeat
        p_var_back = {}

        for p in pset.particles:
            ptype = p.getPType()
            # Don't execute particles that aren't started yet
            sign_end_part = np.sign(endtime - p.time)
            if (sign_end_part != sign_dt) and (dt != 0):
                continue

            # Compute min/max dt for first timestep
            dt_pos = min(abs(p.dt), abs(endtime - p.time))
            while dt_pos > 1e-6 or dt == 0:
                for var in ptype.variables:
                    p_var_back[var.name] = getattr(p, var.name)
                try:
                    p.dt = sign_dt * dt_pos
                    res = self.pyfunc(p, pset.fieldset, p.time)
                except FieldSamplingError as fse:
                    res = ErrorCode.ErrorOutOfBounds
                    p.exception = fse
                except Exception as e:
                    res = ErrorCode.Error
                    p.exception = e

                # Update particle state for explicit returns
                if res is not None:
                    p.state = res

                # Handle particle time and time loop
                if res is None or res == ErrorCode.Success:
                    # Update time and repeat
                    p.time += sign_dt * dt_pos
                    dt_pos = min(abs(p.dt), abs(endtime - p.time))
                    if dt == 0:
                        break
                    continue
                elif res == ErrorCode.Repeat:
                    # Try again without time update
                    for var in ptype.variables:
                        if var.name not in ['dt', 'state']:
                            setattr(p, var.name, p_var_back[var.name])
                    dt_pos = min(abs(p.dt), abs(endtime - p.time))
                    break
                else:
                    break  # Failure - stop time loop

    def execute(self, pset, endtime, dt, recovery=None, output_file=None):
        """Execute this Kernel over a ParticleSet for several timesteps"""

        def remove_deleted(pset):
            """Utility to remove all particles that signalled deletion"""
            indices = [i for i, p in enumerate(pset.particles)
                       if p.state in [ErrorCode.Delete]]
            if len(indices) > 0 and output_file is not None:
                output_file.write(pset[indices], endtime, deleted_only=True)
            pset.remove(indices)

        if recovery is None:
            recovery = {}
        recovery_map = recovery_base_map.copy()
        recovery_map.update(recovery)

        # Execute the kernel over the particle set
        if self.ptype.uses_jit:
            self.execute_jit(pset, endtime, dt)
        else:
            self.execute_python(pset, endtime, dt)

        # Remove all particles that signalled deletion
        remove_deleted(pset)

        # Identify particles that threw errors
        error_particles = [p for p in pset.particles
                           if p.state != ErrorCode.Success]
        while len(error_particles) > 0:
            # Apply recovery kernel
            for p in error_particles:
                if p.state == ErrorCode.Repeat:
                    p.state = ErrorCode.Success
                else:
                    recovery_kernel = recovery_map[p.state]
                    p.state = ErrorCode.Success
                    recovery_kernel(p, self.fieldset, p.time)

            # Remove all particles that signalled deletion
            remove_deleted(pset)

            # Execute core loop again to continue interrupted particles
            if self.ptype.uses_jit:
                self.execute_jit(pset, endtime, dt)
            else:
                self.execute_python(pset, endtime, dt)

            error_particles = [p for p in pset.particles
                               if p.state != ErrorCode.Success]

    def merge(self, kernel):
        funcname = self.funcname + kernel.funcname
        func_ast = FunctionDef(name=funcname, args=self.py_ast.args,
                               body=self.py_ast.body + kernel.py_ast.body,
                               decorator_list=[], lineno=1, col_offset=0)
        return Kernel(self.fieldset, self.ptype, pyfunc=None,
                      funcname=funcname, funccode=self.funccode + kernel.funccode,
                      py_ast=func_ast, funcvars=self.funcvars + kernel.funcvars)

    def __add__(self, kernel):
        if not isinstance(kernel, Kernel):
            kernel = Kernel(self.fieldset, self.ptype, pyfunc=kernel)
        return self.merge(kernel)

    def __radd__(self, kernel):
        if not isinstance(kernel, Kernel):
            kernel = Kernel(self.fieldset, self.ptype, pyfunc=kernel)
        return kernel.merge(self)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import time
import base64

from tools.translate import _
import tools
from decimal import Decimal
from osv import fields, osv

class vat_listing_clients(osv.osv_memory):
    _name = 'vat.listing.clients'
    _columns = {
        'name': fields.char('Client Name', size=64),
        'vat': fields.char('VAT', size=64),
        'country': fields.char('Country', size=64),
        'amount': fields.float('Amount'),
        'turnover': fields.float('Turnover'),
            }
    def name_get(self, cr, uid, ids, context=None):
        if not len(ids):
            return []
        return [(r['id'], r['name'] or '' + ' - ' + r['vat'] or '') \
                for r in self.read(cr, uid, ids, ['name', 'vat'],
                    context, load='_classic_write')]

    def name_search(self, cr, uid, name, args=None, operator='ilike', context=None, limit=100):
        client = self.search(cr, uid, [('vat', '=', name)]+args, limit=limit, context=context)
        if not client:
            client = self.search(cr, uid, [('name', 'ilike', '%%%s%%' % name)]+args, limit=limit, context=context)
        return self.name_get(cr, uid, client, context=context)

vat_listing_clients()

class partner_vat(osv.osv_memory):
    """ Vat Listing """
    _name = "partner.vat"

    def get_partner(self, cursor, user, ids, context=None):
        obj_period = self.pool.get('account.period')
        obj_partner = self.pool.get('res.partner')
        obj_vat_lclient = self.pool.get('vat.listing.clients')
        obj_model_data = self.pool.get('ir.model.data')
        data  = self.read(cursor, user, ids)[0]
        period = obj_period.search(cursor, user, [('fiscalyear_id', '=', data['fyear'][0])], context=context)
        p_id_list = obj_partner.search(cursor, user, [('vat_subjected', '!=', False),('customer','=',True)], context=context)
        if not p_id_list:
             raise osv.except_osv(_('Data Insufficient!'), _('No partner has a VAT Number asociated with him.'))
        partners = []
        records = []
        for obj_partner in obj_partner.browse(cursor, user, p_id_list, context=context):
            record = {} # this holds record per partner

            #This listing is only for customers located in belgium, that's the
            #reason why we skip all the partners that haven't their
            #(or one of their) default address(es) located in Belgium.
            go_ahead = False
            for ads in obj_partner.address:
                if ads.type == 'default' and (ads.country_id and ads.country_id.code == 'BE') and (obj_partner.vat or '').startswith('BE'):
                    go_ahead = True
                    break
            if not go_ahead:
                continue
            cursor.execute('select b.code, sum(credit)-sum(debit) from account_move_line l left join account_account a on (l.account_id=a.id) left join account_account_type b on (a.user_type=b.id) where b.code IN %s and l.partner_id=%s and l.period_id IN %s group by b.code',(('produit', 'tax', 'income'),obj_partner.id,tuple(period),))
            line_info = cursor.fetchall()
            if not line_info:
                continue

            record['vat'] = obj_partner.vat

            #it seems that this listing is only for belgian customers
            record['country'] = 'BE'

            record['amount'] = Decimal(str(0.0))
            record['turnover'] = Decimal(str(0.0))
            record['name'] = obj_partner.name
            for item in line_info:
                if item[0] in ('produit','income'):
                    record['turnover'] += Decimal(str(item[1]))
                else:
                    record['amount'] += Decimal(str(item[1]))
            id_client = obj_vat_lclient.create(cursor, user, record, context=context)
            partners.append(id_client)
            records.append(record)
        context.update({'partner_ids': partners, 'fyear': data['fyear'], 'limit_amount': data['limit_amount'], 'mand_id':data['mand_id']})
        model_data_ids = obj_model_data.search(cursor, user, [('model','=','ir.ui.view'), ('name','=','view_vat_listing')])
        resource_id = obj_model_data.read(cursor, user, model_data_ids, fields=['res_id'])[0]['res_id']
        return {
            'name': 'Vat Listing',
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'partner.vat.list',
            'views': [(resource_id,'form')],
            'context': context,
            'type': 'ir.actions.act_window',
            'target': 'new',
            }

    _columns = {
        'fyear': fields.many2one('account.fiscalyear','Fiscal Year', required=True),
        'mand_id': fields.char('MandataireId', size=14, required=True,  help="This identifies the representative of the sending company. This is a string of 14 characters"),
        'limit_amount': fields.integer('Limit Amount', required=True),
        'test_xml': fields.boolean('Test XML file', help="Sets the XML output as test file"),
        }
partner_vat()

class partner_vat_list(osv.osv_memory):

    """ Partner Vat Listing """
    _name = "partner.vat.list"
    _columns = {
        'partner_ids': fields.many2many('vat.listing.clients', 'vat_partner_rel', 'vat_id', 'partner_id', 'Clients', required=False, help='You can remove clients/partners which you do not want to show in xml file'),
        'name': fields.char('File Name', size=32),
        'msg': fields.text('File created', size=64, readonly=True),
        'file_save' : fields.binary('Save File', readonly=True),
        }

    def _get_partners(self, cursor, user, context=None):
        return context.get('partner_ids', [])

    _defaults={
        'partner_ids': _get_partners
            }

    def create_xml(self, cursor, user, ids, context=None):
        datas = []
        obj_sequence = self.pool.get('ir.sequence')
        obj_users = self.pool.get('res.users')
        obj_partner = self.pool.get('res.partner')
        obj_fyear = self.pool.get('account.fiscalyear')
        obj_addr = self.pool.get('res.partner.address')
        obj_vat_lclient = self.pool.get('vat.listing.clients')

        seq_controlref = obj_sequence.get(cursor, user, 'controlref')
        seq_declarantnum = obj_sequence.get(cursor, user, 'declarantnum')
        obj_cmpny = obj_users.browse(cursor, user, user, context=context).company_id
        company_vat = obj_cmpny.partner_id.vat
        if not company_vat:
            raise osv.except_osv(_('Data Insufficient'),_('No VAT Number Associated with Main Company!'))

        cref = company_vat + seq_controlref
        dnum = cref[2:] + (seq_declarantnum or '')
        obj_year= obj_fyear.browse(cursor, user, context['fyear'][0], context=context)
        street = zip_city = country = ''
        addr = obj_partner.address_get(cursor, user, [obj_cmpny.partner_id.id], ['invoice'])
        if addr.get('invoice',False):
            ads = obj_addr.browse(cursor, user, [addr['invoice']], context=context)[0]

            zip_city = obj_addr.get_city(cursor, user, ads.id)
            if not zip_city:
                zip_city = ''
            if ads.street:
                street = ads.street
            if ads.street2:
                street += ads.street2
            if ads.country_id:
                country = ads.country_id.code

        sender_date = time.strftime('%Y-%m-%d')
#        comp_name = obj_cmpny.name
        data_file = '<?xml version="1.0"?>\n<VatList xmlns="http://www.minfin.fgov.be/VatList" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.minfin.fgov.be/VatList VatList.xml" RecipientId="VAT-ADMIN" SenderId="'+ str(company_vat) + '"'
        data_file +=' ControlRef="'+ cref[2:] + '" MandataireId="'+ tools.ustr(context['mand_id']) + '" SenderDate="'+ str(sender_date)+ '"'
        if ['test_xml']:
            data_file += ' Test="0"'
        data_file += ' VersionTech="1.3">'
        data_file += '\n<AgentRepr DecNumber="1">\n\t<CompanyInfo>\n\t\t<VATNum>'+str(company_vat)+'</VATNum>\n\t\t<Name>'+ tools.ustr(obj_cmpny.name) +'</Name>\n\t\t<Street>'+ tools.ustr(street) +'</Street>\n\t\t<CityAndZipCode>'+ tools.ustr(zip_city) +'</CityAndZipCode>'
        data_file += '\n\t\t<Country>'+ tools.ustr(country) +'</Country>\n\t</CompanyInfo>\n</AgentRepr>'
        data_comp = '\n<CompanyInfo>\n\t<VATNum>'+str(company_vat)+'</VATNum>\n\t<Name>'+ tools.ustr(obj_cmpny.name) +'</Name>\n\t<Street>'+ tools.ustr(street) +'</Street>\n\t<CityAndZipCode>'+ tools.ustr(zip_city) +'</CityAndZipCode>\n\t<Country>'+ tools.ustr(country) +'</Country>\n</CompanyInfo>'
        data_period = '\n<Period>'+ tools.ustr(obj_year.date_stop[:4]) +'</Period>'
        error_message = []
        data = self.read(cursor, user, ids)[0]
        for partner in data['partner_ids']:
            if isinstance(partner, list) and partner:
                datas.append(partner[2])
            else:
                client_data = obj_vat_lclient.read(cursor, user, partner, context=context)
                datas.append(client_data)
        seq = 0
        data_clientinfo = ''
        sum_tax = Decimal(str(0.00))
        sum_turnover=Decimal(str(0.00))
        if len(error_message):
            return 'Exception : \n' +'-'*50+'\n'+ '\n'.join(error_message)
        for line in datas:
            if not line:
                continue
            if Decimal(str(line['turnover'])) < Decimal(str(context['limit_amount'])):
                continue
            seq += 1
            sum_tax +=Decimal(str(line['amount']))
            sum_turnover +=Decimal(str(line['turnover']))
            data_clientinfo += '\n<ClientList SequenceNum="'+str(seq)+'">\n\t<CompanyInfo>\n\t\t<VATNum>'+ (line['vat'] or '')[2:] +'</VATNum>\n\t\t<Country>' + tools.ustr(line['country']) +'</Country>\n\t</CompanyInfo>\n\t<Amount>'+ str(int(Decimal(str(line['amount'] * 100)))) +'</Amount>\n\t<TurnOver>'+ str(int(Decimal(str(line['turnover'] * 100)))) +'</TurnOver>\n</ClientList>'

        data_decl ='\n<DeclarantList SequenceNum="1" DeclarantNum="'+ dnum + '" ClientNbr="'+ str(seq) +'" TurnOverSum="'+ str(int(Decimal(str(sum_turnover * 100)))) +'" TaxSum="'+ str(int(Decimal(str(sum_tax * 100)))) +'" />'
        data_file += tools.ustr(data_decl) + tools.ustr(data_comp) + tools.ustr(data_period) + tools.ustr(data_clientinfo) + '\n</DeclarantList></VatList>'
        msg = 'Save the File with '".xml"' extension.'
        file_save = base64.encodestring(data_file.encode('utf8'))
        self.write(cursor, user, ids, {'file_save':file_save, 'msg':msg, 'name':'vat_list.xml'}, context=context)
        return True

partner_vat_list()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[FIX] l10n_be, vat listing: tag mismatch on DeclarantList<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import time
import base64

from tools.translate import _
import tools
from decimal import Decimal
from osv import fields, osv

class vat_listing_clients(osv.osv_memory):
    _name = 'vat.listing.clients'
    _columns = {
        'name': fields.char('Client Name', size=64),
        'vat': fields.char('VAT', size=64),
        'country': fields.char('Country', size=64),
        'amount': fields.float('Amount'),
        'turnover': fields.float('Turnover'),
            }
    def name_get(self, cr, uid, ids, context=None):
        if not len(ids):
            return []
        return [(r['id'], r['name'] or '' + ' - ' + r['vat'] or '') \
                for r in self.read(cr, uid, ids, ['name', 'vat'],
                    context, load='_classic_write')]

    def name_search(self, cr, uid, name, args=None, operator='ilike', context=None, limit=100):
        client = self.search(cr, uid, [('vat', '=', name)]+args, limit=limit, context=context)
        if not client:
            client = self.search(cr, uid, [('name', 'ilike', '%%%s%%' % name)]+args, limit=limit, context=context)
        return self.name_get(cr, uid, client, context=context)

vat_listing_clients()

class partner_vat(osv.osv_memory):
    """ Vat Listing """
    _name = "partner.vat"

    def get_partner(self, cursor, user, ids, context=None):
        obj_period = self.pool.get('account.period')
        obj_partner = self.pool.get('res.partner')
        obj_vat_lclient = self.pool.get('vat.listing.clients')
        obj_model_data = self.pool.get('ir.model.data')
        data  = self.read(cursor, user, ids)[0]
        period = obj_period.search(cursor, user, [('fiscalyear_id', '=', data['fyear'][0])], context=context)
        p_id_list = obj_partner.search(cursor, user, [('vat_subjected', '!=', False),('customer','=',True)], context=context)
        if not p_id_list:
             raise osv.except_osv(_('Data Insufficient!'), _('No partner has a VAT Number asociated with him.'))
        partners = []
        records = []
        for obj_partner in obj_partner.browse(cursor, user, p_id_list, context=context):
            record = {} # this holds record per partner

            #This listing is only for customers located in belgium, that's the
            #reason why we skip all the partners that haven't their
            #(or one of their) default address(es) located in Belgium.
            go_ahead = False
            for ads in obj_partner.address:
                if ads.type == 'default' and (ads.country_id and ads.country_id.code == 'BE') and (obj_partner.vat or '').startswith('BE'):
                    go_ahead = True
                    break
            if not go_ahead:
                continue
            cursor.execute('select b.code, sum(credit)-sum(debit) from account_move_line l left join account_account a on (l.account_id=a.id) left join account_account_type b on (a.user_type=b.id) where b.code IN %s and l.partner_id=%s and l.period_id IN %s group by b.code',(('produit', 'tax', 'income'),obj_partner.id,tuple(period),))
            line_info = cursor.fetchall()
            if not line_info:
                continue

            record['vat'] = obj_partner.vat

            #it seems that this listing is only for belgian customers
            record['country'] = 'BE'

            record['amount'] = Decimal(str(0.0))
            record['turnover'] = Decimal(str(0.0))
            record['name'] = obj_partner.name
            for item in line_info:
                if item[0] in ('produit','income'):
                    record['turnover'] += Decimal(str(item[1]))
                else:
                    record['amount'] += Decimal(str(item[1]))
            id_client = obj_vat_lclient.create(cursor, user, record, context=context)
            partners.append(id_client)
            records.append(record)
        context.update({'partner_ids': partners, 'fyear': data['fyear'], 'limit_amount': data['limit_amount'], 'mand_id':data['mand_id']})
        model_data_ids = obj_model_data.search(cursor, user, [('model','=','ir.ui.view'), ('name','=','view_vat_listing')])
        resource_id = obj_model_data.read(cursor, user, model_data_ids, fields=['res_id'])[0]['res_id']
        return {
            'name': 'Vat Listing',
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'partner.vat.list',
            'views': [(resource_id,'form')],
            'context': context,
            'type': 'ir.actions.act_window',
            'target': 'new',
            }

    _columns = {
        'fyear': fields.many2one('account.fiscalyear','Fiscal Year', required=True),
        'mand_id': fields.char('MandataireId', size=14, required=True,  help="This identifies the representative of the sending company. This is a string of 14 characters"),
        'limit_amount': fields.integer('Limit Amount', required=True),
        'test_xml': fields.boolean('Test XML file', help="Sets the XML output as test file"),
        }
partner_vat()

class partner_vat_list(osv.osv_memory):

    """ Partner Vat Listing """
    _name = "partner.vat.list"
    _columns = {
        'partner_ids': fields.many2many('vat.listing.clients', 'vat_partner_rel', 'vat_id', 'partner_id', 'Clients', required=False, help='You can remove clients/partners which you do not want to show in xml file'),
        'name': fields.char('File Name', size=32),
        'msg': fields.text('File created', size=64, readonly=True),
        'file_save' : fields.binary('Save File', readonly=True),
        }

    def _get_partners(self, cursor, user, context=None):
        return context.get('partner_ids', [])

    _defaults={
        'partner_ids': _get_partners
            }

    def create_xml(self, cursor, user, ids, context=None):
        datas = []
        obj_sequence = self.pool.get('ir.sequence')
        obj_users = self.pool.get('res.users')
        obj_partner = self.pool.get('res.partner')
        obj_fyear = self.pool.get('account.fiscalyear')
        obj_addr = self.pool.get('res.partner.address')
        obj_vat_lclient = self.pool.get('vat.listing.clients')

        seq_controlref = obj_sequence.get(cursor, user, 'controlref')
        seq_declarantnum = obj_sequence.get(cursor, user, 'declarantnum')
        obj_cmpny = obj_users.browse(cursor, user, user, context=context).company_id
        company_vat = obj_cmpny.partner_id.vat
        if not company_vat:
            raise osv.except_osv(_('Data Insufficient'),_('No VAT Number Associated with Main Company!'))

        cref = company_vat + seq_controlref
        dnum = cref[2:] + (seq_declarantnum or '')
        obj_year= obj_fyear.browse(cursor, user, context['fyear'][0], context=context)
        street = zip_city = country = ''
        addr = obj_partner.address_get(cursor, user, [obj_cmpny.partner_id.id], ['invoice'])
        if addr.get('invoice',False):
            ads = obj_addr.browse(cursor, user, [addr['invoice']], context=context)[0]

            zip_city = obj_addr.get_city(cursor, user, ads.id)
            if not zip_city:
                zip_city = ''
            if ads.street:
                street = ads.street
            if ads.street2:
                street += ads.street2
            if ads.country_id:
                country = ads.country_id.code

        sender_date = time.strftime('%Y-%m-%d')
#        comp_name = obj_cmpny.name
        data_file = '<?xml version="1.0"?>\n<VatList xmlns="http://www.minfin.fgov.be/VatList" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.minfin.fgov.be/VatList VatList.xml" RecipientId="VAT-ADMIN" SenderId="'+ str(company_vat) + '"'
        data_file +=' ControlRef="'+ cref[2:] + '" MandataireId="'+ tools.ustr(context['mand_id']) + '" SenderDate="'+ str(sender_date)+ '"'
        if ['test_xml']:
            data_file += ' Test="0"'
        data_file += ' VersionTech="1.3">'
        data_file += '\n<AgentRepr DecNumber="1">\n\t<CompanyInfo>\n\t\t<VATNum>'+str(company_vat)+'</VATNum>\n\t\t<Name>'+ tools.ustr(obj_cmpny.name) +'</Name>\n\t\t<Street>'+ tools.ustr(street) +'</Street>\n\t\t<CityAndZipCode>'+ tools.ustr(zip_city) +'</CityAndZipCode>'
        data_file += '\n\t\t<Country>'+ tools.ustr(country) +'</Country>\n\t</CompanyInfo>\n</AgentRepr>'
        data_comp = '\n<CompanyInfo>\n\t<VATNum>'+str(company_vat)+'</VATNum>\n\t<Name>'+ tools.ustr(obj_cmpny.name) +'</Name>\n\t<Street>'+ tools.ustr(street) +'</Street>\n\t<CityAndZipCode>'+ tools.ustr(zip_city) +'</CityAndZipCode>\n\t<Country>'+ tools.ustr(country) +'</Country>\n</CompanyInfo>'
        data_period = '\n<Period>'+ tools.ustr(obj_year.date_stop[:4]) +'</Period>'
        error_message = []
        data = self.read(cursor, user, ids)[0]
        for partner in data['partner_ids']:
            if isinstance(partner, list) and partner:
                datas.append(partner[2])
            else:
                client_data = obj_vat_lclient.read(cursor, user, partner, context=context)
                datas.append(client_data)
        seq = 0
        data_clientinfo = ''
        sum_tax = Decimal(str(0.00))
        sum_turnover=Decimal(str(0.00))
        if len(error_message):
            return 'Exception : \n' +'-'*50+'\n'+ '\n'.join(error_message)
        for line in datas:
            if not line:
                continue
            if Decimal(str(line['turnover'])) < Decimal(str(context['limit_amount'])):
                continue
            seq += 1
            sum_tax +=Decimal(str(line['amount']))
            sum_turnover +=Decimal(str(line['turnover']))
            data_clientinfo += '\n<ClientList SequenceNum="'+str(seq)+'">\n\t<CompanyInfo>\n\t\t<VATNum>'+ (line['vat'] or '')[2:] +'</VATNum>\n\t\t<Country>' + tools.ustr(line['country']) +'</Country>\n\t</CompanyInfo>\n\t<Amount>'+ str(int(Decimal(str(line['amount'] * 100)))) +'</Amount>\n\t<TurnOver>'+ str(int(Decimal(str(line['turnover'] * 100)))) +'</TurnOver>\n</ClientList>'

        data_decl ='\n<DeclarantList SequenceNum="1" DeclarantNum="'+ dnum + '" ClientNbr="'+ str(seq) +'" TurnOverSum="'+ str(int(Decimal(str(sum_turnover * 100)))) +'" TaxSum="'+ str(int(Decimal(str(sum_tax * 100)))) +'">'
        data_file += tools.ustr(data_decl) + tools.ustr(data_comp) + tools.ustr(data_period) + tools.ustr(data_clientinfo) + '\n</DeclarantList></VatList>'
        msg = 'Save the File with '".xml"' extension.'
        file_save = base64.encodestring(data_file.encode('utf8'))
        self.write(cursor, user, ids, {'file_save':file_save, 'msg':msg, 'name':'vat_list.xml'}, context=context)
        return True

partner_vat_list()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>"""Base settings available to all environments."""

# Based on https://github.com/rdegges/django-skel and
# https://github.com/cyberdelia/django-heroku-template.

import os
import sys
from datetime import timedelta

from django.contrib.messages import constants as messages
from djcelery import setup_loader

## Path configuration
# Absolute path to our site directory.
# NOTE: Use dirname() twice as the settings file is two levels deep.
SITE_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Create an absolute path from a path relative to the site root.
site_path = lambda path: os.path.normpath(os.path.join(SITE_ROOT, path))

# Name of the site.
SITE_NAME = os.path.basename(SITE_ROOT)

# Absolute path to our project directory.
PROJECT_ROOT = os.path.dirname(SITE_ROOT)

# Create an absolute path from a path relative to the project root.
project_path = lambda path: os.path.normpath(os.path.join(PROJECT_ROOT, path))

# Add our site root to the pythonpath so that we can refer to project
# modules without having to use the project's name.
sys.path.append(SITE_ROOT)

## Debug configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#debug
DEBUG = False

# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-debug
TEMPLATE_DEBUG = DEBUG

## Admin and manager configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#admins
ADMINS = (
    ('Marcus R. Brown', 'mrbrown@precision-mojo.com'),
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#managers
MANAGERS = ADMINS

## Authentication backends configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#authentication-backends
AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
    'account.auth_backends.EmailAuthenticationBackend',
)

## Timezone and localization configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#time-zone
TIME_ZONE = 'America/Phoenix'

# See: https://docs.djangoproject.com/en/dev/ref/settings/#language-code
LANGUAGE_CODE = 'en-us'

# See: https://docs.djangoproject.com/en/dev/ref/settings/#site-id
SITE_ID = 1

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-i18n
USE_I18N = True

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-l10n
USE_L10N = True

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-tz
USE_TZ = True

## Media configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
MEDIA_ROOT = site_path('media/')

# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
MEDIA_URL = '/media/'

## Static files configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-root
# NOTE: We place this path at the our *site* root, not the project root. This
# allows us to run collectstatic during development without worrying about
# overwriting static files in app-relative static/ directories.
STATIC_ROOT = project_path('static_collected/')

# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
STATIC_URL = '/static/'

# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-dirs
STATICFILES_DIRS = ()

# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-finders
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)

# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-storage
STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'

## Secret key configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
SECRET_KEY = '4=py*2g1gl=cgjqcb4v*gr^&5nxt2@d%b+!#l(2eeb#sa_0yxq'

## Template configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-loaders
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-dirs
TEMPLATE_DIRS = (
    site_path('templates/')
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-context-processors
# NOTE: All except the last are defaults.
TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.static',
    'django.core.context_processors.tz',
    'django.contrib.messages.context_processors.messages',
    'django.core.context_processors.request',
    'account.context_processors.account',
    '%s.context_processors.analytical_domain' % SITE_NAME,
)

## Middleware configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#middleware-classes
MIDDLEWARE_CLASSES = (
    # TODO: Disable GZIP middleware until we have a clean way of ensuring that
    # it's first in the list.
    #'django.middleware.gzip.GZipMiddleware',
    'htmlmin.middleware.HtmlMinifyMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'account.middleware.LocaleMiddleware',
    'account.middleware.TimezoneMiddleware',
)

## URL configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#root-urlconf
ROOT_URLCONF = '%s.urls' % SITE_NAME

## WSGI configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
WSGI_APPLICATION = '%s.wsgi.application' % SITE_NAME

## Cache configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#caches
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
    },
    'staticfiles': {
        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
        'LOCATION': os.path.join(PROJECT_ROOT, 'static_cache'),
        'TIMEOUT': 100 * 365 * 24 * 60 * 60,
        'OPTIONS': {
            'MAX_ENTRIES': 100 * 1000
        },
    },
}

## Installed app configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    # This app contains templates for django.contrib.admin, so it must precede
    # it.
    # django-bootstrap-admin-templates
    'bootstrap_admin_templates',

    # See: https://docs.djangoproject.com/en/dev/ref/contrib/admin/
    'django.contrib.admin',
    # https://docs.djangoproject.com/en/dev/ref/contrib/admin/admindocs/
    'django.contrib.admindocs',

    # See: https://docs.djangoproject.com/en/dev/ref/contrib/humanize/
    'django.contrib.humanize',

    # Third-party apps
    # django-pipeline
    'pipeline',
    # django-analytical,
    'analytical',
    # South
    'south',
    # django-robots
    'robots',
    # django-celery
    'djcelery',
    # django-social-auth
    'social_auth',
    # django-bootstrap-toolkit
    'bootstrap_toolkit',

    # Project libraries
    '%s' % SITE_NAME,
    'libs.bootstrap',
)

## Logging configuration
# NOTE: Don't filter HTTP 500 error emails in not DEBUG environments (the
# default configuration does this).
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

## Messages configuration
# See: https://docs.djangoproject.com/en/dev/ref/contrib/messages/#message-tags
# Setup tags to use Twitter Bootstrap alert styles (from
# https://github.com/estebistec/bootstrap-pipelined)
MESSAGE_TAGS = {
    messages.DEBUG: '',
    messages.INFO: 'alert-info',
    messages.SUCCESS: 'alert-success',
    messages.WARNING: '',
    messages.ERROR: 'alert-error',
}

## django-pipeline configuration
# See: http://django-pipeline.readthedocs.org/en/latest/configuration.html
# TODO: This uses a workaround for PipelineFinderStorage for running on
# Windows.
PIPELINE_STORAGE = '%s.settings.pipeline_helpers.PipelineFinderStorageCorrected' % SITE_NAME

# See: http://django-pipeline.readthedocs.org/en/latest/configuration.html#specifying-files
PIPELINE_CSS = {
    'standard': {
        'source_filenames': (
            # TODO: Use the checked-in .css until django-pipeline places
            # compiled output into the STATIC_ROOT directory.
            #'less/bootstrap.less',
            'bootstrap/css/bootstrap.css',
        ),
        'output_filename': 'css/s.min.css',
    },
    'responsive': {
        'source_filenames': (
            # TODO: Use the checked-in .css until django-pipeline places
            # compiled output into the STATIC_ROOT directory.
            #'less/responsive.less',
            'bootstrap/css/bootstrap-responsive.css',
        ),
        'output_filename': 'css/r.min.css',
    },
}

PIPELINE_JS = {
    'standard': {
        'source_filenames': (
            'bootstrap/js/bootstrap-affix.js',
            'bootstrap/js/bootstrap-alert.js',
            'bootstrap/js/bootstrap-button.js',
            'bootstrap/js/bootstrap-carousel.js',
            'bootstrap/js/bootstrap-collapse.js',
            'bootstrap/js/bootstrap-dropdown.js',
            'bootstrap/js/bootstrap-modal.js',
            'bootstrap/js/bootstrap-scrollspy.js',
            'bootstrap/js/bootstrap-tab.js',
            'bootstrap/js/bootstrap-tooltip.js',
            'bootstrap/js/bootstrap-popover.js',
            'bootstrap/js/bootstrap-transition.js',
            'bootstrap/js/bootstrap-typeahead.js',
        ),
        'output_filename': 'js/s.min.js',
    }
}

# See: http://django-pipeline.readthedocs.org/en/latest/configuration.html#pipeline-css-compressor
PIPELINE_CSS_COMPRESSOR = 'pipeline.compressors.yui.YUICompressor'

# See: http://django-pipeline.readthedocs.org/en/latest/configuration.html#pipeline-js-compressor
PIPELINE_JS_COMPRESSOR = 'pipeline.compressors.yui.YUICompressor'

# See: http://django-pipeline.readthedocs.org/en/latest/compilers.html#compilers
PIPELINE_COMPILERS = (
    'pipeline.compilers.less.LessCompiler',
)
# TODO: Strip off the /usr/bin/env portion of the value provided by
# django-pipeline if env isn't on the path.
#PIPELINE_LESS_BINARY = 'lessc'
PIPELINE_YUI_BINARY = 'yuicompressor'

## django-htmlmin configuration
# See: http://pypi.python.org/pypi/django-htmlmin
# TODO: Disable HTML minifying because it breaks the IE-specific code at the
# top of base.html.
HTML_MINIFY = False
KEEP_COMMENTS_ON_MINIFYING = True

## django-analytical configuration
# See: http://packages.python.org/django-analytical/services/google_analytics.html
GOOGLE_ANALYTICS_PROPERTY_ID = os.environ.get('GOOGLE_ANALYTICS_TRACKING_ID', '')
GOOGLE_ANALYTICS_TRACKING_STYLE = 3  # google_analytics.TRACK_MULTIPLE_DOMAINS'

# See: http://packages.python.org/django-analytical/services/gauges.html
GAUGES_SITE_ID = os.environ.get('GAUGES_SITE_ID', '')

# See: http://packages.python.org/django-analytical/services/mixpanel.html
MIXPANEL_API_TOKEN = os.environ.get('MIXPANEL_API_TOKEN', '')

## django-celery / celery configuration
# See: http://celery.readthedocs.org/en/latest/configuration.html#celery-task-result-expires
CELERY_TASK_RESULT_EXPIRES = timedelta(minutes=30)

# See: http://docs.celeryproject.org/en/latest/django/first-steps-with-django.html
setup_loader()

## django-user-accounts configuration
# See: http://django-user-accounts.readthedocs.org/en/latest/usage.html
ACCOUNT_EMAIL_UNIQUE = True
ACCOUNT_EMAIL_CONFIRMATION_REQUIRED = True
<commit_msg>Remove additional django-user-accounts configuration.<commit_after>"""Base settings available to all environments."""

# Based on https://github.com/rdegges/django-skel and
# https://github.com/cyberdelia/django-heroku-template.

import os
import sys
from datetime import timedelta

from django.contrib.messages import constants as messages
from djcelery import setup_loader

## Path configuration
# Absolute path to our site directory.
# NOTE: Use dirname() twice as the settings file is two levels deep.
SITE_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Create an absolute path from a path relative to the site root.
site_path = lambda path: os.path.normpath(os.path.join(SITE_ROOT, path))

# Name of the site.
SITE_NAME = os.path.basename(SITE_ROOT)

# Absolute path to our project directory.
PROJECT_ROOT = os.path.dirname(SITE_ROOT)

# Create an absolute path from a path relative to the project root.
project_path = lambda path: os.path.normpath(os.path.join(PROJECT_ROOT, path))

# Add our site root to the pythonpath so that we can refer to project
# modules without having to use the project's name.
sys.path.append(SITE_ROOT)

## Debug configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#debug
DEBUG = False

# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-debug
TEMPLATE_DEBUG = DEBUG

## Admin and manager configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#admins
ADMINS = (
    ('Marcus R. Brown', 'mrbrown@precision-mojo.com'),
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#managers
MANAGERS = ADMINS

## Authentication backends configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#authentication-backends
AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
)

## Timezone and localization configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#time-zone
TIME_ZONE = 'America/Phoenix'

# See: https://docs.djangoproject.com/en/dev/ref/settings/#language-code
LANGUAGE_CODE = 'en-us'

# See: https://docs.djangoproject.com/en/dev/ref/settings/#site-id
SITE_ID = 1

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-i18n
USE_I18N = True

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-l10n
USE_L10N = True

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-tz
USE_TZ = True

## Media configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
MEDIA_ROOT = site_path('media/')

# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
MEDIA_URL = '/media/'

## Static files configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-root
# NOTE: We place this path at the our *site* root, not the project root. This
# allows us to run collectstatic during development without worrying about
# overwriting static files in app-relative static/ directories.
STATIC_ROOT = project_path('static_collected/')

# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
STATIC_URL = '/static/'

# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-dirs
STATICFILES_DIRS = ()

# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-finders
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)

# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-storage
STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'

## Secret key configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
SECRET_KEY = '4=py*2g1gl=cgjqcb4v*gr^&5nxt2@d%b+!#l(2eeb#sa_0yxq'

## Template configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-loaders
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-dirs
TEMPLATE_DIRS = (
    site_path('templates/')
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-context-processors
# NOTE: All except the last are defaults.
TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.static',
    'django.core.context_processors.tz',
    'django.contrib.messages.context_processors.messages',
    'django.core.context_processors.request',
    '%s.context_processors.analytical_domain' % SITE_NAME,
)

## Middleware configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#middleware-classes
MIDDLEWARE_CLASSES = (
    # TODO: Disable GZIP middleware until we have a clean way of ensuring that
    # it's first in the list.
    #'django.middleware.gzip.GZipMiddleware',
    'htmlmin.middleware.HtmlMinifyMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)

## URL configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#root-urlconf
ROOT_URLCONF = '%s.urls' % SITE_NAME

## WSGI configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
WSGI_APPLICATION = '%s.wsgi.application' % SITE_NAME

## Cache configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#caches
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
    },
    'staticfiles': {
        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
        'LOCATION': os.path.join(PROJECT_ROOT, 'static_cache'),
        'TIMEOUT': 100 * 365 * 24 * 60 * 60,
        'OPTIONS': {
            'MAX_ENTRIES': 100 * 1000
        },
    },
}

## Installed app configuration
# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    # This app contains templates for django.contrib.admin, so it must precede
    # it.
    # django-bootstrap-admin-templates
    'bootstrap_admin_templates',

    # See: https://docs.djangoproject.com/en/dev/ref/contrib/admin/
    'django.contrib.admin',
    # https://docs.djangoproject.com/en/dev/ref/contrib/admin/admindocs/
    'django.contrib.admindocs',

    # See: https://docs.djangoproject.com/en/dev/ref/contrib/humanize/
    'django.contrib.humanize',

    # Third-party apps
    # django-pipeline
    'pipeline',
    # django-analytical,
    'analytical',
    # South
    'south',
    # django-robots
    'robots',
    # django-celery
    'djcelery',
    # django-social-auth
    'social_auth',
    # django-bootstrap-toolkit
    'bootstrap_toolkit',

    # Project libraries
    '%s' % SITE_NAME,
    'libs.bootstrap',
)

## Logging configuration
# NOTE: Don't filter HTTP 500 error emails in not DEBUG environments (the
# default configuration does this).
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

## Messages configuration
# See: https://docs.djangoproject.com/en/dev/ref/contrib/messages/#message-tags
# Setup tags to use Twitter Bootstrap alert styles (from
# https://github.com/estebistec/bootstrap-pipelined)
MESSAGE_TAGS = {
    messages.DEBUG: '',
    messages.INFO: 'alert-info',
    messages.SUCCESS: 'alert-success',
    messages.WARNING: '',
    messages.ERROR: 'alert-error',
}

## django-pipeline configuration
# See: http://django-pipeline.readthedocs.org/en/latest/configuration.html
# TODO: This uses a workaround for PipelineFinderStorage for running on
# Windows.
PIPELINE_STORAGE = '%s.settings.pipeline_helpers.PipelineFinderStorageCorrected' % SITE_NAME

# See: http://django-pipeline.readthedocs.org/en/latest/configuration.html#specifying-files
PIPELINE_CSS = {
    'standard': {
        'source_filenames': (
            # TODO: Use the checked-in .css until django-pipeline places
            # compiled output into the STATIC_ROOT directory.
            #'less/bootstrap.less',
            'bootstrap/css/bootstrap.css',
        ),
        'output_filename': 'css/s.min.css',
    },
    'responsive': {
        'source_filenames': (
            # TODO: Use the checked-in .css until django-pipeline places
            # compiled output into the STATIC_ROOT directory.
            #'less/responsive.less',
            'bootstrap/css/bootstrap-responsive.css',
        ),
        'output_filename': 'css/r.min.css',
    },
}

PIPELINE_JS = {
    'standard': {
        'source_filenames': (
            'bootstrap/js/bootstrap-affix.js',
            'bootstrap/js/bootstrap-alert.js',
            'bootstrap/js/bootstrap-button.js',
            'bootstrap/js/bootstrap-carousel.js',
            'bootstrap/js/bootstrap-collapse.js',
            'bootstrap/js/bootstrap-dropdown.js',
            'bootstrap/js/bootstrap-modal.js',
            'bootstrap/js/bootstrap-scrollspy.js',
            'bootstrap/js/bootstrap-tab.js',
            'bootstrap/js/bootstrap-tooltip.js',
            'bootstrap/js/bootstrap-popover.js',
            'bootstrap/js/bootstrap-transition.js',
            'bootstrap/js/bootstrap-typeahead.js',
        ),
        'output_filename': 'js/s.min.js',
    }
}

# See: http://django-pipeline.readthedocs.org/en/latest/configuration.html#pipeline-css-compressor
PIPELINE_CSS_COMPRESSOR = 'pipeline.compressors.yui.YUICompressor'

# See: http://django-pipeline.readthedocs.org/en/latest/configuration.html#pipeline-js-compressor
PIPELINE_JS_COMPRESSOR = 'pipeline.compressors.yui.YUICompressor'

# See: http://django-pipeline.readthedocs.org/en/latest/compilers.html#compilers
PIPELINE_COMPILERS = (
    'pipeline.compilers.less.LessCompiler',
)
# TODO: Strip off the /usr/bin/env portion of the value provided by
# django-pipeline if env isn't on the path.
#PIPELINE_LESS_BINARY = 'lessc'
PIPELINE_YUI_BINARY = 'yuicompressor'

## django-htmlmin configuration
# See: http://pypi.python.org/pypi/django-htmlmin
# TODO: Disable HTML minifying because it breaks the IE-specific code at the
# top of base.html.
HTML_MINIFY = False
KEEP_COMMENTS_ON_MINIFYING = True

## django-analytical configuration
# See: http://packages.python.org/django-analytical/services/google_analytics.html
GOOGLE_ANALYTICS_PROPERTY_ID = os.environ.get('GOOGLE_ANALYTICS_TRACKING_ID', '')
GOOGLE_ANALYTICS_TRACKING_STYLE = 3  # google_analytics.TRACK_MULTIPLE_DOMAINS'

# See: http://packages.python.org/django-analytical/services/gauges.html
GAUGES_SITE_ID = os.environ.get('GAUGES_SITE_ID', '')

# See: http://packages.python.org/django-analytical/services/mixpanel.html
MIXPANEL_API_TOKEN = os.environ.get('MIXPANEL_API_TOKEN', '')

## django-celery / celery configuration
# See: http://celery.readthedocs.org/en/latest/configuration.html#celery-task-result-expires
CELERY_TASK_RESULT_EXPIRES = timedelta(minutes=30)

# See: http://docs.celeryproject.org/en/latest/django/first-steps-with-django.html
setup_loader()
<|endoftext|>


<commit_before>#!/usr/bin/env python

NAME = 'Radware AppWall'

def is_waf(self):
    if self.matchheader(('X-SL-CompState', '.')):
        return True
    for attack in self.attacks:
        r = attack(self)
        if r is None:
            return
        _, responsebody = r
        # Most reliable fingerprint is this on block page
        if all(i in responsebody for i in (b'because we have detected unauthorized activity',
            b'<TITLE>Unauthorized Request Blocked</TITLE>', b'mailto:CloudWebSec@radware.com')):
            return True
    return False
<commit_msg>Reverted back some fingerprints due to false positives<commit_after>#!/usr/bin/env python

NAME = 'Radware AppWall'

def is_waf(self):
    if self.matchheader(('X-SL-CompState', '.')):
        return True
    for attack in self.attacks:
        r = attack(self)
        if r is None:
            return
        _, responsebody = r
        # Most reliable fingerprint is this on block page
        if all(i in responsebody for i in (b'because we have detected unauthorized activity',
            b'<TITLE>Unauthorized Request Blocked</TITLE>', b'If you believe that there has been some mistake',
            b'?Subject=Security Page - Case Number')):
            return True
    return False
<|endoftext|>


<commit_before>"""
Run this file to load the newest menus.
"""

from app import db
from app.models import (Servery, MealTime, Meal, MealDish,
                        Dish, AllergyFlag)
from downloadmenu import process_servery_menu

from datetime import timedelta

import sys

def load_meals(base_address=None):

    for servery in db.session.query(Servery):
        print servery.name
        if base_address is not None:
            menu = process_servery_menu(servery.name,base_address)
        else:
            menu = process_servery_menu(servery.name)

        base_date = menu['base_date']

        for meal_type in ['lunch', 'dinner']:
            for day_of_the_week in menu[meal_type]:
                actual_date = base_date + timedelta(days=day_of_the_week)

                print meal_type, day_of_the_week, servery.name
                mealtime = db.session.query(MealTime).filter(
                    MealTime.meal_type == meal_type,
                    MealTime.servery == servery,
                    MealTime.day_of_the_week == day_of_the_week).scalar()

                if mealtime is not None:

                    meal = Meal(date=actual_date, mealtime=mealtime)
                    db.session.add(meal)

                    for dish_info in menu[meal_type][day_of_the_week]:
                        create_dish_from_dish_info(dish_info, servery, meal)
    db.session.commit()

def create_dish_from_dish_info(dish_info, servery, meal):
    dish = Dish(
        dish_description=dish_info.dish_description, score=0, servery=servery)

    for flag in dish_info.allergy_flags:
        flagobj = AllergyFlag(dish=dish, allergyflag=flag)
        db.session.add(flagobj)

    mealdish = MealDish(meal=meal, dish=dish)
    db.session.add(mealdish)

import urlparse, urllib

def path2url(path):
    return urlparse.urljoin(
      'file:', urllib.pathname2url(path))

if __name__ == "__main__":
    if len(sys.argv) == 2:
        load_meals(path2url(sys.argv[1]))
    else:
        load_meals()
<commit_msg>Created filtering for bad menu items<commit_after>"""
Run this file to load the newest menus.
"""

from app import db
from app.models import (Servery, MealTime, Meal, MealDish,
                        Dish, AllergyFlag)
from downloadmenu import process_servery_menu

from datetime import timedelta

import sys

def load_meals(base_address=None):

    for servery in db.session.query(Servery):
        print servery.name
        if base_address is not None:
            menu = process_servery_menu(servery.name,base_address)
        else:
            menu = process_servery_menu(servery.name)

        base_date = menu['base_date']

        for meal_type in ['lunch', 'dinner']:
            for day_of_the_week in menu[meal_type]:
                actual_date = base_date + timedelta(days=day_of_the_week)

                print meal_type, day_of_the_week, servery.name
                mealtime = db.session.query(MealTime).filter(
                    MealTime.meal_type == meal_type,
                    MealTime.servery == servery,
                    MealTime.day_of_the_week == day_of_the_week).scalar()

                if mealtime is not None:

                    meal = Meal(date=actual_date, mealtime=mealtime)
                    db.session.add(meal)
                    no_menu_flag = True
                    dish_index = 0
                    while dish_index < len(menu[meal_type][day_of_the_week]):
                    # for dish_info in menu[meal_type][day_of_the_week]:
                        dish_info = menu[meal_type][day_of_the_week][dish_index]
                        dish_description = dish_info.dish_description
                        dish_words = dish_description.split(" ")
                        # filtering
                        # if "Brunch" in dish_words:
                        #     print dish_words
                        #     dish_words.remove("Brunch")
                        # if "Available" in dish_words:
                        #     dish_words.remove("Available")
                        # if "-" in dish_words:
                        #     dish_words.remove("-")
                        i = 0
                        while i < len(dish_words):
                            # print i
                            # print dish_words
                            word = dish_words[i]
                            if "PM" in word or "AM" in word or "Brunch" in word or "Available" in word or "-" in word:
                                del dish_words[i]
                                i-=1
                            i+=1
                        dish_info.dish_description = " ".join(dish_words)
                        # if meal_type == "lunch" and day_of_the_week == 6:
                        #     # print dish_info
                        dish_index+=1
                        if dish_info.dish_description != "":
                            no_menu_flag = False
                            create_dish_from_dish_info(dish_info, servery, meal)
                        # if no_menu_flag and dish_index == len(menu[meal_type][day_of_the_week]):
                        #     dish_info.dish_description = "There's a meal but no menu!"
                        #     create_dish_from_dish_info(dish_info, servery, meal)

    db.session.commit()

def create_dish_from_dish_info(dish_info, servery, meal):

    dish = Dish(
        dish_description=dish_info.dish_description, score=0, servery=servery)

    for flag in dish_info.allergy_flags:
        flagobj = AllergyFlag(dish=dish, allergyflag=flag)
        db.session.add(flagobj)

    mealdish = MealDish(meal=meal, dish=dish)
    db.session.add(mealdish)

import urlparse, urllib

def path2url(path):
    return urlparse.urljoin(
      'file:', urllib.pathname2url(path))

if __name__ == "__main__":
    if len(sys.argv) == 2:
        load_meals(path2url(sys.argv[1]))
    else:
        load_meals()
<|endoftext|>


<commit_before>import os, json
from igf_data.illumina.samplesheet import SampleSheet

class ProcessSingleCellSamplesheet:
  '''
  A class for processing samplesheet containing single cell (10X) index barcodes
  It requires a json format file listing all the single
  cell barcodes downloaded from this page
  https://support.10xgenomics.com/single-cell-gene-expression/sequencing/doc/
  specifications-sample-index-sets-for-single-cell-3

  required params:
  samplesheet_file: A samplesheet containing single cell samples
  singlecell_barcode_json: A JSON file listing single cell indexes
  singlecell_lebel: A text lebel for the single cell sample description
  index_column: Column name for index lookup
  sample_id_column: Column name for sample_id lookup
  sample_name_column: Column name for sample_name lookup
  '''

  def __init__(self,samplesheet_file,singlecell_barcode_json,\
               singlecell_lebel='10X',index_column='index',\
               sample_id_column='Sample_ID', sample_name_column='Sample_Name',
               sample_description_column='Description'):
    self.samplesheet_file=samplesheet_file
    self.singlecell_barcode_json=singlecell_barcode_json
    self.index_column=index_column
    self.sample_id_column=sample_id_column
    self.sample_name_column=sample_name_column
    self.sample_description_column=sample_description_column
    self.singlecell_barcodes=ProcessSingleCellSamplesheet.\
                             _get_index_data(singlecell_barcode_json)           # get single cell indexes
    self.singlecell_lebel=singlecell_lebel

  @staticmethod
  def _get_index_data(singlecell_barcode_json):
    '''
    An internal static method for reading single cell index data json files

    required param:
    singlecell_barcode_json: A JSON file containing single cell barcodes

    returns: A dictionary
    '''
    with open(singlecell_barcode_json) as json_data:
      index_json=json.load(json_data)

    index_data=dict()
    for index_line in index_json:
      index_name,index_list=index_line
      index_data[index_name]=index_list
    return index_data

  def _process_samplesheet_lines(data):
    '''
    An internal method for processing single cell indexes. Four lines of sample
    information are added to the output for each of the single cell samples

    required params:
    data: A dictionary containing data of single line of samplesheet

    returns a list of samplesheet data
    '''
    try:
      if not isinstance(data,dict):
        raise ValueError('expecting a dictionary and got {0}'.format(type(data)))
      if self.index_column in data:
        sc_index=data[self.index_column]
        if sc_index in index_data:
          final_data=list()
          suffix=0
          for index_seq in index_data[sc_index]:
            suffix +=1
            mod_data=dict(data)
            mod_data[self.index_column]=index_seq
            mod_data[self.sample_id_column]='{0}_{1}'.\
                                             format(mod_data[sample_id_column],\
                                                    suffix)
            mod_data[self.sample_name_column]='{0}_{1}'.\
                                              format(mod_data[sample_name_column],\
                                                     suffix)
            final_data.append(mod_data)
        else:
          raise ValueError('index {0} not found in file {1}'.\
                           format(sc_index,self.singlecell_barcode_json))
      else:
        raise ValueError('index column {0} not found in samplesheet data'.\
                         format(index_column))
      return final_data
    except:
      raise

  def replace_singlecell_barcodes(self,output_samplesheet):
    '''
    A method for replacing single cell index codes present in the samplesheet
    with the four index sequences. This method will create 4 samplesheet entries
    for each of the single cell samples with _1 to _4 suffix and relevant indexes

    required params:
    output_samplesheet: A file name of the output samplesheet
    '''
    try:
      samplesheet=SampleSheet(infile=self.samplesheet_file)
      samplesheet.filter_sample_data(condition_key=self.sample_description_column,
                                     condition_value=self.singlecell_lebel,
                                     method='exclude')                          # filter single cell samples

      samplesheet_sc=SampleSheet(infile=self.samplesheet_file)
      samplesheet_sc.filter_sample_data(condition_key=self.sample_description_column,
                                        condition_value=self.singlecell_lebel,
                                        method='include')
      new_samplesheet_data=list()
      if len(samplesheet_sc._data) > 0:                                         # single cell samples are present
        for data in samplesheet_sc._data:
          processed_data=self._process_samplesheet_lines(data)
          if isinstance(processed_data,list):
            new_single_cell_data.extend(processed_data)
          else:
            new_single_cell_data.append(processed_data)
        samplesheet._data.append(new_single_cell_data)                          # add modified single cell records
      samplesheet.print_sampleSheet(outfile=output_samplesheet)                 # write modified samplesheet
    except:
      raise<commit_msg>minor change of the method name<commit_after>import os, json
from igf_data.illumina.samplesheet import SampleSheet

class ProcessSingleCellSamplesheet:
  '''
  A class for processing samplesheet containing single cell (10X) index barcodes
  It requires a json format file listing all the single
  cell barcodes downloaded from this page
  https://support.10xgenomics.com/single-cell-gene-expression/sequencing/doc/
  specifications-sample-index-sets-for-single-cell-3

  required params:
  samplesheet_file: A samplesheet containing single cell samples
  singlecell_barcode_json: A JSON file listing single cell indexes
  singlecell_lebel: A text lebel for the single cell sample description
  index_column: Column name for index lookup
  sample_id_column: Column name for sample_id lookup
  sample_name_column: Column name for sample_name lookup
  '''

  def __init__(self,samplesheet_file,singlecell_barcode_json,\
               singlecell_lebel='10X',index_column='index',\
               sample_id_column='Sample_ID', sample_name_column='Sample_Name',
               sample_description_column='Description'):
    self.samplesheet_file=samplesheet_file
    self.singlecell_barcode_json=singlecell_barcode_json
    self.index_column=index_column
    self.sample_id_column=sample_id_column
    self.sample_name_column=sample_name_column
    self.sample_description_column=sample_description_column
    self.singlecell_barcodes=ProcessSingleCellSamplesheet.\
                             _get_index_data(singlecell_barcode_json)           # get single cell indexes
    self.singlecell_lebel=singlecell_lebel

  @staticmethod
  def _get_index_data(singlecell_barcode_json):
    '''
    An internal static method for reading single cell index data json files

    required param:
    singlecell_barcode_json: A JSON file containing single cell barcodes

    returns: A dictionary
    '''
    with open(singlecell_barcode_json) as json_data:
      index_json=json.load(json_data)

    index_data=dict()
    for index_line in index_json:
      index_name,index_list=index_line
      index_data[index_name]=index_list
    return index_data

  def _process_samplesheet_lines(data):
    '''
    An internal method for processing single cell indexes. Four lines of sample
    information are added to the output for each of the single cell samples

    required params:
    data: A dictionary containing data of single line of samplesheet

    returns a list of samplesheet data
    '''
    try:
      if not isinstance(data,dict):
        raise ValueError('expecting a dictionary and got {0}'.format(type(data)))
      if self.index_column in data:
        sc_index=data[self.index_column]
        if sc_index in index_data:
          final_data=list()
          suffix=0
          for index_seq in index_data[sc_index]:
            suffix +=1
            mod_data=dict(data)
            mod_data[self.index_column]=index_seq
            mod_data[self.sample_id_column]='{0}_{1}'.\
                                             format(mod_data[sample_id_column],\
                                                    suffix)
            mod_data[self.sample_name_column]='{0}_{1}'.\
                                              format(mod_data[sample_name_column],\
                                                     suffix)
            final_data.append(mod_data)
        else:
          raise ValueError('index {0} not found in file {1}'.\
                           format(sc_index,self.singlecell_barcode_json))
      else:
        raise ValueError('index column {0} not found in samplesheet data'.\
                         format(index_column))
      return final_data
    except:
      raise

  def change_singlecell_barcodes(self,output_samplesheet):
    '''
    A method for replacing single cell index codes present in the samplesheet
    with the four index sequences. This method will create 4 samplesheet entries
    for each of the single cell samples with _1 to _4 suffix and relevant indexes

    required params:
    output_samplesheet: A file name of the output samplesheet
    '''
    try:
      samplesheet=SampleSheet(infile=self.samplesheet_file)
      samplesheet.filter_sample_data(condition_key=self.sample_description_column,
                                     condition_value=self.singlecell_lebel,
                                     method='exclude')                          # filter single cell samples

      samplesheet_sc=SampleSheet(infile=self.samplesheet_file)
      samplesheet_sc.filter_sample_data(condition_key=self.sample_description_column,
                                        condition_value=self.singlecell_lebel,
                                        method='include')
      new_samplesheet_data=list()
      if len(samplesheet_sc._data) > 0:                                         # single cell samples are present
        for data in samplesheet_sc._data:
          processed_data=self._process_samplesheet_lines(data)
          if isinstance(processed_data,list):
            new_single_cell_data.extend(processed_data)
          else:
            new_single_cell_data.append(processed_data)
        samplesheet._data.append(new_single_cell_data)                          # add modified single cell records
      samplesheet.print_sampleSheet(outfile=output_samplesheet)                 # write modified samplesheet
    except:
      raise<|endoftext|>