<commit_before>import sys
if sys.version_info[0] < 3:
    from ConfigParser import SafeConfigParser, NoOptionError
else:
    from configparser import SafeConfigParser, NoOptionError
import re
import os
import shlex

__all__ = ['FormatError', 'PkgNotFound', 'LibraryInfo', 'VariableSet',
        'read_config', 'parse_flags']

_VAR = re.compile('\$\{([a-zA-Z0-9_-]+)\}')

class FormatError(IOError):
    """
    Exception thrown when there is a problem parsing a configuration file.

    """
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg

class PkgNotFound(IOError):
    """Exception raised when a package can not be located."""
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg

def parse_flags(line):
    """
    Parse a line from a config file containing compile flags.

    Parameters
    ----------
    line : str
        A single line containing one or more compile flags.

    Returns
    -------
    d : dict
        Dictionary of parsed flags, split into relevant categories.
        These categories are the keys of `d`:

        * 'include_dirs'
        * 'library_dirs'
        * 'libraries'
        * 'macros'
        * 'ignored'

    """
    lexer = shlex.shlex(line)
    lexer.whitespace_split = True

    d = {'include_dirs': [], 'library_dirs': [], 'libraries': [],
            'macros': [], 'ignored': []}
    def next_token(t):
        if t.startswith('-I'):
            if len(t) > 2:
                d['include_dirs'].append(t[2:])
            else:
                t = lexer.get_token()
                d['include_dirs'].append(t)
        elif t.startswith('-L'):
            if len(t) > 2:
                d['library_dirs'].append(t[2:])
            else:
                t = lexer.get_token()
                d['library_dirs'].append(t)
        elif t.startswith('-l'):
            d['libraries'].append(t[2:])
        elif t.startswith('-D'):
            d['macros'].append(t[2:])
        else:
            d['ignored'].append(t)
        return lexer.get_token()

    t = lexer.get_token()
    while t:
        t = next_token(t)

    return d

def _escape_backslash(val):
    return val.replace('\\', '\\\\')

class LibraryInfo(object):
    """
    Object containing build information about a library.

    Parameters
    ----------
    name : str
        The library name.
    description : str
        Description of the library.
    version : str
        Version string.
    sections : dict
        The sections of the configuration file for the library. The keys are
        the section headers, the values the text under each header.
    vars : class instance
        A `VariableSet` instance, which contains ``(name, value)`` pairs for
        variables defined in the configuration file for the library.
    requires : sequence, optional
        The required libraries for the library to be installed.

    Notes
    -----
    All input parameters (except "sections" which is a method) are available as
    attributes of the same name.

    """
    def __init__(self, name, description, version, sections, vars, requires=None):
        self.name = name
        self.description = description
        if requires:
            self.requires = requires
        else:
            self.requires = []
        self.version = version
        self._sections = sections
        self.vars = vars

    def sections(self):
        """
        Return the section headers of the config file.

        Parameters
        ----------
        None

        Returns
        -------
        keys : list of str
            The list of section headers.

        """
        return self._sections.keys()

    def cflags(self, section="default"):
        val = self.vars.interpolate(self._sections[section]['cflags'])
        return _escape_backslash(val)

    def libs(self, section="default"):
        val = self.vars.interpolate(self._sections[section]['libs'])
        return _escape_backslash(val)

    def __str__(self):
        m = ['Name: %s' % self.name]
        m.append('Description: %s' % self.description)
        if self.requires:
            m.append('Requires:')
        else:
            m.append('Requires: %s' % ",".join(self.requires))
        m.append('Version: %s' % self.version)

        return "\n".join(m)

class VariableSet(object):
    """
    Container object for the variables defined in a config file.

    `VariableSet` can be used as a plain dictionary, with the variable names
    as keys.

    Parameters
    ----------
    d : dict
        Dict of items in the "variables" section of the configuration file.

    """
    def __init__(self, d):
        self._raw_data = dict([(k, v) for k, v in d.items()])

        self._re = {}
        self._re_sub = {}

        self._init_parse()

    def _init_parse(self):
        for k, v in self._raw_data.items():
            self._init_parse_var(k, v)

    def _init_parse_var(self, name, value):
        self._re[name] = re.compile(r'\$\{%s\}' % name)
        self._re_sub[name] = value

    def interpolate(self, value):
        # Brute force: we keep interpolating until there is no '${var}' anymore
        # or until interpolated string is equal to input string
        def _interpolate(value):
            for k in self._re.keys():
                value = self._re[k].sub(self._re_sub[k], value)
            return value
        while _VAR.search(value):
            nvalue = _interpolate(value)
            if nvalue == value:
                break
            value = nvalue

        return value

    def variables(self):
        """
        Return the list of variable names.

        Parameters
        ----------
        None

        Returns
        -------
        names : list of str
            The names of all variables in the `VariableSet` instance.

        """
        return self._raw_data.keys()

    # Emulate a dict to set/get variables values
    def __getitem__(self, name):
        return self._raw_data[name]

    def __setitem__(self, name, value):
        self._raw_data[name] = value
        self._init_parse_var(name, value)

def parse_meta(config):
    if not config.has_section('meta'):
        raise FormatError("No meta section found !")

    d = {}
    for name, value in config.items('meta'):
        d[name] = value

    for k in ['name', 'description', 'version']:
        if not d.has_key(k):
            raise FormatError("Option %s (section [meta]) is mandatory, "
                "but not found" % k)

    if not d.has_key('requires'):
        d['requires'] = []

    return d

def parse_variables(config):
    if not config.has_section('variables'):
        raise FormatError("No variables section found !")

    d = {}

    for name, value in config.items("variables"):
        d[name] = value

    return VariableSet(d)

def parse_sections(config):
    return meta_d, r

def pkg_to_filename(pkg_name):
    return "%s.ini" % pkg_name

def parse_config(filename, dirs=None):
    if dirs:
        filenames = [os.path.join(d, filename) for d in dirs]
    else:
        filenames = [filename]

    config = SafeConfigParser()
    n = config.read(filenames)
    if not len(n) >= 1:
        raise PkgNotFound("Could not find file(s) %s" % str(filenames))

    # Parse meta and variables sections
    meta = parse_meta(config)

    vars = {}
    if config.has_section('variables'):
        for name, value in config.items("variables"):
            vars[name] = _escape_backslash(value)

    # Parse "normal" sections
    secs = [s for s in config.sections() if not s in ['meta', 'variables']]
    sections = {}

    requires = {}
    for s in secs:
        d = {}
        if config.has_option(s, "requires"):
            requires[s] = config.get(s, 'requires')

        for name, value in config.items(s):
            d[name] = value
        sections[s] = d

    return meta, vars, sections, requires

def _read_config_imp(filenames, dirs=None):
    def _read_config(f):
        meta, vars, sections, reqs = parse_config(f, dirs)
        # recursively add sections and variables of required libraries
        for rname, rvalue in reqs.items():
            nmeta, nvars, nsections, nreqs = _read_config(pkg_to_filename(rvalue))

            # Update var dict for variables not in 'top' config file
            for k, v in nvars.items():
                if not vars.has_key(k):
                    vars[k] = v

            # Update sec dict
            for oname, ovalue in nsections[rname].items():
                sections[rname][oname] += ' %s' % ovalue

        return meta, vars, sections, reqs

    meta, vars, sections, reqs = _read_config(filenames)

    # FIXME: document this. If pkgname is defined in the variables section, and
    # there is no pkgdir variable defined, pkgdir is automatically defined to
    # the path of pkgname. This requires the package to be imported to work
    if not vars.has_key("pkgdir") and vars.has_key("pkgname"):
        pkgname = vars["pkgname"]
        if not pkgname in sys.modules:
            raise ValueError("You should import %s to get information on %s" %
                             (pkgname, meta["name"]))

        mod = sys.modules[pkgname]
        vars["pkgdir"] = os.path.dirname(mod.__file__)

    return LibraryInfo(name=meta["name"], description=meta["description"],
            version=meta["version"], sections=sections, vars=VariableSet(vars))

# Trivial cache to cache LibraryInfo instances creation. To be really
# efficient, the cache should be handled in read_config, since a same file can
# be parsed many time outside LibraryInfo creation, but I doubt this will be a
# problem in practice
_CACHE = {}
def read_config(pkgname, dirs=None):
    try:
        return _CACHE[pkgname]
    except KeyError:
        v = _read_config_imp(pkg_to_filename(pkgname), dirs)
        _CACHE[pkgname] = v
        return v

# TODO:
#   - implements version comparison (modversion + atleast)

# pkg-config simple emulator - useful for debugging, and maybe later to query
# the system
if __name__ == '__main__':
    import sys
    from optparse import OptionParser
    import glob

    parser = OptionParser()
    parser.add_option("--cflags", dest="cflags", action="store_true",
                      help="output all preprocessor and compiler flags")
    parser.add_option("--libs", dest="libs", action="store_true",
                      help="output all linker flags")
    parser.add_option("--use-section", dest="section",
                      help="use this section instead of default for options")
    parser.add_option("--version", dest="version", action="store_true",
                      help="output version")
    parser.add_option("--atleast-version", dest="min_version",
                      help="Minimal version")
    parser.add_option("--list-all", dest="list_all", action="store_true",
                      help="Minimal version")
    parser.add_option("--define-variable", dest="define_variable",
                      help="Replace variable with the given value")

    (options, args) = parser.parse_args(sys.argv)

    if len(args) < 2:
        raise ValueError("Expect package name on the command line:")

    if options.list_all:
        files = glob.glob("*.ini")
        for f in files:
            info = read_config(f)
            print ("%s\t%s - %s" % (info.name, info.name, info.description))

    pkg_name = args[1]
    import os
    d = os.environ.get('NPY_PKG_CONFIG_PATH')
    if d:
        info = read_config(pkg_name, ['numpy/core/lib/npy-pkg-config', '.', d])
    else:
        info = read_config(pkg_name, ['numpy/core/lib/npy-pkg-config', '.'])

    if options.section:
        section = options.section
    else:
        section = "default"

    if options.define_variable:
        m = re.search('([\S]+)=([\S]+)', options.define_variable)
        if not m:
            raise ValueError("--define-variable option should be of " \
                             "the form --define-variable=foo=bar")
        else:
            name = m.group(1)
            value = m.group(2)
        info.vars[name] = value

    if options.cflags:
        print (info.cflags(section))
    if options.libs:
        print (info.libs(section))
    if options.version:
        print (info.version)
    if options.min_version:
        print (info.version >= options.min_version)
<commit_msg>BUG: fix escaping of pkgdir on windows.<commit_after>import sys
if sys.version_info[0] < 3:
    from ConfigParser import SafeConfigParser, NoOptionError
else:
    from configparser import SafeConfigParser, NoOptionError
import re
import os
import shlex

__all__ = ['FormatError', 'PkgNotFound', 'LibraryInfo', 'VariableSet',
        'read_config', 'parse_flags']

_VAR = re.compile('\$\{([a-zA-Z0-9_-]+)\}')

class FormatError(IOError):
    """
    Exception thrown when there is a problem parsing a configuration file.

    """
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg

class PkgNotFound(IOError):
    """Exception raised when a package can not be located."""
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg

def parse_flags(line):
    """
    Parse a line from a config file containing compile flags.

    Parameters
    ----------
    line : str
        A single line containing one or more compile flags.

    Returns
    -------
    d : dict
        Dictionary of parsed flags, split into relevant categories.
        These categories are the keys of `d`:

        * 'include_dirs'
        * 'library_dirs'
        * 'libraries'
        * 'macros'
        * 'ignored'

    """
    lexer = shlex.shlex(line)
    lexer.whitespace_split = True

    d = {'include_dirs': [], 'library_dirs': [], 'libraries': [],
            'macros': [], 'ignored': []}
    def next_token(t):
        if t.startswith('-I'):
            if len(t) > 2:
                d['include_dirs'].append(t[2:])
            else:
                t = lexer.get_token()
                d['include_dirs'].append(t)
        elif t.startswith('-L'):
            if len(t) > 2:
                d['library_dirs'].append(t[2:])
            else:
                t = lexer.get_token()
                d['library_dirs'].append(t)
        elif t.startswith('-l'):
            d['libraries'].append(t[2:])
        elif t.startswith('-D'):
            d['macros'].append(t[2:])
        else:
            d['ignored'].append(t)
        return lexer.get_token()

    t = lexer.get_token()
    while t:
        t = next_token(t)

    return d

def _escape_backslash(val):
    return val.replace('\\', '\\\\')

class LibraryInfo(object):
    """
    Object containing build information about a library.

    Parameters
    ----------
    name : str
        The library name.
    description : str
        Description of the library.
    version : str
        Version string.
    sections : dict
        The sections of the configuration file for the library. The keys are
        the section headers, the values the text under each header.
    vars : class instance
        A `VariableSet` instance, which contains ``(name, value)`` pairs for
        variables defined in the configuration file for the library.
    requires : sequence, optional
        The required libraries for the library to be installed.

    Notes
    -----
    All input parameters (except "sections" which is a method) are available as
    attributes of the same name.

    """
    def __init__(self, name, description, version, sections, vars, requires=None):
        self.name = name
        self.description = description
        if requires:
            self.requires = requires
        else:
            self.requires = []
        self.version = version
        self._sections = sections
        self.vars = vars

    def sections(self):
        """
        Return the section headers of the config file.

        Parameters
        ----------
        None

        Returns
        -------
        keys : list of str
            The list of section headers.

        """
        return self._sections.keys()

    def cflags(self, section="default"):
        val = self.vars.interpolate(self._sections[section]['cflags'])
        return _escape_backslash(val)

    def libs(self, section="default"):
        val = self.vars.interpolate(self._sections[section]['libs'])
        return _escape_backslash(val)

    def __str__(self):
        m = ['Name: %s' % self.name]
        m.append('Description: %s' % self.description)
        if self.requires:
            m.append('Requires:')
        else:
            m.append('Requires: %s' % ",".join(self.requires))
        m.append('Version: %s' % self.version)

        return "\n".join(m)

class VariableSet(object):
    """
    Container object for the variables defined in a config file.

    `VariableSet` can be used as a plain dictionary, with the variable names
    as keys.

    Parameters
    ----------
    d : dict
        Dict of items in the "variables" section of the configuration file.

    """
    def __init__(self, d):
        self._raw_data = dict([(k, v) for k, v in d.items()])

        self._re = {}
        self._re_sub = {}

        self._init_parse()

    def _init_parse(self):
        for k, v in self._raw_data.items():
            self._init_parse_var(k, v)

    def _init_parse_var(self, name, value):
        self._re[name] = re.compile(r'\$\{%s\}' % name)
        self._re_sub[name] = value

    def interpolate(self, value):
        # Brute force: we keep interpolating until there is no '${var}' anymore
        # or until interpolated string is equal to input string
        def _interpolate(value):
            for k in self._re.keys():
                value = self._re[k].sub(self._re_sub[k], value)
            return value
        while _VAR.search(value):
            nvalue = _interpolate(value)
            if nvalue == value:
                break
            value = nvalue

        return value

    def variables(self):
        """
        Return the list of variable names.

        Parameters
        ----------
        None

        Returns
        -------
        names : list of str
            The names of all variables in the `VariableSet` instance.

        """
        return self._raw_data.keys()

    # Emulate a dict to set/get variables values
    def __getitem__(self, name):
        return self._raw_data[name]

    def __setitem__(self, name, value):
        self._raw_data[name] = value
        self._init_parse_var(name, value)

def parse_meta(config):
    if not config.has_section('meta'):
        raise FormatError("No meta section found !")

    d = {}
    for name, value in config.items('meta'):
        d[name] = value

    for k in ['name', 'description', 'version']:
        if not d.has_key(k):
            raise FormatError("Option %s (section [meta]) is mandatory, "
                "but not found" % k)

    if not d.has_key('requires'):
        d['requires'] = []

    return d

def parse_variables(config):
    if not config.has_section('variables'):
        raise FormatError("No variables section found !")

    d = {}

    for name, value in config.items("variables"):
        d[name] = value

    return VariableSet(d)

def parse_sections(config):
    return meta_d, r

def pkg_to_filename(pkg_name):
    return "%s.ini" % pkg_name

def parse_config(filename, dirs=None):
    if dirs:
        filenames = [os.path.join(d, filename) for d in dirs]
    else:
        filenames = [filename]

    config = SafeConfigParser()
    n = config.read(filenames)
    if not len(n) >= 1:
        raise PkgNotFound("Could not find file(s) %s" % str(filenames))

    # Parse meta and variables sections
    meta = parse_meta(config)

    vars = {}
    if config.has_section('variables'):
        for name, value in config.items("variables"):
            vars[name] = _escape_backslash(value)

    # Parse "normal" sections
    secs = [s for s in config.sections() if not s in ['meta', 'variables']]
    sections = {}

    requires = {}
    for s in secs:
        d = {}
        if config.has_option(s, "requires"):
            requires[s] = config.get(s, 'requires')

        for name, value in config.items(s):
            d[name] = value
        sections[s] = d

    return meta, vars, sections, requires

def _read_config_imp(filenames, dirs=None):
    def _read_config(f):
        meta, vars, sections, reqs = parse_config(f, dirs)
        # recursively add sections and variables of required libraries
        for rname, rvalue in reqs.items():
            nmeta, nvars, nsections, nreqs = _read_config(pkg_to_filename(rvalue))

            # Update var dict for variables not in 'top' config file
            for k, v in nvars.items():
                if not vars.has_key(k):
                    vars[k] = v

            # Update sec dict
            for oname, ovalue in nsections[rname].items():
                sections[rname][oname] += ' %s' % ovalue

        return meta, vars, sections, reqs

    meta, vars, sections, reqs = _read_config(filenames)

    # FIXME: document this. If pkgname is defined in the variables section, and
    # there is no pkgdir variable defined, pkgdir is automatically defined to
    # the path of pkgname. This requires the package to be imported to work
    if not vars.has_key("pkgdir") and vars.has_key("pkgname"):
        pkgname = vars["pkgname"]
        if not pkgname in sys.modules:
            raise ValueError("You should import %s to get information on %s" %
                             (pkgname, meta["name"]))

        mod = sys.modules[pkgname]
        vars["pkgdir"] = _escape_backslash(os.path.dirname(mod.__file__))

    return LibraryInfo(name=meta["name"], description=meta["description"],
            version=meta["version"], sections=sections, vars=VariableSet(vars))

# Trivial cache to cache LibraryInfo instances creation. To be really
# efficient, the cache should be handled in read_config, since a same file can
# be parsed many time outside LibraryInfo creation, but I doubt this will be a
# problem in practice
_CACHE = {}
def read_config(pkgname, dirs=None):
    try:
        return _CACHE[pkgname]
    except KeyError:
        v = _read_config_imp(pkg_to_filename(pkgname), dirs)
        _CACHE[pkgname] = v
        return v

# TODO:
#   - implements version comparison (modversion + atleast)

# pkg-config simple emulator - useful for debugging, and maybe later to query
# the system
if __name__ == '__main__':
    import sys
    from optparse import OptionParser
    import glob

    parser = OptionParser()
    parser.add_option("--cflags", dest="cflags", action="store_true",
                      help="output all preprocessor and compiler flags")
    parser.add_option("--libs", dest="libs", action="store_true",
                      help="output all linker flags")
    parser.add_option("--use-section", dest="section",
                      help="use this section instead of default for options")
    parser.add_option("--version", dest="version", action="store_true",
                      help="output version")
    parser.add_option("--atleast-version", dest="min_version",
                      help="Minimal version")
    parser.add_option("--list-all", dest="list_all", action="store_true",
                      help="Minimal version")
    parser.add_option("--define-variable", dest="define_variable",
                      help="Replace variable with the given value")

    (options, args) = parser.parse_args(sys.argv)

    if len(args) < 2:
        raise ValueError("Expect package name on the command line:")

    if options.list_all:
        files = glob.glob("*.ini")
        for f in files:
            info = read_config(f)
            print ("%s\t%s - %s" % (info.name, info.name, info.description))

    pkg_name = args[1]
    import os
    d = os.environ.get('NPY_PKG_CONFIG_PATH')
    if d:
        info = read_config(pkg_name, ['numpy/core/lib/npy-pkg-config', '.', d])
    else:
        info = read_config(pkg_name, ['numpy/core/lib/npy-pkg-config', '.'])

    if options.section:
        section = options.section
    else:
        section = "default"

    if options.define_variable:
        m = re.search('([\S]+)=([\S]+)', options.define_variable)
        if not m:
            raise ValueError("--define-variable option should be of " \
                             "the form --define-variable=foo=bar")
        else:
            name = m.group(1)
            value = m.group(2)
        info.vars[name] = value

    if options.cflags:
        print (info.cflags(section))
    if options.libs:
        print (info.libs(section))
    if options.version:
        print (info.version)
    if options.min_version:
        print (info.version >= options.min_version)
<|endoftext|>


<commit_before>"""Django related settings."""
from django.contrib import messages

# =================
# General settings
# =================
SITE_ID = 1
ROOT_URLCONF = 'myproject.urls'

MESSAGE_TAGS = {
    messages.DEBUG: 'debug',
    messages.INFO: 'alert-info',
    messages.SUCCESS: 'alert-success',
    messages.WARNING: 'alert-warning',
    messages.ERROR: 'alert-error',
}

# =======================
# Email related settings
# =======================
SEND_BROKEN_LINK_EMAILS = True

# =============================
# Language & Timezone settings
# =============================
gettext = lambda s: s

# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
TIME_ZONE = 'UTC'
USE_TZ = True

# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en'
USE_I18N = True
USE_L10N = True
LANGUAGES = (
    ('en', gettext('English')),
)
<commit_msg>Added FIXTURE_DIRS setting<commit_after>"""Django related settings."""
import os

from django.contrib import messages

from myproject.settings.base_settings import PROJECT_ROOT

# =================
# General settings
# =================
SITE_ID = 1
ROOT_URLCONF = 'myproject.urls'

MESSAGE_TAGS = {
    messages.DEBUG: 'debug',
    messages.INFO: 'alert-info',
    messages.SUCCESS: 'alert-success',
    messages.WARNING: 'alert-warning',
    messages.ERROR: 'alert-error',
}

FIXTURE_DIRS = (
    os.path.join(PROJECT_ROOT, 'myproject/fixtures'),
)

# =======================
# Email related settings
# =======================
SEND_BROKEN_LINK_EMAILS = True

# =============================
# Language & Timezone settings
# =============================
gettext = lambda s: s

# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
TIME_ZONE = 'UTC'
USE_TZ = True

# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en'
USE_I18N = True
USE_L10N = True
LANGUAGES = (
    ('en', gettext('English')),
)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Run REDbot as a daemon.
"""

from configparser import ConfigParser, SectionProxy
from functools import partial
import locale
import os
import signal
import sys
import traceback
from types import FrameType
from typing import Dict
from urllib.parse import urlsplit

import thor
from thor.loop import _loop

from redbot import __version__
from redbot.type import RawHeaderListType
from redbot.webui import RedWebUi

if os.environ.get("SYSTEMD_WATCHDOG"):
    try:
        from cysystemd.daemon import notify, Notification  # type: ignore
    except ImportError:
        notify = Notification = None  # pylint: disable=invalid-name
else:
    notify = Notification = None  # pylint: disable=invalid-name

_loop.precision = 0.1

class RedBotServer:
    """Run REDbot as a standalone Web server."""

    watchdog_freq = 3

    def __init__(self, config: SectionProxy) -> None:
        self.config = config
        self.handler = partial(RedHandler, server=self)

        # Set up the watchdog
        if notify is not None:
            thor.schedule(self.watchdog_freq, self.watchdog_ping)
            signal.signal(signal.SIGABRT, self.abrt_handler)

        # Read static files
        self.static_files = self.walk_files(self.config["asset_dir"], b"static/")
        if self.config.get("extra_base_dir"):
            self.static_files.update(self.walk_files(self.config["extra_base_dir"]))

        # Set up the server
        server = thor.http.HttpServer(
            self.config.get("host", "").encode("utf-8"), int(self.config["port"])
        )
        server.on("exchange", self.handler)
        try:
            thor.run()
        except KeyboardInterrupt:
            sys.stderr.write("Stopping...\n")
            thor.stop()

    def watchdog_ping(self) -> None:
        notify(Notification.WATCHDOG)
        thor.schedule(self.watchdog_freq, self.watchdog_ping)

    @staticmethod
    def abrt_handler(signum: int, frame: FrameType) -> None:
        sys.stderr.write("* ABORT\n")
        traceback.print_stack(frame)
        sys.exit(0)

    @staticmethod
    def walk_files(dir_name: str, base: bytes = b"") -> Dict[bytes, bytes]:
        out: Dict[bytes, bytes] = {}
        for root, _, files in os.walk(dir_name):
            for name in files:
                try:
                    path = os.path.join(root, name)
                    uri = os.path.relpath(path, dir_name).encode("utf-8")
                    with open(path, "rb") as fh:
                        out[b"/%s%s" % (base, uri)] = fh.read()
                    if uri.endswith(b"/index.html"):
                        with open(path, "rb") as fh:
                            out[b"/%s%s" % (base, uri[:-10])] = fh.read()
                except IOError:
                    sys.stderr.write(f"* Problem loading {path}\n")
        return out

class RedHandler:
    static_types = {
        b".html": b"text/html",
        b".js": b"text/javascript",
        b".css": b"text/css",
        b".png": b"image/png",
        b".txt": b"text/plain",
        b".woff": b"font/woff",
        b".ttf": b"font/ttf",
        b".eot": b"application/vnd.ms-fontobject",
        b".svg": b"image/svg+xml",
    }

    def __init__(
        self, exchange: thor.http.server.HttpServerExchange, server: RedBotServer
    ) -> None:
        self.exchange = exchange
        self.config = server.config
        self.static_files = server.static_files
        self.method = b""
        self.uri = b""
        self.req_hdrs: RawHeaderListType = []
        self.req_body = b""
        exchange.on("request_start", self.request_start)
        exchange.on("request_body", self.request_body)
        exchange.on("request_done", self.request_done)

    def request_start(
        self, method: bytes, uri: bytes, req_hdrs: RawHeaderListType
    ) -> None:
        self.method = method
        self.uri = uri
        self.req_hdrs = req_hdrs

    def request_body(self, chunk: bytes) -> None:
        self.req_body += chunk

    def request_done(self, trailers: RawHeaderListType) -> None:
        p_uri = urlsplit(self.uri)
        if p_uri.path in self.static_files:
            file_ext = os.path.splitext(p_uri.path)[1].lower() or b".html"
            content_type = self.static_types.get(file_ext, b"application/octet-stream")
            headers = []
            headers.append((b"Content-Type", content_type))
            headers.append((b"Cache-Control", b"max-age=3600"))
            self.exchange.response_start(b"200", b"OK", headers)
            self.exchange.response_body(self.static_files[p_uri.path])
            self.exchange.response_done([])
        elif p_uri.path == b"/":
            try:
                self.req_hdrs.append(
                    (
                        b"client-ip",
                        self.exchange.http_conn.tcp_conn.socket.getpeername()[0].encode(
                            "idna"
                        ),
                    )
                )
                RedWebUi(
                    self.config,
                    self.method.decode(self.config["charset"]),
                    p_uri.query,
                    self.req_hdrs,
                    self.req_body,
                    self.exchange,
                    self.error_log,
                )
            except Exception:  # pylint: disable=broad-except
                self.error_log(
                    """
*** FATAL ERROR
REDbot has encountered a fatal error which it really, really can't recover from
in standalone server mode. Details follow.
"""
                )

                dump = traceback.format_exc()
                thor.stop()
                self.error_log(dump)
                sys.exit(1)
        else:
            headers = []
            headers.append((b"Content-Type", b"text/plain"))
            headers.append((b"Cache-Control", b"max-age=3600"))
            self.exchange.response_start(b"404", b"Not Found", headers)
            self.exchange.response_body(b"'%s' not found." % p_uri.path)
            self.exchange.response_done([])

    @staticmethod
    def error_log(message: str) -> None:
        sys.stderr.write(f"{message}\n")

if __name__ == "__main__":

    import argparse

    parser = argparse.ArgumentParser(description="REDbot daemon")
    parser.add_argument(
        "config_file", type=argparse.FileType("r"), help="configuration file"
    )
    args = parser.parse_args()
    conf = ConfigParser()
    conf.read_file(args.config_file)

    try:
        locale.setlocale(locale.LC_ALL, locale.normalize(conf["redbot"]["lang"]))
    except ValueError:
        locale.setlocale(locale.LC_ALL, "")

    sys.stderr.write(
        f"Starting on PID {os.getpid()}... (thor {thor.__version__})\n"
        + f"http://{conf['redbot'].get('host', '')}:{conf['redbot']['port']}/\n"
    )

    RedBotServer(conf["redbot"])
<commit_msg>Always run loop debug for daemon<commit_after>#!/usr/bin/env python3

"""
Run REDbot as a daemon.
"""

from configparser import ConfigParser, SectionProxy
from functools import partial
import locale
import os
import signal
import sys
import traceback
from types import FrameType
from typing import Dict
from urllib.parse import urlsplit

import thor
from thor.loop import _loop

from redbot import __version__
from redbot.type import RawHeaderListType
from redbot.webui import RedWebUi

if os.environ.get("SYSTEMD_WATCHDOG"):
    try:
        from cysystemd.daemon import notify, Notification  # type: ignore
    except ImportError:
        notify = Notification = None  # pylint: disable=invalid-name
else:
    notify = Notification = None  # pylint: disable=invalid-name

_loop.precision = 0.1
_loop.debug = True

class RedBotServer:
    """Run REDbot as a standalone Web server."""

    watchdog_freq = 3

    def __init__(self, config: SectionProxy) -> None:
        self.config = config
        self.handler = partial(RedHandler, server=self)

        # Set up the watchdog
        if notify is not None:
            thor.schedule(self.watchdog_freq, self.watchdog_ping)
            signal.signal(signal.SIGABRT, self.abrt_handler)

        # Read static files
        self.static_files = self.walk_files(self.config["asset_dir"], b"static/")
        if self.config.get("extra_base_dir"):
            self.static_files.update(self.walk_files(self.config["extra_base_dir"]))

        # Set up the server
        server = thor.http.HttpServer(
            self.config.get("host", "").encode("utf-8"), int(self.config["port"])
        )
        server.on("exchange", self.handler)
        try:
            thor.run()
        except KeyboardInterrupt:
            sys.stderr.write("Stopping...\n")
            thor.stop()

    def watchdog_ping(self) -> None:
        notify(Notification.WATCHDOG)
        thor.schedule(self.watchdog_freq, self.watchdog_ping)

    @staticmethod
    def abrt_handler(signum: int, frame: FrameType) -> None:
        sys.stderr.write("* ABORT\n")
        traceback.print_stack(frame)
        sys.exit(0)

    @staticmethod
    def walk_files(dir_name: str, base: bytes = b"") -> Dict[bytes, bytes]:
        out: Dict[bytes, bytes] = {}
        for root, _, files in os.walk(dir_name):
            for name in files:
                try:
                    path = os.path.join(root, name)
                    uri = os.path.relpath(path, dir_name).encode("utf-8")
                    with open(path, "rb") as fh:
                        out[b"/%s%s" % (base, uri)] = fh.read()
                    if uri.endswith(b"/index.html"):
                        with open(path, "rb") as fh:
                            out[b"/%s%s" % (base, uri[:-10])] = fh.read()
                except IOError:
                    sys.stderr.write(f"* Problem loading {path}\n")
        return out

class RedHandler:
    static_types = {
        b".html": b"text/html",
        b".js": b"text/javascript",
        b".css": b"text/css",
        b".png": b"image/png",
        b".txt": b"text/plain",
        b".woff": b"font/woff",
        b".ttf": b"font/ttf",
        b".eot": b"application/vnd.ms-fontobject",
        b".svg": b"image/svg+xml",
    }

    def __init__(
        self, exchange: thor.http.server.HttpServerExchange, server: RedBotServer
    ) -> None:
        self.exchange = exchange
        self.config = server.config
        self.static_files = server.static_files
        self.method = b""
        self.uri = b""
        self.req_hdrs: RawHeaderListType = []
        self.req_body = b""
        exchange.on("request_start", self.request_start)
        exchange.on("request_body", self.request_body)
        exchange.on("request_done", self.request_done)

    def request_start(
        self, method: bytes, uri: bytes, req_hdrs: RawHeaderListType
    ) -> None:
        self.method = method
        self.uri = uri
        self.req_hdrs = req_hdrs

    def request_body(self, chunk: bytes) -> None:
        self.req_body += chunk

    def request_done(self, trailers: RawHeaderListType) -> None:
        p_uri = urlsplit(self.uri)
        if p_uri.path in self.static_files:
            file_ext = os.path.splitext(p_uri.path)[1].lower() or b".html"
            content_type = self.static_types.get(file_ext, b"application/octet-stream")
            headers = []
            headers.append((b"Content-Type", content_type))
            headers.append((b"Cache-Control", b"max-age=3600"))
            self.exchange.response_start(b"200", b"OK", headers)
            self.exchange.response_body(self.static_files[p_uri.path])
            self.exchange.response_done([])
        elif p_uri.path == b"/":
            try:
                self.req_hdrs.append(
                    (
                        b"client-ip",
                        self.exchange.http_conn.tcp_conn.socket.getpeername()[0].encode(
                            "idna"
                        ),
                    )
                )
                RedWebUi(
                    self.config,
                    self.method.decode(self.config["charset"]),
                    p_uri.query,
                    self.req_hdrs,
                    self.req_body,
                    self.exchange,
                    self.error_log,
                )
            except Exception:  # pylint: disable=broad-except
                self.error_log(
                    """
*** FATAL ERROR
REDbot has encountered a fatal error which it really, really can't recover from
in standalone server mode. Details follow.
"""
                )

                dump = traceback.format_exc()
                thor.stop()
                self.error_log(dump)
                sys.exit(1)
        else:
            headers = []
            headers.append((b"Content-Type", b"text/plain"))
            headers.append((b"Cache-Control", b"max-age=3600"))
            self.exchange.response_start(b"404", b"Not Found", headers)
            self.exchange.response_body(b"'%s' not found." % p_uri.path)
            self.exchange.response_done([])

    @staticmethod
    def error_log(message: str) -> None:
        sys.stderr.write(f"{message}\n")

if __name__ == "__main__":

    import argparse

    parser = argparse.ArgumentParser(description="REDbot daemon")
    parser.add_argument(
        "config_file", type=argparse.FileType("r"), help="configuration file"
    )
    args = parser.parse_args()
    conf = ConfigParser()
    conf.read_file(args.config_file)

    try:
        locale.setlocale(locale.LC_ALL, locale.normalize(conf["redbot"]["lang"]))
    except ValueError:
        locale.setlocale(locale.LC_ALL, "")

    sys.stderr.write(
        f"Starting on PID {os.getpid()}... (thor {thor.__version__})\n"
        + f"http://{conf['redbot'].get('host', '')}:{conf['redbot']['port']}/\n"
    )

    RedBotServer(conf["redbot"])
<|endoftext|>


<commit_before>#!/usr/bin/env python

import flask
import json
from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import get_lexer_for_filename
from sphinxit.core.helpers import BaseSearchConfig
from sphinxit.core.processor import Search

app = flask.Flask(__name__)

def main():
    app.debug = True
    app.run(host='0.0.0.0')

@app.route("/favicon.ico")
def favicon():
    return app.send_static_file("favicon.ico")

@app.route('/')
def search_page():
    return flask.render_template('codesearch.html')

@app.route('/search')
def search():
    term = flask.request.args.get('q')
    query = Search(indexes=['sourcecode'], config=BaseSearchConfig)
    query = query.match(term)
    results = query.ask()
    filenames = [result['path'] for result in results['result']['items']]
    return flask.Response(json.dumps(filenames), mimetype='application/json')

@app.route('/display')
def display():
    filename = flask.request.args.get('f').strip()
    lexer = get_lexer_for_filename(filename)
    formatter = HtmlFormatter(noclasses=True)
    code = ''
    with open(filename) as f:
        code = f.read()
    result = highlight(code, lexer, formatter)

    return flask.render_template('display.html', filename=filename,
                                 code=result)

if __name__ == '__main__':
    main()
<commit_msg>The display page title is the basename.<commit_after>#!/usr/bin/env python

import flask
import json
import os
from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import get_lexer_for_filename
from sphinxit.core.helpers import BaseSearchConfig
from sphinxit.core.processor import Search

app = flask.Flask(__name__)

def main():
    app.debug = True
    app.run(host='0.0.0.0')

@app.route("/favicon.ico")
def favicon():
    return app.send_static_file("favicon.ico")

@app.route('/')
def search_page():
    return flask.render_template('codesearch.html')

@app.route('/search')
def search():
    term = flask.request.args.get('q')
    query = Search(indexes=['sourcecode'], config=BaseSearchConfig)
    query = query.match(term)
    results = query.ask()
    filenames = [result['path'] for result in results['result']['items']]
    return flask.Response(json.dumps(filenames), mimetype='application/json')

@app.route('/display')
def display():
    filename = flask.request.args.get('f').strip()
    lexer = get_lexer_for_filename(filename)
    formatter = HtmlFormatter(noclasses=True)
    code = ''
    with open(filename) as f:
        code = f.read()
    result = highlight(code, lexer, formatter)

    return flask.render_template('display.html',
                                 filename=os.path.basename(filename),
                                 code=result)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import collections
import copy
from casexml.apps.case.xform import extract_case_blocks, get_case_ids_from_form
from corehq.apps.change_feed import topics
from corehq.apps.change_feed.consumer.feed import KafkaChangeFeed
from corehq.elastic import get_es_new
from corehq.form_processor.interfaces.dbaccessors import FormAccessors
from corehq.pillows.mappings.xform_mapping import XFORM_MAPPING, XFORM_INDEX
from .base import HQPillow
from couchforms.const import RESERVED_WORDS
from couchforms.models import XFormInstance
from dateutil import parser
from dimagi.utils.decorators.memoized import memoized
from pillowtop.checkpoints.manager import PillowCheckpoint, get_django_checkpoint_store, \
    PillowCheckpointEventHandler
from pillowtop.listener import send_to_elasticsearch
from pillowtop.pillow.interface import ConstructedPillow
from pillowtop.processor import PillowProcessor

UNKNOWN_VERSION = 'XXX'
UNKNOWN_UIVERSION = 'XXX'
XFORM_ES_TYPE = 'xform'

def is_valid_date(txt):
    try:
        if txt and parser.parse(txt):
            return True
    except Exception:
        pass
    return False

# modified from: http://stackoverflow.com/questions/6027558/flatten-nested-python-dictionaries-compressing-keys
def flatten(d, parent_key='', delimiter='/'):
    items = []
    for k, v in d.items():
        if k in RESERVED_WORDS:
            continue
        new_key = parent_key + delimiter + k if parent_key else k
        if isinstance(v, collections.MutableMapping):
            items.extend(flatten(v, new_key, delimiter).items())
        elif not isinstance(v, list):
            items.append((new_key, v))
    return dict(items)

class XFormPillow(HQPillow):
    document_class = XFormInstance
    couch_filter = "couchforms/xforms"
    es_alias = "xforms"
    es_type = XFORM_ES_TYPE
    es_index = XFORM_INDEX
    include_docs = False

    # for simplicity, the handlers are managed on the domain level
    handler_domain_map = {}
    default_mapping = XFORM_MAPPING

    @classmethod
    def get_unique_id(self):
        return XFORM_INDEX

    def change_transform(self, doc_dict, include_props=True):
        return transform_xform_for_elasticsearch(doc_dict, include_props)

def transform_xform_for_elasticsearch(doc_dict, include_props=True):
    """
    Given an XFormInstance, return a copy that is ready to be sent to elasticsearch,
    or None, if the form should not be saved to elasticsearch
    """
    if doc_dict.get('domain', None) is None:
        # if there is no domain don't bother processing it
        return None
    else:
        doc_ret = copy.deepcopy(doc_dict)

        if 'meta' in doc_ret['form']:
            if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)):
                doc_ret['form']['meta']['timeEnd'] = None
            if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)):
                doc_ret['form']['meta']['timeStart'] = None

            # Some docs have their @xmlns and #text here
            if isinstance(doc_ret['form']['meta'].get('appVersion'), dict):
                doc_ret['form']['meta']['appVersion'] = doc_ret['form']['meta']['appVersion'].get('#text')

        case_blocks = extract_case_blocks(doc_ret)
        for case_dict in case_blocks:
            for date_modified_key in ['date_modified', '@date_modified']:
                if not is_valid_date(case_dict.get(date_modified_key, None)):
                    if case_dict.get(date_modified_key) == '':
                        case_dict[date_modified_key] = None
                    else:
                        case_dict.pop(date_modified_key, None)

            # convert all mapped dict properties to nulls if they are empty strings
            for object_key in ['index', 'attachment', 'create', 'update']:
                if object_key in case_dict and not isinstance(case_dict[object_key], dict):
                    case_dict[object_key] = None

        doc_ret["__retrieved_case_ids"] = list(get_case_ids_from_form(doc_dict))
        if include_props:
            form_props = ["%s:%s" % (k, v) for k, v in flatten(doc_ret['form']).iteritems()]
            doc_ret["__props_for_querying"] = form_props
        return doc_ret

class XFormToElasticProcessor(PillowProcessor):

    @property
    @memoized
    def elasticsearch(self):
        return get_es_new()

    def process_change(self, pillow_instance, change, do_set_checkpoint):
        # if you don't worry about the various configuration options for reindexing,
        # bootstrapping / dealing with the elasticsearch index, and bulk operations,
        # this is all the current code does too
        form_ready_to_go = transform_xform_for_elasticsearch(change.get_document())
        # todo: this is required for our queries, figure out how best to deal with it
        form_ready_to_go['doc_type'] = 'XFormInstance'
        doc_exists = self.elasticsearch.exists(XFORM_INDEX, change.id, XFORM_ES_TYPE)
        send_to_elasticsearch(
            index=XFORM_INDEX,
            doc_type=XFORM_ES_TYPE,
            doc_id=change.id,
            es_getter=get_es_new,
            name=pillow_instance.get_name(),
            data=form_ready_to_go,
            update=doc_exists,
        )

def get_sql_xform_to_elasticsearch_pillow():
    checkpoint = PillowCheckpoint(
        get_django_checkpoint_store(),
        'sql-xforms-to-elasticsearch',
    )
    return ConstructedPillow(
        name='SqlXFormToElasticsearchPillow',
        document_store=None,
        checkpoint=checkpoint,
        change_feed=KafkaChangeFeed(topic=topics.SQL_FORM, group_id='sql-forms-to-es'),
        processor=XFormToElasticProcessor(),
        change_processed_event_handler=PillowCheckpointEventHandler(
            checkpoint=checkpoint, checkpoint_frequency=100,
        ),
    )
<commit_msg>remove unused import<commit_after>import collections
import copy
from casexml.apps.case.xform import extract_case_blocks, get_case_ids_from_form
from corehq.apps.change_feed import topics
from corehq.apps.change_feed.consumer.feed import KafkaChangeFeed
from corehq.elastic import get_es_new
from corehq.pillows.mappings.xform_mapping import XFORM_MAPPING, XFORM_INDEX
from .base import HQPillow
from couchforms.const import RESERVED_WORDS
from couchforms.models import XFormInstance
from dateutil import parser
from dimagi.utils.decorators.memoized import memoized
from pillowtop.checkpoints.manager import PillowCheckpoint, get_django_checkpoint_store, \
    PillowCheckpointEventHandler
from pillowtop.listener import send_to_elasticsearch
from pillowtop.pillow.interface import ConstructedPillow
from pillowtop.processor import PillowProcessor

UNKNOWN_VERSION = 'XXX'
UNKNOWN_UIVERSION = 'XXX'
XFORM_ES_TYPE = 'xform'

def is_valid_date(txt):
    try:
        if txt and parser.parse(txt):
            return True
    except Exception:
        pass
    return False

# modified from: http://stackoverflow.com/questions/6027558/flatten-nested-python-dictionaries-compressing-keys
def flatten(d, parent_key='', delimiter='/'):
    items = []
    for k, v in d.items():
        if k in RESERVED_WORDS:
            continue
        new_key = parent_key + delimiter + k if parent_key else k
        if isinstance(v, collections.MutableMapping):
            items.extend(flatten(v, new_key, delimiter).items())
        elif not isinstance(v, list):
            items.append((new_key, v))
    return dict(items)

class XFormPillow(HQPillow):
    document_class = XFormInstance
    couch_filter = "couchforms/xforms"
    es_alias = "xforms"
    es_type = XFORM_ES_TYPE
    es_index = XFORM_INDEX
    include_docs = False

    # for simplicity, the handlers are managed on the domain level
    handler_domain_map = {}
    default_mapping = XFORM_MAPPING

    @classmethod
    def get_unique_id(self):
        return XFORM_INDEX

    def change_transform(self, doc_dict, include_props=True):
        return transform_xform_for_elasticsearch(doc_dict, include_props)

def transform_xform_for_elasticsearch(doc_dict, include_props=True):
    """
    Given an XFormInstance, return a copy that is ready to be sent to elasticsearch,
    or None, if the form should not be saved to elasticsearch
    """
    if doc_dict.get('domain', None) is None:
        # if there is no domain don't bother processing it
        return None
    else:
        doc_ret = copy.deepcopy(doc_dict)

        if 'meta' in doc_ret['form']:
            if not is_valid_date(doc_ret['form']['meta'].get('timeEnd', None)):
                doc_ret['form']['meta']['timeEnd'] = None
            if not is_valid_date(doc_ret['form']['meta'].get('timeStart', None)):
                doc_ret['form']['meta']['timeStart'] = None

            # Some docs have their @xmlns and #text here
            if isinstance(doc_ret['form']['meta'].get('appVersion'), dict):
                doc_ret['form']['meta']['appVersion'] = doc_ret['form']['meta']['appVersion'].get('#text')

        case_blocks = extract_case_blocks(doc_ret)
        for case_dict in case_blocks:
            for date_modified_key in ['date_modified', '@date_modified']:
                if not is_valid_date(case_dict.get(date_modified_key, None)):
                    if case_dict.get(date_modified_key) == '':
                        case_dict[date_modified_key] = None
                    else:
                        case_dict.pop(date_modified_key, None)

            # convert all mapped dict properties to nulls if they are empty strings
            for object_key in ['index', 'attachment', 'create', 'update']:
                if object_key in case_dict and not isinstance(case_dict[object_key], dict):
                    case_dict[object_key] = None

        doc_ret["__retrieved_case_ids"] = list(get_case_ids_from_form(doc_dict))
        if include_props:
            form_props = ["%s:%s" % (k, v) for k, v in flatten(doc_ret['form']).iteritems()]
            doc_ret["__props_for_querying"] = form_props
        return doc_ret

class XFormToElasticProcessor(PillowProcessor):

    @property
    @memoized
    def elasticsearch(self):
        return get_es_new()

    def process_change(self, pillow_instance, change, do_set_checkpoint):
        # if you don't worry about the various configuration options for reindexing,
        # bootstrapping / dealing with the elasticsearch index, and bulk operations,
        # this is all the current code does too
        form_ready_to_go = transform_xform_for_elasticsearch(change.get_document())
        # todo: this is required for our queries, figure out how best to deal with it
        form_ready_to_go['doc_type'] = 'XFormInstance'
        doc_exists = self.elasticsearch.exists(XFORM_INDEX, change.id, XFORM_ES_TYPE)
        send_to_elasticsearch(
            index=XFORM_INDEX,
            doc_type=XFORM_ES_TYPE,
            doc_id=change.id,
            es_getter=get_es_new,
            name=pillow_instance.get_name(),
            data=form_ready_to_go,
            update=doc_exists,
        )

def get_sql_xform_to_elasticsearch_pillow():
    checkpoint = PillowCheckpoint(
        get_django_checkpoint_store(),
        'sql-xforms-to-elasticsearch',
    )
    return ConstructedPillow(
        name='SqlXFormToElasticsearchPillow',
        document_store=None,
        checkpoint=checkpoint,
        change_feed=KafkaChangeFeed(topic=topics.SQL_FORM, group_id='sql-forms-to-es'),
        processor=XFormToElasticProcessor(),
        change_processed_event_handler=PillowCheckpointEventHandler(
            checkpoint=checkpoint, checkpoint_frequency=100,
        ),
    )
<|endoftext|>


<commit_before>from django.db.models import Q, Sum, Avg

from rest_framework import status
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework.views import APIView

from ratechecker.models import Region, Rate, Adjustment, Fee
from ratechecker.ratechecker_parameters import ParamsSerializer

def get_rates(params_data, data_load_testing=False, return_fees=False):
    """ params_data is a method parameter of type RateCheckerParameters."""

    # the precalculated results are done by favoring negative points over
    # positive ones, and the API does the opposite
    factor = 1
    if data_load_testing:
        factor = -1

    region_ids = list(Region.objects.filter(
        state_id=params_data.get('state')).values_list('region_id', flat=True))

    rates = Rate.objects.filter(
        region_id__in=region_ids,
        product__loan_purpose=params_data.get('loan_purpose'),
        product__pmt_type=params_data.get('rate_structure'),
        product__loan_type=params_data.get('loan_type'),
        product__max_ltv__gte=params_data.get('max_ltv'),
        product__loan_term=params_data.get('loan_term'),
        product__max_loan_amt__gte=params_data.get('loan_amount'),
        product__max_fico__gte=params_data.get('maxfico'),
        product__min_fico__lte=params_data.get('minfico'))

    if params_data.get('loan_type') != 'FHA-HB':
        rates = rates.filter(
            product__min_loan_amt__lte=params_data.get('loan_amount'))

    if params_data.get('rate_structure') == 'ARM':
        rates = rates.filter(
            product__int_adj_term=params_data.get('arm_type')[:-2],
            product__io=bool(params_data.get('io')))

    if data_load_testing:
        rates = rates.filter(
            product__institution=params_data.get('institution'),
            lock=params_data.get('lock'))
    else:
        rates = rates.filter(
            lock__lte=params_data.get('max_lock', 0),
            lock__gt=params_data.get('min_lock', 0))

    all_rates = []
    products = {}
    for rate in rates:
        all_rates.append(rate)
        products["{}{}".format(rate.product_id, rate.region_id)] = rate.product_id
    product_ids = products.values()

    adjustments = Adjustment.objects.filter(product__plan_id__in=product_ids).filter(
        Q(max_loan_amt__gte=params_data.get('loan_amount')) | Q(max_loan_amt__isnull=True),
        Q(min_loan_amt__lte=params_data.get('loan_amount')) | Q(min_loan_amt__isnull=True),
        Q(prop_type=params_data.get('property_type')) | Q(prop_type__isnull=True) | Q(prop_type=""),
        Q(state=params_data.get('state')) | Q(state__isnull=True) | Q(state=""),
        Q(max_fico__gte=params_data.get('maxfico')) | Q(max_fico__isnull=True),
        Q(min_fico__lte=params_data.get('minfico')) | Q(min_fico__isnull=True),
        Q(min_ltv__lte=params_data.get('min_ltv')) | Q(min_ltv__isnull=True),
        Q(max_ltv__gte=params_data.get('max_ltv')) | Q(max_ltv__isnull=True),
    ).values('product_id',
             'affect_rate_type').annotate(sum_of_adjvalue=Sum('adj_value'))

    summed_adj_dict = {}
    for adj in adjustments:
        current = summed_adj_dict.get(adj['product_id'], {})
        current[adj['affect_rate_type']] = adj['sum_of_adjvalue']
        summed_adj_dict[adj['product_id']] = current
    available_rates = {}
    data_timestamp = ""
    for rate in all_rates:
        # TODO: check that it the same all the time, and do what if it is not?
        data_timestamp = rate.data_timestamp
        product = summed_adj_dict.get(rate.product_id, {})
        rate.total_points += product.get('P', 0)
        rate.base_rate += product.get('R', 0)
        distance = abs(params_data.get('points') - rate.total_points)
        if float(distance) > 0.5:
            continue
        if rate.product_id not in available_rates:
            available_rates[rate.product_id] = rate
        else:
            current_difference = abs(
                params_data.get('points') -
                available_rates[rate.product_id].total_points
                )
            new_difference = abs(params_data.get('points') - rate.total_points)
            if new_difference < current_difference or (
                    new_difference == current_difference and
                    factor * available_rates[
                        rate.product_id].total_points < 0 and
                    factor * rate.total_points > 0):
                available_rates[rate.product_id] = rate

    data = {}
    for rate in available_rates:
        key = str(available_rates[rate].base_rate)
        current_value = data.get(key, 0)
        if data_load_testing:
            data[key] = "%s" % available_rates[rate].total_points
        else:
            data[key] = current_value + 1

    results = {'data': data, 'timestamp': data_timestamp}
    if return_fees and data:
        fees = Fee.objects.filter(plan__plan_id__in=available_rates.keys(),
                                  state_id=params_data.get('state'))

        if params_data.get('property_type', 'SF') == 'SF':
            fees = fees.filter(single_family=True)
        elif params_data.get('property_type', 'SF') == 'CONDO':
            fees = fees.filter(condo=True)
        elif params_data.get('property_type', 'SF') == 'COOP':
            fees = fees.filter(coop=True)

        averages = fees.aggregate(
            origination_dollar=Avg('origination_dollar'),
            origination_percent=Avg('origination_percent'),
            third_party=Avg('third_party'))

        results['fees'] = averages

    if not data:
        obj = Region.objects.all()[0]
        results['timestamp'] = obj.data_timestamp

    return results

def set_lock_max_min(data):
    """Set max and min lock values before serializer validation"""
    lock_map = {
        '30': (0, 30),
        '45': (31, 45),
        '60': (46, 60)
    }
    lock = data.get('lock')
    if lock and lock in lock_map:
        data['min_lock'] = lock_map[lock][0]
        data['max_lock'] = lock_map[lock][1]
        return data
    else:
        return data

@api_view(['GET'])
def rate_checker(request):
    """
    Return available rates in percentage and number of institutions
    with the corresponding rate
    (i.e. "4.75": 2 means there are 2 institutions with the rate of 4.75%)
    """

    if request.method == 'GET':

        # Clean the parameters, make sure no leading or trailing spaces,
        # transform them to upper cases
        fixed_data = dict(map(
            lambda (k, v): (k, v.strip().upper()),
            request.query_params.iteritems()))
        fixed_data = set_lock_max_min(fixed_data)
        serializer = ParamsSerializer(data=fixed_data)

        if serializer.is_valid():
            rate_results = get_rates(serializer.validated_data)
            rate_results['request'] = serializer.validated_data
            return Response(rate_results)
        else:
            return Response(serializer.errors,
                            status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET'])
def rate_checker_fees(request):
    """
    Return available rates in percentage and number of institutions
    with the corresponding rate along with fees data
    """

    if request.method == 'GET':
        # Clean the parameters, make sure no leading or trailing spaces,
        # transform them to upper cases
        fixed_data = dict(map(
            lambda (k, v): (k, v.strip().upper()),
            request.query_params.iteritems()))
        serializer = ParamsSerializer(data=fixed_data)

        if serializer.is_valid():
            rate_results = get_rates(
                serializer.validated_data, return_fees=True)
            rate_results['request'] = serializer.validated_data
            return Response(rate_results)
        else:
            return Response(
                serializer.errors, status=status.HTTP_400_BAD_REQUEST
            )

class RateCheckerStatus(APIView):
    def get(self, request, format=None):
        try:
            load_ts = Region.objects.latest('data_timestamp').data_timestamp
        except Region.DoesNotExist:
            load_ts = None

        return Response({'load': load_ts})
<commit_msg>Prevent IndexError when building date stamp<commit_after>from django.db.models import Q, Sum, Avg

from rest_framework import status
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework.views import APIView

from ratechecker.models import Region, Rate, Adjustment, Fee
from ratechecker.ratechecker_parameters import ParamsSerializer

def get_rates(params_data, data_load_testing=False, return_fees=False):
    """ params_data is a method parameter of type RateCheckerParameters."""

    # the precalculated results are done by favoring negative points over
    # positive ones, and the API does the opposite
    factor = 1
    if data_load_testing:
        factor = -1

    region_ids = list(Region.objects.filter(
        state_id=params_data.get('state')).values_list('region_id', flat=True))

    rates = Rate.objects.filter(
        region_id__in=region_ids,
        product__loan_purpose=params_data.get('loan_purpose'),
        product__pmt_type=params_data.get('rate_structure'),
        product__loan_type=params_data.get('loan_type'),
        product__max_ltv__gte=params_data.get('max_ltv'),
        product__loan_term=params_data.get('loan_term'),
        product__max_loan_amt__gte=params_data.get('loan_amount'),
        product__max_fico__gte=params_data.get('maxfico'),
        product__min_fico__lte=params_data.get('minfico'))

    if params_data.get('loan_type') != 'FHA-HB':
        rates = rates.filter(
            product__min_loan_amt__lte=params_data.get('loan_amount'))

    if params_data.get('rate_structure') == 'ARM':
        rates = rates.filter(
            product__int_adj_term=params_data.get('arm_type')[:-2],
            product__io=bool(params_data.get('io')))

    if data_load_testing:
        rates = rates.filter(
            product__institution=params_data.get('institution'),
            lock=params_data.get('lock'))
    else:
        rates = rates.filter(
            lock__lte=params_data.get('max_lock', 0),
            lock__gt=params_data.get('min_lock', 0))

    all_rates = []
    products = {}
    for rate in rates:
        all_rates.append(rate)
        products["{}{}".format(
            rate.product_id, rate.region_id)] = rate.product_id
    product_ids = products.values()

    adjustments = Adjustment.objects.filter(
        product__plan_id__in=product_ids).filter(
        Q(max_loan_amt__gte=params_data.get('loan_amount'))
            | Q(max_loan_amt__isnull=True),
        Q(min_loan_amt__lte=params_data.get('loan_amount'))
            | Q(min_loan_amt__isnull=True),
        Q(prop_type=params_data.get('property_type'))
            | Q(prop_type__isnull=True) | Q(prop_type=""),
        Q(state=params_data.get('state'))
            | Q(state__isnull=True) | Q(state=""),
        Q(max_fico__gte=params_data.get('maxfico'))
            | Q(max_fico__isnull=True),
        Q(min_fico__lte=params_data.get('minfico'))
            | Q(min_fico__isnull=True),
        Q(min_ltv__lte=params_data.get('min_ltv'))
            | Q(min_ltv__isnull=True),
        Q(max_ltv__gte=params_data.get('max_ltv'))
            | Q(max_ltv__isnull=True),
    ).values('product_id',
             'affect_rate_type').annotate(sum_of_adjvalue=Sum('adj_value'))

    summed_adj_dict = {}
    for adj in adjustments:
        current = summed_adj_dict.get(adj['product_id'], {})
        current[adj['affect_rate_type']] = adj['sum_of_adjvalue']
        summed_adj_dict[adj['product_id']] = current
    available_rates = {}
    data_timestamp = ""
    for rate in all_rates:
        # TODO: check that it the same all the time, and do what if it is not?
        data_timestamp = rate.data_timestamp
        product = summed_adj_dict.get(rate.product_id, {})
        rate.total_points += product.get('P', 0)
        rate.base_rate += product.get('R', 0)
        distance = abs(params_data.get('points') - rate.total_points)
        if float(distance) > 0.5:
            continue
        if rate.product_id not in available_rates:
            available_rates[rate.product_id] = rate
        else:
            current_difference = abs(
                params_data.get('points') -
                available_rates[rate.product_id].total_points
            )
            new_difference = abs(params_data.get('points') - rate.total_points)
            if new_difference < current_difference or (
                    new_difference == current_difference and
                    factor * available_rates[
                        rate.product_id].total_points < 0 and
                    factor * rate.total_points > 0):
                available_rates[rate.product_id] = rate

    data = {}
    for rate in available_rates:
        key = str(available_rates[rate].base_rate)
        current_value = data.get(key, 0)
        if data_load_testing:
            data[key] = "%s" % available_rates[rate].total_points
        else:
            data[key] = current_value + 1

    results = {'data': data, 'timestamp': data_timestamp}
    if return_fees and data:
        fees = Fee.objects.filter(plan__plan_id__in=available_rates.keys(),
                                  state_id=params_data.get('state'))

        if params_data.get('property_type', 'SF') == 'SF':
            fees = fees.filter(single_family=True)
        elif params_data.get('property_type', 'SF') == 'CONDO':
            fees = fees.filter(condo=True)
        elif params_data.get('property_type', 'SF') == 'COOP':
            fees = fees.filter(coop=True)

        averages = fees.aggregate(
            origination_dollar=Avg('origination_dollar'),
            origination_percent=Avg('origination_percent'),
            third_party=Avg('third_party'))

        results['fees'] = averages

    if not data:
        obj = Region.objects.first()
        if obj:
            results['timestamp'] = obj.data_timestamp

    return results

def set_lock_max_min(data):
    """Set max and min lock values before serializer validation"""
    lock_map = {
        '30': (0, 30),
        '45': (31, 45),
        '60': (46, 60)
    }
    lock = data.get('lock')
    if lock and lock in lock_map:
        data['min_lock'] = lock_map[lock][0]
        data['max_lock'] = lock_map[lock][1]
        return data
    else:
        return data

@api_view(['GET'])
def rate_checker(request):
    """
    Return available rates in percentage and number of institutions
    with the corresponding rate
    (i.e. "4.75": 2 means there are 2 institutions with the rate of 4.75%)
    """

    if request.method == 'GET':

        # Clean the parameters, make sure no leading or trailing spaces,
        # transform them to upper cases
        fixed_data = dict(map(
            lambda (k, v): (k, v.strip().upper()),
            request.query_params.iteritems()))
        fixed_data = set_lock_max_min(fixed_data)
        serializer = ParamsSerializer(data=fixed_data)

        if serializer.is_valid():
            rate_results = get_rates(serializer.validated_data)
            rate_results['request'] = serializer.validated_data
            return Response(rate_results)
        else:
            return Response(serializer.errors,
                            status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET'])
def rate_checker_fees(request):
    """
    Return available rates in percentage and number of institutions
    with the corresponding rate along with fees data
    """

    if request.method == 'GET':
        # Clean the parameters, make sure no leading or trailing spaces,
        # transform them to upper cases
        fixed_data = dict(map(
            lambda (k, v): (k, v.strip().upper()),
            request.query_params.iteritems()))
        serializer = ParamsSerializer(data=fixed_data)

        if serializer.is_valid():
            rate_results = get_rates(
                serializer.validated_data, return_fees=True)
            rate_results['request'] = serializer.validated_data
            return Response(rate_results)
        else:
            return Response(
                serializer.errors, status=status.HTTP_400_BAD_REQUEST
            )

class RateCheckerStatus(APIView):
    def get(self, request, format=None):
        try:
            load_ts = Region.objects.latest('data_timestamp').data_timestamp
        except Region.DoesNotExist:
            load_ts = None

        return Response({'load': load_ts})
<|endoftext|>


<commit_before>import threading
from logging import getLogger
from os import urandom
from hashlib import sha1

from redis import StrictRedis
from redis.exceptions import NoScriptError

__version__ = "2.2.0"

logger = getLogger(__name__)

UNLOCK_SCRIPT = b"""
    if redis.call("get", KEYS[1]) == ARGV[1] then
        redis.call("del", KEYS[2])
        redis.call("lpush", KEYS[2], 1)
        return redis.call("del", KEYS[1])
    else
        return 0
    end
"""
UNLOCK_SCRIPT_HASH = sha1(UNLOCK_SCRIPT).hexdigest()

class AlreadyAcquired(RuntimeError):
    pass

class NotAcquired(RuntimeError):
    pass

class AlreadyStarted(RuntimeError):
    pass

class TimeoutNotUsable(RuntimeError):
    pass

class InvalidTimeout(RuntimeError):
    pass

class TimeoutTooLarge(RuntimeError):
    pass

class Lock(object):
    """
    A Lock context manager implemented via redis SETNX/BLPOP.
    """

    def __init__(self, redis_client, name, expire=None, id=None, auto_renewal=False):
        """
        :param redis_client:
            An instance of :class:`~StrictRedis`.
        :param name:
            The name (redis key) the lock should have.
        :param expire:
            The lock expiry time in seconds. If left at the default (None)
            the lock will not expire.
        :param id:
            The ID (redis value) the lock should have. A random value is
            generated when left at the default.
        :param auto_renewal:
            If set to True, Lock will automatically renew the lock so that it
            doesn't expire for as long as the lock is held (acquire() called
            or running in a context manager).

            Implementation note: Renewal will happen using a daemon thread with
            an interval of expire*2/3. If wishing to use a different renewal
            time, subclass Lock, call super().__init__() then set
            self._lock_renewal_interval to your desired interval.
        """
        assert isinstance(redis_client, StrictRedis)
        if auto_renewal and expire is None:
            raise ValueError("Expire may not be None when auto_renewal is set")

        self._client = redis_client
        self._expire = expire if expire is None else int(expire)
        self._id = urandom(16) if id is None else id
        self._held = False
        self._name = 'lock:'+name
        self._signal = 'lock-signal:'+name
        self._lock_renewal_interval = expire*2/3 if auto_renewal else None
        self._lock_renewal_thread = None

    def reset(self):
        """
        Forcibly deletes the lock. Use this with care.
        """
        self._client.delete(self._name)
        self._client.delete(self._signal)

    @property
    def id(self):
        return self._id

    def get_owner_id(self):
        return self._client.get(self._name)

    def acquire(self, blocking=True, timeout=None):
        """
        :param blocking:
            Boolean value specifying whether lock should be blocking or not.
        :param timeout:
            An integer value specifying the maximum number of seconds to block.
        """
        logger.debug("Getting %r ...", self._name)

        if self._held:
            raise AlreadyAcquired("Already acquired from this Lock instance.")

        if not blocking and timeout is not None:
            raise TimeoutNotUsable("Timeout cannot be used if blocking=False")

        timeout = timeout if timeout is None else int(timeout)
        if timeout is not None and timeout <= 0:
            raise InvalidTimeout("Timeout (%d) cannot be less than or equal to 0" % timeout)

        if timeout and self._expire and timeout > self._expire:
            raise TimeoutTooLarge("Timeout (%d) cannot be greater than expire (%d)" % (timeout, self._expire))

        busy = True
        blpop_timeout = timeout or self._expire or 0
        while busy:
            busy = not self._client.set(self._name, self._id, nx=True, ex=self._expire)
            if busy:
                if blocking:
                    timed_out = not self._client.blpop(self._signal, blpop_timeout)
                    if timeout and timed_out:
                        return False
                else:
                    logger.debug("Failed to get %r.", self._name)
                    return False

        logger.debug("Got lock for %r.", self._name)
        self._held = True
        if self._lock_renewal_interval is not None:
            self._start_lock_renewer()
        return True

    def _lock_renewer(self, interval):
        """
        Renew the lock key in redis every `interval` seconds for as long
        as `self._lock_renewal_thread.should_exit` is False.
        """
        log = getLogger("%s.lock_refresher" % __name__)
        while not self._lock_renewal_thread.wait_for_exit_request(timeout=interval):
            log.debug("Refreshing lock")
            self._client.set(self._name, self._id, xx=True, ex=self._expire)
        log.debug("Exit requested, stopping lock refreshing")

    def _start_lock_renewer(self):
        """
        Starts the lock refresher thread.
        """
        if self._lock_renewal_thread is not None:
            raise AlreadyStarted("Lock refresh thread already started")

        logger.debug(
            "Starting thread to refresh lock every %s seconds",
            self._lock_renewal_interval
        )
        self._lock_renewal_thread = InterruptableThread(
            group=None,
            target=self._lock_renewer,
            kwargs={'interval': self._lock_renewal_interval}
        )
        self._lock_renewal_thread.setDaemon(True)
        self._lock_renewal_thread.start()

    def _stop_lock_renewer(self):
        """
        Stop the lock renewer.

        This signals the renewal thread and waits for its exit.
        """
        if self._lock_renewal_thread is None or not self._lock_renewal_thread.is_alive():
            return
        logger.debug("Signalling the lock refresher to stop")
        self._lock_renewal_thread.request_exit()
        self._lock_renewal_thread.join()
        self._lock_renewal_thread = None
        logger.debug("Lock refresher has stopped")

    def __enter__(self):
        assert self.acquire(blocking=True)
        return self

    def __exit__(self, exc_type=None, exc_value=None, traceback=None, force=False):
        if not (self._held or force):
            raise NotAcquired("This Lock instance didn't acquire the lock.")
        if self._lock_renewal_thread is not None:
            self._stop_lock_renewer()
        logger.debug("Releasing %r.", self._name)
        try:
            self._client.evalsha(UNLOCK_SCRIPT_HASH, 2, self._name, self._signal, self._id)
        except NoScriptError:
            logger.warn("UNLOCK_SCRIPT not cached.")
            self._client.eval(UNLOCK_SCRIPT, 2, self._name, self._signal, self._id)
        self._held = False
    release = __exit__

class InterruptableThread(threading.Thread):
    """
    A Python thread that can be requested to stop by calling request_exit()
    on it.

    Code running inside this thread should periodically check the
    `should_exit` property (or use wait_for_exit_request) on the thread
    object and stop further processing once it returns True.
    """
    def __init__(self, *args, **kwargs):
        self._should_exit = threading.Event()
        super(InterruptableThread, self).__init__(*args, **kwargs)

    def request_exit(self):
        """
        Signal the thread that it should stop performing more work and exit.
        """
        self._should_exit.set()

    @property
    def should_exit(self):
        return self._should_exit.isSet()

    def wait_for_exit_request(self, timeout=None):
        """
        Wait until the thread has been signalled to exit.

        If timeout is specified (as a float of seconds to wait) then wait
        up to this many seconds before returning the value of `should_exit`.
        """
        should_exit = self._should_exit.wait(timeout)
        if should_exit is None:
            # Python 2.6 compatibility which doesn't return self.__flag when
            # calling Event.wait()
            should_exit = self.should_exit
        return should_exit

def reset_all(redis_client):
    """
    Forcibly deletes all locks if its remains (like a crash reason). Use this with care.
    """
    for lock_key in redis_client.keys('lock:*'):
        redis_client.delete(lock_key)
    for lock_key in redis_client.keys('lock-signal:*'):
        redis_client.delete(lock_key)
<commit_msg>Improve assertion here so that it doesn't cause change of behavior if assertions are disabled.<commit_after>import threading
from logging import getLogger
from os import urandom
from hashlib import sha1

from redis import StrictRedis
from redis.exceptions import NoScriptError

__version__ = "2.2.0"

logger = getLogger(__name__)

UNLOCK_SCRIPT = b"""
    if redis.call("get", KEYS[1]) == ARGV[1] then
        redis.call("del", KEYS[2])
        redis.call("lpush", KEYS[2], 1)
        return redis.call("del", KEYS[1])
    else
        return 0
    end
"""
UNLOCK_SCRIPT_HASH = sha1(UNLOCK_SCRIPT).hexdigest()

class AlreadyAcquired(RuntimeError):
    pass

class NotAcquired(RuntimeError):
    pass

class AlreadyStarted(RuntimeError):
    pass

class TimeoutNotUsable(RuntimeError):
    pass

class InvalidTimeout(RuntimeError):
    pass

class TimeoutTooLarge(RuntimeError):
    pass

class Lock(object):
    """
    A Lock context manager implemented via redis SETNX/BLPOP.
    """

    def __init__(self, redis_client, name, expire=None, id=None, auto_renewal=False):
        """
        :param redis_client:
            An instance of :class:`~StrictRedis`.
        :param name:
            The name (redis key) the lock should have.
        :param expire:
            The lock expiry time in seconds. If left at the default (None)
            the lock will not expire.
        :param id:
            The ID (redis value) the lock should have. A random value is
            generated when left at the default.
        :param auto_renewal:
            If set to True, Lock will automatically renew the lock so that it
            doesn't expire for as long as the lock is held (acquire() called
            or running in a context manager).

            Implementation note: Renewal will happen using a daemon thread with
            an interval of expire*2/3. If wishing to use a different renewal
            time, subclass Lock, call super().__init__() then set
            self._lock_renewal_interval to your desired interval.
        """
        assert isinstance(redis_client, StrictRedis)
        if auto_renewal and expire is None:
            raise ValueError("Expire may not be None when auto_renewal is set")

        self._client = redis_client
        self._expire = expire if expire is None else int(expire)
        self._id = urandom(16) if id is None else id
        self._held = False
        self._name = 'lock:'+name
        self._signal = 'lock-signal:'+name
        self._lock_renewal_interval = expire*2/3 if auto_renewal else None
        self._lock_renewal_thread = None

    def reset(self):
        """
        Forcibly deletes the lock. Use this with care.
        """
        self._client.delete(self._name)
        self._client.delete(self._signal)

    @property
    def id(self):
        return self._id

    def get_owner_id(self):
        return self._client.get(self._name)

    def acquire(self, blocking=True, timeout=None):
        """
        :param blocking:
            Boolean value specifying whether lock should be blocking or not.
        :param timeout:
            An integer value specifying the maximum number of seconds to block.
        """
        logger.debug("Getting %r ...", self._name)

        if self._held:
            raise AlreadyAcquired("Already acquired from this Lock instance.")

        if not blocking and timeout is not None:
            raise TimeoutNotUsable("Timeout cannot be used if blocking=False")

        timeout = timeout if timeout is None else int(timeout)
        if timeout is not None and timeout <= 0:
            raise InvalidTimeout("Timeout (%d) cannot be less than or equal to 0" % timeout)

        if timeout and self._expire and timeout > self._expire:
            raise TimeoutTooLarge("Timeout (%d) cannot be greater than expire (%d)" % (timeout, self._expire))

        busy = True
        blpop_timeout = timeout or self._expire or 0
        while busy:
            busy = not self._client.set(self._name, self._id, nx=True, ex=self._expire)
            if busy:
                if blocking:
                    timed_out = not self._client.blpop(self._signal, blpop_timeout)
                    if timeout and timed_out:
                        return False
                else:
                    logger.debug("Failed to get %r.", self._name)
                    return False

        logger.debug("Got lock for %r.", self._name)
        self._held = True
        if self._lock_renewal_interval is not None:
            self._start_lock_renewer()
        return True

    def _lock_renewer(self, interval):
        """
        Renew the lock key in redis every `interval` seconds for as long
        as `self._lock_renewal_thread.should_exit` is False.
        """
        log = getLogger("%s.lock_refresher" % __name__)
        while not self._lock_renewal_thread.wait_for_exit_request(timeout=interval):
            log.debug("Refreshing lock")
            self._client.set(self._name, self._id, xx=True, ex=self._expire)
        log.debug("Exit requested, stopping lock refreshing")

    def _start_lock_renewer(self):
        """
        Starts the lock refresher thread.
        """
        if self._lock_renewal_thread is not None:
            raise AlreadyStarted("Lock refresh thread already started")

        logger.debug(
            "Starting thread to refresh lock every %s seconds",
            self._lock_renewal_interval
        )
        self._lock_renewal_thread = InterruptableThread(
            group=None,
            target=self._lock_renewer,
            kwargs={'interval': self._lock_renewal_interval}
        )
        self._lock_renewal_thread.setDaemon(True)
        self._lock_renewal_thread.start()

    def _stop_lock_renewer(self):
        """
        Stop the lock renewer.

        This signals the renewal thread and waits for its exit.
        """
        if self._lock_renewal_thread is None or not self._lock_renewal_thread.is_alive():
            return
        logger.debug("Signalling the lock refresher to stop")
        self._lock_renewal_thread.request_exit()
        self._lock_renewal_thread.join()
        self._lock_renewal_thread = None
        logger.debug("Lock refresher has stopped")

    def __enter__(self):
        acquired = self.acquire(blocking=True)
        assert acquired, "Lock wasn't acquired, but blocking=True"
        return self

    def __exit__(self, exc_type=None, exc_value=None, traceback=None, force=False):
        if not (self._held or force):
            raise NotAcquired("This Lock instance didn't acquire the lock.")
        if self._lock_renewal_thread is not None:
            self._stop_lock_renewer()
        logger.debug("Releasing %r.", self._name)
        try:
            self._client.evalsha(UNLOCK_SCRIPT_HASH, 2, self._name, self._signal, self._id)
        except NoScriptError:
            logger.warn("UNLOCK_SCRIPT not cached.")
            self._client.eval(UNLOCK_SCRIPT, 2, self._name, self._signal, self._id)
        self._held = False
    release = __exit__

class InterruptableThread(threading.Thread):
    """
    A Python thread that can be requested to stop by calling request_exit()
    on it.

    Code running inside this thread should periodically check the
    `should_exit` property (or use wait_for_exit_request) on the thread
    object and stop further processing once it returns True.
    """
    def __init__(self, *args, **kwargs):
        self._should_exit = threading.Event()
        super(InterruptableThread, self).__init__(*args, **kwargs)

    def request_exit(self):
        """
        Signal the thread that it should stop performing more work and exit.
        """
        self._should_exit.set()

    @property
    def should_exit(self):
        return self._should_exit.isSet()

    def wait_for_exit_request(self, timeout=None):
        """
        Wait until the thread has been signalled to exit.

        If timeout is specified (as a float of seconds to wait) then wait
        up to this many seconds before returning the value of `should_exit`.
        """
        should_exit = self._should_exit.wait(timeout)
        if should_exit is None:
            # Python 2.6 compatibility which doesn't return self.__flag when
            # calling Event.wait()
            should_exit = self.should_exit
        return should_exit

def reset_all(redis_client):
    """
    Forcibly deletes all locks if its remains (like a crash reason). Use this with care.
    """
    for lock_key in redis_client.keys('lock:*'):
        redis_client.delete(lock_key)
    for lock_key in redis_client.keys('lock-signal:*'):
        redis_client.delete(lock_key)
<|endoftext|>


<commit_before>
import numpy
import pyopencl, pyopencl.array

def make_vertices_np(nr, max_val=1024):
    """
    Generates a set of vertices as nr-tuple of 2-tuple of integers
    """
    return numpy.random.randint(0, max_val, nr * 2).reshape((nr, 2)).astype(numpy.float32)

if __name__ == "__main__":
    import time
    N = 24
    L = 1024
    vertices = make_vertices_np(N, L)
    print vertices

    ctx = pyopencl.create_some_context()
    queue = pyopencl.CommandQueue(ctx,
                properties=pyopencl.command_queue_properties.PROFILING_ENABLE)
    src = open("inside_polygon.cl").read()
    prg = pyopencl.Program(ctx, src).build()

    t0 = time.time()
    d_vertices = pyopencl.array.to_device(queue, vertices)
    d_result = pyopencl.array.empty(queue, (L,L), numpy.uint8)
    evt = prg.insidePolygon(queue, (L,L), (32,1),
                            d_vertices.data, numpy.int32(N), numpy.uint8(1), d_result.data)
    res = d_result.get()
    print("execution time OpenCL: %.3fs Actual kernel execution time: %.6fs" % (time.time() - t0, 1e-9 * (evt.profile.end - evt.profile.start)))
    msk = res

    import pylab
    pylab.imshow(msk)
    last = vertices[-1]
    for v in vertices:
        pylab.annotate("", xy=v, xytext=last, xycoords="data",
                 arrowprops=dict(arrowstyle="-"))
        last = v
    pylab.show()
<commit_msg>vary workgroup size<commit_after>
import numpy
import pyopencl, pyopencl.array

def make_vertices_np(nr, max_val=1024):
    """
    Generates a set of vertices as nr-tuple of 2-tuple of integers
    """
    return numpy.random.randint(0, max_val, nr * 2).reshape((nr, 2)).astype(numpy.float32)

if __name__ == "__main__":
    import time
    N = 24
    L = 1024
    vertices = make_vertices_np(N, L)
    print vertices

    ctx = pyopencl.create_some_context()
    queue = pyopencl.CommandQueue(ctx,
                properties=pyopencl.command_queue_properties.PROFILING_ENABLE)
    src = open("inside_polygon.cl").read()
    prg = pyopencl.Program(ctx, src).build()

    for i in range(10):
        j = 2 ** i
        t0 = time.time()
        d_vertices = pyopencl.array.to_device(queue, vertices)
        d_result = pyopencl.array.empty(queue, (L, L), numpy.uint8)
        evt = prg.insidePolygon(queue, (L, L), (j, 1),
                                d_vertices.data, numpy.int32(N), numpy.uint8(1), d_result.data)
        res = d_result.get()
        print("execution time OpenCL all: %.3fs kernel: %.6fs, workgoup: %i" % (time.time() - t0, 1e-9 * (evt.profile.end - evt.profile.start), j))
    msk = res

    import pylab
    pylab.imshow(msk)
    last = vertices[-1]
    for v in vertices:
        pylab.annotate("", xy=v, xytext=last, xycoords="data",
                 arrowprops=dict(arrowstyle="-"))
        last = v
    pylab.show()
<|endoftext|>