<commit_before>import numpy as np
from numpy import pi, cos

from .generate import F64

class PyModel(object):
    def __init__(self, info):
        self.info = info

    def __call__(self, q_input):
        kernel = self.info['Iqxy'] if q_input.is_2D else self.info['Iq']
        return PyKernel(kernel, self.info, q_input)

    # pylint: disable=no-self-use
    def make_input(self, q_vectors):
        return PyInput(q_vectors, dtype=F64)

    def release(self):
        pass

class PyInput(object):
    """
    Make q data available to the gpu.

    *q_vectors* is a list of q vectors, which will be *[q]* for 1-D data,
    and *[qx, qy]* for 2-D data.  Internally, the vectors will be reallocated
    to get the best performance on OpenCL, which may involve shifting and
    stretching the array to better match the memory architecture.  Additional
    points will be evaluated with *q=1e-3*.

    *dtype* is the data type for the q vectors. The data type should be
    set to match that of the kernel, which is an attribute of
    :class:`GpuProgram`.  Note that not all kernels support double
    precision, so even if the program was created for double precision,
    the *GpuProgram.dtype* may be single precision.

    Call :meth:`release` when complete.  Even if not called directly, the
    buffer will be released when the data object is freed.
    """
    def __init__(self, q_vectors, dtype):
        self.nq = q_vectors[0].size
        self.dtype = dtype
        self.is_2D = (len(q_vectors) == 2)
        self.q_vectors = [np.ascontiguousarray(q, self.dtype) for q in q_vectors]
        self.q_pointers = [q.ctypes.data for q in self.q_vectors]

    def release(self):
        self.q_vectors = []

class PyKernel(object):
    """
    Callable SAS kernel.

    *kernel* is the DllKernel object to call.

    *info* is the module information

    *q_input* is the DllInput q vectors at which the kernel should be
    evaluated.

    The resulting call method takes the *pars*, a list of values for
    the fixed parameters to the kernel, and *pd_pars*, a list of (value,weight)
    vectors for the polydisperse parameters.  *cutoff* determines the
    integration limits: any points with combined weight less than *cutoff*
    will not be calculated.

    Call :meth:`release` when done with the kernel instance.
    """
    def __init__(self, kernel, info, q_input):
        self.info = info
        self.q_input = q_input
        self.res = np.empty(q_input.nq, q_input.dtype)
        dim = '2d' if q_input.is_2D else '1d'
        # Loop over q unless user promises that the kernel is vectorized by
        # taggining it with vectorized=True
        if not getattr(kernel, 'vectorized', False):
            if dim == '2d':
                def vector_kernel(qx, qy, *args):
                    return np.array([kernel(qxi, qyi, *args)
                                     for qxi, qyi in zip(qx, qy)])
            else:
                def vector_kernel(q, *args):
                    return np.array([kernel(qi, *args)
                                     for qi in q])
            self.kernel = vector_kernel
        else:
            self.kernel = kernel
        fixed_pars = info['partype']['fixed-' + dim]
        pd_pars = info['partype']['pd-' + dim]
        vol_pars = info['partype']['volume']

        # First two fixed pars are scale and background
        pars = [p[0] for p in info['parameters'][2:]]
        offset = len(self.q_input.q_vectors)
        self.args = self.q_input.q_vectors + [None] * len(pars)
        self.fixed_index = np.array([pars.index(p) + offset
                                     for p in fixed_pars[2:]])
        self.pd_index = np.array([pars.index(p) + offset
                                  for p in pd_pars])
        self.vol_index = np.array([pars.index(p) + offset
                                   for p in vol_pars])
        try: self.theta_index = pars.index('theta') + offset
        except ValueError: self.theta_index = -1

        # Caller needs fixed_pars and pd_pars
        self.fixed_pars = fixed_pars
        self.pd_pars = pd_pars

    def __call__(self, fixed, pd, cutoff=1e-5):
        #print("fixed",fixed)
        #print("pd", pd)
        args = self.args[:]  # grab a copy of the args
        form, form_volume = self.kernel, self.info['form_volume']
        # First two fixed
        scale, background = fixed[:2]
        for index, value in zip(self.fixed_index, fixed[2:]):
            args[index] = float(value)
        res = _loops(form, form_volume, cutoff, scale, background, args,
                     pd, self.pd_index, self.vol_index, self.theta_index)

        return res

    def release(self):
        self.q_input = None

def _loops(form, form_volume, cutoff, scale, background,
           args, pd, pd_index, vol_index, theta_index):
    """
    *form* is the name of the form function, which should be vectorized over
    q, but otherwise have an interface like the opencl kernels, with the
    q parameters first followed by the individual parameters in the order
    given in model.parameters (see :mod:`sasmodels.generate`).

    *form_volume* calculates the volume of the shape.  *vol_index* gives
    the list of volume parameters

    *cutoff* ignores the corners of the dispersion hypercube

    *scale*, *background* multiplies the resulting form and adds an offset

    *args* is the prepopulated set of arguments to the form function, starting
    with the q vectors, and including slots for all the parameters.  The
    values for the parameters will be substituted with values from the
    dispersion functions.

    *pd* is the list of dispersion parameters

    *pd_index* are the indices of the dispersion parameters in *args*

    *vol_index* are the indices of the volume parameters in *args*

    *theta_index* is the index of the theta parameter for the sasview
    spherical correction, or -1 if there is no angular dispersion
    """

    ################################################################
    #                                                              #
    #   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   #
    #   !!                                                    !!   #
    #   !!  KEEP THIS CODE CONSISTENT WITH KERNEL_TEMPLATE.C  !!   #
    #   !!                                                    !!   #
    #   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   #
    #                                                              #
    ################################################################

    weight = np.empty(len(pd), 'd')
    if weight.size > 0:
        # weight vector, to be populated by polydispersity loops

        # identify which pd parameters are volume parameters
        vol_weight_index = np.array([(index in vol_index) for index in pd_index])

        # Sort parameters in decreasing order of pd length
        Npd = np.array([len(pdi[0]) for pdi in pd], 'i')
        order = np.argsort(Npd)[::-1]
        stride = np.cumprod(Npd[order])
        pd = [pd[index] for index in order]
        pd_index = pd_index[order]
        vol_weight_index = vol_weight_index[order]

        fast_value = pd[0][0]
        fast_weight = pd[0][1]
    else:
        stride = np.array([1])
        vol_weight_index = slice(None, None)
        # keep lint happy
        fast_value = [None]
        fast_weight = [None]

    ret = np.zeros_like(args[0])
    norm = np.zeros_like(ret)
    vol = np.zeros_like(ret)
    vol_norm = np.zeros_like(ret)
    for k in range(stride[-1]):
        # update polydispersity parameter values
        fast_index = k % stride[0]
        if fast_index == 0:  # bottom loop complete ... check all other loops
            if weight.size > 0:
                for i, index, in enumerate(k % stride):
                    args[pd_index[i]] = pd[i][0][index]
                    weight[i] = pd[i][1][index]
        else:
            args[pd_index[0]] = fast_value[fast_index]
            weight[0] = fast_weight[fast_index]
        # This computes the weight, and if it is sufficient, calls the
        # scattering function and adds it to the total.  If there is a volume
        # normalization, it will also be added here.
        # Note: make sure this is consistent with the code in PY_LOOP_BODY!!
        # Note: can precompute w1*w2*...*wn
        w = np.prod(weight)
        if w > cutoff:
            I = form(*args)
            #positive = (I >= 0.0)

            # Note: can precompute spherical correction if theta_index is not
            # the fast index. Correction factor for spherical integration
            #spherical_correction = abs(cos(pi*args[phi_index])) if phi_index>=0 else 1.0
            spherical_correction = (abs(cos(pi * args[theta_index])) * pi / 2
                                    if theta_index >= 0 else 1.0)
            #spherical_correction = 1.0
            ret += w * I * spherical_correction #* positive
            norm += w #* positive

            # Volume normalization.
            # If there are "volume" polydispersity parameters, then these
            # will be used to call the form_volume function from the user
            # supplied kernel, and accumulate a normalized weight.
            # Note: can precompute volume norm if fast index is not a volume
            if form_volume:
                vol_args = [args[index] for index in vol_index]
                vol_weight = np.prod(weight[vol_weight_index])
                vol += vol_weight * form_volume(*vol_args) #* positive
                vol_norm += vol_weight #* positive

    positive = (vol * vol_norm != 0.0)
    ret[positive] *= vol_norm[positive] / vol[positive]
    result = scale * ret / norm + background
    return result
<commit_msg>return NAN to signal invalid kernel parameters (for python as well)<commit_after>import numpy as np
from numpy import pi, cos

from .generate import F64

class PyModel(object):
    def __init__(self, info):
        self.info = info

    def __call__(self, q_input):
        kernel = self.info['Iqxy'] if q_input.is_2D else self.info['Iq']
        return PyKernel(kernel, self.info, q_input)

    # pylint: disable=no-self-use
    def make_input(self, q_vectors):
        return PyInput(q_vectors, dtype=F64)

    def release(self):
        pass

class PyInput(object):
    """
    Make q data available to the gpu.

    *q_vectors* is a list of q vectors, which will be *[q]* for 1-D data,
    and *[qx, qy]* for 2-D data.  Internally, the vectors will be reallocated
    to get the best performance on OpenCL, which may involve shifting and
    stretching the array to better match the memory architecture.  Additional
    points will be evaluated with *q=1e-3*.

    *dtype* is the data type for the q vectors. The data type should be
    set to match that of the kernel, which is an attribute of
    :class:`GpuProgram`.  Note that not all kernels support double
    precision, so even if the program was created for double precision,
    the *GpuProgram.dtype* may be single precision.

    Call :meth:`release` when complete.  Even if not called directly, the
    buffer will be released when the data object is freed.
    """
    def __init__(self, q_vectors, dtype):
        self.nq = q_vectors[0].size
        self.dtype = dtype
        self.is_2D = (len(q_vectors) == 2)
        self.q_vectors = [np.ascontiguousarray(q, self.dtype) for q in q_vectors]
        self.q_pointers = [q.ctypes.data for q in self.q_vectors]

    def release(self):
        self.q_vectors = []

class PyKernel(object):
    """
    Callable SAS kernel.

    *kernel* is the DllKernel object to call.

    *info* is the module information

    *q_input* is the DllInput q vectors at which the kernel should be
    evaluated.

    The resulting call method takes the *pars*, a list of values for
    the fixed parameters to the kernel, and *pd_pars*, a list of (value,weight)
    vectors for the polydisperse parameters.  *cutoff* determines the
    integration limits: any points with combined weight less than *cutoff*
    will not be calculated.

    Call :meth:`release` when done with the kernel instance.
    """
    def __init__(self, kernel, info, q_input):
        self.info = info
        self.q_input = q_input
        self.res = np.empty(q_input.nq, q_input.dtype)
        dim = '2d' if q_input.is_2D else '1d'
        # Loop over q unless user promises that the kernel is vectorized by
        # taggining it with vectorized=True
        if not getattr(kernel, 'vectorized', False):
            if dim == '2d':
                def vector_kernel(qx, qy, *args):
                    return np.array([kernel(qxi, qyi, *args)
                                     for qxi, qyi in zip(qx, qy)])
            else:
                def vector_kernel(q, *args):
                    return np.array([kernel(qi, *args)
                                     for qi in q])
            self.kernel = vector_kernel
        else:
            self.kernel = kernel
        fixed_pars = info['partype']['fixed-' + dim]
        pd_pars = info['partype']['pd-' + dim]
        vol_pars = info['partype']['volume']

        # First two fixed pars are scale and background
        pars = [p[0] for p in info['parameters'][2:]]
        offset = len(self.q_input.q_vectors)
        self.args = self.q_input.q_vectors + [None] * len(pars)
        self.fixed_index = np.array([pars.index(p) + offset
                                     for p in fixed_pars[2:]])
        self.pd_index = np.array([pars.index(p) + offset
                                  for p in pd_pars])
        self.vol_index = np.array([pars.index(p) + offset
                                   for p in vol_pars])
        try: self.theta_index = pars.index('theta') + offset
        except ValueError: self.theta_index = -1

        # Caller needs fixed_pars and pd_pars
        self.fixed_pars = fixed_pars
        self.pd_pars = pd_pars

    def __call__(self, fixed, pd, cutoff=1e-5):
        #print("fixed",fixed)
        #print("pd", pd)
        args = self.args[:]  # grab a copy of the args
        form, form_volume = self.kernel, self.info['form_volume']
        # First two fixed
        scale, background = fixed[:2]
        for index, value in zip(self.fixed_index, fixed[2:]):
            args[index] = float(value)
        res = _loops(form, form_volume, cutoff, scale, background, args,
                     pd, self.pd_index, self.vol_index, self.theta_index)

        return res

    def release(self):
        self.q_input = None

def _loops(form, form_volume, cutoff, scale, background,
           args, pd, pd_index, vol_index, theta_index):
    """
    *form* is the name of the form function, which should be vectorized over
    q, but otherwise have an interface like the opencl kernels, with the
    q parameters first followed by the individual parameters in the order
    given in model.parameters (see :mod:`sasmodels.generate`).

    *form_volume* calculates the volume of the shape.  *vol_index* gives
    the list of volume parameters

    *cutoff* ignores the corners of the dispersion hypercube

    *scale*, *background* multiplies the resulting form and adds an offset

    *args* is the prepopulated set of arguments to the form function, starting
    with the q vectors, and including slots for all the parameters.  The
    values for the parameters will be substituted with values from the
    dispersion functions.

    *pd* is the list of dispersion parameters

    *pd_index* are the indices of the dispersion parameters in *args*

    *vol_index* are the indices of the volume parameters in *args*

    *theta_index* is the index of the theta parameter for the sasview
    spherical correction, or -1 if there is no angular dispersion
    """

    ################################################################
    #                                                              #
    #   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   #
    #   !!                                                    !!   #
    #   !!  KEEP THIS CODE CONSISTENT WITH KERNEL_TEMPLATE.C  !!   #
    #   !!                                                    !!   #
    #   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   #
    #                                                              #
    ################################################################

    weight = np.empty(len(pd), 'd')
    if weight.size > 0:
        # weight vector, to be populated by polydispersity loops

        # identify which pd parameters are volume parameters
        vol_weight_index = np.array([(index in vol_index) for index in pd_index])

        # Sort parameters in decreasing order of pd length
        Npd = np.array([len(pdi[0]) for pdi in pd], 'i')
        order = np.argsort(Npd)[::-1]
        stride = np.cumprod(Npd[order])
        pd = [pd[index] for index in order]
        pd_index = pd_index[order]
        vol_weight_index = vol_weight_index[order]

        fast_value = pd[0][0]
        fast_weight = pd[0][1]
    else:
        stride = np.array([1])
        vol_weight_index = slice(None, None)
        # keep lint happy
        fast_value = [None]
        fast_weight = [None]

    ret = np.zeros_like(args[0])
    norm = np.zeros_like(ret)
    vol = np.zeros_like(ret)
    vol_norm = np.zeros_like(ret)
    for k in range(stride[-1]):
        # update polydispersity parameter values
        fast_index = k % stride[0]
        if fast_index == 0:  # bottom loop complete ... check all other loops
            if weight.size > 0:
                for i, index, in enumerate(k % stride):
                    args[pd_index[i]] = pd[i][0][index]
                    weight[i] = pd[i][1][index]
        else:
            args[pd_index[0]] = fast_value[fast_index]
            weight[0] = fast_weight[fast_index]

        # Computes the weight, and if it is not sufficient then ignore this
        # parameter set.
        # Note: could precompute w1*...*wn so we only need to multiply by w0
        w = np.prod(weight)
        if w > cutoff:
            # Note: can precompute spherical correction if theta_index is not
            # the fast index. Correction factor for spherical integration
            #spherical_correction = abs(cos(pi*args[phi_index])) if phi_index>=0 else 1.0
            spherical_correction = (abs(cos(pi * args[theta_index])) * pi / 2
                                    if theta_index >= 0 else 1.0)
            #spherical_correction = 1.0

            # Call the scattering function and adds it to the total.
            I = form(*args)
            if np.isnan(I).any(): continue
            ret += w * I * spherical_correction
            norm += w

            # Volume normalization.
            # If there are "volume" polydispersity parameters, then these
            # will be used to call the form_volume function from the user
            # supplied kernel, and accumulate a normalized weight.
            # Note: can precompute volume norm if fast index is not a volume
            if form_volume:
                vol_args = [args[index] for index in vol_index]
                vol_weight = np.prod(weight[vol_weight_index])
                vol += vol_weight * form_volume(*vol_args)
                vol_norm += vol_weight

    positive = (vol * vol_norm != 0.0)
    ret[positive] *= vol_norm[positive] / vol[positive]
    result = scale * ret / norm + background
    return result
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8
import os
from pymongo import MongoClient
import random

class TvrainData():
    def __init__(self):
        """
        Just load data from Mongo.
        """
        self.sequences = MongoClient(os.environ['MONGODB_URL']).tvrain.sequences
        self.collection = MongoClient(os.environ['MONGODB_URL']).tvrain.articles
        self.collection.create_index("time")

    def get_random_articles(self, n):
        """Returns N of topics for index.html"""
        articles = self.collection.find().skip(random.randint(0, self.collection.count())).limit(n)
        return list(articles)

    def get_article_id(self, url):
        """Get id by url"""
        return self.collection.find_one({'url': url})['_id']

    def get_articles_data(self, articles_urls):
        """
        Get data from MongoDB for articles urls
        :param articles_urls: ['article_url', ...]
        :return: list of MongoDB documents
        """
        articles = []
        for url in articles_urls:
            articles.append(self.collection.find_one({'url': url}))
        return articles

    def iterate_articles(self, except_articles, skip=0, limit=None, query={}):
        """
        Iteate throw all articles without ids of except articles
        :param except_articles: list of ids
        :return:
        """
        if limit is None:
            data = self.collection.find(query).skip(skip)
        else:
            data = self.collection.find(query).skip(skip).limit(limit)

        for value in data:
            if value not in except_articles:
                yield value

    def get_sequences(self):
        """Return all sequences for train"""
        return list(self.sequences.find().limit(-1))

<commit_msg>Add log message<commit_after>#!/usr/bin/env python
# coding=utf-8
import os
from pymongo import MongoClient
import random

class TvrainData():
    def __init__(self):
        """
        Just load data from Mongo.
        """
        self.sequences = MongoClient(os.environ['MONGODB_URL']).tvrain.sequences
        self.collection = MongoClient(os.environ['MONGODB_URL']).tvrain.articles
        self.collection.create_index("time")

    def get_random_articles(self, n):
        """Returns N of topics for index.html"""
        articles = self.collection.find().skip(random.randint(0, self.collection.count())).limit(n)
        return list(articles)

    def get_article_id(self, url):
        """Get id by url"""
        return self.collection.find_one({'url': url})['_id']

    def get_articles_data(self, articles_urls):
        """
        Get data from MongoDB for articles urls
        :param articles_urls: ['article_url', ...]
        :return: list of MongoDB documents
        """
        articles = []
        for url in articles_urls:
            articles.append(self.collection.find_one({'url': url}))
        return articles

    def iterate_articles(self, except_articles, skip=0, limit=None, query={}):
        """
        Iteate throw all articles without ids of except articles
        :param except_articles: list of ids
        :return:
        """
        if limit is None:
            data = self.collection.find(query).skip(skip)
        else:
            data = self.collection.find(query).skip(skip).limit(limit)

        print(data.count())

        for value in data:
            if value not in except_articles:
                yield value

    def get_sequences(self):
        """Return all sequences for train"""
        return list(self.sequences.find().limit(-1))

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sublime
import sublime_plugin

import os
import sys
from time import time

PLUGIN_DIR = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.join(PLUGIN_DIR, 'libs'))

from fluent import sender, event

SETTINGS_FILE = 'fluent-logger-sublimetext.sublime-settings'

class FluentLoggerSublimetext(sublime_plugin.EventListener):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.settings = sublime.load_settings(SETTINGS_FILE)
        sender.setup(
            str(self.settings.get('tagprefix')),
            host=str(self.settings.get('host')),
            port=int(self.settings.get('port'))
        )

    def _get_data_from_view(self, view):
        window = view.window()
        return {
            "version": sublime.version(),
            "platform": sublime.platform(),
            "arch": sublime.arch(),
            "file_name": view.file_name(),
            "size": view.size(),
            "viewport_extent": view.viewport_extent(),
            "layout_extent": view.layout_extent(),
            "line_height": view.line_height(),
            "em_width": view.em_width(),
            "change_count": view.change_count(),
            "encoding": view.encoding(),
            "line_endings": view.line_endings(),
            "overwrite_status": view.overwrite_status(),
            "syntax": view.settings().get('syntax'),
            "project": window.project_data() if hasattr(window, 'project_data') else None,
            "folders": window.folders()
        }

    def on_new(self, view):
        event.Event('on_new', self._get_data_from_view(view))

    def on_clone(self, view):
        event.Event('on_clone', self._get_data_from_view(view))

    def on_load(self, view):
        event.Event('on_load', self._get_data_from_view(view))

    def on_pre_close(self, view):
        event.Event('on_pre_close', self._get_data_from_view(view))

    def on_close(self, view):
        event.Event('on_close', self._get_data_from_view(view))

    def on_pre_save(self, view):
        event.Event('on_pre_save', self._get_data_from_view(view))

    def on_post_save(self, view):
        event.Event('on_post_save', self._get_data_from_view(view))

    def on_modified(self, view):
        event.Event('on_modified', self._get_data_from_view(view))

    def on_selection_modified(self, view):
        event.Event('on_selection_modified', self._get_data_from_view(view))

    def on_activated(self, view):
        self.active_from = time()
        event.Event('on_activated', self._get_data_from_view(view))

    def on_deactivated(self, view):
        data = self._get_data_from_view(view)
        active_time = time() - self.active_from
        data['active_time'] = active_time
        self.active_from = None
        event.Event('on_deactivated', data)

    def on_text_command(self, view, command_name, args):
        data = self._get_data_from_view(view)
        data['command_name'] = command_name
        data['args'] = args
        event.Event('on_text_command', data)

    def on_window_command(self, window, command_name, args):
        view = window.active_view()
        data = self._get_data_from_view(view)
        data['command_name'] = command_name
        data['args'] = args
        event.Event('on_window_command', data)

    def post_text_command(self, view, command_name, args):
        data = self._get_data_from_view(view)
        data['command_name'] = command_name
        data['args'] = args
        event.Event('post_text_command', data)

    def post_window_command(self, window, command_name, args):
        data = self._get_data_from_view(view)
        data['command_name'] = command_name
        data['args'] = args
        event.Event('post_window_command', data)

    def on_query_context(self, view, key, operator, operand, match_all):
        data = self._get_data_from_view(view)
        data['key'] = key
        data['operator'] = operator
        data['operand'] = operand
        data['match_all'] = match_all
        event.Event('on_query_context', data)

    def on_query_completions(self, view, prefix, locations):
        data = self._get_data_from_view(view)
        data['prefix'] = prefix
        data['locations'] = locations
        event.Event('on_query_completions', data)
<commit_msg>Remove window data<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sublime
import sublime_plugin

import os
import sys
from time import time

PLUGIN_DIR = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, os.path.join(PLUGIN_DIR, 'libs'))

from fluent import sender, event

SETTINGS_FILE = 'fluent-logger-sublimetext.sublime-settings'

class FluentLoggerSublimetext(sublime_plugin.EventListener):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.settings = sublime.load_settings(SETTINGS_FILE)
        sender.setup(
            str(self.settings.get('tagprefix')),
            host=str(self.settings.get('host')),
            port=int(self.settings.get('port'))
        )

    def _get_data_from_view(self, view):
        window = view.window()
        return {
            "version": sublime.version(),
            "platform": sublime.platform(),
            "arch": sublime.arch(),
            "file_name": view.file_name(),
            "size": view.size(),
            "viewport_extent": view.viewport_extent(),
            "layout_extent": view.layout_extent(),
            "line_height": view.line_height(),
            "em_width": view.em_width(),
            "change_count": view.change_count(),
            "encoding": view.encoding(),
            "line_endings": view.line_endings(),
            "overwrite_status": view.overwrite_status(),
            "syntax": view.settings().get('syntax')
        }

    def on_new(self, view):
        event.Event('on_new', self._get_data_from_view(view))

    def on_clone(self, view):
        event.Event('on_clone', self._get_data_from_view(view))

    def on_load(self, view):
        event.Event('on_load', self._get_data_from_view(view))

    def on_pre_close(self, view):
        event.Event('on_pre_close', self._get_data_from_view(view))

    def on_close(self, view):
        event.Event('on_close', self._get_data_from_view(view))

    def on_pre_save(self, view):
        event.Event('on_pre_save', self._get_data_from_view(view))

    def on_post_save(self, view):
        event.Event('on_post_save', self._get_data_from_view(view))

    def on_modified(self, view):
        event.Event('on_modified', self._get_data_from_view(view))

    def on_selection_modified(self, view):
        event.Event('on_selection_modified', self._get_data_from_view(view))

    def on_activated(self, view):
        self.active_from = time()
        event.Event('on_activated', self._get_data_from_view(view))

    def on_deactivated(self, view):
        data = self._get_data_from_view(view)
        active_time = time() - self.active_from
        data['active_time'] = active_time
        self.active_from = None
        event.Event('on_deactivated', data)

    def on_text_command(self, view, command_name, args):
        data = self._get_data_from_view(view)
        data['command_name'] = command_name
        data['args'] = args
        event.Event('on_text_command', data)

    def on_window_command(self, window, command_name, args):
        view = window.active_view()
        data = self._get_data_from_view(view)
        data['command_name'] = command_name
        data['args'] = args
        event.Event('on_window_command', data)

    def post_text_command(self, view, command_name, args):
        data = self._get_data_from_view(view)
        data['command_name'] = command_name
        data['args'] = args
        event.Event('post_text_command', data)

    def post_window_command(self, window, command_name, args):
        data = self._get_data_from_view(view)
        data['command_name'] = command_name
        data['args'] = args
        event.Event('post_window_command', data)

    def on_query_context(self, view, key, operator, operand, match_all):
        data = self._get_data_from_view(view)
        data['key'] = key
        data['operator'] = operator
        data['operand'] = operand
        data['match_all'] = match_all
        event.Event('on_query_context', data)

    def on_query_completions(self, view, prefix, locations):
        data = self._get_data_from_view(view)
        data['prefix'] = prefix
        data['locations'] = locations
        event.Event('on_query_completions', data)
<|endoftext|>


<commit_before>import fmn.lib.models
from fmn.consumer.backends.base import BaseBackend
import fedmsg.meta

import twisted.internet.protocol
import twisted.words.protocols.irc

from twisted.internet import reactor

confirmation_template = """
{username} has requested that notifications be sent to this nick
* To accept, visit this address:
  {acceptance_url}
* Or, to reject you can visit this address:
  {rejection_url}
Alternatively, you can ignore this.  This is an automated message, please
email {support_email} if you have any concerns/issues/abuse.
I am run by Fedora Infrastructure.  Type 'help' for more information.
"""

help_template = """
I am a notifications bot run by Fedora Infrastructure.  My commands are:
  'stop'  -- stops all messages
  'start' -- starts sending messages again
  'help'  -- produces this help message
You can update your preferences at {base_url}
You can contact {support_email} if you have any concerns/issues/abuse.
"""

class IRCBackend(BaseBackend):
    __context_name__ = 'irc'

    def __init__(self, *args, **kwargs):
        super(IRCBackend, self).__init__(*args, **kwargs)
        self.network = self.config['fmn.irc.network']
        self.nickname = self.config['fmn.irc.nickname']
        self.port = int(self.config['fmn.irc.port'])
        self.timeout = int(self.config['fmn.irc.timeout'])

        self.clients = []
        factory = IRCClientFactory(self)

        # These are callbacks we use when people try to message us.
        self.commands = {
            'start': self.cmd_start,
            'stop': self.cmd_stop,
            'help': self.cmd_help,
            'default': self.cmd_default,
        }

        reactor.connectTCP(
            self.network,
            self.port,
            factory,
            timeout=self.timeout,
        )

    def send(self, nick, line):
        for client in self.clients:
            client.msg(nick.encode('utf-8'), line.encode('utf-8'))

    def cmd_start(self, nick, message):
        self.log.info("CMD start: %r sent us %r" % (nick, message))
        if self.disabled_for(nick):
            self.enable(nick)
            self.send(nick, "OK")
        else:
            self.send(nick, "Messages not currently stopped.  Nothing to do.")

    def cmd_stop(self, nick, message):
        self.log.info("CMD stop:  %r sent us %r" % (nick, message))
        if self.disabled_for(nick):
            self.send(nick, "Messages already stopped.  Nothing to do.")
        else:
            self.disable(nick)
            self.send(nick, "OK")

    def cmd_help(self, nick, message):
        self.log.info("CMD help:  %r sent us %r" % (nick, message))

        lines = help_template.format(
            support_email=self.config['fmn.support_email'],
            base_url=self.config['fmn.base_url'],
        ).strip().split('\n')

        for line in lines:
            self.send(nick, line)

    def cmd_default(self, nick, message):

        # Ignore notices from freenode services
        if message.startswith('***'):
            return

        self.log.info("CMD unk:   %r sent us %r" % (nick, message))
        self.send(nick, "say 'help' for help or 'stop' to stop messages")

    def handle(self, recipient, msg):
        if not self.clients:
            self.log.warning("IRCBackend has no clients to work with.")
            return

        self.log.debug("Notifying via irc %r" % recipient)

        if 'irc nick' not in recipient:
            self.log.warning("No irc nick found.  Bailing.")
            return

        message = fedmsg.meta.msg2repr(msg, **self.config)

        nickname = recipient['irc nick']

        if self.disabled_for(detail_value=nickname):
            self.log.debug("Messages stopped for %r, not sending." % nickname)
            return

        for client in self.clients:
            getattr(client, recipient.get('method', 'msg'))(
                nickname.encode('utf-8'),
                message.encode('utf-8'),
            )

    def handle_batch(self, recipient, queued_messages):
        for queued_message in queued_messages:
            self.handle(recipient, queued_message.message)

    def handle_confirmation(self, confirmation):
        if not self.clients:
            self.log.warning("IRCBackend has no clients to work with.")
            return

        self.log.debug("Handling confirmation via irc %r" % confirmation)

        query = "ACC %s" % confirmation.detail_value
        for client in self.clients:
            client.msg((u'NickServ').encode('utf-8'), query.encode('utf-8'))

    def handle_confirmation_valid_nick(self, nick):
        if not self.clients:
            self.log.warning("IRCBackend has no clients to work with.")
            return

        confirmations = fmn.lib.models.Confirmation.by_detail(
            self.session, context="irc", value=nick)

        for confirmation in confirmations:
            confirmation.set_status(self.session, 'valid')
            acceptance_url = self.config['fmn.acceptance_url'].format(
                secret=confirmation.secret)
            rejection_url = self.config['fmn.rejection_url'].format(
                secret=confirmation.secret)

            lines = confirmation_template.format(
                acceptance_url=acceptance_url,
                rejection_url=rejection_url,
                support_email=self.config['fmn.support_email'],
                username=confirmation.openid,
            ).strip().split('\n')

            for line in lines:
                for client in self.clients:
                    client.msg(nick.encode('utf-8'), line.encode('utf-8'))

    def handle_confirmation_invalid_nick(self, nick):
        confirmations = fmn.lib.models.Confirmation.by_detail(
            self.session, context="irc", value=nick)
        for confirmation in confirmations:
            confirmation.set_status(self.session, 'invalid')

    def cleanup_clients(self, factory):
        self.clients = [c for c in self.clients if c.factory != factory]

    def add_client(self, client):
        self.clients.append(client)

class IRCBackendProtocol(twisted.words.protocols.irc.IRCClient):
    lineRate = 0.6
    sourceURL = "http://github.com/fedora-infra/fmn"

    @property
    def nickname(self):
        return self.factory.parent.nickname

    @property
    def log(self):
        return self.factory.parent.log

    @property
    def commands(self):
        return self.factory.parent.commands

    def signedOn(self):
        self.log.info("Signed on as %r." % self.nickname)
        # Attach ourselves back on the consumer to be used.
        self.factory.parent.add_client(self)

    def privmsg(self, user, channel, msg):
        """ Called when a user privmsgs me. """
        return self.noticed(user, channel, msg)

    def noticed(self, user, channel, msg):
        """ Called when I have a notice from a user. """
        self.factory.parent.log.debug(
            "Received notice %r %r %r" % (user, channel, msg))

        # We query NickServ off the bat.  This is probably her responding.
        # We check NickServ before doing confirmations, so, let's do that now.
        if user == "NickServ!NickServ@services.":
            nickname, commands, result = msg.split(None, 2)
            if result.strip() == '3':
                # Then all is good.
                # 1) the nickname is registered
                # 2) the person is currently logged in and identified.
                self.factory.parent.handle_confirmation_valid_nick(nickname)
            else:
                # Something is off.  There are a number of possible scenarios,
                # but we'll just report back "invalid"
                self.factory.parent.handle_confirmation_invalid_nick(nickname)
        else:
            # If it's not NickServ, then it might be a user asking for help
            nick = user.split("!")[0]
            cmd_string = msg.split(None, 1)[0].lower()
            self.commands.get(cmd_string, self.commands['default'])(nick, msg)

class IRCClientFactory(twisted.internet.protocol.ClientFactory):
    protocol = IRCBackendProtocol

    def __init__(self, parent):
        self.parent = parent

    def clientConnectionLost(self, connector, reason):
        self.parent.log.warning("Lost connection %r, reconnecting." % reason)
        self.parent.cleanup_clients(factory=self)
        connector.connect()

    def clientConnectionFailed(self, connector, reason):
        self.parent.log.error("Could not connect: %r" % reason)
<commit_msg>Shorten links with dagd for irc.<commit_after>import fmn.lib.models
from fmn.consumer.backends.base import BaseBackend
import fedmsg.meta

import requests

import twisted.internet.protocol
import twisted.words.protocols.irc

from twisted.internet import reactor

confirmation_template = """
{username} has requested that notifications be sent to this nick
* To accept, visit this address:
  {acceptance_url}
* Or, to reject you can visit this address:
  {rejection_url}
Alternatively, you can ignore this.  This is an automated message, please
email {support_email} if you have any concerns/issues/abuse.
I am run by Fedora Infrastructure.  Type 'help' for more information.
"""

help_template = """
I am a notifications bot run by Fedora Infrastructure.  My commands are:
  'stop'  -- stops all messages
  'start' -- starts sending messages again
  'help'  -- produces this help message
You can update your preferences at {base_url}
You can contact {support_email} if you have any concerns/issues/abuse.
"""

def _shorten(link):
    if not link:
        return ''
    return requests.get('http://da.gd/s', params=dict(url=link)).text.strip()

def _format_message(msg, config):
    template = u"{title} -- {subtitle} {link}"
    title = fedmsg.meta.msg2title(msg, **config)
    subtitle = fedmsg.meta.msg2subtitle(msg, **config)
    link = _shorten(fedmsg.meta.msg2link(msg, **config))
    return template.format(title=title, subtitle=subtitle, link=link)

class IRCBackend(BaseBackend):
    __context_name__ = 'irc'

    def __init__(self, *args, **kwargs):
        super(IRCBackend, self).__init__(*args, **kwargs)
        self.network = self.config['fmn.irc.network']
        self.nickname = self.config['fmn.irc.nickname']
        self.port = int(self.config['fmn.irc.port'])
        self.timeout = int(self.config['fmn.irc.timeout'])

        self.clients = []
        factory = IRCClientFactory(self)

        # These are callbacks we use when people try to message us.
        self.commands = {
            'start': self.cmd_start,
            'stop': self.cmd_stop,
            'help': self.cmd_help,
            'default': self.cmd_default,
        }

        reactor.connectTCP(
            self.network,
            self.port,
            factory,
            timeout=self.timeout,
        )

    def send(self, nick, line):
        for client in self.clients:
            client.msg(nick.encode('utf-8'), line.encode('utf-8'))

    def cmd_start(self, nick, message):
        self.log.info("CMD start: %r sent us %r" % (nick, message))
        if self.disabled_for(nick):
            self.enable(nick)
            self.send(nick, "OK")
        else:
            self.send(nick, "Messages not currently stopped.  Nothing to do.")

    def cmd_stop(self, nick, message):
        self.log.info("CMD stop:  %r sent us %r" % (nick, message))
        if self.disabled_for(nick):
            self.send(nick, "Messages already stopped.  Nothing to do.")
        else:
            self.disable(nick)
            self.send(nick, "OK")

    def cmd_help(self, nick, message):
        self.log.info("CMD help:  %r sent us %r" % (nick, message))

        lines = help_template.format(
            support_email=self.config['fmn.support_email'],
            base_url=self.config['fmn.base_url'],
        ).strip().split('\n')

        for line in lines:
            self.send(nick, line)

    def cmd_default(self, nick, message):

        # Ignore notices from freenode services
        if message.startswith('***'):
            return

        self.log.info("CMD unk:   %r sent us %r" % (nick, message))
        self.send(nick, "say 'help' for help or 'stop' to stop messages")

    def handle(self, recipient, msg):
        if not self.clients:
            self.log.warning("IRCBackend has no clients to work with.")
            return

        self.log.debug("Notifying via irc %r" % recipient)

        if 'irc nick' not in recipient:
            self.log.warning("No irc nick found.  Bailing.")
            return

        message = _format_message(msg, self.config)

        nickname = recipient['irc nick']

        if self.disabled_for(detail_value=nickname):
            self.log.debug("Messages stopped for %r, not sending." % nickname)
            return

        for client in self.clients:
            getattr(client, recipient.get('method', 'msg'))(
                nickname.encode('utf-8'),
                message.encode('utf-8'),
            )

    def handle_batch(self, recipient, queued_messages):
        for queued_message in queued_messages:
            self.handle(recipient, queued_message.message)

    def handle_confirmation(self, confirmation):
        if not self.clients:
            self.log.warning("IRCBackend has no clients to work with.")
            return

        self.log.debug("Handling confirmation via irc %r" % confirmation)

        query = "ACC %s" % confirmation.detail_value
        for client in self.clients:
            client.msg((u'NickServ').encode('utf-8'), query.encode('utf-8'))

    def handle_confirmation_valid_nick(self, nick):
        if not self.clients:
            self.log.warning("IRCBackend has no clients to work with.")
            return

        confirmations = fmn.lib.models.Confirmation.by_detail(
            self.session, context="irc", value=nick)

        for confirmation in confirmations:
            confirmation.set_status(self.session, 'valid')
            acceptance_url = self.config['fmn.acceptance_url'].format(
                secret=confirmation.secret)
            rejection_url = self.config['fmn.rejection_url'].format(
                secret=confirmation.secret)

            lines = confirmation_template.format(
                acceptance_url=acceptance_url,
                rejection_url=rejection_url,
                support_email=self.config['fmn.support_email'],
                username=confirmation.openid,
            ).strip().split('\n')

            for line in lines:
                for client in self.clients:
                    client.msg(nick.encode('utf-8'), line.encode('utf-8'))

    def handle_confirmation_invalid_nick(self, nick):
        confirmations = fmn.lib.models.Confirmation.by_detail(
            self.session, context="irc", value=nick)
        for confirmation in confirmations:
            confirmation.set_status(self.session, 'invalid')

    def cleanup_clients(self, factory):
        self.clients = [c for c in self.clients if c.factory != factory]

    def add_client(self, client):
        self.clients.append(client)

class IRCBackendProtocol(twisted.words.protocols.irc.IRCClient):
    lineRate = 0.6
    sourceURL = "http://github.com/fedora-infra/fmn"

    @property
    def nickname(self):
        return self.factory.parent.nickname

    @property
    def log(self):
        return self.factory.parent.log

    @property
    def commands(self):
        return self.factory.parent.commands

    def signedOn(self):
        self.log.info("Signed on as %r." % self.nickname)
        # Attach ourselves back on the consumer to be used.
        self.factory.parent.add_client(self)

    def privmsg(self, user, channel, msg):
        """ Called when a user privmsgs me. """
        return self.noticed(user, channel, msg)

    def noticed(self, user, channel, msg):
        """ Called when I have a notice from a user. """
        self.factory.parent.log.debug(
            "Received notice %r %r %r" % (user, channel, msg))

        # We query NickServ off the bat.  This is probably her responding.
        # We check NickServ before doing confirmations, so, let's do that now.
        if user == "NickServ!NickServ@services.":
            nickname, commands, result = msg.split(None, 2)
            if result.strip() == '3':
                # Then all is good.
                # 1) the nickname is registered
                # 2) the person is currently logged in and identified.
                self.factory.parent.handle_confirmation_valid_nick(nickname)
            else:
                # Something is off.  There are a number of possible scenarios,
                # but we'll just report back "invalid"
                self.factory.parent.handle_confirmation_invalid_nick(nickname)
        else:
            # If it's not NickServ, then it might be a user asking for help
            nick = user.split("!")[0]
            cmd_string = msg.split(None, 1)[0].lower()
            self.commands.get(cmd_string, self.commands['default'])(nick, msg)

class IRCClientFactory(twisted.internet.protocol.ClientFactory):
    protocol = IRCBackendProtocol

    def __init__(self, parent):
        self.parent = parent

    def clientConnectionLost(self, connector, reason):
        self.parent.log.warning("Lost connection %r, reconnecting." % reason)
        self.parent.cleanup_clients(factory=self)
        connector.connect()

    def clientConnectionFailed(self, connector, reason):
        self.parent.log.error("Could not connect: %r" % reason)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models

class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding model 'HeadTeacher'
        db.create_table(u'data_headteacher', (
            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('first_name', self.gf('django.db.models.fields.CharField')(max_length=50)),
            ('last_name', self.gf('django.db.models.fields.CharField')(max_length=50)),
            ('gender', self.gf('django.db.models.fields.CharField')(max_length=6)),
            ('msisdn', self.gf('django.db.models.fields.CharField')(max_length=20)),
            ('date_of_birth', self.gf('django.db.models.fields.DateField')()),
            ('is_zonal_head', self.gf('django.db.models.fields.BooleanField')(default=False)),
            ('emis_id', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['hierarchy.School'], null=True, blank=True)),
            ('created_at', self.gf('django.db.models.fields.DateField')()),
        ))
        db.send_create_signal(u'data', ['HeadTeacher'])

        # Adding model 'SchoolData'
        db.create_table(u'data_schooldata', (
            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('emis_id', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['hierarchy.School'])),
            ('classrooms', self.gf('django.db.models.fields.IntegerField')()),
            ('teachers', self.gf('django.db.models.fields.IntegerField')()),
            ('teachers_g1', self.gf('django.db.models.fields.IntegerField')()),
            ('teachers_g2', self.gf('django.db.models.fields.IntegerField')()),
            ('boys_g2', self.gf('django.db.models.fields.IntegerField')()),
            ('girls_g2', self.gf('django.db.models.fields.IntegerField')()),
            ('created_at', self.gf('django.db.models.fields.DateField')()),
            ('created_by', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['data.HeadTeacher'])),
        ))
        db.send_create_signal(u'data', ['SchoolData'])

    def backwards(self, orm):
        # Deleting model 'HeadTeacher'
        db.delete_table(u'data_headteacher')

        # Deleting model 'SchoolData'
        db.delete_table(u'data_schooldata')

    models = {
        u'data.headteacher': {
            'Meta': {'object_name': 'HeadTeacher'},
            'created_at': ('django.db.models.fields.DateField', [], {}),
            'date_of_birth': ('django.db.models.fields.DateField', [], {}),
            'emis_id': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']", 'null': 'True', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'gender': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_zonal_head': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'msisdn': ('django.db.models.fields.CharField', [], {'max_length': '20'})
        },
        u'data.schooldata': {
            'Meta': {'object_name': 'SchoolData'},
            'boys_g2': ('django.db.models.fields.IntegerField', [], {}),
            'classrooms': ('django.db.models.fields.IntegerField', [], {}),
            'created_at': ('django.db.models.fields.DateField', [], {}),
            'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['data.HeadTeacher']"}),
            'emis_id': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']"}),
            'girls_g2': ('django.db.models.fields.IntegerField', [], {}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'teachers': ('django.db.models.fields.IntegerField', [], {}),
            'teachers_g1': ('django.db.models.fields.IntegerField', [], {}),
            'teachers_g2': ('django.db.models.fields.IntegerField', [], {})
        },
        u'hierarchy.district': {
            'Meta': {'object_name': 'District'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'province_id': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'province_id'", 'to': u"orm['hierarchy.Province']"})
        },
        u'hierarchy.province': {
            'Meta': {'object_name': 'Province'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        u'hierarchy.school': {
            'EMIS': ('django.db.models.fields.IntegerField', [], {'primary_key': 'True'}),
            'Meta': {'object_name': 'School'},
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'zone_id': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'zone_id'", 'to': u"orm['hierarchy.Zone']"})
        },
        u'hierarchy.zone': {
            'Meta': {'object_name': 'Zone'},
            'district_id': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'district_id'", 'to': u"orm['hierarchy.District']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        }
    }

    complete_apps = ['data']<commit_msg>added migration for data<commit_after># -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models

class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding model 'HeadTeacher'
        db.create_table(u'data_headteacher', (
            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('first_name', self.gf('django.db.models.fields.CharField')(max_length=50)),
            ('last_name', self.gf('django.db.models.fields.CharField')(max_length=50)),
            ('gender', self.gf('django.db.models.fields.CharField')(max_length=6)),
            ('msisdn', self.gf('django.db.models.fields.CharField')(max_length=20)),
            ('date_of_birth', self.gf('django.db.models.fields.DateField')()),
            ('is_zonal_head', self.gf('django.db.models.fields.BooleanField')(default=False)),
            ('emis_id', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['hierarchy.School'], null=True, blank=True)),
            ('created_at', self.gf('django.db.models.fields.DateField')()),
        ))
        db.send_create_signal(u'data', ['HeadTeacher'])

        # Adding model 'SchoolData'
        db.create_table(u'data_schooldata', (
            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('emis_id', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['hierarchy.School'])),
            ('classrooms', self.gf('django.db.models.fields.IntegerField')()),
            ('teachers', self.gf('django.db.models.fields.IntegerField')()),
            ('teachers_g1', self.gf('django.db.models.fields.IntegerField')()),
            ('teachers_g2', self.gf('django.db.models.fields.IntegerField')()),
            ('boys_g2', self.gf('django.db.models.fields.IntegerField')()),
            ('girls_g2', self.gf('django.db.models.fields.IntegerField')()),
            ('created_at', self.gf('django.db.models.fields.DateField')()),
            ('created_by', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['data.HeadTeacher'])),
        ))
        db.send_create_signal(u'data', ['SchoolData'])

    def backwards(self, orm):
        # Deleting model 'HeadTeacher'
        db.delete_table(u'data_headteacher')

        # Deleting model 'SchoolData'
        db.delete_table(u'data_schooldata')

    models = {
        u'data.headteacher': {
            'Meta': {'object_name': 'HeadTeacher'},
            'created_at': ('django.db.models.fields.DateField', [], {}),
            'date_of_birth': ('django.db.models.fields.DateField', [], {}),
            'emis_id': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']", 'null': 'True', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'gender': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_zonal_head': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'msisdn': ('django.db.models.fields.CharField', [], {'max_length': '20'})
        },
        u'data.schooldata': {
            'Meta': {'object_name': 'SchoolData'},
            'boys_g2': ('django.db.models.fields.IntegerField', [], {}),
            'classrooms': ('django.db.models.fields.IntegerField', [], {}),
            'created_at': ('django.db.models.fields.DateField', [], {}),
            'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['data.HeadTeacher']"}),
            'emis_id': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']"}),
            'girls_g2': ('django.db.models.fields.IntegerField', [], {}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'teachers': ('django.db.models.fields.IntegerField', [], {}),
            'teachers_g1': ('django.db.models.fields.IntegerField', [], {}),
            'teachers_g2': ('django.db.models.fields.IntegerField', [], {})
        },
        u'hierarchy.district': {
            'Meta': {'object_name': 'District'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'province': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.Province']"})
        },
        u'hierarchy.province': {
            'Meta': {'object_name': 'Province'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        u'hierarchy.school': {
            'EMIS': ('django.db.models.fields.IntegerField', [], {'unique': 'True'}),
            'Meta': {'object_name': 'School'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'zone': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.Zone']"})
        },
        u'hierarchy.zone': {
            'Meta': {'object_name': 'Zone'},
            'district': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.District']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        }
    }

    complete_apps = ['data']<|endoftext|>


<commit_before>#!/usr/bin/env python

'''Spatial filtering of image arrays with the FFT'''

from functools import partial
from numpy import (
    float32 as _float32,
    indices as _indices,
    multiply as _multiply,
)
from numpy.fft import (
    fftshift as _fftshift,
    ifftshift as _ifftshift,
    irfft2 as _irfft2,
    rfft2 as _rfft2,
)

from . import image

def _low_pass_filter(shape, radius):
    m, n = shape
    a, b = m // 2, n // 2
    i = _indices((m, n), dtype=_float32)
    rows, columns = i[0, :], i[1, :]
    d = (rows - a) ** 2 + (columns - b) ** 2
    d[d < radius ** 2] = 1.0
    d[d >= radius ** 2] = 0.0
    return d

def _high_pass_filter(shape, radius):
    return 1.0 - _low_pass_filter(shape, radius)

def _filter(filter, array):
    X = _rfft2(array)
    X = _fftshift(X)
    _multiply(X, filter, out=X)
    X = _ifftshift(X)
    x = _irfft2(X, s=array.shape)
    return x

def _rshape(shape):
    m, n = shape
    if n % 2 == 0:
        return (m, (n // 2) + 1)
    else:
        return (m, (n + 1) // 2)

def _lowpass(radius, array):
    rfilter = _low_pass_filter(_rshape(array.shape), radius)
    return _filter(rfilter, array)

def lowpass(img, radius):
    '''Apply a 2D low-pass filter to an image array.'''
    return image.map_to_channels(
        partial(_lowpass, radius),
        lambda shape: shape,
        img,
    )

def _highpass(radius, array):
    rfilter = _high_pass_filter(_rshape(array.shape), radius)
    return _filter(rfilter, array)

def highpass(img, radius):
    '''Apply a 2D high-pass filter to an image array.'''
    return image.map_to_channels(
        partial(_highpass, radius),
        lambda shape: shape,
        img,
    )

if __name__ == '__main__':
    pass
<commit_msg>Fix an off by one error in the filter module.<commit_after>#!/usr/bin/env python

'''Spatial filtering of image arrays with the FFT'''

from functools import partial
from numpy import (
    float32 as _float32,
    indices as _indices,
    multiply as _multiply,
)
from numpy.fft import (
    fftshift as _fftshift,
    ifftshift as _ifftshift,
    irfft2 as _irfft2,
    rfft2 as _rfft2,
)

from . import image

def _low_pass_filter(shape, radius):
    m, n = shape
    a, b = m // 2, n // 2
    i = _indices((m, n), dtype=_float32)
    rows, columns = i[0, :], i[1, :]
    d = (rows - a) ** 2 + (columns - b) ** 2
    d[d <= radius ** 2] = 1.0
    d[d > radius ** 2] = 0.0
    return d

def _high_pass_filter(shape, radius):
    return 1.0 - _low_pass_filter(shape, radius)

def _filter(filter, array):
    X = _rfft2(array)
    X = _fftshift(X)
    _multiply(X, filter, out=X)
    X = _ifftshift(X)
    x = _irfft2(X, s=array.shape)
    return x

def _rshape(shape):
    m, n = shape
    if n % 2 == 0:
        return (m, (n // 2) + 1)
    else:
        return (m, (n + 1) // 2)

def _lowpass(radius, array):
    rfilter = _low_pass_filter(_rshape(array.shape), radius)
    return _filter(rfilter, array)

def lowpass(img, radius):
    '''Apply a 2D low-pass filter to an image array.'''
    return image.map_to_channels(
        partial(_lowpass, radius),
        lambda shape: shape,
        img,
    )

def _highpass(radius, array):
    rfilter = _high_pass_filter(_rshape(array.shape), radius)
    return _filter(rfilter, array)

def highpass(img, radius):
    '''Apply a 2D high-pass filter to an image array.'''
    return image.map_to_channels(
        partial(_highpass, radius),
        lambda shape: shape,
        img,
    )

if __name__ == '__main__':
    pass
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

import json
from itertools import groupby, chain
import sys

def podemos_proportion_rounded_and_duplicates(data_list, women_names, proportions, withdrawed_candidates=[]):
    '''
    Given a list of woman names, returns a list of winners where the proportions
    of each sex is between the number provided.

    It also checks that the winners in the first question cannot win in the
    second.

    NOTE: it assumes the list of answers is already sorted.
    '''
    data = data_list[0]
    total = sum(proportions)
    proportions.sort()
    questions = data['results']['questions']
    first_question_winners = []
    first_question_winner_is_woman = None
    for question, question_index in zip(questions, range(len(questions))):
        num_winners = question['num_winners']
        max_samesex = round(num_winners*(proportions[1]/total))
        q_withdrawed = [a['id'] for a in withdrawed_candidates if a['question_num'] == question_index]

        if question['tally_type'] not in ["plurality-at-large"] or len(question['answers']) < 2 or question['num_winners'] < 2:
            last_winner = None
            num_won = 0
            for answer, i in zip(question['answers'], range(len(question['answers']))):
                if num_won < question['num_winners'] and answer['id'] not in q_withdrawed:
                    answer['winner_position'] = i
                    num_won += 1
                    last_winner = answer
                elif num_won == question['num_winners']:
                    answer['winner_position'] = None
                    if answer['total_count'] == last_winner['total_count'] and answer['id'] not in q_withdrawed:
                        print("it seems there is a tie in question '%s' with total_count = %d, '%s' vs '%s'" % (
                            question['title'], last_winner['total_count'], last_winner['text'], answer['text']), file=sys.stderr)
                else:
                    answer['winner_position'] = None

            # check if there's any removed winner that could have won and show
            # a warning if so
            last_winner_count = last_winner['total_count']
            removed_candidates = [
                '\'%s\'' % a['text']
                for a in question['answers']
                if a['text'] in first_question_winners and a['total_count'] >= last_winner_count]
            if len(removed_candidates) > 0:
                print("removed candidates that could have won: [%s]" % ", ".join(removed_candidates), file=sys.stderr)

            if question_index == 0:
                first_question_winners = [a['text'] for a in question['answers'] if a['winner_position'] is not None]
                first_question_winner_is_woman = first_question_winners[0] in women_names
            continue

        for answer, i in zip(question['answers'], range(len(question['answers']))):
            answer['winner_position'] = None

        def filter_women(l, women_names, question_index):
          return [
            a
            for a in l
            if a['text'] in women_names and a['text'] not in first_question_winners and\
            a['id'] not in q_withdrawed]
        def filter_men(l, women_names, question_index):
          return [
              a
              for a in l
              if a['text'] not in women_names and a['text'] not in first_question_winners and\
                a['id'] not in q_withdrawed]

        # remove withdrawed candidates
        running_candidates = [a for a in question['answers'] if a['text'] not in first_question_winners and\
                a['id'] not in q_withdrawed]
        women = filter_women(running_candidates, women_names, question_index)
        men = filter_men(running_candidates, women_names, question_index)
        num_winners = question['num_winners']

        base_winners = running_candidates[:num_winners]
        base_women_winners = filter_women(base_winners, women_names, question_index)
        base_men_winners = filter_men(base_winners, women_names, question_index)

        winners = base_women_winners + base_men_winners
        if len(base_women_winners) > max_samesex:
            n_diff =len(base_women_winners) - max_samesex
            winners = base_women_winners[:max_samesex] + men[:num_winners - max_samesex]
            print("too many women, len(base_women_winners)(%d) > max_samesex(%d)" % (len(base_women_winners), max_samesex), file=sys.stderr)
        elif len(base_men_winners) > max_samesex:
            n_diff =len(base_men_winners) - max_samesex
            winners = base_men_winners[:max_samesex] + women[:num_winners - max_samesex]
            print("too many men, len(base_men_winners)(%d) > max_samesex(%d)" % (len(base_men_winners), max_samesex), file=sys.stderr)

        winners = sorted(winners, reverse=True, key=lambda a: a['total_count'])

        # check for ties, filtering if there are other non winners with the same
        # total count of votes
        tie_val = winners[-1]['total_count']
        last_winner_name = winners[-1]['text']
        ties = [
            a for a in question['answers']
            if a['total_count'] == tie_val and a not in winners and answer['id'] not in q_withdrawed]
        if len(ties) > 0:
            tie_names = ", ".join(['\'%s\'' % tie['text'] for tie in ties])
            print("it seems there is a tie in question '%s' with total_count = %d, '%s' vs [%s]" % (
                question['title'], tie_val, last_winner_name, tie_names), file=sys.stderr)

        for answer, i in zip(winners, range(len(winners))):
            answer['winner_position'] = i

        # check if there's any removed winner that could have won and show
        # a warning if so
        removed_candidates = [
            '\'%s\'' % a['text']
            for a in question['answers']
            if a['text'] in first_question_winners and a['total_count'] >= tie_val]
        if len(removed_candidates) > 0:
            print("removed candidates that could have won: [%s]" % ", ".join(removed_candidates), file=sys.stderr)

        if question_index == 0:
          first_question_winners = [a['text'] for a in winners]
          first_question_winner_is_woman = first_question_winners[0] in women_names
<commit_msg>use truncation instead of rounding, as requested<commit_after># -*- coding:utf-8 -*-

import json
from itertools import groupby, chain
import sys

def podemos_proportion_rounded_and_duplicates(data_list, women_names, proportions, withdrawed_candidates=[]):
    '''
    Given a list of woman names, returns a list of winners where the proportions
    of each sex is between the number provided.

    It also checks that the winners in the first question cannot win in the
    second.

    NOTE: it assumes the list of answers is already sorted.
    '''
    data = data_list[0]
    total = sum(proportions)
    proportions.sort()
    questions = data['results']['questions']
    first_question_winners = []
    first_question_winner_is_woman = None
    for question, question_index in zip(questions, range(len(questions))):
        num_winners = question['num_winners']
        max_samesex = int(num_winners*(proportions[1]/total))
        q_withdrawed = [a['id'] for a in withdrawed_candidates if a['question_num'] == question_index]

        if question['tally_type'] not in ["plurality-at-large"] or len(question['answers']) < 2 or question['num_winners'] < 2:
            last_winner = None
            num_won = 0
            for answer, i in zip(question['answers'], range(len(question['answers']))):
                if num_won < question['num_winners'] and answer['id'] not in q_withdrawed:
                    answer['winner_position'] = i
                    num_won += 1
                    last_winner = answer
                elif num_won == question['num_winners']:
                    answer['winner_position'] = None
                    if answer['total_count'] == last_winner['total_count'] and answer['id'] not in q_withdrawed:
                        print("it seems there is a tie in question '%s' with total_count = %d, '%s' vs '%s'" % (
                            question['title'], last_winner['total_count'], last_winner['text'], answer['text']), file=sys.stderr)
                else:
                    answer['winner_position'] = None

            # check if there's any removed winner that could have won and show
            # a warning if so
            last_winner_count = last_winner['total_count']
            removed_candidates = [
                '\'%s\'' % a['text']
                for a in question['answers']
                if a['text'] in first_question_winners and a['total_count'] >= last_winner_count]
            if len(removed_candidates) > 0:
                print("removed candidates that could have won: [%s]" % ", ".join(removed_candidates), file=sys.stderr)

            if question_index == 0:
                first_question_winners = [a['text'] for a in question['answers'] if a['winner_position'] is not None]
                first_question_winner_is_woman = first_question_winners[0] in women_names
            continue

        for answer, i in zip(question['answers'], range(len(question['answers']))):
            answer['winner_position'] = None

        def filter_women(l, women_names, question_index):
          return [
            a
            for a in l
            if a['text'] in women_names and a['text'] not in first_question_winners and\
            a['id'] not in q_withdrawed]
        def filter_men(l, women_names, question_index):
          return [
              a
              for a in l
              if a['text'] not in women_names and a['text'] not in first_question_winners and\
                a['id'] not in q_withdrawed]

        # remove withdrawed candidates
        running_candidates = [a for a in question['answers'] if a['text'] not in first_question_winners and\
                a['id'] not in q_withdrawed]
        women = filter_women(running_candidates, women_names, question_index)
        men = filter_men(running_candidates, women_names, question_index)
        num_winners = question['num_winners']

        base_winners = running_candidates[:num_winners]
        base_women_winners = filter_women(base_winners, women_names, question_index)
        base_men_winners = filter_men(base_winners, women_names, question_index)

        winners = base_women_winners + base_men_winners
        if len(base_women_winners) > max_samesex:
            n_diff =len(base_women_winners) - max_samesex
            winners = base_women_winners[:max_samesex] + men[:num_winners - max_samesex]
            print("too many women, len(base_women_winners)(%d) > max_samesex(%d)" % (len(base_women_winners), max_samesex), file=sys.stderr)
        elif len(base_men_winners) > max_samesex:
            n_diff =len(base_men_winners) - max_samesex
            winners = base_men_winners[:max_samesex] + women[:num_winners - max_samesex]
            print("too many men, len(base_men_winners)(%d) > max_samesex(%d)" % (len(base_men_winners), max_samesex), file=sys.stderr)

        winners = sorted(winners, reverse=True, key=lambda a: a['total_count'])

        # check for ties, filtering if there are other non winners with the same
        # total count of votes
        tie_val = winners[-1]['total_count']
        last_winner_name = winners[-1]['text']
        ties = [
            a for a in question['answers']
            if a['total_count'] == tie_val and a not in winners and answer['id'] not in q_withdrawed]
        if len(ties) > 0:
            tie_names = ", ".join(['\'%s\'' % tie['text'] for tie in ties])
            print("it seems there is a tie in question '%s' with total_count = %d, '%s' vs [%s]" % (
                question['title'], tie_val, last_winner_name, tie_names), file=sys.stderr)

        for answer, i in zip(winners, range(len(winners))):
            answer['winner_position'] = i

        # check if there's any removed winner that could have won and show
        # a warning if so
        removed_candidates = [
            '\'%s\'' % a['text']
            for a in question['answers']
            if a['text'] in first_question_winners and a['total_count'] >= tie_val]
        if len(removed_candidates) > 0:
            print("removed candidates that could have won: [%s]" % ", ".join(removed_candidates), file=sys.stderr)

        if question_index == 0:
          first_question_winners = [a['text'] for a in winners]
          first_question_winner_is_woman = first_question_winners[0] in women_names
<|endoftext|>


<commit_before>import re
import sentry_sdk

from cached_property import cached_property
from datetime import datetime
from flask import current_app, g, request, session
from itsdangerous import BadSignature, JSONWebSignatureSerializer
from sqlalchemy.orm import joinedload
from typing import Mapping, Optional
from urllib.parse import urlparse, urljoin
from uuid import UUID

from zeus.config import db
from zeus.constants import Permission
from zeus.exceptions import AuthenticationFailed
from zeus.models import (
    ApiToken,
    ApiTokenRepositoryAccess,
    Repository,
    RepositoryAccess,
    RepositoryApiToken,
    User,
    UserApiToken,
)
from zeus.utils import timezone

class Tenant(object):
    def __init__(self, access: Optional[Mapping[UUID, Optional[Permission]]] = None):
        self.access = access or {}

    def __repr__(self):
        return "<{} repository_ids={}>".format(type(self).__name__, self.repository_ids)

    @cached_property
    def repository_ids(self):
        return list(self.access.keys())

    def get_permission(self, repository_id: UUID):
        return self.access.get(repository_id)

    def has_permission(self, repository_id: UUID, permission: Permission = None):
        if permission is None:
            return repository_id in self.access

        access = self.get_permission(repository_id)
        if not access:
            return False

        return permission in access

    @classmethod
    def from_user(cls, user: User):
        if not user:
            return cls()

        g.current_user = user
        return UserTenant(user_id=user.id)

    @classmethod
    def from_repository(
        cls, repository: Repository, permission: Optional[Permission] = None
    ):
        if not repository:
            return cls()

        return RepositoryTenant(access={repository.id: permission})

    @classmethod
    def from_api_token(cls, token: ApiToken):
        if not token:
            return cls()

        return ApiTokenTenant(token_id=token.id)

class ApiTokenTenant(Tenant):
    def __init__(self, token_id: str):
        self.token_id = token_id

    def __repr__(self):
        return "<{} token_id={}>".format(type(self).__name__, self.token_id)

    @cached_property
    def access(self) -> Mapping[UUID, Permission]:
        if not self.token_id:
            return None

        return dict(
            db.session.query(
                ApiTokenRepositoryAccess.repository_id,
                ApiTokenRepositoryAccess.permission,
            ).filter(ApiTokenRepositoryAccess.apitoken_id == self.token_id)
        )

class UserTenant(Tenant):
    def __init__(self, user_id: str):
        self.user_id = user_id

    def __repr__(self):
        return "<{} user_id={}>".format(type(self).__name__, self.user_id)

    @cached_property
    def access(self) -> Mapping[UUID, Permission]:
        if not self.user_id:
            return None

        return dict(
            db.session.query(
                RepositoryAccess.repository_id, RepositoryAccess.permission
            ).filter(RepositoryAccess.user_id == self.user_id)
        )

class RepositoryTenant(Tenant):
    def __init__(self, repository_id: str, permission: Optional[Permission] = None):
        self.repository_id = repository_id
        self.permission = permission

    def __repr__(self):
        return "<{} repository_id={} permisison={}>".format(
            type(self).__name__, self.repository_id, self.permission
        )

    @cached_property
    def access(self) -> Mapping[UUID, Permission]:
        if not self.repository_id:
            return None

        return {self.repository_id: self.permission}

def get_tenant_from_token():
    header = request.headers.get("Authorization", "").lower()
    if not header:
        return None

    if not header.startswith("bearer"):
        return None

    token = re.sub(r"^bearer(:|\s)\s*", "", header).strip()
    if not token.startswith("zeus-"):
        # Assuming this is a legacy token
        return get_tenant_from_legacy_token(token)

    parts = token.split("-", 2)
    if not len(parts) == 3:
        raise AuthenticationFailed

    if parts[1] == "u":
        return get_tenant_from_user_token(parts[2])

    if parts[1] == "r":
        return get_tenant_from_repository_token(parts[2])

    raise AuthenticationFailed

def get_tenant_from_user_token(key):
    token = (
        UserApiToken.query.options(joinedload("user"))
        .filter(UserApiToken.key == key)
        .first()
    )

    if not token:
        raise AuthenticationFailed

    return Tenant.from_user(token.user)

def get_tenant_from_repository_token(key):
    token = (
        RepositoryApiToken.query.options(joinedload("repository"))
        .filter(RepositoryApiToken.key == key)
        .first()
    )

    if not token:
        raise AuthenticationFailed

    return Tenant.from_repository(token.repository)

def get_tenant_from_legacy_token(token):
    api_token = ApiToken.query.filter(ApiToken.access_token == token).first()

    if not api_token:
        raise AuthenticationFailed

    if api_token.is_expired():
        raise AuthenticationFailed

    if api_token:
        g.current_api_token = api_token

    return Tenant.from_api_token(api_token)

def get_user_from_request() -> Optional[User]:
    expire = session.get("expire")
    if not expire:
        return None

    try:
        expire = datetime.utcfromtimestamp(expire).replace(tzinfo=timezone.utc)
    except Exception:
        current_app.logger.exception("invalid session expirey")
        del session["expire"]
        return None

    if expire <= timezone.now():
        current_app.logger.info("session expired")
        del session["expire"]
        return None

    try:
        uid = session["uid"]
    except KeyError:
        current_app.logger.error("missing uid session key", exc_info=True)
        del session["expire"]
        return None

    return User.query.get(uid)

def login_user(user_id: str, session=session, current_datetime=None):
    session["uid"] = str(user_id)
    session["expire"] = int(
        (
            (current_datetime or timezone.now())
            + current_app.config["PERMANENT_SESSION_LIFETIME"]
        ).strftime("%s")
    )
    session.permanent = True

def logout():
    session.clear()
    g.current_user = None
    g.current_tenant = None

def get_current_user(fetch=True) -> Optional[User]:
    rv = getattr(g, "current_user", None)
    if not rv and fetch:
        rv = get_user_from_request()
        g.current_user = rv
        with sentry_sdk.configure_scope() as scope:
            scope.user = {"id": str(rv.id), "email": rv.email} if rv else None
    return rv

def get_tenant_from_request():
    tenant = get_tenant_from_token()
    if tenant:
        return tenant

    user = get_current_user()
    return Tenant.from_user(user)

def set_current_tenant(tenant: Tenant):
    current_app.logger.info("Binding tenant as %r", tenant)
    g.current_tenant = tenant
    with sentry_sdk.configure_scope() as scope:
        scope.set_tag(
            "repository_id",
            str(tenant.repository_ids[0]) if len(tenant.repository_ids) == 1 else None,
        )
        scope.set_context(
            "tenant", {"repository_ids": [str(i) for i in tenant.repository_ids]}
        )

def get_current_tenant() -> Tenant:
    rv = getattr(g, "current_tenant", None)
    if rv is None:
        rv = get_tenant_from_request()
        set_current_tenant(rv)
    return rv

def generate_token(tenant: Tenant) -> str:
    s = JSONWebSignatureSerializer(current_app.secret_key, salt="auth")
    payload = {"repo_ids": [str(o) for o in tenant.repository_ids]}
    if getattr(tenant, "user_id", None):
        payload["uid"] = str(tenant.user_id)
    return s.dumps(payload)

def parse_token(token: str) -> str:
    s = JSONWebSignatureSerializer(current_app.secret_key, salt="auth")
    try:
        return s.loads(token)

    except BadSignature:
        return None

def is_safe_url(target: str) -> bool:
    ref_url = urlparse(request.host_url)
    test_url = urlparse(urljoin(request.host_url, target))
    return (
        # same scheme
        test_url.scheme in ("http", "https")
        and
        # same host and port
        ref_url.netloc == test_url.netloc
        and
        # and different endoint
        ref_url.path != test_url.path
    )

def get_redirect_target(clear=True, session=session) -> str:
    if clear:
        session_target = session.pop("next", None)
    else:
        session_target = session.get("next")

    for target in request.values.get("next"), session_target:
        if not target:
            continue

        if is_safe_url(target):
            return target

def bind_redirect_target(target: str = None, session=session):
    if not target:
        target = request.values.get("next")
    if not target and request.referrer != request.url:
        target = request.referrer
    if target and is_safe_url(target):
        session["next"] = target
    else:
        session.pop("next", None)
<commit_msg>Bind user context in other various scenarios<commit_after>import re
import sentry_sdk

from cached_property import cached_property
from datetime import datetime
from flask import current_app, g, request, session
from itsdangerous import BadSignature, JSONWebSignatureSerializer
from sqlalchemy.orm import joinedload
from typing import Mapping, Optional
from urllib.parse import urlparse, urljoin
from uuid import UUID

from zeus.config import db
from zeus.constants import Permission
from zeus.exceptions import AuthenticationFailed
from zeus.models import (
    ApiToken,
    ApiTokenRepositoryAccess,
    Repository,
    RepositoryAccess,
    RepositoryApiToken,
    User,
    UserApiToken,
)
from zeus.utils import timezone

class Tenant(object):
    def __init__(self, access: Optional[Mapping[UUID, Optional[Permission]]] = None):
        self.access = access or {}

    def __repr__(self):
        return "<{} repository_ids={}>".format(type(self).__name__, self.repository_ids)

    @cached_property
    def repository_ids(self):
        return list(self.access.keys())

    def get_permission(self, repository_id: UUID):
        return self.access.get(repository_id)

    def has_permission(self, repository_id: UUID, permission: Permission = None):
        if permission is None:
            return repository_id in self.access

        access = self.get_permission(repository_id)
        if not access:
            return False

        return permission in access

    @classmethod
    def from_user(cls, user: User):
        if not user:
            return cls()

        g.current_user = user
        with sentry_sdk.configure_scope() as scope:
            scope.user = {"id": str(user.id), "email": user.email}
        return UserTenant(user_id=user.id)

    @classmethod
    def from_repository(
        cls, repository: Repository, permission: Optional[Permission] = None
    ):
        if not repository:
            return cls()

        return RepositoryTenant(access={repository.id: permission})

    @classmethod
    def from_api_token(cls, token: ApiToken):
        if not token:
            return cls()

        return ApiTokenTenant(token_id=token.id)

class ApiTokenTenant(Tenant):
    def __init__(self, token_id: str):
        self.token_id = token_id

    def __repr__(self):
        return "<{} token_id={}>".format(type(self).__name__, self.token_id)

    @cached_property
    def access(self) -> Mapping[UUID, Permission]:
        if not self.token_id:
            return None

        return dict(
            db.session.query(
                ApiTokenRepositoryAccess.repository_id,
                ApiTokenRepositoryAccess.permission,
            ).filter(ApiTokenRepositoryAccess.apitoken_id == self.token_id)
        )

class UserTenant(Tenant):
    def __init__(self, user_id: str):
        self.user_id = user_id

    def __repr__(self):
        return "<{} user_id={}>".format(type(self).__name__, self.user_id)

    @cached_property
    def access(self) -> Mapping[UUID, Permission]:
        if not self.user_id:
            return None

        return dict(
            db.session.query(
                RepositoryAccess.repository_id, RepositoryAccess.permission
            ).filter(RepositoryAccess.user_id == self.user_id)
        )

class RepositoryTenant(Tenant):
    def __init__(self, repository_id: str, permission: Optional[Permission] = None):
        self.repository_id = repository_id
        self.permission = permission

    def __repr__(self):
        return "<{} repository_id={} permisison={}>".format(
            type(self).__name__, self.repository_id, self.permission
        )

    @cached_property
    def access(self) -> Mapping[UUID, Permission]:
        if not self.repository_id:
            return None

        return {self.repository_id: self.permission}

def get_tenant_from_token():
    header = request.headers.get("Authorization", "").lower()
    if not header:
        return None

    if not header.startswith("bearer"):
        return None

    token = re.sub(r"^bearer(:|\s)\s*", "", header).strip()
    if not token.startswith("zeus-"):
        # Assuming this is a legacy token
        return get_tenant_from_legacy_token(token)

    parts = token.split("-", 2)
    if not len(parts) == 3:
        raise AuthenticationFailed

    if parts[1] == "u":
        return get_tenant_from_user_token(parts[2])

    if parts[1] == "r":
        return get_tenant_from_repository_token(parts[2])

    raise AuthenticationFailed

def get_tenant_from_user_token(key):
    token = (
        UserApiToken.query.options(joinedload("user"))
        .filter(UserApiToken.key == key)
        .first()
    )

    if not token:
        raise AuthenticationFailed

    return Tenant.from_user(token.user)

def get_tenant_from_repository_token(key):
    token = (
        RepositoryApiToken.query.options(joinedload("repository"))
        .filter(RepositoryApiToken.key == key)
        .first()
    )

    if not token:
        raise AuthenticationFailed

    return Tenant.from_repository(token.repository)

def get_tenant_from_legacy_token(token):
    api_token = ApiToken.query.filter(ApiToken.access_token == token).first()

    if not api_token:
        raise AuthenticationFailed

    if api_token.is_expired():
        raise AuthenticationFailed

    if api_token:
        g.current_api_token = api_token

    return Tenant.from_api_token(api_token)

def get_user_from_request() -> Optional[User]:
    expire = session.get("expire")
    if not expire:
        return None

    try:
        expire = datetime.utcfromtimestamp(expire).replace(tzinfo=timezone.utc)
    except Exception:
        current_app.logger.exception("invalid session expirey")
        del session["expire"]
        return None

    if expire <= timezone.now():
        current_app.logger.info("session expired")
        del session["expire"]
        return None

    try:
        uid = session["uid"]
    except KeyError:
        current_app.logger.error("missing uid session key", exc_info=True)
        del session["expire"]
        return None

    return User.query.get(uid)

def login_user(user_id: str, session=session, current_datetime=None):
    session["uid"] = str(user_id)
    session["expire"] = int(
        (
            (current_datetime or timezone.now())
            + current_app.config["PERMANENT_SESSION_LIFETIME"]
        ).strftime("%s")
    )
    session.permanent = True
    with sentry_sdk.configure_scope() as scope:
        scope.user = {"id": str(user_id)}

def logout():
    session.clear()
    g.current_user = None
    g.current_tenant = None
    with sentry_sdk.configure_scope() as scope:
        scope.user = None

def get_current_user(fetch=True) -> Optional[User]:
    rv = getattr(g, "current_user", None)
    if not rv and fetch:
        rv = get_user_from_request()
        g.current_user = rv
        with sentry_sdk.configure_scope() as scope:
            scope.user = {"id": str(rv.id), "email": rv.email} if rv else None
    return rv

def get_tenant_from_request():
    tenant = get_tenant_from_token()
    if tenant:
        return tenant

    user = get_current_user()
    return Tenant.from_user(user)

def set_current_tenant(tenant: Tenant):
    current_app.logger.info("Binding tenant as %r", tenant)
    g.current_tenant = tenant
    with sentry_sdk.configure_scope() as scope:
        scope.set_tag(
            "repository_id",
            str(tenant.repository_ids[0]) if len(tenant.repository_ids) == 1 else None,
        )
        scope.set_context(
            "tenant", {"repository_ids": [str(i) for i in tenant.repository_ids]}
        )

def get_current_tenant() -> Tenant:
    rv = getattr(g, "current_tenant", None)
    if rv is None:
        rv = get_tenant_from_request()
        set_current_tenant(rv)
    return rv

def generate_token(tenant: Tenant) -> str:
    s = JSONWebSignatureSerializer(current_app.secret_key, salt="auth")
    payload = {"repo_ids": [str(o) for o in tenant.repository_ids]}
    if getattr(tenant, "user_id", None):
        payload["uid"] = str(tenant.user_id)
    return s.dumps(payload)

def parse_token(token: str) -> str:
    s = JSONWebSignatureSerializer(current_app.secret_key, salt="auth")
    try:
        return s.loads(token)

    except BadSignature:
        return None

def is_safe_url(target: str) -> bool:
    ref_url = urlparse(request.host_url)
    test_url = urlparse(urljoin(request.host_url, target))
    return (
        # same scheme
        test_url.scheme in ("http", "https")
        and
        # same host and port
        ref_url.netloc == test_url.netloc
        and
        # and different endoint
        ref_url.path != test_url.path
    )

def get_redirect_target(clear=True, session=session) -> str:
    if clear:
        session_target = session.pop("next", None)
    else:
        session_target = session.get("next")

    for target in request.values.get("next"), session_target:
        if not target:
            continue

        if is_safe_url(target):
            return target

def bind_redirect_target(target: str = None, session=session):
    if not target:
        target = request.values.get("next")
    if not target and request.referrer != request.url:
        target = request.referrer
    if target and is_safe_url(target):
        session["next"] = target
    else:
        session.pop("next", None)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymongo

from scrapy.exceptions import DropItem
from scrapy.conf import settings
from scrapy import log
from elasticsearch import Elasticsearch
from uuid import uuid1

class savollarPipeline(object):
    def process_item(self, item, spider):
        es = Elasticsearch([
            {"host": settings["ELASTICSEARCH_HOST"]},
        ])
        valid = True
        for data in item:
          # here we only check if the data is not null
          # but we could do any crazy validation we want
          if not data:

            raise DropItem("Missing %s of blogpost from %s" %(data, item['url']))
        if valid:
          es.index(
              index=settings["ELASTICSEARCH_INDEX"],
              doc_type="info",
              id=str(uuid1()),
              body=dict(item)
          )
          log.msg("Item indexed to ElasticSearch database %s:%s" %
                  (settings['ELASTICSEARCH_HOST'], settings['ELASTICSEARCH_PORT']),
                  level=log.DEBUG, spider=spider)
        return item
<commit_msg>pymongo no longer needed<commit_after># -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
from scrapy.exceptions import DropItem
from scrapy.conf import settings
from scrapy import log
from elasticsearch import Elasticsearch
from uuid import uuid1

class savollarPipeline(object):
    def process_item(self, item, spider):
        es = Elasticsearch([
            {"host": settings["ELASTICSEARCH_HOST"]},
        ])
        valid = True
        for data in item:
          # here we only check if the data is not null
          # but we could do any crazy validation we want
          if not data:

            raise DropItem("Missing %s of blogpost from %s" %(data, item['url']))
        if valid:
          es.index(
              index=settings["ELASTICSEARCH_INDEX"],
              doc_type="info",
              id=str(uuid1()),
              body=dict(item)
          )
          log.msg("Item indexed to ElasticSearch database %s:%s" %
                  (settings['ELASTICSEARCH_HOST'], settings['ELASTICSEARCH_PORT']),
                  level=log.DEBUG, spider=spider)
        return item
<|endoftext|>


<commit_before>### General Default

# Browsing initial dir (if recent videos file's is empty)
def_initialDir='/Users/francescomalatto/'

# File to store recent videos (extension must be .xml)
def_recentFile = './data/recent_files.xml'

# Max number of recent videos
def_maxRecentVideos = 10

# Language
def_lang = 'it'

# Omx player additional options (add a blank space at the end)
def_moreOptions = ''

### Menu default

# Output Audio
# hdmi, local, both
def_menuOutputAudio = 'hdmi'

# Adjust framerate/resolution to video
# True, False
def_menuVideoRefresh = True

# Set background to black
# True, False
def_menuBgBlack = False

### Debug option

# Enable debug
isDebug = True<commit_msg>Update defaultOpt.py<commit_after>### General Default

# Browsing initial dir (if recent videos file's is empty)
def_initialDir='/'

# File to store recent videos (extension must be .xml)
def_recentFile = './data/recent_files.xml'

# Max number of recent videos
def_maxRecentVideos = 10

# Language
def_lang = 'en'

# Omx player additional options (add a blank space at the end)
def_moreOptions = ''

### Menu default

# Output Audio
# hdmi, local, both
def_menuOutputAudio = 'hdmi'

# Adjust framerate/resolution to video
# True, False
def_menuVideoRefresh = True

# Set background to black
# True, False
def_menuBgBlack = False

### Debug option

# Enable debug
isDebug = True
<|endoftext|>