<commit_before>import numpy as np
import pandas as pd
from scipy import sparse, ndimage as ndi
from scipy.sparse import csgraph
import numba

from .nputil import pad, raveled_steps_to_neighbors

## CSGraph and Numba-based implementation

@numba.jit(nopython=True, cache=True, nogil=True)
def _write_pixel_graph(image, steps, distances, row, col, data):
    """Step over `image` to build a graph of nonzero pixel neighbors.

    Parameters
    ----------
    image : int array
        The input image.
    steps : int array, shape (N,)
        The raveled index steps to find a pixel's neighbors in `image`.
    distances : float array, shape (N,)
        The euclidean distance from a pixel to its corresponding
        neighbor in `steps`.
    row : int array
        Output array to be filled with the "center" pixel IDs.
    col : int array
        Output array to be filled with the "neighbor" pixel IDs.
    data : float array
        Output array to be filled with the distances from center to
        neighbor pixels.

    Notes
    -----
    No size or bounds checking is performed. Users should ensure that
    - No index in `indices` falls on any edge of `image` (or the
      neighbor computation will fail or segfault).
    - The `steps` and `distances` arrays have the same shape.
    - The `row`, `col`, `data` are long enough to hold all of the
      edges.
    """
    image = image.ravel()
    n_neighbors = steps.size
    start_idx = np.max(steps)
    end_idx = image.size + np.min(steps)
    k = 0
    for i in range(start_idx, end_idx + 1):
        if image[i] != 0:
            for j in range(n_neighbors):
                n = steps[j] + i
                if image[n] != 0:
                    row[k] = image[i]
                    col[k] = image[n]
                    data[k] = distances[j]
                    k += 1

def skeleton_to_csgraph(skel, *, spacing=1):
    """Convert a skeleton image of thin lines to a graph of neighbor pixels.

    Parameters
    ----------
    skel : array
        An input image in which every nonzero pixel is considered part of
        the skeleton, and links between pixels are determined by a full
        n-dimensional neighborhood.
    spacing : float, or array-like of float, shape `(skel.ndim,)`
        A value indicating the distance between adjacent pixels. This can
        either be a single value if the data has the same resolution along
        all axes, or it can be an array of the same shape as `skel` to
        indicate spacing along each axis.

    Returns
    -------
    graph : sparse.csr_matrix
        A graph of shape (Nnz + 1, Nnz + 1), where Nnz is the number of
        nonzero pixels in `skel`. The value graph[i, j] is the distance
        between adjacent pixels i and j. In a 2D image, that would be
        1 for immediately adjacent pixels and sqrt(2) for diagonally
        adjacent ones.
    pixel_indices : array of int
        An array of shape (Nnz + 1,), mapping indices in `graph` to
        raveled indices in `degree_image` or `skel`.
    degree_image : array of int, same shape as skel
        An image where each pixel value contains the degree of its
        corresponding node in `graph`. This is useful to classify nodes.
    """
    skel = skel.astype(bool)  # ensure we have a bool image
                              # since we later use it for bool indexing
    spacing = np.ones(skel.ndim, dtype=float) * spacing

    ndim = skel.ndim
    pixel_indices = np.concatenate(([0], np.flatnonzero(skel)))
    skelint = np.zeros(skel.shape, int)
    skelint.ravel()[pixel_indices] = np.arange(pixel_indices.size + 1)
    skelint = pad(skelint, 0)

    degree_kernel = np.ones((3,) * ndim)
    degree_kernel.ravel()[3**ndim // 2] = 0  # remove centre pix
    degree_image = ndi.convolve(skel.astype(int), degree_kernel,
                                mode='constant') * skel
    num_edges = np.sum(degree_image)  # *2, which is how many we need to store
    row, col = np.zeros(num_edges, dtype=int), np.zeros(num_edges, dtype=int)
    data = np.zeros(num_edges, dtype=float)
    steps, distances = raveled_steps_to_neighbors(skelint.shape, ndim)
    _write_pixel_graph(skelint, steps, distances, row, col, data)
    graph = sparse.coo_matrix((data, (row, col))).tocsr()
    return graph, pixel_indices, degree_image

@numba.jit(nopython=True, cache=True)
def _csrget(indices, indptr, data, row, col):
    start, end = indptr[row], indptr[row+1]
    for i in range(start, end):
        if indices[i] == col:
            return data[i]
    return 0.

@numba.jit(nopython=True, cache=True)
def _expand_path(indices, indptr, data, source, step, visited, degrees):
    d = _csrget(indices, indptr, data, source, step)
    while degrees[step] == 2 and not visited[step]:
        loc = indptr[step]
        n1, n2 = indices[loc], indices[loc+1]
        nextstep = n1 if n1 != source else n2
        source, step = step, nextstep
        d += _csrget(indices, indptr, data, source, step)
        visited[source] = True
    return step, d, degrees[step]

def branch_statistics(graph, pixel_indices, degree_image, *,
                      buffer_size_offset=0):
    """Compute the length and type of each branch in a skeleton graph.

    Parameters
    ----------
    graph : sparse.csr_matrix, shape (N, N)
        A skeleton graph.
    pixel_indices : array of int, shape (N,)
        A map from rows/cols of `graph` to image coordinates.
    degree_image : array of int, shape (P, Q, ...)
        The image corresponding to the skeleton, where each value is
        its degree in `graph`.
    buffer_size_offset : int, optional
        The buffer size is given by the sum of the degrees of non-path
        nodes. This is usually 2x the amount needed, allowing room for
        extra cycles of path-only nodes. However, if the image consists
        *only* of such cycles, the buffer size will be 0, resulting in
        an error. Until a more sophisticated, expandable-buffer
        solution is implemented, you can manually set a bigger buffer
        size using this parameter.

    Returns
    -------
    branches : array of float, shape (N, 4)
        An array containing branch endpoint IDs, length, and branch type.
        The types are:
        - tip-tip (0)
        - tip-junction (1)
        - junction-junction (2)
        - path-path (3) (This can only be a standalone cycle)
    """
    degree_image = degree_image.ravel()
    degrees = degree_image[pixel_indices]
    visited = np.zeros(pixel_indices.shape, dtype=bool)
    endpoints = (degrees != 2)
    num_paths = np.sum(degrees[endpoints])
    result = np.zeros((num_paths + buffer_size_offset, 4), dtype=float)
    num_results = 0
    for node in range(1, graph.shape[0]):
        if degrees[node] == 2 and not visited[node]:
            visited[node] = True
            loc = graph.indptr[node]
            left, right = graph.indices[loc:loc+2]
            id0, d0, deg0 = _expand_path(graph.indices, graph.indptr,
                                         graph.data, node, left, visited,
                                         degrees)
            if id0 == node:  # standalone cycle
                id1, d1, deg1 = node, 0., 2
                kind = 3
            else:
                id1, d1, deg1 = _expand_path(graph.indices, graph.indptr,
                                             graph.data, node, right, visited,
                                             degrees)
                kind = 2  # default: junction-to-junction
                if deg0 == 1 and deg1 == 1:  # tip-tip
                    kind = 0
                elif deg0 == 1 or deg1 == 1:  # tip-junct, tip-path impossible
                    kind = 1
            result[num_results, :] = id0, id1, d0 + d1, kind
            num_results += 1
    return result[:num_results]

def submatrix(M, idxs):
    """Return the outer-index product submatrix, `M[idxs, :][:, idxs]`.

    Parameters
    ----------
    M : scipy.sparse.spmatrix
        Input (square) matrix
    idxs : array of int
        The indices to subset. No index in `idxs` should exceed the
        number of rows of `M`.

    Returns
    -------
    Msub : scipy.sparse.spmatrix
        The subsetted matrix.

    Examples
    --------
    >>> Md = np.arange(16).reshape((4, 4))
    >>> M = sparse.csr_matrix(Md)
    >>> submatrix(M, [0, 2]).toarray()
    array([[ 0,  2],
           [ 8, 10]], dtype=int64)
    """
    Msub = M[idxs, :][:, idxs]
    return Msub

def summarise(skelimage):
    ndim = skelimage.ndim
    g, pixels, degrees = skeleton_to_csgraph(skelimage)
    coords = np.transpose(np.unravel_index(pixels, skelimage.shape))
    num_skeletons, skeleton_ids = csgraph.connected_components(g,
                                                               directed=False)
    stats = branch_statistics(g, pixels, degree_image=degrees)
    coords0 = coords[stats[:, 0].astype(int)]
    coords1 = coords[stats[:, 1].astype(int)]
    distances = np.sqrt(np.sum((coords0 - coords1)**2, axis=1))
    skeleton_id = skeleton_ids[stats[:, 0].astype(int)]
    table = np.column_stack((skeleton_id, stats,
                             coords0, coords1, distances))
    columns = (['skeleton-id', 'node-id-0', 'node-id-1', 'branch-distance',
                'branch-type'] +
               ['coord-0-%i' % i for i in range(ndim)] +
               ['coord-1-%i' % i for i in range(ndim)] +
               ['euclidean-distance'])
    column_types = [int, int, int, float, int] + 2*ndim*[int] + [float]
    data_dict = {col: dat.astype(dtype)
                 for col, dat, dtype in zip(columns, table.T, column_types)}
    df = pd.DataFrame(data_dict)
    return df
<commit_msg>Bug fix: pass spacing kwarg correctly<commit_after>import numpy as np
import pandas as pd
from scipy import sparse, ndimage as ndi
from scipy.sparse import csgraph
import numba

from .nputil import pad, raveled_steps_to_neighbors

## CSGraph and Numba-based implementation

@numba.jit(nopython=True, cache=True, nogil=True)
def _write_pixel_graph(image, steps, distances, row, col, data):
    """Step over `image` to build a graph of nonzero pixel neighbors.

    Parameters
    ----------
    image : int array
        The input image.
    steps : int array, shape (N,)
        The raveled index steps to find a pixel's neighbors in `image`.
    distances : float array, shape (N,)
        The euclidean distance from a pixel to its corresponding
        neighbor in `steps`.
    row : int array
        Output array to be filled with the "center" pixel IDs.
    col : int array
        Output array to be filled with the "neighbor" pixel IDs.
    data : float array
        Output array to be filled with the distances from center to
        neighbor pixels.

    Notes
    -----
    No size or bounds checking is performed. Users should ensure that
    - No index in `indices` falls on any edge of `image` (or the
      neighbor computation will fail or segfault).
    - The `steps` and `distances` arrays have the same shape.
    - The `row`, `col`, `data` are long enough to hold all of the
      edges.
    """
    image = image.ravel()
    n_neighbors = steps.size
    start_idx = np.max(steps)
    end_idx = image.size + np.min(steps)
    k = 0
    for i in range(start_idx, end_idx + 1):
        if image[i] != 0:
            for j in range(n_neighbors):
                n = steps[j] + i
                if image[n] != 0:
                    row[k] = image[i]
                    col[k] = image[n]
                    data[k] = distances[j]
                    k += 1

def skeleton_to_csgraph(skel, *, spacing=1):
    """Convert a skeleton image of thin lines to a graph of neighbor pixels.

    Parameters
    ----------
    skel : array
        An input image in which every nonzero pixel is considered part of
        the skeleton, and links between pixels are determined by a full
        n-dimensional neighborhood.
    spacing : float, or array-like of float, shape `(skel.ndim,)`
        A value indicating the distance between adjacent pixels. This can
        either be a single value if the data has the same resolution along
        all axes, or it can be an array of the same shape as `skel` to
        indicate spacing along each axis.

    Returns
    -------
    graph : sparse.csr_matrix
        A graph of shape (Nnz + 1, Nnz + 1), where Nnz is the number of
        nonzero pixels in `skel`. The value graph[i, j] is the distance
        between adjacent pixels i and j. In a 2D image, that would be
        1 for immediately adjacent pixels and sqrt(2) for diagonally
        adjacent ones.
    pixel_indices : array of int
        An array of shape (Nnz + 1,), mapping indices in `graph` to
        raveled indices in `degree_image` or `skel`.
    degree_image : array of int, same shape as skel
        An image where each pixel value contains the degree of its
        corresponding node in `graph`. This is useful to classify nodes.
    """
    skel = skel.astype(bool)  # ensure we have a bool image
                              # since we later use it for bool indexing
    spacing = np.ones(skel.ndim, dtype=float) * spacing

    ndim = skel.ndim
    pixel_indices = np.concatenate(([0], np.flatnonzero(skel)))
    skelint = np.zeros(skel.shape, int)
    skelint.ravel()[pixel_indices] = np.arange(pixel_indices.size + 1)
    skelint = pad(skelint, 0)

    degree_kernel = np.ones((3,) * ndim)
    degree_kernel.ravel()[3**ndim // 2] = 0  # remove centre pix
    degree_image = ndi.convolve(skel.astype(int), degree_kernel,
                                mode='constant') * skel
    num_edges = np.sum(degree_image)  # *2, which is how many we need to store
    row, col = np.zeros(num_edges, dtype=int), np.zeros(num_edges, dtype=int)
    data = np.zeros(num_edges, dtype=float)
    steps, distances = raveled_steps_to_neighbors(skelint.shape, ndim,
                                                  spacing=spacing)
    _write_pixel_graph(skelint, steps, distances, row, col, data)
    graph = sparse.coo_matrix((data, (row, col))).tocsr()
    return graph, pixel_indices, degree_image

@numba.jit(nopython=True, cache=True)
def _csrget(indices, indptr, data, row, col):
    start, end = indptr[row], indptr[row+1]
    for i in range(start, end):
        if indices[i] == col:
            return data[i]
    return 0.

@numba.jit(nopython=True, cache=True)
def _expand_path(indices, indptr, data, source, step, visited, degrees):
    d = _csrget(indices, indptr, data, source, step)
    while degrees[step] == 2 and not visited[step]:
        loc = indptr[step]
        n1, n2 = indices[loc], indices[loc+1]
        nextstep = n1 if n1 != source else n2
        source, step = step, nextstep
        d += _csrget(indices, indptr, data, source, step)
        visited[source] = True
    return step, d, degrees[step]

def branch_statistics(graph, pixel_indices, degree_image, *,
                      buffer_size_offset=0):
    """Compute the length and type of each branch in a skeleton graph.

    Parameters
    ----------
    graph : sparse.csr_matrix, shape (N, N)
        A skeleton graph.
    pixel_indices : array of int, shape (N,)
        A map from rows/cols of `graph` to image coordinates.
    degree_image : array of int, shape (P, Q, ...)
        The image corresponding to the skeleton, where each value is
        its degree in `graph`.
    buffer_size_offset : int, optional
        The buffer size is given by the sum of the degrees of non-path
        nodes. This is usually 2x the amount needed, allowing room for
        extra cycles of path-only nodes. However, if the image consists
        *only* of such cycles, the buffer size will be 0, resulting in
        an error. Until a more sophisticated, expandable-buffer
        solution is implemented, you can manually set a bigger buffer
        size using this parameter.

    Returns
    -------
    branches : array of float, shape (N, 4)
        An array containing branch endpoint IDs, length, and branch type.
        The types are:
        - tip-tip (0)
        - tip-junction (1)
        - junction-junction (2)
        - path-path (3) (This can only be a standalone cycle)
    """
    degree_image = degree_image.ravel()
    degrees = degree_image[pixel_indices]
    visited = np.zeros(pixel_indices.shape, dtype=bool)
    endpoints = (degrees != 2)
    num_paths = np.sum(degrees[endpoints])
    result = np.zeros((num_paths + buffer_size_offset, 4), dtype=float)
    num_results = 0
    for node in range(1, graph.shape[0]):
        if degrees[node] == 2 and not visited[node]:
            visited[node] = True
            loc = graph.indptr[node]
            left, right = graph.indices[loc:loc+2]
            id0, d0, deg0 = _expand_path(graph.indices, graph.indptr,
                                         graph.data, node, left, visited,
                                         degrees)
            if id0 == node:  # standalone cycle
                id1, d1, deg1 = node, 0., 2
                kind = 3
            else:
                id1, d1, deg1 = _expand_path(graph.indices, graph.indptr,
                                             graph.data, node, right, visited,
                                             degrees)
                kind = 2  # default: junction-to-junction
                if deg0 == 1 and deg1 == 1:  # tip-tip
                    kind = 0
                elif deg0 == 1 or deg1 == 1:  # tip-junct, tip-path impossible
                    kind = 1
            result[num_results, :] = id0, id1, d0 + d1, kind
            num_results += 1
    return result[:num_results]

def submatrix(M, idxs):
    """Return the outer-index product submatrix, `M[idxs, :][:, idxs]`.

    Parameters
    ----------
    M : scipy.sparse.spmatrix
        Input (square) matrix
    idxs : array of int
        The indices to subset. No index in `idxs` should exceed the
        number of rows of `M`.

    Returns
    -------
    Msub : scipy.sparse.spmatrix
        The subsetted matrix.

    Examples
    --------
    >>> Md = np.arange(16).reshape((4, 4))
    >>> M = sparse.csr_matrix(Md)
    >>> submatrix(M, [0, 2]).toarray()
    array([[ 0,  2],
           [ 8, 10]], dtype=int64)
    """
    Msub = M[idxs, :][:, idxs]
    return Msub

def summarise(skelimage):
    ndim = skelimage.ndim
    g, pixels, degrees = skeleton_to_csgraph(skelimage)
    coords = np.transpose(np.unravel_index(pixels, skelimage.shape))
    num_skeletons, skeleton_ids = csgraph.connected_components(g,
                                                               directed=False)
    stats = branch_statistics(g, pixels, degree_image=degrees)
    coords0 = coords[stats[:, 0].astype(int)]
    coords1 = coords[stats[:, 1].astype(int)]
    distances = np.sqrt(np.sum((coords0 - coords1)**2, axis=1))
    skeleton_id = skeleton_ids[stats[:, 0].astype(int)]
    table = np.column_stack((skeleton_id, stats,
                             coords0, coords1, distances))
    columns = (['skeleton-id', 'node-id-0', 'node-id-1', 'branch-distance',
                'branch-type'] +
               ['coord-0-%i' % i for i in range(ndim)] +
               ['coord-1-%i' % i for i in range(ndim)] +
               ['euclidean-distance'])
    column_types = [int, int, int, float, int] + 2*ndim*[int] + [float]
    data_dict = {col: dat.astype(dtype)
                 for col, dat, dtype in zip(columns, table.T, column_types)}
    df = pd.DataFrame(data_dict)
    return df
<|endoftext|>


<commit_before># this is the class that defines the player character, and holds all of its animations
# + their animation timings

from direct.actor.Actor import Actor
from direct.task import Task
from tricks import *

from panda3d.core import *
# from direct.interval.IntervalGlobal import *

class Tricker(object):
    def __init__(self):
        self.actor = Actor("tp/models/tricker-model",
                           {"gainer": "tp/anims/tricker-gainer",
                            "gainer_bad": "tp/anims/tricker-gainer-bad",
                            "gswitch": "tp/anims/tricker-gswitch"})

        self.trickList = {"gainer": 0,
                          "gswitch": 0}

        # Load tricks
        self.gainer = Gainer(self)
        self.gswitch = Gswitch(self)

        self.prevTrick = None

        self.totalStam = self.stamina = 100

        self.grade = ' '

    def stamPercentage(self):
        sp = self.stamina/self.totalStam
        if sp < 0 : sp = 0
        return sp

    def getGrade(self):
        return self.grade

    def tryTrick(self, animation, trick, taskMgr):
        if self.stamina <= 0:
            print("no stamina!")
            return

        grade = 'A'
        currAnim = self.actor.getCurrentAnim()
        if currAnim:
            if self.prevTrick.getExitTransition() != trick.getEntryTransition():
                print("invalid transition")
                return
            currFrame = self.actor.getCurrentFrame(currAnim)
            numFrames = self.actor.getNumFrames(currAnim)
            framesLeft = numFrames - currFrame

            grade = self.prevTrick.getGrade(currFrame)
            self.grade = grade
            self.drawGrade(grade)
            # TODO: organize the grading system
            if grade == 'E': return

            # 0.06 is the time it takes for 2 frames - smooth blending
            delayTime = framesLeft / 30 - 0.06
            taskMgr.doMethodLater(delayTime, self.doTrickTask, 'doTrick',
                             extraArgs=[animation], appendTask=True)
        else:
            taskMgr.add(self.doTrickTask, 'doTrick',
                             extraArgs=[animation], appendTask=True)

        stamCost = trick.getStamCost(grade)
        self.stamina -= stamCost

        self.prevTrick = trick

    def doTrickTask(self, animation, task):
        airTime = self.actor.getNumFrames(animation) / 30
        moveInterval = self.actor.posInterval(airTime,
                                                Point3(0, .1, 0),
                                                other=self.actor)
        self.actor.play(animation)

        moveInterval.start()
        return Task.done

    def drawGrade(self, grade):
        if grade == 'D':
            print('Grade: D. pretty shit.')
        elif grade == 'C':
            print('Grade: C. MEDIOCRE')
        elif grade == 'B':
            print('Grade: B. Aight')
        elif grade == 'A':
            print("Grade: A. PERFFECT")
        elif grade == 'E':
            print("Grade: E. Trick failed")<commit_msg>clean up grade vars<commit_after># this is the class that defines the player character, and holds all of its animations
# + their animation timings

from direct.actor.Actor import Actor
from direct.task import Task
from tricks import *

from panda3d.core import *
# from direct.interval.IntervalGlobal import *

class Tricker(object):
    def __init__(self):
        self.actor = Actor("tp/models/tricker-model",
                           {"gainer": "tp/anims/tricker-gainer",
                            "gainer_bad": "tp/anims/tricker-gainer-bad",
                            "gswitch": "tp/anims/tricker-gswitch"})

        self.trickList = {"gainer": 0,
                          "gswitch": 0}

        # Load tricks
        self.gainer = Gainer(self)
        self.gswitch = Gswitch(self)

        self.prevTrick = None

        self.totalStam = self.stamina = 100

        self.grade = ' '

    def stamPercentage(self):
        sp = self.stamina/self.totalStam
        if sp < 0 : sp = 0
        return sp

    def getGrade(self):
        return self.grade

    def tryTrick(self, animation, trick, taskMgr):
        if self.stamina <= 0:
            print("no stamina!")
            return

        grade = 'A'
        currAnim = self.actor.getCurrentAnim()
        if currAnim:
            if self.prevTrick.getExitTransition() != trick.getEntryTransition():
                print("invalid transition")
                return
            currFrame = self.actor.getCurrentFrame(currAnim)
            numFrames = self.actor.getNumFrames(currAnim)
            framesLeft = numFrames - currFrame

            self.grade = self.prevTrick.getGrade(currFrame)
            # TODO: organize the grading system
            if self.grade == 'E': return

            # 0.06 is the time it takes for 2 frames - smooth blending
            delayTime = framesLeft / 30 - 0.06
            taskMgr.doMethodLater(delayTime, self.doTrickTask, 'doTrick',
                             extraArgs=[animation], appendTask=True)
        else:
            taskMgr.add(self.doTrickTask, 'doTrick',
                             extraArgs=[animation], appendTask=True)

        stamCost = trick.getStamCost(grade)
        self.stamina -= stamCost

        self.prevTrick = trick

    def doTrickTask(self, animation, task):
        airTime = self.actor.getNumFrames(animation) / 30
        moveInterval = self.actor.posInterval(airTime,
                                                Point3(0, .1, 0),
                                                other=self.actor)
        self.actor.play(animation)

        moveInterval.start()
        return Task.done<|endoftext|>


<commit_before>""" Mailu admin app utilities
"""

try:
    import cPickle as pickle
except ImportError:
    import pickle

import dns
import dns.resolver

import hmac
import secrets
import time

from multiprocessing import Value

from mailu import limiter

import flask
import flask_login
import flask_migrate
import flask_babel
import redis

from flask.sessions import SessionMixin, SessionInterface
from itsdangerous.encoding import want_bytes
from werkzeug.datastructures import CallbackDict
from werkzeug.contrib import fixers

# Login configuration
login = flask_login.LoginManager()
login.login_view = "ui.login"

@login.unauthorized_handler
def handle_needs_login():
    """ redirect unauthorized requests to login page """
    return flask.redirect(
        flask.url_for('ui.login', next=flask.request.endpoint)
    )

# DNS stub configured to do DNSSEC enabled queries
resolver = dns.resolver.Resolver()
resolver.use_edns(0, 0, 1500)
resolver.flags = dns.flags.AD | dns.flags.RD

def has_dane_record(domain, timeout=5):
    try:
        result = resolver.query(f'_25._tcp.{domain}', dns.rdatatype.TLSA,dns.rdataclass.IN, lifetime=timeout)
        if (result.response.flags & dns.flags.AD) == dns.flags.AD:
            for record in result:
                if isinstance(record, dns.rdtypes.ANY.TLSA.TLSA):
                    record.validate()
                    if record.usage in [2,3]: # postfix wants DANE-only
                        return True
    except dns.resolver.NoNameservers:
        # If the DNSSEC data is invalid and the DNS resolver is DNSSEC enabled
        # we will receive this non-specific exception. The safe behaviour is to
        # accept to defer the email.
        return app.config['DEFER_ON_TLS_ERROR']
    except:
        pass

# Rate limiter
limiter = limiter.LimitWraperFactory()

# Application translation
babel = flask_babel.Babel()

@babel.localeselector
def get_locale():
    """ selects locale for translation """
    translations = [str(translation) for translation in babel.list_translations()]
    return flask.request.accept_languages.best_match(translations)

# Proxy fixer
class PrefixMiddleware(object):
    """ fix proxy headers """
    def __init__(self):
        self.app = None

    def __call__(self, environ, start_response):
        prefix = environ.get('HTTP_X_FORWARDED_PREFIX', '')
        if prefix:
            environ['SCRIPT_NAME'] = prefix
        return self.app(environ, start_response)

    def init_app(self, app):
        self.app = fixers.ProxyFix(app.wsgi_app, x_for=1, x_proto=1)
        app.wsgi_app = self

proxy = PrefixMiddleware()

# Data migrate
migrate = flask_migrate.Migrate()

# session store (inspired by https://github.com/mbr/flask-kvsession)
class RedisStore:
    """ Stores session data in a redis db. """

    has_ttl = True

    def __init__(self, redisstore):
        self.redis = redisstore

    def get(self, key):
        """ load item from store. """
        value = self.redis.get(key)
        if value is None:
            raise KeyError(key)
        return value

    def put(self, key, value, ttl=None):
        """ save item to store. """
        if ttl:
            self.redis.setex(key, int(ttl), value)
        else:
            self.redis.set(key, value)

    def delete(self, key):
        """ delete item from store. """
        self.redis.delete(key)

    def list(self, prefix=None):
        """ return list of keys starting with prefix """
        if prefix:
            prefix += b'*'
        return list(self.redis.scan_iter(match=prefix))

class DictStore:
    """ Stores session data in a python dict. """

    has_ttl = False

    def __init__(self):
        self.dict = {}

    def get(self, key):
        """ load item from store. """
        return self.dict[key]

    def put(self, key, value, ttl_secs=None):
        """ save item to store. """
        self.dict[key] = value

    def delete(self, key):
        """ delete item from store. """
        try:
            del self.dict[key]
        except KeyError:
            pass

    def list(self, prefix=None):
        """ return list of keys starting with prefix """
        if prefix is None:
            return list(self.dict.keys())
        return [key for key in self.dict if key.startswith(prefix)]

class MailuSession(CallbackDict, SessionMixin):
    """ Custom flask session storage. """

    # default modified to false
    modified = False

    def __init__(self, key=None, app=None):

        self.app = app or flask.current_app

        initial = None

        key = want_bytes(key)
        if parsed := self.app.session_config.parse_key(key, self.app):
            try:
                initial = pickle.loads(app.session_store.get(key))
            except (KeyError, EOFError, pickle.UnpicklingError):
                # either the cookie was manipulated or we did not find the
                # session in the backend or the pickled data is invalid.
                # => start new session
                pass
            else:
                (self._uid, self._sid, self._created) = parsed
                self._key = key

        if initial is None:
            # start new session
            self.new = True
            self._uid = None
            self._sid = None
            self._created = self.app.session_config.gen_created()
            self._key = None

        def _on_update(obj):
            obj.modified = True

        CallbackDict.__init__(self, initial, _on_update)

    @property
    def saved(self):
        """ this reflects if the session was saved. """
        return self._key is not None

    @property
    def sid(self):
        """ this reflects the session's id. """
        if self._sid is None or self._uid is None or self._created is None:
            return None
        return b''.join([self._uid, self._sid, self._created])

    def destroy(self):
        """ destroy session for security reasons. """

        self.delete()

        self._uid = None
        self._sid = None
        self._created = None

        self.clear()

        self.modified = True
        self.new = False

    def regenerate(self):
        """ generate new id for session to avoid `session fixation`. """

        self.delete()

        self._sid = None
        self._created = self.app.session_config.gen_created()

        self.modified = True

    def delete(self):
        """ Delete stored session. """
        if self.saved:
            self.app.session_store.delete(self._key)
            self._key = None

    def save(self):
        """ Save session to store. """

        set_cookie = False

        # set uid from dict data
        if self._uid is None:
            self._uid = self.app.session_config.gen_uid(self.get('user_id', ''))

        # create new session id for new or regenerated sessions and force setting the cookie
        if self._sid is None:
            self._sid = self.app.session_config.gen_sid()
            set_cookie = True

        # get new session key
        key = self.sid

        # delete old session if key has changed
        if key != self._key:
            self.delete()

        # remember time to refresh
        self['_refresh'] = int(time.time()) + self.app.permanent_session_lifetime.total_seconds()/2

        # save session
        self.app.session_store.put(
            key,
            pickle.dumps(dict(self)),
            self.app.permanent_session_lifetime.total_seconds()
        )

        self._key = key

        self.new = False
        self.modified = False

        return set_cookie

    def needs_refresh(self):
        """ Checks if server side session needs to be refreshed. """

        return int(time.time()) > self.get('_refresh', 0)

class MailuSessionConfig:
    """ Stores sessions crypto config """

    # default size of session key parts
    uid_bits = 64 # default if SESSION_KEY_BITS is not set in config
    sid_bits = 128 # for now. must be multiple of 8!
    time_bits = 32 # for now. must be multiple of 8!

    def __init__(self, app=None):

        if app is None:
            app = flask.current_app

        bits = app.config.get('SESSION_KEY_BITS', self.uid_bits)
        if not 64 <= bits <= 256:
            raise ValueError('SESSION_KEY_BITS must be between 64 and 256!')

        uid_bytes = bits//8 + (bits%8>0)
        sid_bytes = self.sid_bits//8

        key = want_bytes(app.secret_key)

        self._hmac    = hmac.new(hmac.digest(key, b'SESSION_UID_HASH', digest='sha256'), digestmod='sha256')
        self._uid_len = uid_bytes
        self._uid_b64 = len(self._encode(bytes(uid_bytes)))
        self._sid_len = sid_bytes
        self._sid_b64 = len(self._encode(bytes(sid_bytes)))
        self._key_min = self._uid_b64 + self._sid_b64
        self._key_max = self._key_min + len(self._encode(bytes(self.time_bits//8)))

    def gen_sid(self):
        """ Generate random session id. """
        return self._encode(secrets.token_bytes(self._sid_len))

    def gen_uid(self, uid):
        """ Generate hashed user id part of session key. """
        _hmac = self._hmac.copy()
        _hmac.update(want_bytes(uid))
        return self._encode(_hmac.digest()[:self._uid_len])

    def gen_created(self, now=None):
        """ Generate base64 representation of creation time. """
        return self._encode(int(now or time.time()).to_bytes(8, byteorder='big').lstrip(b'\0'))

    def parse_key(self, key, app=None, validate=False, now=None):
        """ Split key into sid, uid and creation time. """

        if not (isinstance(key, bytes) and self._key_min <= len(key) <= self._key_max):
            return None

        uid = key[:self._uid_b64]
        sid = key[self._uid_b64:self._key_min]
        crt = key[self._key_min:]

        # validate if parts are decodeable
        created = self._decode(crt)
        if created is None or self._decode(uid) is None or self._decode(sid) is None:
            return None

        # validate creation time when requested or store does not support ttl
        if validate or not app.session_store.has_ttl:
            if now is None:
                now = int(time.time())
            created = int.from_bytes(created, byteorder='big')
            if not created < now < created + app.permanent_session_lifetime.total_seconds():
                return None

        return (uid, sid, crt)

    def _encode(self, value):
        return secrets.base64.urlsafe_b64encode(value).rstrip(b'=')

    def _decode(self, value):
        try:
            return secrets.base64.urlsafe_b64decode(value + b'='*(4-len(value)%4))
        except secrets.binascii.Error:
            return None

class MailuSessionInterface(SessionInterface):
    """ Custom flask session interface. """

    def open_session(self, app, request):
        """ Load or create session. """
        return MailuSession(request.cookies.get(app.config['SESSION_COOKIE_NAME'], None), app)

    def save_session(self, app, session, response):
        """ Save modified session. """

        # If the session is modified to be empty, remove the cookie.
        # If the session is empty, return without setting the cookie.
        if not session:
            if session.modified:
                session.delete()
                response.delete_cookie(
                    app.session_cookie_name,
                    domain=self.get_cookie_domain(app),
                    path=self.get_cookie_path(app),
                )
            return

        # Add a "Vary: Cookie" header if the session was accessed
        if session.accessed:
            response.vary.add('Cookie')

        set_cookie = session.permanent and app.config['SESSION_REFRESH_EACH_REQUEST']
        need_refresh = session.needs_refresh()

        # save modified session or refresh unmodified session
        if session.modified or need_refresh:
            set_cookie |= session.save()

        # set cookie on refreshed permanent sessions
        if need_refresh and session.permanent:
            set_cookie = True

        # set or update cookie if necessary
        if set_cookie:
            response.set_cookie(
                app.session_cookie_name,
                session.sid,
                expires=self.get_expiration_time(app, session),
                httponly=self.get_cookie_httponly(app),
                domain=self.get_cookie_domain(app),
                path=self.get_cookie_path(app),
                secure=self.get_cookie_secure(app),
                samesite=self.get_cookie_samesite(app)
            )

class MailuSessionExtension:
    """ Server side session handling """

    @staticmethod
    def cleanup_sessions(app=None):
        """ Remove invalid or expired sessions. """

        app = app or flask.current_app
        now = int(time.time())

        count = 0
        for key in app.session_store.list():
            if not app.session_config.parse_key(key, app, validate=True, now=now):
                app.session_store.delete(key)
                count += 1

        return count

    @staticmethod
    def prune_sessions(uid=None, keep=None, app=None):
        """ Remove sessions
            uid: remove all sessions (NONE) or sessions belonging to a specific user
            keep: keep listed sessions
        """

        keep = keep or set()
        app = app or flask.current_app

        prefix = None if uid is None else app.session_config.gen_uid(uid)

        count = 0
        for key in app.session_store.list(prefix):
            if key not in keep:
                app.session_store.delete(key)
                count += 1

        return count

    def init_app(self, app):
        """ Replace session management of application. """

        if app.config.get('MEMORY_SESSIONS'):
            # in-memory session store for use in development
            app.session_store = DictStore()

        else:
            # redis-based session store for use in production
            app.session_store = RedisStore(
                redis.StrictRedis().from_url(app.config['SESSION_STORAGE_URL'])
            )

            # clean expired sessions oonce on first use in case lifetime was changed
            def cleaner():
                with cleaned.get_lock():
                    if not cleaned.value:
                        cleaned.value = True
                        flask.current_app.logger.error('cleaning')
                        MailuSessionExtension.cleanup_sessions(app)

            app.before_first_request(cleaner)

        app.session_config = MailuSessionConfig(app)
        app.session_interface = MailuSessionInterface()

cleaned = Value('i', False)
session = MailuSessionExtension()
<commit_msg>Ensure that postfix will be able to use the TLSA records<commit_after>""" Mailu admin app utilities
"""

try:
    import cPickle as pickle
except ImportError:
    import pickle

import dns
import dns.resolver

import hmac
import secrets
import time

from multiprocessing import Value

from mailu import limiter

import flask
import flask_login
import flask_migrate
import flask_babel
import redis

from flask.sessions import SessionMixin, SessionInterface
from itsdangerous.encoding import want_bytes
from werkzeug.datastructures import CallbackDict
from werkzeug.contrib import fixers

# Login configuration
login = flask_login.LoginManager()
login.login_view = "ui.login"

@login.unauthorized_handler
def handle_needs_login():
    """ redirect unauthorized requests to login page """
    return flask.redirect(
        flask.url_for('ui.login', next=flask.request.endpoint)
    )

# DNS stub configured to do DNSSEC enabled queries
resolver = dns.resolver.Resolver()
resolver.use_edns(0, 0, 1500)
resolver.flags = dns.flags.AD | dns.flags.RD

def has_dane_record(domain, timeout=10):
    try:
        result = resolver.query(f'_25._tcp.{domain}', dns.rdatatype.TLSA,dns.rdataclass.IN, lifetime=timeout)
        if (result.response.flags & dns.flags.AD) == dns.flags.AD:
            for record in result:
                if isinstance(record, dns.rdtypes.ANY.TLSA.TLSA):
                    record.validate()
                    if record.usage in [2,3] and record.selector in [0,1] and record.mtype in [0,1,2]:
                        return True
    except dns.resolver.NoNameservers:
        # If the DNSSEC data is invalid and the DNS resolver is DNSSEC enabled
        # we will receive this non-specific exception. The safe behaviour is to
        # accept to defer the email.
        return app.config['DEFER_ON_TLS_ERROR']
    except:
        pass

# Rate limiter
limiter = limiter.LimitWraperFactory()

# Application translation
babel = flask_babel.Babel()

@babel.localeselector
def get_locale():
    """ selects locale for translation """
    translations = [str(translation) for translation in babel.list_translations()]
    return flask.request.accept_languages.best_match(translations)

# Proxy fixer
class PrefixMiddleware(object):
    """ fix proxy headers """
    def __init__(self):
        self.app = None

    def __call__(self, environ, start_response):
        prefix = environ.get('HTTP_X_FORWARDED_PREFIX', '')
        if prefix:
            environ['SCRIPT_NAME'] = prefix
        return self.app(environ, start_response)

    def init_app(self, app):
        self.app = fixers.ProxyFix(app.wsgi_app, x_for=1, x_proto=1)
        app.wsgi_app = self

proxy = PrefixMiddleware()

# Data migrate
migrate = flask_migrate.Migrate()

# session store (inspired by https://github.com/mbr/flask-kvsession)
class RedisStore:
    """ Stores session data in a redis db. """

    has_ttl = True

    def __init__(self, redisstore):
        self.redis = redisstore

    def get(self, key):
        """ load item from store. """
        value = self.redis.get(key)
        if value is None:
            raise KeyError(key)
        return value

    def put(self, key, value, ttl=None):
        """ save item to store. """
        if ttl:
            self.redis.setex(key, int(ttl), value)
        else:
            self.redis.set(key, value)

    def delete(self, key):
        """ delete item from store. """
        self.redis.delete(key)

    def list(self, prefix=None):
        """ return list of keys starting with prefix """
        if prefix:
            prefix += b'*'
        return list(self.redis.scan_iter(match=prefix))

class DictStore:
    """ Stores session data in a python dict. """

    has_ttl = False

    def __init__(self):
        self.dict = {}

    def get(self, key):
        """ load item from store. """
        return self.dict[key]

    def put(self, key, value, ttl_secs=None):
        """ save item to store. """
        self.dict[key] = value

    def delete(self, key):
        """ delete item from store. """
        try:
            del self.dict[key]
        except KeyError:
            pass

    def list(self, prefix=None):
        """ return list of keys starting with prefix """
        if prefix is None:
            return list(self.dict.keys())
        return [key for key in self.dict if key.startswith(prefix)]

class MailuSession(CallbackDict, SessionMixin):
    """ Custom flask session storage. """

    # default modified to false
    modified = False

    def __init__(self, key=None, app=None):

        self.app = app or flask.current_app

        initial = None

        key = want_bytes(key)
        if parsed := self.app.session_config.parse_key(key, self.app):
            try:
                initial = pickle.loads(app.session_store.get(key))
            except (KeyError, EOFError, pickle.UnpicklingError):
                # either the cookie was manipulated or we did not find the
                # session in the backend or the pickled data is invalid.
                # => start new session
                pass
            else:
                (self._uid, self._sid, self._created) = parsed
                self._key = key

        if initial is None:
            # start new session
            self.new = True
            self._uid = None
            self._sid = None
            self._created = self.app.session_config.gen_created()
            self._key = None

        def _on_update(obj):
            obj.modified = True

        CallbackDict.__init__(self, initial, _on_update)

    @property
    def saved(self):
        """ this reflects if the session was saved. """
        return self._key is not None

    @property
    def sid(self):
        """ this reflects the session's id. """
        if self._sid is None or self._uid is None or self._created is None:
            return None
        return b''.join([self._uid, self._sid, self._created])

    def destroy(self):
        """ destroy session for security reasons. """

        self.delete()

        self._uid = None
        self._sid = None
        self._created = None

        self.clear()

        self.modified = True
        self.new = False

    def regenerate(self):
        """ generate new id for session to avoid `session fixation`. """

        self.delete()

        self._sid = None
        self._created = self.app.session_config.gen_created()

        self.modified = True

    def delete(self):
        """ Delete stored session. """
        if self.saved:
            self.app.session_store.delete(self._key)
            self._key = None

    def save(self):
        """ Save session to store. """

        set_cookie = False

        # set uid from dict data
        if self._uid is None:
            self._uid = self.app.session_config.gen_uid(self.get('user_id', ''))

        # create new session id for new or regenerated sessions and force setting the cookie
        if self._sid is None:
            self._sid = self.app.session_config.gen_sid()
            set_cookie = True

        # get new session key
        key = self.sid

        # delete old session if key has changed
        if key != self._key:
            self.delete()

        # remember time to refresh
        self['_refresh'] = int(time.time()) + self.app.permanent_session_lifetime.total_seconds()/2

        # save session
        self.app.session_store.put(
            key,
            pickle.dumps(dict(self)),
            self.app.permanent_session_lifetime.total_seconds()
        )

        self._key = key

        self.new = False
        self.modified = False

        return set_cookie

    def needs_refresh(self):
        """ Checks if server side session needs to be refreshed. """

        return int(time.time()) > self.get('_refresh', 0)

class MailuSessionConfig:
    """ Stores sessions crypto config """

    # default size of session key parts
    uid_bits = 64 # default if SESSION_KEY_BITS is not set in config
    sid_bits = 128 # for now. must be multiple of 8!
    time_bits = 32 # for now. must be multiple of 8!

    def __init__(self, app=None):

        if app is None:
            app = flask.current_app

        bits = app.config.get('SESSION_KEY_BITS', self.uid_bits)
        if not 64 <= bits <= 256:
            raise ValueError('SESSION_KEY_BITS must be between 64 and 256!')

        uid_bytes = bits//8 + (bits%8>0)
        sid_bytes = self.sid_bits//8

        key = want_bytes(app.secret_key)

        self._hmac    = hmac.new(hmac.digest(key, b'SESSION_UID_HASH', digest='sha256'), digestmod='sha256')
        self._uid_len = uid_bytes
        self._uid_b64 = len(self._encode(bytes(uid_bytes)))
        self._sid_len = sid_bytes
        self._sid_b64 = len(self._encode(bytes(sid_bytes)))
        self._key_min = self._uid_b64 + self._sid_b64
        self._key_max = self._key_min + len(self._encode(bytes(self.time_bits//8)))

    def gen_sid(self):
        """ Generate random session id. """
        return self._encode(secrets.token_bytes(self._sid_len))

    def gen_uid(self, uid):
        """ Generate hashed user id part of session key. """
        _hmac = self._hmac.copy()
        _hmac.update(want_bytes(uid))
        return self._encode(_hmac.digest()[:self._uid_len])

    def gen_created(self, now=None):
        """ Generate base64 representation of creation time. """
        return self._encode(int(now or time.time()).to_bytes(8, byteorder='big').lstrip(b'\0'))

    def parse_key(self, key, app=None, validate=False, now=None):
        """ Split key into sid, uid and creation time. """

        if not (isinstance(key, bytes) and self._key_min <= len(key) <= self._key_max):
            return None

        uid = key[:self._uid_b64]
        sid = key[self._uid_b64:self._key_min]
        crt = key[self._key_min:]

        # validate if parts are decodeable
        created = self._decode(crt)
        if created is None or self._decode(uid) is None or self._decode(sid) is None:
            return None

        # validate creation time when requested or store does not support ttl
        if validate or not app.session_store.has_ttl:
            if now is None:
                now = int(time.time())
            created = int.from_bytes(created, byteorder='big')
            if not created < now < created + app.permanent_session_lifetime.total_seconds():
                return None

        return (uid, sid, crt)

    def _encode(self, value):
        return secrets.base64.urlsafe_b64encode(value).rstrip(b'=')

    def _decode(self, value):
        try:
            return secrets.base64.urlsafe_b64decode(value + b'='*(4-len(value)%4))
        except secrets.binascii.Error:
            return None

class MailuSessionInterface(SessionInterface):
    """ Custom flask session interface. """

    def open_session(self, app, request):
        """ Load or create session. """
        return MailuSession(request.cookies.get(app.config['SESSION_COOKIE_NAME'], None), app)

    def save_session(self, app, session, response):
        """ Save modified session. """

        # If the session is modified to be empty, remove the cookie.
        # If the session is empty, return without setting the cookie.
        if not session:
            if session.modified:
                session.delete()
                response.delete_cookie(
                    app.session_cookie_name,
                    domain=self.get_cookie_domain(app),
                    path=self.get_cookie_path(app),
                )
            return

        # Add a "Vary: Cookie" header if the session was accessed
        if session.accessed:
            response.vary.add('Cookie')

        set_cookie = session.permanent and app.config['SESSION_REFRESH_EACH_REQUEST']
        need_refresh = session.needs_refresh()

        # save modified session or refresh unmodified session
        if session.modified or need_refresh:
            set_cookie |= session.save()

        # set cookie on refreshed permanent sessions
        if need_refresh and session.permanent:
            set_cookie = True

        # set or update cookie if necessary
        if set_cookie:
            response.set_cookie(
                app.session_cookie_name,
                session.sid,
                expires=self.get_expiration_time(app, session),
                httponly=self.get_cookie_httponly(app),
                domain=self.get_cookie_domain(app),
                path=self.get_cookie_path(app),
                secure=self.get_cookie_secure(app),
                samesite=self.get_cookie_samesite(app)
            )

class MailuSessionExtension:
    """ Server side session handling """

    @staticmethod
    def cleanup_sessions(app=None):
        """ Remove invalid or expired sessions. """

        app = app or flask.current_app
        now = int(time.time())

        count = 0
        for key in app.session_store.list():
            if not app.session_config.parse_key(key, app, validate=True, now=now):
                app.session_store.delete(key)
                count += 1

        return count

    @staticmethod
    def prune_sessions(uid=None, keep=None, app=None):
        """ Remove sessions
            uid: remove all sessions (NONE) or sessions belonging to a specific user
            keep: keep listed sessions
        """

        keep = keep or set()
        app = app or flask.current_app

        prefix = None if uid is None else app.session_config.gen_uid(uid)

        count = 0
        for key in app.session_store.list(prefix):
            if key not in keep:
                app.session_store.delete(key)
                count += 1

        return count

    def init_app(self, app):
        """ Replace session management of application. """

        if app.config.get('MEMORY_SESSIONS'):
            # in-memory session store for use in development
            app.session_store = DictStore()

        else:
            # redis-based session store for use in production
            app.session_store = RedisStore(
                redis.StrictRedis().from_url(app.config['SESSION_STORAGE_URL'])
            )

            # clean expired sessions oonce on first use in case lifetime was changed
            def cleaner():
                with cleaned.get_lock():
                    if not cleaned.value:
                        cleaned.value = True
                        flask.current_app.logger.error('cleaning')
                        MailuSessionExtension.cleanup_sessions(app)

            app.before_first_request(cleaner)

        app.session_config = MailuSessionConfig(app)
        app.session_interface = MailuSessionInterface()

cleaned = Value('i', False)
session = MailuSessionExtension()
<|endoftext|>


<commit_before>'''
Project: Farnsworth

Author: Karandeep Singh Nagra
'''

from django import forms
from farnsworth.settings import house
from utils.variables import time_formats

def EventForm(manager_positions, initial=None, post=None):
	''' Return a form with an as_manager position if the user is a manager.
	Parameters:
		manager_positions should be a choice set of manager positions the user making the request holds
		initial should be a dictionary of initial values.
		post should be a request.POST
	'''
	class InnerEventForm(forms.Form):
		''' A form to post an event. '''
		title = forms.CharField(max_length=100, widget=forms.TextInput())
		description = forms.CharField(widget=forms.Textarea())
		location = forms.CharField(max_length=100, widget=forms.TextInput())
		rsvp = forms.BooleanField(required=False, label="RSVP")
		start_time = forms.DateTimeField(widget=forms.DateTimeInput(format="%m/%d/%Y %I:%M %p"),
						 input_formats=time_formats)
		end_time = forms.DateTimeField(widget=forms.DateTimeInput(format="%m/%d/%Y %I:%M %p"),
					       input_formats=time_formats)
		as_manager = forms.ModelChoiceField(queryset=manager_positions, required=False,
						    label="As manager (if manager event)")
		cancelled = forms.BooleanField(required=False, label="Mark Cancelled")

		def is_valid(self):
			if not super(InnerEventForm, self).is_valid():
				return False
			start_time = self.cleaned_data['start_time']
			end_time = self.cleaned_data['end_time']
			if start_time > end_time:
				self.errors['__all__'] = self.error_class(["Start time is later than end time. Unless this event involves time travel, please change the start or end time."])
				return False
			return True

	if post is None:
		if initial is None:
			return InnerEventForm(initial={'rsvp': True, 'location': house})
		else:
			return InnerEventForm(initial=initial)
	else:
		return InnerEventForm(post)

class RsvpForm(forms.Form):
	''' Form to RSVP or un-RSVP from an event. '''
	event_pk = forms.IntegerField(widget=forms.HiddenInput())
<commit_msg>Made the event form use the time_formats variable as well<commit_after>'''
Project: Farnsworth

Author: Karandeep Singh Nagra
'''

from django import forms
from farnsworth.settings import house
from utils.variables import time_formats

def EventForm(manager_positions, initial=None, post=None):
	''' Return a form with an as_manager position if the user is a manager.
	Parameters:
		manager_positions should be a choice set of manager positions the user making the request holds
		initial should be a dictionary of initial values.
		post should be a request.POST
	'''
	class InnerEventForm(forms.Form):
		''' A form to post an event. '''
		title = forms.CharField(max_length=100, widget=forms.TextInput())
		description = forms.CharField(widget=forms.Textarea())
		location = forms.CharField(max_length=100, widget=forms.TextInput())
		rsvp = forms.BooleanField(required=False, label="RSVP")
		start_time = forms.DateTimeField(widget=forms.DateTimeInput(format=time_formats[0]),
						 input_formats=time_formats)
		end_time = forms.DateTimeField(widget=forms.DateTimeInput(format=time_formats[0]),
					       input_formats=time_formats)
		as_manager = forms.ModelChoiceField(queryset=manager_positions, required=False,
						    label="As manager (if manager event)")
		cancelled = forms.BooleanField(required=False, label="Mark Cancelled")

		def is_valid(self):
			if not super(InnerEventForm, self).is_valid():
				return False
			start_time = self.cleaned_data['start_time']
			end_time = self.cleaned_data['end_time']
			if start_time > end_time:
				self.errors['__all__'] = self.error_class(["Start time is later than end time. Unless this event involves time travel, please change the start or end time."])
				return False
			return True

	if post is None:
		if initial is None:
			return InnerEventForm(initial={'rsvp': True, 'location': house})
		else:
			return InnerEventForm(initial=initial)
	else:
		return InnerEventForm(post)

class RsvpForm(forms.Form):
	''' Form to RSVP or un-RSVP from an event. '''
	event_pk = forms.IntegerField(widget=forms.HiddenInput())
<|endoftext|>


<commit_before>

import urllib3, json, math
import numpy as np
from datetime import datetime
from core.art.modelsART import ARTSubResult
from django.db import transaction, DatabaseError
from core.settings import defaultDatetimeFormat
import logging

_logger_error = logging.getLogger('bigpandamon-error')
_logger = logging.getLogger('bigpandamon')

def getJobReport(guid, lfn, scope):
    filebrowserURL = "http://bigpanda.cern.ch/filebrowser/"  # This is deployment specific because memory monitoring is intended to work in ATLAS
    jobSubResult = []
    http = urllib3.PoolManager()
    resp = http.request('GET', filebrowserURL, fields={'guid': guid, 'lfn': lfn, 'scope': scope, 'json': 1})
    if resp and len(resp.data) > 0:
        try:
            data = json.loads(resp.data)
            HOSTNAME = data['HOSTNAME']
            tardir = data['tardir']
            MEDIA_URL = data['MEDIA_URL']
            dirprefix = data['dirprefix']
            files = data['files']
            files = [f for f in files if 'jobReport.json' in f['name']]
        except:
            return -2
    else:
        return -2

    urlBase = "http://" + HOSTNAME + "/" + MEDIA_URL + dirprefix + "/" + tardir

    for f in files:
        url = urlBase + "/" + f['name']
        response = http.request('GET', url)
        data = json.loads(response.data)

    return data

def getARTjobSubResults(data):
    jobSubResult = {}

    if isinstance(data, dict) and 'art' in data:
        jobSubResult = data['art']

    # protection of json format change from list to list of dicts
    if 'result' in jobSubResult and isinstance(jobSubResult['result'], list):
        resultlist = []
        for r in jobSubResult['result']:
            if not isinstance(r, dict):
                resultlist.append({'name': '', 'result': r})
            else:
                resultlist.append({'name': r['name'] if 'name' in r else '', 'result': r['result'] if 'result' in r else r})
        jobSubResult['result'] = resultlist
    return jobSubResult

def subresults_getter(url_params_str):
    """
    A function for getting ART jobs sub results in multithreading mode
    :return: dictionary with sub-results
    """
    base_url = "http://bigpanda.cern.ch/"
    fb_path = "filebrowser/?json=1"
    subresults_dict = {}

    pandaidstr = url_params_str.split('=')[-1]
    try:
        pandaid = int(pandaidstr)
    except:
        _logger.exception('Exception was caught while transforming pandaid from str to int.')
        raise

    http = urllib3.PoolManager()
    resp = http.request('GET', base_url + fb_path + url_params_str, timeout=300)
    if resp and resp.status == 200 and len(resp.data) > 0:
        try:
            data = json.loads(resp.data)
            tardir = data['tardir']
            MEDIA_URL = data['MEDIA_URL']
            dirprefix = data['dirprefix']
            files = data['files']
            files = [f for f in files if 'artReport.json' in f['name']]
        except:
            _logger.exception('Exception was caught while seeking artReport.json in logs for PanDA job: {}'.format(str(pandaid)))
            return {pandaid: subresults_dict}
    elif resp and resp.status == 429:
        _logger.info('Too many requests to filebrowser, return None for now, will try in next loop. PanDA job : {}'.format(str(pandaid)))
        return {pandaid: None}
    else:
        _logger.exception('Exception was caught while downloading logs using Rucio for PanDA job: {}'.format(str(pandaid)))
        return {pandaid: subresults_dict}

    if len(files) > 0:
        media_path = base_url + MEDIA_URL + dirprefix + "/" + tardir
        for f in files:
            url = media_path + "/" + f['name']
            response = http.request('GET', url)
            data = json.loads(response.data)
    else:
        _logger.error('No artReport.json file found in log tarball for PanDA job: {}'.format(str(pandaid)))
        return {pandaid: subresults_dict}

    if isinstance(data, dict) and 'art' in data:
        subresults_dict = data['art']

    # protection of json format change from list to list of dicts
    if 'result' in subresults_dict and isinstance(subresults_dict['result'], list):
        resultlist = []
        for r in subresults_dict['result']:
            if not isinstance(r, dict):
                resultlist.append({'name': '', 'result': r})
            else:
                resultlist.append({'name': r['name'] if 'name' in r else '', 'result': r['result'] if 'result' in r else r})
            subresults_dict['result'] = resultlist

    _logger.info('ART Results for {} is {}'.format(str(pandaid), str(subresults_dict)))

    return {pandaid: subresults_dict}

def save_subresults(subResultsDict):
    """
    A function to save subresults of ART jobs to the special table - ART_SUBRESULT
    :param subResultsDict:
    :return: True or False
    """

    with transaction.atomic():
        for pandaid, data in subResultsDict.items():
            try:
                row = ARTSubResult(pandaid=pandaid,
                                   subresult=data)
                row.save()
            except DatabaseError as e:
                _logger_error.error(e)
                return False

    return True

def lock_nqueuedjobs(cur, nrows):
    """
    Function to lock first N rows for futher processing
    :param nrows:
    :return: lock_time
    """

    lock_time = datetime.now().strftime(defaultDatetimeFormat)
    lquery = """UPDATE atlas_pandabigmon.art_results_queue
                SET IS_LOCKED = 1,
                    LOCK_TIME = to_date('%s', 'YYYY-MM-DD HH24:MI:SS')
                WHERE rownum <= %i AND IS_LOCKED = 0""" % (lock_time, nrows)
    try:
        cur.execute(lquery)
    except DatabaseError as e:
        _logger_error.error(e)
        raise

    return lock_time

def delete_queuedjobs(cur, lock_time):
    """
    A function to delete processed jobs from ART_RESULTS_QUEUE
    :param lock_time:
    :return:
    """

    dquery = """DELETE FROM atlas_pandabigmon.art_results_queue
                  WHERE IS_LOCKED = 1
                    AND LOCK_TIME = to_date('%s', 'YYYY-MM-DD HH24:MI:SS')""" % (lock_time)
    try:
        cur.execute(dquery)
    except DatabaseError as e:
        _logger_error.error(e)
        raise

    return True

def clear_queue(cur):
    """
    A function to delete processed jobs from ART_RESULTS_QUEUE
    :param lock_time:
    :return:
    """

    cquery = """DELETE FROM atlas_pandabigmon.art_results_queue
                  WHERE IS_LOCKED = 1"""
    try:
        cur.execute(cquery)
    except DatabaseError as e:
        _logger_error.error(e)
        raise

    return True

def get_final_result(job):
    """ A function to determine the real ART test result depending on sub-step results, exitcode and PanDA job state
    0 - succeeded - green
    1 - finished - yellow
    2 - active - blue
    3 - failed - red
    """
    finalResultDict = {0: 'succeeded', 1: 'finished', 2: 'active', 3: 'failed'}
    extraParamsDict = {
        'testexitcode': None,
        'subresults': None,
        'testdirectory': None,
        'reportjira': None,
        'reportmail': None,
        'description': None,
        'linktoplots': None,
        }
    finalresult = ''
    if job['jobstatus'] == 'finished':
        finalresult = 0
    elif job['jobstatus'] in ('failed', 'cancelled'):
        finalresult = 3
    else:
        finalresult = 2
    try:
        job['result'] = json.loads(job['result'])
    except:
        job['result'] = None
    try:
        extraParamsDict['testexitcode'] = job['result']['exit_code'] if 'exit_code' in job['result'] else None
    except:
        pass
    try:
        extraParamsDict['subresults'] = job['result']['result'] if 'result' in job['result'] else []
    except:
        pass
    try:
        extraParamsDict['testdirectory'] = job['result']['test_directory'] if 'test_directory' in job['result'] else []
    except:
        pass
    try:
        extraParamsDict['reportjira'] = job['result']['report-to']['jira'] if 'report-to' in job['result'] and 'jira' in job['result']['report-to'] else None
    except:
        pass
    try:
        extraParamsDict['reportmail'] = job['result']['report-to']['mail'] if 'report-to' in job['result'] and 'mail' in job['result']['report-to'] else None
    except:
        pass
    try:
        extraParamsDict['description'] = job['result']['description'] if 'description' in job['result'] else None
    except:
        pass
    try:
        extraParamsDict['linktoplots'] = job['result']['url'] if 'url' in job['result'] else None
    except:
        pass

    if job['result'] is not None:
        if 'result' in job['result'] and len(job['result']['result']) > 0:
            finalresult = analize_test_subresults(job['result']['result'])
            # for r in job['result']['result']:
            #     if int(r['result']) > 0:
            #         finalresult = 'failed'
        elif 'exit_code' in job['result'] and job['result']['exit_code'] > 0:
            finalresult = 3

    finalresult = finalResultDict[finalresult]

    return finalresult, extraParamsDict

def analize_test_subresults(subresults):
    """A function for analysing a sub-step results to decide if test failed in its first steps which are Athena or Reco
    or in additional checks and comparison steps
    """
    finalstate = -1

    if not any([r['result'] for r in subresults]) > 0:
        finalstate = 0
    else:
        weights = [math.exp(-i) for i in range(0, len(subresults))]
        weights_normalized = [w/sum(weights) for w in weights]
        bin_subresults = [0 if int(r['result']) > 0 else 1 for r in subresults]
        weighted_subresults = np.multiply(weights_normalized, bin_subresults)
        if sum(weighted_subresults) > 0.5:
            finalstate = 1
        else:
            finalstate = 3

    return finalstate<commit_msg>ART | fix of showing tests as green while subresults are not loaded yet<commit_after>

import urllib3, json, math
import numpy as np
from datetime import datetime
from core.art.modelsART import ARTSubResult
from django.db import transaction, DatabaseError
from core.settings import defaultDatetimeFormat
import logging

_logger_error = logging.getLogger('bigpandamon-error')
_logger = logging.getLogger('bigpandamon')

def getJobReport(guid, lfn, scope):
    filebrowserURL = "http://bigpanda.cern.ch/filebrowser/"  # This is deployment specific because memory monitoring is intended to work in ATLAS
    jobSubResult = []
    http = urllib3.PoolManager()
    resp = http.request('GET', filebrowserURL, fields={'guid': guid, 'lfn': lfn, 'scope': scope, 'json': 1})
    if resp and len(resp.data) > 0:
        try:
            data = json.loads(resp.data)
            HOSTNAME = data['HOSTNAME']
            tardir = data['tardir']
            MEDIA_URL = data['MEDIA_URL']
            dirprefix = data['dirprefix']
            files = data['files']
            files = [f for f in files if 'jobReport.json' in f['name']]
        except:
            return -2
    else:
        return -2

    urlBase = "http://" + HOSTNAME + "/" + MEDIA_URL + dirprefix + "/" + tardir

    for f in files:
        url = urlBase + "/" + f['name']
        response = http.request('GET', url)
        data = json.loads(response.data)

    return data

def getARTjobSubResults(data):
    jobSubResult = {}

    if isinstance(data, dict) and 'art' in data:
        jobSubResult = data['art']

    # protection of json format change from list to list of dicts
    if 'result' in jobSubResult and isinstance(jobSubResult['result'], list):
        resultlist = []
        for r in jobSubResult['result']:
            if not isinstance(r, dict):
                resultlist.append({'name': '', 'result': r})
            else:
                resultlist.append({'name': r['name'] if 'name' in r else '', 'result': r['result'] if 'result' in r else r})
        jobSubResult['result'] = resultlist
    return jobSubResult

def subresults_getter(url_params_str):
    """
    A function for getting ART jobs sub results in multithreading mode
    :return: dictionary with sub-results
    """
    base_url = "http://bigpanda.cern.ch/"
    fb_path = "filebrowser/?json=1"
    subresults_dict = {}

    pandaidstr = url_params_str.split('=')[-1]
    try:
        pandaid = int(pandaidstr)
    except:
        _logger.exception('Exception was caught while transforming pandaid from str to int.')
        raise

    http = urllib3.PoolManager()
    resp = http.request('GET', base_url + fb_path + url_params_str, timeout=300)
    if resp and resp.status == 200 and len(resp.data) > 0:
        try:
            data = json.loads(resp.data)
            tardir = data['tardir']
            MEDIA_URL = data['MEDIA_URL']
            dirprefix = data['dirprefix']
            files = data['files']
            files = [f for f in files if 'artReport.json' in f['name']]
        except:
            _logger.exception('Exception was caught while seeking artReport.json in logs for PanDA job: {}'.format(str(pandaid)))
            return {pandaid: subresults_dict}
    elif resp and resp.status == 429:
        _logger.info('Too many requests to filebrowser, return None for now, will try in next loop. PanDA job : {}'.format(str(pandaid)))
        return {pandaid: None}
    else:
        _logger.exception('Exception was caught while downloading logs using Rucio for PanDA job: {}'.format(str(pandaid)))
        return {pandaid: subresults_dict}

    if len(files) > 0:
        media_path = base_url + MEDIA_URL + dirprefix + "/" + tardir
        for f in files:
            url = media_path + "/" + f['name']
            response = http.request('GET', url)
            data = json.loads(response.data)
    else:
        _logger.error('No artReport.json file found in log tarball for PanDA job: {}'.format(str(pandaid)))
        return {pandaid: subresults_dict}

    if isinstance(data, dict) and 'art' in data:
        subresults_dict = data['art']

    # protection of json format change from list to list of dicts
    if 'result' in subresults_dict and isinstance(subresults_dict['result'], list):
        resultlist = []
        for r in subresults_dict['result']:
            if not isinstance(r, dict):
                resultlist.append({'name': '', 'result': r})
            else:
                resultlist.append({'name': r['name'] if 'name' in r else '', 'result': r['result'] if 'result' in r else r})
            subresults_dict['result'] = resultlist

    _logger.info('ART Results for {} is {}'.format(str(pandaid), str(subresults_dict)))

    return {pandaid: subresults_dict}

def save_subresults(subResultsDict):
    """
    A function to save subresults of ART jobs to the special table - ART_SUBRESULT
    :param subResultsDict:
    :return: True or False
    """

    with transaction.atomic():
        for pandaid, data in subResultsDict.items():
            try:
                row = ARTSubResult(pandaid=pandaid,
                                   subresult=data)
                row.save()
            except DatabaseError as e:
                _logger_error.error(e)
                return False

    return True

def lock_nqueuedjobs(cur, nrows):
    """
    Function to lock first N rows for futher processing
    :param nrows:
    :return: lock_time
    """

    lock_time = datetime.now().strftime(defaultDatetimeFormat)
    lquery = """UPDATE atlas_pandabigmon.art_results_queue
                SET IS_LOCKED = 1,
                    LOCK_TIME = to_date('%s', 'YYYY-MM-DD HH24:MI:SS')
                WHERE rownum <= %i AND IS_LOCKED = 0""" % (lock_time, nrows)
    try:
        cur.execute(lquery)
    except DatabaseError as e:
        _logger_error.error(e)
        raise

    return lock_time

def delete_queuedjobs(cur, lock_time):
    """
    A function to delete processed jobs from ART_RESULTS_QUEUE
    :param lock_time:
    :return:
    """

    dquery = """DELETE FROM atlas_pandabigmon.art_results_queue
                  WHERE IS_LOCKED = 1
                    AND LOCK_TIME = to_date('%s', 'YYYY-MM-DD HH24:MI:SS')""" % (lock_time)
    try:
        cur.execute(dquery)
    except DatabaseError as e:
        _logger_error.error(e)
        raise

    return True

def clear_queue(cur):
    """
    A function to delete processed jobs from ART_RESULTS_QUEUE
    :param lock_time:
    :return:
    """

    cquery = """DELETE FROM atlas_pandabigmon.art_results_queue
                  WHERE IS_LOCKED = 1"""
    try:
        cur.execute(cquery)
    except DatabaseError as e:
        _logger_error.error(e)
        raise

    return True

def get_final_result(job):
    """ A function to determine the real ART test result depending on sub-step results, exitcode and PanDA job state
    0 - succeeded - green
    1 - finished - yellow
    2 - active - blue
    3 - failed - red
    """
    finalResultDict = {0: 'succeeded', 1: 'finished', 2: 'active', 3: 'failed'}
    extraParamsDict = {
        'testexitcode': None,
        'subresults': None,
        'testdirectory': None,
        'reportjira': None,
        'reportmail': None,
        'description': None,
        'linktoplots': None,
        }
    finalresult = ''
    if job['jobstatus'] == 'finished':
        finalresult = 0
    elif job['jobstatus'] in ('failed', 'cancelled'):
        finalresult = 3
    else:
        finalresult = 2
    try:
        job['result'] = json.loads(job['result'])
    except:
        job['result'] = None
    try:
        extraParamsDict['testexitcode'] = job['result']['exit_code'] if 'exit_code' in job['result'] else None
    except:
        pass
    try:
        extraParamsDict['subresults'] = job['result']['result'] if 'result' in job['result'] else None
    except:
        pass
    try:
        extraParamsDict['testdirectory'] = job['result']['test_directory'] if 'test_directory' in job['result'] else []
    except:
        pass
    try:
        extraParamsDict['reportjira'] = job['result']['report-to']['jira'] if 'report-to' in job['result'] and 'jira' in job['result']['report-to'] else None
    except:
        pass
    try:
        extraParamsDict['reportmail'] = job['result']['report-to']['mail'] if 'report-to' in job['result'] and 'mail' in job['result']['report-to'] else None
    except:
        pass
    try:
        extraParamsDict['description'] = job['result']['description'] if 'description' in job['result'] else None
    except:
        pass
    try:
        extraParamsDict['linktoplots'] = job['result']['url'] if 'url' in job['result'] else None
    except:
        pass

    if job['result'] is not None:
        if 'result' in job['result'] and len(job['result']['result']) > 0:
            finalresult = analize_test_subresults(job['result']['result'])
            # for r in job['result']['result']:
            #     if int(r['result']) > 0:
            #         finalresult = 'failed'
        elif 'exit_code' in job['result'] and job['result']['exit_code'] > 0:
            finalresult = 3

    finalresult = finalResultDict[finalresult]

    return finalresult, extraParamsDict

def analize_test_subresults(subresults):
    """A function for analysing a sub-step results to decide if test failed in its first steps which are Athena or Reco
    or in additional checks and comparison steps
    """
    finalstate = -1

    if not any([r['result'] for r in subresults]) > 0:
        finalstate = 0
    else:
        weights = [math.exp(-i) for i in range(0, len(subresults))]
        weights_normalized = [w/sum(weights) for w in weights]
        bin_subresults = [0 if int(r['result']) > 0 else 1 for r in subresults]
        weighted_subresults = np.multiply(weights_normalized, bin_subresults)
        if sum(weighted_subresults) > 0.5:
            finalstate = 1
        else:
            finalstate = 3

    return finalstate<|endoftext|>


<commit_before>import requests
import re
import urllib
from flask import Flask
from flask import jsonify

app = Flask(__name__)
app.debug = True

@app.route('/')
def index():
	return 'SkipBait'

# this is the route to access the meat of the application. you can test it with the following urls.
# since the app consumes the url to skip as a url parameter, it is necessary to escape the url to skip.
# http://127.0.0.1:5000/skip/http%3A%2F%2Fwww.buzzfeed.com%2Fbradesposito%2Fno-pants-dance
# http://127.0.0.1:5000/skip/http%3A%2F%2Fsamjbrenner.com%2Fnotes%2Fmariah-careys-hand%2F
@app.route('/skip/<path:path>')
def skip_url(path):
	site_url = urllib.unquote(path)
	site_html = requests.get(site_url).text
	sources = get_sources(site_html)

	return jsonify(sources = sources, original_url = site_url)

if __name__ == '__main__':
	app.run()

def get_sources(site_html):
	# here are example uris for embedded content to help build the regexes. currently, it seems like this
	# app will always have a bias towards what is considered a primary source - a factor of what sites I build
	# regexes to identify. I can limit the bias by building a final regex that will check for all src attributes
	# of <iframe> tags, but this will throw a lot of false positives and miss a lot of actual primary sources.
	# it will be necessary to keep these regexes up-to-date with popular sources for stolen material.

	# vine
	# https://vine.co/v/MPu0Tq52wiv/card

	# youtube
	# https://www.youtube.com/watch?v=ZwfLgXJpQd0

	# vimeo
	# http://player.vimeo.com/video/55880815
	# http://vimeo.com/55880815

	# when building the patterns, be sure to name the entire url as a capture group
	# called <url> and the id of the embedded content as <id>. we will refer to the
	# groups in get_sources_from_regex_search()
	vine_pattern = '(?P<url>(https?://(www\.)?vine.co/v/(?P<id>([A-Za-z0-9]+)/?)))'
	youtube_pattern = '(?P<url>(https?://(www\.)?youtube.com/watch\?v=(?P<id>([A-Za-z0-9]+))))'
	vimeo_pattern = '(?P<url>(https?://(www\.)?(player\.)?vimeo.com/(video/)?(?P<id>([0-9]+))))'

	patterns = [vine_pattern, youtube_pattern, vimeo_pattern]
	sources = []

	for pattern in patterns:
		# perform regex find
		find = re.finditer(pattern, site_html)

		# extract full string and store in array
		sources.extend(get_sources_from_regex_search(find))

	return sources

def get_sources_from_regex_search(regex_search):
	sources = []
	ids = []

	for match in regex_search:
		# use the <id> capturing group to prevent duplicates
		id = match.group('id')

		if not (id in ids):
			ids.append(id)
			sources.append(match.group('url'))

	return sources<commit_msg>adding more regexes<commit_after>import requests
import re
import urllib
from flask import Flask
from flask import jsonify

app = Flask(__name__)
app.debug = True

@app.route('/')
def index():
	return 'SkipBait'

# this is the route to access the meat of the application. you can test it with the following urls.
# since the app consumes the url to skip as a url parameter, it is necessary to escape the url to skip.
# http://127.0.0.1:5000/skip/http%3A%2F%2Fwww.buzzfeed.com%2Fbradesposito%2Fno-pants-dance
# http://127.0.0.1:5000/skip/http%3A%2F%2Fsamjbrenner.com%2Fnotes%2Fmariah-careys-hand%2F
@app.route('/skip/<path:path>')
def skip_url(path):
	site_url = urllib.unquote(path)
	site_html = requests.get(site_url).text
	sources = get_sources(site_html)

	return jsonify(sources = sources, original_url = site_url)

if __name__ == '__main__':
	app.run()

def get_sources(site_html):
	# currently, it seems like this app will always have a bias towards what is considered
	# a primary source - a factor of what sites I build regexes to identify. I can limit the bias
	# by building a final regex that will check for all src attributes of <iframe> tags, but this
	# will throw a lot of false positives and miss a lot of actual primary sources. it will be
	# necessary to keep these regexes up-to-date with popular sources for stolen material.

	# when building the patterns, be sure to name the entire url as a capture group
	# called <url> and the id of the embedded content as <id>. we will refer to the
	# groups in get_sources_from_regex_search()
	vine_pattern = '(?P<url>(https?://(www\.)?vine.co/v/(?P<id>([A-Za-z0-9]+))/?))'
	youtube_pattern = '(?P<url>(https?://(www\.)?youtube.com/watch\?v=(?P<id>([A-Za-z0-9]+))))'
	vimeo_pattern = '(?P<url>(https?://(www\.)?(player\.)?vimeo.com/(video/)?(?P<id>([0-9]+))))'
	instagram_pattern = '(?P<url>(https?://(www\.)?instagram.com/p/(?P<id>([A-Za-z0-9]+))/?))'
	twitter_pattern = '(?P<url>(https?://(www\.)?twitter.com/[A-Za-z0-9]+/status/(?P<id>([0-9]+))))'
	# reddit_pattern =
	# http://www.reddit.com/r/soccer/comments/2a6ejg/match_thread_second_half_brazil_vs_germany_world/cirxo6e

	patterns = [vine_pattern, youtube_pattern, vimeo_pattern, instagram_pattern, twitter_pattern]
	sources = []

	for pattern in patterns:
		# perform regex find
		find = re.finditer(pattern, site_html)

		# extract full string and store in array
		sources.extend(get_sources_from_regex_search(find))

	return sources

def get_sources_from_regex_search(regex_search):
	sources = []
	ids = []

	for match in regex_search:
		# use the <id> capturing group to prevent duplicates
		id = match.group('id')

		if not (id in ids):
			ids.append(id)
			sources.append(match.group('url'))

	return sources<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2020 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Soft sort based losses for different learning tasks.

The main paradigm shift here, is to design losses that penalizes the ranks
instead of the values of the predictions.
"""

from __future__ import absolute_import
from __future__ import division

from __future__ import print_function

import gin
import tensorflow.compat.v2 as tf

from soft_sort import metrics
from soft_sort import ops

@gin.configurable
class TopKErrorLoss(tf.losses.Loss):
  """Based on a ranks function, compute the topk loss."""

  def __init__(self, topk, rescale_ranks=True, name=None):
    name = name if name is not None else 'topk_error'
    super(TopKErrorLoss, self).__init__(name=name)
    self._topk = topk
    self._rescale_ranks = rescale_ranks

  def get_ranks(self, y):
    """Compute the (hard) ranks of each entry in y."""
    diff = y[:, :, tf.newaxis] - y[:, tf.newaxis, :]
    diff -= tf.eye(y.shape[-1])[tf.newaxis, :, :] * 1e9
    s = tf.cast(diff > 0.0, dtype=tf.float32)
    return tf.math.reduce_sum(s, axis=-1)

  def rescale_ranks(self, ranks):
    mu = tf.math.reduce_mean(ranks, axis=-1)
    std = tf.math.reduce_std(ranks, axis=-1)
    n = tf.cast(tf.shape(ranks)[-1], tf.float32)
    tgt_mu = (n - 1) / 2
    tgt_std = tf.sqrt((n - 1) * (n + 1) / 12)
    return (ranks - mu[:, tf.newaxis]) / std[:, tf.newaxis] * tgt_std + tgt_mu

  def accuracy(self, y_true, y_pred):
    """Computes the soft topk accuracy of the prediction w.r.t the true values.

    Args:
     y_true: Tensor<float>[batch]: the true labels in [0, n-1].
     y_pred: Tensor<float>[batch, n]: n activation values for each input.

    Returns:
     A Tensor<float>[batch] of accuracy per batch.
    """
    ranks = self.get_ranks(y_pred)
    if self._rescale_ranks:
      ranks = self.rescale_ranks(ranks)

    # If the ranks are above topk then the accuracy is 1. Below that threshold
    # the accuracy decreases to 0.
    topk = tf.cast(self._topk, dtype=y_pred.dtype)
    num_activations = tf.shape(y_pred)[-1]
    accuracies = tf.math.minimum(
        1.0, ranks / (tf.cast(num_activations, dtype=y_pred.dtype) - topk))
    # Multiply with the one hot encoding of the label to select only the soft
    # topk accuracy of the true labels.
    true_labels = tf.one_hot(tf.cast(y_true, dtype=tf.int32),
                             depth=num_activations,
                             dtype=y_pred.dtype)
    return tf.reduce_sum(accuracies * true_labels, axis=-1)

  def call(self, y_true, y_pred):
    return 1.0 - self.accuracy(y_true, y_pred)

@gin.configurable
class SoftHeavisideErrorLoss(TopKErrorLoss):
  """Implements an error loss based on soft heavysides."""

  def __init__(self, topk=1, rescale_ranks=True, epsilon=1e-3):
    super(SoftHeavisideErrorLoss, self).__init__(
        topk=topk, rescale_ranks=rescale_ranks, name='soft_heaviside_error')
    self._epsilon = epsilon

  def get_ranks(self, y):

    @tf.custom_gradient
    def _ranks(y):
      diff = y[:, :, tf.newaxis] - y[:, tf.newaxis, :]
      diff -= tf.eye(y.shape[-1])[tf.newaxis, :, :] * 1e6
      s = 1.0 / (1 + tf.math.exp(-diff / self._epsilon))

      def grad(dy):
        return dy * tf.math.reduce_sum(s * (1.0 - s), axis=1) / self._epsilon

      return tf.math.reduce_sum(s, axis=-1), grad

    return _ranks(y)

@gin.configurable
class SoftErrorLoss(TopKErrorLoss):
  """Implementation of the soft error loss for classification."""

  def __init__(self, topk=1, rescale_ranks=True, **kwargs):
    super(SoftErrorLoss, self).__init__(
        topk=topk, rescale_ranks=rescale_ranks, name='soft_error')
    self._kwargs = kwargs

  @tf.function
  def get_ranks(self, y):
    return ops.softranks(
        y, direction='ASCENDING', axis=-1, zero_based=True, **self._kwargs)

@gin.configurable
class LeastQuantileRegressionLoss(tf.losses.Loss):
  """A loss for least quantile regression based on plain sort operations."""

  def __init__(self, quantile=0.5, power=1.0, **kwargs):
    self._quantile = quantile
    self._power = power
    self._kwargs = kwargs
    super(LeastQuantileRegressionLoss, self).__init__(name='lqr')

  def call(self, y_true, y_pred):
    return tf.reduce_mean(metrics.quantile_error(
        y_true, y_pred, quantile=self._quantile, power=self._power))

@gin.configurable
class SoftLeastQuantileRegressionLoss(tf.losses.Loss):
  """A loss for least quantile regression based on soft sort operators."""

  def __init__(self, quantile=0.5, power=1.0, **kwargs):
    self._quantile = quantile
    self._power = power
    self._kwargs = kwargs
    super(SoftLeastQuantileRegressionLoss, self).__init__(name='soft_lqr')

  def call(self, y_true, y_pred):
    error = tf.pow(tf.abs(tf.squeeze(y_pred) - y_true), self._power)
    return ops.softquantiles(error, self._quantile, axis=0, **self._kwargs)

@gin.configurable
class TrimmedRegressionLoss(tf.losses.Loss):
  """A loss for trimmed quantile regression."""

  def __init__(self, start_quantile, end_quantile, power=1.0, **kwargs):
    self._start_quantile = start_quantile
    self._end_quantile = end_quantile
    self._power = power
    self._kwargs = kwargs
    super(TrimmedRegressionLoss, self).__init__(name='trimmed')

  def call(self, y_true, y_pred):
    return tf.reduce_mean(metrics.trimmed_error(
        y_true, y_pred,
        self._start_quantile, self._end_quantile, power=self._power))

@gin.configurable
class SoftTrimmedRegressionLoss(tf.losses.Loss):
  """A loss for least quantile regression based on soft sort operators.

  In trimmed regression, we want to minimize the (usually quadratic) error
  between the prediction and the true value, discarding both the best errors and
  the worst ones and focusing on the median-ish ones, more precisely the ones
  between two given quantiles.

  In terms of soft sorting operator, this is obtained by transporting the
  predictions errors onto 3 sorted targets with weights based on those quantile
  values.

  To do this, we use the softquantiles operator with the proper target quantile
  and target quantile width.
  """

  def __init__(self, start_quantile, end_quantile, power=1.0, **kwargs):
    if end_quantile < start_quantile:
      raise ValueError(
          'Start quantile {:.3f} should be lower than end {:.3f}'.format(
              start_quantile, end_quantile))

    self._start_quantile = start_quantile
    self._end_quantile = end_quantile
    self._power = power
    self._kwargs = kwargs
    super(SoftTrimmedRegressionLoss, self).__init__(name='soft_trimmed')

  def call(self, y_true, y_pred):
    error = tf.pow(tf.abs(tf.squeeze(y_pred) - y_true), self._power)
    width = self._end_quantile - self._start_quantile
    quantile = 0.5 * (self._end_quantile + self._start_quantile)
    return ops.softquantiles(
        error, quantile, quantile_width=width, axis=0, **self._kwargs)
<commit_msg>Fix a dimension->int error in SoftHeavisideERrorLoss<commit_after># coding=utf-8
# Copyright 2020 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Soft sort based losses for different learning tasks.

The main paradigm shift here, is to design losses that penalizes the ranks
instead of the values of the predictions.
"""

from __future__ import absolute_import
from __future__ import division

from __future__ import print_function

import gin
import tensorflow.compat.v2 as tf

from soft_sort import metrics
from soft_sort import ops

@gin.configurable
class TopKErrorLoss(tf.losses.Loss):
  """Based on a ranks function, compute the topk loss."""

  def __init__(self, topk, rescale_ranks=True, name=None):
    name = name if name is not None else 'topk_error'
    super(TopKErrorLoss, self).__init__(name=name)
    self._topk = topk
    self._rescale_ranks = rescale_ranks

  def get_ranks(self, y):
    """Compute the (hard) ranks of each entry in y."""
    diff = y[:, :, tf.newaxis] - y[:, tf.newaxis, :]
    diff -= tf.eye(y.shape[-1])[tf.newaxis, :, :] * 1e9
    s = tf.cast(diff > 0.0, dtype=tf.float32)
    return tf.math.reduce_sum(s, axis=-1)

  def rescale_ranks(self, ranks):
    mu = tf.math.reduce_mean(ranks, axis=-1)
    std = tf.math.reduce_std(ranks, axis=-1)
    n = tf.cast(tf.shape(ranks)[-1], tf.float32)
    tgt_mu = (n - 1) / 2
    tgt_std = tf.sqrt((n - 1) * (n + 1) / 12)
    return (ranks - mu[:, tf.newaxis]) / std[:, tf.newaxis] * tgt_std + tgt_mu

  def accuracy(self, y_true, y_pred):
    """Computes the soft topk accuracy of the prediction w.r.t the true values.

    Args:
     y_true: Tensor<float>[batch]: the true labels in [0, n-1].
     y_pred: Tensor<float>[batch, n]: n activation values for each input.

    Returns:
     A Tensor<float>[batch] of accuracy per batch.
    """
    ranks = self.get_ranks(y_pred)
    if self._rescale_ranks:
      ranks = self.rescale_ranks(ranks)

    # If the ranks are above topk then the accuracy is 1. Below that threshold
    # the accuracy decreases to 0.
    topk = tf.cast(self._topk, dtype=y_pred.dtype)
    num_activations = tf.shape(y_pred)[-1]
    accuracies = tf.math.minimum(
        1.0, ranks / (tf.cast(num_activations, dtype=y_pred.dtype) - topk))
    # Multiply with the one hot encoding of the label to select only the soft
    # topk accuracy of the true labels.
    true_labels = tf.one_hot(tf.cast(y_true, dtype=tf.int32),
                             depth=num_activations,
                             dtype=y_pred.dtype)
    return tf.reduce_sum(accuracies * true_labels, axis=-1)

  def call(self, y_true, y_pred):
    return 1.0 - self.accuracy(y_true, y_pred)

@gin.configurable
class SoftHeavisideErrorLoss(TopKErrorLoss):
  """Implements an error loss based on soft heavysides."""

  def __init__(self, topk=1, rescale_ranks=True, epsilon=1e-3):
    super(SoftHeavisideErrorLoss, self).__init__(
        topk=topk, rescale_ranks=rescale_ranks, name='soft_heaviside_error')
    self._epsilon = epsilon

  def get_ranks(self, y):

    @tf.custom_gradient
    def _ranks(y):
      diff = y[:, :, tf.newaxis] - y[:, tf.newaxis, :]
      diff -= tf.eye(tf.shape(y)[-1], dtype=y.dtype)[tf.newaxis, :, :] * 1e6
      s = 1.0 / (1 + tf.math.exp(-diff / self._epsilon))

      def grad(dy):
        return dy * tf.math.reduce_sum(s * (1.0 - s), axis=1) / self._epsilon

      return tf.math.reduce_sum(s, axis=-1), grad

    return _ranks(y)

@gin.configurable
class SoftErrorLoss(TopKErrorLoss):
  """Implementation of the soft error loss for classification."""

  def __init__(self, topk=1, rescale_ranks=True, **kwargs):
    super(SoftErrorLoss, self).__init__(
        topk=topk, rescale_ranks=rescale_ranks, name='soft_error')
    self._kwargs = kwargs

  @tf.function
  def get_ranks(self, y):
    return ops.softranks(
        y, direction='ASCENDING', axis=-1, zero_based=True, **self._kwargs)

@gin.configurable
class LeastQuantileRegressionLoss(tf.losses.Loss):
  """A loss for least quantile regression based on plain sort operations."""

  def __init__(self, quantile=0.5, power=1.0, **kwargs):
    self._quantile = quantile
    self._power = power
    self._kwargs = kwargs
    super(LeastQuantileRegressionLoss, self).__init__(name='lqr')

  def call(self, y_true, y_pred):
    return tf.reduce_mean(metrics.quantile_error(
        y_true, y_pred, quantile=self._quantile, power=self._power))

@gin.configurable
class SoftLeastQuantileRegressionLoss(tf.losses.Loss):
  """A loss for least quantile regression based on soft sort operators."""

  def __init__(self, quantile=0.5, power=1.0, **kwargs):
    self._quantile = quantile
    self._power = power
    self._kwargs = kwargs
    super(SoftLeastQuantileRegressionLoss, self).__init__(name='soft_lqr')

  def call(self, y_true, y_pred):
    error = tf.pow(tf.abs(tf.squeeze(y_pred) - y_true), self._power)
    return ops.softquantiles(error, self._quantile, axis=0, **self._kwargs)

@gin.configurable
class TrimmedRegressionLoss(tf.losses.Loss):
  """A loss for trimmed quantile regression."""

  def __init__(self, start_quantile, end_quantile, power=1.0, **kwargs):
    self._start_quantile = start_quantile
    self._end_quantile = end_quantile
    self._power = power
    self._kwargs = kwargs
    super(TrimmedRegressionLoss, self).__init__(name='trimmed')

  def call(self, y_true, y_pred):
    return tf.reduce_mean(metrics.trimmed_error(
        y_true, y_pred,
        self._start_quantile, self._end_quantile, power=self._power))

@gin.configurable
class SoftTrimmedRegressionLoss(tf.losses.Loss):
  """A loss for least quantile regression based on soft sort operators.

  In trimmed regression, we want to minimize the (usually quadratic) error
  between the prediction and the true value, discarding both the best errors and
  the worst ones and focusing on the median-ish ones, more precisely the ones
  between two given quantiles.

  In terms of soft sorting operator, this is obtained by transporting the
  predictions errors onto 3 sorted targets with weights based on those quantile
  values.

  To do this, we use the softquantiles operator with the proper target quantile
  and target quantile width.
  """

  def __init__(self, start_quantile, end_quantile, power=1.0, **kwargs):
    if end_quantile < start_quantile:
      raise ValueError(
          'Start quantile {:.3f} should be lower than end {:.3f}'.format(
              start_quantile, end_quantile))

    self._start_quantile = start_quantile
    self._end_quantile = end_quantile
    self._power = power
    self._kwargs = kwargs
    super(SoftTrimmedRegressionLoss, self).__init__(name='soft_trimmed')

  def call(self, y_true, y_pred):
    error = tf.pow(tf.abs(tf.squeeze(y_pred) - y_true), self._power)
    width = self._end_quantile - self._start_quantile
    quantile = 0.5 * (self._end_quantile + self._start_quantile)
    return ops.softquantiles(
        error, quantile, quantile_width=width, axis=0, **self._kwargs)
<|endoftext|>