<commit_before>read = open('data.txt','r')
write = open('result.txt','w')

R = read.readline().rstrip()
B = read.readline().rstrip()
n = len(B)
m = len(R)
k = 3

print(R)
print(B)
print(m, n)

MAXLENGTH = [[0 for i in range(m)] for j in range(m)]
for i in range(m):
    for j in range(i, m):
        length = 0
        for p in range(m-j):
            if (R[i+p] != R[j+p]):
                break
            length = length+1
        MAXLENGTH[i][j] = length
        MAXLENGTH[j][i] = length

j = 0
choice = 0
for i in range(n-m+k+1):
    L = dict()
    T = dict()

    L[(d, abs(d)-2)] = -999
    if (d < 0):
        L[d, abs(d)-1] = abs(d)-1
    else:
        L[d, abs(d)-1] = -1

    for e in range(k+1):
        for d in range(-e, e+1):
            row = max(L[(d, e-1)]+1, L[(d-1, e-1)], L[(d+1, e-1)]+1)

            print(row)

            if (row == L[(d-1, e-1)]):
                    choice = 0
                    T[(d, e)] = T[(d-1, e-1)]
                    T[(d, e)].append((i+row+d-1,0,0))
                elif (row == L[(d, e-1)]+1):
                    choice = 1
                    T[(d, e)] = T[(d, e-1)]
                    T[(d, e)].append((i+row+d-1,0,0))
                elif (row == L[(d+1, e-1)]+1):
                    choice = 2
                    T[(d, e)] = T[(d+1, e-1)]

            print(T(d, e)]

            done=0
            while ((i+row+d+1) <= j) and (row < m) and ((i+row+d) < n):
                ##c i f
                if f>=1:
                    if f!=MAXLENGHT[c][row]:
                        row=row+min(f,MAXLENGHT(c,row))
                        done=1
                    else:
                        row=row+f
                else:
                    if b[i+row+d+1]!=r[row+1]:
                        done=1
                    else:
                        row=row+1
            while r[row+1]==b[i+row+1+d] and done==0 and row<m:
                row=row+1
            L[d,e]=row
            if L[d,e]==m:
                print('YES')
                break ##exits from last for loop
    ##novi simboli

read.close()
write.close()
<commit_msg>radi, treba vidjet sta je ispis i dodat za citanje fasta formata<commit_after>read = open('data.txt','r')
write = open('result.txt','w')

R = read.readline().rstrip()
B = read.readline().rstrip()
n = len(B)
m = len(R)
k = 3

MAXLENGTH = [[0 for i in range(m)] for j in range(m)]
for i in range(m):
    for j in range(i, m):
        length = 0
        for p in range(m-j):
            if (R[i+p] != R[j+p]):
                break
            length = length+1
        MAXLENGTH[i][j] = length
        MAXLENGTH[j][i] = length

S_ij = None
newS_ij = None
j = 0
max_j = 0
choice = 0
for i in range(n-m+k+1):
    L = dict()
    T = dict()
    best_match = 0

    for d in range(-(k+1), (k+1)+1):
        L[(d, abs(d)-2)] = -999
        T[(d, abs(d)-2)] = list()
        if (d < 0):
            L[(d, abs(d)-1)] = abs(d)-1
            T[(d, abs(d)-1)] = list()
        else:
            L[(d, abs(d)-1)] = -1
            T[(d, abs(d)-1)] = list()

    for e in range(k+1):
        for d in range(-e, e+1):
            row = max(L[(d, e-1)]+1, L[(d-1, e-1)], L[(d+1, e-1)]+1)
            old_row = row

            if (row == L[(d-1, e-1)]):
                choice = 0
                T[(d, e)] = T[(d-1, e-1)]
                T[(d, e)].append((i+row+d-1,0,0))
            elif (row == L[(d, e-1)]+1):
                choice = 1
                T[(d, e)] = T[(d, e-1)]
                T[(d, e)].append((i+row+d-1,0,0))
            elif (row == L[(d+1, e-1)]+1):
                choice = 2
                T[(d, e)] = T[(d+1, e-1)]

            done = 0
            while ((i+row+d+1) <= j) and (row < m) and ((i+row+d) < n):
                c=0
                f=0
                if (S_ij is not None):
                    j=i+row+d
                    for x in range(len(S_ij)):
                        triple = S_ij[x]
                        if (j == triple[0]) and (triple[2] == 0):
                            c = f = 0
                            break
                        else:
                            if (j <= triple[0]+triple[2]) and (j > triple[0]):
                                difference = j-triple[0]
                                c = triple[1]+difference
                                f = triple[2]-difference
                                break

                if (f >= 1):
                    if (f != MAXLENGTH[c][row]):
                        row = row+min(f, MAXLENGTH[c][row])
                        done = 1
                        break
                    else:
                        row = row+f
                else:
                    try:
                        if B[i+row+d] != R[row]:
                            done = 1
                            break
                    except IndexError:
                        print(row, i+row+d, j)
                    else:
                        row = row+1

            if (done == 0):
                while (row < m) and (i+row+d < n) and (R[row] == B[i+row+d]):
                    row = row+1

            L[(d,e)] = row

            if (old_row < row):
                T[(d,e)].append((i+old_row+d, old_row, row-old_row))
            if (i+row+d > max_j):
                max_j = i+row+d
                newS_ij = (d,e)
            if (row == m) and (best_match == 0):
                ##nest upisat u result
                print('start =', i, 'end =', i+m+d-1, 'diff =', e)
                best_match = 1

    if (max_j > j):
        j= max_j
        S_ij = T[newS_ij]

read.close()
write.close()
<|endoftext|>


<commit_before>"""
This implementation demonstrates how to generate the
elements of a Pascal's triangle. The element having
a row index of r and column index of c can be derived
as follows:
triangle[r][c] = triangle[r-1][c-1]+triangle[r-1][c]
What is Pascal's triangle?
- It is a triangular array containing binomial coefficients.
Refer to (https://en.wikipedia.org/wiki/Pascal%27s_triangle)
for more info about this triangle.
"""

def print_pascal_triangle(num_rows: int) -> None:
    """
    Print Pascal's triangle for different number of rows
    >>> print_pascal_triangle(5)
        1
       1 1
      1 2 1
     1 3 3 1
    1 4 6 4 1
    """
    triangle = generate_pascal_triangle(num_rows)
    for row_idx in range(num_rows):
        # Print left spaces
        for _ in range(num_rows - row_idx - 1):
            print(end=" ")
        # Print row values
        for col_idx in range(row_idx + 1):
            if col_idx != row_idx:
                print(triangle[row_idx][col_idx], end=" ")
            else:
                print(triangle[row_idx][col_idx], end="")
        print()

def generate_pascal_triangle(num_rows: int) -> list[list[int]]:
    """
    Create Pascal's triangle for different number of rows
    >>> generate_pascal_triangle(1)
    [[1]]
    >>> generate_pascal_triangle(2)
    [[1], [1, 1]]
    >>> generate_pascal_triangle(3)
    [[1], [1, 1], [1, 2, 1]]
    >>> generate_pascal_triangle(4)
    [[1], [1, 1], [1, 2, 1], [1, 3, 3, 1]]
    >>> generate_pascal_triangle(5)
    [[1], [1, 1], [1, 2, 1], [1, 3, 3, 1], [1, 4, 6, 4, 1]]
    """
    triangle: list[list[int]] = []
    for current_row_idx in range(num_rows):
        current_row = populate_current_row(triangle, current_row_idx)
        triangle.append(current_row)
    return triangle

def populate_current_row(triangle: list[list[int]], current_row_idx: int) -> list[int]:
    """
    >>> triangle = [[1]]
    >>> populate_current_row(triangle, 1)
    [1, 1]
    """
    current_row = [-1] * (current_row_idx + 1)
    # first and last elements of current row are equal to 1
    current_row[0], current_row[-1] = 1, 1
    for current_col_idx in range(1, current_row_idx):
        calculate_current_element(
            triangle, current_row, current_row_idx, current_col_idx
        )
    return current_row

def calculate_current_element(
    triangle: list[list[int]],
    current_row: list[int],
    current_row_idx: int,
    current_col_idx: int,
) -> None:
    """
    >>> triangle = [[1], [1, 1]]
    >>> current_row = [1, -1, 1]
    >>> calculate_current_element(triangle, current_row, 2, 1)
    >>> current_row
    [1, 2, 1]
    """
    above_to_left_elt = triangle[current_row_idx - 1][current_col_idx - 1]
    above_to_right_elt = triangle[current_row_idx - 1][current_col_idx]
    current_row[current_col_idx] = above_to_left_elt + above_to_right_elt

if __name__ == "__main__":
    import doctest

    doctest.testmod()
<commit_msg>Modified 'pascal_triangle.py' program (#7901)<commit_after>"""
This implementation demonstrates how to generate the elements of a Pascal's triangle.
The element havingva row index of r and column index of c can be derivedvas follows:
triangle[r][c] = triangle[r-1][c-1]+triangle[r-1][c]

A Pascal's triangle is a triangular array containing binomial coefficients.
https://en.wikipedia.org/wiki/Pascal%27s_triangle
"""

def print_pascal_triangle(num_rows: int) -> None:
    """
    Print Pascal's triangle for different number of rows
    >>> print_pascal_triangle(5)
        1
       1 1
      1 2 1
     1 3 3 1
    1 4 6 4 1
    """
    triangle = generate_pascal_triangle(num_rows)
    for row_idx in range(num_rows):
        # Print left spaces
        for _ in range(num_rows - row_idx - 1):
            print(end=" ")
        # Print row values
        for col_idx in range(row_idx + 1):
            if col_idx != row_idx:
                print(triangle[row_idx][col_idx], end=" ")
            else:
                print(triangle[row_idx][col_idx], end="")
        print()

def generate_pascal_triangle(num_rows: int) -> list[list[int]]:
    """
    Create Pascal's triangle for different number of rows
    >>> generate_pascal_triangle(0)
    []
    >>> generate_pascal_triangle(1)
    [[1]]
    >>> generate_pascal_triangle(2)
    [[1], [1, 1]]
    >>> generate_pascal_triangle(3)
    [[1], [1, 1], [1, 2, 1]]
    >>> generate_pascal_triangle(4)
    [[1], [1, 1], [1, 2, 1], [1, 3, 3, 1]]
    >>> generate_pascal_triangle(5)
    [[1], [1, 1], [1, 2, 1], [1, 3, 3, 1], [1, 4, 6, 4, 1]]
    >>> generate_pascal_triangle(-5)
    Traceback (most recent call last):
        ...
    ValueError: The input value of 'num_rows' should be greater than or equal to 0
    >>> generate_pascal_triangle(7.89)
    Traceback (most recent call last):
        ...
    TypeError: The input value of 'num_rows' should be 'int'
    """

    if not isinstance(num_rows, int):
        raise TypeError("The input value of 'num_rows' should be 'int'")

    if num_rows == 0:
        return []
    elif num_rows < 0:
        raise ValueError(
            "The input value of 'num_rows' should be greater than or equal to 0"
        )

    triangle: list[list[int]] = []
    for current_row_idx in range(num_rows):
        current_row = populate_current_row(triangle, current_row_idx)
        triangle.append(current_row)
    return triangle

def populate_current_row(triangle: list[list[int]], current_row_idx: int) -> list[int]:
    """
    >>> triangle = [[1]]
    >>> populate_current_row(triangle, 1)
    [1, 1]
    """
    current_row = [-1] * (current_row_idx + 1)
    # first and last elements of current row are equal to 1
    current_row[0], current_row[-1] = 1, 1
    for current_col_idx in range(1, current_row_idx):
        calculate_current_element(
            triangle, current_row, current_row_idx, current_col_idx
        )
    return current_row

def calculate_current_element(
    triangle: list[list[int]],
    current_row: list[int],
    current_row_idx: int,
    current_col_idx: int,
) -> None:
    """
    >>> triangle = [[1], [1, 1]]
    >>> current_row = [1, -1, 1]
    >>> calculate_current_element(triangle, current_row, 2, 1)
    >>> current_row
    [1, 2, 1]
    """
    above_to_left_elt = triangle[current_row_idx - 1][current_col_idx - 1]
    above_to_right_elt = triangle[current_row_idx - 1][current_col_idx]
    current_row[current_col_idx] = above_to_left_elt + above_to_right_elt

def generate_pascal_triangle_optimized(num_rows: int) -> list[list[int]]:
    """
    This function returns a matrix representing the corresponding pascal's triangle
    according to the given input of number of rows of Pascal's triangle to be generated.
    It reduces the operations done to generate a row by half
    by eliminating redundant calculations.

    :param num_rows: Integer specifying the number of rows in the Pascal's triangle
    :return: 2-D List (matrix) representing the Pascal's triangle

    Return the Pascal's triangle of given rows
    >>> generate_pascal_triangle_optimized(3)
    [[1], [1, 1], [1, 2, 1]]
    >>> generate_pascal_triangle_optimized(1)
    [[1]]
    >>> generate_pascal_triangle_optimized(0)
    []
    >>> generate_pascal_triangle_optimized(-5)
    Traceback (most recent call last):
        ...
    ValueError: The input value of 'num_rows' should be greater than or equal to 0
    >>> generate_pascal_triangle_optimized(7.89)
    Traceback (most recent call last):
        ...
    TypeError: The input value of 'num_rows' should be 'int'
    """

    if not isinstance(num_rows, int):
        raise TypeError("The input value of 'num_rows' should be 'int'")

    if num_rows == 0:
        return []
    elif num_rows < 0:
        raise ValueError(
            "The input value of 'num_rows' should be greater than or equal to 0"
        )

    result: list[list[int]] = [[1]]

    for row_index in range(1, num_rows):
        temp_row = [0] + result[-1] + [0]
        row_length = row_index + 1
        # Calculate the number of distinct elements in a row
        distinct_elements = sum(divmod(row_length, 2))
        row_first_half = [
            temp_row[i - 1] + temp_row[i] for i in range(1, distinct_elements + 1)
        ]
        row_second_half = row_first_half[: (row_index + 1) // 2]
        row_second_half.reverse()
        row = row_first_half + row_second_half
        result.append(row)

    return result

def benchmark() -> None:
    """
    Benchmark multiple functions, with three different length int values.
    """
    from collections.abc import Callable
    from timeit import timeit

    def benchmark_a_function(func: Callable, value: int) -> None:
        call = f"{func.__name__}({value})"
        timing = timeit(f"__main__.{call}", setup="import __main__")
        # print(f"{call:38} = {func(value)} -- {timing:.4f} seconds")
        print(f"{call:38} -- {timing:.4f} seconds")

    for value in range(15):  # (1, 7, 14):
        for func in (generate_pascal_triangle, generate_pascal_triangle_optimized):
            benchmark_a_function(func, value)
        print()

if __name__ == "__main__":
    import doctest

    doctest.testmod()
    benchmark()
<|endoftext|>


<commit_before>"""
Interface that uses Blizzard's pages as the source.
"""
import functools
import logging

import asyncio
from lxml import etree

import aiohttp
from kyokai.context import HTTPRequestContext

from owapi import util

B_BASE_URL = "https://playoverwatch.com/en-us/"
B_PAGE_URL = B_BASE_URL + "career/{platform}{region}/{btag}"

logger = logging.getLogger("OWAPI")

async def get_page_body(ctx: HTTPRequestContext, url: str, cache_time=300) -> str:
    """
    Downloads page body from MasterOverwatch and caches it.
    """
    session = aiohttp.ClientSession(headers={"User-Agent": "OWAPI Scraper/1.0.0"})

    async def _real_get_body(_, url: str):
        # Real function.
        logger.info("GET => {}".format(url))
        async with session.get(url) as req:
            assert isinstance(req, aiohttp.ClientResponse)
            if req.status != 200:
                return None
            return (await req.read()).decode()

    result = await util.with_cache(ctx, _real_get_body, url, expires=cache_time)
    session.close()
    return result

def _parse_page(content: str) -> etree._Element:
    """
    Internal function to parse a page and return the data.
    """
    if content:
        data = etree.HTML(content)
        return data

async def get_user_page(ctx: HTTPRequestContext, battletag: str, platform: str="pc", region: str = "us", extra="",
                        cache_time=300) -> etree._Element:
    """
    Downloads the BZ page for a user, and parses it.
    """
    if platform != "pc":
        region = ""
    built_url = B_PAGE_URL.format(
        region=region, btag=battletag.replace("#", "-"), platform=platform) + "{}".format(extra)
    page_body = await get_page_body(ctx, built_url, cache_time=cache_time)

    if not page_body:
        return None

    # parse the page
    parse_partial = functools.partial(_parse_page, page_body)
    loop = asyncio.get_event_loop()
    parsed = await loop.run_in_executor(None, parse_partial)

    return parsed

async def region_helper(ctx: HTTPRequestContext, battletag: str, platform="pc", region=None, extra=""):
    """
    Downloads the correct page for a user in the right region.

    This will return either (etree._Element, region) or (None, None).
    """
    if region is None:
        reg_l = ["/eu", "/us", "/cn", "/kr"]
    else:
        reg_l = [region]

    for reg in reg_l:
        # Get the user page.
        page = await get_user_page(ctx, battletag, platform, reg, extra)
        # Check if the page was returned successfully.
        # If it was, return it.
        if page is not None:
            return page, reg
    else:
        # Since we continued without returning, give back the None, None.
        return None, None
<commit_msg>Return the "region" without /<commit_after>"""
Interface that uses Blizzard's pages as the source.
"""
import functools
import logging

import asyncio
from lxml import etree

import aiohttp
from kyokai.context import HTTPRequestContext

from owapi import util

B_BASE_URL = "https://playoverwatch.com/en-us/"
B_PAGE_URL = B_BASE_URL + "career/{platform}{region}/{btag}"

logger = logging.getLogger("OWAPI")

async def get_page_body(ctx: HTTPRequestContext, url: str, cache_time=300) -> str:
    """
    Downloads page body from MasterOverwatch and caches it.
    """
    session = aiohttp.ClientSession(headers={"User-Agent": "OWAPI Scraper/1.0.0"})

    async def _real_get_body(_, url: str):
        # Real function.
        logger.info("GET => {}".format(url))
        async with session.get(url) as req:
            assert isinstance(req, aiohttp.ClientResponse)
            if req.status != 200:
                return None
            return (await req.read()).decode()

    result = await util.with_cache(ctx, _real_get_body, url, expires=cache_time)
    session.close()
    return result

def _parse_page(content: str) -> etree._Element:
    """
    Internal function to parse a page and return the data.
    """
    if content:
        data = etree.HTML(content)
        return data

async def get_user_page(ctx: HTTPRequestContext, battletag: str, platform: str="pc", region: str = "us", extra="",
                        cache_time=300) -> etree._Element:
    """
    Downloads the BZ page for a user, and parses it.
    """
    if platform != "pc":
        region = ""
    built_url = B_PAGE_URL.format(
        region=region, btag=battletag.replace("#", "-"), platform=platform) + "{}".format(extra)
    page_body = await get_page_body(ctx, built_url, cache_time=cache_time)

    if not page_body:
        return None

    # parse the page
    parse_partial = functools.partial(_parse_page, page_body)
    loop = asyncio.get_event_loop()
    parsed = await loop.run_in_executor(None, parse_partial)

    return parsed

async def region_helper(ctx: HTTPRequestContext, battletag: str, platform="pc", region=None, extra=""):
    """
    Downloads the correct page for a user in the right region.

    This will return either (etree._Element, region) or (None, None).
    """
    if region is None:
        reg_l = ["/eu", "/us", "/cn", "/kr"]
    else:
        reg_l = [region]

    for reg in reg_l:
        # Get the user page.
        page = await get_user_page(ctx, battletag, platform, reg, extra)
        # Check if the page was returned successfully.
        # If it was, return it.
        if page is not None:
            return page, reg[1:]
    else:
        # Since we continued without returning, give back the None, None.
        return None, None
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Name :          qconcurrency.widgets._progressbar_.py
Created :       Apr 17, 2017
Author :        Will Pittman
Contact :       willjpittman@gmail.com
________________________________________________________________________________
Description :   ProgressBar, that can be updated from a separate thread.
________________________________________________________________________________
"""
#builtin
from __future__    import unicode_literals
from __future__    import absolute_import
from __future__    import division
from __future__    import print_function
from   collections import Iterable
import functools
import uuid
import copy
#external
from Qt            import QtWidgets
#internal
from qconcurrency.threading_  import ThreadedTask, SoloThreadedTask

#!TODO: http://stackoverflow.com/questions/7108715/how-to-show-hide-a-child-qwidget-with-a-motion-animation

class _ProgressSoloThreadedTask( SoloThreadedTask ):
    """
    customized :py:obj:`qconcurrency.threading_.SoloThreadedTask` that
    assigns a new jobid for each thread it starts, and each thread's progress
    is measured entirely independently.

    This way, there is no race-condition where cancelling one thread can knock out
    progress on *all* threads.
    """
    def __init__(self, progressbar, callback, signals=None, connections=None, mutex_expiry=5000 ):
        self._progressbar = progressbar
        SoloThreadedTask.__init__(self,
            callback     = callback,
            signals      = signals,
            connections  = connections,
            mutex_expiry = mutex_expiry,
        )

    def start(self, expiryTimeout=-1, threadpool=None, wait=False, *args, **kwds ):
        """
        Wraps :py:meth:`qconcurrency.threading_.SoloThreadedTask.start` ,
        adding signal-connections so that they update a progressbar.

        See Also:

            * :py:meth:`qconcurrency.threading_.SoloThreadedTask.start`
        """
        jobid = uuid.uuid4().hex

        connections = self._connections.copy()
        if not connections:
            connections = {}

        progbar_connections = {
            'incr_progress' : functools.partial( self._progressbar.incr_progress, jobid=jobid ),
            'add_progress'  : functools.partial( self._progressbar.add_progress,  jobid=jobid ),
            'returned'      : functools.partial( self._progressbar._handle_return_or_abort, jobid=jobid ),
            'exception'     : functools.partial( self._progressbar._handle_return_or_abort, jobid=jobid ),
        }

        for signal in progbar_connections:
            if signal in connections:
                connections[ signal ].append( progbar_connections[signal] )
            else:
                connections[ signal ] = [ progbar_connections[signal] ]

        SoloThreadedTask.start(self,
                           expiryTimeout = expiryTimeout,
                           threadpool    = threadpool,
                           wait          = wait,
                           _connections  = connections,
                           *args,**kwds
                       )

class ProgressBar( QtWidgets.QProgressBar ):
    """
    A ProgressBar designed to track progress of several threads,
    that is hidden automatically whenever all threads have exited
    (by unhandled exception, or return).

    Each thread is assigned it's own `jobid`, so that it's progress
    is tracked entirely independently of all other pending tasks.
    (my hope is that if errors appear in total-calculated progress in the
    codebase, this will lessen the appearance of an error for the user - each
    thread's progress being set to 100% once it exits).
    """
    def __init__(self):
        QtWidgets.QProgressBar.__init__(self)

        self._progress = {}  # { jobid: {'total':10, 'current':3} }
        self.setHidden(True)

    def add_progress(self, amount, jobid=None):
        """
        Adds to the total number of required steps
        required to complete.
        """

        if jobid not in self._progress:
            self._progress[ jobid ] = {'total':amount, 'current':0}
        else:
            self._progress[ jobid ]['total'] += amount

        self.refresh_progress()
        self.setHidden(False)

    def refresh_progress(self):
        """
        ReCalculates current/total progress, and updates the progressbar
        """

        total_progress   = 0
        current_progress = 0
        for jobid in self._progress.keys():
            total_progress   += self._progress[ jobid ]['total']
            current_progress += self._progress[ jobid ]['current']

            if current_progress == total_progress:
                self._progress.pop(jobid)

        self.setMaximum( total_progress   )
        self.setValue(   current_progress )

        if total_progress <= current_progress:
            self.setHidden(True)

    def incr_progress(self, amount, jobid):
        """
        Completes a particular number of steps for
        a particular jobid.
        """

        if amount == None:
            amount = 1

        if jobid not in self._progress:
            return

        self._progress[ jobid ]['current'] += amount
        self.refresh_progress()

    def reset(self, jobid=None ):
        """
        If not `jobid`, sets all required progress-steps back to 0.
        Otherwise, removes all required progress associated with that
        jobid.
        """

        if not jobid:
            self._progress = {}
            QtWidgets.QProgressBar.reset(self)

        elif jobid in self._progress:
            self._progress.pop( jobid )
            self.refresh_progress()

    def new_task(self, callback, signals=None, *args, **kwds ):
        """
        Creates a new :py:obj:`ThreadedTask` object, adding
        signals to it so that it can update this :py:obj:`ProgressBar`.

        :py:obj:`ThreadedTask` objects are most suitable for
        producer/consumer patterns (multiple threads running at once),
        and no thread depends on another.

        See also:
            * :py:obj:`ThreadedTask`
            * :py:obj:`SoloThreadedTask`
            * :py:meth:`ProgressBar.new_solotask`
        """

        jobid = uuid.uuid4().hex

        # assign signals
        default_signals = {
            'incr_progress': int,
            'add_progress':  int,
        }

        if signals:
            default_signals.update( signals )

        # create task
        task = ThreadedTask(
            callback = callback,
            signals  = default_signals,
            *args, **kwds
        )

        # Connections
        task.signal('incr_progress').connect(
            functools.partial( self.incr_progress, jobid=jobid )
        )
        task.signal('add_progress').connect(
            functools.partial( self.add_progress, jobid=jobid )
        )
        task.signal('returned').connect(
            functools.partial( self._handle_return_or_abort, jobid=jobid )
        )
        task.signal('exception').connect(
            functools.partial( self._handle_return_or_abort, jobid=jobid )
        )

        return task

    def new_solotask(self, callback, signals=None, connections=None, mutex_expiry=5000 ):
        """
        Creates a new :py:obj:`SoloThreadedTask` object, adding
        signals to it so that it can update this :py:obj:`ProgressBar`.
        """

        jobid = uuid.uuid4().hex

        # assign signals
        default_signals = {
            'incr_progress': int,
            'add_progress':  int,
        }
        if signals:
            default_signals.update( signals )

        # assign connections
        default_connections = {}
        if connections:
            for signal in connections:
                if signal in default_connections:
                    if isinstance( connections[ signal ], Iterable ):
                        for _callable in connections[ signal ]:
                            default_connections[ signal ].append( _callable )
                    else:
                        _callable = connections[ signal ]
                        default_connections[ signal ].append( _callable )
                else:
                    default_connections[ signal ] = connections[ signal ]

        # create task
        solotask = _ProgressSoloThreadedTask(
            progressbar  = self,
            callback     = callback,
            signals      = default_signals,
            connections  = default_connections,
            mutex_expiry = mutex_expiry,
        )

        return solotask

    def _handle_return_or_abort(self, *args,**kwds):
        if 'jobid' in kwds:
            self.reset( jobid=kwds['jobid'] )

if __name__ == '__main__':
    from   qconcurrency import QApplication, Fake
    from   Qt           import QtWidgets, QtCore, QtGui
    import supercli.logging
    import time
    supercli.logging.SetLog(lv=20)

    def update_progbar( start_wait=0, signalmgr=None ):
        if not signalmgr:
            signalmgr = Fake()

        signalmgr.add_progress.emit(5)
        time.sleep( start_wait )

        for i in range(5):
            signalmgr.handle_if_abort()
            time.sleep(0.2)
            signalmgr.incr_progress.emit(1)
        print('done')

    with QApplication():
        win = QtWidgets.QWidget()
        lyt = QtWidgets.QVBoxLayout()
        bar = ProgressBar()
        win.setLayout(lyt)
        lyt.addWidget(bar)

        win.show()

        solotask = bar.new_solotask(
            callback = update_progbar,
        )
        solotask.start( start_wait=0   )

        # wait before starting next job,
        # keeping eventloop alive so progress can be seen
        for i in range(2):
            time.sleep(0.2)
            QtCore.QCoreApplication.instance().processEvents()

        solotask.start( start_wait=0.5 )
        solotask.start()
        solotask.start()

<commit_msg>py3 bugfix for dict iteration<commit_after>#!/usr/bin/env python
"""
Name :          qconcurrency.widgets._progressbar_.py
Created :       Apr 17, 2017
Author :        Will Pittman
Contact :       willjpittman@gmail.com
________________________________________________________________________________
Description :   ProgressBar, that can be updated from a separate thread.
________________________________________________________________________________
"""
#builtin
from __future__    import unicode_literals
from __future__    import absolute_import
from __future__    import division
from __future__    import print_function
from   collections import Iterable
import functools
import uuid
import copy
#external
from Qt            import QtWidgets
#internal
from qconcurrency.threading_  import ThreadedTask, SoloThreadedTask

#!TODO: http://stackoverflow.com/questions/7108715/how-to-show-hide-a-child-qwidget-with-a-motion-animation

class _ProgressSoloThreadedTask( SoloThreadedTask ):
    """
    customized :py:obj:`qconcurrency.threading_.SoloThreadedTask` that
    assigns a new jobid for each thread it starts, and each thread's progress
    is measured entirely independently.

    This way, there is no race-condition where cancelling one thread can knock out
    progress on *all* threads.
    """
    def __init__(self, progressbar, callback, signals=None, connections=None, mutex_expiry=5000 ):
        self._progressbar = progressbar
        SoloThreadedTask.__init__(self,
            callback     = callback,
            signals      = signals,
            connections  = connections,
            mutex_expiry = mutex_expiry,
        )

    def start(self, expiryTimeout=-1, threadpool=None, wait=False, *args, **kwds ):
        """
        Wraps :py:meth:`qconcurrency.threading_.SoloThreadedTask.start` ,
        adding signal-connections so that they update a progressbar.

        See Also:

            * :py:meth:`qconcurrency.threading_.SoloThreadedTask.start`
        """
        jobid = uuid.uuid4().hex

        connections = self._connections.copy()
        if not connections:
            connections = {}

        progbar_connections = {
            'incr_progress' : functools.partial( self._progressbar.incr_progress, jobid=jobid ),
            'add_progress'  : functools.partial( self._progressbar.add_progress,  jobid=jobid ),
            'returned'      : functools.partial( self._progressbar._handle_return_or_abort, jobid=jobid ),
            'exception'     : functools.partial( self._progressbar._handle_return_or_abort, jobid=jobid ),
        }

        for signal in progbar_connections:
            if signal in connections:
                connections[ signal ].append( progbar_connections[signal] )
            else:
                connections[ signal ] = [ progbar_connections[signal] ]

        SoloThreadedTask.start(self,
                           expiryTimeout = expiryTimeout,
                           threadpool    = threadpool,
                           wait          = wait,
                           _connections  = connections,
                           *args,**kwds
                       )

class ProgressBar( QtWidgets.QProgressBar ):
    """
    A ProgressBar designed to track progress of several threads,
    that is hidden automatically whenever all threads have exited
    (by unhandled exception, or return).

    Each thread is assigned it's own `jobid`, so that it's progress
    is tracked entirely independently of all other pending tasks.
    (my hope is that if errors appear in total-calculated progress in the
    codebase, this will lessen the appearance of an error for the user - each
    thread's progress being set to 100% once it exits).
    """
    def __init__(self):
        QtWidgets.QProgressBar.__init__(self)

        self._progress = {}  # { jobid: {'total':10, 'current':3} }
        self.setHidden(True)

    def add_progress(self, amount, jobid=None):
        """
        Adds to the total number of required steps
        required to complete.
        """

        if jobid not in self._progress:
            self._progress[ jobid ] = {'total':amount, 'current':0}
        else:
            self._progress[ jobid ]['total'] += amount

        self.refresh_progress()
        self.setHidden(False)

    def refresh_progress(self):
        """
        ReCalculates current/total progress, and updates the progressbar
        """

        total_progress   = 0
        current_progress = 0
        for jobid in list(self._progress.keys()):
            total_progress   += self._progress[ jobid ]['total']
            current_progress += self._progress[ jobid ]['current']

            if current_progress == total_progress:
                self._progress.pop(jobid)

        self.setMaximum( total_progress   )
        self.setValue(   current_progress )

        if total_progress <= current_progress:
            self.setHidden(True)

    def incr_progress(self, amount, jobid):
        """
        Completes a particular number of steps for
        a particular jobid.
        """

        if amount == None:
            amount = 1

        if jobid not in self._progress:
            return

        self._progress[ jobid ]['current'] += amount
        self.refresh_progress()

    def reset(self, jobid=None ):
        """
        If not `jobid`, sets all required progress-steps back to 0.
        Otherwise, removes all required progress associated with that
        jobid.
        """

        if not jobid:
            self._progress = {}
            QtWidgets.QProgressBar.reset(self)

        elif jobid in self._progress:
            self._progress.pop( jobid )
            self.refresh_progress()

    def new_task(self, callback, signals=None, *args, **kwds ):
        """
        Creates a new :py:obj:`ThreadedTask` object, adding
        signals to it so that it can update this :py:obj:`ProgressBar`.

        :py:obj:`ThreadedTask` objects are most suitable for
        producer/consumer patterns (multiple threads running at once),
        and no thread depends on another.

        See also:
            * :py:obj:`ThreadedTask`
            * :py:obj:`SoloThreadedTask`
            * :py:meth:`ProgressBar.new_solotask`
        """

        jobid = uuid.uuid4().hex

        # assign signals
        default_signals = {
            'incr_progress': int,
            'add_progress':  int,
        }

        if signals:
            default_signals.update( signals )

        # create task
        task = ThreadedTask(
            callback = callback,
            signals  = default_signals,
            *args, **kwds
        )

        # Connections
        task.signal('incr_progress').connect(
            functools.partial( self.incr_progress, jobid=jobid )
        )
        task.signal('add_progress').connect(
            functools.partial( self.add_progress, jobid=jobid )
        )
        task.signal('returned').connect(
            functools.partial( self._handle_return_or_abort, jobid=jobid )
        )
        task.signal('exception').connect(
            functools.partial( self._handle_return_or_abort, jobid=jobid )
        )

        return task

    def new_solotask(self, callback, signals=None, connections=None, mutex_expiry=5000 ):
        """
        Creates a new :py:obj:`SoloThreadedTask` object, adding
        signals to it so that it can update this :py:obj:`ProgressBar`.
        """

        jobid = uuid.uuid4().hex

        # assign signals
        default_signals = {
            'incr_progress': int,
            'add_progress':  int,
        }
        if signals:
            default_signals.update( signals )

        # assign connections
        default_connections = {}
        if connections:
            for signal in connections:
                if signal in default_connections:
                    if isinstance( connections[ signal ], Iterable ):
                        for _callable in connections[ signal ]:
                            default_connections[ signal ].append( _callable )
                    else:
                        _callable = connections[ signal ]
                        default_connections[ signal ].append( _callable )
                else:
                    default_connections[ signal ] = connections[ signal ]

        # create task
        solotask = _ProgressSoloThreadedTask(
            progressbar  = self,
            callback     = callback,
            signals      = default_signals,
            connections  = default_connections,
            mutex_expiry = mutex_expiry,
        )

        return solotask

    def _handle_return_or_abort(self, *args,**kwds):
        if 'jobid' in kwds:
            self.reset( jobid=kwds['jobid'] )

if __name__ == '__main__':
    from   qconcurrency import QApplication, Fake
    from   Qt           import QtWidgets, QtCore, QtGui
    import supercli.logging
    import time
    supercli.logging.SetLog(lv=20)

    def update_progbar( start_wait=0, signalmgr=None ):
        if not signalmgr:
            signalmgr = Fake()

        signalmgr.add_progress.emit(5)
        time.sleep( start_wait )

        for i in range(5):
            signalmgr.handle_if_abort()
            time.sleep(0.2)
            signalmgr.incr_progress.emit(1)
        print('done')

    with QApplication():
        win = QtWidgets.QWidget()
        lyt = QtWidgets.QVBoxLayout()
        bar = ProgressBar()
        win.setLayout(lyt)
        lyt.addWidget(bar)

        win.show()

        solotask = bar.new_solotask(
            callback = update_progbar,
        )
        solotask.start( start_wait=0   )

        # wait before starting next job,
        # keeping eventloop alive so progress can be seen
        for i in range(2):
            time.sleep(0.2)
            QtCore.QCoreApplication.instance().processEvents()

        solotask.start( start_wait=0.5 )
        solotask.start()
        solotask.start()

<|endoftext|>


<commit_before>from datetime import timedelta
import re

from django.utils import timezone
from django.db import transaction
from django.contrib.sessions.models import Session

from froide.helper.email_sending import (
    send_mail, mail_middleware_registry, mail_registry
)

from . import account_canceled, account_merged, account_made_private
from .models import User

POSTCODE_RE = re.compile(r'(\d{5})\s+(.*)')

EXPIRE_UNCONFIRMED_USERS_AGE = timedelta(days=30)
CANCEL_DEACTIVATED_USERS_AGE = timedelta(days=100)

def send_mail_users(subject, body, users,
              **kwargs):
    for user in users:
        send_mail_user(
            subject, body, user,
            **kwargs
        )

class OnlyActiveUsersMailMiddleware:
    def should_mail(self, mail_intent, context, email_kwargs):
        user = context.get('user')
        if not user:
            # No user, not our concern here
            return

        if not email_kwargs.get('ignore_active', False) and not user.is_active:
            return False
        if not user.email:
            return False

mail_middleware_registry.register(OnlyActiveUsersMailMiddleware())

def send_mail_user(subject, body, user: User,
                   ignore_active=False, **kwargs):
    if not ignore_active and not user.is_active:
        return
    if not user.email:
        return

    return send_mail(subject, body, user.email, bounce_check=False, **kwargs)

user_email = mail_registry.register(
    'account/emails/user_email',
    ('subject', 'body')
)

def send_template_mail(user: User, subject: str, body: str, **kwargs):
    mail_context = {
        'first_name': user.first_name,
        'last_name': user.last_name,
        'name': user.get_full_name(),
        'url': user.get_autologin_url('/'),
    }
    user_subject = subject.format(**mail_context)
    user_body = body.format(**mail_context)
    return user_email.send(
        user=user,
        context={
            'subject': user_subject,
            'body': user_body
        }, **kwargs
    )

def make_account_private(user):
    user.private = True
    user.save()

    account_made_private.send(sender=User, user=user)

def merge_accounts(old_user, new_user):
    account_merged.send(
        sender=User, old_user=old_user, new_user=new_user
    )
    start_cancel_account_process(old_user)

def move_ownership(model, attr, old_user, new_user, dupe=None):
    qs = model.objects.filter(**{attr: old_user})

    if dupe:
        collision_key = dupe[1]
        collision_list = model.objects.filter(
            **{attr: new_user}
        ).values_list(collision_key, flat=True)
        qs = qs.exclude(
            **{'%s__in' % collision_key: collision_list}
        )

    qs.update(**{attr: new_user})

    if dupe is None:
        return

    # rest are collisions, delete
    qs = model.objects.filter(**{attr: old_user}).delete()

def all_unexpired_sessions_for_user(user):
    user_sessions = []
    all_sessions = Session.objects.filter(expire_date__gte=timezone.now())
    for session in all_sessions:
        session_data = session.get_decoded()
        if user.pk == session_data.get('_auth_user_id'):
            user_sessions.append(session.pk)
    return Session.objects.filter(pk__in=user_sessions)

def delete_all_unexpired_sessions_for_user(user, session_to_omit=None):
    session_list = all_unexpired_sessions_for_user(user)
    if session_to_omit is not None:
        session_list.exclude(session_key=session_to_omit.session_key)
    session_list.delete()

def start_cancel_account_process(user, delete=False):
    from .tasks import cancel_account_task

    user.private = True
    user.email = None
    user.is_active = False
    user.set_unusable_password()
    user.date_deactivated = timezone.now()

    user.save()
    delete_all_unexpired_sessions_for_user(user)

    # Asynchronously delete account
    # So more expensive anonymization can run in the background
    cancel_account_task.delay(user.pk, delete=delete)

def cancel_user(user, delete=False):
    with transaction.atomic():
        account_canceled.send(sender=User, user=user)

    delete_all_unexpired_sessions_for_user(user)

    if delete:
        user.delete()
        return

    user.organization = ''
    user.organization_url = ''
    user.private = True
    user.newsletter = False
    user.terms = False
    user.address = ''
    user.profile_text = ''
    user.profile_photo.delete()
    user.save()
    user.first_name = ''
    user.last_name = ''
    user.is_trusted = False
    user.is_staff = False
    user.is_superuser = False
    user.is_active = False
    user.date_deactivated = timezone.now()
    user.is_deleted = True
    user.date_left = timezone.now()
    user.email = None
    user.set_unusable_password()
    user.username = 'u%s' % user.pk
    user.save()

def delete_unconfirmed_users():
    time_ago = timezone.now() - EXPIRE_UNCONFIRMED_USERS_AGE
    expired_users = User.objects.filter(
        is_active=False,
        is_deleted=False,
        last_login__isnull=True,
        date_joined__lt=time_ago
    )
    for user in expired_users:
        start_cancel_account_process(user, delete=True)

def delete_deactivated_users():
    time_ago = timezone.now() - CANCEL_DEACTIVATED_USERS_AGE
    expired_users = User.objects.filter(
        is_active=False,
        is_deleted=False,
        date_deactivated__lt=time_ago
    )
    for user in expired_users:
        start_cancel_account_process(user)

def parse_address(address):
    match = POSTCODE_RE.search(address)
    if match is None:
        return {}
    postcode = match.group(1)
    city = match.group(2)
    refined = address.replace(match.group(0), '').strip().splitlines()
    return {
        'address': refined[0],
        'postcode': postcode,
        'city': city
    }
<commit_msg>Fix error on parsing bad addresses<commit_after>from datetime import timedelta
import re

from django.utils import timezone
from django.db import transaction
from django.contrib.sessions.models import Session

from froide.helper.email_sending import (
    send_mail, mail_middleware_registry, mail_registry
)

from . import account_canceled, account_merged, account_made_private
from .models import User

POSTCODE_RE = re.compile(r'(\d{5})\s+(.*)')

EXPIRE_UNCONFIRMED_USERS_AGE = timedelta(days=30)
CANCEL_DEACTIVATED_USERS_AGE = timedelta(days=100)

def send_mail_users(subject, body, users,
              **kwargs):
    for user in users:
        send_mail_user(
            subject, body, user,
            **kwargs
        )

class OnlyActiveUsersMailMiddleware:
    def should_mail(self, mail_intent, context, email_kwargs):
        user = context.get('user')
        if not user:
            # No user, not our concern here
            return

        if not email_kwargs.get('ignore_active', False) and not user.is_active:
            return False
        if not user.email:
            return False

mail_middleware_registry.register(OnlyActiveUsersMailMiddleware())

def send_mail_user(subject, body, user: User,
                   ignore_active=False, **kwargs):
    if not ignore_active and not user.is_active:
        return
    if not user.email:
        return

    return send_mail(subject, body, user.email, bounce_check=False, **kwargs)

user_email = mail_registry.register(
    'account/emails/user_email',
    ('subject', 'body')
)

def send_template_mail(user: User, subject: str, body: str, **kwargs):
    mail_context = {
        'first_name': user.first_name,
        'last_name': user.last_name,
        'name': user.get_full_name(),
        'url': user.get_autologin_url('/'),
    }
    user_subject = subject.format(**mail_context)
    user_body = body.format(**mail_context)
    return user_email.send(
        user=user,
        context={
            'subject': user_subject,
            'body': user_body
        }, **kwargs
    )

def make_account_private(user):
    user.private = True
    user.save()

    account_made_private.send(sender=User, user=user)

def merge_accounts(old_user, new_user):
    account_merged.send(
        sender=User, old_user=old_user, new_user=new_user
    )
    start_cancel_account_process(old_user)

def move_ownership(model, attr, old_user, new_user, dupe=None):
    qs = model.objects.filter(**{attr: old_user})

    if dupe:
        collision_key = dupe[1]
        collision_list = model.objects.filter(
            **{attr: new_user}
        ).values_list(collision_key, flat=True)
        qs = qs.exclude(
            **{'%s__in' % collision_key: collision_list}
        )

    qs.update(**{attr: new_user})

    if dupe is None:
        return

    # rest are collisions, delete
    qs = model.objects.filter(**{attr: old_user}).delete()

def all_unexpired_sessions_for_user(user):
    user_sessions = []
    all_sessions = Session.objects.filter(expire_date__gte=timezone.now())
    for session in all_sessions:
        session_data = session.get_decoded()
        if user.pk == session_data.get('_auth_user_id'):
            user_sessions.append(session.pk)
    return Session.objects.filter(pk__in=user_sessions)

def delete_all_unexpired_sessions_for_user(user, session_to_omit=None):
    session_list = all_unexpired_sessions_for_user(user)
    if session_to_omit is not None:
        session_list.exclude(session_key=session_to_omit.session_key)
    session_list.delete()

def start_cancel_account_process(user, delete=False):
    from .tasks import cancel_account_task

    user.private = True
    user.email = None
    user.is_active = False
    user.set_unusable_password()
    user.date_deactivated = timezone.now()

    user.save()
    delete_all_unexpired_sessions_for_user(user)

    # Asynchronously delete account
    # So more expensive anonymization can run in the background
    cancel_account_task.delay(user.pk, delete=delete)

def cancel_user(user, delete=False):
    with transaction.atomic():
        account_canceled.send(sender=User, user=user)

    delete_all_unexpired_sessions_for_user(user)

    if delete:
        user.delete()
        return

    user.organization = ''
    user.organization_url = ''
    user.private = True
    user.newsletter = False
    user.terms = False
    user.address = ''
    user.profile_text = ''
    user.profile_photo.delete()
    user.save()
    user.first_name = ''
    user.last_name = ''
    user.is_trusted = False
    user.is_staff = False
    user.is_superuser = False
    user.is_active = False
    user.date_deactivated = timezone.now()
    user.is_deleted = True
    user.date_left = timezone.now()
    user.email = None
    user.set_unusable_password()
    user.username = 'u%s' % user.pk
    user.save()

def delete_unconfirmed_users():
    time_ago = timezone.now() - EXPIRE_UNCONFIRMED_USERS_AGE
    expired_users = User.objects.filter(
        is_active=False,
        is_deleted=False,
        last_login__isnull=True,
        date_joined__lt=time_ago
    )
    for user in expired_users:
        start_cancel_account_process(user, delete=True)

def delete_deactivated_users():
    time_ago = timezone.now() - CANCEL_DEACTIVATED_USERS_AGE
    expired_users = User.objects.filter(
        is_active=False,
        is_deleted=False,
        date_deactivated__lt=time_ago
    )
    for user in expired_users:
        start_cancel_account_process(user)

def parse_address(address):
    match = POSTCODE_RE.search(address)
    if match is None:
        return {}
    postcode = match.group(1)
    city = match.group(2)
    refined = address.replace(match.group(0), '').strip().splitlines()
    refined = refined or ['']
    return {
        'address': refined[0],
        'postcode': postcode,
        'city': city
    }
<|endoftext|>


<commit_before>#!/usr/bin/env python
# This script drops one or multiple versions of a release
#
from __future__ import print_function
import json
import sys

def load_package(filename):
    if filename == "-":
        pkg = json.load(sys.stdin)['packages'][0]
    else:
        pkg = json.load(open(filename))['packages'][0]
    print("Loaded package {0} from {1}".format(pkg['name'], filename), file=sys.stderr)
    print("{0} platform(s), {1} tools".format(len(pkg['platforms']), len(pkg['tools'])), file=sys.stderr)
    return pkg

# There's probably a lambda way of doing this, but I can't figure it out...
def drop_version(todrop, obj):
    out = [];
    for o in obj:
        version = o['version'].encode('ascii')
        if version != todrop:
            out.append(o)
    return out

def main(args):
    if len(args) < 3:
        print("Usage: {0} <inpackage> <section> <version-to-remove> ...".format(args[0]), file=sys.stderr)
        return 1

    pkg = load_package(args[1])
    section = args[2]
    sub = pkg[section]
    for ver in args[3:]:
        sub = drop_version(ver, sub)
    pkg[section] = sub

    json.dump({'packages':[pkg]}, sys.stdout, indent=2)

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<commit_msg>Moved 'Dropping' print from stdout to stderr in drop_versions.py (#6071)<commit_after>#!/usr/bin/env python
# This script drops one or multiple versions of a release
#
from __future__ import print_function
import json
import sys

def load_package(filename):
    if filename == "-":
        pkg = json.load(sys.stdin)['packages'][0]
    else:
        pkg = json.load(open(filename))['packages'][0]
    print("Loaded package {0} from {1}".format(pkg['name'], filename), file=sys.stderr)
    print("{0} platform(s), {1} tools".format(len(pkg['platforms']), len(pkg['tools'])), file=sys.stderr)
    return pkg

# There's probably a lambda way of doing this, but I can't figure it out...
def drop_version(todrop, obj):
    out = [];
    for o in obj:
        version = o['version'].encode('ascii')
        if version == todrop:
            print("Dropping version {0}".format(todrop), file=sys.stderr)
        else:
            out.append(o)
    return out

def main(args):
    if len(args) < 3:
        print("Usage: {0} <inpackage> <section> <version-to-remove> ...".format(args[0]), file=sys.stderr)
        return 1

    pkg = load_package(args[1])
    section = args[2]
    sub = pkg[section]
    for ver in args[3:]:
        sub = drop_version(ver, sub)
    pkg[section] = sub

    json.dump({'packages':[pkg]}, sys.stdout, indent=2)

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>import importlib

from django.conf import settings
from django.db import models
from django.utils.http import urlencode
from django.urls import reverse
from django.core.paginator import Paginator
from django.utils.safestring import mark_safe

from django_elasticsearch_dsl import Index
from django_elasticsearch_dsl.registries import registry
from django_elasticsearch_dsl.signals import RealTimeSignalProcessor

from elasticsearch_dsl import analyzer, tokenizer, A
from elasticsearch_dsl.query import Q

from .tasks import (
    search_instance_save, search_instance_pre_delete,
    search_instance_delete
)

def get_index(name):
    index = Index('%s_%s' % (
        settings.ELASTICSEARCH_INDEX_PREFIX,
        name
    ))

    # See Elasticsearch Indices API reference for available settings
    index.settings(
        number_of_shards=1,
        number_of_replicas=0
    )
    return index

def get_pagination_vars(data):
    d = data.copy()
    d.pop('page', None)
    return '&' + urlencode(d)

def get_default_text_analyzer():
    return analyzer(
        'froide_analyzer',
        tokenizer='standard',
        filter=[
            'standard',
            'lowercase',
            'asciifolding',
        ]
    )

def get_default_ngram_analyzer():
    return analyzer(
        'froide_ngram_analyzer',
        tokenizer=tokenizer(
            'froide_ngram_tokenzier',
            type='edge_ngram',
            min_gram=1,
            max_gram=15,
            token_chars=['letter', 'digit']
        ),
        filter=[
            'standard',
            'lowercase',
            'asciifolding',
        ]
    )

def get_func(config_name, default_func):
    def get_it():
        from django.conf import settings
        func_path = settings.FROIDE_CONFIG.get(config_name, None)
        if not func_path:
            return default_func()

        module, func = func_path.rsplit('.', 1)
        module = importlib.import_module(module)
        analyzer_func = getattr(module, func)
        return analyzer_func()
    return get_it

get_text_analyzer = get_func('search_text_analyzer', get_default_text_analyzer)
get_ngram_analyzer = get_func('search_ngram_analyzer', get_default_ngram_analyzer)

def make_filter_url(url_name, data=None, get_active_filters=None):
    if data is None:
        data = {}
    data = dict(data)
    url_kwargs = {}

    if get_active_filters is None:
        active_filters = {}
    else:
        active_filters = get_active_filters(data)

    for key in active_filters:
        url_kwargs[key] = data.pop(key)

    query_string = ''
    data = {k: v for k, v in data.items() if v}
    if data:
        query_string = '?' + urlencode(data)
    return reverse(url_name, kwargs=url_kwargs) + query_string

def get_facet_with_label(info, model=None, attr='name'):
    pks = [item['key'] for item in info['buckets']]
    objs = {
        o.pk: o for o in model.objects.filter(pk__in=pks)
    }
    for item in info['buckets']:
        yield {
            'label': getattr(objs[item['key']], attr),
            'id': item['key'],
            'count': item['doc_count']
        }

def key_getter(item):
    return item['key']

def resolve_facet(data, getter=None, label_getter=None,
                  model=None, make_url=None):
    if getter is None:
        getter = key_getter

    if label_getter is None:
        label_getter = key_getter

    def resolve(key, info):
        if model is not None:
            pks = [item['key'] for item in info['buckets']]
            objs = {
                o.pk: o for o in model.objects.filter(pk__in=pks)
            }
            for item in info['buckets']:
                item['object'] = objs[item['key']]
        for item in info['buckets']:
            item['active'] = getter(item) == data.get(key)
            item['label'] = label_getter(item)
            d = data.copy()
            d[key] = getter(item)
            item['url'] = make_url(d)
            d.pop(key)
            item['clear_url'] = make_url(d)
        return info
    return resolve

def _make_values_lists(kwargs):
    return {
        k: v if isinstance(v, (list, tuple)) else [v]
        for k, v in kwargs.items()
    }

class EmtpyResponse(list):
    class hits:
        total = 0

class SearchQuerySetWrapper(object):
    """
    Decorates a SearchQuerySet object using a generator for efficient iteration
    """
    def __init__(self, sqs, model):
        self.sqs = sqs
        self.sqs.model = model
        self.model = model
        self.filters = []
        self.query = None
        self.aggs = []
        self.broken_query = False

    def count(self):
        return self.response.hits.total

    def to_queryset(self):
        if self.broken_query:
            return self.model.objects.none()
        return self.sqs.to_queryset()

    def wrap_queryset(self, qs):
        return ESQuerySetWrapper(qs, self.response)

    def update_query(self):
        if self.filters:
            self.sqs.post_filter = Q('bool', must=self.filters)

    def all(self):
        return self

    @property
    def response(self):
        return self.get_response()

    def get_response(self):
        if not hasattr(self.sqs, '_response'):
            self.sqs = self.sqs.source(excludes=['*'])
        try:
            return self.sqs.execute()
        except Exception:
            self.broken_query = True
            return EmtpyResponse()

    def add_aggregation(self, aggs):
        for field in aggs:
            a = A('terms', field=field)
            self.sqs.aggs.bucket(field, a)
        return self

    def get_facets(self, resolvers=None):
        if resolvers is None:
            resolvers = {}
        return {
            k: resolvers[k](k, self.response['aggregations'][k])
            for k in self.response['aggregations']
            if k in resolvers
        }

    def get_aggregations(self):
        return {'fields': {
            k: [
                [i['key'], i['doc_count']]
                for i in self.response['aggregations'][k]['buckets']
            ]
            for k in self.response['aggregations']
        }}

    def filter(self, *args, **kwargs):
        if kwargs:
            value = Q('terms', **_make_values_lists(kwargs))
        else:
            value = args[0]
        self.filters.append(value)
        self.update_query()
        return self

    def set_query(self, q):
        self.query = q
        self.sqs = self.sqs.query(self.query)
        return self

    def add_sort(self, *sorts):
        self.sqs = self.sqs.sort(*sorts)
        return self

    def __getitem__(self, key):
        self.sqs = self.sqs[key]
        return self

    def __iter__(self):
        return iter(self.sqs)

class ESQuerySetWrapper(object):
    def __init__(self, qs, es_response):
        self.__class__ = type(
            qs.__class__.__name__,
            (self.__class__, qs.__class__),
            {}
        )
        self.__dict__ = qs.__dict__
        self._qs = qs
        self._es_response = es_response
        self._es_map = {
            int(hit.meta.id): hit for hit in es_response
        }

    def __iter__(self):
        for obj in self._qs:
            hit = self._es_map[obj.pk]
            # mark_safe should work because highlight_options
            # has been set with encoder="html"
            obj.es_highlight = mark_safe(' '.join(self._get_highlight(hit)))
            yield obj

    def _get_highlight(self, hit):
        if hasattr(hit.meta, 'highlight'):
            for key in hit.meta.highlight:
                yield from hit.meta.highlight[key]

class ElasticsearchPaginator(Paginator):
    """
    Paginator that prevents two queries to ES (for count and objects)
    as ES gives count with objects
    """
    MAX_ES_OFFSET = 10000

    def page(self, number):
        """
        Returns a Page object for the given 1-based page number.
        """
        bottom = (number - 1) * self.per_page
        if bottom >= self.MAX_ES_OFFSET:
            # Only validate if bigger than offset
            number = self.validate_number(number)
            bottom = (number - 1) * self.per_page

        top = bottom + self.per_page
        self.object_list = self.object_list[bottom:top]

        # ignore top boundary
        # if top + self.orphans >= self.count:
        #     top = self.count

        # Validate number after limit/offset has been set
        number = self.validate_number(number)
        return self._get_page(self.object_list, number, self)

class CelerySignalProcessor(RealTimeSignalProcessor):
    def setup(self):
        for model in registry.get_models():
            models.signals.post_save.connect(self.handle_save, sender=model)
            models.signals.post_delete.connect(self.handle_delete, sender=model)

            models.signals.m2m_changed.connect(self.handle_m2m_changed, sender=model)
            models.signals.pre_delete.connect(self.handle_pre_delete, sender=model)

    def teardown(self):
        # Listen to all model saves.
        for model in registry.get_models():
            models.signals.post_save.disconnect(self.handle_save, sender=model)
            models.signals.post_delete.disconnect(self.handle_delete, sender=model)
            models.signals.m2m_changed.disconnect(self.handle_m2m_changed, sender=model)
            models.signals.pre_delete.disconnect(self.handle_pre_delete, sender=model)

    def handle_save(self, sender, instance, **kwargs):
        """Handle save.

        Given an individual model instance, update the object in the index.
        Update the related objects either.
        """
        search_instance_save.delay(instance)

    def handle_pre_delete(self, sender, instance, **kwargs):
        """Handle removing of instance object from related models instance.
        We need to do this before the real delete otherwise the relation
        doesn't exists anymore and we can't get the related models instance.
        """
        search_instance_pre_delete.delay(instance)

    def handle_delete(self, sender, instance, **kwargs):
        """Handle delete.

        Given an individual model instance, delete the object from index.
        """
        search_instance_delete.delay(instance)
<commit_msg>Use default manager when loading facet objects<commit_after>import importlib

from django.conf import settings
from django.db import models
from django.utils.http import urlencode
from django.urls import reverse
from django.core.paginator import Paginator
from django.utils.safestring import mark_safe

from django_elasticsearch_dsl import Index
from django_elasticsearch_dsl.registries import registry
from django_elasticsearch_dsl.signals import RealTimeSignalProcessor

from elasticsearch_dsl import analyzer, tokenizer, A
from elasticsearch_dsl.query import Q

from .tasks import (
    search_instance_save, search_instance_pre_delete,
    search_instance_delete
)

def get_index(name):
    index = Index('%s_%s' % (
        settings.ELASTICSEARCH_INDEX_PREFIX,
        name
    ))

    # See Elasticsearch Indices API reference for available settings
    index.settings(
        number_of_shards=1,
        number_of_replicas=0
    )
    return index

def get_pagination_vars(data):
    d = data.copy()
    d.pop('page', None)
    return '&' + urlencode(d)

def get_default_text_analyzer():
    return analyzer(
        'froide_analyzer',
        tokenizer='standard',
        filter=[
            'standard',
            'lowercase',
            'asciifolding',
        ]
    )

def get_default_ngram_analyzer():
    return analyzer(
        'froide_ngram_analyzer',
        tokenizer=tokenizer(
            'froide_ngram_tokenzier',
            type='edge_ngram',
            min_gram=1,
            max_gram=15,
            token_chars=['letter', 'digit']
        ),
        filter=[
            'standard',
            'lowercase',
            'asciifolding',
        ]
    )

def get_func(config_name, default_func):
    def get_it():
        from django.conf import settings
        func_path = settings.FROIDE_CONFIG.get(config_name, None)
        if not func_path:
            return default_func()

        module, func = func_path.rsplit('.', 1)
        module = importlib.import_module(module)
        analyzer_func = getattr(module, func)
        return analyzer_func()
    return get_it

get_text_analyzer = get_func('search_text_analyzer', get_default_text_analyzer)
get_ngram_analyzer = get_func('search_ngram_analyzer', get_default_ngram_analyzer)

def make_filter_url(url_name, data=None, get_active_filters=None):
    if data is None:
        data = {}
    data = dict(data)
    url_kwargs = {}

    if get_active_filters is None:
        active_filters = {}
    else:
        active_filters = get_active_filters(data)

    for key in active_filters:
        url_kwargs[key] = data.pop(key)

    query_string = ''
    data = {k: v for k, v in data.items() if v}
    if data:
        query_string = '?' + urlencode(data)
    return reverse(url_name, kwargs=url_kwargs) + query_string

def get_facet_with_label(info, model=None, attr='name'):
    pks = [item['key'] for item in info['buckets']]
    objs = {
        o.pk: o for o in model.objects.filter(pk__in=pks)
    }
    for item in info['buckets']:
        yield {
            'label': getattr(objs[item['key']], attr),
            'id': item['key'],
            'count': item['doc_count']
        }

def key_getter(item):
    return item['key']

def resolve_facet(data, getter=None, label_getter=None,
                  model=None, make_url=None):
    if getter is None:
        getter = key_getter

    if label_getter is None:
        label_getter = key_getter

    def resolve(key, info):
        if model is not None:
            pks = [item['key'] for item in info['buckets']]
            objs = {
                o.pk: o for o in model._default_manager.filter(pk__in=pks)
            }
            for item in info['buckets']:
                item['object'] = objs[item['key']]
        for item in info['buckets']:
            item['active'] = getter(item) == data.get(key)
            item['label'] = label_getter(item)
            d = data.copy()
            d[key] = getter(item)
            item['url'] = make_url(d)
            d.pop(key)
            item['clear_url'] = make_url(d)
        return info
    return resolve

def _make_values_lists(kwargs):
    return {
        k: v if isinstance(v, (list, tuple)) else [v]
        for k, v in kwargs.items()
    }

class EmtpyResponse(list):
    class hits:
        total = 0

class SearchQuerySetWrapper(object):
    """
    Decorates a SearchQuerySet object using a generator for efficient iteration
    """
    def __init__(self, sqs, model):
        self.sqs = sqs
        self.sqs.model = model
        self.model = model
        self.filters = []
        self.query = None
        self.aggs = []
        self.broken_query = False

    def count(self):
        return self.response.hits.total

    def to_queryset(self):
        if self.broken_query:
            return self.model.objects.none()
        return self.sqs.to_queryset()

    def wrap_queryset(self, qs):
        return ESQuerySetWrapper(qs, self.response)

    def update_query(self):
        if self.filters:
            self.sqs.post_filter = Q('bool', must=self.filters)

    def all(self):
        return self

    @property
    def response(self):
        return self.get_response()

    def get_response(self):
        if not hasattr(self.sqs, '_response'):
            self.sqs = self.sqs.source(excludes=['*'])
        try:
            return self.sqs.execute()
        except Exception:
            self.broken_query = True
            return EmtpyResponse()

    def add_aggregation(self, aggs):
        for field in aggs:
            a = A('terms', field=field)
            self.sqs.aggs.bucket(field, a)
        return self

    def get_facets(self, resolvers=None):
        if resolvers is None:
            resolvers = {}
        return {
            k: resolvers[k](k, self.response['aggregations'][k])
            for k in self.response['aggregations']
            if k in resolvers
        }

    def get_aggregations(self):
        return {'fields': {
            k: [
                [i['key'], i['doc_count']]
                for i in self.response['aggregations'][k]['buckets']
            ]
            for k in self.response['aggregations']
        }}

    def filter(self, *args, **kwargs):
        if kwargs:
            value = Q('terms', **_make_values_lists(kwargs))
        else:
            value = args[0]
        self.filters.append(value)
        self.update_query()
        return self

    def set_query(self, q):
        self.query = q
        self.sqs = self.sqs.query(self.query)
        return self

    def add_sort(self, *sorts):
        self.sqs = self.sqs.sort(*sorts)
        return self

    def __getitem__(self, key):
        self.sqs = self.sqs[key]
        return self

    def __iter__(self):
        return iter(self.sqs)

class ESQuerySetWrapper(object):
    def __init__(self, qs, es_response):
        self.__class__ = type(
            qs.__class__.__name__,
            (self.__class__, qs.__class__),
            {}
        )
        self.__dict__ = qs.__dict__
        self._qs = qs
        self._es_response = es_response
        self._es_map = {
            int(hit.meta.id): hit for hit in es_response
        }

    def __iter__(self):
        for obj in self._qs:
            hit = self._es_map[obj.pk]
            # mark_safe should work because highlight_options
            # has been set with encoder="html"
            obj.es_highlight = mark_safe(' '.join(self._get_highlight(hit)))
            yield obj

    def _get_highlight(self, hit):
        if hasattr(hit.meta, 'highlight'):
            for key in hit.meta.highlight:
                yield from hit.meta.highlight[key]

class ElasticsearchPaginator(Paginator):
    """
    Paginator that prevents two queries to ES (for count and objects)
    as ES gives count with objects
    """
    MAX_ES_OFFSET = 10000

    def page(self, number):
        """
        Returns a Page object for the given 1-based page number.
        """
        bottom = (number - 1) * self.per_page
        if bottom >= self.MAX_ES_OFFSET:
            # Only validate if bigger than offset
            number = self.validate_number(number)
            bottom = (number - 1) * self.per_page

        top = bottom + self.per_page
        self.object_list = self.object_list[bottom:top]

        # ignore top boundary
        # if top + self.orphans >= self.count:
        #     top = self.count

        # Validate number after limit/offset has been set
        number = self.validate_number(number)
        return self._get_page(self.object_list, number, self)

class CelerySignalProcessor(RealTimeSignalProcessor):
    def setup(self):
        for model in registry.get_models():
            models.signals.post_save.connect(self.handle_save, sender=model)
            models.signals.post_delete.connect(self.handle_delete, sender=model)

            models.signals.m2m_changed.connect(self.handle_m2m_changed, sender=model)
            models.signals.pre_delete.connect(self.handle_pre_delete, sender=model)

    def teardown(self):
        # Listen to all model saves.
        for model in registry.get_models():
            models.signals.post_save.disconnect(self.handle_save, sender=model)
            models.signals.post_delete.disconnect(self.handle_delete, sender=model)
            models.signals.m2m_changed.disconnect(self.handle_m2m_changed, sender=model)
            models.signals.pre_delete.disconnect(self.handle_pre_delete, sender=model)

    def handle_save(self, sender, instance, **kwargs):
        """Handle save.

        Given an individual model instance, update the object in the index.
        Update the related objects either.
        """
        search_instance_save.delay(instance)

    def handle_pre_delete(self, sender, instance, **kwargs):
        """Handle removing of instance object from related models instance.
        We need to do this before the real delete otherwise the relation
        doesn't exists anymore and we can't get the related models instance.
        """
        search_instance_pre_delete.delay(instance)

    def handle_delete(self, sender, instance, **kwargs):
        """Handle delete.

        Given an individual model instance, delete the object from index.
        """
        search_instance_delete.delay(instance)
<|endoftext|>