<commit_before># -*- coding: utf-8 -*-
"""
Created on Sun Feb 26 12:08:35 2017

@author: Calixi
"""

import CSVreader
import matplotlib.pyplot as plt
import numpy as np
import random

def dist_matrix(list_points,list_means,k):
    """
    This method computes all the distances between the points and the actual means
    of the clusters, taking in account that the last coordinate is the cluster number

    @param list_points: an array of points
    @param list_means: an array of k points which are the means of the clusters
    @return: a matrix which contains the distances between the ith point and the jth center
    """

    nb_points = len(list_points)

    real_points = list_points[:,0:2]
    square_points_distances = np.sum(real_points*real_points, axis=1)
    A = np.transpose(np.asmatrix([square_points_distances for i in range(k)]))
    # A is a nb_points * k matrix. Aij is the squared norm of the ith point of list_points

    if isinstance(list_means[0],np.float64):
        real_means = list_means[0:-2:]
        print(real_means)
        square_means_distances = np.linalg.norm(real_means)
    else:
        real_means = list_means[:,0:2]
        real_means = real_means[0:k:1]
        square_means_distances = np.sum(real_means*real_means, axis=1)
    B = np.asmatrix([square_means_distances for i in range(nb_points)])
    # B is a nb_points * k matrix. Bij is the squared norm of the jth point of list_means

    C = np.dot(real_points,np.transpose(real_means))
    # C is a nb_points * k matrix. Cij is the scalar product of the ith point of list_points and the jth point of list_means

    return np.sqrt(A+B-2*C)

def initialization_random(list_point,k):
    """
    This method returns an array of k points which will be used in order to
    initialize a k_means algorithm

    @param list_points: an array of points
    @param k: the number of clusters
    @return: an array of k points which are means of clusters
    """

    nb_points = len(list_points)
    idx = np.random.randint(nb_points,size = k)
    return list_points[idx,:]

def initialization_plus_plus(list_points,k):
    """
    This method returns an array of k points which will be used in order to
    initialize a k_means algorithm

    @param list_points: an array of points
    @param k: the number of clusters
    @return: an array of k points which are means of clusters
    """
    nb_points = len(list_points)
    dimension = len(list_points[0])
    probability_vector = np.arange(nb_points)/nb_points #All points have the same probability to be chosen the first time
    list_points[:,-1] = probability_vector

    list_means = np.zeros((k,dimension))

    for i in range(k):
        total_dst = 0

        #Choice of a new seed
        value_chosen = random.uniform(0,1)
        idx_point = 0
        value = 0
        while (value<value_chosen) and (idx_point+1<nb_points):
            idx_point +=1
            value = list_points[idx_point][-1]
        list_means[i] = list_points[idx_point]

        #Calculation of distances for each point in order to find the probabilities to choose each point
        M = dist_matrix(list_points,list_means,i+1)

        for point in list_points:
            dst = np.min(M[i])
            total_dst += dst**2
            point[-1] = total_dst

        list_points[:,-1] = list_points[:,-1]/total_dst

    return list_means

def step_E(list_points,list_means):
    """
    This method assign a cluster number to each point by changing its last coordinate
    Ex : if P belongs to the first cluster, then P[-1] = 0.

    @param list_points: an array of points
    @param list_means: an array of k points which are the means of the clusters
    """
    M = dist_matrix(list_points,list_means,k)
    nb_points = len(list_points)
    for i in range(nb_points):
        index_min = np.argmin(M[i]) #the cluster number of the ith point is index_min
        if (isinstance(index_min,np.int64)):
            list_points[i][-1] = index_min
        else:
            list_points[i][-1] = index_min[0]

def step_M(list_points,list_means):
    """
    This method computes the new position of each means by minimizing the distortion

    @param list_points: an array of points
    @param list_means: an array of k points which are the means of the clusters
    """
    k = len(list_means)
    length = len(list_points)
    for j in range(k):
        list_set = [list_points[i] for i in range(length) if (list_points[i][-1]==j)]
        list_means[j] = np.mean(list_set, axis=0)

def create_graph(list_points,list_means,t):
    """
    This method draws a 2D graph displaying the clusters and their means and saves it as a PNG file.
    If points have more than two coordinates, then it will be a projection including only the first coordinates.

    @param list_points: an array of points
    @param list_means: an array of k points which are the means of the clusters
    @param k: the figure number

    """

    k=len(list_means)

    x_points = [[] for i in range(k)]
    y_points = [[] for i in range(k)]

    fig = plt.figure()
    ax = fig.add_subplot(111)

    for i in range(k):
        x_points[i] = [point[0] for point in list_points if (point[-1]==i)]
        y_points[i] = [point[1] for point in list_points if (point[-1]==i)]

        ax.plot(x_points[i],y_points[i],'o')
        ax.plot(list_means[i][0],list_means[i][1],'x')

    titre = 'figure_' + str(t)
    plt.savefig(titre)
    plt.close("all")

def distortion(list_points,list_means):
    """
    This method returns the distortion measurement at the end of the k_means.

    @param list_points: an array of points
    @param list_means: an array of k points which are the means of the clusters
    @return: distortion measurement (float)
    """
    distortion = 0
    for i in range(k):
        list_set = [point for point in list_points if (point[-1]==i)]
        list_set = np.asarray(list_set)
        M = dist_matrix(list_set,list_means[i],1) #TODO
        distortion += np.sum(M)
    return distortion

def k_means(list_points,k,draw_graphs=False,initialization=initialization_plus_plus):
    """
    This method returns an array of k points which will be used in order to
    initialize a k_means algorithm

    @param list_points: an array of points
    @param k: the number of clusters
    @param draw_graphs: a boolean to allow the algorithm to draw graphs if True
    @param initialization: a method in order to initialize the k_means algorithm
    @return: an array of k points which are means of clusters
    """

    #K-means++ initialization
    list_means = initialization(list_points,k)
    list_means_pre = list_means.copy()

    resume_iter = True
    t=0

    #K-means beginning
    while resume_iter:

        step_E(list_points,list_means)
        list_means_pre = list_means.copy()
        step_M(list_points,list_means)

        #Graphic part
        if draw_graphs:
            create_graph(list_points,list_means,t)

        t+=1
        resume_iter = not np.array_equal(list_means,list_means_pre)

    print(distortion(list_points,list_means))

if __name__ == '__main__':

    #Lecture du fichier
    list_points = CSVreader.read("D:/Mines/Cours/Stages/Stage_ENS/Problem/EMGaussienne.data")

    #k-means
    k=1
    k_means(list_points,k,draw_graphs=False)<commit_msg>ajout du calcul de la distortion<commit_after># -*- coding: utf-8 -*-
"""
Created on Sun Feb 26 12:08:35 2017

@author: Calixi
"""

import CSVreader
import matplotlib.pyplot as plt
import numpy as np
import random

def dist_matrix(list_points,list_means,k):
    """
    This method computes all the distances between the points and the actual means
    of the clusters, taking in account that the last coordinate is the cluster number

    @param list_points: an array of points
    @param list_means: an array of k points which are the means of the clusters
    @return: a matrix which contains the distances between the ith point and the jth center
    """

    nb_points = len(list_points)

    real_points = list_points[:,0:2]
    square_points_distances = np.sum(real_points*real_points, axis=1)
    A = np.transpose(np.asmatrix([square_points_distances for i in range(k)]))
    # A is a nb_points * k matrix. Aij is the squared norm of the ith point of list_points

    real_means = list_means[:,0:2]
    real_means = real_means[0:k:1]
    real_means = np.asarray(real_means)
    square_means_distances = np.sum(real_means*real_means, axis=1)
    B = np.asmatrix([square_means_distances for i in range(nb_points)])
    # B is a nb_points * k matrix. Bij is the squared norm of the jth point of list_means

    C = np.dot(real_points,np.transpose(real_means))
    # C is a nb_points * k matrix. Cij is the scalar product of the ith point of list_points and the jth point of list_means

    return np.sqrt(A+B-2*C)

def initialization_random(list_point,k):
    """
    This method returns an array of k points which will be used in order to
    initialize a k_means algorithm

    @param list_points: an array of points
    @param k: the number of clusters
    @return: an array of k points which are means of clusters
    """

    nb_points = len(list_points)
    idx = np.random.randint(nb_points,size = k)
    return list_points[idx,:]

def initialization_plus_plus(list_points,k):
    """
    This method returns an array of k points which will be used in order to
    initialize a k_means algorithm

    @param list_points: an array of points
    @param k: the number of clusters
    @return: an array of k points which are means of clusters
    """
    nb_points = len(list_points)
    dimension = len(list_points[0])
    probability_vector = np.arange(nb_points)/nb_points #All points have the same probability to be chosen the first time
    list_points[:,-1] = probability_vector

    list_means = np.zeros((k,dimension))

    for i in range(k):
        total_dst = 0

        #Choice of a new seed
        value_chosen = random.uniform(0,1)
        idx_point = 0
        value = 0
        while (value<value_chosen) and (idx_point+1<nb_points):
            idx_point +=1
            value = list_points[idx_point][-1]
        list_means[i] = list_points[idx_point]

        #Calculation of distances for each point in order to find the probabilities to choose each point
        M = dist_matrix(list_points,list_means,i+1)

        for point in list_points:
            dst = np.min(M[i])
            total_dst += dst**2
            point[-1] = total_dst

        list_points[:,-1] = list_points[:,-1]/total_dst

    return list_means

def step_E(list_points,list_means):
    """
    This method assign a cluster number to each point by changing its last coordinate
    Ex : if P belongs to the first cluster, then P[-1] = 0.

    @param list_points: an array of points
    @param list_means: an array of k points which are the means of the clusters
    """
    M = dist_matrix(list_points,list_means,k)
    nb_points = len(list_points)
    for i in range(nb_points):
        index_min = np.argmin(M[i]) #the cluster number of the ith point is index_min
        if (isinstance(index_min,np.int64)):
            list_points[i][-1] = index_min
        else:
            list_points[i][-1] = index_min[0]

def step_M(list_points,list_means):
    """
    This method computes the new position of each means by minimizing the distortion

    @param list_points: an array of points
    @param list_means: an array of k points which are the means of the clusters
    """
    k = len(list_means)
    length = len(list_points)
    for j in range(k):
        list_set = [list_points[i] for i in range(length) if (list_points[i][-1]==j)]
        list_means[j] = np.mean(list_set, axis=0)

def create_graph(list_points,list_means,t):
    """
    This method draws a 2D graph displaying the clusters and their means and saves it as a PNG file.
    If points have more than two coordinates, then it will be a projection including only the first coordinates.

    @param list_points: an array of points
    @param list_means: an array of k points which are the means of the clusters
    @param k: the figure number
    """

    k=len(list_means)

    x_points = [[] for i in range(k)]
    y_points = [[] for i in range(k)]

    fig = plt.figure()
    ax = fig.add_subplot(111)

    for i in range(k):
        x_points[i] = [point[0] for point in list_points if (point[-1]==i)]
        y_points[i] = [point[1] for point in list_points if (point[-1]==i)]

        ax.plot(x_points[i],y_points[i],'o')
        ax.plot(list_means[i][0],list_means[i][1],'x')

    titre = 'figure_' + str(t)
    plt.savefig(titre)
    plt.close("all")

def distortion(list_points,list_means):
    """
    This method returns the distortion measurement at the end of the k_means.

    @param list_points: an array of points
    @param list_means: an array of k points which are the means of the clusters
    @return: distortion measurement (float)
    """
    distortion = 0
    for i in range(k):
        list_set = [point for point in list_points if (point[-1]==i)]
        list_set = np.asarray(list_set)
        M = dist_matrix(list_set,np.asmatrix(list_means[i]),1)
        distortion += np.sum(M)
    return distortion

def k_means(list_points,k,draw_graphs=False,initialization=initialization_plus_plus):
    """
    This method returns an array of k points which will be used in order to
    initialize a k_means algorithm

    @param list_points: an array of points
    @param k: the number of clusters
    @param draw_graphs: a boolean to allow the algorithm to draw graphs if True
    @param initialization: a method in order to initialize the k_means algorithm
    @return: an array of k points which are means of clusters
    """

    #K-means++ initialization
    list_means = initialization(list_points,k)
    list_means_pre = list_means.copy()

    resume_iter = True
    t=0

    #K-means beginning
    while resume_iter:

        step_E(list_points,list_means)
        list_means_pre = list_means.copy()
        step_M(list_points,list_means)

        #Graphic part
        if draw_graphs:
            create_graph(list_points,list_means,t)

        t+=1
        resume_iter = not np.array_equal(list_means,list_means_pre)

    print(distortion(list_points,list_means))

if __name__ == '__main__':

    #Lecture du fichier
    list_points = CSVreader.read("D:/Mines/Cours/Stages/Stage_ENS/Problem/EMGaussienne.data")

    #k-means
    k=3
    k_means(list_points,k,draw_graphs=False)<|endoftext|>


<commit_before># coding=utf-8
from __future__ import absolute_import, print_function, unicode_literals

import logging
import re
import signal
import threading
from Queue import Queue
from contextlib import contextmanager
from itertools import chain, islice

from celery import Celery, states
from celery.backends.base import DisabledBackend
from celery.events import EventReceiver
from celery.events.state import State, Task, Worker
from kombu import log as kombu_log

from clearly.safe_compiler import safe_compile_text
from .expected_state import ExpectedStateHandler, setup_task_states, \
    setup_worker_states
from .serializer import serialize_task, serialize_worker

kombu_log.get_logger('').setLevel(logging.ERROR)

class ClearlyServer(object):
    """Simple and real-time monitor for celery.
    Server object, to capture events and handle tasks and workers.

        Attributes:
            _app: Celery
            _memory: celery.events.State
            _task_states: ExpectedStateHandler
            _worker_states: ExpectedStateHandler
    """

    def __init__(self, app=None, ignore_result_backend=False,
                 max_tasks_in_memory=1000, max_workers_in_memory=100):
        """Constructs a server instance.

        Args:
            app (Celery): a configured celery app instance
            ignore_result_backend (bool): if True, do not fetch results from the actual result backend
            max_tasks_in_memory (int): max tasks stored
            max_workers_in_memory (int): max workers stored

        """
        if isinstance(app.backend, DisabledBackend):
            ignore_result_backend = True

        self._app = app
        self._ignore_result_backend = ignore_result_backend

        self._memory = self._app.events.State(
            max_tasks_in_memory=max_tasks_in_memory,
            max_workers_in_memory=max_workers_in_memory,
        )  # type: State
        self._task_states = setup_task_states()  # type: ExpectedStateHandler
        self._worker_states = setup_worker_states()  # type: ExpectedStateHandler

        # concurrency control
        self._background_lock = threading.RLock()
        self._background_connected = threading.Event()

        # running engine
        self._background_thread = None  # type:threading.Thread
        self._background_receiver = None  # type:EventReceiver

        # connected client
        self._client_queue = None  # type:Queue
        self._client_regex = None
        self._client_negate = None

        # detect shutdown.
        def sigterm_handler(_signo, _stack_frame):
            self.stop()

        signal.signal(signal.SIGTERM, sigterm_handler)

    def start(self):
        """Starts the real-time engine that captures tasks.

        """

        with self._background_lock:
            if self._background_thread:
                return
            self._background_thread = threading.Thread(
                target=self.__run_server, name='clearly-agent')
            self._background_thread.daemon = True

            self._background_connected.clear()
            self._background_thread.start()
            self._background_connected.wait()

    def stop(self):
        """Stops the background engine, without losing anything already
        captured.

        """

        with self._background_lock:
            if not self._background_thread:
                return
            print('Stopping server')
            self._background_receiver.should_stop = True
            while self._background_thread.is_alive():
                self._background_thread.join()
            self._background_thread = None
            self._background_receiver = None

    @contextmanager
    def client_connect(self, pattern=None, negate=False):
        """Connects a client to this server, filtering the tasks that are sent
        to it.

        Args:
            pattern (Optional[str]): a pattern to filter tasks to capture.
                ex.: '^dispatch|^email' to filter names starting with that
                      or 'dispatch.*123456' to filter that exact name and number
                      or even '123456' to filter that exact number anywhere.
            negate (bool): if True, finds tasks that do not match criteria

        """

        self._client_regex = re.compile(pattern or '.')
        self._client_negate = negate

        with self._background_lock:
            self._client_queue = Queue()
            yield self._client_queue
            self._client_queue = None

    def __run_server(self):
        import sys
        print('Starting server', threading.current_thread())
        sys.stdout.flush()

        def client_accepts(values):
            return any(v and self._client_regex.search(v)
                       for v in values) ^ self._client_negate

        def maybe_publish(func, *values):
            if self._client_queue:
                if values and client_accepts(values):
                    obj = func()
                    try:
                        self._client_queue.put(obj)
                    except AttributeError:
                        # the capture is not synced with terminal anymore.
                        # no problem.
                        pass

        def process_event(event):
            event_type = event['type']
            if event_type.startswith('task'):
                for task, state, created in process_task_event(event):
                    maybe_publish(lambda: serialize_task(task, state, created),
                                  task.name, task.routing_key)

            elif event_type.startswith('worker'):
                for worker in process_worker_event(event):
                    maybe_publish(lambda: serialize_worker(worker))

            else:
                maybe_publish(lambda: event)

        def process_task_event(event):
            task, created = self._memory.get_or_create_task(event['uuid'])
            with self._task_states.track_changes(task):
                (_, _), subject = self._memory.event(event)
            if task.state == states.SUCCESS:
            for state in chain(range(created),
                               self._task_states.states_through()):
                yield task, state, created
                created = False
                if self._ignore_result_backend:
                    # celery tasks' results are escaped, so we must compile them.
                    task.result = safe_compile_text(task.result)
                else:
                    task.result = self._app.AsyncResult(task.uuid).result

        def process_worker_event(event):
            worker, _ = self._memory.get_or_create_worker(event['hostname'])
            with self._worker_states.track_changes(worker):
                (_, _), subject = self._memory.event(event)
            for _ in self._worker_states.states_through():
                yield worker

        with self._app.connection() as connection:
            self._background_receiver = self._app.events.Receiver(
                connection, handlers={
                    '*': process_event,
                })  # type: EventReceiver
            self._background_connected.set()
            self._background_receiver.capture(limit=None, timeout=None,
                                              wakeup=True)

        print('Server stopped', threading.current_thread())
        sys.stdout.flush()

    def tasks(self, pattern=None, state=None, negate=False):
        """Filters captured tasks.

        Args:
            pattern (Optional[str]): any part of the task name or routing key
            state (Optional[str]): a state to filter tasks
            negate (bool): if True, finds tasks that do not match criteria

        """
        pcondition = scondition = lambda task: True
        if pattern:
            regex = re.compile(pattern)
            pcondition = lambda task: \
                regex.search(task.name or '') or \
                regex.search(task.routing_key or '')
        if state:
            scondition = lambda task: task.state == state

        found_tasks = islice(
            (task for _, task in self._memory.itertasks()
             if bool(pcondition(task) and scondition(task)) ^ negate
             ), 0, None)
        for task in found_tasks:  # type:Task
            yield serialize_task(task, task.state, False)

    def workers(self, pattern=None, negate=False):
        """Filters known workers and prints their current status.

        Args:
            pattern (Optional[str]): any part of the task name or routing key
            negate (bool): if True, finds tasks that do not match criteria

        """
        regex = re.compile(pattern or '.')
        found_workers = islice(
            (worker for worker in map(lambda w: self._memory.workers[w],
                                      sorted(self._memory.workers))
             if bool(regex.search(worker.hostname)) ^ negate
             ), 0, None)
        for worker in found_workers:  # type:Worker
            yield serialize_worker(worker)

    def task(self, task_uuid):
        """Finds one specific task.

        Args:
            task_uuid (str): any part of the task name or routing key

        """
        task = self._memory.tasks.get(task_uuid)
        if task:
            return serialize_task(task, task.state, False)

    def seen_tasks(self):
        return self._memory.task_types()

    def reset(self):
        """Resets all captured tasks.

        """
        self._memory.clear_tasks()

    def stats(self):
        m = self._memory
        return m.task_count, m.event_count, len(m.tasks), len(m.workers)
<commit_msg>refac(server) clearer code<commit_after># coding=utf-8
from __future__ import absolute_import, print_function, unicode_literals

import logging
import re
import signal
import threading
from Queue import Queue
from contextlib import contextmanager
from itertools import islice

from celery import Celery, states
from celery.backends.base import DisabledBackend
from celery.events import EventReceiver
from celery.events.state import State, Task, Worker
from kombu import log as kombu_log

from clearly.safe_compiler import safe_compile_text
from .expected_state import ExpectedStateHandler, setup_task_states, \
    setup_worker_states
from .serializer import serialize_task, serialize_worker

kombu_log.get_logger('').setLevel(logging.ERROR)

class ClearlyServer(object):
    """Simple and real-time monitor for celery.
    Server object, to capture events and handle tasks and workers.

        Attributes:
            _app: Celery
            _memory: celery.events.State
            _task_states: ExpectedStateHandler
            _worker_states: ExpectedStateHandler
    """

    def __init__(self, app=None, ignore_result_backend=False,
                 max_tasks_in_memory=1000, max_workers_in_memory=100):
        """Constructs a server instance.

        Args:
            app (Celery): a configured celery app instance
            ignore_result_backend (bool): if True, do not fetch results from the actual result backend
            max_tasks_in_memory (int): max tasks stored
            max_workers_in_memory (int): max workers stored

        """
        if isinstance(app.backend, DisabledBackend):
            ignore_result_backend = True

        self._app = app
        self._ignore_result_backend = ignore_result_backend

        self._memory = self._app.events.State(
            max_tasks_in_memory=max_tasks_in_memory,
            max_workers_in_memory=max_workers_in_memory,
        )  # type: State
        self._task_states = setup_task_states()  # type: ExpectedStateHandler
        self._worker_states = setup_worker_states()  # type: ExpectedStateHandler

        # concurrency control
        self._background_lock = threading.RLock()
        self._background_connected = threading.Event()

        # running engine
        self._background_thread = None  # type:threading.Thread
        self._background_receiver = None  # type:EventReceiver

        # connected client
        self._client_queue = None  # type:Queue
        self._client_regex = None
        self._client_negate = None

        # detect shutdown.
        def sigterm_handler(_signo, _stack_frame):
            self.stop()

        signal.signal(signal.SIGTERM, sigterm_handler)

    def start(self):
        """Starts the real-time engine that captures tasks.

        """

        with self._background_lock:
            if self._background_thread:
                return
            self._background_thread = threading.Thread(
                target=self.__run_server, name='clearly-agent')
            self._background_thread.daemon = True

            self._background_connected.clear()
            self._background_thread.start()
            self._background_connected.wait()

    def stop(self):
        """Stops the background engine, without losing anything already
        captured.

        """

        with self._background_lock:
            if not self._background_thread:
                return
            print('Stopping server')
            self._background_receiver.should_stop = True
            while self._background_thread.is_alive():
                self._background_thread.join()
            self._background_thread = None
            self._background_receiver = None

    @contextmanager
    def client_connect(self, pattern=None, negate=False):
        """Connects a client to this server, filtering the tasks that are sent
        to it.

        Args:
            pattern (Optional[str]): a pattern to filter tasks to capture.
                ex.: '^dispatch|^email' to filter names starting with that
                      or 'dispatch.*123456' to filter that exact name and number
                      or even '123456' to filter that exact number anywhere.
            negate (bool): if True, finds tasks that do not match criteria

        """

        self._client_regex = re.compile(pattern or '.')
        self._client_negate = negate

        with self._background_lock:
            self._client_queue = Queue()
            yield self._client_queue
            self._client_queue = None

    def __run_server(self):
        import sys
        print('Starting server', threading.current_thread())
        sys.stdout.flush()

        def client_accepts(values):
            return any(v and self._client_regex.search(v)
                       for v in values) ^ self._client_negate

        def maybe_publish(func, *values):
            if self._client_queue:
                if values and client_accepts(values):
                    obj = func()
                    try:
                        self._client_queue.put(obj)
                    except AttributeError:
                        # the capture is not synced with terminal anymore.
                        # no problem.
                        pass

        def process_event(event):
            event_type = event['type']
            if event_type.startswith('task'):
                for task, state, created in process_task_event(event):
                    maybe_publish(lambda: serialize_task(task, state, created),
                                  task.name, task.routing_key)

            elif event_type.startswith('worker'):
                for worker in process_worker_event(event):
                    maybe_publish(lambda: serialize_worker(worker))

            else:
                maybe_publish(lambda: event)

        def process_task_event(event):
            task, created = self._memory.get_or_create_task(event['uuid'])
            with self._task_states.track_changes(task):
                (_, _), subject = self._memory.event(event)
            if task.state == states.SUCCESS:
                if self._ignore_result_backend:
                    # celery tasks' results are escaped, so we must compile them.
                    task.result = safe_compile_text(task.result)
                else:
                    task.result = self._app.AsyncResult(task.uuid).result
            if created:
                yield task, '-', True
            for state in self._task_states.states_through():
                yield task, state, False

        def process_worker_event(event):
            worker, _ = self._memory.get_or_create_worker(event['hostname'])
            with self._worker_states.track_changes(worker):
                (_, _), subject = self._memory.event(event)
            for _ in self._worker_states.states_through():
                yield worker

        with self._app.connection() as connection:
            self._background_receiver = self._app.events.Receiver(
                connection, handlers={
                    '*': process_event,
                })  # type: EventReceiver
            self._background_connected.set()
            self._background_receiver.capture(limit=None, timeout=None,
                                              wakeup=True)

        print('Server stopped', threading.current_thread())
        sys.stdout.flush()

    def tasks(self, pattern=None, state=None, negate=False):
        """Filters captured tasks.

        Args:
            pattern (Optional[str]): any part of the task name or routing key
            state (Optional[str]): a state to filter tasks
            negate (bool): if True, finds tasks that do not match criteria

        """
        pcondition = scondition = lambda task: True
        if pattern:
            regex = re.compile(pattern)
            pcondition = lambda task: \
                regex.search(task.name or '') or \
                regex.search(task.routing_key or '')
        if state:
            scondition = lambda task: task.state == state

        found_tasks = islice(
            (task for _, task in self._memory.itertasks()
             if bool(pcondition(task) and scondition(task)) ^ negate
             ), 0, None)
        for task in found_tasks:  # type:Task
            yield serialize_task(task, task.state, False)

    def workers(self, pattern=None, negate=False):
        """Filters known workers and prints their current status.

        Args:
            pattern (Optional[str]): any part of the task name or routing key
            negate (bool): if True, finds tasks that do not match criteria

        """
        regex = re.compile(pattern or '.')
        found_workers = islice(
            (worker for worker in map(lambda w: self._memory.workers[w],
                                      sorted(self._memory.workers))
             if bool(regex.search(worker.hostname)) ^ negate
             ), 0, None)
        for worker in found_workers:  # type:Worker
            yield serialize_worker(worker)

    def task(self, task_uuid):
        """Finds one specific task.

        Args:
            task_uuid (str): any part of the task name or routing key

        """
        task = self._memory.tasks.get(task_uuid)
        if task:
            return serialize_task(task, task.state, False)

    def seen_tasks(self):
        return self._memory.task_types()

    def reset(self):
        """Resets all captured tasks.

        """
        self._memory.clear_tasks()

    def stats(self):
        m = self._memory
        return m.task_count, m.event_count, len(m.tasks), len(m.workers)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
"""
- Query workflow --------------------------------------------------------------

To query Aleph, just create one of the Queries - ISBNQuery for example and put
it into SearchRequest wrapper. Then encode it by calling toAMQPMessage() and
send the message to the Aleph's exchange.

---
isbnq = ISBNQuery("80-251-0225-4")
request = SearchRequest(isbnq)

amqp.send(
    message    = convertors.toJSON(request),
    properties = "..",
    exchange   = "ALEPH'S_EXCHANGE"
)
---

and you will get back AMQP message, and after decoding with fromAMQPMessage()
also SearchResult.

If you want to just get count of how many items is there in Aleph, just wrap
the ISBNQuery with CountRequest (you should use this instead of just calling
len() to SearchResult.records - it doesn't put that much load to Aleph):

---
isbnq = ISBNQuery("80-251-0225-4")
request = CountRequest(isbnq)

# rest is same..
---

and you will get back (after decoding) CountResult.

Here is ASCII flow diagram for you:

ISBNQuery      ----.                                 ,--> CountResult
AuthorQuery    ----|                                 |        `- num_of_records
PublisherQuery ----|                                 |
GenericQuery   ----|                                 |
                   |                                 |--> SearchResult
                   V                                 |        `- AlephRecord
         Count/SearchRequest                         |
                   |                                 |
                   |                                 |
         convertors.toJSON()               convertors.fromJSON()
                   |                                 ^
                   V             Client              |
              AMQPMessage ------> AMQP -------> AMQPMessage
                                 |    ^
                                 V    |
                                 |    ^
                                 V    |
                                 |    ^
                                 V    |
              AMQPMessage <------ AMQP <-------- AMQPMessage
                   |             Service              ^
                   |                                  |
                   V                                  |
          reactToAMQPMessage() ............... magic_happens()

Neat, isn't it?

- Export workflow -------------------------------------------------------------
TODO: implement, then write docstring
"""
#= Imports ====================================================================
from collections import namedtuple

import aleph
import convertors

# Datatypes for searching in Aleph ############################################
class CountRequest(namedtuple("CountRequest", ["query"])):
    """
    Put one of the Queries to .query property and it will return just
    number of records, instead of records itself.

    This help to saves some of Aleph resources (yeah, it is restricted to give
    too much queries by license).

    query -- GenericQuery, ISBNQuery, .. *Query structures in this module
    """
    pass

class SearchRequest(namedtuple("SearchRequest", ['query'])):
    """
    query -- GenericQuery, ISBNQuery, .. *Query structures in this module
    """
    pass

class AlephRecord(namedtuple("AlephRecord",
                             ['library',
                              'docNumber',
                              'xml',
                              'epublication'])):
    """
    This structure is returned as response to SearchRequest inside
    SearchResult.

    library -- library string, used for downloading documents from Aleph when
               you know proper docNumber
    docNumber -- identificator in Aleph. It is not that much unique as it could
                 be, but with .library string, you can fetch documents from
                 Aleph if you know this.
    xml -- MARC XML source returned from Aleph. Quite complicated stuff.
    epublication -- parsed .xml to EPublication structure
    """
    pass

class SearchResult(namedtuple("SearchResult", ['records'])):
    """
    This is response structure, which is sent back when SearchRequest is
    received.

    records -- array of AlephRecord structures
    """
    pass

class CountResult(namedtuple("CountResult", ['num_of_records'])):
    """
    This is returned back to client when he send CountRequest.
    """
    pass

class _QueryTemplate:
    """
    This class is here to just save some effort by using common ancestor with
    same .getSearchResult() and .getCountResult() definition.

    You probably shouldn't use it.
    """
    def getSearchResult(self):
        records = []
        for doc_id, library in self._getIDs():
            xml = aleph.downloadAlephDocument(doc_id, library)

            records.append(
                AlephRecord(
                    library,
                    doc_id,
                    xml,
                    convertors.toEPublication(xml)
                )
            )

        return SearchResult(records)

    def getCountResult(self):
        return CountResult(self._getCount())

class GenericQuery(namedtuple("GenericQuery",
                              ['base',
                               'phrase',
                               'considerSimilar',
                               'field']), _QueryTemplate):
    """
    Used for generic queries to aleph.

    For details of base/phrase/.. parameters, see aleph.py : searchInAleph().

    This is used mainly if you want to search by your own parameters and don't
    want to use prepared wrappers (AuthorQuery/ISBNQuery/..).
    """
    def _getIDs(self):
        return aleph.getDocumentIDs(
            aleph.searchInAleph(
                self.base,
                self.phrase,
                self.considerSimilar,
                self.field
            )
        )

    def _getCount(self):
        return aleph.searchInAleph(
            self.base,
            self.phrase,
            self.considerSimilar,
            self.field
        )["no_entries"]

class ISBNQuery(namedtuple("ISBNQuery", ["ISBN"]), _QueryTemplate):
    """
    Query for ISBN.

    Note: ISBN is not unique, so you can get back lot of books with same ISBN.
          some books also have two or more ISBNs.
    """
    def _getIDs(self):
        return aleph.getISBNsIDs(self.ISBN)

    def _getCount(self):
        return aleph.getISBNCount(self.ISBN)

class AuthorQuery(namedtuple("AuthorQuery", ["author"]), _QueryTemplate):
    def _getIDs(self):
        return aleph.getAuthorsBooksIDs(self.author)

    def _getCount(self):
        return aleph.getAuthorsBooksCount(self.author)

class PublisherQuery(namedtuple("PublisherQuery", ["publisher"]), _QueryTemplate):
    def _getIDs(self):
        return aleph.getPublishersBooksIDs(self.publisher)

    def _getCount(self):
        return aleph.getPublishersBooksCount(self.publisher)

###############################################################################
# Add new record to Aleph #####################################################
# TODO: implement exports
#
class Author(namedtuple("Author", ['firstName', 'lastName', 'title'])):
    pass

class Producent(namedtuple("Producent",
                           ['title',
                            'phone',
                            'fax',
                            'email',
                            'url',
                            'identificator',
                            'ico'])):
    pass

class EPublication(namedtuple("EPublication",
                              ['nazev',
                               'podnazev',
                               'vazba',
                               'cena',
                               'castDil',
                               'nazevCasti',
                               'nakladatelVydavatel',
                               'datumVydani',
                               'poradiVydani',
                               'zpracovatelZaznamu',
                               'kategorieProRIV',
                               'mistoDistribuce',
                               'distributor',
                               'datumDistribuce',
                               'datumProCopyright',
                               'format',
                               'url',
                               'mistoVydani',
                               'ISBNSouboruPublikaci',
                               'autori',
                               'originaly'])):
    """
    This structure is returned as result of users SearchRequest. It will be
    also used in exporting new data to aleph, but that is not implemented yet.

    In case of Search/Count requests, this structure is filled with data from
    MARC XML record parsed by marcxml.py.
    """
    pass

class OriginalFile(namedtuple("OriginalFile",
                              ['url', 'format', 'file', 'isbns'])):
    """ type of isbn: ISBN"""
    pass

## Export protocol query wrappers #############################################
class AlephExport(namedtuple("AlephExport",
                             ['epublication',
                              'linkOfEPublication'])):
    """ epublication ... type of EPublication
    linkOfEPublication  ... url with epublication

    User will fill this record.
    """
    pass

class ExportRequest(namedtuple("ExportRequest", ['export'])):
    pass

class AlephExportResult(namedtuple("AlephExportResult",
                                   ['docNumber',
                                    'base',
                                    'xml',
                                    'success',
                                    'message'])):
    """ docNumber ... docNumber of a record in Aleph
    base      ... base of Aleph
    success   ... whether import was successful
    message   ... message of error or success
    """
    pass

class ExportResult(namedtuple("ExportResult", ['result'])):
    """
    ... result is type of AlephExportResult
    """
    pass

# Interface for an external world #############################################

# Variables ###################################################################
QUERY_TYPES = [
    ISBNQuery,
    AuthorQuery,
    PublisherQuery,
    GenericQuery
]

REQUEST_TYPES = [
    SearchRequest,
    CountRequest
]

def iiOfAny(instance, classes):
    """
    Returns true, if `instance` is instance of any (iiOfAny) of the `classes`.

    This function doesn't use isinstance() check, it just compares the
    classnames.

    This can be generaly dangerous, but it is really useful when you are
    comparing class serialized in one module and deserialized in another.

    This causes, that module paths in class internals are different and
    isinstance() and type() comparsions thus fails.

    Use this function instead, if you wan't to check what type is your
    deserialized message.

    instance -- class instance you want to know the type
    classes -- list of classes, or just the class you want to compare - func
               automatically retypes nonlist/nontuple parameters to list
    """
    if type(classes) not in [list, tuple]:
        classes = [classes]

    return any(map(lambda x: type(instance).__name__ == x.__name__, classes))

# Functions ###################################################################
def reactToAMQPMessage(message, response_callback, UUID):
    """
    React to given AMQPMessage. Return data thru given callback function.

    message -- message encoded in JSON by convertors.toJSON()
    response_callback -- function taking exactly ONE parameter - message's body
                         with response. Function take care of sending the
                         response over AMQP.

    Returns result of response_callback() call.

    Raise:
        ValueError if bad type of `message` structure is given.

    TODO:
        React to Export requests.
    """
    req = convertors.fromJSON(message)

    # TODO: pridat podporu exportnich typu
    if not iiOfAny(req, REQUEST_TYPES):
        raise ValueError(
            "Unknown type of request: '" + str(type(req)) + "'!"
        )

    response = None
    if iiOfAny(req, CountRequest) and iiOfAny(req.query, QUERY_TYPES):
        response = req.query.getCountResult()
    elif iiOfAny(req, SearchRequest) and iiOfAny(req.query, QUERY_TYPES):
        response = req.query.getSearchResult()
    else:
        raise ValueError(
            "Unknown type of request: '" + str(type(req)) + "' or query: '" +
            str(type(req.query)) + "'!"
        )

    if response is not None:
        return response_callback(convertors.toJSON(response), UUID)
<commit_msg>Reformated for better readability.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
"""
- Query workflow --------------------------------------------------------------

To query Aleph, just create one of the Queries - ISBNQuery for example and put
it into SearchRequest wrapper. Then encode it by calling toAMQPMessage() and
send the message to the Aleph's exchange.

---
isbnq = ISBNQuery("80-251-0225-4")
request = SearchRequest(isbnq)

amqp.send(
    message    = convertors.toJSON(request),
    properties = "..",
    exchange   = "ALEPH'S_EXCHANGE"
)
---

and you will get back AMQP message, and after decoding with fromAMQPMessage()
also SearchResult.

If you want to just get count of how many items is there in Aleph, just wrap
the ISBNQuery with CountRequest (you should use this instead of just calling
len() to SearchResult.records - it doesn't put that much load to Aleph):

---
isbnq = ISBNQuery("80-251-0225-4")
request = CountRequest(isbnq)

# rest is same..
---

and you will get back (after decoding) CountResult.

Here is ASCII flow diagram for you:

ISBNQuery      ----.                                 ,--> CountResult
AuthorQuery    ----|                                 |        `- num_of_records
PublisherQuery ----|                                 |
GenericQuery   ----|                                 |
                   |                                 |--> SearchResult
                   V                                 |        `- AlephRecord
         Count/SearchRequest                         |
                   |                                 |
                   |                                 |
         convertors.toJSON()               convertors.fromJSON()
                   |                                 ^
                   V             Client              |
              AMQPMessage ------> AMQP -------> AMQPMessage
                                 |    ^
                                 V    |
                                 |    ^
                                 V    |
                                 |    ^
                                 V    |
              AMQPMessage <------ AMQP <-------- AMQPMessage
                   |             Service              ^
                   |                                  |
                   V                                  |
          reactToAMQPMessage() ............... magic_happens()

Neat, isn't it?

- Export workflow -------------------------------------------------------------
TODO: implement, then write docstring
"""
#= Imports ====================================================================
from collections import namedtuple

import aleph
import convertors

# Datatypes for searching in Aleph ############################################
class CountRequest(namedtuple("CountRequest", ["query"])):
    """
    Put one of the Queries to .query property and it will return just
    number of records, instead of records itself.

    This help to saves some of Aleph resources (yeah, it is restricted to give
    too much queries by license).

    query -- GenericQuery, ISBNQuery, .. *Query structures in this module
    """
    pass

class SearchRequest(namedtuple("SearchRequest", ['query'])):
    """
    query -- GenericQuery, ISBNQuery, .. *Query structures in this module
    """
    pass

class AlephRecord(namedtuple("AlephRecord", ['library',
                                             'docNumber',
                                             'xml',
                                             'epublication'])):
    """
    This structure is returned as response to SearchRequest inside
    SearchResult.

    library -- library string, used for downloading documents from Aleph when
               you know proper docNumber
    docNumber -- identificator in Aleph. It is not that much unique as it could
                 be, but with .library string, you can fetch documents from
                 Aleph if you know this.
    xml -- MARC XML source returned from Aleph. Quite complicated stuff.
    epublication -- parsed .xml to EPublication structure
    """
    pass

class SearchResult(namedtuple("SearchResult", ['records'])):
    """
    This is response structure, which is sent back when SearchRequest is
    received.

    records -- array of AlephRecord structures
    """
    pass

class CountResult(namedtuple("CountResult", ['num_of_records'])):
    """
    This is returned back to client when he send CountRequest.
    """
    pass

class _QueryTemplate:
    """
    This class is here to just save some effort by using common ancestor with
    same .getSearchResult() and .getCountResult() definition.

    You probably shouldn't use it.
    """
    def getSearchResult(self):
        records = []
        for doc_id, library in self._getIDs():
            xml = aleph.downloadAlephDocument(doc_id, library)

            records.append(
                AlephRecord(
                    library,
                    doc_id,
                    xml,
                    convertors.toEPublication(xml)
                )
            )

        return SearchResult(records)

    def getCountResult(self):
        return CountResult(self._getCount())

class GenericQuery(namedtuple("GenericQuery", ['base',
                                               'phrase',
                                               'considerSimilar',
                                               'field']),
                   _QueryTemplate):
    """
    Used for generic queries to aleph.

    For details of base/phrase/.. parameters, see aleph.py : searchInAleph().

    This is used mainly if you want to search by your own parameters and don't
    want to use prepared wrappers (AuthorQuery/ISBNQuery/..).
    """
    def _getIDs(self):
        return aleph.getDocumentIDs(
            aleph.searchInAleph(
                self.base,
                self.phrase,
                self.considerSimilar,
                self.field
            )
        )

    def _getCount(self):
        return aleph.searchInAleph(
            self.base,
            self.phrase,
            self.considerSimilar,
            self.field
        )["no_entries"]

class ISBNQuery(namedtuple("ISBNQuery", ["ISBN"]), _QueryTemplate):
    """
    Query for ISBN.

    Note: ISBN is not unique, so you can get back lot of books with same ISBN.
          some books also have two or more ISBNs.
    """
    def _getIDs(self):
        return aleph.getISBNsIDs(self.ISBN)

    def _getCount(self):
        return aleph.getISBNCount(self.ISBN)

class AuthorQuery(namedtuple("AuthorQuery", ["author"]), _QueryTemplate):
    def _getIDs(self):
        return aleph.getAuthorsBooksIDs(self.author)

    def _getCount(self):
        return aleph.getAuthorsBooksCount(self.author)

class PublisherQuery(namedtuple("PublisherQuery", ["publisher"]), _QueryTemplate):
    def _getIDs(self):
        return aleph.getPublishersBooksIDs(self.publisher)

    def _getCount(self):
        return aleph.getPublishersBooksCount(self.publisher)

###############################################################################
# Add new record to Aleph #####################################################
# TODO: implement exports
#
class Author(namedtuple("Author", ['firstName', 'lastName', 'title'])):
    pass

class Producent(namedtuple("Producent", ['title',
                                         'phone',
                                         'fax',
                                         'email',
                                         'url',
                                         'identificator',
                                         'ico'])):
    pass

class EPublication(namedtuple("EPublication", ['nazev',
                                               'podnazev',
                                               'vazba',
                                               'cena',
                                               'castDil',
                                               'nazevCasti',
                                               'nakladatelVydavatel',
                                               'datumVydani',
                                               'poradiVydani',
                                               'zpracovatelZaznamu',
                                               'kategorieProRIV',
                                               'mistoDistribuce',
                                               'distributor',
                                               'datumDistribuce',
                                               'datumProCopyright',
                                               'format',
                                               'url',
                                               'mistoVydani',
                                               'ISBNSouboruPublikaci',
                                               'autori',
                                               'originaly'])):
    """
    This structure is returned as result of users SearchRequest. It will be
    also used in exporting new data to aleph, but that is not implemented yet.

    In case of Search/Count requests, this structure is filled with data from
    MARC XML record parsed by marcxml.py.
    """
    pass

class OriginalFile(namedtuple("OriginalFile", ['url',
                                               'format',
                                               'file',
                                               'isbns'])):
    """ type of isbn: ISBN"""
    pass

## Export protocol query wrappers #############################################
class ExportRequest(namedtuple("AlephExport", ['epublication',
                                               'linkOfEPublication'])):
    """
    epublication -- type of EPublication
    linkOfEPublication -- url with epublication
    """
    pass

class AlephExportResult(namedtuple("AlephExportResult", ['docNumber',
                                                         'base',
                                                         'xml',
                                                         'success',
                                                         'message'])):
    """ docNumber ... docNumber of a record in Aleph
    base      ... base of Aleph
    success   ... whether import was successful
    message   ... message of error or success
    """
    pass

class ExportResult(namedtuple("ExportResult", ['result'])):
    """
    ... result is type of AlephExportResult
    """
    pass

# Interface for an external world #############################################

# Variables ###################################################################
QUERY_TYPES = [
    ISBNQuery,
    AuthorQuery,
    PublisherQuery,
    GenericQuery
]

REQUEST_TYPES = [
    SearchRequest,
    CountRequest
]

def iiOfAny(instance, classes):
    """
    Returns true, if `instance` is instance of any (iiOfAny) of the `classes`.

    This function doesn't use isinstance() check, it just compares the
    classnames.

    This can be generaly dangerous, but it is really useful when you are
    comparing class serialized in one module and deserialized in another.

    This causes, that module paths in class internals are different and
    isinstance() and type() comparsions thus fails.

    Use this function instead, if you wan't to check what type is your
    deserialized message.

    instance -- class instance you want to know the type
    classes -- list of classes, or just the class you want to compare - func
               automatically retypes nonlist/nontuple parameters to list
    """
    if type(classes) not in [list, tuple]:
        classes = [classes]

    return any(map(lambda x: type(instance).__name__ == x.__name__, classes))

# Functions ###################################################################
def reactToAMQPMessage(message, response_callback, UUID):
    """
    React to given AMQPMessage. Return data thru given callback function.

    message -- message encoded in JSON by convertors.toJSON()
    response_callback -- function taking exactly ONE parameter - message's body
                         with response. Function take care of sending the
                         response over AMQP.

    Returns result of response_callback() call.

    Raise:
        ValueError if bad type of `message` structure is given.

    TODO:
        React to Export requests.
    """
    req = convertors.fromJSON(message)

    # TODO: pridat podporu exportnich typu
    if not iiOfAny(req, REQUEST_TYPES):
        raise ValueError(
            "Unknown type of request: '" + str(type(req)) + "'!"
        )

    response = None
    if iiOfAny(req, CountRequest) and iiOfAny(req.query, QUERY_TYPES):
        response = req.query.getCountResult()
    elif iiOfAny(req, SearchRequest) and iiOfAny(req.query, QUERY_TYPES):
        response = req.query.getSearchResult()
    else:
        raise ValueError(
            "Unknown type of request: '" + str(type(req)) + "' or query: '" +
            str(type(req.query)) + "'!"
        )

    if response is not None:
        return response_callback(convertors.toJSON(response), UUID)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    click
    ~~~~~

    Click is a simple Python module that wraps the stdlib's optparse to make
    writing command line scripts fun.  Unlike other modules, it's based around
    a simple API that does not come with too much magic and is composable.

    In case optparse ever gets removed from the stdlib, it will be shipped by
    this module.

    :copyright: (c) 2014 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

# Core classes
from .core import Context, BaseCommand, Command, MultiCommand, Group, \
     CommandCollection, Parameter, Option, Argument

# Globals
from .globals import get_current_context

# Decorators
from .decorators import pass_context, pass_obj, make_pass_decorator, \
     command, group, argument, option, confirmation_option, \
     password_option, version_option, help_option

# Types
from .types import ParamType, File, Path, Choice, IntRange, Tuple, \
     STRING, INT, FLOAT, BOOL, UUID, UNPROCESSED, FloatRange

# Utilities
from .utils import echo, get_binary_stream, get_text_stream, open_file, \
     format_filename, get_app_dir, get_os_args

# Terminal functions
from .termui import prompt, confirm, get_terminal_size, echo_via_pager, \
     progressbar, clear, style, unstyle, secho, edit, launch, getchar, \
     pause

# Exceptions
from .exceptions import ClickException, UsageError, BadParameter, \
     FileError, Abort, NoSuchOption, BadOptionUsage, BadArgumentUsage, \
     MissingParameter

# Formatting
from .formatting import HelpFormatter, wrap_text

# Parsing
from .parser import OptionParser

__all__ = [
    # Core classes
    'Context', 'BaseCommand', 'Command', 'MultiCommand', 'Group',
    'CommandCollection', 'Parameter', 'Option', 'Argument',

    # Globals
    'get_current_context',

    # Decorators
    'pass_context', 'pass_obj', 'make_pass_decorator', 'command', 'group',
    'argument', 'option', 'confirmation_option', 'password_option',
    'version_option', 'help_option',

    # Types
    'ParamType', 'File', 'Path', 'Choice', 'IntRange', 'Tuple', 'STRING',
    'INT', 'FLOAT', 'BOOL', 'UUID', 'UNPROCESSED', 'FloatRange'

    # Utilities
    'echo', 'get_binary_stream', 'get_text_stream', 'open_file',
    'format_filename', 'get_app_dir', 'get_os_args',

    # Terminal functions
    'prompt', 'confirm', 'get_terminal_size', 'echo_via_pager',
    'progressbar', 'clear', 'style', 'unstyle', 'secho', 'edit', 'launch',
    'getchar', 'pause',

    # Exceptions
    'ClickException', 'UsageError', 'BadParameter', 'FileError',
    'Abort', 'NoSuchOption', 'BadOptionUsage', 'BadArgumentUsage',
    'MissingParameter',

    # Formatting
    'HelpFormatter', 'wrap_text',

    # Parsing
    'OptionParser',
]

# Controls if click should emit the warning about the use of unicode
# literals.
disable_unicode_literals_warning = False

__version__ = '7.0-dev'
<commit_msg>Fix missing comma in __all__ list (#935)<commit_after># -*- coding: utf-8 -*-
"""
    click
    ~~~~~

    Click is a simple Python module that wraps the stdlib's optparse to make
    writing command line scripts fun.  Unlike other modules, it's based around
    a simple API that does not come with too much magic and is composable.

    In case optparse ever gets removed from the stdlib, it will be shipped by
    this module.

    :copyright: (c) 2014 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

# Core classes
from .core import Context, BaseCommand, Command, MultiCommand, Group, \
     CommandCollection, Parameter, Option, Argument

# Globals
from .globals import get_current_context

# Decorators
from .decorators import pass_context, pass_obj, make_pass_decorator, \
     command, group, argument, option, confirmation_option, \
     password_option, version_option, help_option

# Types
from .types import ParamType, File, Path, Choice, IntRange, Tuple, \
     STRING, INT, FLOAT, BOOL, UUID, UNPROCESSED, FloatRange

# Utilities
from .utils import echo, get_binary_stream, get_text_stream, open_file, \
     format_filename, get_app_dir, get_os_args

# Terminal functions
from .termui import prompt, confirm, get_terminal_size, echo_via_pager, \
     progressbar, clear, style, unstyle, secho, edit, launch, getchar, \
     pause

# Exceptions
from .exceptions import ClickException, UsageError, BadParameter, \
     FileError, Abort, NoSuchOption, BadOptionUsage, BadArgumentUsage, \
     MissingParameter

# Formatting
from .formatting import HelpFormatter, wrap_text

# Parsing
from .parser import OptionParser

__all__ = [
    # Core classes
    'Context', 'BaseCommand', 'Command', 'MultiCommand', 'Group',
    'CommandCollection', 'Parameter', 'Option', 'Argument',

    # Globals
    'get_current_context',

    # Decorators
    'pass_context', 'pass_obj', 'make_pass_decorator', 'command', 'group',
    'argument', 'option', 'confirmation_option', 'password_option',
    'version_option', 'help_option',

    # Types
    'ParamType', 'File', 'Path', 'Choice', 'IntRange', 'Tuple', 'STRING',
    'INT', 'FLOAT', 'BOOL', 'UUID', 'UNPROCESSED', 'FloatRange',

    # Utilities
    'echo', 'get_binary_stream', 'get_text_stream', 'open_file',
    'format_filename', 'get_app_dir', 'get_os_args',

    # Terminal functions
    'prompt', 'confirm', 'get_terminal_size', 'echo_via_pager',
    'progressbar', 'clear', 'style', 'unstyle', 'secho', 'edit', 'launch',
    'getchar', 'pause',

    # Exceptions
    'ClickException', 'UsageError', 'BadParameter', 'FileError',
    'Abort', 'NoSuchOption', 'BadOptionUsage', 'BadArgumentUsage',
    'MissingParameter',

    # Formatting
    'HelpFormatter', 'wrap_text',

    # Parsing
    'OptionParser',
]

# Controls if click should emit the warning about the use of unicode
# literals.
disable_unicode_literals_warning = False

__version__ = '7.0-dev'
<|endoftext|>


<commit_before>#!/usr/bin/python

## Binary Analysis Tool
## Copyright 2013 Armijn Hemel for Tjaldur Software Governance Solutions
## Licensed under Apache 2.0, see LICENSE file for details

'''
This script can be used to initialize the clone database. The clone database is
used to record information about which packages should be treated as the same
package, or under which alternative names it is known, and so on.

Cloning of packages happens frequently:

* renaming: packages are renamed for some reason, like politics
* bundling: one package is copied entirely into another package, for example
  as "third party software"
* partial copying: parts of a package have been copied into another package.
  Examples are glue code for some Python packages, where a large amount of
  packages share just one file

Apart from verbatim cloning there is also cloning that happens in a more subtle
way. For example, code was copied, then slightly adapted. It might not be the
same when looking at SHA256 checksums of the files, but it might still look
the same when looking at strings or function names.
'''

import os, sys, sqlite3
from optparse import OptionParser

def main(argv):
        parser = OptionParser()
	parser.add_option("-d", "--database", dest="db", help="path to clone database", metavar="FILE")
	(options, args) = parser.parse_args()
	if options.db == None:
                parser.error("Path to clone database file needed")
        try:
                conn = sqlite3.connect(options.db)
        except:
                print "Can't open clone database file"
                sys.exit(1)

	c = conn.cursor()

	## create table for renamed packages
	c.execute('''create table if not exists renames (originalname text, newname text)''')
	c.execute('''create index if not exists renames_index on renames (originalname)''')
	c.execute('''create index if not exists renames_index on renames (newname)''')

	## insert some values as examples
	c.execute('''insert into renames values ('ethereal', 'wireshark')''')
	c.execute('''insert into renames values ('koffice', 'calligra')''')
	c.execute('''insert into renames values ('ucd-snmp', 'net-snmp')''')
	c.execute('''insert into renames values ('iproute', 'iproute2')''')
	c.execute('''insert into renames values ('gaim', 'pidgin')''')
	c.execute('''insert into renames values ('kdebase-runtime', 'kde-runtime')''')
	c.execute('''insert into renames values ('kdebase-workspace', 'kde-workspace')''')
	c.execute('''insert into renames values ('eglibc', 'glibc')''')
	c.execute('''insert into renames values ('org.apache.servicemix.bundles.ant', 'apache-ant')''')
	c.execute('''insert into renames values ('wengophone', 'qutecom')''')
	c.execute('''insert into renames values ('gaim-plugin_pack', 'purple-plugin_pack')''')

	conn.commit()
	c.close()
	conn.close()

if __name__ == "__main__":
        main(sys.argv)
<commit_msg>fix index name<commit_after>#!/usr/bin/python

## Binary Analysis Tool
## Copyright 2013 Armijn Hemel for Tjaldur Software Governance Solutions
## Licensed under Apache 2.0, see LICENSE file for details

'''
This script can be used to initialize the clone database. The clone database is
used to record information about which packages should be treated as the same
package, or under which alternative names it is known, and so on.

Cloning of packages happens frequently:

* renaming: packages are renamed for some reason, like politics
* bundling: one package is copied entirely into another package, for example
  as "third party software"
* partial copying: parts of a package have been copied into another package.
  Examples are glue code for some Python packages, where a large amount of
  packages share just one file

Apart from verbatim cloning there is also cloning that happens in a more subtle
way. For example, code was copied, then slightly adapted. It might not be the
same when looking at SHA256 checksums of the files, but it might still look
the same when looking at strings or function names.
'''

import os, sys, sqlite3
from optparse import OptionParser

def main(argv):
        parser = OptionParser()
	parser.add_option("-d", "--database", dest="db", help="path to clone database", metavar="FILE")
	(options, args) = parser.parse_args()
	if options.db == None:
                parser.error("Path to clone database file needed")
        try:
                conn = sqlite3.connect(options.db)
        except:
                print "Can't open clone database file"
                sys.exit(1)

	c = conn.cursor()

	## create table for renamed packages
	c.execute('''create table if not exists renames (originalname text, newname text)''')
	c.execute('''create index if not exists renames_index_originalname on renames (originalname)''')
	c.execute('''create index if not exists renames_index_newname on renames (newname)''')

	## insert some values as examples
	c.execute('''insert into renames values ('ethereal', 'wireshark')''')
	c.execute('''insert into renames values ('koffice', 'calligra')''')
	c.execute('''insert into renames values ('ucd-snmp', 'net-snmp')''')
	c.execute('''insert into renames values ('iproute', 'iproute2')''')
	c.execute('''insert into renames values ('gaim', 'pidgin')''')
	c.execute('''insert into renames values ('kdebase-runtime', 'kde-runtime')''')
	c.execute('''insert into renames values ('kdebase-workspace', 'kde-workspace')''')
	c.execute('''insert into renames values ('eglibc', 'glibc')''')
	c.execute('''insert into renames values ('org.apache.servicemix.bundles.ant', 'apache-ant')''')
	c.execute('''insert into renames values ('wengophone', 'qutecom')''')
	c.execute('''insert into renames values ('gaim-plugin_pack', 'purple-plugin_pack')''')

	conn.commit()
	c.close()
	conn.close()

if __name__ == "__main__":
        main(sys.argv)
<|endoftext|>