<commit_before>"""
A set of useful modules for reading data with the Steam API

Copyright (c) 2010, Anthony Garcia <lagg@lavabit.com>

Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
"""

__version__ = "2.0"

from base import *
<commit_msg>Bump version to 3.0<commit_after>"""
A set of useful modules for reading data with the Steam API

Copyright (c) 2010, Anthony Garcia <lagg@lavabit.com>

Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
"""

__version__ = "3.0"

from base import *
<|endoftext|>


<commit_before>import numpy as np

def simon(message, **kwargs):
    """
    The Statistical Interpretation MONitor.

    A warning system designed to always remind the user that Simon
    is watching him/her.

    Parameters
    ---------
    message : string
        The message that is thrown
    kwargs : dict
        The rest of the arguments that are passed to warnings.warn
    """
    warnings.warn("SIMON says: {0}".format(message), **kwargs)

def rebin_data(x, y, dx_new, method='sum'):

    """
    Rebin some data to an arbitrary new data resolution. Either sum
    the data points in the new bins or average them.

    Parameters
    ----------
    x: iterable
        The dependent variable with some resolution dx_old = x[1]-x[0]

    y: interable
        The independent variable to be binned

    dx_new: float
        The new resolution of the dependent variable x

    method: {"sum" | "average" | "mean"}, optional, default "sum"
        The method to be used in binning. Either sum the samples y in
        each new bin of x, or take the arithmetic mean.

    Returns
    -------
    xbin: numpy.ndarray
        The midpoints of the new bins in x

    ybin: numpy.ndarray
        The binned quantity y
    """

    dx_old = x[1] - x[0]

    assert dx_new >= dx_old, "New frequency resolution must be larger than " \
                             "old frequency resolution."

    step_size = np.float(dx_new)/np.float(dx_old)

    output = []
    for i in np.arange(0, y.shape[0], step_size):
        total = 0

        prev_frac = int(i+1) - i
        prev_bin = int(i)
        total += prev_frac * y[prev_bin]

        if i + step_size < len(x):
            # Fractional part of next bin:
            next_frac = i+step_size - int(i+step_size)
            next_bin = int(i+step_size)
            total += next_frac * y[next_bin]

        total += sum(y[int(i+1):int(i+step_size)])
        output.append(total)

    output = np.asarray(output)

    if method in ['mean', 'avg', 'average', 'arithmetic mean']:
        ybin = output/np.float(step_size)

    elif method == "sum":
        ybin = output
    else:
        raise Exception("Method for summing or averaging not recognized. "
                        "Please enter either 'sum' or 'mean'.")

    tseg = x[-1]-x[0]+dx_old

    if tseg/dx_new % 1.0 > 0.0:
        ybin = ybin[:-1]

    xbin = np.arange(ybin.shape[0])*dx_new + x[0]-dx_old/2.+dx_new/2.

    return xbin, ybin, step_size
<commit_msg>Corrected bug when calling SIMON; import jit from numba, or create a fake one<commit_after>import numpy as np
import warnings
# If numba is installed, import jit. Otherwise, define an empty decorator with the same name.
try:
    from numba import jit
except:
    def jit(fun):
        return fun

def simon(message, **kwargs):
    """
    The Statistical Interpretation MONitor.

    A warning system designed to always remind the user that Simon
    is watching him/her.

    Parameters
    ---------
    message : string
        The message that is thrown
    kwargs : dict
        The rest of the arguments that are passed to warnings.warn
    """
    warnings.warn("SIMON says: {0}".format(message), **kwargs)

def rebin_data(x, y, dx_new, method='sum'):

    """
    Rebin some data to an arbitrary new data resolution. Either sum
    the data points in the new bins or average them.

    Parameters
    ----------
    x: iterable
        The dependent variable with some resolution dx_old = x[1]-x[0]

    y: interable
        The independent variable to be binned

    dx_new: float
        The new resolution of the dependent variable x

    method: {"sum" | "average" | "mean"}, optional, default "sum"
        The method to be used in binning. Either sum the samples y in
        each new bin of x, or take the arithmetic mean.

    Returns
    -------
    xbin: numpy.ndarray
        The midpoints of the new bins in x

    ybin: numpy.ndarray
        The binned quantity y
    """

    dx_old = x[1] - x[0]

    assert dx_new >= dx_old, "New frequency resolution must be larger than " \
                             "old frequency resolution."

    step_size = np.float(dx_new)/np.float(dx_old)

    output = []
    for i in np.arange(0, y.shape[0], step_size):
        total = 0

        prev_frac = int(i+1) - i
        prev_bin = int(i)
        total += prev_frac * y[prev_bin]

        if i + step_size < len(x):
            # Fractional part of next bin:
            next_frac = i+step_size - int(i+step_size)
            next_bin = int(i+step_size)
            total += next_frac * y[next_bin]

        total += sum(y[int(i+1):int(i+step_size)])
        output.append(total)

    output = np.asarray(output)

    if method in ['mean', 'avg', 'average', 'arithmetic mean']:
        ybin = output/np.float(step_size)

    elif method == "sum":
        ybin = output
    else:
        raise Exception("Method for summing or averaging not recognized. "
                        "Please enter either 'sum' or 'mean'.")

    tseg = x[-1]-x[0]+dx_old

    if tseg/dx_new % 1.0 > 0.0:
        ybin = ybin[:-1]

    xbin = np.arange(ybin.shape[0])*dx_new + x[0]-dx_old/2.+dx_new/2.

    return xbin, ybin, step_size
<|endoftext|>


<commit_before># TODO: technically incorrect on 0 robustness since conflates < and >

from functools import singledispatch
from operator import sub, add

from lenses import lens

import stl.ast

oo = float('inf')

@singledispatch
def pointwise_robustness(stl):
    raise NotImplementedError

@pointwise_robustness.register(stl.Or)
def _(stl):
    return lambda x, t: max(pointwise_robustness(arg)(x, t) for arg in stl.args)

@pointwise_robustness.register(stl.And)
def _(stl):
    return lambda x, t: min(pointwise_robustness(arg)(x, t) for arg in stl.args)

@pointwise_robustness.register(stl.F)
def _(stl):
    lo, hi = stl.interval
    return lambda x, t: max((pointwise_robustness(stl.arg)(x, t + t2)
                             for t2 in x[lo:hi].index), default=-oo)

@pointwise_robustness.register(stl.G)
def _(stl):
    lo, hi = stl.interval
    return lambda x, t: min((pointwise_robustness(stl.arg)(x, t + t2)
                             for t2 in x[lo:hi].index), default=oo)

@pointwise_robustness.register(stl.Neg)
def _(stl):
    return lambda x, t: -pointwise_robustness(arg)(x, t)

op_lookup = {
    ">": sub,
    ">=": sub,
    "<": lambda x, y: sub(y, x),
    "<=": lambda x, y: sub(y, x),
    "=": lambda a, b: -abs(a - b),
}

@pointwise_robustness.register(stl.LinEq)
def _(stl):
    op = op_lookup[stl.op]
    return lambda x, t: op(eval_terms(stl, x, t), stl.const)

def eval_terms(lineq, x, t):
    psi = lens(lineq).terms.each_().modify(eval_term(x, t))
    return sum(psi.terms)

def eval_term(x, t):
    return lambda term: term.coeff*x[term.id.name][t]
<commit_msg>interpolate if necessary<commit_after># TODO: technically incorrect on 0 robustness since conflates < and >

from functools import singledispatch
from operator import sub, add

import numpy as np
from lenses import lens

import stl.ast

oo = float('inf')

@singledispatch
def pointwise_robustness(stl):
    raise NotImplementedError

@pointwise_robustness.register(stl.Or)
def _(stl):
    return lambda x, t: max(pointwise_robustness(arg)(x, t) for arg in stl.args)

@pointwise_robustness.register(stl.And)
def _(stl):
    return lambda x, t: min(pointwise_robustness(arg)(x, t) for arg in stl.args)

@pointwise_robustness.register(stl.F)
def _(stl):
    lo, hi = stl.interval
    return lambda x, t: max((pointwise_robustness(stl.arg)(x, t + t2)
                             for t2 in x[lo:hi].index), default=-oo)

@pointwise_robustness.register(stl.G)
def _(stl):
    lo, hi = stl.interval
    return lambda x, t: min((pointwise_robustness(stl.arg)(x, t + t2)
                             for t2 in x[lo:hi].index), default=oo)

@pointwise_robustness.register(stl.Neg)
def _(stl):
    return lambda x, t: -pointwise_robustness(arg)(x, t)

op_lookup = {
    ">": sub,
    ">=": sub,
    "<": lambda x, y: sub(y, x),
    "<=": lambda x, y: sub(y, x),
    "=": lambda a, b: -abs(a - b),
}

@pointwise_robustness.register(stl.LinEq)
def _(stl):
    op = op_lookup[stl.op]
    return lambda x, t: op(eval_terms(stl, x, t), stl.const)

def eval_terms(lineq, x, t):
    psi = lens(lineq).terms.each_().modify(eval_term(x, t))
    return sum(psi.terms)

def eval_term(x, t):
    # TODO(lift interpolation much higher)
    return lambda term: term.coeff*np.interp(t, x.index, x[term.id.name])
<|endoftext|>


<commit_before>import threading
import time

import utils
from constants import *

try:
    from fractions import gcd
except ImportError:
    from backward import gcd

import logging
log = logging.getLogger('stomp.py')

class ConnectionListener(object):
    """
    This class should be used as a base class for objects registered
    using Connection.set_listener().
    """
    def on_connecting(self, host_and_port):
        """
        Called by the STOMP connection once a TCP/IP connection to the
        STOMP server has been established or re-established. Note that
        at this point, no connection has been established on the STOMP
        protocol level. For this, you need to invoke the "connect"
        method on the connection.

        \param host_and_port a tuple containing the host name and port
        number to which the connection has been established.
        """
        pass

    def on_connected(self, headers, body):
        """
        Called by the STOMP connection when a CONNECTED frame is
        received, that is after a connection has been established or
        re-established.

        \param headers a dictionary containing all headers sent by the
        server as key/value pairs.

        \param body the frame's payload. This is usually empty for
        CONNECTED frames.
        """
        pass

    def on_disconnected(self):
        """
        Called by the STOMP connection when a TCP/IP connection to the
        STOMP server has been lost.  No messages should be sent via
        the connection until it has been reestablished.
        """
        pass

    def on_heartbeat_timeout(self):
        """
        Called by the STOMP connection when a heartbeat message has not been
        received beyond the specified period.
        """
        pass

    def on_before_message(self, headers, body):
        pass

    def on_message(self, headers, body):
        """
        Called by the STOMP connection when a MESSAGE frame is
        received.

        \param headers a dictionary containing all headers sent by the
        server as key/value pairs.

        \param body the frame's payload - the message body.
        """
        pass

    def on_receipt(self, headers, body):
        """
        Called by the STOMP connection when a RECEIPT frame is
        received, sent by the server if requested by the client using
        the 'receipt' header.

        \param headers a dictionary containing all headers sent by the
        server as key/value pairs.

        \param body the frame's payload. This is usually empty for
        RECEIPT frames.
        """
        pass

    def on_error(self, headers, body):
        """
        Called by the STOMP connection when an ERROR frame is
        received.

        \param headers a dictionary containing all headers sent by the
        server as key/value pairs.

        \param body the frame's payload - usually a detailed error
        description.
        """
        pass

    def on_send(self, frame):
        """
        Called by the STOMP connection when it is in the process of sending a message

        \param frame the frame to be sent
        """
        pass

    def on_heartbeat(self, heartbeat, *args):
        """
        Called on receipt of a heartbeat.
        """
        pass

class HeartbeatListener(ConnectionListener):
    def __init__(self, heartbeats):
        self.connected = False
        self.running = False
        self.heartbeats = heartbeats
        self.__received_heartbeat = time.time()
        print('>>>>>>>>>>>>> HEARTBEAT LISTENER')

    def on_connected(self, headers, body):
        if 'heart-beat' in headers.keys():
            self.heartbeats = utils.calculate_heartbeats(headers['heart-beat'].replace(' ', '').split(','), self.heartbeats)
            if self.heartbeats != (0,0):
                utils.default_create_thread(self.__heartbeat_loop)

    def on_message(self, headers, body):
        # reset the heartbeat for any received message
        print('>>>>>>>>>>>>>>>>>> RESET HEARTBEATS')
        self.__received_heartbeat = time.time()

    def on_send(self, frame):
        if frame.cmd == 'CONNECT' or frame.cmd == 'STOMP':
            print('>>>>>>>>>>>>>>> ON SEND -- connect')
            if self.heartbeats != (0, 0):
                frame.headers[HDR_HEARTBEAT] = '%s,%s' % self.heartbeats

    def __heartbeat_loop(self):
        """
        Loop for sending (and monitoring received) heartbeats
        """
        send_sleep = self.heartbeats[0] / 1000

        # receive gets an additional threshold of 3 additional seconds
        receive_sleep = (self.heartbeats[1] / 1000) + 3

        if send_sleep == 0:
            sleep_time = receive_sleep
        elif receive_sleep == 0:
            sleep_time = send_sleep
        else:
            # sleep is the GCD of the send and receive times
            sleep_time = gcd(send_sleep, receive_sleep) / 2.0

        send_time = time.time()
        receive_time = time.time()

        while self.running:
            time.sleep(sleep_time)

            if time.time() - send_time > send_sleep:
                send_time = time.time()
                log.debug('Sending a heartbeat message')
                self.send_frame(utils.Frame(None, {}, None))

            if time.time() - receive_time > receive_sleep:
                if time.time() - self.__received_heartbeat > receive_sleep:
                    log.debug('Heartbeat timeout')
                    # heartbeat timeout
                    for listener in self.listeners.values():
                        listener.on_heartbeat_timeout()
                    self.disconnect_socket()
                    self.set_connected(False)

class WaitingListener(ConnectionListener):
    """
    A listener which waits for a specific receipt to arrive
    """
    def __init__(self, receipt):
        self.condition = threading.Condition()
        self.receipt = receipt
        self.received = False

    def on_receipt(self, headers, body):
        if 'receipt-id' in headers and headers['receipt-id'] == self.receipt:
            self.condition.acquire()
            self.received = True
            self.condition.notify()
            self.condition.release()

    def wait_on_receipt(self):
        self.condition.acquire()
        while not self.received:
            self.condition.wait()
        self.condition.release()

class StatsListener(ConnectionListener):
    """
    A connection listener for recording statistics on messages sent and received.
    """
    def __init__(self):
        self.errors = 0
        self.connections = 0
        self.messages_recd = 0
        self.messages_sent = 0

    def on_error(self, headers, message):
        """
        \see ConnectionListener::on_error
        """
        self.errors += 1

    def on_connecting(self, host_and_port):
        """
        \see ConnectionListener::on_connecting
        """
        self.connections += 1

    def on_message(self, headers, body):
        """
        \see ConnectionListener::on_message
        """
        self.messages_recd += 1

    def on_send(self, frame):
        """
        \see ConnectionListener::on_send
        """
        self.messages_sent += 1

    def __str__(self):
        """
        Return a string containing the current statistics (messages sent and received,
        errors, etc)
        """
        return '''Connections: %s
Messages sent: %s
Messages received: %s
Errors: %s''' % (self.connections, self.messages_sent, self.messages_recd, self.errors)
<commit_msg>finish heartbeat listener<commit_after>import threading
import time

import utils
from constants import *

try:
    from fractions import gcd
except ImportError:
    from backward import gcd

import logging
log = logging.getLogger('stomp.py')

class ConnectionListener(object):
    """
    This class should be used as a base class for objects registered
    using Connection.set_listener().
    """
    def on_connecting(self, host_and_port):
        """
        Called by the STOMP connection once a TCP/IP connection to the
        STOMP server has been established or re-established. Note that
        at this point, no connection has been established on the STOMP
        protocol level. For this, you need to invoke the "connect"
        method on the connection.

        \param host_and_port a tuple containing the host name and port
        number to which the connection has been established.
        """
        pass

    def on_connected(self, headers, body):
        """
        Called by the STOMP connection when a CONNECTED frame is
        received, that is after a connection has been established or
        re-established.

        \param headers a dictionary containing all headers sent by the
        server as key/value pairs.

        \param body the frame's payload. This is usually empty for
        CONNECTED frames.
        """
        pass

    def on_disconnected(self):
        """
        Called by the STOMP connection when a TCP/IP connection to the
        STOMP server has been lost.  No messages should be sent via
        the connection until it has been reestablished.
        """
        pass

    def on_heartbeat_timeout(self):
        """
        Called by the STOMP connection when a heartbeat message has not been
        received beyond the specified period.
        """
        pass

    def on_before_message(self, headers, body):
        pass

    def on_message(self, headers, body):
        """
        Called by the STOMP connection when a MESSAGE frame is
        received.

        \param headers a dictionary containing all headers sent by the
        server as key/value pairs.

        \param body the frame's payload - the message body.
        """
        pass

    def on_receipt(self, headers, body):
        """
        Called by the STOMP connection when a RECEIPT frame is
        received, sent by the server if requested by the client using
        the 'receipt' header.

        \param headers a dictionary containing all headers sent by the
        server as key/value pairs.

        \param body the frame's payload. This is usually empty for
        RECEIPT frames.
        """
        pass

    def on_error(self, headers, body):
        """
        Called by the STOMP connection when an ERROR frame is
        received.

        \param headers a dictionary containing all headers sent by the
        server as key/value pairs.

        \param body the frame's payload - usually a detailed error
        description.
        """
        pass

    def on_send(self, frame):
        """
        Called by the STOMP connection when it is in the process of sending a message

        \param frame the frame to be sent
        """
        pass

    def on_heartbeat(self):
        """
        Called on receipt of a heartbeat.
        """
        pass

class HeartbeatListener(ConnectionListener):
    def __init__(self, heartbeats):
        self.connected = False
        self.running = False
        self.heartbeats = heartbeats
        self.__received_heartbeat = time.time()

    def on_connected(self, headers, body):
        if 'heart-beat' in headers.keys():
            self.heartbeats = utils.calculate_heartbeats(headers['heart-beat'].replace(' ', '').split(','), self.heartbeats)
            if self.heartbeats != (0,0):
                utils.default_create_thread(self.__heartbeat_loop)

    def on_message(self, headers, body):
        # reset the heartbeat for any received message
        self.__received_heartbeat = time.time()

    def on_heartbeat(self):
        self.__received_heartbeat = time.time()

    def on_send(self, frame):
        if frame.cmd == 'CONNECT' or frame.cmd == 'STOMP':
            if self.heartbeats != (0, 0):
                frame.headers[HDR_HEARTBEAT] = '%s,%s' % self.heartbeats

    def __heartbeat_loop(self):
        """
        Loop for sending (and monitoring received) heartbeats
        """
        send_sleep = self.heartbeats[0] / 1000

        # receive gets an additional threshold of 3 additional seconds
        receive_sleep = (self.heartbeats[1] / 1000) + 3

        if send_sleep == 0:
            sleep_time = receive_sleep
        elif receive_sleep == 0:
            sleep_time = send_sleep
        else:
            # sleep is the GCD of the send and receive times
            sleep_time = gcd(send_sleep, receive_sleep) / 2.0

        send_time = time.time()
        receive_time = time.time()

        while self.running:
            time.sleep(sleep_time)

            now = time.time()

            if now - send_time > send_sleep:
                send_time = now
                log.debug('Sending a heartbeat message at %s' % now)
                self.send_frame(utils.Frame(None, {}, None))

            diff_receive = now - receive_time
            if diff_receive > receive_sleep:
                diff_heartbeat = now - self.__received_heartbeat
                if diff_heartbeat > receive_sleep:
                    log.debug('Heartbeat timeout: diff_receive=%s, diff_heartbeat=%s, time=%s, lastrec=%s' % (diff_receive, diff_heartbeat, now, self.__received_heartbeat))
                    # heartbeat timeout
                    for listener in self.listeners.values():
                        listener.on_heartbeat_timeout()
                    self.disconnect_socket()
                    self.set_connected(False)

class WaitingListener(ConnectionListener):
    """
    A listener which waits for a specific receipt to arrive
    """
    def __init__(self, receipt):
        self.condition = threading.Condition()
        self.receipt = receipt
        self.received = False

    def on_receipt(self, headers, body):
        if 'receipt-id' in headers and headers['receipt-id'] == self.receipt:
            self.condition.acquire()
            self.received = True
            self.condition.notify()
            self.condition.release()

    def wait_on_receipt(self):
        self.condition.acquire()
        while not self.received:
            self.condition.wait()
        self.condition.release()

class StatsListener(ConnectionListener):
    """
    A connection listener for recording statistics on messages sent and received.
    """
    def __init__(self):
        self.errors = 0
        self.connections = 0
        self.messages_recd = 0
        self.messages_sent = 0

    def on_error(self, headers, message):
        """
        \see ConnectionListener::on_error
        """
        self.errors += 1

    def on_connecting(self, host_and_port):
        """
        \see ConnectionListener::on_connecting
        """
        self.connections += 1

    def on_message(self, headers, body):
        """
        \see ConnectionListener::on_message
        """
        self.messages_recd += 1

    def on_send(self, frame):
        """
        \see ConnectionListener::on_send
        """
        self.messages_sent += 1

    def __str__(self):
        """
        Return a string containing the current statistics (messages sent and received,
        errors, etc)
        """
        return '''Connections: %s
Messages sent: %s
Messages received: %s
Errors: %s''' % (self.connections, self.messages_sent, self.messages_recd, self.errors)
<|endoftext|>


<commit_before>"""
TXFMTrackService
(C) 2015
David Rieger
"""

import redis
import re
from pymongo import MongoClient
from collections import OrderedDict
from time import strptime

class Cache:

    def __init__(self, host="localhost", port="6379", db="0"):
        self.storage = redis.StrictRedis(host=host, port=port, db=db)
        self.TIMES_KEY = "times"

    def add_song(self, song, time):
        """
        Adds [song] to the DB as hash with key [time]
        """

        # First remove the song that has been the "playing_song"
        # from the storage if it was a TXFM Show rather than an actual song
        last_song = self.get_latest_songs(1)
        ignore_pattern = re.compile("((nialler9[\s]?.*)|(txfm[\s]?.*))")

        if ignore_pattern.match(last_song["title"].strip().lower()) and not last_song["artist"].strip():
            self.storage.delete(self.storage.lpop(self.TIMES_KEY))

        self.storage.lpush(self.TIMES_KEY, time)
        self.storage.hmset(time, song)

    def get_times(self, amount=None):
        """
        Returns the latest [amount] stored times
        All if [amount] is ::None::
        Format is String
        """
        if amount is None:
            end_idx = -1
        else:
            end_idx = min(amount-1, self.get_amount_songs()-1)
        return [t.decode("UTF-8") for t in self.storage.lrange(self.TIMES_KEY, start=0, end=end_idx)]

    def get_amount_songs(self):
        """
        Returns the number of songs stored in the cache
        """
        return self.storage.llen(self.TIMES_KEY)

    def get_latest_songs(self, amount=50):
        """
        Returns the latest [amount] songs
        All if [amount] is ::None::
        if [amount] is ::1:: only the value of the hash
        for the latest song is returned i.e. no time
        """
        if amount is 1:
            return self.get_song_for_key(self.get_times(amount=1)[0])

        _times = self.get_times(amount)
        _all_songs = OrderedDict()
        for t in self.get_times(amount):
            _all_songs[t] = self.get_song_for_key(t)

        return _all_songs

    def get_song_for_key(self, time):
        """
        Returns a song for the given [time]
        whereas time has to be an actual existing hash key
        """
        return {k.decode("UTF-8"): v.decode("UTF-8") for (k, v) in self.storage.hgetall(time).items()}

    def get_songs_by_time(self, req_time, scope=30):
        """
        Returns [scope] songs stored around the given [req_time]
        [req_time] must be in the correct format _times %d.%m.%Y %H:%M:%S
        but does not have to be an existing key
        """
        for t in self.get_times():
            # Go through the list from HEAD to TAIL and get the [scope] times arounc the
            # time that closestly matches [req_time] but is definitly smaller than [req_time]
            if strptime(t, "%d.%m.%Y %H:%M:%S") > strptime(str(req_time), "%d.%m.%Y %H:%M:%S"):
                continue

            # index of this time in the list
            _idx = self.get_times().index(t)

            _start_idx = max(_idx-int((scope/2)), 0)
            _end_idx = min(_idx+int((scope/2)), self.get_amount_songs()-1)

            # Get the songs for all scope times and add to dict
            _songs = OrderedDict()
            for _timekey in self.get_times()[_start_idx:_end_idx]:
                _songs[_timekey] = self.get_song_for_key(_timekey)

            return _songs

    def get_songs_by_text(self, text, scope=None):
        """
        returns all songs that match [text] within the latest [scope] songs
        """
        _result = OrderedDict()
        for k, v in self.get_latest_songs(amount=self.get_amount_songs()).items():
            if (v["title"] + v["artist"]).replace(" ", "").lower().find(text.replace(" ", "").lower()) != -1:
                _result[k] = v
        return _result

    def remove_song(self, time):
        """
        removes [time] from the list of times and the accoring song from the cache
        """
        self.storage.lrem(self.TIMES_KEY, count=1, value=time)
        self.storage.delete(time)

class Persistence:

    def __init__(self, host="localhost", port=6380, dbname="txfmstore"):
        self.db = MongoClient(host, port)[dbname]
        self.col_songs = self.db.songs

    def add_song(self, song, time):
        song["_id"] = time
        self.col_songs.insert_one(song)

    def is_stored(self, time):
        _res = self.col_songs.find_one({"_id": time})
        if _res is None:
            return False
        return True

    def get_songs_by_text(self, text, scope=None):

        _songs = OrderedDict()
        for t in self.get_all_times():
            s = self.col_songs.find_one({"_id": t}, {"_id": False})

            if (s["title"] + s["artist"]).replace(" ", "").lower().find(text.replace(" ", "").lower()) != -1:
                _songs[t] = s

        return _songs

    def get_songs_by_time(self, req_time, scope=30):

        for t in self.get_all_times():
            if t > strptime(str(req_time), "%d.%m.%Y %H:%M:%S"):
                continue

            # index of this time in the list
            _idx = self.get_all_times().index(t)

            _start_idx = max(_idx-int((scope/2)), 0)
            _end_idx = min(_idx+int((scope/2)), len(self.get_all_times())-1)

            _songs = OrderedDict()
            for _timekey in self.get_all_times()[_start_idx:_end_idx]:
                _songs[_timekey] = self.col_songs.find_one({"_id": t}, {"_id": False})

            return _songs

    def get_all_times(self):
        """
        Returns all times for which a song is stored
        Format is a time-object
        """
        return sorted([strptime(t["_id"], "%d.%m.%Y %H:%M:%S")
                      for t in self.col_songs.find({}, {"_id": True})], reverse=True)
<commit_msg>bugfix for persistence search<commit_after>"""
TXFMTrackService
(C) 2015
David Rieger
"""

import redis
import re
from pymongo import MongoClient
from collections import OrderedDict
from time import strptime, strftime

class Cache:

    def __init__(self, host="localhost", port="6379", db="0"):
        self.storage = redis.StrictRedis(host=host, port=port, db=db)
        self.TIMES_KEY = "times"

    def add_song(self, song, time):
        """
        Adds [song] to the DB as hash with key [time]
        """

        # First remove the song that has been the "playing_song"
        # from the storage if it was a TXFM Show rather than an actual song
        last_song = self.get_latest_songs(1)
        ignore_pattern = re.compile("((nialler9[\s]?.*)|(txfm[\s]?.*))")

        if ignore_pattern.match(last_song["title"].strip().lower()) and not last_song["artist"].strip():
            self.storage.delete(self.storage.lpop(self.TIMES_KEY))

        self.storage.lpush(self.TIMES_KEY, time)
        self.storage.hmset(time, song)

    def get_times(self, amount=None):
        """
        Returns the latest [amount] stored times
        All if [amount] is ::None::
        Format is String
        """
        if amount is None:
            end_idx = -1
        else:
            end_idx = min(amount-1, self.get_amount_songs()-1)
        return [t.decode("UTF-8") for t in self.storage.lrange(self.TIMES_KEY, start=0, end=end_idx)]

    def get_amount_songs(self):
        """
        Returns the number of songs stored in the cache
        """
        return self.storage.llen(self.TIMES_KEY)

    def get_latest_songs(self, amount=50):
        """
        Returns the latest [amount] songs
        All if [amount] is ::None::
        if [amount] is ::1:: only the value of the hash
        for the latest song is returned i.e. no time
        """
        if amount is 1:
            return self.get_song_for_key(self.get_times(amount=1)[0])

        _times = self.get_times(amount)
        _all_songs = OrderedDict()
        for t in self.get_times(amount):
            _all_songs[t] = self.get_song_for_key(t)

        return _all_songs

    def get_song_for_key(self, time):
        """
        Returns a song for the given [time]
        whereas time has to be an actual existing hash key
        """
        return {k.decode("UTF-8"): v.decode("UTF-8") for (k, v) in self.storage.hgetall(time).items()}

    def get_songs_by_time(self, req_time, scope=30):
        """
        Returns [scope] songs stored around the given [req_time]
        [req_time] must be in the correct format _times %d.%m.%Y %H:%M:%S
        but does not have to be an existing key
        """
        for t in self.get_times():
            # Go through the list from HEAD to TAIL and get the [scope] times arounc the
            # time that closestly matches [req_time] but is definitly smaller than [req_time]
            if strptime(t, "%d.%m.%Y %H:%M:%S") > strptime(str(req_time), "%d.%m.%Y %H:%M:%S"):
                continue

            # index of this time in the list
            _idx = self.get_times().index(t)

            _start_idx = max(_idx-int((scope/2)), 0)
            _end_idx = min(_idx+int((scope/2)), self.get_amount_songs()-1)

            # Get the songs for all scope times and add to dict
            _songs = OrderedDict()
            for _timekey in self.get_times()[_start_idx:_end_idx]:
                _songs[_timekey] = self.get_song_for_key(_timekey)

            return _songs

    def get_songs_by_text(self, text, scope=None):
        """
        returns all songs that match [text] within the latest [scope] songs
        """
        _result = OrderedDict()
        for k, v in self.get_latest_songs(amount=self.get_amount_songs()).items():
            if (v["title"] + v["artist"]).replace(" ", "").lower().find(text.replace(" ", "").lower()) != -1:
                _result[k] = v
        return _result

    def remove_song(self, time):
        """
        removes [time] from the list of times and the accoring song from the cache
        """
        self.storage.lrem(self.TIMES_KEY, count=1, value=time)
        self.storage.delete(time)

class Persistence:

    def __init__(self, host="localhost", port=6380, dbname="txfmstore"):
        self.db = MongoClient(host, port)[dbname]
        self.col_songs = self.db.songs

    def add_song(self, song, time):
        song["_id"] = time
        self.col_songs.insert_one(song)

    def is_stored(self, time):
        _res = self.col_songs.find_one({"_id": time})
        if _res is None:
            return False
        return True

    def get_songs_by_text(self, text, scope=None):

        _songs = OrderedDict()
        for t in self.get_all_times():
            s = self.col_songs.find_one({"_id": t}, {"_id": False})

            if (s["title"] + s["artist"]).replace(" ", "").lower().find(text.replace(" ", "").lower()) != -1:
                _songs[t] = s

        return _songs

    def get_songs_by_time(self, req_time, scope=30):

        for t in self.get_all_times():
            if strptime(t, "%d.%m.%Y %H:%M:%S") > strptime(str(req_time), "%d.%m.%Y %H:%M:%S"):
                continue

            # index of this time in the list
            _idx = self.get_all_times().index(t)

            _start_idx = max(_idx-int((scope/2)), 0)
            _end_idx = min(_idx+int((scope/2)), len(self.get_all_times())-1)

            _songs = OrderedDict()
            for _timekey in self.get_all_times()[_start_idx:_end_idx]:
                _songs[_timekey] = self.col_songs.find_one({"_id": _timekey}, {"_id": False})

            return _songs

    def get_all_times(self):
        """
        Returns all times for which a song is stored
        Format is a time-object
        """
        _times = sorted([strptime(t["_id"], "%d.%m.%Y %H:%M:%S")
                        for t in self.col_songs.find({}, {"_id": True})], reverse=True)

        return [strftime("%d.%m.%Y %H:%M:%S", t) for t in _times]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import time
import pickle
import logging
import datetime

from pprint import pprint

from django.shortcuts import render_to_response
from django.http import HttpResponseRedirect
from django.template import RequestContext

from django.http import HttpResponseForbidden
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import redirect, render
from django.utils import simplejson

logger = logging.getLogger(__name__)

import boto.ec2
import boto.ec2.cloudwatch

from django.contrib.auth.models import User
from userprofile.models import Profile as userprofile

from amazon import s3_funcs
from amazon import s3_funcs_shortcuts

from django import forms
from forms import UploadFileForm
from cloud_storage.models import Files
from cloud_storage.models import Uploaded_Files

from django.template.defaultfilters import filesizeformat, upper
from django.contrib.humanize.templatetags.humanize import naturalday
from cloudly.templatetags.cloud_extras import clear_filename, get_file_extension

def delete_file(request, file_id):

	print '-- delete_file:'

	if not request.user.is_authenticated():
		return HttpResponseRedirect("/")

	user = request.user
	profile = userprofile.objects.get(user=request.user)
	secret = profile.secret

	print 'user', request.user

	try:
		f = Uploaded_Files.objects.get(pk=file_id)
		if(f.user!=request.user):
			return HttpResponse("access denied")
	except:
		return HttpResponse("access denied")

	if(request.GET):
		if(request.GET['confirm']=="True"):
			f.delete()
			return HttpResponseRedirect("/cloud/storage/")

	return render_to_response('cloud_file_delete.html', {'f':f,'profile':profile,}, context_instance=RequestContext(request))

def cloud_dropzone(request):

	print '-- cloud_storage:'

	if not request.user.is_authenticated():
		return HttpResponseRedirect("/")

	user = request.user
	profile = userprofile.objects.get(user=request.user)
	secret = profile.secret

	print 'user', request.user

	uploaded_files = []

	if request.method == 'POST':
		form = UploadFileForm(request.POST, request.FILES)
		if form.is_valid():
			new_file = Files(file=request.FILES['file'])
			new_file.save()
			uploaded_file = Uploaded_Files.objects.create(file=new_file,user=request.user)

	cloud_storage_menu_open = True

	return render_to_response('cloud_dropzone.html', {'cloud_storage_menu_open':cloud_storage_menu_open,'uploaded_files':uploaded_files,'user':user,'profile':profile,}, context_instance=RequestContext(request))

def cloud_sharing(request):

	print '-- cloud sharing:'

	if not request.user.is_authenticated():
		return HttpResponseRedirect("/")

	user = request.user
	profile = userprofile.objects.get(user=request.user)
	secret = profile.secret

	print 'user', request.user

	return render_to_response('cloud_sharing.html', {'profile':profile,}, context_instance=RequestContext(request))

def dropzone_uploader(request):

	print '-- dropzone_uploader:'

	if not request.user.is_authenticated():
		return HttpResponseRedirect("/")

	print request.user

	if request.method == 'POST':

		#form = UploadFileForm(request.POST, request.FILES)
		#if form.is_valid():
		#print 'valid form', form

		new_file = Files(file=request.FILES['files[]'])
		new_file.save()
		f = Uploaded_Files.objects.create(file=new_file,user=request.user)

		simple_json = {
			"thumbnailUrl": "/media/"+str(new_file.file), # XXX add thumbnail here
			"name": new_file.file,
			"size": os.path.getsize("media/"+str(new_file.file)),
			"type": str(new_file.file).split('.')[:-1][0]
		}

		response_type = "application/json"
		if "text/html" in request.META["HTTP_ACCEPT"]:
		   response_type = "text/html"

		return HttpResponse(simple_json, mimetype=response_type)

	return HttpResponse("invalid form")

def cloud_storage(request):

	print '-- cloud_storage:'

	if not request.user.is_authenticated():
		return HttpResponseRedirect("/")

	user = request.user
	profile = userprofile.objects.get(user=request.user)
	secret = profile.secret

	print request.user

	if request.method == 'POST':
		form = UploadFileForm(request.POST, request.FILES)
		if form.is_valid():
			new_file = Files(file=request.FILES['file'])
			new_file.save()
			Uploaded_Files.objects.create(file=new_file,user=request.user)

	uploaded_files = Uploaded_Files.objects.filter(user=request.user).order_by('-pk')
	cloud_storage_menu_open = True

	# XXX batch sync files to the S3.....

	return render_to_response('cloud_storage.html', {'cloud_storage_menu_open':cloud_storage_menu_open,'uploaded_files':uploaded_files,'user':user,'profile':profile,}, context_instance=RequestContext(request))

def ajax_cloud_storage(request):

	print '-- ajax_cloud_storage:'

	if not request.user.is_authenticated():
		return HttpResponse("access denied")

	user = request.user
	profile = userprofile.objects.get(user=request.user)
	secret = profile.secret

	print request.user

	uploaded_files = Uploaded_Files.objects.filter(user=request.user).order_by('-pk')

	return render_to_response('cloud_storage-list.html', {'uploaded_files':uploaded_files,'profile':profile,}, context_instance=RequestContext(request))

<commit_msg>one more try to serialise json<commit_after># -*- coding: utf-8 -*-

import os
import time
import pickle
import logging
import datetime

from pprint import pprint

from django.shortcuts import render_to_response
from django.http import HttpResponseRedirect
from django.template import RequestContext

from django.http import HttpResponseForbidden
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import redirect, render
from django.utils import simplejson

logger = logging.getLogger(__name__)

import boto.ec2
import boto.ec2.cloudwatch

from django.contrib.auth.models import User
from userprofile.models import Profile as userprofile

from amazon import s3_funcs
from amazon import s3_funcs_shortcuts

from django import forms
from forms import UploadFileForm
from cloud_storage.models import Files
from cloud_storage.models import Uploaded_Files

from django.template.defaultfilters import filesizeformat, upper
from django.contrib.humanize.templatetags.humanize import naturalday
from cloudly.templatetags.cloud_extras import clear_filename, get_file_extension

def delete_file(request, file_id):

	print '-- delete_file:'

	if not request.user.is_authenticated():
		return HttpResponseRedirect("/")

	user = request.user
	profile = userprofile.objects.get(user=request.user)
	secret = profile.secret

	print 'user', request.user

	try:
		f = Uploaded_Files.objects.get(pk=file_id)
		if(f.user!=request.user):
			return HttpResponse("access denied")
	except:
		return HttpResponse("access denied")

	if(request.GET):
		if(request.GET['confirm']=="True"):
			f.delete()
			return HttpResponseRedirect("/cloud/storage/")

	return render_to_response('cloud_file_delete.html', {'f':f,'profile':profile,}, context_instance=RequestContext(request))

def cloud_dropzone(request):

	print '-- cloud_storage:'

	if not request.user.is_authenticated():
		return HttpResponseRedirect("/")

	user = request.user
	profile = userprofile.objects.get(user=request.user)
	secret = profile.secret

	print 'user', request.user

	uploaded_files = []

	if request.method == 'POST':
		form = UploadFileForm(request.POST, request.FILES)
		if form.is_valid():
			new_file = Files(file=request.FILES['file'])
			new_file.save()
			uploaded_file = Uploaded_Files.objects.create(file=new_file,user=request.user)

	cloud_storage_menu_open = True

	return render_to_response('cloud_dropzone.html', {'cloud_storage_menu_open':cloud_storage_menu_open,'uploaded_files':uploaded_files,'user':user,'profile':profile,}, context_instance=RequestContext(request))

def cloud_sharing(request):

	print '-- cloud sharing:'

	if not request.user.is_authenticated():
		return HttpResponseRedirect("/")

	user = request.user
	profile = userprofile.objects.get(user=request.user)
	secret = profile.secret

	print 'user', request.user

	return render_to_response('cloud_sharing.html', {'profile':profile,}, context_instance=RequestContext(request))

def dropzone_uploader(request):

	print '-- dropzone_uploader:'

	if not request.user.is_authenticated():
		return HttpResponseRedirect("/")

	print request.user

	if request.method == 'POST':

		#form = UploadFileForm(request.POST, request.FILES)
		#if form.is_valid():
		#print 'valid form', form

		new_file = Files(file=request.FILES['files[]'])
		new_file.save()
		f = Uploaded_Files.objects.create(file=new_file,user=request.user)

		simple_json = {
			"thumbnailUrl": "/media/"+str(new_file.file), # XXX add thumbnail here
			"name": new_file.file,
			"size": os.path.getsize("media/"+str(new_file.file)),
			"type": str(new_file.file).split('.')[:-1][0]
		}

		response_type = "application/json"
		if "text/html" in request.META["HTTP_ACCEPT"]:
		   response_type = "text/html"

		return HttpResponse(str(simple_json), mimetype=response_type)

	return HttpResponse("invalid form")

def cloud_storage(request):

	print '-- cloud_storage:'

	if not request.user.is_authenticated():
		return HttpResponseRedirect("/")

	user = request.user
	profile = userprofile.objects.get(user=request.user)
	secret = profile.secret

	print request.user

	if request.method == 'POST':
		form = UploadFileForm(request.POST, request.FILES)
		if form.is_valid():
			new_file = Files(file=request.FILES['file'])
			new_file.save()
			Uploaded_Files.objects.create(file=new_file,user=request.user)

	uploaded_files = Uploaded_Files.objects.filter(user=request.user).order_by('-pk')
	cloud_storage_menu_open = True

	# XXX batch sync files to the S3.....

	return render_to_response('cloud_storage.html', {'cloud_storage_menu_open':cloud_storage_menu_open,'uploaded_files':uploaded_files,'user':user,'profile':profile,}, context_instance=RequestContext(request))

def ajax_cloud_storage(request):

	print '-- ajax_cloud_storage:'

	if not request.user.is_authenticated():
		return HttpResponse("access denied")

	user = request.user
	profile = userprofile.objects.get(user=request.user)
	secret = profile.secret

	print request.user

	uploaded_files = Uploaded_Files.objects.filter(user=request.user).order_by('-pk')

	return render_to_response('cloud_storage-list.html', {'uploaded_files':uploaded_files,'profile':profile,}, context_instance=RequestContext(request))

<|endoftext|>


<commit_before>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
############

import os
import copy
import json
import uuid
import click
import logging
import logging.config

import colorama

from cloudify import logs

from . import env
from .config.config import is_use_colors
from .config.config import CloudifyConfig
from .colorful_event import ColorfulEvent

DEFAULT_LOG_FILE = os.path.join(env.CLOUDIFY_WORKDIR, 'logs', 'cli.log')

HIGH_VERBOSE = 3
MEDIUM_VERBOSE = 2
LOW_VERBOSE = 1
NO_VERBOSE = 0
QUIET = -1

verbosity_level = NO_VERBOSE
json_output = False

_lgr = None

LOGGER = {
    "version": 1,
    "formatters": {
        "file": {
            "format": "%(asctime)s [%(levelname)s] %(message)s"
        },
        "console": {
            "format": "%(message)s"
        }
    },
    "handlers": {
        "file": {
            "class": "logging.handlers.RotatingFileHandler",
            "formatter": "file",
            "maxBytes": 5000000,
            "backupCount": "20"
        },
        "console": {
            "class": "logging.StreamHandler",
            "stream": "ext://sys.stdout",
            "formatter": "console"
        }
    },
    "loggers": {
        "cloudify.cli.main": {
            "handlers": ["console", "file"],
            "level": "INFO"
        }
    }
}
# logger that goes only to the file, for use when logging table data
logfile_logger = logging.getLogger('logfile')

def get_logger():
    if not _lgr:
        configure_loggers()
    return _lgr

def configure_loggers():
    # first off, configure defaults
    # to enable the use of the logger
    # even before the init was executed.
    logger_config = copy.deepcopy(LOGGER)
    _configure_defaults(logger_config)

    if env.is_initialized():
        # init was already called
        # use the configuration file.
        _configure_from_file(logger_config)
    _set_loggers_verbosity(logger_config)
    logging.config.dictConfig(logger_config)

    global _lgr
    _lgr = logging.getLogger('cloudify.cli.main')

    # configuring events/logs loggers
    # (this will also affect local workflow loggers, which don't use
    # the get_events_logger method of this module)
    if is_use_colors():
        logs.EVENT_CLASS = ColorfulEvent
        # refactor this elsewhere if colorama is further used in CLI
        colorama.init(autoreset=True)

def _set_loggers_verbosity(logger_config):
    if get_global_json_output():
        logger_config['loggers']['cloudify.cli.main']['level'] = 'ERROR'
    for logger in logger_config['loggers'].values():
        if verbosity_level >= HIGH_VERBOSE:
            logger['level'] = logging.DEBUG
        elif verbosity_level == LOW_VERBOSE:
            logger['level'] = logging.INFO
        elif verbosity_level <= QUIET:
            logger['level'] = logging.ERROR

def _configure_defaults(logger_config):
    if get_global_json_output():
        logger_config['loggers']['logfile'] = {
            "level": "DEBUG",
            "propagate": False,
            "handlers": ["file"]
        }
        logger_config['handlers']['console']['stream'] = 'ext://sys.stderr'

    logger_config['handlers']['file']['filename'] = DEFAULT_LOG_FILE
    logfile_dir = os.path.dirname(DEFAULT_LOG_FILE)
    if not os.path.exists(logfile_dir):
        os.makedirs(logfile_dir)

def _configure_from_file(loggers_config):
    config = CloudifyConfig()

    # set filename on file handler
    logger_dict = copy.deepcopy(LOGGER)
    loggers_config['handlers']['file']['filename'] = config.logging.filename
    logfile_dir = os.path.dirname(config.logging.filename)
    if not os.path.exists(logfile_dir):
        os.makedirs(logfile_dir)

    # add handlers to every logger specified in the file
    for logger_name, logging_level in config.logging.loggers.items():
        loggers_config['loggers'][logger_name] = {
            'handlers': list(logger_dict['handlers'].keys()),
            'level': logging_level.upper()
        }

def get_events_logger(json_output):
    json_output = json_output or get_global_json_output()

    def json_events_logger(events):
        """The json events logger prints events as consumable JSON formatted
        entries. Each event appears in its own line.

        :param events: The events to print.
        :return:
        """
        for event in events:
            click.echo(json.dumps(event))

    def text_events_logger(events):
        """The default events logger prints events as short messages.

        :param events: The events to print.
        :return:
        """
        for event in events:
            output = logs.create_event_message_prefix(event)
            if output:
                click.echo(output)

    return json_events_logger if json_output else text_events_logger

def set_global_verbosity_level(verbose):
    """Set the global verbosity level.
    """
    global verbosity_level
    verbosity_level = verbose
    logs.EVENT_VERBOSITY_LEVEL = verbosity_level

def get_global_verbosity():
    """Return the globally set verbosity
    """
    return verbosity_level

def set_global_json_output(enabled=False):
    global json_output
    json_output = enabled

def get_global_json_output():
    return json_output

def output(line):
    logfile_logger.info(line)
    click.echo(line)

class CloudifyJSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, uuid.UUID):
            return obj.hex
        return super(CloudifyJSONEncoder, self).default(obj)
<commit_msg>logger: change backupCount to an int (#1073)<commit_after>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
############

import os
import copy
import json
import uuid
import click
import logging
import logging.config

import colorama

from cloudify import logs

from . import env
from .config.config import is_use_colors
from .config.config import CloudifyConfig
from .colorful_event import ColorfulEvent

DEFAULT_LOG_FILE = os.path.join(env.CLOUDIFY_WORKDIR, 'logs', 'cli.log')

HIGH_VERBOSE = 3
MEDIUM_VERBOSE = 2
LOW_VERBOSE = 1
NO_VERBOSE = 0
QUIET = -1

verbosity_level = NO_VERBOSE
json_output = False

_lgr = None

LOGGER = {
    "version": 1,
    "formatters": {
        "file": {
            "format": "%(asctime)s [%(levelname)s] %(message)s"
        },
        "console": {
            "format": "%(message)s"
        }
    },
    "handlers": {
        "file": {
            "class": "logging.handlers.RotatingFileHandler",
            "formatter": "file",
            "maxBytes": 5000000,
            "backupCount": 20
        },
        "console": {
            "class": "logging.StreamHandler",
            "stream": "ext://sys.stdout",
            "formatter": "console"
        }
    },
    "loggers": {
        "cloudify.cli.main": {
            "handlers": ["console", "file"],
            "level": "INFO"
        }
    }
}
# logger that goes only to the file, for use when logging table data
logfile_logger = logging.getLogger('logfile')

def get_logger():
    if not _lgr:
        configure_loggers()
    return _lgr

def configure_loggers():
    # first off, configure defaults
    # to enable the use of the logger
    # even before the init was executed.
    logger_config = copy.deepcopy(LOGGER)
    _configure_defaults(logger_config)

    if env.is_initialized():
        # init was already called
        # use the configuration file.
        _configure_from_file(logger_config)
    _set_loggers_verbosity(logger_config)
    logging.config.dictConfig(logger_config)

    global _lgr
    _lgr = logging.getLogger('cloudify.cli.main')

    # configuring events/logs loggers
    # (this will also affect local workflow loggers, which don't use
    # the get_events_logger method of this module)
    if is_use_colors():
        logs.EVENT_CLASS = ColorfulEvent
        # refactor this elsewhere if colorama is further used in CLI
        colorama.init(autoreset=True)

def _set_loggers_verbosity(logger_config):
    if get_global_json_output():
        logger_config['loggers']['cloudify.cli.main']['level'] = 'ERROR'
    for logger in logger_config['loggers'].values():
        if verbosity_level >= HIGH_VERBOSE:
            logger['level'] = logging.DEBUG
        elif verbosity_level == LOW_VERBOSE:
            logger['level'] = logging.INFO
        elif verbosity_level <= QUIET:
            logger['level'] = logging.ERROR

def _configure_defaults(logger_config):
    if get_global_json_output():
        logger_config['loggers']['logfile'] = {
            "level": "DEBUG",
            "propagate": False,
            "handlers": ["file"]
        }
        logger_config['handlers']['console']['stream'] = 'ext://sys.stderr'

    logger_config['handlers']['file']['filename'] = DEFAULT_LOG_FILE
    logfile_dir = os.path.dirname(DEFAULT_LOG_FILE)
    if not os.path.exists(logfile_dir):
        os.makedirs(logfile_dir)

def _configure_from_file(loggers_config):
    config = CloudifyConfig()

    # set filename on file handler
    logger_dict = copy.deepcopy(LOGGER)
    loggers_config['handlers']['file']['filename'] = config.logging.filename
    logfile_dir = os.path.dirname(config.logging.filename)
    if not os.path.exists(logfile_dir):
        os.makedirs(logfile_dir)

    # add handlers to every logger specified in the file
    for logger_name, logging_level in config.logging.loggers.items():
        loggers_config['loggers'][logger_name] = {
            'handlers': list(logger_dict['handlers'].keys()),
            'level': logging_level.upper()
        }

def get_events_logger(json_output):
    json_output = json_output or get_global_json_output()

    def json_events_logger(events):
        """The json events logger prints events as consumable JSON formatted
        entries. Each event appears in its own line.

        :param events: The events to print.
        :return:
        """
        for event in events:
            click.echo(json.dumps(event))

    def text_events_logger(events):
        """The default events logger prints events as short messages.

        :param events: The events to print.
        :return:
        """
        for event in events:
            output = logs.create_event_message_prefix(event)
            if output:
                click.echo(output)

    return json_events_logger if json_output else text_events_logger

def set_global_verbosity_level(verbose):
    """Set the global verbosity level.
    """
    global verbosity_level
    verbosity_level = verbose
    logs.EVENT_VERBOSITY_LEVEL = verbosity_level

def get_global_verbosity():
    """Return the globally set verbosity
    """
    return verbosity_level

def set_global_json_output(enabled=False):
    global json_output
    json_output = enabled

def get_global_json_output():
    return json_output

def output(line):
    logfile_logger.info(line)
    click.echo(line)

class CloudifyJSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, uuid.UUID):
            return obj.hex
        return super(CloudifyJSONEncoder, self).default(obj)
<|endoftext|>


<commit_before>import ldap

class ldap_user:
    """ Provides a class for getting the user/project
    data that will come from LDAP. """

    # put these in a config file?
    LDAP_HOST = 'im3r.idp.iu.futuregrid.org'
    LDAP_PERSONBASE = 'ou=People,dc=futuregrid,dc=org'
    LDAP_PROJECTBASE = 'ou=Groups,dc=futuregrid,dc=org'

    def __init__(self, username):
        self._username = username
        self._data = None

    def _load_data(self):
        # specify ca cert (make sure TLS_CACERTDIR is defined in
        # /etc/openldap/ldap.conf, or optionally set LDAPTLS_CACERTDIR
        # in environment)
        ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_DEMAND)

        # connect
        ldapconn = ldap.initialize('ldap://%s' % self.LDAP_HOST)

        # start tls connection and bind
        ldapconn.start_tls_s()
        ldapconn.simple_bind_s()

        # Get profile and project data from LDAP
        self._data = {}

        search_result_person = ldapconn.search_s(self.LDAP_PERSONBASE, ldap.SCOPE_SUBTREE, '(cn=%s)' % self.username)
        self._data['person'] = search_result_person[0][1] if search_result_person is not None else None

        self._data['projects'] = ldapconn.search_s(self.LDAP_PROJECTBASE, ldap.SCOPE_SUBTREE, '(&(memberUid=%s)(cn=fg*))' % self.username, ['cn'])

    @property
    def username(self):
        return self._username

    @property
    def data(self):
        if self._data is None:
            self._load_data()
        return self._data

    @property
    def uid(self):
        return self.data['person']['uidNumber'][0] if 'uidNumber' in self.data['person'] else None

    @property
    def gid(self):
        return self.data['person']['gidNumber'][0] if 'gidNumber' in self.data['person'] else None

    @property
    def firstname(self):
        return self.data['person']['givenName'][0] if 'givenName' in self.data['person'] else None

    @property
    def lastname(self):
        return self.data['person']['sn'][0] if 'sn' in self.data['person'] else None

    @property
    def phone(self):
        return self.data['person']['telephoneNumber'][0] if 'telephoneNumber' in self.data['person'] else None

    @property
    def email(self):
        return self.data['person']['mail'][0] if 'mail' in self.data['person'] else None

    @property
    def address(self):
        return self.data['person']['homePostalAddress'] if 'homePostalAddress' in self.data['person'] else None

    @property
    def keys(self):
        keys = {}
        if 'sshPublicKey' in self.data['person']:
            for sshkey in self.data['person']['sshPublicKey']:
                if sshkey.strip():
                    (keytype, key, nickname) = (sshkey.strip().split(None, 2) + [''])[0:3]
                    keys[nickname] = "key %s" % sshkey
            return keys
        else:
            return None

    @property
    def activeclouds(self):
        """Not yet implemented"""
        return ['sierra-openstack-grizzly']

    @property
    def defaultcloud(self):
        """Not yet implemented"""
        return 'sierra-openstack-grizzly'

    @property
    def activeprojects(self):
        """List of active projects"""
        if self.data['projects']:
            projects = []
            for project in self.data['projects']:
                (projectDN, projectAttrs) = project
                projectCN = projectAttrs['cn'][0]
                projects.append("%s" % projectCN)
        return projects

    @property
    def completedprojects(self):
        """Not yet implemented"""
        return []

    @property
    def defaultproject(self):
        """The user's default project (currently just the first active project)"""
        return self.activeprojects[0]

if __name__ == "__main__":
    luser = ldap_user('astreib')
    print "Username: %s" % luser.username
    print "Uid: %s" % luser.uid
    print "Gid: %s" % luser.gid
    print "First Name: %s" % luser.firstname
    print "Last Name: %s" % luser.lastname
    print "Keys: %s" % luser.keys
    print "Phone: %s" % luser.phone
    print "Email: %s" % luser.email
    print "Address: %s" % luser.address
    print "Projects - Active: %s" % luser.activeprojects
    print "Projects - Completed: %s" % luser.completedprojects
    print "Projects - Default: %s" % luser.defaultproject
    print "Active Clouds: %s" % luser.activeclouds
    print "Default Cloud: %s" % luser.defaultcloud
<commit_msg>'TBD' for unknown phone/address; address is a list<commit_after>import ldap

class ldap_user:
    """ Provides a class for getting the user/project
    data that will come from LDAP. """

    # put these in a config file?
    LDAP_HOST = 'im3r.idp.iu.futuregrid.org'
    LDAP_PERSONBASE = 'ou=People,dc=futuregrid,dc=org'
    LDAP_PROJECTBASE = 'ou=Groups,dc=futuregrid,dc=org'

    def __init__(self, username):
        self._username = username
        self._data = None

    def _load_data(self):
        # specify ca cert (make sure TLS_CACERTDIR is defined in
        # /etc/openldap/ldap.conf, or optionally set LDAPTLS_CACERTDIR
        # in environment)
        ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_DEMAND)

        # connect
        ldapconn = ldap.initialize('ldap://%s' % self.LDAP_HOST)

        # start tls connection and bind
        ldapconn.start_tls_s()
        ldapconn.simple_bind_s()

        # Get profile and project data from LDAP
        self._data = {}

        search_result_person = ldapconn.search_s(self.LDAP_PERSONBASE, ldap.SCOPE_SUBTREE, '(cn=%s)' % self.username)
        self._data['person'] = search_result_person[0][1] if search_result_person is not None else None

        self._data['projects'] = ldapconn.search_s(self.LDAP_PROJECTBASE, ldap.SCOPE_SUBTREE, '(&(memberUid=%s)(cn=fg*))' % self.username, ['cn'])

    @property
    def username(self):
        return self._username

    @property
    def data(self):
        if self._data is None:
            self._load_data()
        return self._data

    @property
    def uid(self):
        return self.data['person']['uidNumber'][0] if 'uidNumber' in self.data['person'] else None

    @property
    def gid(self):
        return self.data['person']['gidNumber'][0] if 'gidNumber' in self.data['person'] else None

    @property
    def firstname(self):
        return self.data['person']['givenName'][0] if 'givenName' in self.data['person'] else None

    @property
    def lastname(self):
        return self.data['person']['sn'][0] if 'sn' in self.data['person'] else None

    @property
    def phone(self):
        p = self.data['person']['telephoneNumber'][0] if 'telephoneNumber' in self.data['person'] else None
        return p or 'TBD'

    @property
    def email(self):
        return self.data['person']['mail'][0] if 'mail' in self.data['person'] else None

    @property
    def address(self):
        ## This is currently not in LDAP; also needs to split address
        ## lines into a list, not sure how ldap will represent this so
        ## it is not yet done.
        # return self.data['person']['homePostalAddress'] if 'homePostalAddress' in self.data['person'] else None
        return ['TBD']

    @property
    def keys(self):
        keys = {}
        if 'sshPublicKey' in self.data['person']:
            for sshkey in self.data['person']['sshPublicKey']:
                if sshkey.strip():
                    (keytype, key, nickname) = (sshkey.strip().split(None, 2) + [''])[0:3]
                    keys[nickname] = "key %s" % sshkey
            return keys
        else:
            return None

    @property
    def activeclouds(self):
        """Not yet implemented"""
        return ['sierra-openstack-grizzly']

    @property
    def defaultcloud(self):
        """Not yet implemented"""
        return 'sierra-openstack-grizzly'

    @property
    def activeprojects(self):
        """List of active projects"""
        if self.data['projects']:
            projects = []
            for project in self.data['projects']:
                (projectDN, projectAttrs) = project
                projectCN = projectAttrs['cn'][0]
                projects.append("%s" % projectCN)
        return projects

    @property
    def completedprojects(self):
        """Not yet implemented"""
        return []

    @property
    def defaultproject(self):
        """The user's default project (currently just the first active project)"""
        return self.activeprojects[0]

if __name__ == "__main__":
    luser = ldap_user('astreib')
    print "Username: %s" % luser.username
    print "Uid: %s" % luser.uid
    print "Gid: %s" % luser.gid
    print "First Name: %s" % luser.firstname
    print "Last Name: %s" % luser.lastname
    print "Keys: %s" % luser.keys
    print "Phone: %s" % luser.phone
    print "Email: %s" % luser.email
    print "Address: %s" % luser.address
    print "Projects - Active: %s" % luser.activeprojects
    print "Projects - Completed: %s" % luser.completedprojects
    print "Projects - Default: %s" % luser.defaultproject
    print "Active Clouds: %s" % luser.activeclouds
    print "Default Cloud: %s" % luser.defaultcloud
<|endoftext|>


<commit_before>from collections import Counter
from collections import defaultdict
import bisect
import calendar
import csv
import datetime
import glob
import os
import re

import numpy as np
from sklearn import metrics
from sklearn.cluster import KMeans

__all__ = ['get_all_log_types', 'extract_feature_vectors', 'cluster', 'run_clustering']

def get_all_log_types(node_dirs, regex_pattern='.'):
    prog = re.compile(regex_pattern, flags=re.MULTILINE)

    # Find all the log types
    #print 'Find all the log types...'
    all_log_types = set()
    matched_log_types = set()  # Log types matched with the regex pattern
    for node_dir in node_dirs:
        log_files = glob.glob(os.path.join(node_dir, '*.csv'))

        for log_file in log_files:
            log_type_str = os.path.splitext(os.path.basename(log_file))[0]
            if log_type_str in all_log_types:
                continue

            all_log_types.add(log_type_str)

            matched = True
            txt_file = os.path.splitext(log_file)[0] + '.txt'
            try:
                with open(txt_file, 'r') as f:
                    common_strs = f.read()
                    if prog.search(common_strs) is None:
                        matched = False
            except IOError:
                with open(log_file, 'rb') as f:
                    reader = csv.DictReader(f)
                    for row in reader:
                        if prog.search(row['log_msg']) is None:
                            matched = False
                            break
            if matched:
                matched_log_types.add(log_type_str)
    all_log_types = frozenset(all_log_types)
    matched_log_types = frozenset(matched_log_types)
    return all_log_types, matched_log_types

def read_dates_from_csv_file(fname):
    with open(fname, 'rb') as f:
        reader = csv.DictReader(f)

        # Get the date column
        dates = []
        timestamps = []
        for row in reader:
            try:
                date = datetime.datetime.strptime(row['date'],
                        '%Y-%m-%d %H:%M:%S.%f')
            except ValueError:
                date = datetime.datetime.strptime(row['date'],
                        '%Y-%m-%d %H:%M:%S')
            dates.append(date)
            timestamps.append(calendar.timegm(date.utctimetuple()))
    return dates, timestamps

def extract_feature_vectors(node_dirs, start_time=0, end_time=float('inf')):
    start_dt = datetime.datetime.utcfromtimestamp(start_time)
    end_dt = datetime.datetime.utcfromtimestamp(end_time)

    all_log_types, _ = get_all_log_types(node_dirs)

    # Extract feature vectors
    #print 'Extract feature vectors...'
    fvs_by_node = defaultdict(lambda: np.zeros(len(all_log_types), dtype=np.int))
    min_time = float('inf')
    max_time = 0
    for node_dir in node_dirs:
        #print 'Node', node_dir
        for i, log_type in enumerate(all_log_types):
            out_file = os.path.join(node_dir, '%s.npz' % log_type)

            try:
                npz_file = np.load(out_file)
                timestamps = npz_file['timestamps']
                cumulative_cnts = npz_file['cumulative_cnts']
            except IOError:
                #print 'No .npz file exists.', 'Rebuilding from CSV file...'
                csv_file = os.path.join(node_dir, '%s.csv' % log_type)
                try:
                    dates, timestamps = read_dates_from_csv_file(csv_file)
                except IOError:
                    fvs_by_node[node_dir][i] = 0
                    continue

                # Build data structures for fast range queries
                c = Counter(timestamps)
                timestamps, counts = zip(*sorted(c.items()))

                # I used reverse prefix sums because it was easier to think.
                cumulative_cnts = []
                total_cnt = 0
                for count in reversed(counts):
                    total_cnt += count
                    cumulative_cnts.append(total_cnt)
                cumulative_cnts.reverse()

                # Convert to NumPy arrays
                timestamps = np.asarray(timestamps, dtype=np.uint)
                cumulative_cnts = np.asarray(cumulative_cnts, dtype=np.uint)

                # Save
                np.savez(out_file, timestamps=timestamps, cumulative_cnts=cumulative_cnts)

            min_time = min(timestamps[0], min_time);
            max_time = max(timestamps[-1], max_time);

            # Binary search
            n_matched = 0
            beg_idx = bisect.bisect_left(timestamps, start_time)  # GTEQ
            if beg_idx != len(timestamps):
                n_matched =  cumulative_cnts[beg_idx]

                end_idx = bisect.bisect_left(timestamps, end_time) # GTEQ
                if end_idx != len(timestamps):
                    n_matched -= cumulative_cnts[end_idx]
            fvs_by_node[node_dir][i] = n_matched
    return fvs_by_node, all_log_types, min_time, max_time

def cluster(fvs, n_clusters):
    #print 'Start clustering for %d clusters...' % n_clusters
    fvs = np.asarray(fvs)
    kmeans = KMeans(init='k-means++', n_clusters=n_clusters)
    return kmeans.fit(fvs)

def run_clustering(data_dir, n_clusters, start_time, end_time, regex_pattern):
    node_dirs = [os.path.join(data_dir, node_dir) for node_dir in os.listdir(data_dir)]

    fvs_by_node, all_log_types, min_time, max_time = \
            extract_feature_vectors(node_dirs, start_time, end_time)
    fvs_np = np.array(fvs_by_node.values(), dtype=np.float)

    km = cluster(fvs_np, n_clusters)
    pred = km.predict(fvs_np)
    from collections import Counter
    c = Counter(pred)
    c = dict(c)

    dists = km.transform(fvs_np)
    closest_nodes = np.argmin(dists, axis=0)
    closest_node_dirs = []
    for node in closest_nodes:
        closest_node_dirs.append(fvs_by_node.keys()[node])

    matched_log_types = get_all_log_types(node_dirs, regex_pattern)[1]
    return {'closest_nodes': closest_node_dirs, \
            'matched_log_types': matched_log_types, \
            'num_nodes': len(node_dirs), \
            'num_messages': len(matched_log_types), \
            'min_time': min_time, \
            'max_time': max_time,
            'nodes_per_cluster': c
            }
<commit_msg>Fix a bug<commit_after>from collections import Counter
from collections import defaultdict
import bisect
import calendar
import csv
import datetime
import glob
import os
import re

import numpy as np
from sklearn import metrics
from sklearn.cluster import KMeans

__all__ = ['get_all_log_types', 'extract_feature_vectors', 'cluster', 'run_clustering']

def get_all_log_types(node_dirs, regex_pattern='.'):
    prog = re.compile(regex_pattern, flags=re.MULTILINE)

    # Find all the log types
    #print 'Find all the log types...'
    all_log_types = set()
    matched_log_types = set()  # Log types matched with the regex pattern
    for node_dir in node_dirs:
        log_files = glob.glob(os.path.join(node_dir, '*.csv'))

        for log_file in log_files:
            log_type_str = os.path.splitext(os.path.basename(log_file))[0]
            if log_type_str in all_log_types:
                continue

            all_log_types.add(log_type_str)

            matched = True
            txt_file = os.path.splitext(log_file)[0] + '.txt'
            try:
                with open(txt_file, 'r') as f:
                    common_strs = f.read()
                    if prog.search(common_strs) is None:
                        matched = False
            except IOError:
                with open(log_file, 'rb') as f:
                    reader = csv.DictReader(f)
                    for row in reader:
                        if prog.search(row['log_msg']) is None:
                            matched = False
                            break
            if matched:
                matched_log_types.add(log_type_str)
    all_log_types = frozenset(all_log_types)
    matched_log_types = frozenset(matched_log_types)
    return all_log_types, matched_log_types

def read_dates_from_csv_file(fname):
    with open(fname, 'rb') as f:
        reader = csv.DictReader(f)

        # Get the date column
        dates = []
        timestamps = []
        for row in reader:
            try:
                date = datetime.datetime.strptime(row['date'],
                        '%Y-%m-%d %H:%M:%S.%f')
            except ValueError:
                try:
                    date = datetime.datetime.strptime(row['date'],
                            '%Y-%m-%d %H:%M:%S')
                except ValueError:
                    print fname, row, row['date']
                    date = datetime.datetime.now()
            dates.append(date)
            timestamps.append(calendar.timegm(date.utctimetuple()))
    return dates, timestamps

def extract_feature_vectors(node_dirs, start_time=0, end_time=float('inf')):
    start_dt = datetime.datetime.utcfromtimestamp(start_time)
    end_dt = datetime.datetime.utcfromtimestamp(end_time)

    all_log_types, _ = get_all_log_types(node_dirs)

    # Extract feature vectors
    #print 'Extract feature vectors...'
    fvs_by_node = defaultdict(lambda: np.zeros(len(all_log_types), dtype=np.int))
    min_time = float('inf')
    max_time = 0
    for node_dir in node_dirs:
        #print 'Node', node_dir
        for i, log_type in enumerate(all_log_types):
            out_file = os.path.join(node_dir, '%s.npz' % log_type)

            try:
                npz_file = np.load(out_file)
                timestamps = npz_file['timestamps']
                cumulative_cnts = npz_file['cumulative_cnts']
            except IOError:
                #print 'No .npz file exists.', 'Rebuilding from CSV file...'
                csv_file = os.path.join(node_dir, '%s.csv' % log_type)
                try:
                    dates, timestamps = read_dates_from_csv_file(csv_file)
                except IOError:
                    fvs_by_node[node_dir][i] = 0
                    continue

                if len(timestamps) == 0:
                    fvs_by_node[node_dir][i] = 0
                    continue

                # Build data structures for fast range queries
                c = Counter(timestamps)
                timestamps, counts = zip(*sorted(c.items()))

                # I used reverse prefix sums because it was easier to think.
                cumulative_cnts = []
                total_cnt = 0
                for count in reversed(counts):
                    total_cnt += count
                    cumulative_cnts.append(total_cnt)
                cumulative_cnts.reverse()

                # Convert to NumPy arrays
                timestamps = np.asarray(timestamps, dtype=np.uint)
                cumulative_cnts = np.asarray(cumulative_cnts, dtype=np.uint)

                # Save
                np.savez(out_file, timestamps=timestamps, cumulative_cnts=cumulative_cnts)

            min_time = min(timestamps[0], min_time);
            max_time = max(timestamps[-1], max_time);

            # Binary search
            n_matched = 0
            beg_idx = bisect.bisect_left(timestamps, start_time)  # GTEQ
            if beg_idx != len(timestamps):
                n_matched =  cumulative_cnts[beg_idx]

                end_idx = bisect.bisect_left(timestamps, end_time) # GTEQ
                if end_idx != len(timestamps):
                    n_matched -= cumulative_cnts[end_idx]
            fvs_by_node[node_dir][i] = n_matched
    return fvs_by_node, all_log_types, min_time, max_time

def cluster(fvs, n_clusters):
    #print 'Start clustering for %d clusters...' % n_clusters
    fvs = np.asarray(fvs)
    kmeans = KMeans(init='k-means++', n_clusters=n_clusters)
    return kmeans.fit(fvs)

def run_clustering(data_dir, n_clusters, start_time, end_time, regex_pattern):
    node_dirs = [os.path.join(data_dir, node_dir) for node_dir in os.listdir(data_dir)]

    fvs_by_node, all_log_types, min_time, max_time = \
            extract_feature_vectors(node_dirs, start_time, end_time)
    fvs_np = np.array(fvs_by_node.values(), dtype=np.float)

    km = cluster(fvs_np, n_clusters)
    pred = km.predict(fvs_np)
    from collections import Counter
    c = Counter(pred)
    c = dict(c)

    dists = km.transform(fvs_np)
    closest_nodes = np.argmin(dists, axis=0)
    closest_node_dirs = []
    for node in closest_nodes:
        closest_node_dirs.append(fvs_by_node.keys()[node])

    matched_log_types = get_all_log_types(node_dirs, regex_pattern)[1]
    return {'closest_nodes': closest_node_dirs, \
            'matched_log_types': matched_log_types, \
            'num_nodes': len(node_dirs), \
            'num_messages': len(matched_log_types), \
            'min_time': min_time, \
            'max_time': max_time,
            'nodes_per_cluster': c
            }
<|endoftext|>