<commit_before>import matplotlib.pyplot as plt
from matplotlib.ticker import MultipleLocator
import math
from sys import maxsize

linewidth = 1.5
xoffset = 0.5
color = "k"
xgrid = False

class Classifier:
    # weights = (size weight, average weight, stddev weight)
    # priority = desirable value of size
    def __init__(self, weights, priority, interval):
        self.weights = weights
        self.priority = priority
        self.interval = interval

    def __call__(self, values):
        vs = []
        vs.append(1 / (abs(self.priority - values[0]) + 1))
        vs.append(self.min_average / values[1])
        vs.append(self.min_stddev / values[2])
        return sum([w * v for w, v in zip(self.weights, vs)])

def visit_cluster(tree, labels, xticks, counter = 0):
    if isinstance(tree[0], tuple):
        (blx, bly) = visit_cluster(tree[0], labels, xticks)
    else:
        (blx, bly) = (0, len(labels))
        labels.append(tree[0])
    if isinstance(tree[1], tuple):
        (brx, bry) = visit_cluster(tree[1], labels, xticks)
    else:
        (brx, bry) = (0, len(labels))
        labels.append(tree[1])

    (ulx, uly) = (tree[2], bly)
    (urx, ury) = (tree[2], bry)

    plt.plot(
        [blx, ulx, urx, brx],
        [bly, uly, ury, bry],
        color,
        linewidth = linewidth)

    xticks.append(tree[2])

    return (tree[2], (bly + bry) / 2)

# tree = (((A, B, dist), C, dist), )
def plot(tree):
    labels = []
    xticks = [0]

    fig, ax = plt.subplots()

    visit_cluster(tree, labels, xticks)

    ml = MultipleLocator(xoffset)
    ax.set_yticks(list(range(len(labels))))
    ax.set_yticklabels(labels)
    ax.set_xlim(0, max(xticks) + xoffset)
    ax.set_xticks(xticks)
    ax.xaxis.set_minor_locator(ml)
    ax.yaxis.set_tick_params(width = linewidth)

    if xgrid:
        ax.xaxis.grid()

    fig.show()

    return fig

def labels_of(tree):
    labels = []
    if isinstance(tree[0], tuple):
        labels += labels_of(tree[0])
    else:
        labels.append(tree[0])
    if isinstance(tree[1], tuple):
        labels += labels_of(tree[1])
    else:
        labels.append(tree[1])
    return labels

def levelize(tree, levels, counter = 0):
    if isinstance(tree[0], tuple):
        levelize(tree[0], levels, counter + 1)
    if isinstance(tree[1], tuple):
        levelize(tree[1], levels, counter + 1)
    if counter not in levels.keys():
        levels[counter] = []
    levels[counter].append(tree)

def levels_of(tree):
    levels = {}
    levelize(tree, levels)
    return levels

def best_level(levels, weights, interval, priority):
    score_it = Classifier(weights, priority, interval)
    statistics = {}
    min_average = maxsize
    min_stddev = maxsize

    for (i, level) in levels.items():
        distances = list(map((lambda x: x[2]), level))
        groups = len(distances)
        xm = sum(distances) / groups
        stddev = math.sqrt(sum(map(lambda x: (x - xm) ** 2, distances)))
        statistics[i] = (groups, xm, stddev)
        if xm < min_average:
            min_average = xm
        if stddev < min_stddev:
            min_stddev = stddev

    score_it.min_average = min_average
    score_it.min_stddev = min_stddev
    best_level = -1
    best_score = 0

    for (i, values) in statistics.items():
        score = score_it(values)
        if score > best_score:
            best_score = score
            best_level = i

    print(levels)
    print("best level is", best_level)
    print("best score is", best_score)
    return best_level

def cut(tree, weights, interval, priority = None):
    labels = labels_of(tree)
    levels = levels_of(tree)
    max_level = max(levels.keys())
    if not priority:
        priority = round((interval[0] + interval[1]) / 2)

    for i in range(max_level + 1):
        level_labels = []
        for tup in levels[i]:
            level_labels += labels_of(tup)
        length = len(levels[i])
        if len(level_labels) != len(labels)
         or length > interval[1] or length < interval[0]:
            del levels[i]

    chosen_level = best_level(levels, weights, interval, priority)

    trees = [tree]
    for i in range(chosen_level):
        new_trees = []
        for j in range(trees):
            trees.append(trees[j][0], trees[j][1])
            del trees[j]
    <commit_msg>Partial implementation of cut heuristics - Pt. 2<commit_after>import matplotlib.pyplot as plt
from matplotlib.ticker import MultipleLocator
import math
from sys import maxsize

linewidth = 1.5
xoffset = 0.5
color = "k"
xgrid = False

class Classifier:
    # weights = (size weight, average weight, stddev weight)
    # priority = desirable value of size
    def __init__(self, weights, priority, interval):
        self.weights = weights
        self.priority = priority
        self.interval = interval

    def __call__(self, values):
        vs = []
        vs.append(1 / (abs(self.priority - values[0]) + 1))
        vs.append(self.min_average / values[1])
        vs.append(self.min_stddev / values[2])
        return sum([w * v for w, v in zip(self.weights, vs)])

def __visit_cluster(tree, labels, xticks, counter = 0):
    if isinstance(tree[0], tuple):
        (blx, bly) = __visit_cluster(tree[0], labels, xticks)
    else:
        (blx, bly) = (0, len(labels))
        labels.append(tree[0])
    if isinstance(tree[1], tuple):
        (brx, bry) = __visit_cluster(tree[1], labels, xticks)
    else:
        (brx, bry) = (0, len(labels))
        labels.append(tree[1])

    (ulx, uly) = (tree[2], bly)
    (urx, ury) = (tree[2], bry)

    plt.plot(
        [blx, ulx, urx, brx],
        [bly, uly, ury, bry],
        color,
        linewidth = linewidth)

    xticks.append(tree[2])

    return (tree[2], (bly + bry) / 2)

# def plot_twist(tree):
#     labels = []
#     xticks = [0]
#     label_values = {}

#     for node in tree:
#         if len(node[0]) == 1:
#             label_values[node[0][0]] = len(labels)
#             labels.append(node[0][0])
#         else:

#         if len(node[1]) == 1:
#             label_values[node[1][0]] = len(labels)
#             labels.append(node[1][0])
#         else:

# tree = (((A, B, dist), C, dist), )
def plot(tree):
    labels = []
    xticks = [0]

    fig, ax = plt.subplots()

    __visit_cluster(tree, labels, xticks)

    ml = MultipleLocator(xoffset)
    ax.set_yticks(list(range(len(labels))))
    ax.set_yticklabels(labels)
    ax.set_xlim(0, max(xticks) + xoffset)
    ax.set_xticks(xticks)
    ax.xaxis.set_minor_locator(ml)
    ax.yaxis.set_tick_params(width = linewidth)

    if xgrid:
        ax.xaxis.grid()

    # fig.show()

    return fig

def labels_of(tree):
    labels = []
    if isinstance(tree[0], tuple):
        labels += labels_of(tree[0])
    else:
        labels.append(tree[0])
    if isinstance(tree[1], tuple):
        labels += labels_of(tree[1])
    else:
        labels.append(tree[1])
    return labels

def __levelize(tree, levels, counter = 0):
    if isinstance(tree[0], tuple):
        __levelize(tree[0], levels, counter + 1)
    if isinstance(tree[1], tuple):
        __levelize(tree[1], levels, counter + 1)
    if counter not in levels.keys():
        levels[counter] = []
    levels[counter].append(tree)

def levels_of(tree):
    levels = {}
    __levelize(tree, levels)
    return levels

def best_level(levels, weights, interval, priority):
    score_it = Classifier(weights, priority, interval)
    statistics = {}
    min_average = maxsize
    min_stddev = maxsize

    for (i, level) in levels.items():
        distances = list(map((lambda x: x[2]), level))
        groups = len(distances)
        xm = sum(distances) / groups
        stddev = math.sqrt(sum(map(lambda x: (x - xm) ** 2, distances)))
        statistics[i] = (groups, xm, stddev)
        if xm < min_average:
            min_average = xm
        if stddev < min_stddev:
            min_stddev = stddev

    score_it.min_average = min_average
    score_it.min_stddev = min_stddev
    best_level = -1
    best_score = 0

    for (i, values) in statistics.items():
        score = score_it(values)
        if score > best_score:
            best_score = score
            best_level = i

    print(levels)
    print("best level is", best_level)
    print("best score is", best_score)
    return best_level

def cut(tree, weights, interval, priority = None):
    labels = labels_of(tree)
    levels = levels_of(tree)
    max_level = max(levels.keys())
    if not priority:
        priority = round((interval[0] + interval[1]) / 2)

    for i in range(max_level + 1):
        level_labels = []
        for tup in levels[i]:
            level_labels += labels_of(tup)
        length = len(levels[i])
        # len(level_labels) != len(labels)
        if length > interval[1] or length < interval[0]:
            del levels[i]

    chosen_level = best_level(levels, weights, interval, priority)

    trees = [tree]
    for i in range(chosen_level):
        for j in range(len(trees)):
            if isinstance(trees[j], str):
                continue
            trees.append(trees[j][0])
            trees.append(trees[j][1])
            del trees[j]

    return trees
<|endoftext|>


<commit_before>import os
import ctypes
import array
import fcntl

class I2CError(IOError):
    """Base class for I2C errors."""
    pass

class _CI2CMessage(ctypes.Structure):
    _fields_ = [
        ("addr", ctypes.c_ushort),
        ("flags", ctypes.c_ushort),
        ("len", ctypes.c_ushort),
        ("buf", ctypes.POINTER(ctypes.c_ubyte)),
    ]

class _CI2CIocTransfer(ctypes.Structure):
    _fields_ = [
        ("msgs", ctypes.POINTER(_CI2CMessage)),
        ("nmsgs", ctypes.c_uint),
    ]

class I2C(object):
    # Constants scraped from <linux/i2c-dev.h> and <linux/i2c.h>
    _I2C_IOC_FUNCS      = 0x705
    _I2C_IOC_RDWR       = 0x707
    _I2C_FUNC_I2C       = 0x1
    _I2C_M_TEN          = 0x0010
    _I2C_M_RD           = 0x0001
    _I2C_M_STOP         = 0x8000
    _I2C_M_NOSTART      = 0x4000
    _I2C_M_REV_DIR_ADDR = 0x2000
    _I2C_M_IGNORE_NAK   = 0x1000
    _I2C_M_NO_RD_ACK    = 0x0800
    _I2C_M_RECV_LEN     = 0x0400

    def __init__(self, devpath):
        """Instantiate an I2C object and open the i2c-dev device at the
        specified path.

        Args:
            devpath (str): i2c-dev device path.

        Returns:
            I2C: I2C object.

        Raises:
            I2CError: if an I/O or OS error occurs.

        """
        self._fd = None
        self._devpath = None
        self._open(devpath)

    def __del__(self):
        self.close()

    def __enter__(self):
        pass

    def __exit__(self, t, value, traceback):
        self.close()

    def _open(self, devpath):
        # Open i2c device
        try:
            self._fd = os.open(devpath, os.O_RDWR)
        except OSError as e:
            raise I2CError(e.errno, "Opening I2C device: " + e.strerror)

        self._devpath = devpath

        # Query supported functions
        buf = array.array('I', [0])
        try:
            fcntl.ioctl(self._fd, I2C._I2C_IOC_FUNCS, buf, True)
        except OSError as e:
            self.close()
            raise I2CError(e.errno, "Querying supported functions: " + e.strerror)

        # Check that I2C_RDWR ioctl() is supported on this device
        if (buf[0] & I2C._I2C_FUNC_I2C) == 0:
            self.close()
            raise I2CError(None, "I2C not supported on device %s." % devpath)

    # Methods

    def transfer(self, address, messages):
        """Transfer `messages` to the specified I2C `address`. Modifies the
        `messages` array with the results of any read transactions.

        Args:
            address (int): I2C address.
            messages (list): list of I2C.Message messages.

        Raises:
            I2CError: if an I/O or OS error occurs.
            TypeError: if `messages` type is not list.
            ValueError: if `messages` length is zero, or if message data is not valid bytes.

        """
        if not isinstance(messages, list):
            raise TypeError("Invalid messages type, should be list of I2C.Message.")
        elif len(messages) == 0:
            raise ValueError("Invalid messages data, should be non-zero length.")

        # Convert I2C.Message messages to _CI2CMessage messages
        cmessages = (_CI2CMessage * len(messages))()
        for i in range(len(messages)):
            # Convert I2C.Message data to bytes
            if isinstance(messages[i].data, bytes):
                data = messages[i].data
            elif isinstance(messages[i].data, bytearray):
                data = bytes(messages[i].data)
            elif isinstance(messages[i].data, list):
                data = bytes(bytearray(messages[i].data))

            cmessages[i].addr = address
            cmessages[i].flags = messages[i].flags | (I2C._I2C_M_RD if messages[i].read else 0)
            cmessages[i].len = len(data)
            cmessages[i].buf = ctypes.cast(ctypes.create_string_buffer(data, len(data)), ctypes.POINTER(ctypes.c_ubyte))

        # Prepare transfer structure
        i2c_xfer = _CI2CIocTransfer()
        i2c_xfer.nmsgs = len(cmessages)
        i2c_xfer.msgs = cmessages

        # Transfer
        try:
            fcntl.ioctl(self._fd, I2C._I2C_IOC_RDWR, i2c_xfer, False)
        except IOError as e:
            raise I2CError(e.errno, "I2C transfer: " + e.strerror)

        # Update any read I2C.Message messages
        for i in range(len(messages)):
            if messages[i].read:
                data = [cmessages[i].buf[j] for j in range(cmessages[i].len)]
                # Convert read data to type used in I2C.Message messages
                if isinstance(messages[i].data, list):
                    messages[i].data = data
                elif isinstance(messages[i].data, bytearray):
                    messages[i].data = bytearray(data)
                elif isinstance(messages[i].data, bytes):
                    messages[i].data = bytes(bytearray(data))

    def close(self):
        """Close the i2c-dev I2C device.

        Raises:
            I2CError: if an I/O or OS error occurs.

        """
        if self._fd is None:
            return

        try:
            os.close(self._fd)
        except OSError as e:
            raise I2CError(e.errno, "Closing I2C device: " + e.strerror)

        self._fd = None

    # Immutable properties

    @property
    def fd(self):
        """Get the file descriptor of the underlying i2c-dev device.

        :type: int
        """
        return self._fd

    @property
    def devpath(self):
        """Get the device path of the underlying i2c-dev device.

        :type: str
        """
        return self._devpath

    # String representation

    def __str__(self):
        return "I2C (device=%s, fd=%d)" % (self.devpath, self.fd)

    class Message:
        def __init__(self, data, read=False, flags=0):
            """Instantiate an I2C Message object.

            Args:
                data (bytes, bytearray, list): a byte array or list of 8-bit integers to write.
                read (bool): specify this as a read transaction, where `data` serves as placeholder bytes for the read.
                flags (int): additional i2c-dev flags.

            Returns:
                Message: Message object.

            Raises:
                TypeError: if `data`, `read`, or `flags` types are invalid.

            """
            if not isinstance(data, (bytes, bytearray, list)):
                raise TypeError("Invalid data type, should be bytes, bytearray, or list.")
            if not isinstance(read, bool):
                raise TypeError("Invalid read type, should be boolean.")
            if not isinstance(flags, int):
                raise TypeError("Invalid flags type, should be integer.")

            self.data = data
            self.read = read
            self.flags = flags

<commit_msg>i2c: improve wording of docstrings<commit_after>import os
import ctypes
import array
import fcntl

class I2CError(IOError):
    """Base class for I2C errors."""
    pass

class _CI2CMessage(ctypes.Structure):
    _fields_ = [
        ("addr", ctypes.c_ushort),
        ("flags", ctypes.c_ushort),
        ("len", ctypes.c_ushort),
        ("buf", ctypes.POINTER(ctypes.c_ubyte)),
    ]

class _CI2CIocTransfer(ctypes.Structure):
    _fields_ = [
        ("msgs", ctypes.POINTER(_CI2CMessage)),
        ("nmsgs", ctypes.c_uint),
    ]

class I2C(object):
    # Constants scraped from <linux/i2c-dev.h> and <linux/i2c.h>
    _I2C_IOC_FUNCS      = 0x705
    _I2C_IOC_RDWR       = 0x707
    _I2C_FUNC_I2C       = 0x1
    _I2C_M_TEN          = 0x0010
    _I2C_M_RD           = 0x0001
    _I2C_M_STOP         = 0x8000
    _I2C_M_NOSTART      = 0x4000
    _I2C_M_REV_DIR_ADDR = 0x2000
    _I2C_M_IGNORE_NAK   = 0x1000
    _I2C_M_NO_RD_ACK    = 0x0800
    _I2C_M_RECV_LEN     = 0x0400

    def __init__(self, devpath):
        """Instantiate an I2C object and open the i2c-dev device at the
        specified path.

        Args:
            devpath (str): i2c-dev device path.

        Returns:
            I2C: I2C object.

        Raises:
            I2CError: if an I/O or OS error occurs.

        """
        self._fd = None
        self._devpath = None
        self._open(devpath)

    def __del__(self):
        self.close()

    def __enter__(self):
        pass

    def __exit__(self, t, value, traceback):
        self.close()

    def _open(self, devpath):
        # Open i2c device
        try:
            self._fd = os.open(devpath, os.O_RDWR)
        except OSError as e:
            raise I2CError(e.errno, "Opening I2C device: " + e.strerror)

        self._devpath = devpath

        # Query supported functions
        buf = array.array('I', [0])
        try:
            fcntl.ioctl(self._fd, I2C._I2C_IOC_FUNCS, buf, True)
        except OSError as e:
            self.close()
            raise I2CError(e.errno, "Querying supported functions: " + e.strerror)

        # Check that I2C_RDWR ioctl() is supported on this device
        if (buf[0] & I2C._I2C_FUNC_I2C) == 0:
            self.close()
            raise I2CError(None, "I2C not supported on device %s." % devpath)

    # Methods

    def transfer(self, address, messages):
        """Transfer `messages` to the specified I2C `address`. Modifies the
        `messages` array with the results of any read transactions.

        Args:
            address (int): I2C address.
            messages (list): list of I2C.Message messages.

        Raises:
            I2CError: if an I/O or OS error occurs.
            TypeError: if `messages` type is not list.
            ValueError: if `messages` length is zero, or if message data is not valid bytes.

        """
        if not isinstance(messages, list):
            raise TypeError("Invalid messages type, should be list of I2C.Message.")
        elif len(messages) == 0:
            raise ValueError("Invalid messages data, should be non-zero length.")

        # Convert I2C.Message messages to _CI2CMessage messages
        cmessages = (_CI2CMessage * len(messages))()
        for i in range(len(messages)):
            # Convert I2C.Message data to bytes
            if isinstance(messages[i].data, bytes):
                data = messages[i].data
            elif isinstance(messages[i].data, bytearray):
                data = bytes(messages[i].data)
            elif isinstance(messages[i].data, list):
                data = bytes(bytearray(messages[i].data))

            cmessages[i].addr = address
            cmessages[i].flags = messages[i].flags | (I2C._I2C_M_RD if messages[i].read else 0)
            cmessages[i].len = len(data)
            cmessages[i].buf = ctypes.cast(ctypes.create_string_buffer(data, len(data)), ctypes.POINTER(ctypes.c_ubyte))

        # Prepare transfer structure
        i2c_xfer = _CI2CIocTransfer()
        i2c_xfer.nmsgs = len(cmessages)
        i2c_xfer.msgs = cmessages

        # Transfer
        try:
            fcntl.ioctl(self._fd, I2C._I2C_IOC_RDWR, i2c_xfer, False)
        except IOError as e:
            raise I2CError(e.errno, "I2C transfer: " + e.strerror)

        # Update any read I2C.Message messages
        for i in range(len(messages)):
            if messages[i].read:
                data = [cmessages[i].buf[j] for j in range(cmessages[i].len)]
                # Convert read data to type used in I2C.Message messages
                if isinstance(messages[i].data, list):
                    messages[i].data = data
                elif isinstance(messages[i].data, bytearray):
                    messages[i].data = bytearray(data)
                elif isinstance(messages[i].data, bytes):
                    messages[i].data = bytes(bytearray(data))

    def close(self):
        """Close the i2c-dev I2C device.

        Raises:
            I2CError: if an I/O or OS error occurs.

        """
        if self._fd is None:
            return

        try:
            os.close(self._fd)
        except OSError as e:
            raise I2CError(e.errno, "Closing I2C device: " + e.strerror)

        self._fd = None

    # Immutable properties

    @property
    def fd(self):
        """Get the file descriptor of the underlying i2c-dev device.

        :type: int
        """
        return self._fd

    @property
    def devpath(self):
        """Get the device path of the underlying i2c-dev device.

        :type: str
        """
        return self._devpath

    # String representation

    def __str__(self):
        return "I2C (device=%s, fd=%d)" % (self.devpath, self.fd)

    class Message:
        def __init__(self, data, read=False, flags=0):
            """Instantiate an I2C Message object.

            Args:
                data (bytes, bytearray, list): a byte array or list of 8-bit
                             integers to write.
                read (bool): specify this as a read message, where `data`
                             serves as placeholder bytes for the read.
                flags (int): additional i2c-dev flags for this message.

            Returns:
                Message: Message object.

            Raises:
                TypeError: if `data`, `read`, or `flags` types are invalid.

            """
            if not isinstance(data, (bytes, bytearray, list)):
                raise TypeError("Invalid data type, should be bytes, bytearray, or list.")
            if not isinstance(read, bool):
                raise TypeError("Invalid read type, should be boolean.")
            if not isinstance(flags, int):
                raise TypeError("Invalid flags type, should be integer.")

            self.data = data
            self.read = read
            self.flags = flags

<|endoftext|>


<commit_before>from __future__ import print_function
import time, requests, json, textwrap
from binascii import hexlify, unhexlify
from spake2 import SPAKE2_A, SPAKE2_B
from nacl.secret import SecretBox
from nacl.exceptions import CryptoError
from nacl import utils
from .. import codes
from ..util.hkdf import HKDF

SECOND = 1
MINUTE = 60*SECOND

class Timeout(Exception):
    pass

class WrongPasswordError(Exception):
    """
    Key confirmation failed.
    """
    # or the data blob was corrupted, and that's why decrypt failed
    def explain(self):
        return textwrap.dedent(self.__doc__)

class InitiatorWrongPasswordError(WrongPasswordError):
    """
    Key confirmation failed. Either your correspondent typed the code wrong,
    or a would-be man-in-the-middle attacker guessed incorrectly. You could
    try again, giving both your correspondent and the attacker another
    chance.
    """

class ReceiverWrongPasswordError(WrongPasswordError):
    """
    Key confirmation failed. Either you typed the code wrong, or a would-be
    man-in-the-middle attacker guessed incorrectly. You could try again,
    giving both you and the attacker another chance.
    """

# relay URLs are:
#
# POST /allocate                                  -> {channel-id: INT}
#  these return all messages for CHANNEL-ID= and MSGNUM= but SIDE!= :
# POST /CHANNEL-ID/SIDE/post/MSGNUM  {message: STR} -> {messages: [STR..]}
# POST /CHANNEL-ID/SIDE/poll/MSGNUM                 -> {messages: [STR..]}
# GET  /CHANNEL-ID/SIDE/poll/MSGNUM (eventsource)   -> STR, STR, ..
#
# POST /CHANNEL-ID/SIDE/deallocate                  -> waiting | deleted

class EventSourceFollower:
    def __init__(self, url, timeout):
        self.resp = requests.get(url,
                                 headers={"accept": "text/event-stream"},
                                 stream=True,
                                 timeout=timeout)
        self.resp.raise_for_status()

    def close(self):
        self.resp.close()

    def _get_fields(self, lines):
        while True:
            first_line = lines.next() # raises StopIteration when closed
            fieldname, data = first_line.split(": ", 1)
            data_lines = [data]
            while True:
                next_line = lines.next()
                if not next_line: # empty string, original was "\n"
                    yield (fieldname, "\n".join(data_lines))
                    break
                data_lines.append(next_line)

    def iter_events(self):
        # I think Request.iter_lines and .iter_content use chunk_size= in a
        # funny way, and nothing happens until at least that much data has
        # arrived. So unless we set chunk_size=1, we won't hear about lines
        # for a long time. I'd prefer that chunk_size behaved like
        # read(size), and gave you 1<=x<=size bytes in response.
        eventtype = "message"
        lines_iter = self.resp.iter_lines(chunk_size=1)
        for (fieldname, data) in self._get_fields(lines_iter):
            if fieldname == "data":
                yield (eventtype, data)
                eventtype = "message"
            elif fieldname == "event":
                eventtype = data
            else:
                print("weird fieldname", fieldname, data)

class Common:
    def url(self, verb, msgnum=None):
        url = "%s%d/%s/%s" % (self.relay, self.channel_id, self.side, verb)
        if msgnum is not None:
            url += "/" + msgnum
        return url

    def get(self, old_msgs, verb, msgnum):
        # For now, server errors cause the client to fail. TODO: don't. This
        # will require changing the client to re-post messages when the
        # server comes back up.

        # note: while this passes around msgs (plural), our callers really
        # only care about the first one. we use "WHICH" and "SIDE" so that we
        # only expect to see a single message (not our own, where "SIDE" is
        # our own, and not messages for earlier stages, where "WHICH" is
        # different)
        msgs = old_msgs
        while not msgs:
            remaining = self.started + self.timeout - time.time()
            if remaining < 0:
                raise Timeout
            #time.sleep(self.wait)
            f = EventSourceFollower(self.url(verb, msgnum), remaining)
            for (eventtype, data) in f.iter_events():
                if eventtype == "message":
                    msgs = [json.loads(data)["message"]]
                    break
            f.close()
        return msgs

    def _allocate(self):
        r = requests.post(self.relay + "allocate")
        r.raise_for_status()
        channel_id = r.json()["channel-id"]
        return channel_id

    def _post_pake(self):
        msg = self.sp.start()
        post_data = {"message": hexlify(msg).decode("ascii")}
        r = requests.post(self.url("post", "pake"), data=json.dumps(post_data))
        r.raise_for_status()
        other_msgs = r.json()["messages"]
        return other_msgs

    def _get_pake(self, other_msgs):
        msgs = self.get(other_msgs, "poll", "pake")
        pake_msg = unhexlify(msgs[0].encode("ascii"))
        key = self.sp.finish(pake_msg)
        return key

    def _encrypt_data(self, key, data):
        assert len(key) == SecretBox.KEY_SIZE
        box = SecretBox(key)
        nonce = utils.random(SecretBox.NONCE_SIZE)
        return box.encrypt(data, nonce)

    def _post_data(self, data):
        post_data = json.dumps({"message": hexlify(data).decode("ascii")})
        r = requests.post(self.url("post", "data"), data=post_data)
        r.raise_for_status()
        other_msgs = r.json()["messages"]
        return other_msgs

    def _get_data(self, other_msgs):
        msgs = self.get(other_msgs, "poll", "data")
        data = unhexlify(msgs[0].encode("ascii"))
        return data

    def _decrypt_data(self, key, encrypted):
        assert len(key) == SecretBox.KEY_SIZE
        box = SecretBox(key)
        data = box.decrypt(encrypted)
        return data

    def _deallocate(self):
        r = requests.post(self.url("deallocate"))
        r.raise_for_status()

    def derive_key(self, purpose, length=SecretBox.KEY_SIZE):
        assert type(purpose) == type(b"")
        return HKDF(self.key, length, CTXinfo=purpose)

class Initiator(Common):
    def __init__(self, appid, relay):
        self.appid = appid
        self.relay = relay
        assert self.relay.endswith("/")
        self.started = time.time()
        self.wait = 0.5*SECOND
        self.timeout = 3*MINUTE
        self.side = "initiator"
        self.key = None
        self.verifier = None

    def get_code(self, code_length=2):
        self.channel_id = self._allocate() # allocate channel
        self.code = codes.make_code(self.channel_id, code_length)
        self.sp = SPAKE2_A(self.code.encode("ascii"),
                           idA=self.appid+":Initiator",
                           idB=self.appid+":Receiver")
        self._post_pake()
        return self.code

    def _wait_for_key(self):
        if not self.key:
            key = self._get_pake([])
            self.key = key
            self.verifier = self.derive_key(self.appid+b":Verifier")

    def get_verifier(self):
        self._wait_for_key()
        return self.verifier

    def get_data(self, outbound_data):
        self._wait_for_key()
        try:
            outbound_key = self.derive_key(b"sender")
            outbound_encrypted = self._encrypt_data(outbound_key, outbound_data)
            other_msgs = self._post_data(outbound_encrypted)

            inbound_encrypted = self._get_data(other_msgs)
            inbound_key = self.derive_key(b"receiver")
            try:
                inbound_data = self._decrypt_data(inbound_key,
                                                  inbound_encrypted)
            except CryptoError:
                raise InitiatorWrongPasswordError
        finally:
            self._deallocate()
        return inbound_data

class Receiver(Common):
    def __init__(self, appid, relay):
        self.appid = appid
        self.relay = relay
        assert self.relay.endswith("/")
        self.started = time.time()
        self.wait = 0.5*SECOND
        self.timeout = 3*MINUTE
        self.side = "receiver"
        self.code = None
        self.channel_id = None
        self.key = None
        self.verifier = None

    def list_channels(self):
        r = requests.get(self.relay + "list")
        r.raise_for_status()
        channel_ids = r.json()["channel-ids"]
        return channel_ids

    def input_code(self, prompt="Enter wormhole code: ", code_length=2):
        code = codes.input_code_with_completion(prompt, self.list_channels,
                                                code_length)
        return code

    def set_code(self, code):
        assert self.code is None
        assert self.channel_id is None
        self.code = code
        self.channel_id = codes.extract_channel_id(code)
        self.sp = SPAKE2_B(code.encode("ascii"),
                           idA=self.appid+":Initiator",
                           idB=self.appid+":Receiver")

    def _wait_for_key(self):
        if not self.key:
            other_msgs = self._post_pake()
            key = self._get_pake(other_msgs)
            self.key = key
            self.verifier = self.derive_key(self.appid+b":Verifier")

    def get_verifier(self):
        self._wait_for_key()
        return self.verifier

    def get_data(self, outbound_data):
        assert self.code is not None
        assert self.channel_id is not None
        self._wait_for_key()

        try:
            outbound_key = self.derive_key(b"receiver")
            outbound_encrypted = self._encrypt_data(outbound_key, outbound_data)
            other_msgs = self._post_data(outbound_encrypted)

            inbound_encrypted = self._get_data(other_msgs)
            inbound_key = self.derive_key(b"sender")
            try:
                inbound_data = self._decrypt_data(inbound_key,
                                                  inbound_encrypted)
            except CryptoError:
                raise ReceiverWrongPasswordError
        finally:
            self._deallocate()
        return inbound_data
<commit_msg>warn (to stderr) if the client version differs from the server's<commit_after>from __future__ import print_function
import sys, time, requests, json, textwrap
from binascii import hexlify, unhexlify
from spake2 import SPAKE2_A, SPAKE2_B
from nacl.secret import SecretBox
from nacl.exceptions import CryptoError
from nacl import utils
from .. import __version__
from .. import codes
from ..util.hkdf import HKDF

SECOND = 1
MINUTE = 60*SECOND

class Timeout(Exception):
    pass

class WrongPasswordError(Exception):
    """
    Key confirmation failed.
    """
    # or the data blob was corrupted, and that's why decrypt failed
    def explain(self):
        return textwrap.dedent(self.__doc__)

class InitiatorWrongPasswordError(WrongPasswordError):
    """
    Key confirmation failed. Either your correspondent typed the code wrong,
    or a would-be man-in-the-middle attacker guessed incorrectly. You could
    try again, giving both your correspondent and the attacker another
    chance.
    """

class ReceiverWrongPasswordError(WrongPasswordError):
    """
    Key confirmation failed. Either you typed the code wrong, or a would-be
    man-in-the-middle attacker guessed incorrectly. You could try again,
    giving both you and the attacker another chance.
    """

# relay URLs are:
#
# POST /allocate                                  -> {channel-id: INT}
#  these return all messages for CHANNEL-ID= and MSGNUM= but SIDE!= :
# POST /CHANNEL-ID/SIDE/post/MSGNUM  {message: STR} -> {messages: [STR..]}
# POST /CHANNEL-ID/SIDE/poll/MSGNUM                 -> {messages: [STR..]}
# GET  /CHANNEL-ID/SIDE/poll/MSGNUM (eventsource)   -> STR, STR, ..
#
# POST /CHANNEL-ID/SIDE/deallocate                  -> waiting | deleted

class EventSourceFollower:
    def __init__(self, url, timeout):
        self.resp = requests.get(url,
                                 headers={"accept": "text/event-stream"},
                                 stream=True,
                                 timeout=timeout)
        self.resp.raise_for_status()

    def close(self):
        self.resp.close()

    def _get_fields(self, lines):
        while True:
            first_line = lines.next() # raises StopIteration when closed
            fieldname, data = first_line.split(": ", 1)
            data_lines = [data]
            while True:
                next_line = lines.next()
                if not next_line: # empty string, original was "\n"
                    yield (fieldname, "\n".join(data_lines))
                    break
                data_lines.append(next_line)

    def iter_events(self):
        # I think Request.iter_lines and .iter_content use chunk_size= in a
        # funny way, and nothing happens until at least that much data has
        # arrived. So unless we set chunk_size=1, we won't hear about lines
        # for a long time. I'd prefer that chunk_size behaved like
        # read(size), and gave you 1<=x<=size bytes in response.
        eventtype = "message"
        lines_iter = self.resp.iter_lines(chunk_size=1)
        for (fieldname, data) in self._get_fields(lines_iter):
            if fieldname == "data":
                yield (eventtype, data)
                eventtype = "message"
            elif fieldname == "event":
                eventtype = data
            else:
                print("weird fieldname", fieldname, data)

class Common:
    def url(self, verb, msgnum=None):
        url = "%s%d/%s/%s" % (self.relay, self.channel_id, self.side, verb)
        if msgnum is not None:
            url += "/" + msgnum
        return url

    version_warning_displayed = False

    def handle_welcome(self, welcome):
        if self.version_warning_displayed:
            return
        if "-" in __version__:
            # only warn if we're running a release version (e.g. 0.0.6, not
            # 0.0.6-DISTANCE-gHASH)
            return
        if welcome["current_version"] != __version__:
            print("Warning: errors may occur unless both sides are running the same version", file=sys.stderr)
            print("Server claims %s is current, but ours is %s"
                  % (welcome["current_version"], __version__), file=sys.stderr)
            self.version_warning_displayed = True

    def get(self, old_msgs, verb, msgnum):
        # For now, server errors cause the client to fail. TODO: don't. This
        # will require changing the client to re-post messages when the
        # server comes back up.

        # note: while this passes around msgs (plural), our callers really
        # only care about the first one. we use "WHICH" and "SIDE" so that we
        # only expect to see a single message (not our own, where "SIDE" is
        # our own, and not messages for earlier stages, where "WHICH" is
        # different)
        msgs = old_msgs
        while not msgs:
            remaining = self.started + self.timeout - time.time()
            if remaining < 0:
                raise Timeout
            #time.sleep(self.wait)
            f = EventSourceFollower(self.url(verb, msgnum), remaining)
            for (eventtype, data) in f.iter_events():
                if eventtype == "welcome":
                    self.handle_welcome(json.loads(data))
                if eventtype == "message":
                    msgs = [json.loads(data)["message"]]
                    break
            f.close()
        return msgs

    def _allocate(self):
        r = requests.post(self.relay + "allocate")
        r.raise_for_status()
        channel_id = r.json()["channel-id"]
        return channel_id

    def _post_pake(self):
        msg = self.sp.start()
        post_data = {"message": hexlify(msg).decode("ascii")}
        r = requests.post(self.url("post", "pake"), data=json.dumps(post_data))
        r.raise_for_status()
        other_msgs = r.json()["messages"]
        return other_msgs

    def _get_pake(self, other_msgs):
        msgs = self.get(other_msgs, "poll", "pake")
        pake_msg = unhexlify(msgs[0].encode("ascii"))
        key = self.sp.finish(pake_msg)
        return key

    def _encrypt_data(self, key, data):
        assert len(key) == SecretBox.KEY_SIZE
        box = SecretBox(key)
        nonce = utils.random(SecretBox.NONCE_SIZE)
        return box.encrypt(data, nonce)

    def _post_data(self, data):
        post_data = json.dumps({"message": hexlify(data).decode("ascii")})
        r = requests.post(self.url("post", "data"), data=post_data)
        r.raise_for_status()
        other_msgs = r.json()["messages"]
        return other_msgs

    def _get_data(self, other_msgs):
        msgs = self.get(other_msgs, "poll", "data")
        data = unhexlify(msgs[0].encode("ascii"))
        return data

    def _decrypt_data(self, key, encrypted):
        assert len(key) == SecretBox.KEY_SIZE
        box = SecretBox(key)
        data = box.decrypt(encrypted)
        return data

    def _deallocate(self):
        r = requests.post(self.url("deallocate"))
        r.raise_for_status()

    def derive_key(self, purpose, length=SecretBox.KEY_SIZE):
        assert type(purpose) == type(b"")
        return HKDF(self.key, length, CTXinfo=purpose)

class Initiator(Common):
    def __init__(self, appid, relay):
        self.appid = appid
        self.relay = relay
        assert self.relay.endswith("/")
        self.started = time.time()
        self.wait = 0.5*SECOND
        self.timeout = 3*MINUTE
        self.side = "initiator"
        self.key = None
        self.verifier = None

    def get_code(self, code_length=2):
        self.channel_id = self._allocate() # allocate channel
        self.code = codes.make_code(self.channel_id, code_length)
        self.sp = SPAKE2_A(self.code.encode("ascii"),
                           idA=self.appid+":Initiator",
                           idB=self.appid+":Receiver")
        self._post_pake()
        return self.code

    def _wait_for_key(self):
        if not self.key:
            key = self._get_pake([])
            self.key = key
            self.verifier = self.derive_key(self.appid+b":Verifier")

    def get_verifier(self):
        self._wait_for_key()
        return self.verifier

    def get_data(self, outbound_data):
        self._wait_for_key()
        try:
            outbound_key = self.derive_key(b"sender")
            outbound_encrypted = self._encrypt_data(outbound_key, outbound_data)
            other_msgs = self._post_data(outbound_encrypted)

            inbound_encrypted = self._get_data(other_msgs)
            inbound_key = self.derive_key(b"receiver")
            try:
                inbound_data = self._decrypt_data(inbound_key,
                                                  inbound_encrypted)
            except CryptoError:
                raise InitiatorWrongPasswordError
        finally:
            self._deallocate()
        return inbound_data

class Receiver(Common):
    def __init__(self, appid, relay):
        self.appid = appid
        self.relay = relay
        assert self.relay.endswith("/")
        self.started = time.time()
        self.wait = 0.5*SECOND
        self.timeout = 3*MINUTE
        self.side = "receiver"
        self.code = None
        self.channel_id = None
        self.key = None
        self.verifier = None

    def list_channels(self):
        r = requests.get(self.relay + "list")
        r.raise_for_status()
        channel_ids = r.json()["channel-ids"]
        return channel_ids

    def input_code(self, prompt="Enter wormhole code: ", code_length=2):
        code = codes.input_code_with_completion(prompt, self.list_channels,
                                                code_length)
        return code

    def set_code(self, code):
        assert self.code is None
        assert self.channel_id is None
        self.code = code
        self.channel_id = codes.extract_channel_id(code)
        self.sp = SPAKE2_B(code.encode("ascii"),
                           idA=self.appid+":Initiator",
                           idB=self.appid+":Receiver")

    def _wait_for_key(self):
        if not self.key:
            other_msgs = self._post_pake()
            key = self._get_pake(other_msgs)
            self.key = key
            self.verifier = self.derive_key(self.appid+b":Verifier")

    def get_verifier(self):
        self._wait_for_key()
        return self.verifier

    def get_data(self, outbound_data):
        assert self.code is not None
        assert self.channel_id is not None
        self._wait_for_key()

        try:
            outbound_key = self.derive_key(b"receiver")
            outbound_encrypted = self._encrypt_data(outbound_key, outbound_data)
            other_msgs = self._post_data(outbound_encrypted)

            inbound_encrypted = self._get_data(other_msgs)
            inbound_key = self.derive_key(b"sender")
            try:
                inbound_data = self._decrypt_data(inbound_key,
                                                  inbound_encrypted)
            except CryptoError:
                raise ReceiverWrongPasswordError
        finally:
            self._deallocate()
        return inbound_data
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#  This file is part of TinyIDS.
#
#  TinyIDS is a distributed Intrusion Detection System (IDS) for Unix systems.
#
#  Project development web site:
#
#      http://www.codetrax.org/projects/tinyids
#
#  Copyright (c) 2010 George Notaras, G-Loaded.eu, CodeTRAX.org
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import os
import logging
import glob
import socket
import getpass
import time

import TinyIDS.backends
from TinyIDS import config
from TinyIDS import crypto
from TinyIDS.util import sha1, load_backend

logger = logging.getLogger()

class NoBackendsToRun(Exception):
    pass

class TinyIDSClient:
    """A client implementation of the TinyIDS protocol."""

    cmd_end = '\r\n'
    max_response_len = 1024

    def __init__(self, command):

        self.command = command  # TEST | CHECK | UPDATE | DELETE | CHANGEPHRASE
        self.hasher = sha1()
        self.cfg = config.get_client_configuration()

        # Default hashing delay
        self.default_hashing_delay = self._get_hashing_delay()

        # Debug protocol
        self.debug_protocol = self.cfg.getboolean('main', 'debug_protocol')

        # PKI Module
        _keys_dir = self.cfg.get('main', 'keys_dir')
        self.pki = crypto.RSAModule(_keys_dir)

        # This is socket.socket object while connected to server
        # Should be set to None as soon the connection is closed
        # or a socket error occurs.
        self.sock = None

        # Should hold the name of the server as long as there is a
        # valid connection to it.
        self.server_name = None

    def _close_socket(self):
        """Should be called after all socket errors."""
        if isinstance(self.sock, socket.socket):
            self.sock.close()
            if self.server_name:
                logger.info('- Closed connection with server: %s' % self.server_name)
        self.sock = None
        self.server_name = None

    def _get_hashing_delay(self):
        """Returns the hashing delay in seconds."""
        delay_msec = self.default_hashing_delay = self.cfg.getfloat('main', 'hashing_delay')
        delay_sec = delay_msec / 1000
        logger.debug('Hashing delay set to %.3f seconds' % delay_sec)
        return delay_sec

    def _run_backends(self):
        """Runs all collector backends and passes the yielded information
        through the hashing algorithm."""

        backends_conf_dir = self.cfg.get('main', 'backends_conf_dir')

        # Get a list of all backend paths
        backend_paths = []
        core_backends_dir = TinyIDS.backends.__path__[0]
        backend_paths.extend(glob.glob(os.path.join(core_backends_dir, '*.py')))
        extra_backends_dir = self.cfg.get('main', 'extra_backends_dir')
        backend_paths.extend(glob.glob(os.path.join(extra_backends_dir, '*.py')))

        if not backend_paths:
            raise NoBackendsToRun

        #
        user_defined_backend_list = self.cfg.getlist('main', 'tests')
        user_defined_backend_list_finished = [] # holds names of backends that have finished
        if user_defined_backend_list:
            logger.debug('Using user-defined list of backends')

        # Load all needed backends and store them in a list
        for backend_path in backend_paths:
            backend_name = os.path.basename(backend_path)[:-3]
            backend_dir = os.path.dirname(backend_path)
            if backend_name == '__init__':
                continue
            elif user_defined_backend_list:
                #print "checking user list"
                if not backend_name in user_defined_backend_list:
                    logger.debug('Skipping backend: %s' % backend_name)
                    continue
            # Load backend
            m = load_backend(backend_dir, backend_name)
            if not hasattr(m, 'CollectorBackend'):
                logger.warning('Skipping invalid backend: %s' % backend_path)
                continue

            backend_config_file = os.path.join(backends_conf_dir, m.__name__ + '.conf')
            b = m.CollectorBackend(config_path=backend_config_file)
            if not hasattr(b, 'collect'):
                logger.warning('Skipping invalid backend: %s' % backend_path)
                continue

            # Run backend
            logger.info('Processing backend: %s' % backend_name)
            for data in b.collect():
                self.hash_data(data)
            logger.info('- Complete')

            if user_defined_backend_list:
                # If a user-defined list of backends is used, add the name of
                # the backend that has just run to the list:
                # user_defined_backend_list_finished
                user_defined_backend_list_finished.append(backend_name)

        if user_defined_backend_list:
            invalid_user_defined_tests = []
            for test in user_defined_backend_list:
                if test not in user_defined_backend_list_finished:
                    invalid_user_defined_tests.append(test)
            if invalid_user_defined_tests:
                logger.warning('Invalid user-defined test(s): %s' % ', '.join(invalid_user_defined_tests))
                if not user_defined_backend_list_finished:
                    raise NoBackendsToRun

    def _get_server_canonical_name(self, server_name):
        """Returns the name of the server after stripping the 'server__' prefix'"""
        return server_name.split('__')[1]

    def _get_enabled_server_list(self):
        """Returns a list of servers that have been enabled in the
        client configuration."""
        enabled_servers = []
        for section in self.cfg.sections():
            if not section.startswith('server__'):
                continue
            server_name = self._get_server_canonical_name(section)
            if not self.cfg.has_option(section, 'host'):
                logger.warning('Misconfigured server: %s' % server_name)
                continue
            if self.cfg.has_option(section, 'enabled'):
                is_enabled = self.cfg.getboolean(section, 'enabled')
                if not is_enabled:
                    continue
                enabled_servers.append(section)
        return enabled_servers

    def _send(self, host, port, data):
        """Sends the command to a TinyIDS server."""
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.connect((host, port))
        logger.info('- Established connection to server: %s' % self.server_name)
        if self.debug_protocol:
            logger.debug('-> Sending command: %s' % data)
        if self.pki.public_key is not None:
            data = self.pki.encrypt(data)
            logger.info('- PKI: data encrypted')
        self.sock.send(data + self.cmd_end)
        logger.info('- Sent %s command to server: %s' % (self.command, self.server_name))

    def _get_server_response(self):
        """Receives a response from a TinyIDS server."""
        logger.info('- Awaiting server response...')
        response = self.sock.recv(self.max_response_len).strip()
        response = response.rstrip(self.cmd_end)
        logger.info('- Received response from server: %s' % self.server_name)
        if self.pki.public_key is not None:
            response = self.pki.verify(response)
            logger.info('- PKI: data verified')
        if self.debug_protocol:
            logger.debug('-> Received response: %s' % response.strip())
        return response.strip()

    def _check_command_status(self, response):
        if response.startswith('20'):
            logger.info('- RESULT: %s on %s: SUCCESS' % (self.command, self.server_name))
        else:
            logger.warning('- RESULT: %s on %s: FAILURE with error: %s' % (self.command, self.server_name, response))

    def _communicate(self, host, port, data):
        self._send(host, port, data)
        response = self._get_server_response()
        self._check_command_status(response)

    def _get_passphrase(self, msg):
        """Prompts the user for a passphrase."""
        data = ''
        while not data:
            data = getpass.getpass('%s: ' % msg)
        return data

    def _com_TEST(self, host, port):
        """
        Syntax: TEST <protocol_revision>
        """
        data = '%s %s' % (self.command, config.PROTOCOL_REVISION)
        self._communicate(host, port, data)

    def _com_CHECK(self, host, port):
        """
        Syntax: CHECK <hash>
        """
        data = '%s %s' % (self.command, self.get_checksum())
        self._communicate(host, port, data)

    def _com_UPDATE(self, host, port):
        """
        Syntax: UPDATE <hash> <passphrase>
        """
        passphrase = self._get_passphrase('Passphrase')
        data = '%s %s %s' % (self.command, self.get_checksum(), passphrase)
        self._communicate(host, port, data)

    def _com_DELETE(self, host, port):
        """
        Syntax: DELETE <passphrase>
        """
        passphrase = self._get_passphrase('Passphrase')
        data = '%s %s' % (self.command, passphrase)
        self._communicate(host, port, data)

    def _com_CHANGEPHRASE(self, host, port):
        """
        Syntax: CHANGEPHRASE <old_passphrase> <new_passphrase>
        """
        passphrase_old = self._get_passphrase('Old passphrase')
        while True:
            passphrase_new = self._get_passphrase('New passphrase')
            passphrase_new_confirm = self._get_passphrase('Confirm new passphrase')
            if passphrase_new == passphrase_new_confirm:
                break
            logger.error('passphrases do not match. try again...')
        data = '%s %s %s' % (self.command, passphrase_old, passphrase_new)
        self._communicate(host, port, data)

    # Public API

    def get_checksum(self):
        return self.hasher.hexdigest()

    def hash_data(self, data):
        """Passes data through the hashing algorithm."""
        self.hasher.update(data)
        time.sleep(self.default_hashing_delay)

    def run(self):
        """Main client method."""

        logger.info('Getting list of enabled servers')
        enabled_servers = self._get_enabled_server_list()
        if not enabled_servers:
            logger.warning('No servers configured. Shutting down...')
            self.client_close()
            return
        logger.info('Servers found. Proceeding with data hashing...')

        # Decide which method to execute
        func = getattr(self, '_com_%s' % self.command)

        # Tests are required to run only with the CHECK and UPDATE commands
        if self.command in ('CHECK', 'UPDATE'):
            logger.info('Hashing data. Please wait...')
            try:
                self._run_backends()
            except NoBackendsToRun:
                logger.warning('No valid backends. Shutting down...')
                self.client_close()
                return
            else:
                logger.info('Hashing complete')

        # Execute command on server
        logger.info('Preparing to contact servers')
        for server in enabled_servers:
            # server is in 'server__<name>' format (a section name)
            # Here we set self.server_name to the canonical server name
            self.server_name = self._get_server_canonical_name(server)
            logger.info('* Contacting server: %s' % self.server_name)

            # Get server settings
            host = self.cfg.get(server, 'host')
            port = config.DEFAULT_PORT
            if self.cfg.has_option(server, 'port'):
                port = self.cfg.getint(server, 'port')
            if self.cfg.has_option(server, 'public_key'):
                public_key_fname = self.cfg.get(server, 'public_key')
                if public_key_fname:
                    try:
                        self.pki.load_external_public_key(public_key_fname) # sets self.pki.public_key
                    except crypto.InvalidPublicKey:
                        logger.warning('- RESULT: %s on %s: FAILURE with PKI error: invalid server public key: %s' % (self.command, self.server_name, public_key_fname))
                        self.server_name = None
                        continue

            # Run command on the server
            try:
                func(host, port)
            except socket.error, (errno, strerror):
                logger.error('Connection error: \'%s\'' % strerror)
                logger.warning('Skipping server: %s' % self.server_name)
                self.server_name = None
            except crypto.DataEncryptionError:
                logger.warning('- RESULT: %s on %s: FAILURE with PKI error: could not encrypt data for server' % (self.command, self.server_name))
            except crypto.DataVerificationError:
                logger.warning('- RESULT: %s on %s: FAILURE with PKI error: could not verify response from server' % (self.command, self.server_name))

            self._close_socket()
            self.pki.reset()    # sets self.pki.public_key to None

        logger.info('Finished with servers')

        self.client_close()

    def client_close(self):
        logger.info('Client shutting down...')

<commit_msg>CLOSED #248: The hashing algorithm should always hash the machine's hostname http://www.codetrax.org/issues/show/248<commit_after># -*- coding: utf-8 -*-
#
#  This file is part of TinyIDS.
#
#  TinyIDS is a distributed Intrusion Detection System (IDS) for Unix systems.
#
#  Project development web site:
#
#      http://www.codetrax.org/projects/tinyids
#
#  Copyright (c) 2010 George Notaras, G-Loaded.eu, CodeTRAX.org
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import os
import logging
import glob
import socket
import getpass
import time

import TinyIDS.backends
from TinyIDS import config
from TinyIDS import crypto
from TinyIDS.util import sha1, load_backend

logger = logging.getLogger()

class NoBackendsToRun(Exception):
    pass

class TinyIDSClient:
    """A client implementation of the TinyIDS protocol."""

    cmd_end = '\r\n'
    max_response_len = 1024

    def __init__(self, command):

        # Client configuration (config.TinyIDSConfigParser instance)
        self.cfg = config.get_client_configuration()

        # Data hashing object
        self.hasher = sha1()
        # Immediately hash the machine's hostname (issue: #248)
        self.hash_data(socket.gethostname())

        # Holds the current command
        self.command = command  # TEST | CHECK | UPDATE | DELETE | CHANGEPHRASE

        # Default hashing delay
        self.default_hashing_delay = self._get_hashing_delay()

        # Debug protocol
        self.debug_protocol = self.cfg.getboolean('main', 'debug_protocol')

        # PKI Module
        _keys_dir = self.cfg.get('main', 'keys_dir')
        self.pki = crypto.RSAModule(_keys_dir)

        # This is socket.socket object while connected to server
        # Should be set to None as soon the connection is closed
        # or a socket error occurs.
        self.sock = None

        # Should hold the name of the server as long as there is a
        # valid connection to it.
        self.server_name = None

    def _close_socket(self):
        """Should be called after all socket errors."""
        if isinstance(self.sock, socket.socket):
            self.sock.close()
            if self.server_name:
                logger.info('- Closed connection with server: %s' % self.server_name)
        self.sock = None
        self.server_name = None

    def _get_hashing_delay(self):
        """Returns the hashing delay in seconds."""
        delay_msec = self.default_hashing_delay = self.cfg.getfloat('main', 'hashing_delay')
        delay_sec = delay_msec / 1000
        logger.debug('Hashing delay set to %.3f seconds' % delay_sec)
        return delay_sec

    def _run_backends(self):
        """Runs all collector backends and passes the yielded information
        through the hashing algorithm."""

        backends_conf_dir = self.cfg.get('main', 'backends_conf_dir')

        # Get a list of all backend paths
        backend_paths = []
        core_backends_dir = TinyIDS.backends.__path__[0]
        backend_paths.extend(glob.glob(os.path.join(core_backends_dir, '*.py')))
        extra_backends_dir = self.cfg.get('main', 'extra_backends_dir')
        backend_paths.extend(glob.glob(os.path.join(extra_backends_dir, '*.py')))

        if not backend_paths:
            raise NoBackendsToRun

        #
        user_defined_backend_list = self.cfg.getlist('main', 'tests')
        user_defined_backend_list_finished = [] # holds names of backends that have finished
        if user_defined_backend_list:
            logger.debug('Using user-defined list of backends')

        # Load all needed backends and store them in a list
        for backend_path in backend_paths:
            backend_name = os.path.basename(backend_path)[:-3]
            backend_dir = os.path.dirname(backend_path)
            if backend_name == '__init__':
                continue
            elif user_defined_backend_list:
                #print "checking user list"
                if not backend_name in user_defined_backend_list:
                    logger.debug('Skipping backend: %s' % backend_name)
                    continue
            # Load backend
            m = load_backend(backend_dir, backend_name)
            if not hasattr(m, 'CollectorBackend'):
                logger.warning('Skipping invalid backend: %s' % backend_path)
                continue

            backend_config_file = os.path.join(backends_conf_dir, m.__name__ + '.conf')
            b = m.CollectorBackend(config_path=backend_config_file)
            if not hasattr(b, 'collect'):
                logger.warning('Skipping invalid backend: %s' % backend_path)
                continue

            # Run backend
            logger.info('Processing backend: %s' % backend_name)
            for data in b.collect():
                self.hash_data(data)
            logger.info('- Complete')

            if user_defined_backend_list:
                # If a user-defined list of backends is used, add the name of
                # the backend that has just run to the list:
                # user_defined_backend_list_finished
                user_defined_backend_list_finished.append(backend_name)

        if user_defined_backend_list:
            invalid_user_defined_tests = []
            for test in user_defined_backend_list:
                if test not in user_defined_backend_list_finished:
                    invalid_user_defined_tests.append(test)
            if invalid_user_defined_tests:
                logger.warning('Invalid user-defined test(s): %s' % ', '.join(invalid_user_defined_tests))
                if not user_defined_backend_list_finished:
                    raise NoBackendsToRun

    def _get_server_canonical_name(self, server_name):
        """Returns the name of the server after stripping the 'server__' prefix'"""
        return server_name.split('__')[1]

    def _get_enabled_server_list(self):
        """Returns a list of servers that have been enabled in the
        client configuration."""
        enabled_servers = []
        for section in self.cfg.sections():
            if not section.startswith('server__'):
                continue
            server_name = self._get_server_canonical_name(section)
            if not self.cfg.has_option(section, 'host'):
                logger.warning('Misconfigured server: %s' % server_name)
                continue
            if self.cfg.has_option(section, 'enabled'):
                is_enabled = self.cfg.getboolean(section, 'enabled')
                if not is_enabled:
                    continue
                enabled_servers.append(section)
        return enabled_servers

    def _send(self, host, port, data):
        """Sends the command to a TinyIDS server."""
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.connect((host, port))
        logger.info('- Established connection to server: %s' % self.server_name)
        if self.debug_protocol:
            logger.debug('-> Sending command: %s' % data)
        if self.pki.public_key is not None:
            data = self.pki.encrypt(data)
            logger.info('- PKI: data encrypted')
        self.sock.send(data + self.cmd_end)
        logger.info('- Sent %s command to server: %s' % (self.command, self.server_name))

    def _get_server_response(self):
        """Receives a response from a TinyIDS server."""
        logger.info('- Awaiting server response...')
        response = self.sock.recv(self.max_response_len).strip()
        response = response.rstrip(self.cmd_end)
        logger.info('- Received response from server: %s' % self.server_name)
        if self.pki.public_key is not None:
            response = self.pki.verify(response)
            logger.info('- PKI: data verified')
        if self.debug_protocol:
            logger.debug('-> Received response: %s' % response.strip())
        return response.strip()

    def _check_command_status(self, response):
        if response.startswith('20'):
            logger.info('- RESULT: %s on %s: SUCCESS' % (self.command, self.server_name))
        else:
            logger.warning('- RESULT: %s on %s: FAILURE with error: %s' % (self.command, self.server_name, response))

    def _communicate(self, host, port, data):
        self._send(host, port, data)
        response = self._get_server_response()
        self._check_command_status(response)

    def _get_passphrase(self, msg):
        """Prompts the user for a passphrase."""
        data = ''
        while not data:
            data = getpass.getpass('%s: ' % msg)
        return data

    def _com_TEST(self, host, port):
        """
        Syntax: TEST <protocol_revision>
        """
        data = '%s %s' % (self.command, config.PROTOCOL_REVISION)
        self._communicate(host, port, data)

    def _com_CHECK(self, host, port):
        """
        Syntax: CHECK <hash>
        """
        data = '%s %s' % (self.command, self.get_checksum())
        self._communicate(host, port, data)

    def _com_UPDATE(self, host, port):
        """
        Syntax: UPDATE <hash> <passphrase>
        """
        passphrase = self._get_passphrase('Passphrase')
        data = '%s %s %s' % (self.command, self.get_checksum(), passphrase)
        self._communicate(host, port, data)

    def _com_DELETE(self, host, port):
        """
        Syntax: DELETE <passphrase>
        """
        passphrase = self._get_passphrase('Passphrase')
        data = '%s %s' % (self.command, passphrase)
        self._communicate(host, port, data)

    def _com_CHANGEPHRASE(self, host, port):
        """
        Syntax: CHANGEPHRASE <old_passphrase> <new_passphrase>
        """
        passphrase_old = self._get_passphrase('Old passphrase')
        while True:
            passphrase_new = self._get_passphrase('New passphrase')
            passphrase_new_confirm = self._get_passphrase('Confirm new passphrase')
            if passphrase_new == passphrase_new_confirm:
                break
            logger.error('passphrases do not match. try again...')
        data = '%s %s %s' % (self.command, passphrase_old, passphrase_new)
        self._communicate(host, port, data)

    # Public API

    def get_checksum(self):
        return self.hasher.hexdigest()

    def hash_data(self, data):
        """Passes data through the hashing algorithm."""
        self.hasher.update(data)
        time.sleep(self.default_hashing_delay)

    def run(self):
        """Main client method."""

        logger.info('Getting list of enabled servers')
        enabled_servers = self._get_enabled_server_list()
        if not enabled_servers:
            logger.warning('No servers configured. Shutting down...')
            self.client_close()
            return
        logger.info('Servers found. Proceeding with data hashing...')

        # Decide which method to execute
        func = getattr(self, '_com_%s' % self.command)

        # Tests are required to run only with the CHECK and UPDATE commands
        if self.command in ('CHECK', 'UPDATE'):
            logger.info('Hashing data. Please wait...')
            try:
                self._run_backends()
            except NoBackendsToRun:
                logger.warning('No valid backends. Shutting down...')
                self.client_close()
                return
            else:
                logger.info('Hashing complete')

        # Execute command on server
        logger.info('Preparing to contact servers')
        for server in enabled_servers:
            # server is in 'server__<name>' format (a section name)
            # Here we set self.server_name to the canonical server name
            self.server_name = self._get_server_canonical_name(server)
            logger.info('* Contacting server: %s' % self.server_name)

            # Get server settings
            host = self.cfg.get(server, 'host')
            port = config.DEFAULT_PORT
            if self.cfg.has_option(server, 'port'):
                port = self.cfg.getint(server, 'port')
            if self.cfg.has_option(server, 'public_key'):
                public_key_fname = self.cfg.get(server, 'public_key')
                if public_key_fname:
                    try:
                        self.pki.load_external_public_key(public_key_fname) # sets self.pki.public_key
                    except crypto.InvalidPublicKey:
                        logger.warning('- RESULT: %s on %s: FAILURE with PKI error: invalid server public key: %s' % (self.command, self.server_name, public_key_fname))
                        self.server_name = None
                        continue

            # Run command on the server
            try:
                func(host, port)
            except socket.error, (errno, strerror):
                logger.error('Connection error: \'%s\'' % strerror)
                logger.warning('Skipping server: %s' % self.server_name)
                self.server_name = None
            except crypto.DataEncryptionError:
                logger.warning('- RESULT: %s on %s: FAILURE with PKI error: could not encrypt data for server' % (self.command, self.server_name))
            except crypto.DataVerificationError:
                logger.warning('- RESULT: %s on %s: FAILURE with PKI error: could not verify response from server' % (self.command, self.server_name))

            self._close_socket()
            self.pki.reset()    # sets self.pki.public_key to None

        logger.info('Finished with servers')

        self.client_close()

    def client_close(self):
        logger.info('Client shutting down...')

<|endoftext|>


<commit_before>import triage_fits_files as tff
import ccd_characterization as ccd_char
from astropysics import ccd
from os import path
import pyfits
from datetime import datetime

combiner = ccd.ImageCombiner()

def master_frame(data, img_type, T, Terr, sample=None,combiner=None):
    copy_from_sample = ['xbinning', 'ybinning',
                        'xpixsz', 'ypixsz', 'exptime','filter']
    img = ccd.FitsImage(data)
    hdr = img.fitsfile[0].header
    hdr.update('imagetyp',img_type)
    now = datetime.utcnow()
    now = now.replace(microsecond=0)
    hdr.update('date', now.isoformat(),
               'Creation date of file')
    hdr.update('ccd-temp', T, 'Average temperature of CCD')
    hdr.update('temp-dev', Terr,
               'Standard deviation of CCD temperature')
    if combiner is not None:
        hdr.update('cmbn-mth',combiner.method,
                   'Combination method for producing master')

    if sample is not None:
        if not isinstance(sample, pyfits.Header):
            raise TypeError
        cards = sample.ascard
        for key in copy_from_sample:
            hdr.update(key,cards[key].value,cards[key].comment)
    return img

for currentDir in foo:
    keywords = ['imagetyp', 'exptime', 'filter', 'ccd-temp']
    image_collection = tff.ImageFileCollection(location=currentDir,
                                               keywords=keywords,
                                               info_file=None)
    images = image_collection.summary_info
    master_dark_files = images.where(images['imagetyp'] == 'MASTER DARK')
    all_flats = images.where(images['imagetyp'] == 'FLAT')
    exposure_times = set(all_flats['exptime'])
    print exposure_times
    for time in exposure_times:
        these_flats = all_flats.where(all_flats['exptime'] == time)
        flat_filter = these_flats['filter'][0]
        master_dark = master_dark_files.where(master_dark_files['exptime']==time)
        if not master_dark:
            print 'Sorry, no dark for the exposure %f, skipping....' %time
            continue
        master_dark = ccd.FitsImage(path.join(currentDir,master_dark['file'][0]))
        flats = []
        for flat_file in these_flats['file']:
            flat = ccd.FitsImage(path.join(currentDir,flat_file))
            flats.append(flat.data - master_dark.data)
        master_flat = combiner.combineImages(flats)
        avg_temp = these_flats['ccd-temp'].mean()
        temp_dev = these_flats['ccd-temp'].std()
        sample = pyfits.open(path.join(currentDir,these_flats['file'][0]))
        flat_im = master_frame(master_flat, 'MASTER FLAT', avg_temp,
                               temp_dev, sample=sample[0].header,
                               combiner=combiner)
        flat_fn = 'Master_Flat_%s_band.fit' % flat_filter
        flat_im.save(path.join(currentDir,flat_fn))
<commit_msg>Crude check that all flats of same exposure time have same filter<commit_after>import triage_fits_files as tff
import ccd_characterization as ccd_char
from astropysics import ccd
from os import path
import pyfits
from datetime import datetime

combiner = ccd.ImageCombiner()

def master_frame(data, img_type, T, Terr, sample=None,combiner=None):
    copy_from_sample = ['xbinning', 'ybinning',
                        'xpixsz', 'ypixsz', 'exptime','filter']
    img = ccd.FitsImage(data)
    hdr = img.fitsfile[0].header
    hdr.update('imagetyp',img_type)
    now = datetime.utcnow()
    now = now.replace(microsecond=0)
    hdr.update('date', now.isoformat(),
               'Creation date of file')
    hdr.update('ccd-temp', T, 'Average temperature of CCD')
    hdr.update('temp-dev', Terr,
               'Standard deviation of CCD temperature')
    if combiner is not None:
        hdr.update('cmbn-mth',combiner.method,
                   'Combination method for producing master')

    if sample is not None:
        if not isinstance(sample, pyfits.Header):
            raise TypeError
        cards = sample.ascard
        for key in copy_from_sample:
            hdr.update(key,cards[key].value,cards[key].comment)
    return img

for currentDir in foo:
    keywords = ['imagetyp', 'exptime', 'filter', 'ccd-temp']
    image_collection = tff.ImageFileCollection(location=currentDir,
                                               keywords=keywords,
                                               info_file=None)
    images = image_collection.summary_info
    master_dark_files = images.where(images['imagetyp'] == 'MASTER DARK')
    all_flats = images.where(images['imagetyp'] == 'FLAT')
    exposure_times = set(all_flats['exptime'])
    print exposure_times
    for time in exposure_times:
        these_flats = all_flats.where(all_flats['exptime'] == time)
        flat_filter = these_flats['filter'][0]
        same_filter = (these_flats['filter'] == flat_filter)
        if not same_filter.all():
            raise RuntimeError('Holy crap, my flats have mixed filters!')
        master_dark = master_dark_files.where(master_dark_files['exptime']==time)
        if not master_dark:
            print 'Sorry, no dark for the exposure %f, skipping....' %time
            continue
        master_dark = ccd.FitsImage(path.join(currentDir,master_dark['file'][0]))
        flats = []
        for flat_file in these_flats['file']:
            flat = ccd.FitsImage(path.join(currentDir,flat_file))
            flats.append(flat.data - master_dark.data)
        master_flat = combiner.combineImages(flats)
        avg_temp = these_flats['ccd-temp'].mean()
        temp_dev = these_flats['ccd-temp'].std()
        sample = pyfits.open(path.join(currentDir,these_flats['file'][0]))
        flat_im = master_frame(master_flat, 'MASTER FLAT', avg_temp,
                               temp_dev, sample=sample[0].header,
                               combiner=combiner)
        flat_fn = 'Master_Flat_%s_band.fit' % flat_filter
        flat_im.save(path.join(currentDir,flat_fn))
<|endoftext|>


<commit_before>from kivy.app import App
from kivy.uix.widget import Widget
from kivy.properties import *
from kivy.vector import Vector

class PongGame(Widget):
    pass

class PongBall(Widget):
    velocity_x = NumericProperty(0)
    velocity_y = NumericProperty(0)

    velocity = ReferenceListProperty(velocity_x, velocity_y)

    def move(self):
        self.pos = Vector(*self.velocity) + self.pos

class Tackle(App):
    def build(self):
        return PongGame()

if __name__ == '__main__':
    Tackle().run()
<commit_msg>module, entry point<commit_after>from kivy.app import App
from kivy.uix.widget import Widget

class Ticket(Widget):
    pass

class Manifest(Widget):
    pass

class Tackle(App):
    def build(self):
        pass

if __name__ == '__main__':
    Tackle().run()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import, division

import datetime
import math

from lxml import html
import requests

class GithubMeta(object):
    def __init__(self, response):
        self.tree = html.fromstring(response.text)

    def _get_num(self, css_expression, index):
        return int(self.tree.cssselect(css_expression)[index].text.strip())

    @property
    def watchers(self):
        return self._get_num('.social-count', 0)

    @property
    def forks(self):
        return self._get_num('.social-count', -1)

    @property
    def last_commit(self):
        commit_time = self.tree.cssselect('time')[0].get('datetime')
        return _parse_date(commit_time)

    @property
    def contributors(self):
        return self._get_num('.text-emphasized', -1)

    @property
    def commits(self):
        return self._get_num('.text-emphasized', 0)

    @property
    def issues(self):
        return self._get_num('.counter', 0)

    @property
    def pull_requests(self):
        return self._get_num('.counter', -1)

def get_first_commit(url):
    # Steal this idea from https://github.com/wong2/first-commit.
    commits = GithubMeta(requests.get(url)).commits
    first_commit_page = int(math.ceil(commits / 35))
    page_url = "{0}/commits?page={1}".format(url, first_commit_page)
    tree = html.fromstring(requests.get(page_url).text)
    first_commit_time = tree.cssselect('time')[-1].get('datetime')
    return _parse_date(first_commit_time)

def get_development_activity(url):
    owner, repo = url.split('/')[-2:]
    url = 'https://api.github.com/repos/{0}/{1}/commits'.format(owner, repo)
    response = requests.get(url)
    epoch = datetime.datetime.utcfromtimestamp(0)
    deltas = [_parse_date(commit['commit']['committer']['date']) - epoch
              for commit in response.json()]
    average_delta = (
        sum(delta.total_seconds() for delta in deltas) // len(response.json()))
    average_date = epoch + datetime.timedelta(seconds=average_delta)
    delta_of_day = (datetime.datetime.now() - average_date).days
    if delta_of_day in range(0, 7+1):
        return 'Very active'
    elif delta_of_day in range(8, 31+1):
        return 'Active'
    elif delta_of_day in range(32, 365+1):
        return 'Less Active'
    else:
        return 'Inactive'

def _parse_date(date_string):
    return datetime.datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ')
<commit_msg>Fix the bug that commits num may hava a comma<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import, division

import datetime
import math

from lxml import html
import requests

class GithubMeta(object):
    def __init__(self, response):
        self.tree = html.fromstring(response.text)

    def _custom_int(self, string_number):
        # 1999 commits will be 1,999 commits on github.
        if ',' in string_number:
            return int(string_number.replace(',', ''))
        return int(string_number)

    def _get_num(self, css_expression, index):
        number = self._custom_int(
            self.tree.cssselect(css_expression)[index].text.strip())
        return number

    @property
    def watchers(self):
        return self._get_num('.social-count', 0)

    @property
    def forks(self):
        return self._get_num('.social-count', -1)

    @property
    def last_commit(self):
        commit_time = self.tree.cssselect('time')[0].get('datetime')
        return _parse_date(commit_time)

    @property
    def contributors(self):
        return self._get_num('.text-emphasized', -1)

    @property
    def commits(self):
        return self._get_num('.text-emphasized', 0)

    @property
    def issues(self):
        return self._get_num('.counter', 0)

    @property
    def pull_requests(self):
        return self._get_num('.counter', -1)

def get_first_commit(url):
    # Steal this idea from https://github.com/wong2/first-commit.
    commits = GithubMeta(requests.get(url)).commits
    first_commit_page = int(math.ceil(commits / 35))
    page_url = "{0}/commits?page={1}".format(url, first_commit_page)
    tree = html.fromstring(requests.get(page_url).text)
    first_commit_time = tree.cssselect('time')[-1].get('datetime')
    return _parse_date(first_commit_time)

def get_development_activity(url):
    owner, repo = url.split('/')[-2:]
    url = 'https://api.github.com/repos/{0}/{1}/commits'.format(owner, repo)
    response = requests.get(url)
    epoch = datetime.datetime.utcfromtimestamp(0)
    deltas = [_parse_date(commit['commit']['committer']['date']) - epoch
              for commit in response.json()]
    average_delta = (
        sum(delta.total_seconds() for delta in deltas) // len(response.json()))
    average_date = epoch + datetime.timedelta(seconds=average_delta)
    delta_of_day = (datetime.datetime.now() - average_date).days
    if delta_of_day in range(0, 7+1):
        return 'Very active'
    elif delta_of_day in range(8, 31+1):
        return 'Active'
    elif delta_of_day in range(32, 365+1):
        return 'Less Active'
    else:
        return 'Inactive'

def _parse_date(date_string):
    return datetime.datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ')
<|endoftext|>