<commit_before>'''
Created on 23.01.2015

@author: marscher
'''
import tempfile
import unittest

import mdtraj
import pkg_resources

from pyemma.coordinates.coordinate_transformation.feature_reader import FeatureReader
from pyemma.coordinates.coordinate_transformation.featurizer import Featurizer
from pyemma.coordinates.coordinate_transformation.transformer import Transformer
import numpy as np

def map_return_input(traj):
    return traj.xyz

class MemoryStorage(Transformer):

    def __init__(self, source, chunksize=100, lag=0):
        Transformer.__init__(self)
        self.lagged_chunks = []
        self.final = None
        self.data_producer = source
        self._param_finished = False

    def add_chunk(self, X, itraj, t, first_chunk, last_chunk_in_traj, last_chunk, ipass, Y=None):
        self.lagged_chunks.append(Y)
        # print t
        if last_chunk:
            # TODO: fix shape
            new_shape = ()
            # .reshape((2, 1000, 2, 3))
            self.final = np.array(self.lagged_chunks)
            self._param_finished = True

class TestFeatureReader(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        c = super(TestFeatureReader, cls).setUpClass()
        # create a fake trajectory which has 2 atoms and coordinates are just a range
        # over all frames.
        cls.trajfile = tempfile.mktemp('.xtc')
        xyz = np.arange(1000 * 2 * 3).reshape((1000, 2, 3))
        cls.topfile = pkg_resources.resource_filename(
            'test_featurereader', 'data/test.pdb')
        t = mdtraj.load(cls.topfile)
        t.xyz = xyz
        t.time = np.arange(1000)
        t.save(cls.trajfile)
        return c

    def testTimeLaggedAccess(self):
        # each frame has 2 atoms with 3 coords = 6 coords per frame.
        # coords are sequential through all frames and start with 0.
        trajfiles = [self.trajfile]

        # so first lagged frame is 18, since 3 frames a 6 coords are skipped
        lags = [1, 3, 5, 10, 29, 100]

        chunksizes = [2, 3, 7, 11, 10, 101]

        for lag in lags:
            for chunksize in chunksizes:
                if chunksize <= lag:
                    continue
                print "lag: %s; chunksize: %s " % (lag, chunksize)
                f = Featurizer(self.topfile)
                f.map = map_return_input

                reader = FeatureReader(trajfiles, self.topfile, f)

                m = MemoryStorage(reader, lag=lag)

                for t in [f, reader, m]:
                    t.chunksize = chunksize

                m.parametrize()

                frist_lagged_chunk = m.lagged_chunks[0]
                self.assertEqual(frist_lagged_chunk.shape[0], chunksize)

                n_coords = 3 * 2 * 10 * chunksize
                coords = np.arange(n_coords).reshape((n_coords / 6, 2, 3))

                # first lagged chunk should start at lag and stop at chunksize +
                # lag
                np.testing.assert_allclose(
                    frist_lagged_chunk, coords[lag:chunksize + lag])
                # TODO: check last lagged frame

if __name__ == "__main__":
    unittest.main()
<commit_msg>fix test<commit_after>'''
Created on 23.01.2015

@author: marscher
'''
import tempfile
import unittest

import mdtraj
import pkg_resources

from pyemma.coordinates.coordinate_transformation.feature_reader import FeatureReader
from pyemma.coordinates.coordinate_transformation.featurizer import Featurizer
from pyemma.coordinates.coordinate_transformation.transformer import Transformer
import numpy as np

def map_return_input(traj):
    return traj.xyz

class MemoryStorage(Transformer):

    def __init__(self, source, chunksize=100, lag=0):
        Transformer.__init__(self, chunksize, lag)

        self.lagged_chunks = []
        self.final = None
        self.data_producer = source
        self._param_finished = False

    def add_chunk(self, X, itraj, t, first_chunk, last_chunk_in_traj, last_chunk, ipass, Y=None):
        self.lagged_chunks.append(Y)
        # print t
        if last_chunk:
            # TODO: fix shape
            new_shape = ()
            # .reshape((2, 1000, 2, 3))
            self.final = np.array(self.lagged_chunks)
            self._param_finished = True

class TestFeatureReader(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        c = super(TestFeatureReader, cls).setUpClass()
        # create a fake trajectory which has 2 atoms and coordinates are just a range
        # over all frames.
        cls.trajfile = tempfile.mktemp('.xtc')
        xyz = np.arange(1000 * 2 * 3).reshape((1000, 2, 3))
        cls.topfile = pkg_resources.resource_filename(
            'test_featurereader', 'data/test.pdb')
        t = mdtraj.load(cls.topfile)
        t.xyz = xyz
        t.time = np.arange(1000)
        t.save(cls.trajfile)
        return c

    def testTimeLaggedAccess(self):
        # each frame has 2 atoms with 3 coords = 6 coords per frame.
        # coords are sequential through all frames and start with 0.
        trajfiles = [self.trajfile]

        # so first lagged frame is 18, since 3 frames a 6 coords are skipped
        lags = [1, 3, 5, 10, 29, 100]

        chunksizes = [2, 3, 7, 11, 10, 101]

        for lag in lags:
            for chunksize in chunksizes:
                if chunksize <= lag:
                    continue
                print "lag: %s; chunksize: %s " % (lag, chunksize)
                f = Featurizer(self.topfile)
                f.map = map_return_input

                reader = FeatureReader(trajfiles, self.topfile, f)

                m = MemoryStorage(reader, lag=lag)
                chain = [f, reader, m]
                for t in chain:
                    t.chunksize = chunksize

                m.parametrize()

                frist_lagged_chunk = m.lagged_chunks[0]
                self.assertEqual(frist_lagged_chunk.shape[0], chunksize)

                n_coords = 3 * 2 * 10 * chunksize
                coords = np.arange(n_coords).reshape((n_coords / 6, 2, 3))

                # first lagged chunk should start at lag and stop at chunksize +
                # lag
                np.testing.assert_allclose(
                    frist_lagged_chunk, coords[lag:chunksize + lag])
                # TODO: check last lagged frame

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>class Clock:

    def __init__(self, hour: int, minute: int):
        self.minute = minute % 60
        overflow = minute // 60
        self.hour = (hour + overflow) % 24

    def __repr__(self) -> str:
        return f'{str(self.hour).rjust(2, "0")}:{str(self.minute).rjust(2, "0")}'

    def __eq__(self, other) -> bool:
        if isinstance(other, Clock):
            return self.hour == other.hour and self.minute == other.minute
        return False

    def __add__(self, minutes: int):
        total_minutes = self.minute + minutes
        self.minute = total_minutes % 60
        overflow = total_minutes // 60
        self.hour = (self.hour + overflow) % 24
        return self

    def __sub__(self, minutes: int):
        total_minutes = self.minute - minutes
        self.minute = total_minutes % 60
        overflow = total_minutes // 60
        self.hour = (self.hour + overflow) % 24
        return self
<commit_msg>Refactoring the initial solution of the exercise Clock from Exercism in the Python Track<commit_after>class Clock:
    def __init__(self, hour: int, minute: int):
        self.minute = minute % 60
        overflow = minute // 60
        self.hour = (hour + overflow) % 24

    def __repr__(self) -> str:
        return f'{str(self.hour).rjust(2, "0")}:{str(self.minute).rjust(2, "0")}'

    def __eq__(self, other) -> bool:
        if isinstance(other, Clock):
            return self.hour == other.hour and self.minute == other.minute
        return False

    def __add__(self, minutes: int):
        return Clock(self.hour, self.minute + minutes)

    def __sub__(self, minutes: int):
        return Clock(self.hour, self.minute - minutes)
<|endoftext|>


<commit_before># python 3
import os

import pandas
import re

import numpy
from threading import Thread
import matplotlib
from matplotlib import pyplot

matplotlib.style.use('ggplot')
pyplot.interactive(False)

def get_speed_stats(speed_data_path):
    df = pandas.read_csv(speed_data_path, sep=',', thousands=',')
    try:
        node_nr = re.search('RSUExampleScenario.(.+?).veinsmobility.speed', df.columns[1]).group(1)
    except AttributeError:
        node_nr = '??'  # apply your error handling
    df.columns = ['time', 'speed']
    mean = df['speed'].mean()
    std = df['speed'].std()

    return (node_nr, mean, std)

def analysis_speed():

    per10_speedpath = os.path.abspath('.\PER1.0_100Demand\speed_data')
    per00_speedpath = os.path.abspath('.\PER0.0_100Demand_5secs\speed_data')

    per10_df = pandas.DataFrame(columns=['node', 'mean_speed', 'std'])
    per00_df = pandas.DataFrame(columns=['node', 'mean_speed', 'std'])

    for file in os.listdir(per10_speedpath):
        stats = get_speed_stats(os.path.join(per10_speedpath, file))
        per10_df.loc[len(per10_df)] = stats

    for file in os.listdir(per00_speedpath):
        stats = get_speed_stats(os.path.join(per00_speedpath, file))
        per00_df.loc[len(per00_df)] = stats

    per00_df.set_index(['node'], inplace=True)
    per10_df.set_index(['node'], inplace=True)

    per00_df.to_csv('speed_stats_per00.csv', encoding='utf-8')
    per10_df.to_csv('speed_stats_per10.csv', encoding='utf-8')

    #fig, axes = pyplot.subplots(nrows=1, ncols=2)
    #per00_df['mean_speed'].plot.hist(ax=axes[0], color='DarkBlue', bins=10)
    #per10_df['mean_speed'].plot.hist(ax=axes[1], color='DarkRed', bins=10)
    #pyplot.show()

def analysis_distance():
    per00_df = pandas.read_csv('./PER0.0_100Demand_5secs_clean.csv', sep=';')
    per10_df = pandas.read_csv('./PER1.0_100Demand_clean.csv', sep=';')

    per00_df.set_index(['Module'], inplace=True)
    per10_df.set_index(['Module'], inplace=True)

    # fig, axes = pyplot.subplots(nrows=1, ncols=2)
    axes = per00_df['totalDistance'].plot.hist(color='DarkBlue', alpha=0.75, bins=15, cumulative=False)
    axes = per10_df['totalDistance'].plot.hist(ax=axes, color='DarkRed', alpha=0.75, cumulative=False, sharex=True)
    handles, labels = axes.get_legend_handles_labels()
    axes.legend(handles=handles, labels=['PER 0.0', 'PER 1.0'])
    pyplot.show()

def analysis_time():
    per00_df = pandas.read_csv('./PER0.0_100Demand_5secs_clean.csv', sep=';')
    per10_df = pandas.read_csv('./PER1.0_100Demand_clean.csv', sep=';')

    per00_df.set_index(['Module'], inplace=True)
    per10_df.set_index(['Module'], inplace=True)

    bins = numpy.linspace(0, 1000, 50)

    pyplot.hist(per00_df['totalTime'], bins, alpha=0.75, label='PER 0.0')
    pyplot.hist(per10_df['totalTime'], bins, alpha=0.75, label='PER 1.0')
    pyplot.legend(loc='upper right')
    pyplot.show()

if __name__ == '__main__':
    #analysis_speed()
    #analysis_distance()
    analysis_time()
<commit_msg>normalized histograms<commit_after># python 3
import os

import pandas
import re

import numpy
from threading import Thread
import matplotlib
from matplotlib import pyplot

matplotlib.style.use('ggplot')
pyplot.interactive(False)

def get_speed_stats(speed_data_path):
    df = pandas.read_csv(speed_data_path, sep=',', thousands=',')
    try:
        node_nr = re.search('RSUExampleScenario.(.+?).veinsmobility.speed', df.columns[1]).group(1)
    except AttributeError:
        node_nr = '??'  # apply your error handling
    df.columns = ['time', 'speed']
    mean = df['speed'].mean()
    std = df['speed'].std()

    return (node_nr, mean, std)

def store_speed_data_csv():

    per10_speedpath = os.path.abspath('.\PER1.0_100Demand\speed_data')
    per00_speedpath = os.path.abspath('.\PER0.0_100Demand_5secs\speed_data')

    per10_df = pandas.DataFrame(columns=['node', 'mean_speed', 'std'])
    per00_df = pandas.DataFrame(columns=['node', 'mean_speed', 'std'])

    for file in os.listdir(per10_speedpath):
        stats = get_speed_stats(os.path.join(per10_speedpath, file))
        per10_df.loc[len(per10_df)] = stats

    for file in os.listdir(per00_speedpath):
        stats = get_speed_stats(os.path.join(per00_speedpath, file))
        per00_df.loc[len(per00_df)] = stats

    per00_df.set_index(['node'], inplace=True)
    per10_df.set_index(['node'], inplace=True)

    per00_df.to_csv('speed_stats_per00.csv', encoding='utf-8')
    per10_df.to_csv('speed_stats_per10.csv', encoding='utf-8')

def analysis_speed():
    per00_df = pandas.read_csv('speed_stats_per00.csv')
    per10_df = pandas.read_csv('speed_stats_per10.csv')

    per00_df.set_index(['node'], inplace=True)
    per10_df.set_index(['node'], inplace=True)

    bins = numpy.linspace(0, per00_df['mean_speed'].max(), 50)

    pyplot.hist(per00_df['mean_speed'], bins, alpha=0.75, label='PER 0.0', normed=True)
    pyplot.hist(per10_df['mean_speed'], bins, alpha=0.75, label='PER 1.0', normed=True)
    pyplot.legend(loc='upper right')
    pyplot.title('Histograma normalizado de velocidades promedio')
    pyplot.show()

    #fig, axes = pyplot.subplots(nrows=1, ncols=2)
    #per00_df['mean_speed'].plot.hist(ax=axes[0], color='DarkBlue', bins=10)
    #per10_df['mean_speed'].plot.hist(ax=axes[1], color='DarkRed', bins=10)
    #pyplot.show()

def analysis_distance():
    per00_df = pandas.read_csv('./PER0.0_100Demand_5secs_clean.csv', sep=';')
    per10_df = pandas.read_csv('./PER1.0_100Demand_clean.csv', sep=';')

    per00_df.set_index(['Module'], inplace=True)
    per10_df.set_index(['Module'], inplace=True)

    bins = numpy.linspace(0, per00_df['totalDistance'].max(), 50)

    pyplot.hist(per00_df['totalDistance'], bins, alpha=0.75, label='PER 0.0', normed=True)
    pyplot.hist(per10_df['totalDistance'], bins, alpha=0.75, label='PER 1.0', normed=True)
    pyplot.legend(loc='upper right')
    pyplot.title('Histograma normalizado de distancias totales')
    pyplot.show()

def analysis_time():
    per00_df = pandas.read_csv('./PER0.0_100Demand_5secs_clean.csv', sep=';')
    per10_df = pandas.read_csv('./PER1.0_100Demand_clean.csv', sep=';')

    per00_df.set_index(['Module'], inplace=True)
    per10_df.set_index(['Module'], inplace=True)

    bins = numpy.linspace(0, 1000, 50)

    pyplot.hist(per00_df['totalTime'], bins, alpha=0.75, label='PER 0.0', normed=True)
    pyplot.hist(per10_df['totalTime'], bins, alpha=0.75, label='PER 1.0', normed=True)
    pyplot.legend(loc='upper right')
    pyplot.title('Histograma normalizado de tiempos totales')
    pyplot.show()

if __name__ == '__main__':
    analysis_speed()
    analysis_distance()
    analysis_time()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Class that stores chess moves.
Destination, status and piece making move are required
to initialize Move.

:type: end_loc: Location
:type: piece: Piece
:type: status: int

Copyright © 2016 Aubhro Sengupta. All rights reserved.
"""

import warnings

from .location import Location

class Move:
    def __init__(self, end_loc, piece, status,
                 start_rank=None,
                 start_file=None,
                 promoted_to_piece=None):
        """
        Constructor to create move using object Location

        :type: end_loc: Location
        :type: piece: Piece
        :type: status: int
        """
        if not self.on_board():
            raise Exception("Location of move must be on the board")

        self._end_loc = end_loc
        self._status = status
        self.piece = piece
        self.color = piece.color
        self.start_rank = start_rank
        self.start_file = start_file
        self.promoted_to_piece = promoted_to_piece

    @property
    def end_loc(self):
        return self._end_loc

    @end_loc.setter
    def end_loc(self, end_loc):
        warnings.warn("Mutating end_loc attribute in Move should not be done")
        self._end_loc = end_loc

    @property
    def status(self):
        return self._status

    @status.setter
    def status(self, status):
        warnings.warn("Mutating status attribute in Move should not be done")
        self._status = status

    def __key(self):
        return self._end_loc, self.piece, self._status, \
               self.start_rank, self.start_file, self.promoted_to_piece

    def __hash__(self):
        return hash(self.__key())

    def __eq__(self, other):
        """
        Finds if move is same move as this one.
        :type: other: Move
        """
        if not isinstance(other, self.__class__):
            raise TypeError("Cannot compare other types with Move")

        for index, item in enumerate(self.__key()):
            if not self._check_equals_or_none(item, other.__key()[index]):
                return False

        return True

    @staticmethod
    def _check_equals_or_none(var1, var2):
        """
        If either is None then return True,
        otherwise compare them and return
        if they are equal.

        :type: var1: object
        :type: var2: object
        :rtype: bool
        """
        if var1 is None or var2 is None:
            return True

        return var1 == var2

    def __ne__(self, other):
        return not self.__eq__(other)

    def __str__(self):
        """
        Finds string representation in long algebraic noatation

        :rtype: str
        """
        move_str = str(Location(self.start_rank, self.start_file)) + str(self._end_loc)

        if self.promoted_to_piece is not None:
            move_str += str(self.promoted_to_piece)

        return move_str

    @property
    def start_loc(self):
        """
        Finds start Location of move if specified.
        Otherwise throws an AttributeError

        :rtype: Location
        """
        if self.start_rank is None or self.start_file is None:
            raise AttributeError("start_rank and start_file must be specified")

        return Location(self.start_rank, self.start_file)

    def on_board(self):
        """
        Determines whether end Location of this Move
        is on the board.

        :rtype: bool
        """
        return self._end_loc.on_board()

    def would_move_be_promotion(self):
        """
        Finds if move from current location would be a promotion
        """
        return (self._end_loc.rank == 0 and not self.color) or \
                (self._end_loc.rank == 7 and self.color)
<commit_msg>Fix Move intializing bug<commit_after># -*- coding: utf-8 -*-

"""
Class that stores chess moves.
Destination, status and piece making move are required
to initialize Move.

:type: end_loc: Location
:type: piece: Piece
:type: status: int

Copyright © 2016 Aubhro Sengupta. All rights reserved.
"""

import warnings

from .location import Location

class Move:
    def __init__(self, end_loc, piece, status,
                 start_rank=None,
                 start_file=None,
                 promoted_to_piece=None):
        """
        Constructor to create move using object Location

        :type: end_loc: Location
        :type: piece: Piece
        :type: status: int
        """
        self._end_loc = end_loc
        self._status = status
        self.piece = piece
        self.color = piece.color
        self.start_rank = start_rank
        self.start_file = start_file
        self.promoted_to_piece = promoted_to_piece

        if not self.on_board():
            raise Exception("Location of move must be on the board")

    @property
    def end_loc(self):
        return self._end_loc

    @end_loc.setter
    def end_loc(self, end_loc):
        warnings.warn("Mutating end_loc attribute in Move should not be done")
        self._end_loc = end_loc

    @property
    def status(self):
        return self._status

    @status.setter
    def status(self, status):
        warnings.warn("Mutating status attribute in Move should not be done")
        self._status = status

    def __key(self):
        return self._end_loc, self.piece, self._status, \
               self.start_rank, self.start_file, self.promoted_to_piece

    def __hash__(self):
        return hash(self.__key())

    def __eq__(self, other):
        """
        Finds if move is same move as this one.
        :type: other: Move
        """
        if not isinstance(other, self.__class__):
            raise TypeError("Cannot compare other types with Move")

        for index, item in enumerate(self.__key()):
            if not self._check_equals_or_none(item, other.__key()[index]):
                return False

        return True

    @staticmethod
    def _check_equals_or_none(var1, var2):
        """
        If either is None then return True,
        otherwise compare them and return
        if they are equal.

        :type: var1: object
        :type: var2: object
        :rtype: bool
        """
        if var1 is None or var2 is None:
            return True

        return var1 == var2

    def __ne__(self, other):
        return not self.__eq__(other)

    def __str__(self):
        """
        Finds string representation in long algebraic noatation

        :rtype: str
        """
        move_str = str(Location(self.start_rank, self.start_file)) + str(self._end_loc)

        if self.promoted_to_piece is not None:
            move_str += str(self.promoted_to_piece)

        return move_str

    @property
    def start_loc(self):
        """
        Finds start Location of move if specified.
        Otherwise throws an AttributeError

        :rtype: Location
        """
        if self.start_rank is None or self.start_file is None:
            raise AttributeError("start_rank and start_file must be specified")

        return Location(self.start_rank, self.start_file)

    def on_board(self):
        """
        Determines whether end Location of this Move
        is on the board.

        :rtype: bool
        """
        return self.end_loc.on_board()

    def would_move_be_promotion(self):
        """
        Finds if move from current location would be a promotion
        """
        return (self._end_loc.rank == 0 and not self.color) or \
                (self._end_loc.rank == 7 and self.color)
<|endoftext|>


<commit_before>__author__ = 'Bohdan Mushkevych'

import time
import os

import psutil

from system.repeat_timer import RepeatTimer
from settings import settings

class FootprintCalculator(object):
    def __init__(self):
        self.pid = os.getpid()

    def group(self, number):
        """ method formats number and inserts thousands separators """
        s = '%d' % number
        groups = []
        while s and s[-1].isdigit():
            groups.append(s[-3:])
            s = s[:-3]
        return s + '\''.join(reversed(groups))

    def get_snapshot_as_list(self):
        ps = psutil.Process(self.pid)
        return (self.group(ps.get_memory_info()[0]),
                self.group(ps.get_memory_info()[1]),
                '%02d' % ps.get_cpu_percent(),
                self.group(psutil.phymem_usage().free),
                self.group(psutil.virtmem_usage().free))

    def get_snapshot(self):
        resp = 'Footprint: RSS=%r VMS=%r CPU=%r; Available: PHYS=%r VIRT=%r' % self.get_snapshot_as_list()
        return resp

class Tracker(object):
    def __init__(self, name):
        self.name = name
        self.per_24h = 0
        self.per_tick = 0

    def increment(self, delta=1):
        self.per_24h += delta
        self.per_tick += delta

    def reset_tick(self):
        self.per_tick = 0

    def reset_24h(self):
        self.per_24h = 0

class TrackerPair(object):
    def __init__(self, name, success='Success', failure='Failure'):
        self.name = name
        self.success = Tracker(success)
        self.failure = Tracker(failure)

    def increment_success(self, delta=1):
        self.success.increment(delta)

    def increment_failure(self, delta=1):
        self.failure.increment(delta)

    def reset_tick(self):
        self.success.reset_tick()
        self.failure.reset_tick()

    def reset_24h(self):
        self.success.reset_24h()
        self.failure.reset_24h()

    def to_string(self, tick_interval_seconds, show_header=True):
        header = self.name + ' : ' + self.failure.name + '/' + self.success.name + '.' if show_header else ''
        return header + 'In last {0:d} seconds: {1:d}/{2:d}. In last 24 hours: {3:d}/{4:d}'.format(
            tick_interval_seconds,
            self.failure.per_tick,
            self.success.per_tick,
            self.failure.per_24h,
            self.success.per_24h)

class TickerThread(object):
    SECONDS_IN_24_HOURS = 86400
    TICKS_BETWEEN_FOOTPRINTS = 10

    def __init__(self, logger):
        self.logger = logger
        self.trackers = dict()
        self.interval = settings['perf_ticker_interval']
        self.mark_24_hours = time.time()
        self.mark_footprint = time.time()
        self.footprint = FootprintCalculator()
        self.timer = RepeatTimer(self.interval, self._run_tick_thread)

    def add_tracker(self, tracker):
        self.trackers[tracker.name] = tracker

    def get_tracker(self, name):
        return self.trackers[name]

    def start(self):
        self.timer.start()

    def cancel(self):
        self.timer.cancel()

    def is_alive(self):
        return self.timer.is_alive()

    def _print_footprint(self):
        if time.time() - self.mark_footprint > self.TICKS_BETWEEN_FOOTPRINTS * self.interval:
            self.logger.info(self.footprint.get_snapshot())
            self.mark_footprint = time.time()

    def _run_tick_thread(self):
        self._print_footprint()

        current_time = time.time()
        do_24h_reset = current_time - self.mark_24_hours > self.SECONDS_IN_24_HOURS
        if do_24h_reset:
            self.mark_24_hours = current_time

        tracker_outputs = []
        for tracker_name, tracker in self.trackers.iteritems():
            tracker_outputs.append(tracker.to_string(self.interval))
            tracker.reset_tick()
            if do_24h_reset:
                tracker.reset_24h()

        self.logger.info('\n'.join(tracker_outputs))

class SimpleTracker(TickerThread):
    TRACKER_PERFORMANCE = 'Performance'

    def __init__(self, logger):
        super(SimpleTracker, self).__init__(logger)
        self.add_tracker(TrackerPair(self.TRACKER_PERFORMANCE))

    @property
    def tracker(self):
        return self.get_tracker(self.TRACKER_PERFORMANCE)
<commit_msg>- addressing improper success/failure labeling<commit_after>__author__ = 'Bohdan Mushkevych'

import time
import os

import psutil

from system.repeat_timer import RepeatTimer
from settings import settings

class FootprintCalculator(object):
    def __init__(self):
        self.pid = os.getpid()

    def group(self, number):
        """ method formats number and inserts thousands separators """
        s = '%d' % number
        groups = []
        while s and s[-1].isdigit():
            groups.append(s[-3:])
            s = s[:-3]
        return s + '\''.join(reversed(groups))

    def get_snapshot_as_list(self):
        ps = psutil.Process(self.pid)
        return (self.group(ps.get_memory_info()[0]),
                self.group(ps.get_memory_info()[1]),
                '%02d' % ps.get_cpu_percent(),
                self.group(psutil.phymem_usage().free),
                self.group(psutil.virtmem_usage().free))

    def get_snapshot(self):
        resp = 'Footprint: RSS=%r VMS=%r CPU=%r; Available: PHYS=%r VIRT=%r' % self.get_snapshot_as_list()
        return resp

class Tracker(object):
    def __init__(self, name):
        self.name = name
        self.per_24h = 0
        self.per_tick = 0

    def increment(self, delta=1):
        self.per_24h += delta
        self.per_tick += delta

    def reset_tick(self):
        self.per_tick = 0

    def reset_24h(self):
        self.per_24h = 0

class TrackerPair(object):
    def __init__(self, name, success='Success', failure='Failure'):
        self.name = name
        self.success = Tracker(success)
        self.failure = Tracker(failure)

    def increment_success(self, delta=1):
        self.success.increment(delta)

    def increment_failure(self, delta=1):
        self.failure.increment(delta)

    def reset_tick(self):
        self.success.reset_tick()
        self.failure.reset_tick()

    def reset_24h(self):
        self.success.reset_24h()
        self.failure.reset_24h()

    def to_string(self, tick_interval_seconds, show_header=True):
        header = self.name + ' : ' + self.success.name + '/' + self.failure.name + '.' if show_header else ''
        return header + 'In last {0:d} seconds: {1:d}/{2:d}. In last 24 hours: {3:d}/{4:d}'.format(
            tick_interval_seconds,
            self.success.per_tick,
            self.failure.per_tick,
            self.success.per_24h,
            self.failure.per_24h)

class TickerThread(object):
    SECONDS_IN_24_HOURS = 86400
    TICKS_BETWEEN_FOOTPRINTS = 10

    def __init__(self, logger):
        self.logger = logger
        self.trackers = dict()
        self.interval = settings['perf_ticker_interval']
        self.mark_24_hours = time.time()
        self.mark_footprint = time.time()
        self.footprint = FootprintCalculator()
        self.timer = RepeatTimer(self.interval, self._run_tick_thread)

    def add_tracker(self, tracker):
        self.trackers[tracker.name] = tracker

    def get_tracker(self, name):
        return self.trackers[name]

    def start(self):
        self.timer.start()

    def cancel(self):
        self.timer.cancel()

    def is_alive(self):
        return self.timer.is_alive()

    def _print_footprint(self):
        if time.time() - self.mark_footprint > self.TICKS_BETWEEN_FOOTPRINTS * self.interval:
            self.logger.info(self.footprint.get_snapshot())
            self.mark_footprint = time.time()

    def _run_tick_thread(self):
        self._print_footprint()

        current_time = time.time()
        do_24h_reset = current_time - self.mark_24_hours > self.SECONDS_IN_24_HOURS
        if do_24h_reset:
            self.mark_24_hours = current_time

        tracker_outputs = []
        for tracker_name, tracker in self.trackers.iteritems():
            tracker_outputs.append(tracker.to_string(self.interval))
            tracker.reset_tick()
            if do_24h_reset:
                tracker.reset_24h()

        self.logger.info('\n'.join(tracker_outputs))

class SimpleTracker(TickerThread):
    TRACKER_PERFORMANCE = 'Performance'

    def __init__(self, logger):
        super(SimpleTracker, self).__init__(logger)
        self.add_tracker(TrackerPair(self.TRACKER_PERFORMANCE))

    @property
    def tracker(self):
        return self.get_tracker(self.TRACKER_PERFORMANCE)
<|endoftext|>


<commit_before># Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import unittest2

from happybase import Connection

TABLE_NAME = 'table-name'
ALT_TABLE_NAME = 'other-table'
TTL_DAY = 24 * 60 * 60
COL_FAM1 = 'cf1'
COL_FAM2 = 'cf2'
COL_FAM3 = 'cf3'
FAMILIES = {
    COL_FAM1: {'max_versions': 10},
    COL_FAM2: {'max_versions': 1, 'time_to_live': TTL_DAY},
    COL_FAM3: {},  # use defaults
}
ROW_KEY1 = 'row-key1'
ROW_KEY2 = 'row-key2'
COL1 = COL_FAM1 + ':qual1'
COL2 = COL_FAM1 + ':qual2'
COL3 = COL_FAM2 + ':qual1'

class Config(object):
    """Simple namespace for holding test globals."""

    connection = None
    table = None

# BEGIN: Backend dependent values.
IGNORE_TTL = True

def get_connection():
    if Config.connection is None:
        Config.connection = Connection()
    return Config.connection
#   END: Backend dependent values.

def get_table():
    if Config.table is None:
        connection = get_connection()
        Config.table = connection.table(TABLE_NAME)
    return Config.table

def setUpModule():
    if Config.table is None:
        connection = get_connection()
        if TABLE_NAME not in connection.tables():
            connection.create_table(TABLE_NAME, FAMILIES)

def tearDownModule():
    # connection.delete_table(TABLE_NAME, disable=True)
    pass

class TestConnection(unittest2.TestCase):

    @unittest2.skip('Creation takes longer than I want for now')
    def test_create_and_delete_table(self):
        connection = get_connection()

        self.assertFalse(ALT_TABLE_NAME in connection.tables())
        connection.create_table(ALT_TABLE_NAME, {COL_FAM1: {}})
        self.assertTrue(ALT_TABLE_NAME in connection.tables())
        connection.delete_table(ALT_TABLE_NAME, disable=True)
        self.assertFalse(ALT_TABLE_NAME in connection.tables())

    def test_families(self):
        table = get_table()
        families = table.families()

        self.assertEqual(set(families.keys()), set(FAMILIES.keys()))
        for col_fam, settings in FAMILIES.items():
            retrieved = families[col_fam]
            for key, value in settings.items():
                if key == 'time_to_live' and IGNORE_TTL:
                    # The Thrift API seems to fail here for some reason
                    continue
                self.assertEqual(retrieved[key], value)

class TestTable(unittest2.TestCase):

    def setUp(self):
        self.rows_to_delete = []

    def tearDown(self):
        table = get_table()
        for row_key in self.rows_to_delete:
            table.delete(row_key)

    def test_empty_rows(self):
        table = get_table()
        row1 = table.row(ROW_KEY1)
        row2 = table.row(ROW_KEY2)

        self.assertEqual(row1, {})
        self.assertEqual(row2, {})

    def test_put_success(self):
        table = get_table()
        value1 = 'value1'
        value2 = 'value2'
        row1_data = {COL1: value1, COL2: value2}

        # Make sure to delete before put(), in case it fails.
        self.rows_to_delete.append(ROW_KEY1)
        table.put(ROW_KEY1, row1_data)

        row1 = table.row(ROW_KEY1)
        self.assertEqual(row1, row1_data)

        # Check again, but this time with timestamps.
        row1 = table.row(ROW_KEY1, include_timestamp=True)
        timestamp1 = row1[COL1][1]
        timestamp2 = row1[COL2][1]
        self.assertEqual(timestamp1, timestamp2)

        row1_data_with_timestamps = {COL1: (value1, timestamp1),
                                     COL2: (value2, timestamp2)}
        self.assertEqual(row1, row1_data_with_timestamps)
<commit_msg>Adding failed HappyBase system test.<commit_after># Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import time
import unittest2

from happybase import Connection

NOW_MILLIS = int(1000 * time.time())
TABLE_NAME = 'table-name'
ALT_TABLE_NAME = 'other-table'
TTL_DAY = 24 * 60 * 60
COL_FAM1 = 'cf1'
COL_FAM2 = 'cf2'
COL_FAM3 = 'cf3'
FAMILIES = {
    COL_FAM1: {'max_versions': 10},
    COL_FAM2: {'max_versions': 1, 'time_to_live': TTL_DAY},
    COL_FAM3: {},  # use defaults
}
ROW_KEY1 = 'row-key1'
ROW_KEY2 = 'row-key2'
COL1 = COL_FAM1 + ':qual1'
COL2 = COL_FAM1 + ':qual2'
COL3 = COL_FAM2 + ':qual1'

class Config(object):
    """Simple namespace for holding test globals."""

    connection = None
    table = None

# BEGIN: Backend dependent values.
IGNORE_TTL = True

def get_connection():
    if Config.connection is None:
        Config.connection = Connection()
    return Config.connection
#   END: Backend dependent values.

def get_table():
    if Config.table is None:
        connection = get_connection()
        Config.table = connection.table(TABLE_NAME)
    return Config.table

def setUpModule():
    if Config.table is None:
        connection = get_connection()
        if TABLE_NAME not in connection.tables():
            connection.create_table(TABLE_NAME, FAMILIES)

def tearDownModule():
    # connection.delete_table(TABLE_NAME, disable=True)
    pass

class TestConnection(unittest2.TestCase):

    @unittest2.skip('Creation takes longer than I want for now')
    def test_create_and_delete_table(self):
        connection = get_connection()

        self.assertFalse(ALT_TABLE_NAME in connection.tables())
        connection.create_table(ALT_TABLE_NAME, {COL_FAM1: {}})
        self.assertTrue(ALT_TABLE_NAME in connection.tables())
        connection.delete_table(ALT_TABLE_NAME, disable=True)
        self.assertFalse(ALT_TABLE_NAME in connection.tables())

    def test_families(self):
        table = get_table()
        families = table.families()

        self.assertEqual(set(families.keys()), set(FAMILIES.keys()))
        for col_fam, settings in FAMILIES.items():
            retrieved = families[col_fam]
            for key, value in settings.items():
                if key == 'time_to_live' and IGNORE_TTL:
                    # The Thrift API seems to fail here for some reason
                    continue
                self.assertEqual(retrieved[key], value)

class TestTable(unittest2.TestCase):

    def setUp(self):
        self.rows_to_delete = []

    def tearDown(self):
        table = get_table()
        for row_key in self.rows_to_delete:
            table.delete(row_key)

    def test_empty_rows(self):
        table = get_table()
        row1 = table.row(ROW_KEY1)
        row2 = table.row(ROW_KEY2)

        self.assertEqual(row1, {})
        self.assertEqual(row2, {})

    def test_put(self):
        table = get_table()
        value1 = 'value1'
        value2 = 'value2'
        row1_data = {COL1: value1, COL2: value2}

        # Make sure to delete before put(), in case it fails.
        self.rows_to_delete.append(ROW_KEY1)
        table.put(ROW_KEY1, row1_data)

        row1 = table.row(ROW_KEY1)
        self.assertEqual(row1, row1_data)

        # Check again, but this time with timestamps.
        row1 = table.row(ROW_KEY1, include_timestamp=True)
        timestamp1 = row1[COL1][1]
        timestamp2 = row1[COL2][1]
        self.assertEqual(timestamp1, timestamp2)

        row1_data_with_timestamps = {COL1: (value1, timestamp1),
                                     COL2: (value2, timestamp2)}

    @unittest2.skip('HBase fails to write with a timestamp')
    def test_put_with_timestamps(self):
        table = get_table()
        value1 = 'value1'
        value2 = 'value2'
        row1_data = {COL1: value1, COL2: value2}
        ts = NOW_MILLIS

        # Make sure to delete before put(), in case it fails.
        self.rows_to_delete.append(ROW_KEY1)
        table.put(ROW_KEY1, row1_data, timestamp=ts)

        # Check again, but this time with timestamps.
        row1 = table.row(ROW_KEY1, include_timestamp=True)
        row1_data_with_timestamps = {COL1: (value1, ts),
                                     COL2: (value2, ts)}
        self.assertEqual(row1, row1_data_with_timestamps)
<|endoftext|>


<commit_before>#!/usr/bin/env python
from nose.tools import *
from nose import SkipTest
import networkx as nx
import io

class TestGEXF(object):
    @classmethod
    def setupClass(cls):
        try:
            import xml.etree.ElementTree
        except ImportError:
            raise SkipTest('xml.etree.ElementTree not available.')

    def setUp(self):
        self.simple_directed_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="directed">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" />
        </edges>
    </graph>
</gexf>
"""
        self.simple_directed_graph=nx.DiGraph()
        self.simple_directed_graph.add_node('0',label='Hello')
        self.simple_directed_graph.add_node('1',label='World')
        self.simple_directed_graph.add_edge('0','1',id='0')

        self.simple_directed_fh = \
            io.BytesIO(self.simple_directed_data.encode('UTF-8'))

        self.attribute_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.gexf.net/1.1draft http://www.gexf.net/1.1draft/gexf.xsd" version="1.1">
  <meta lastmodifieddate="2009-03-20">
    <creator>Gephi.org</creator>
    <description>A Web network</description>
  </meta>
  <graph defaultedgetype="directed">
    <attributes class="node">
      <attribute id="0" title="url" type="string"/>
      <attribute id="1" title="indegree" type="integer"/>
      <attribute id="2" title="frog" type="boolean">
        <default>true</default>
      </attribute>
    </attributes>
    <nodes>
      <node id="0" label="Gephi">
        <attvalues>
          <attvalue for="0" value="http://gephi.org"/>
          <attvalue for="1" value="1"/>
        </attvalues>
      </node>
      <node id="1" label="Webatlas">
        <attvalues>
          <attvalue for="0" value="http://webatlas.fr"/>
          <attvalue for="1" value="2"/>
        </attvalues>
      </node>
      <node id="2" label="RTGI">
        <attvalues>
          <attvalue for="0" value="http://rtgi.fr"/>
          <attvalue for="1" value="1"/>
        </attvalues>
      </node>
      <node id="3" label="BarabasiLab">
        <attvalues>
          <attvalue for="0" value="http://barabasilab.com"/>
          <attvalue for="1" value="1"/>
          <attvalue for="2" value="false"/>
        </attvalues>
      </node>
    </nodes>
    <edges>
      <edge id="0" source="0" target="1"/>
      <edge id="1" source="0" target="2"/>
      <edge id="2" source="1" target="0"/>
      <edge id="3" source="2" target="1"/>
      <edge id="4" source="0" target="3"/>
    </edges>
  </graph>
</gexf>
"""
        self.attribute_graph=nx.DiGraph()
        self.attribute_graph.graph['node_default']={'frog':True}
        self.attribute_graph.add_node('0',
                                      label='Gephi',
                                      url='http://gephi.org',
                                      indegree=1)
        self.attribute_graph.add_node('1',
                                      label='Webatlas',
                                      url='http://webatlas.fr',
                                      indegree=2)

        self.attribute_graph.add_node('2',
                                      label='RTGI',
                                      url='http://rtgi.fr',
                                      indegree=1)

        self.attribute_graph.add_node('3',
                                      label='BarabasiLab',
                                      url='http://barabasilab.com',
                                      indegree=1,
                                      frog=False)
        self.attribute_graph.add_edge('0','1',id='0')
        self.attribute_graph.add_edge('0','2',id='1')
        self.attribute_graph.add_edge('1','0',id='2')
        self.attribute_graph.add_edge('2','1',id='3')
        self.attribute_graph.add_edge('0','3',id='4')
        self.attribute_fh = io.BytesIO(self.attribute_data.encode('UTF-8'))

        self.simple_undirected_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="undirected">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" />
        </edges>
    </graph>
</gexf>
"""
        self.simple_undirected_graph=nx.Graph()
        self.simple_undirected_graph.add_node('0',label='Hello')
        self.simple_undirected_graph.add_node('1',label='World')
        self.simple_undirected_graph.add_edge('0','1',id='0')

        self.simple_undirected_fh = io.BytesIO(self.simple_undirected_data.encode('UTF-8'))

    def test_read_simple_directed_graphml(self):
        G=self.simple_directed_graph
        H=nx.read_gexf(self.simple_directed_fh)
        assert_equal(sorted(G.nodes()),sorted(H.nodes()))
        assert_equal(sorted(G.edges()),sorted(H.edges()))
        assert_equal(sorted(G.edges(data=True)),
                     sorted(H.edges(data=True)))
        self.simple_directed_fh.seek(0)

    def test_write_read_simple_directed_graphml(self):
        G=self.simple_directed_graph
        fh=io.BytesIO()
        nx.write_gexf(G,fh)
        fh.seek(0)
        H=nx.read_gexf(fh)
        assert_equal(sorted(G.nodes()),sorted(H.nodes()))
        assert_equal(sorted(G.edges()),sorted(H.edges()))
        assert_equal(sorted(G.edges(data=True)),
                     sorted(H.edges(data=True)))
        self.simple_directed_fh.seek(0)

    def test_read_simple_undirected_graphml(self):
        G=self.simple_undirected_graph
        H=nx.read_gexf(self.simple_undirected_fh)
        assert_equal(sorted(G.nodes()),sorted(H.nodes()))
        assert_equal(
            sorted(sorted(e) for e in G.edges()),
            sorted(sorted(e) for e in H.edges()))
        self.simple_undirected_fh.seek(0)

    def test_read_attribute_graphml(self):
        G=self.attribute_graph
        H=nx.read_gexf(self.attribute_fh)
        assert_equal(sorted(G.nodes(True)),sorted(H.nodes(data=True)))
        ge=sorted(G.edges(data=True))
        he=sorted(H.edges(data=True))
        for a,b in zip(ge,he):
            assert_equal(a,b)
        self.attribute_fh.seek(0)

    def test_directed_edge_in_undirected(self):
        s="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="undirected">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" type="directed"/>
        </edges>
    </graph>
</gexf>
"""
        fh = io.BytesIO(s.encode('UTF-8'))
        assert_raises(nx.NetworkXError,nx.read_gexf,fh)

    def test_undirected_edge_in_directed(self):
        s="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="directed">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" type="undirected"/>
        </edges>
    </graph>
</gexf>
"""
        fh = io.BytesIO(s.encode('UTF-8'))
        assert_raises(nx.NetworkXError,nx.read_gexf,fh)

    def test_key_error(self):
        s="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="directed">
        <nodes>
            <node id="0" label="Hello">
              <attvalues>
                <attvalue for='0' value='1'/>
              </attvalues>
            </node>
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" type="undirected"/>
        </edges>
    </graph>
</gexf>
"""
        fh = io.BytesIO(s.encode('UTF-8'))
        assert_raises(nx.NetworkXError,nx.read_gexf,fh)

    def test_relabel(self):
        s="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="directed">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1"/>
        </edges>
    </graph>
</gexf>
"""
        fh = io.BytesIO(s.encode('UTF-8'))
        G=nx.read_gexf(fh,relabel=True)
        assert_equal(sorted(G.nodes()),["Hello","Word"])

    def test_default_attribute(self):
        G=nx.Graph()
        G.add_node(1,label='1',color='green')
        G.add_path([0,1,2,3])
        G.add_edge(1,2,foo=3)
        G.graph['node_default']={'color':'yellow'}
        G.graph['edge_default']={'foo':7}
        fh = io.BytesIO()
        nx.write_gexf(G,fh)
        fh.seek(0)
        H=nx.read_gexf(fh,node_type=int)
        assert_equal(sorted(G.nodes()),sorted(H.nodes()))
        assert_equal(
            sorted(sorted(e) for e in G.edges()),
            sorted(sorted(e) for e in H.edges()))
        assert_equal(G.graph,H.graph)

    def test_serialize_ints_to_strings(self):
        G=nx.Graph()
        G.add_node(1,id=7,label=77)
        fh = io.BytesIO()
        nx.write_gexf(G,fh)
        fh.seek(0)
        H=nx.read_gexf(fh,node_type=int)
        assert_equal(H.nodes(),[7])
        assert_equal(H.node[7]['label'],'77')

    def test_write_with_node_attributes(self):
        # Addresses #673.
        G = nx.path_graph(4)
        for i in range(4):
            G.node[i]['id'] = i
            G.node[i]['label'] = i
            G.node[i]['pid'] = i

        expected = """<gexf version="1.1" xmlns="http://www.gexf.net/1.1draft" xmlns:viz="http://www.gexf.net/1.1draft/viz" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.w3.org/2001/XMLSchema-instance">
  <graph defaultedgetype="undirected" mode="static">
    <nodes>
      <node id="0" label="0" pid="0" />
      <node id="1" label="1" pid="1" />
      <node id="2" label="2" pid="2" />
      <node id="3" label="3" pid="3" />
    </nodes>
    <edges>
      <edge id="0" source="0" target="1" />
      <edge id="1" source="1" target="2" />
      <edge id="2" source="2" target="3" />
    </edges>
  </graph>
</gexf>"""
        obtained = '\n'.join(nx.generate_gexf(G))
        assert_equal( expected, obtained )

    def test_bool(self):
        G=nx.Graph()
        G.add_node(1, testattr=True)
        fh = io.BytesIO()
        nx.write_gexf(G,fh)
        fh.seek(0)
        H=nx.read_gexf(fh,node_type=int)
        assert_true(H.node[1]['testattr'], True)
<commit_msg>Fix assertion bug in test<commit_after>#!/usr/bin/env python
from nose.tools import *
from nose import SkipTest
import networkx as nx
import io

class TestGEXF(object):
    @classmethod
    def setupClass(cls):
        try:
            import xml.etree.ElementTree
        except ImportError:
            raise SkipTest('xml.etree.ElementTree not available.')

    def setUp(self):
        self.simple_directed_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="directed">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" />
        </edges>
    </graph>
</gexf>
"""
        self.simple_directed_graph=nx.DiGraph()
        self.simple_directed_graph.add_node('0',label='Hello')
        self.simple_directed_graph.add_node('1',label='World')
        self.simple_directed_graph.add_edge('0','1',id='0')

        self.simple_directed_fh = \
            io.BytesIO(self.simple_directed_data.encode('UTF-8'))

        self.attribute_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.gexf.net/1.1draft http://www.gexf.net/1.1draft/gexf.xsd" version="1.1">
  <meta lastmodifieddate="2009-03-20">
    <creator>Gephi.org</creator>
    <description>A Web network</description>
  </meta>
  <graph defaultedgetype="directed">
    <attributes class="node">
      <attribute id="0" title="url" type="string"/>
      <attribute id="1" title="indegree" type="integer"/>
      <attribute id="2" title="frog" type="boolean">
        <default>true</default>
      </attribute>
    </attributes>
    <nodes>
      <node id="0" label="Gephi">
        <attvalues>
          <attvalue for="0" value="http://gephi.org"/>
          <attvalue for="1" value="1"/>
        </attvalues>
      </node>
      <node id="1" label="Webatlas">
        <attvalues>
          <attvalue for="0" value="http://webatlas.fr"/>
          <attvalue for="1" value="2"/>
        </attvalues>
      </node>
      <node id="2" label="RTGI">
        <attvalues>
          <attvalue for="0" value="http://rtgi.fr"/>
          <attvalue for="1" value="1"/>
        </attvalues>
      </node>
      <node id="3" label="BarabasiLab">
        <attvalues>
          <attvalue for="0" value="http://barabasilab.com"/>
          <attvalue for="1" value="1"/>
          <attvalue for="2" value="false"/>
        </attvalues>
      </node>
    </nodes>
    <edges>
      <edge id="0" source="0" target="1"/>
      <edge id="1" source="0" target="2"/>
      <edge id="2" source="1" target="0"/>
      <edge id="3" source="2" target="1"/>
      <edge id="4" source="0" target="3"/>
    </edges>
  </graph>
</gexf>
"""
        self.attribute_graph=nx.DiGraph()
        self.attribute_graph.graph['node_default']={'frog':True}
        self.attribute_graph.add_node('0',
                                      label='Gephi',
                                      url='http://gephi.org',
                                      indegree=1)
        self.attribute_graph.add_node('1',
                                      label='Webatlas',
                                      url='http://webatlas.fr',
                                      indegree=2)

        self.attribute_graph.add_node('2',
                                      label='RTGI',
                                      url='http://rtgi.fr',
                                      indegree=1)

        self.attribute_graph.add_node('3',
                                      label='BarabasiLab',
                                      url='http://barabasilab.com',
                                      indegree=1,
                                      frog=False)
        self.attribute_graph.add_edge('0','1',id='0')
        self.attribute_graph.add_edge('0','2',id='1')
        self.attribute_graph.add_edge('1','0',id='2')
        self.attribute_graph.add_edge('2','1',id='3')
        self.attribute_graph.add_edge('0','3',id='4')
        self.attribute_fh = io.BytesIO(self.attribute_data.encode('UTF-8'))

        self.simple_undirected_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="undirected">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" />
        </edges>
    </graph>
</gexf>
"""
        self.simple_undirected_graph=nx.Graph()
        self.simple_undirected_graph.add_node('0',label='Hello')
        self.simple_undirected_graph.add_node('1',label='World')
        self.simple_undirected_graph.add_edge('0','1',id='0')

        self.simple_undirected_fh = io.BytesIO(self.simple_undirected_data.encode('UTF-8'))

    def test_read_simple_directed_graphml(self):
        G=self.simple_directed_graph
        H=nx.read_gexf(self.simple_directed_fh)
        assert_equal(sorted(G.nodes()),sorted(H.nodes()))
        assert_equal(sorted(G.edges()),sorted(H.edges()))
        assert_equal(sorted(G.edges(data=True)),
                     sorted(H.edges(data=True)))
        self.simple_directed_fh.seek(0)

    def test_write_read_simple_directed_graphml(self):
        G=self.simple_directed_graph
        fh=io.BytesIO()
        nx.write_gexf(G,fh)
        fh.seek(0)
        H=nx.read_gexf(fh)
        assert_equal(sorted(G.nodes()),sorted(H.nodes()))
        assert_equal(sorted(G.edges()),sorted(H.edges()))
        assert_equal(sorted(G.edges(data=True)),
                     sorted(H.edges(data=True)))
        self.simple_directed_fh.seek(0)

    def test_read_simple_undirected_graphml(self):
        G=self.simple_undirected_graph
        H=nx.read_gexf(self.simple_undirected_fh)
        assert_equal(sorted(G.nodes()),sorted(H.nodes()))
        assert_equal(
            sorted(sorted(e) for e in G.edges()),
            sorted(sorted(e) for e in H.edges()))
        self.simple_undirected_fh.seek(0)

    def test_read_attribute_graphml(self):
        G=self.attribute_graph
        H=nx.read_gexf(self.attribute_fh)
        assert_equal(sorted(G.nodes(True)),sorted(H.nodes(data=True)))
        ge=sorted(G.edges(data=True))
        he=sorted(H.edges(data=True))
        for a,b in zip(ge,he):
            assert_equal(a,b)
        self.attribute_fh.seek(0)

    def test_directed_edge_in_undirected(self):
        s="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="undirected">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" type="directed"/>
        </edges>
    </graph>
</gexf>
"""
        fh = io.BytesIO(s.encode('UTF-8'))
        assert_raises(nx.NetworkXError,nx.read_gexf,fh)

    def test_undirected_edge_in_directed(self):
        s="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="directed">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" type="undirected"/>
        </edges>
    </graph>
</gexf>
"""
        fh = io.BytesIO(s.encode('UTF-8'))
        assert_raises(nx.NetworkXError,nx.read_gexf,fh)

    def test_key_error(self):
        s="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="directed">
        <nodes>
            <node id="0" label="Hello">
              <attvalues>
                <attvalue for='0' value='1'/>
              </attvalues>
            </node>
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" type="undirected"/>
        </edges>
    </graph>
</gexf>
"""
        fh = io.BytesIO(s.encode('UTF-8'))
        assert_raises(nx.NetworkXError,nx.read_gexf,fh)

    def test_relabel(self):
        s="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="directed">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1"/>
        </edges>
    </graph>
</gexf>
"""
        fh = io.BytesIO(s.encode('UTF-8'))
        G=nx.read_gexf(fh,relabel=True)
        assert_equal(sorted(G.nodes()),["Hello","Word"])

    def test_default_attribute(self):
        G=nx.Graph()
        G.add_node(1,label='1',color='green')
        G.add_path([0,1,2,3])
        G.add_edge(1,2,foo=3)
        G.graph['node_default']={'color':'yellow'}
        G.graph['edge_default']={'foo':7}
        fh = io.BytesIO()
        nx.write_gexf(G,fh)
        fh.seek(0)
        H=nx.read_gexf(fh,node_type=int)
        assert_equal(sorted(G.nodes()),sorted(H.nodes()))
        assert_equal(
            sorted(sorted(e) for e in G.edges()),
            sorted(sorted(e) for e in H.edges()))
        assert_equal(G.graph,H.graph)

    def test_serialize_ints_to_strings(self):
        G=nx.Graph()
        G.add_node(1,id=7,label=77)
        fh = io.BytesIO()
        nx.write_gexf(G,fh)
        fh.seek(0)
        H=nx.read_gexf(fh,node_type=int)
        assert_equal(H.nodes(),[7])
        assert_equal(H.node[7]['label'],'77')

    def test_write_with_node_attributes(self):
        # Addresses #673.
        G = nx.path_graph(4)
        for i in range(4):
            G.node[i]['id'] = i
            G.node[i]['label'] = i
            G.node[i]['pid'] = i

        expected = """<gexf version="1.1" xmlns="http://www.gexf.net/1.1draft" xmlns:viz="http://www.gexf.net/1.1draft/viz" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.w3.org/2001/XMLSchema-instance">
  <graph defaultedgetype="undirected" mode="static">
    <nodes>
      <node id="0" label="0" pid="0" />
      <node id="1" label="1" pid="1" />
      <node id="2" label="2" pid="2" />
      <node id="3" label="3" pid="3" />
    </nodes>
    <edges>
      <edge id="0" source="0" target="1" />
      <edge id="1" source="1" target="2" />
      <edge id="2" source="2" target="3" />
    </edges>
  </graph>
</gexf>"""
        obtained = '\n'.join(nx.generate_gexf(G))
        assert_equal( expected, obtained )

    def test_bool(self):
        G=nx.Graph()
        G.add_node(1, testattr=True)
        fh = io.BytesIO()
        nx.write_gexf(G,fh)
        fh.seek(0)
        H=nx.read_gexf(fh,node_type=int)
        assert_equal(H.node[1]['testattr'], True)
<|endoftext|>


<commit_before>"""
base

"""
import logging
import socket

LOGGER = logging.getLogger(__name__)

class Plugin(object):

    GUID = 'com.meetme.newrelic_plugin_agent'
    MAX_VAL = 2147483647

    def __init__(self, config, poll_interval, last_interval_values=None):
        self.poll_interval = poll_interval
        self.config = config

        self.derive_values = dict()
        self.derive_last_interval = last_interval_values or dict()

        self.derive_last_interval = last_interval_values or dict()
        self.gauge_values = dict()
        self.rate_values = dict()

    def add_derive_value(self, metric_name, units, value, count=None):
        """Add a value that will derive the current value from the difference
        between the last interval value and the current value.

        If this is the first time a stat is being added, it will report a 0
        value until the next poll interval and it is able to calculate the
        derivative value.

        :param str metric_name: The name of the metric
        :param str units: The unit type
        :param int value: The value to add
        :param int count: The number of items the timing is for

        """
        if value is None:
            value = 0
        metric = self.metric_name(metric_name, units)
        if metric not in self.derive_last_interval.keys():
            LOGGER.debug('Bypassing initial %s value for first run', metric)
            self.derive_values[metric] = self.metric_payload(0, count=0)
        else:
            cval = value - self.derive_last_interval[metric]
            self.derive_values[metric] = self.metric_payload(cval, count=count)
            LOGGER.debug('%s: Last: %r, Current: %r, Reporting: %r',
                         metric, self.derive_last_interval[metric], value,
                         self.derive_values[metric])
        self.derive_last_interval[metric] = value

    def add_derive_timing_value(self, metric_name, units, count, total_value,
                                last_value=None):
        """For timing based metrics that have a count of objects for the timing
        and an optional last value.

        :param str metric_name: The name of the metric
        :param str units: The unit type
        :param int count: The number of items the timing is for
        :param int total_value: The timing value
        :param int last_value: The last value

        """
        if last_value is None:
            return self.add_derive_value(metric_name, units, total_value, count)
        self.add_derive_value('%s/Total' % metric_name,
                              units, total_value, count)
        self.add_derive_value('%s/Last' % metric_name,
                              units, last_value, count)

    def add_gauge_value(self, metric_name, units, value,
                        min_val=None, max_val=None, count=None,
                        sum_of_squares=None):
        """Add a value that is not a rolling counter but rather an absolute
        gauge

        :param str metric_name: The name of the metric
        :param str units: The unit type
        :param int value: The value to add
        :param float value: The sum of squares for the values

        """
        metric = self.metric_name(metric_name, units)
        self.gauge_values[metric] = self.metric_payload(value,
                                                        min_val,
                                                        max_val,
                                                        count,
                                                        sum_of_squares)
        LOGGER.debug('%s: %r', metric_name, self.gauge_values[metric])

    def component_data(self):
        """Create the component section of the NewRelic Platform data payload
        message.

        :rtype: dict

        """
        metrics = dict()
        metrics.update(self.derive_values.items())
        metrics.update(self.gauge_values.items())
        metrics.update(self.rate_values.items())
        return {'name': self.name,
                'guid': self.GUID,
                'duration': self.poll_interval,
                'metrics': metrics}

    def initialize_counters(self, keys):
        """Create a new set of counters for the given key list

        :param list keys: Keys to initialize in the counters
        :rtype: tuple

        """
        count, total, min_val, max_val, values = (dict(), dict(), dict(),
                                                  dict(), dict())
        for key in keys:
            (count[key], total[key], min_val[key],
             max_val[key], values[key]) = 0, 0, self.MAX_VAL, 0, list()
        return count, total, min_val, max_val, values

    def metric_name(self, metric, units):
        """Return the metric name in the format for the NewRelic platform

        :param str metric: The name of th metric
        :param str units: The unit name

        """
        if not units:
            return 'Component/%s' % metric
        return 'Component/%s[%s]' % (metric, units)

    def metric_payload(self, value, min_value=None, max_value=None, count=None,
                       squares=None):
        """Return the metric in the standard payload format for the NewRelic
        agent.

        :rtype: dict

        """
        if isinstance(value, basestring):
            value = 0

        sum_of_squares = int(squares or (value * value))
        if sum_of_squares > self.MAX_VAL:
            sum_of_squares = 0

        return {'min': min_value,
                'max': max_value,
                'total': value,
                'count': count or 1,
                'sum_of_squares': sum_of_squares}

    @property
    def name(self):
        """Return the name of the component

        :rtype: str

        """
        return self.config.get('name', socket.gethostname().split('.')[0])

    def poll(self):
        """Poll the server returning the results in the expected component
        format.

        """
        raise NotImplementedError

    def sum_of_squares(self, values):
        """Return the sum_of_squares for the given values

        :param list values: The values list
        :rtype: float

        """
        value_sum = sum(values)
        if not value_sum:
            return 0
        squares = list()
        for value in values:
            squares.append(value * value)
        return sum(squares) - float(value_sum * value_sum) / len(values)

    def values(self):
        """Return the poll results

        :rtype: dict

        """
        return self.component_data()
<commit_msg>New extended base classes to simplify plugin code<commit_after>"""
base

"""
import logging
from os import path
import requests
import socket
import time
import urlparse

LOGGER = logging.getLogger(__name__)

class Plugin(object):

    GUID = 'com.meetme.newrelic_plugin_agent'
    MAX_VAL = 2147483647

    def __init__(self, config, poll_interval, last_interval_values=None):
        self.config = config
        self.poll_interval = poll_interval
        self.poll_start_time = 0

        self.derive_values = dict()
        self.derive_last_interval = last_interval_values or dict()
        self.gauge_values = dict()
        self.rate_values = dict()

    def add_datapoints(self, data):
        """Extend this method to process the data points retrieved during the
        poll process.

        :param mixed data: The data received during the poll process

        """
        raise NotImplementedError

    def add_derive_value(self, metric_name, units, value, count=None):
        """Add a value that will derive the current value from the difference
        between the last interval value and the current value.

        If this is the first time a stat is being added, it will report a 0
        value until the next poll interval and it is able to calculate the
        derivative value.

        :param str metric_name: The name of the metric
        :param str units: The unit type
        :param int value: The value to add
        :param int count: The number of items the timing is for

        """
        if value is None:
            value = 0
        metric = self.metric_name(metric_name, units)
        if metric not in self.derive_last_interval.keys():
            LOGGER.debug('Bypassing initial %s value for first run', metric)
            self.derive_values[metric] = self.metric_payload(0, count=0)
        else:
            cval = value - self.derive_last_interval[metric]
            self.derive_values[metric] = self.metric_payload(cval, count=count)
            LOGGER.debug('%s: Last: %r, Current: %r, Reporting: %r',
                         metric, self.derive_last_interval[metric], value,
                         self.derive_values[metric])
        self.derive_last_interval[metric] = value

    def add_derive_timing_value(self, metric_name, units, count, total_value,
                                last_value=None):
        """For timing based metrics that have a count of objects for the timing
        and an optional last value.

        :param str metric_name: The name of the metric
        :param str units: The unit type
        :param int count: The number of items the timing is for
        :param int total_value: The timing value
        :param int last_value: The last value

        """
        if last_value is None:
            return self.add_derive_value(metric_name, units, total_value, count)
        self.add_derive_value('%s/Total' % metric_name,
                              units, total_value, count)
        self.add_derive_value('%s/Last' % metric_name,
                              units, last_value, count)

    def add_gauge_value(self, metric_name, units, value,
                        min_val=None, max_val=None, count=None,
                        sum_of_squares=None):
        """Add a value that is not a rolling counter but rather an absolute
        gauge

        :param str metric_name: The name of the metric
        :param str units: The unit type
        :param int value: The value to add
        :param float value: The sum of squares for the values

        """
        metric = self.metric_name(metric_name, units)
        self.gauge_values[metric] = self.metric_payload(value,
                                                        min_val,
                                                        max_val,
                                                        count,
                                                        sum_of_squares)
        LOGGER.debug('%s: %r', metric_name, self.gauge_values[metric])

    def component_data(self):
        """Create the component section of the NewRelic Platform data payload
        message.

        :rtype: dict

        """
        metrics = dict()
        metrics.update(self.derive_values.items())
        metrics.update(self.gauge_values.items())
        metrics.update(self.rate_values.items())
        return {'name': self.name,
                'guid': self.GUID,
                'duration': self.poll_interval,
                'metrics': metrics}

    def error_message(self):
        """Output an error message when stats collection fails"""
        LOGGER.error('Error collecting stats data from %s. Please check '
                     'configuration and sure it conforms with YAML '
                     'syntax', self.__class__.__name__)

    def finish(self):
        """Note the end of the stat collection run and let the user know of any
        errors.

        """
        if not self.derive and not self.gauge and not self.rate:
            self.error_message()
        else:
            LOGGER.info('%s poll successful, completed in %.2f seconds',
                        self.__class__.__name__,
                        time.time() - self.poll_start_time)

    def initialize(self):
        """Empty stats collection dictionaries for the polling interval"""
        LOGGER.info('Polling %s', self.__class__.__name__)
        self.poll_start_time = time.time()
        self.derive = dict()
        self.gauge = dict()
        self.rate = dict()

    def initialize_counters(self, keys):
        """Create a new set of counters for the given key list

        :param list keys: Keys to initialize in the counters
        :rtype: tuple

        """
        count, total, min_val, max_val, values = (dict(), dict(), dict(),
                                                  dict(), dict())
        for key in keys:
            (count[key], total[key], min_val[key],
             max_val[key], values[key]) = 0, 0, self.MAX_VAL, 0, list()
        return count, total, min_val, max_val, values

    def metric_name(self, metric, units):
        """Return the metric name in the format for the NewRelic platform

        :param str metric: The name of th metric
        :param str units: The unit name

        """
        if not units:
            return 'Component/%s' % metric
        return 'Component/%s[%s]' % (metric, units)

    def metric_payload(self, value, min_value=None, max_value=None, count=None,
                       squares=None):
        """Return the metric in the standard payload format for the NewRelic
        agent.

        :rtype: dict

        """
        if isinstance(value, basestring):
            value = 0

        sum_of_squares = int(squares or (value * value))
        if sum_of_squares > self.MAX_VAL:
            sum_of_squares = 0

        return {'min': min_value,
                'max': max_value,
                'total': value,
                'count': count or 1,
                'sum_of_squares': sum_of_squares}

    @property
    def name(self):
        """Return the name of the component

        :rtype: str

        """
        return self.config.get('name', socket.gethostname().split('.')[0])

    def poll(self):
        """Poll the server returning the results in the expected component
        format.

        """
        raise NotImplementedError

    def sum_of_squares(self, values):
        """Return the sum_of_squares for the given values

        :param list values: The values list
        :rtype: float

        """
        value_sum = sum(values)
        if not value_sum:
            return 0
        squares = list()
        for value in values:
            squares.append(value * value)
        return sum(squares) - float(value_sum * value_sum) / len(values)

    def values(self):
        """Return the poll results

        :rtype: dict

        """
        return self.component_data()

class SocketStatsPlugin(Plugin):
    """Connect to a socket and collect stats data"""
    DEFAULT_HOST = 'localhost'
    DEFAULT_PORT = 0
    SOCKET_RECV_MAX = 10485760

    def connect(self):
        """Top level interface to create a socket and connect it to the
        socket.

        :rtype: socket

        """
        try:
            connection = self.socket_connect()
        except socket.error as error:
            LOGGER.error('Error connecting to %s: %s',
                         self.__class__.__name__, error)
        else:
            return connection

    def fetch_data(self, connection):
        """Read the data from the socket

        :param  socket connection: The connection

        """
        LOGGER.debug('Fetching data')
        return connection.recv(self.SOCKET_RECV_MAX)

    def poll(self):
        """This method is called after every sleep interval. If the intention
        is to use an IOLoop instead of sleep interval based daemon, override
        the run method.

        """
        self.initialize()

        # Fetch the data from the remote socket
        connection = self.connect()
        if not connection:
            LOGGER.error('%s could not connect, skipping poll interval',
                         self.__class__.__name__)
            return

        data = self.fetch_data(connection)
        connection.close()

        if data:
            self.add_datapoints(data)
            self.finish()
        else:
            self.error_message()

    def socket_connect(self):
        """Low level interface to create a socket and connect to it.

        :rtype: socket

        """
        if 'path' in self.config:
            if path.exists(self.config['path']):
                LOGGER.debug('Connecting to UNIX domain socket: %s',
                             self.config['path'])
                connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
                connection.connect(self.config['path'])
            else:
                LOGGER.error('UNIX domain socket path does not exist: %s',
                             self.config['path'])
                return None
        else:
            remote_host = (self.config.get('host', self.DEFAULT_HOST),
                           self.config.get('port', self.DEFAULT_PORT))
            LOGGER.debug('Connecting to %s:%s', remote_host)
            connection = socket.socket()
            connection.connect(remote_host)
        return connection

class HTTPStatsPlugin(Plugin):
    """Extend the Plugin class overriding poll for targets that provide data
    via HTTP protocol.

    """
    def fetch_data(self):
        """Fetch the data from the stats URL

        :rtype: str

        """
        data = self.http_get()
        return data.content if data else ''

    def http_get(self):
        """Fetch the data from the stats URL

        :rtype: requests.models.Response

        """
        LOGGER.info('Polling %s Stats at %s',
                    self.__class__.__name__, self.stats_url)
        try:
            response = requests.get(**self.request_kwargs)
        except requests.ConnectionError as error:
            LOGGER.error('Error polling stats: %s', error)
            return ''

        if response.status_code >= 300:
            LOGGER.error('Error response from %s (%s): %s', self.stats_url,
                         response.status_code, response.content)
            return None
        return response

    def poll(self):
        """Poll HTTP server for stats data"""
        self.initialize()
        data = self.fetch_data()
        if data:
            self.add_datapoints(data)
        self.finish()

    @property
    def stats_url(self):
        """Return the configured URL in a uniform way for all HTTP based data
        sources.

        :rtype: str

        """
        netloc = self.config.get('host', 'localhost')
        if self.config.get('port'):
            netloc += ':%s' % self.config['port']

        return urlparse.urlunparse((self.config.get('scheme', 'http'),
                                    netloc,
                                    self.config.get('path', '/'),
                                    self.config.get('query', None),
                                    None, None, None))

    @property
    def request_kwargs(self):
        """Return kwargs for a HTTP request.

        :rtype: dict

        """
        kwargs = {'url': self.stats_url}
        if (self.config.get('scheme') == 'https' and
                self.config.get('verify_ssl_cert')):
            kwargs['verify'] = self.config.get('verify_ssl_cert', True)

        if 'username' in self.config and 'password' in self.config:
            kwargs['auth'] = (self.config['username'], self.config['password'])

        return kwargs

class JSONStatsPlugin(HTTPStatsPlugin):
    """Extend the Plugin overriding poll for targets that provide JSON output
    for stats collection

    """
    def fetch_data(self):
        """Fetch the data from the stats URL

        :rtype: dict

        """
        data = self.http_get()
        try:
            return data.json() if data else {}
        except Exception as error:
            LOGGER.error('JSON decoding error: %r', error)
        return {}

    def poll(self):
        """Poll HTTP JSON endpoint for stats data"""
        self.initialize()
        data = self.fetch_data()
        if data:
            self.add_datapoints(data)
        self.finish()
<|endoftext|>


<commit_before>import scipy as sp
import numpy as np

class MultivariateNormal(object):
		# TODO: make n_params general
		n_params = 5

    def __init__(self, params, temp_scale = 1.0):
        self.N, p = params.shape
        self.p = int(0.5 * (np.sqrt(8 * p - 3))

        self.loc = params[0:p, :].T

        self.L = np.zeros((self.p, self.p, self.N))
        self.L[np.tril_indices(self.p)] = params[self.p:, :]
        self.L = np.transpose(self.L, (2, 0, 1))
        self.cov = self.L @ np.transpose(self.L, (0, 2, 1))
        self.cov_inv = np.linalg.inv(self.cov)
        self.dCovdL = self.D_cov_D_L()

    def __getattr__(self, name):
        if name in dir(self.dist):
            return getattr(self.dist, name)
        return None

    def D_cov_D_L(self):
        commutation = np.zeros((self.p**2, self.p**2))
        ind = np.arange(self.p**2).reshape(self.p, self.p).T.flatten()
        commutation[np.arange(self.p**2), ind] = 1.
        dCovdL = (np.identity(self.p**2) + commutation) @\
                 np.kron(self.L, np.identity(self.p))
        dCovdL = dCovdL.reshape(-1,2,2,2,2).swapaxes(-2,-1)
        return dCovdL

		def nll(self, Y):
        diff = Y - self.loc
        M = diff[:,None,:] @ self.cov_inv @ diff[:,:,None]
        half_log_det = np.log(np.diagonal(self.L, axis1=1, axis2=2)).sum(-1)
        const = self.p / 2 * np.log(2*np.pi)
        logpdf = - const - half_log_det - 0.5 * M.flatten()
        return -logpdf

		def D_nll(self, Y_):
				pass

	  def fisher_info(self):
				pass

		def fit(Y):
				pass
<commit_msg>add D_nll and comments<commit_after>import scipy as sp
import numpy as np

class MultivariateNormal(object):
		# TODO: make n_params general
		n_params = 5

    def __init__(self, params, temp_scale = 1.0):
        self.N, p = params.shape
        self.p = int(0.5 * (np.sqrt(8 * p - 3))

        self.loc = params[0:p, :].T

        self.L = np.zeros((self.p, self.p, self.N))
        self.L[np.tril_indices(self.p)] = params[self.p:, :]
        self.L = np.transpose(self.L, (2, 0, 1))
        self.cov = self.L @ np.transpose(self.L, (0, 2, 1))
        self.cov_inv = np.linalg.inv(self.cov)
        self.dCovdL = self.D_cov_D_L()

    def __getattr__(self, name):
        if name in dir(self.dist):
            return getattr(self.dist, name)
        return None

    def D_cov_D_L(self):
        # create commutation matrix
        commutation = np.zeros((self.p**2, self.p**2))
        ind = np.arange(self.p**2).reshape(self.p, self.p).T.flatten()
        commutation[np.arange(self.p**2), ind] = 1.

        # compute Jacobian
        dCovdL = (np.identity(self.p**2) + commutation) @\
                 np.kron(self.L, np.identity(self.p))
        dCovdL = dCovdL.reshape(-1,2,2,2,2).swapaxes(-2,-1)
        return dCovdL

		def nll(self, Y):
        diff = Y - self.loc
        M = diff[:,None,:] @ self.cov_inv @ diff[:,:,None]
        half_log_det = np.log(np.diagonal(self.L, axis1=1, axis2=2)).sum(-1)
        const = self.p / 2 * np.log(2*np.pi)
        logpdf = - const - half_log_det - 0.5 * M.flatten()
        return -logpdf

		def D_nll(self, Y_):
        # reshape Jacobian
        tril_indices = np.tril_indices(self.p)
        J = self.dCovdL[:,:,:,tril_indices[0],tril_indices[1]]
        J = np.transpose(J, (0,3,1,2)).reshape(self.N, -1, self.p**2)
        J = J.swapaxes(-2,-1)

        # compute grad mu
        D = np.zeros((self.N, J.shape[-1] + self.p))
        sigma_inv = np.linalg.inv(self.cov)
        diff = self.loc - Y_
        D[:, :self.p] = (sigma_inv @ diff[:,:,None])[...,0]

        # compute grad sigma
        D_sigma = 0.5*(sigma_inv - sigma_inv @ (diff[:,:,None]*diff[:,None,:]) @ sigma_inv)
        D_sigma = D_sigma.reshape(self.N, -1)
        D_L = J.swapaxes(-2,-1) @ D_sigma[:,:,None]
        D[:, self.p:] = D_L[..., 0]

        return D

	  def fisher_info(self):
				pass

		def fit(Y):
				pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Run as: cat google_ngram_file | ./convert_to_mongo | sort -k1,1 -t $'\t' > output
Then proceed to mongodb import
"""
import sys
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--sub', dest='subst_file', action='store',
                   help='path to sustitutions file')
args = parser.parse_args()
SUBST_SET = set(open(args.subst_file).read().split('\n'))

# input comes from STDIN (standard input)
for line in sys.stdin:
    # remove leading and trailing whitespace
    line = line.strip()
    ngram, count = line.split('\t')
    words = ngram.split()
    for i, word in enumerate(words):
        if word in SUBST_SET:
            ngram = words[:]
            ngram[i] = 'SUB'
            ngram.append(str(i))
            print '\t'.join([' '.join(ngram), word, count])
<commit_msg>Update convert_to_mongo.py<commit_after>#!/usr/bin/env python
"""
Run as: cat google_ngram_file | ./convert_to_mongo --sub subs_file.txt | sort -k1,1 -t $'\t' > output
Then proceed to mongodb import
"""
import sys
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--sub', dest='subst_file', action='store',
                   help='path to sustitutions file')
args = parser.parse_args()
SUBST_SET = set(open(args.subst_file).read().split('\n'))

# input comes from STDIN (standard input)
for line in sys.stdin:
    # remove leading and trailing whitespace
    line = line.strip()
    ngram, count = line.split('\t')
    words = ngram.split()
    for i, word in enumerate(words):
        if word in SUBST_SET:
            ngram = words[:]
            ngram[i] = 'SUB'
            ngram.append(str(i))
            print '\t'.join([' '.join(ngram), word, count])
<|endoftext|>


<commit_before>""" Main taca_ngi_pipeline module
"""

__version__ = '0.2.0'
<commit_msg>bump minor revision<commit_after>""" Main taca_ngi_pipeline module
"""

__version__ = '0.3.0'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2014-2022 Dontnod Entertainment

# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:

# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import argparse
import fnmatch
import json
import itertools
import os
import re

import nimp.command
import nimp.utils.p4

class CreateLoadlist(nimp.command.Command):
	''' Generates a list of modified files from a set of Perforce changelists '''

	def configure_arguments(self, env, parser):
		parser.add_argument('changelists', nargs = argparse.ZERO_OR_MORE, help = 'select the changelists to list files from. Defaults to listing all files in havelist', default=[])
		parser.add_argument('-o', '--output', help = 'output file')
		parser.add_argument('-e', '--extensions', nargs = argparse.ZERO_OR_MORE, help = 'file extensions to include', default = [ 'uasset', 'umap' ])
		parser.add_argument('--dirs', nargs = argparse.ZERO_OR_MORE, help = 'directories to include', default = None)
		parser.add_argument('--exclude-dirs', nargs = argparse.ZERO_OR_MORE, help = 'directories to exclude', default = None)
		parser.add_argument('--check-empty', action = 'store_true', help = 'Returns check empty in json format')
		nimp.utils.p4.add_arguments(parser)
		nimp.command.add_common_arguments(parser, 'dry_run', 'slice_job')
		return True

	def is_available(self, env):
		return env.is_unreal, ''

	@staticmethod
	def _product_dirs_and_extensions(env, extensions):
		if env.dirs is None:
			env.dirs = ['']
		dirs = [env.format(dir) for dir in env.dirs]
		return list(itertools.product(dirs, extensions))

	@staticmethod
	def _product_paths_and_changelists(env, paths):
		changelists = [f'@{cl}' for cl in env.changelists]
		if len(changelists) <= 0:
			changelists = ['#have']
		return list(itertools.product(paths, changelists))

	@staticmethod
	def _exclude_from_modified_files(exclude_dirs, filepath):
		if exclude_dirs is None:
			return False
		for exclude_dir in exclude_dirs:
			if fnmatch.fnmatch(filepath, f'*{exclude_dir}*'):
				return True
		return False

	@staticmethod
	def _normpath_dirs(dirs):
		if dirs is None:
			return dirs
		return [os.path.normpath(dir) for dir in dirs]

	def get_modified_files(self, env, extensions):
		p4 = nimp.utils.p4.get_client(env)

		# Do not use '//...' which will also list files not mapped to workspace
		root = f"//{p4._client}/..."

		paths = []
		for dir, ext in self._product_dirs_and_extensions(env, extensions):
			paths.append(f"{root}{dir}{ext}")
		if len(paths) <= 0:
			paths.append(root)

		filespecs = []
		for path, cl in self._product_paths_and_changelists(env, paths):
			filespecs.append(f"{path}{cl}")

		base_command = [
			"fstat",
			# Only list modified files currently accessible
			"-F", "^headAction=delete & ^headAction=move/delete"
		]

		modified_files = set()
		exclude_dirs = self._normpath_dirs(env.exclude_dirs)
		for (filepath, ) in p4._parse_command_output(base_command + filespecs, r"^\.\.\. clientFile(.*)$", hide_output=True):
			modified_file_path = os.path.normpath(filepath)
			if not self._exclude_from_modified_files(exclude_dirs, modified_file_path):
				modified_files.add(modified_file_path)

		# Needed for sorting and ease debug
		modified_files = list(modified_files)
		modified_files.sort()

		if env.slice_job_count is not None and env.slice_job_count > 1:
			# slice modified files
			# use a simple heuristic to spread the load between slices
			# as demanding files tends to be in the same directory
			slice = []
			for idx, elem in enumerate(modified_files):
				if (idx % env.slice_job_count) == (env.slice_job_index - 1):
					slice.append(elem)

			modified_files = slice

		return modified_files

	def run(self, env):
		loadlist_files = self.get_modified_files(env, env.extensions)

		loadlist_path = env.output if env.output else f'{env.unreal_loadlist}'
		loadlist_path = os.path.abspath(env.format(nimp.system.sanitize_path(loadlist_path)))

		if env.check_empty:
			return self.check_empty_loadlist(loadlist_files)

		if not env.dry_run:
			with open(loadlist_path, 'w') as fp:
				for file in loadlist_files:
					print(file)
					fp.write(f'{file}\n')

		return True

	def check_empty_loadlist(self, modified_files):
		results = {'loadlist_is_empty': True}
		if modified_files:
			results['loadlist_is_empty'] = False

		json_content = json.dumps(results, indent=4)
		print('<loadlist_start>')
		print(json_content)
		print('<loadlist_end>')
		return True
<commit_msg>CreateLoadlist: default dir values to p4client<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2014-2022 Dontnod Entertainment

# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:

# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import argparse
import fnmatch
import json
import itertools
import os
import re

import nimp.command
import nimp.utils.p4

class CreateLoadlist(nimp.command.Command):
	''' Generates a list of modified files from a set of Perforce changelists '''

	def configure_arguments(self, env, parser):
		parser.add_argument('changelists', nargs = argparse.ZERO_OR_MORE, help = 'select the changelists to list files from. Defaults to listing all files in havelist', default=[])
		parser.add_argument('-o', '--output', help = 'output file')
		parser.add_argument('-e', '--extensions', nargs = argparse.ZERO_OR_MORE, help = 'file extensions to include', default = [ 'uasset', 'umap' ])
		parser.add_argument('--dirs', nargs = argparse.ZERO_OR_MORE, help = 'directories to include', default = ['//{p4client}/...'])
		parser.add_argument('--exclude-dirs', nargs = argparse.ZERO_OR_MORE, help = 'directories to exclude', default = None)
		parser.add_argument('--check-empty', action = 'store_true', help = 'Returns check empty in json format')
		nimp.utils.p4.add_arguments(parser)
		nimp.command.add_common_arguments(parser, 'dry_run', 'slice_job')
		return True

	def is_available(self, env):
		return env.is_unreal, ''

	@staticmethod
	def _product_dirs_and_extensions(env, extensions):
		if env.dirs is None:
			env.dirs = ['']
		dirs = [env.format(dir) for dir in env.dirs]
		return list(itertools.product(dirs, extensions))

	@staticmethod
	def _product_paths_and_changelists(env, paths):
		changelists = [f'@{cl}' for cl in env.changelists]
		if len(changelists) <= 0:
			changelists = ['#have']
		return list(itertools.product(paths, changelists))

	@staticmethod
	def _exclude_from_modified_files(exclude_dirs, filepath):
		if exclude_dirs is None:
			return False
		for exclude_dir in exclude_dirs:
			if fnmatch.fnmatch(filepath, f'*{exclude_dir}*'):
				return True
		return False

	@staticmethod
	def _normpath_dirs(dirs):
		if dirs is None:
			return dirs
		return [os.path.normpath(dir) for dir in dirs]

	def get_modified_files(self, env, extensions):
		p4 = nimp.utils.p4.get_client(env)

		paths = []
		for dir, ext in self._product_dirs_and_extensions(env, extensions):
			paths.append(f"{dir}{ext}")

		filespecs = []
		for path, cl in self._product_paths_and_changelists(env, paths):
			filespecs.append(f"{path}{cl}")

		base_command = [
			"fstat",
			# Only list modified files currently accessible
			"-F", "^headAction=delete & ^headAction=move/delete"
		]

		modified_files = set()
		exclude_dirs = self._normpath_dirs(env.exclude_dirs)
		for (filepath, ) in p4._parse_command_output(base_command + filespecs, r"^\.\.\. clientFile(.*)$", hide_output=True):
			modified_file_path = os.path.normpath(filepath)
			if not self._exclude_from_modified_files(exclude_dirs, modified_file_path):
				modified_files.add(modified_file_path)

		# Needed for sorting and ease debug
		modified_files = list(modified_files)
		modified_files.sort()

		if env.slice_job_count is not None and env.slice_job_count > 1:
			# slice modified files
			# use a simple heuristic to spread the load between slices
			# as demanding files tends to be in the same directory
			slice = []
			for idx, elem in enumerate(modified_files):
				if (idx % env.slice_job_count) == (env.slice_job_index - 1):
					slice.append(elem)

			modified_files = slice

		return modified_files

	def run(self, env):
		loadlist_files = self.get_modified_files(env, env.extensions)

		loadlist_path = env.output if env.output else f'{env.unreal_loadlist}'
		loadlist_path = os.path.abspath(env.format(nimp.system.sanitize_path(loadlist_path)))

		if env.check_empty:
			return self.check_empty_loadlist(loadlist_files)

		if not env.dry_run:
			with open(loadlist_path, 'w') as fp:
				for file in loadlist_files:
					print(file)
					fp.write(f'{file}\n')

		return True

	def check_empty_loadlist(self, modified_files):
		results = {'loadlist_is_empty': True}
		if modified_files:
			results['loadlist_is_empty'] = False

		json_content = json.dumps(results, indent=4)
		print('<loadlist_start>')
		print(json_content)
		print('<loadlist_end>')
		return True
<|endoftext|>