<commit_before># coding=utf-8
# Filename: hdf5.py
# pylint: disable=C0103,R0903
# vim:set ts=4 sts=4 sw=4 et:
"""
Pumps for the EVT simulation dataformat.

"""
from __future__ import division, absolute_import, print_function

import os.path

import tables
import numpy as np

from km3pipe import Pump, Module
from km3pipe.dataclasses import HitSeries, TrackSeries
from km3pipe.logger import logging

log = logging.getLogger(__name__)  # pylint: disable=C0103

__author__ = 'tamasgal'

POS_ATOM = tables.FloatAtom(shape=3)

class Hit(tables.IsDescription):
    channel_id = tables.UInt8Col()
    dom_id = tables.UIntCol()
    event_id = tables.UIntCol()
    id = tables.UIntCol()
    pmt_id = tables.UIntCol()
    time = tables.IntCol()
    tot = tables.UInt8Col()
    triggered = tables.BoolCol()

class Track(tables.IsDescription):
    dir = tables.FloatCol(shape=(3,))
    energy = tables.FloatCol()
    event_id = tables.UIntCol()
    id = tables.UIntCol()
    pos = tables.FloatCol(shape=(3,))
    time = tables.IntCol()
    type = tables.IntCol()

class EventInfo(tables.IsDescription):
    id = tables.IntCol()
    det_id = tables.IntCol()
    event_id = tables.UIntCol()
    frame_index = tables.UIntCol()
    mc_id = tables.IntCol()
    mc_t = tables.Float64Col()
    overlays = tables.UInt8Col()
    run_id = tables.UIntCol()
    #timestamp = tables.Float64Col()
    trigger_counter = tables.UInt64Col()
    trigger_mask = tables.UInt64Col()

class HDF5Sink(Module):
    def __init__(self, **context):
        """A Module to convert (KM3NeT) ROOT files to HDF5."""
        super(self.__class__, self).__init__(**context)
        self.filename = self.get('filename') or 'dump.h5'
        self.index = 1
        self.h5file = tables.open_file(self.filename, mode="w", title="Test file")
        self.filters = tables.Filters(complevel=5)
        self.hits = self.h5file.create_table('/', 'hits', Hit, "Hits", filters=self.filters)
        self.mc_hits = self.h5file.create_table('/', 'mc_hits', Hit, "MC Hits", filters=self.filters)
        self.mc_tracks = self.h5file.create_table('/', 'mc_tracks', Track, "MC Tracks", filters=self.filters)
        self.event_info = self.h5file.create_table('/', 'event_info', EventInfo, "Event Info", filters=self.filters)

    def _write_hits(self, hits, hit_row):
        for hit in hits:
            hit_row['channel_id'] = hit.channel_id
            hit_row['dom_id'] = hit.dom_id
            hit_row['event_id'] = self.index
            hit_row['id'] = hit.id
            hit_row['pmt_id'] = hit.pmt_id
            hit_row['time'] = hit.time
            hit_row['tot'] = hit.tot
            hit_row['triggered'] = hit.triggered
            hit_row.append()

    def _write_tracks(self, tracks, track_row):
        for track in tracks:
            track_row['dir'] = track.dir
            track_row['energy'] = track.energy
            track_row['event_id'] = self.index
            track_row['id'] = track.id
            track_row['pos'] = track.pos
            track_row['time'] = track.time
            track_row['type'] = track.type
            track_row.append()

    def _write_eventinfo(self, info, info_row):
        info_row['det_id'] = info.det_id
        info_row['event_id'] = self.index
        info_row['frame_index'] = info.frame_index
        info_row['id'] = info.id
        info_row['mc_id'] = info.mc_id
        info_row['mc_t'] = info.mc_t
        info_row['overlays'] = info.overlays
        info_row['run_id'] = info.run_id
        #info_row['timestamp'] = info.timestamp
        info_row['trigger_counter'] = info.trigger_counter
        info_row['trigger_mask'] = info.trigger_mask
        info_row.append()

    def process(self, blob):
        hits = blob['Hits']
        self._write_hits(hits, self.hits.row)
        if 'MCHits' in blob:
            self._write_hits(blob['MCHits'], self.mc_hits.row)
        if 'MCTracks' in blob:
            self._write_tracks(blob['MCTracks'], self.mc_tracks.row)
        if 'Evt' in blob:
            self._write_event_info(blob['Evt'], self.event_info.row)

        if not self.index % 1000:
            self.hits.flush()
            self.mc_hits.flush()
            self.mc_tracks.flush()
            self.event_info.flush()

        self.index += 1
        return blob

    def finish(self):
        self.hits.flush()
        self.mc_hits.flush()
        self.mc_tracks.flush()
        self.h5file.close()

class HDF5Pump(Pump):
    """Provides a pump for KM3NeT HDF5 files"""
    def __init__(self, **context):
        super(self.__class__, self).__init__(**context)
        self.filename = self.get('filename')
        if os.path.isfile(self.filename):
            self.h5_file = tables.File(self.filename)
        else:
            raise IOError("No such file or directory: '{0}'"
                          .format(self.filename))
        self.index = None
        self._reset_index()

    def process(self, blob):
        try:
            blob = self.get_blob(self.index)
        except KeyError:
            self._reset_index()
            raise StopIteration
        self.index += 1
        return blob

    def _get_hits(self, index, table_name='hits', where='/'):
        table = self.h5_file.get_node(where, table_name)
        _channel_id = table.channel_id[index]
        _dom_id = table.dom_id[index]
        _id = table.id[index]
        _pmt_id = table.pmt_id[index]
        _time = table.time[index]
        _tot = table.tot[index]
        _triggered = table.triggered[index]
        return HitSeries.from_arrays(_channel_id, _dom_id, _id, _pmt_id,
                                     _time, _tot, _triggered)

    def _get_tracks(self, index, table_name='tracks', where='/'):
        table = self.h5_file.get_node(where, table_name)
        _dir = table.dir[index]
        _energy = table.energy[index]
        _id = table.id[index]
        _pos = table.pos[index]
        _time = table.time[index]
        _type = table.type[index]
        return TrackSeries.from_arrays(_dir, _energy, _id, _pos,
                                       _time, _type)

    def _get_event_info(self, index, table_name='info', where='/'):
        table = self.h5_file.get_node(where, table_name)
        info = {}
        info['id'] = table.id[index]
        info['det_id'] = table.det_id[index]
        info['mc_id'] = table.mc_id[index]
        info['run_id'] = table.run_id[index]
        info['trigger_mask'] = table.trigger_mask[index]
        info['trigger_counter'] = table.trigger_counter[index]
        info['overlays'] = table.overlays[index]
        #info['timestamp'] = table.timestamp[index]
        info['mc_t'] = table.mc_t[index]
        return info

    def get_blob(self, index):
        blob = {}
        blob['Hits'] = self._get_hits(index, table_name='hits')
        blob['MCHits'] = self._get_hits(index, table_name='mc_hits')
        blob['MCTracks'] = self._get_tracks(index, table_name='mc_tracks')
        blob['EventInfo'] = self._get_event_info(index, table_name='info')
        return blob

    def finish(self):
        """Clean everything up"""
        self.h5_file.close()

    def _reset_index(self):
        """Reset index to default value"""
        self.index = 0

    def __len__(self):
        return self._n_events

    def __iter__(self):
        return self

    def next(self):
        """Python 2/3 compatibility for iterators"""
        return self.__next__()

    def __next__(self):
        if self.index >= self._n_events:
            self._reset_index()
            raise StopIteration
        blob = self.get_blob(self.index)
        self.index += 1
        return blob

    def __getitem__(self, index):
        if isinstance(index, int):
            return self.get_blob(index)
        elif isinstance(index, slice):
            return self._slice_generator(index)
        else:
            raise TypeError("index must be int or slice")

    def _slice_generator(self, index):
        """A simple slice generator for iterations"""
        start, stop, step = index.indices(len(self))
        for i in range(start, stop, step):
            yield self.get_blob(i)
<commit_msg>fix typo<commit_after># coding=utf-8
# Filename: hdf5.py
# pylint: disable=C0103,R0903
# vim:set ts=4 sts=4 sw=4 et:
"""
Pumps for the EVT simulation dataformat.

"""
from __future__ import division, absolute_import, print_function

import os.path

import tables
import numpy as np

from km3pipe import Pump, Module
from km3pipe.dataclasses import HitSeries, TrackSeries
from km3pipe.logger import logging

log = logging.getLogger(__name__)  # pylint: disable=C0103

__author__ = 'tamasgal'

POS_ATOM = tables.FloatAtom(shape=3)

class Hit(tables.IsDescription):
    channel_id = tables.UInt8Col()
    dom_id = tables.UIntCol()
    event_id = tables.UIntCol()
    id = tables.UIntCol()
    pmt_id = tables.UIntCol()
    time = tables.IntCol()
    tot = tables.UInt8Col()
    triggered = tables.BoolCol()

class Track(tables.IsDescription):
    dir = tables.FloatCol(shape=(3,))
    energy = tables.FloatCol()
    event_id = tables.UIntCol()
    id = tables.UIntCol()
    pos = tables.FloatCol(shape=(3,))
    time = tables.IntCol()
    type = tables.IntCol()

class EventInfo(tables.IsDescription):
    id = tables.IntCol()
    det_id = tables.IntCol()
    event_id = tables.UIntCol()
    frame_index = tables.UIntCol()
    mc_id = tables.IntCol()
    mc_t = tables.Float64Col()
    overlays = tables.UInt8Col()
    run_id = tables.UIntCol()
    #timestamp = tables.Float64Col()
    trigger_counter = tables.UInt64Col()
    trigger_mask = tables.UInt64Col()

class HDF5Sink(Module):
    def __init__(self, **context):
        """A Module to convert (KM3NeT) ROOT files to HDF5."""
        super(self.__class__, self).__init__(**context)
        self.filename = self.get('filename') or 'dump.h5'
        self.index = 1
        self.h5file = tables.open_file(self.filename, mode="w", title="Test file")
        self.filters = tables.Filters(complevel=5)
        self.hits = self.h5file.create_table('/', 'hits', Hit, "Hits", filters=self.filters)
        self.mc_hits = self.h5file.create_table('/', 'mc_hits', Hit, "MC Hits", filters=self.filters)
        self.mc_tracks = self.h5file.create_table('/', 'mc_tracks', Track, "MC Tracks", filters=self.filters)
        self.event_info = self.h5file.create_table('/', 'event_info', EventInfo, "Event Info", filters=self.filters)

    def _write_hits(self, hits, hit_row):
        for hit in hits:
            hit_row['channel_id'] = hit.channel_id
            hit_row['dom_id'] = hit.dom_id
            hit_row['event_id'] = self.index
            hit_row['id'] = hit.id
            hit_row['pmt_id'] = hit.pmt_id
            hit_row['time'] = hit.time
            hit_row['tot'] = hit.tot
            hit_row['triggered'] = hit.triggered
            hit_row.append()

    def _write_tracks(self, tracks, track_row):
        for track in tracks:
            track_row['dir'] = track.dir
            track_row['energy'] = track.energy
            track_row['event_id'] = self.index
            track_row['id'] = track.id
            track_row['pos'] = track.pos
            track_row['time'] = track.time
            track_row['type'] = track.type
            track_row.append()

    def _write_event_info(self, info, info_row):
        info_row['det_id'] = info.det_id
        info_row['event_id'] = self.index
        info_row['frame_index'] = info.frame_index
        info_row['id'] = info.id
        info_row['mc_id'] = info.mc_id
        info_row['mc_t'] = info.mc_t
        info_row['overlays'] = info.overlays
        info_row['run_id'] = info.run_id
        #info_row['timestamp'] = info.timestamp
        info_row['trigger_counter'] = info.trigger_counter
        info_row['trigger_mask'] = info.trigger_mask
        info_row.append()

    def process(self, blob):
        hits = blob['Hits']
        self._write_hits(hits, self.hits.row)
        if 'MCHits' in blob:
            self._write_hits(blob['MCHits'], self.mc_hits.row)
        if 'MCTracks' in blob:
            self._write_tracks(blob['MCTracks'], self.mc_tracks.row)
        if 'Evt' in blob:
            self._write_event_info(blob['Evt'], self.event_info.row)

        if not self.index % 1000:
            self.hits.flush()
            self.mc_hits.flush()
            self.mc_tracks.flush()
            self.event_info.flush()

        self.index += 1
        return blob

    def finish(self):
        self.hits.flush()
        self.mc_hits.flush()
        self.mc_tracks.flush()
        self.h5file.close()

class HDF5Pump(Pump):
    """Provides a pump for KM3NeT HDF5 files"""
    def __init__(self, **context):
        super(self.__class__, self).__init__(**context)
        self.filename = self.get('filename')
        if os.path.isfile(self.filename):
            self.h5_file = tables.File(self.filename)
        else:
            raise IOError("No such file or directory: '{0}'"
                          .format(self.filename))
        self.index = None
        self._reset_index()

    def process(self, blob):
        try:
            blob = self.get_blob(self.index)
        except KeyError:
            self._reset_index()
            raise StopIteration
        self.index += 1
        return blob

    def _get_hits(self, index, table_name='hits', where='/'):
        table = self.h5_file.get_node(where, table_name)
        _channel_id = table.channel_id[index]
        _dom_id = table.dom_id[index]
        _id = table.id[index]
        _pmt_id = table.pmt_id[index]
        _time = table.time[index]
        _tot = table.tot[index]
        _triggered = table.triggered[index]
        return HitSeries.from_arrays(_channel_id, _dom_id, _id, _pmt_id,
                                     _time, _tot, _triggered)

    def _get_tracks(self, index, table_name='tracks', where='/'):
        table = self.h5_file.get_node(where, table_name)
        _dir = table.dir[index]
        _energy = table.energy[index]
        _id = table.id[index]
        _pos = table.pos[index]
        _time = table.time[index]
        _type = table.type[index]
        return TrackSeries.from_arrays(_dir, _energy, _id, _pos,
                                       _time, _type)

    def _get_event_info(self, index, table_name='info', where='/'):
        table = self.h5_file.get_node(where, table_name)
        info = {}
        info['id'] = table.id[index]
        info['det_id'] = table.det_id[index]
        info['mc_id'] = table.mc_id[index]
        info['run_id'] = table.run_id[index]
        info['trigger_mask'] = table.trigger_mask[index]
        info['trigger_counter'] = table.trigger_counter[index]
        info['overlays'] = table.overlays[index]
        #info['timestamp'] = table.timestamp[index]
        info['mc_t'] = table.mc_t[index]
        return info

    def get_blob(self, index):
        blob = {}
        blob['Hits'] = self._get_hits(index, table_name='hits')
        blob['MCHits'] = self._get_hits(index, table_name='mc_hits')
        blob['MCTracks'] = self._get_tracks(index, table_name='mc_tracks')
        blob['EventInfo'] = self._get_event_info(index, table_name='info')
        return blob

    def finish(self):
        """Clean everything up"""
        self.h5_file.close()

    def _reset_index(self):
        """Reset index to default value"""
        self.index = 0

    def __len__(self):
        return self._n_events

    def __iter__(self):
        return self

    def next(self):
        """Python 2/3 compatibility for iterators"""
        return self.__next__()

    def __next__(self):
        if self.index >= self._n_events:
            self._reset_index()
            raise StopIteration
        blob = self.get_blob(self.index)
        self.index += 1
        return blob

    def __getitem__(self, index):
        if isinstance(index, int):
            return self.get_blob(index)
        elif isinstance(index, slice):
            return self._slice_generator(index)
        else:
            raise TypeError("index must be int or slice")

    def _slice_generator(self, index):
        """A simple slice generator for iterations"""
        start, stop, step = index.indices(len(self))
        for i in range(start, stop, step):
            yield self.get_blob(i)
<|endoftext|>


<commit_before>__version__ = "0.2.4"

if __name__ == "__main__":  # pragma: no cover
    print(__version__)
<commit_msg>bump version<commit_after>__version__ = "0.2.5"

if __name__ == "__main__":  # pragma: no cover
    print(__version__)
<|endoftext|>


<commit_before>from django.core.urlresolvers import reverse
from django.http import HttpRequest
from django.test import TestCase
from django.test.testcases import LiveServerTestCase

from selenium import webdriver
from jellyblog.views import index
from .models import Category, Note, Document

class NoteViewTest(LiveServerTestCase):
    def setUp(self):
        self.browser = webdriver.Firefox()
        self.note_content1 = "note test 111"
        self.note_content2 = "note test 222"
        Note.objects.create(content=self.note_content1)
        Note.objects.create(content=self.note_content2)

    def tearDown(self):
        self.browser.quit()

    def test_get_notes(self):
        self.browser.implicitly_wait(5)
        self.browser.get(self.live_server_url + reverse("blog_index"))
        elem = self.browser.find_element_by_xpath("//*")
        source_code = elem.get_attribute("outerHTML")

        self.assertIn(self.note_content1, source_code)
        self.assertIn(self.note_content2, source_code)

class DocumentViewTest(TestCase):
    @classmethod
    def setUpClass(cls):
        super(DocumentViewTest, cls).setUpClass()
        Category.init_category()

    def setUp(self):
        category = Category.objects.get(pk=1)
        self.test_doc1 = Document.objects.create(
            category=category,
            title="Document Test1",
            content="Content content content",
            meta_tag="metametametametametameta",
            public_doc=True
        )

        self.test_doc2 = Document.objects.create(
            category=category,
            title="Document Test2",
            content="Content2 content2 content2",
            meta_tag="metametametametametameta",
            public_doc=False
        )

    def test_get_document(self):
        request = HttpRequest()
        response = index(request)

        self.assertIn(self.test_doc1.title, response.content.decode())
        self.assertIn(self.test_doc1.content, response.content.decode())

    def test_about_private_doc(self):
        request = HttpRequest()
        response = index(request)
        self.assertNotIn(self.test_doc2.title, response.content.decode())
        self.assertNotIn(self.test_doc2.content, response.content.decode())

<commit_msg>update wait time<commit_after>from django.core.urlresolvers import reverse
from django.http import HttpRequest
from django.test import TestCase
from django.test.testcases import LiveServerTestCase

from selenium import webdriver
from jellyblog.views import index
from .models import Category, Note, Document

class NoteViewTest(LiveServerTestCase):
    def setUp(self):
        self.browser = webdriver.Firefox()
        self.note_content1 = "note test 111"
        self.note_content2 = "note test 222"
        Note.objects.create(content=self.note_content1)
        Note.objects.create(content=self.note_content2)

    def tearDown(self):
        self.browser.quit()

    def test_get_notes(self):
        self.browser.implicitly_wait(10)
        self.browser.get(self.live_server_url + reverse("blog_index"))
        elem = self.browser.find_element_by_xpath("//*")
        source_code = elem.get_attribute("outerHTML")

        self.assertIn(self.note_content1, source_code)
        self.assertIn(self.note_content2, source_code)

class DocumentViewTest(TestCase):
    @classmethod
    def setUpClass(cls):
        super(DocumentViewTest, cls).setUpClass()
        Category.init_category()

    def setUp(self):
        category = Category.objects.get(pk=1)
        self.test_doc1 = Document.objects.create(
            category=category,
            title="Document Test1",
            content="Content content content",
            meta_tag="metametametametametameta",
            public_doc=True
        )

        self.test_doc2 = Document.objects.create(
            category=category,
            title="Document Test2",
            content="Content2 content2 content2",
            meta_tag="metametametametametameta",
            public_doc=False
        )

    def test_get_document(self):
        request = HttpRequest()
        response = index(request)

        self.assertIn(self.test_doc1.title, response.content.decode())
        self.assertIn(self.test_doc1.content, response.content.decode())

    def test_about_private_doc(self):
        request = HttpRequest()
        response = index(request)
        self.assertNotIn(self.test_doc2.title, response.content.decode())
        self.assertNotIn(self.test_doc2.content, response.content.decode())

<|endoftext|>


<commit_before>import sys
from piece import King, Queen, Rook, Bishop, Knight, Pawn
from util.enums import Player, Color

class CharacterMap:
    CHARACTERS = {
        (Player.WHITE, King): ('\u2654', 'k', 'K'),
        (Player.WHITE, Queen): ('\u2655', 'q', 'Q'),
        (Player.WHITE, Rook): ('\u2656', 'r', 'R'),
        (Player.WHITE, Bishop): ('\u2657', 'b', 'B'),
        (Player.WHITE, Knight): ('\u2658', 'n', 'N'),
        (Player.WHITE, Pawn): ('\u2659', 'p', ''),
        (Player.BLACK, King): ('\u265A', 'K', 'K'),
        (Player.BLACK, Queen): ('\u265B', 'Q', 'Q'),
        (Player.BLACK, Rook): ('\u265C', 'R', 'R'),
        (Player.BLACK, Bishop): ('\u265D', 'B', 'B'),
        (Player.BLACK, Knight): ('\u265E', 'N', 'N'),
        (Player.BLACK, Pawn): ('\u265F', 'P', ''),
        (Color.WHITE, None): ('\u2610', ' ', ' '),
        (Color.BLACK, None): ('\u2610', ' ', ' '),
    }

    def __init__(self):
        self.unicode = self._unicode()

    @classmethod
    def _unicode(cls):
        for c, _, _ in cls.CHARACTERS.values():
            try:
                c.encode(sys.stdout.encoding)
            except UnicodeEncodeError:
                return False
        return True

    def character(self, player_or_color, piece_type=None):
        return self.CHARACTERS[(player_or_color, piece_type)][1]

    def unicode_character(self, player_or_color, piece_type=None):
        return self.CHARACTERS[(player_or_color, piece_type)][0]

    def algebraic_notation(self, player, piece_type):
        return self.CHARACTERS[(player, piece_type)][2]

    def __call__(self, player_or_color, piece_type=None):
        if self.unicode:
            return self.unicode_character(player_or_color, piece_type)
        return self.character(player_or_color, piece_type)
<commit_msg>update comments<commit_after>import sys
from piece import King, Queen, Rook, Bishop, Knight, Pawn
from util.enums import Player, Color

class CharacterMap:
    """
    Map for various representations of the chess pieces
    """
    CHARACTERS = {
        (Player.WHITE, King): ('\u2654', 'k', 'K'),  # (unicode, ascii, algebraic)
        (Player.WHITE, Queen): ('\u2655', 'q', 'Q'),
        (Player.WHITE, Rook): ('\u2656', 'r', 'R'),
        (Player.WHITE, Bishop): ('\u2657', 'b', 'B'),
        (Player.WHITE, Knight): ('\u2658', 'n', 'N'),
        (Player.WHITE, Pawn): ('\u2659', 'p', ''),
        (Player.BLACK, King): ('\u265A', 'K', 'K'),
        (Player.BLACK, Queen): ('\u265B', 'Q', 'Q'),
        (Player.BLACK, Rook): ('\u265C', 'R', 'R'),
        (Player.BLACK, Bishop): ('\u265D', 'B', 'B'),
        (Player.BLACK, Knight): ('\u265E', 'N', 'N'),
        (Player.BLACK, Pawn): ('\u265F', 'P', ''),
        (Color.WHITE, None): ('\u2610', ' ', ' '),
        (Color.BLACK, None): ('\u2610', ' ', ' '),
    }

    def __init__(self):
        self.unicode = self._unicode()

    @classmethod
    def _unicode(cls):
        """ Check if unicode characters can be printed to stdout

        :return: boolean
        """
        for c, _, _ in cls.CHARACTERS.values():
            try:
                c.encode(sys.stdout.encoding)
            except UnicodeEncodeError:
                return False
        return True

    def unicode_character(self, player_or_color, piece_type=None):
        """ Return the unicode character representation of a piece or tile

        :param player_or_color: Player enum or Color enum
        :param piece_type: Class of piece (e.g. Pawn) or None if looking for tile character
        :return: str
        """
        return self.CHARACTERS[(player_or_color, piece_type)][0]

    def character(self, player_or_color, piece_type=None):
        """ Return the ascii character representation of a piece or tile

        :param player_or_color: Player enum or Color enum
        :param piece_type: Class of piece (e.g. Pawn) or None if looking for tile character
        :return: str
        """
        return self.CHARACTERS[(player_or_color, piece_type)][1]

    def algebraic_notation(self, player, piece_type):
        """ Return the algebraic notation for a piece

        :param player: Player enum
        :param piece_type: Class of piece (e.g. Pawn)
        :return: str
        """
        return self.CHARACTERS[(player, piece_type)][2]

    def __call__(self, player_or_color, piece_type=None):
        """ Return the unicode/ascii character representation of a piece or tile depending on what is printable

        :param player_or_color: Player enum or Color enum
        :param piece_type: Class of piece (e.g. Pawn) or None if looking for tile character
        :return: str
        """
        # return unicode if available to print
        if self.unicode:
            return self.unicode_character(player_or_color, piece_type)
        return self.character(player_or_color, piece_type)
<|endoftext|>


<commit_before>try:
    from ... import editor
except ValueError:
    from floo import editor
from .. import msg, event_emitter, shared as G, utils

class BaseHandler(event_emitter.EventEmitter):
    PROTOCOL = None

    def __init__(self):
        super(BaseHandler, self).__init__()
        self.joined_workspace = False
        G.AGENT = self
        # TODO: removeme?
        utils.reload_settings()
        self.req_ids = {}

    def build_protocol(self, *args):
        self.proto = self.PROTOCOL(*args)
        self.proto.on('data', self.on_data)
        self.proto.on('connect', self.on_connect)
        return self.proto

    def send(self, d):
        """@return the request id"""
        req_id = self.proto.put(d)
        self.req_ids[req_id] = d.get('name', '?')
        return req_id

    def on_data(self, name, data):
        req_id = data.get('res_id')
        if req_id is not None:
            try:
                del self.req_ids[req_id]
            except KeyError:
                msg.warn('No outstanding req_id ', req_id)

        handler = getattr(self, '_on_%s' % name, None)
        if handler:
            return handler(data)
        msg.debug('unknown event name ', name, ' data: ', data)

    @property
    def client(self):
        return editor.name()

    @property
    def codename(self):
        return editor.codename()

    def _on_ack(self, data):
        msg.debug('Ack ', data)

    def _on_error(self, data):
        message = 'Error from Floobits server: %s' % str(data.get('msg'))
        msg.error(message)
        if data.get('flash'):
            editor.error_message(message)

    def _on_disconnect(self, data):
        message = 'Disconnected from server! Reason: %s' % str(data.get('reason'))
        msg.error(message)
        editor.error_message(message)
        self.stop()

    def stop(self):
        from .. import reactor
        if self.req_ids:
            msg.warn("Unresponded msgs", self.req_ids)
            self.req_ids = {}
        reactor.reactor.stop_handler(self)
        if G.AGENT is self:
            G.AGENT = None

    def is_ready(self):
        return self.joined_workspace

    def tick(self):
        pass
<commit_msg>Don't send None.<commit_after>try:
    from ... import editor
except ValueError:
    from floo import editor
from .. import msg, event_emitter, shared as G, utils

class BaseHandler(event_emitter.EventEmitter):
    PROTOCOL = None

    def __init__(self):
        super(BaseHandler, self).__init__()
        self.joined_workspace = False
        G.AGENT = self
        # TODO: removeme?
        utils.reload_settings()
        self.req_ids = {}

    def build_protocol(self, *args):
        self.proto = self.PROTOCOL(*args)
        self.proto.on('data', self.on_data)
        self.proto.on('connect', self.on_connect)
        return self.proto

    def send(self, d):
        """@return the request id"""
        if not d:
            return
        req_id = self.proto.put(d)
        self.req_ids[req_id] = d.get('name', '?')
        return req_id

    def on_data(self, name, data):
        req_id = data.get('res_id')
        if req_id is not None:
            try:
                del self.req_ids[req_id]
            except KeyError:
                msg.warn('No outstanding req_id ', req_id)

        handler = getattr(self, '_on_%s' % name, None)
        if handler:
            return handler(data)
        msg.debug('unknown event name ', name, ' data: ', data)

    @property
    def client(self):
        return editor.name()

    @property
    def codename(self):
        return editor.codename()

    def _on_ack(self, data):
        msg.debug('Ack ', data)

    def _on_error(self, data):
        message = 'Error from Floobits server: %s' % str(data.get('msg'))
        msg.error(message)
        if data.get('flash'):
            editor.error_message(message)

    def _on_disconnect(self, data):
        message = 'Disconnected from server! Reason: %s' % str(data.get('reason'))
        msg.error(message)
        editor.error_message(message)
        self.stop()

    def stop(self):
        from .. import reactor
        if self.req_ids:
            msg.warn("Unresponded msgs", self.req_ids)
            self.req_ids = {}
        reactor.reactor.stop_handler(self)
        if G.AGENT is self:
            G.AGENT = None

    def is_ready(self):
        return self.joined_workspace

    def tick(self):
        pass
<|endoftext|>


<commit_before>"""
A query interface to retrieve blog models and tags.
"""
from calendar import monthrange
from datetime import datetime, timedelta
from django.contrib.contenttypes.models import ContentType
from django.db.models.aggregates import Count
from fluent_blogs.models.db import get_entry_model
from fluent_blogs.utils.compat import utc

__all__ = (
    'query_entries',
    'query_tags',
)

ENTRY_ORDER_BY_FIELDS = {
    'slug': 'slug',
    'title': 'title',
    'author': ('author__first_name', 'author__last_name'),
    'author_slug': ('author__username',),
    'category': ('categories__name',),
    'category_slug': ('categories__slug',),
    'tag': ('tags__name',),
    'tag_slug': ('tags__slug',),
    'date': ('publication_date',),
    'year': ('publication_date',),
}

TAG_ORDER_BY_FIELDS = {
    'slug': ('slug',),
    'name': ('name',),
    'count': ('count',),
}

ORDER_BY_DESC = (
    'date', 'year', 'month', 'day', 'count',
)

def _get_order_by(order, orderby, order_by_fields):
    """
    Return the order by syntax for a model.
    Checks whether use ascending or descending order, and maps the fieldnames.
    """
    try:
        # Find the actual database fieldnames for the keyword.
        db_fieldnames = order_by_fields[orderby]
    except KeyError:
        raise ValueError("Invalid value for 'orderby': '{0}', supported values are: {1}".format(orderby, ', '.join(sorted(order_by_fields.keys()))))

    # Default to descending for some fields, otherwise be ascending
    is_desc = (not order and orderby in ORDER_BY_DESC) \
           or (order or 'asc').lower() in ('desc', 'descending')

    if is_desc:
        return map(lambda name: '-' + name, db_fieldnames)
    else:
        return db_fieldnames

def query_entries(queryset=None,
        year=None, month=None, day=None,
        category=None, category_slug=None,
        tag=None, tag_slug=None,
        author=None, author_slug=None,
        future=False,
        order=None,
        orderby=None,
        limit=None,
    ):
    """
    Query the entries using a set of predefined filters.
    This interface is mainly used by the ``get_entries`` template tag.
    """
    if queryset is None:
        queryset = get_entry_model().objects.all()

    if not future:
        queryset = queryset.published()

    if year:
        queryset = queryset.filter(publication_date__year=year)
    if month:
        queryset = queryset.filter(publication_date__month=month)
    if day:
        queryset = queryset.filter(publication_date__day=day)

    # The main category/tag/author filters
    if category:
        if isinstance(category, basestring):
            queryset = queryset.filter(categories__slug=category)
        elif isinstance(category, (int, long)):
            queryset = queryset.filter(categories=category)
        else:
            raise ValueError("Expected slug or ID for the 'category' parameter")
    if category_slug:
        queryset = queryset.filter(categories__slug=category)

    if tag:
        if isinstance(tag, basestring):
            queryset = queryset.filter(tags__slug=tag)
        elif isinstance(tag, (int, long)):
            queryset = queryset.filter(tags=tag)
        else:
            raise ValueError("Expected slug or ID for 'tag' parameter.")
    if tag_slug:
        queryset = queryset.filter(tags__slug=tag)

    if author:
        if isinstance(author, basestring):
            queryset = queryset.filter(author__username=author)
        elif isinstance(author, (int, long)):
            queryset = queryset.filter(author=author)
        else:
            raise ValueError("Expected slug or ID for 'author' parameter.")
    if author_slug:
        queryset = queryset.filter(author__username=author_slug)

    # Ordering
    if orderby:
        queryset = queryset.order_by(*_get_order_by(order, orderby, ENTRY_ORDER_BY_FIELDS))
    else:
        queryset = queryset.order_by('-publication_date')

    # Limit
    if limit:
        queryset = queryset[:limit]

    return queryset

def query_tags(order=None, orderby=None, limit=None):
    """
    Query the tags, with usage count included.
    This interface is mainly used by the ``get_tags`` template tag.
    """
    from taggit.models import Tag, TaggedItem    # feature is still optional
    EntryModel = get_entry_model()
    ct = ContentType.objects.get_for_model(EntryModel)  # take advantage of local caching.
    entry_tag_ids = TaggedItem.objects.filter(content_type=ct).filter(entry__status=get_entry_model().PUBLISHED).values_list('tag_id')

    # get tags
    queryset = Tag.objects.filter(id__in=entry_tag_ids)
    queryset = queryset.annotate(count=Count('taggit_taggeditem_items'))  # Related name differed in Django 1.2

    # Ordering
    if orderby:
        queryset = queryset.order_by(*_get_order_by(order, orderby, TAG_ORDER_BY_FIELDS))
    else:
        queryset = queryset.order_by('-count')

    # Limit
    if limit:
        queryset = queryset[:limit]

    return queryset

def get_date_range(year=None, month=None, day=None):
    """
    Return a start..end range to query for a specific month, day or year.
    """
    if year is None:
        return None

    if month is None:
        # year only
        start = datetime(year, 1, 1, 0, 0, 0, tzinfo=utc)
        end = datetime(year, 12, 31, 23, 59, 59, 999, tzinfo=utc)
        return (start, end)

    if day is None:
        # year + month only
        start = datetime(year, month, 1, 0, 0, 0, tzinfo=utc)
        end = start + timedelta(days=monthrange(year, month)[1], microseconds=-1)
        return (start, end)
    else:
        # Exact day
        start = datetime(year, month, day, 0, 0, 0, tzinfo=utc)
        end = start + timedelta(days=1, microseconds=-1)
        return (start, end)
<commit_msg>Enhance fix for published tags (#9), support custom model names too.<commit_after>"""
A query interface to retrieve blog models and tags.
"""
from calendar import monthrange
from datetime import datetime, timedelta
from django.contrib.contenttypes.models import ContentType
from django.db.models.aggregates import Count
from fluent_blogs.models.db import get_entry_model
from fluent_blogs.utils.compat import utc

__all__ = (
    'query_entries',
    'query_tags',
)

ENTRY_ORDER_BY_FIELDS = {
    'slug': 'slug',
    'title': 'title',
    'author': ('author__first_name', 'author__last_name'),
    'author_slug': ('author__username',),
    'category': ('categories__name',),
    'category_slug': ('categories__slug',),
    'tag': ('tags__name',),
    'tag_slug': ('tags__slug',),
    'date': ('publication_date',),
    'year': ('publication_date',),
}

TAG_ORDER_BY_FIELDS = {
    'slug': ('slug',),
    'name': ('name',),
    'count': ('count',),
}

ORDER_BY_DESC = (
    'date', 'year', 'month', 'day', 'count',
)

def _get_order_by(order, orderby, order_by_fields):
    """
    Return the order by syntax for a model.
    Checks whether use ascending or descending order, and maps the fieldnames.
    """
    try:
        # Find the actual database fieldnames for the keyword.
        db_fieldnames = order_by_fields[orderby]
    except KeyError:
        raise ValueError("Invalid value for 'orderby': '{0}', supported values are: {1}".format(orderby, ', '.join(sorted(order_by_fields.keys()))))

    # Default to descending for some fields, otherwise be ascending
    is_desc = (not order and orderby in ORDER_BY_DESC) \
           or (order or 'asc').lower() in ('desc', 'descending')

    if is_desc:
        return map(lambda name: '-' + name, db_fieldnames)
    else:
        return db_fieldnames

def query_entries(queryset=None,
        year=None, month=None, day=None,
        category=None, category_slug=None,
        tag=None, tag_slug=None,
        author=None, author_slug=None,
        future=False,
        order=None,
        orderby=None,
        limit=None,
    ):
    """
    Query the entries using a set of predefined filters.
    This interface is mainly used by the ``get_entries`` template tag.
    """
    if queryset is None:
        queryset = get_entry_model().objects.all()

    if not future:
        queryset = queryset.published()

    if year:
        queryset = queryset.filter(publication_date__year=year)
    if month:
        queryset = queryset.filter(publication_date__month=month)
    if day:
        queryset = queryset.filter(publication_date__day=day)

    # The main category/tag/author filters
    if category:
        if isinstance(category, basestring):
            queryset = queryset.filter(categories__slug=category)
        elif isinstance(category, (int, long)):
            queryset = queryset.filter(categories=category)
        else:
            raise ValueError("Expected slug or ID for the 'category' parameter")
    if category_slug:
        queryset = queryset.filter(categories__slug=category)

    if tag:
        if isinstance(tag, basestring):
            queryset = queryset.filter(tags__slug=tag)
        elif isinstance(tag, (int, long)):
            queryset = queryset.filter(tags=tag)
        else:
            raise ValueError("Expected slug or ID for 'tag' parameter.")
    if tag_slug:
        queryset = queryset.filter(tags__slug=tag)

    if author:
        if isinstance(author, basestring):
            queryset = queryset.filter(author__username=author)
        elif isinstance(author, (int, long)):
            queryset = queryset.filter(author=author)
        else:
            raise ValueError("Expected slug or ID for 'author' parameter.")
    if author_slug:
        queryset = queryset.filter(author__username=author_slug)

    # Ordering
    if orderby:
        queryset = queryset.order_by(*_get_order_by(order, orderby, ENTRY_ORDER_BY_FIELDS))
    else:
        queryset = queryset.order_by('-publication_date')

    # Limit
    if limit:
        queryset = queryset[:limit]

    return queryset

def query_tags(order=None, orderby=None, limit=None):
    """
    Query the tags, with usage count included.
    This interface is mainly used by the ``get_tags`` template tag.
    """
    from taggit.models import Tag, TaggedItem    # feature is still optional
    EntryModel = get_entry_model()
    ct = ContentType.objects.get_for_model(EntryModel)  # take advantage of local caching.
    published_filter = {'{0}__status'.format(ct.model.lower()): EntryModel.PUBLISHED}  # e.g. entry__status=, or newsitem__status=
    entry_tag_ids = TaggedItem.objects.filter(content_type=ct).filter(**published_filter).values_list('tag_id')

    # get tags
    queryset = Tag.objects.filter(id__in=entry_tag_ids)
    queryset = queryset.annotate(count=Count('taggit_taggeditem_items'))  # Related name differed in Django 1.2

    # Ordering
    if orderby:
        queryset = queryset.order_by(*_get_order_by(order, orderby, TAG_ORDER_BY_FIELDS))
    else:
        queryset = queryset.order_by('-count')

    # Limit
    if limit:
        queryset = queryset[:limit]

    return queryset

def get_date_range(year=None, month=None, day=None):
    """
    Return a start..end range to query for a specific month, day or year.
    """
    if year is None:
        return None

    if month is None:
        # year only
        start = datetime(year, 1, 1, 0, 0, 0, tzinfo=utc)
        end = datetime(year, 12, 31, 23, 59, 59, 999, tzinfo=utc)
        return (start, end)

    if day is None:
        # year + month only
        start = datetime(year, month, 1, 0, 0, 0, tzinfo=utc)
        end = start + timedelta(days=monthrange(year, month)[1], microseconds=-1)
        return (start, end)
    else:
        # Exact day
        start = datetime(year, month, day, 0, 0, 0, tzinfo=utc)
        end = start + timedelta(days=1, microseconds=-1)
        return (start, end)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import sys
import qi
import random
import threading
from fluent import sender
from fluent import event

class FluentLoggerService:
    def __init__(self):
        self.lock = threading.RLock()
        self.running = False

    def start(self):
        with self.lock:
            if self.running:
                self.stop()
            sender.setup('pepper', host='192.168.10.119', port=24224)
            event.Event('service', {
                'status': 'started'
            })

    def stop(self):
        with self.lock:
            if self.running:
                event.Event('service', {
                    'status': 'stopped'
                })
            self.running = False

def main():
    app = qi.Application()
    app.start()
    session = app.session
    myService = FluentLoggerService()
    session.registerService("FluentLoggerService", myService)
    myService.start()
    app.run()

if __name__ == "__main__":
    main()<commit_msg>Add preference support<commit_after># -*- coding: utf-8 -*-
import sys
import qi
import random
import threading
from fluent import sender
from fluent import event

PREF_DOMAIN = 'com.github.yacchin1205.fluentlogger'

class FluentLoggerService:
    def __init__(self, session):
        self.session = session
        self.lock = threading.RLock()
        self.running = False

    def start(self):
        with self.lock:
            if self.running:
                self.stop()
            host = self._get_pref('host')
            if host is not None:
                tag = self._get_pref('tag', 'pepper')
                sender.setup(tag, host=host,
                             port=int(self._get_pref('port', '24224')))
                event.Event('service', {
                    'status': 'started'
                })
                self.running = True

    def stop(self):
        with self.lock:
            if self.running:
                event.Event('service', {
                    'status': 'stopped'
                })
            self.running = False

    def setForwarder(self, host, port):
        prefManager = self.session.service('ALPreferenceManager')
        prefManager.setValue(PREF_DOMAIN, 'host', host)
        prefManager.setValue(PREF_DOMAIN, 'port', str(port))
        self.start()

    def _get_pref(self, name, default_value=None):
        prefManager = self.session.service('ALPreferenceManager')
        value = prefManager.getValue(PREF_DOMAIN, name)
        if value is None:
            return default_value
        else:
            return value

def main():
    app = qi.Application()
    app.start()
    session = app.session
    myService = FluentLoggerService(session)
    session.registerService("FluentLoggerService", myService)
    myService.start()
    app.run()

if __name__ == "__main__":
    main()<|endoftext|>


<commit_before>from __future__ import print_function

import argparse
import logging
import os
import time

import numpy as np

import paddle
import paddle.fluid as fluid

import reader
from network_conf import ctr_dnn_model
from multiprocessing import cpu_count

# disable gpu training for this example
os.environ["CUDA_VISIBLE_DEVICES"] = ""

logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger("fluid")
logger.setLevel(logging.INFO)

def parse_args():
    parser = argparse.ArgumentParser(description="PaddlePaddle CTR example")
    parser.add_argument(
        '--train_data_path',
        type=str,
        default='./data/raw/train.txt',
        help="The path of training dataset")
    parser.add_argument(
        '--test_data_path',
        type=str,
        default='./data/raw/valid.txt',
        help="The path of testing dataset")
    parser.add_argument(
        '--batch_size',
        type=int,
        default=1000,
        help="The size of mini-batch (default:1000)")
    parser.add_argument(
        '--embedding_size',
        type=int,
        default=10,
        help="The size for embedding layer (default:10)")
    parser.add_argument(
        '--num_passes',
        type=int,
        default=10,
        help="The number of passes to train (default: 10)")
    parser.add_argument(
        '--model_output_dir',
        type=str,
        default='models',
        help='The path for model to store (default: models)')
    parser.add_argument(
        '--sparse_feature_dim',
        type=int,
        default=1000001,
        help='sparse feature hashing space for index processing')
    parser.add_argument(
        '--is_local',
        type=int,
        default=1,
        help='Local train or distributed train (default: 1)')
    parser.add_argument(
        '--cloud_train',
        type=int,
        default=0,
        help='Local train or distributed train on paddlecloud (default: 0)')
    parser.add_argument(
        '--async_mode',
        action='store_true',
        default=False,
        help='Whether start pserver in async mode to support ASGD')
    parser.add_argument(
        '--no_split_var',
        action='store_true',
        default=False,
        help='Whether split variables into blocks when update_method is pserver')
    # the following arguments is used for distributed train, if is_local == false, then you should set them
    parser.add_argument(
        '--role',
        type=str,
        default='pserver', # trainer or pserver
        help='The path for model to store (default: models)')
    parser.add_argument(
        '--endpoints',
        type=str,
        default='127.0.0.1:6000',
        help='The pserver endpoints, like: 127.0.0.1:6000,127.0.0.1:6001')
    parser.add_argument(
        '--current_endpoint',
        type=str,
        default='127.0.0.1:6000',
        help='The path for model to store (default: 127.0.0.1:6000)')
    parser.add_argument(
        '--trainer_id',
        type=int,
        default=0,
        help='The path for model to store (default: models)')
    parser.add_argument(
        '--trainers',
        type=int,
        default=1,
        help='The num of trianers, (default: 1)')
    parser.add_argument(
        '--enable_ce',
        action='store_true',
        help='If set, run the task with continuous evaluation logs.')
    parser.add_argument(
        '--num_threads',
        type=int,
        default=1,
        help='The num of threads, (default: 1)')

    return parser.parse_args()

def train_loop(args, train_program, py_reader, loss, auc_var, batch_auc_var,
               trainer_num, trainer_id):

    if args.enable_ce:
        SEED = 102
        train_program.random_seed = SEED
        fluid.default_startup_program().random_seed = SEED

    dataset = reader.CriteoDataset(args.sparse_feature_dim)
    train_reader = paddle.batch(
        paddle.reader.shuffle(
            dataset.train([args.train_data_path], trainer_num, trainer_id),
            buf_size=args.batch_size * 100),
        batch_size=args.batch_size)

    py_reader.decorate_paddle_reader(train_reader)
    data_name_list = []

    place = fluid.CPUPlace()
    exe = fluid.Executor(place)

    exec_strategy = fluid.ExecutionStrategy()
    build_strategy = fluid.BuildStrategy()

    if os.getenv("NUM_THREADS", ""):
        exec_strategy.num_threads = int(os.getenv("NUM_THREADS"))

    cpu_num = int(os.environ.get('CPU_NUM', cpu_count()))
    build_strategy.reduce_strategy = \
        fluid.BuildStrategy.ReduceStrategy.Reduce if cpu_num > 1 \
            else fluid.BuildStrategy.ReduceStrategy.AllReduce

    pe = fluid.ParallelExecutor(
        use_cuda=False,
        loss_name=loss.name,
        main_program=train_program,
        build_strategy=build_strategy,
        exec_strategy=exec_strategy)

    exe.run(fluid.default_startup_program())

    total_time = 0
    for pass_id in range(args.num_passes):
        pass_start = time.time()
        batch_id = 0
        py_reader.start()

        try:
            while True:
                loss_val, auc_val, batch_auc_val = pe.run(fetch_list=[loss.name, auc_var.name, batch_auc_var.name])
                loss_val = np.mean(loss_val)
                auc_val = np.mean(auc_val)
                batch_auc_val = np.mean(batch_auc_val)

                logger.info("TRAIN --> pass: {} batch: {} loss: {} auc: {}, batch_auc: {}"
                      .format(pass_id, batch_id, loss_val/args.batch_size, auc_val, batch_auc_val))
                if batch_id % 1000 == 0 and batch_id != 0:
                    model_dir = args.model_output_dir + '/batch-' + str(batch_id)
                    if args.trainer_id == 0:
                        fluid.io.save_inference_model(model_dir, data_name_list, [loss, auc_var], exe)
                batch_id += 1
        except fluid.core.EOFException:
            py_reader.reset()
        print("pass_id: %d, pass_time_cost: %f" % (pass_id, time.time() - pass_start))

        total_time += time.time() - pass_start

        model_dir = args.model_output_dir + '/pass-' + str(pass_id)
        if args.trainer_id == 0:
            fluid.io.save_inference_model(model_dir, data_name_list, [loss, auc_var], exe)

    # only for ce
    if args.enable_ce:
        cpu_num = get_cards(args)
        print("cpu_num", cpu_num)
        epoch_idx = args.num_passes
        print("kpis\teach_pass_duration_card%s\t%s" %
                (cpu_num, total_time / epoch_idx))
        print("kpis\ttrain_loss_card%s\t%s" %
                (cpu_num, loss_val/args.batch_size))
        print("kpis\ttrain_auc_val_card%s\t%s" %
                (cpu_num, auc_val))
        print("kpis\ttrain_batch_auc_val_card%s\t%s" %
                (cpu_num, batch_auc_val))

def train():
    args = parse_args()

    if not os.path.isdir(args.model_output_dir):
        os.mkdir(args.model_output_dir)

    loss, auc_var, batch_auc_var, py_reader = ctr_dnn_model(args.embedding_size, args.sparse_feature_dim)
    optimizer = fluid.optimizer.Adam(learning_rate=1e-4)
    optimizer.minimize(loss)
    if args.cloud_train:
        # the port of all pservers, needed by both trainer and pserver
        port = os.getenv("PADDLE_PORT", "6174")
        # comma separated ips of all pservers, needed by trainer and
        pserver_ips = os.getenv("PADDLE_PSERVERS", "")
        eplist = []
        for ip in pserver_ips.split(","):
            eplist.append(':'.join([ip, port]))
        args.endpoints = ",".join(eplist)
        args.trainers = int(os.getenv("PADDLE_TRAINERS_NUM", "1"))
        args.current_endpoint = os.getenv("POD_IP", "localhost") + ":" + port
        args.role = os.getenv("TRAINING_ROLE", "TRAINER")
        args.trainer_id = int(os.getenv("PADDLE_TRAINER_ID", "0"))
        args.is_local = bool(int(os.getenv("PADDLE_IS_LOCAL", 0)))

    if args.is_local:
        logger.info("run local training")
        main_program = fluid.default_main_program()
        train_loop(args, main_program, py_reader, loss, auc_var, batch_auc_var, 1, 0)
    else:
        logger.info("run dist training")
        t = fluid.DistributeTranspiler()
        t.transpile(args.trainer_id, pservers=args.endpoints, trainers=args.trainers)
        if args.role == "pserver" or args.role == "PSERVER":
            logger.info("run pserver")
            prog = t.get_pserver_program(args.current_endpoint)
            startup = t.get_startup_program(args.current_endpoint, pserver_program=prog)
            exe = fluid.Executor(fluid.CPUPlace())
            exe.run(startup)
            exe.run(prog)
        elif args.role == "trainer" or args.role == "TRAINER":
            logger.info("run trainer")
            train_prog = t.get_trainer_program()
            train_loop(args, train_prog, py_reader, loss, auc_var, batch_auc_var,
                       args.trainers, args.trainer_id)
        else:
            raise ValueError(
                'PADDLE_TRAINING_ROLE environment variable must be either TRAINER or PSERVER'
            )

def get_cards(args):
    if args.enable_ce:
        cards = os.environ.get('NUM_THREADS', 1)
        print("cards", cards)

        return int(cards)
    else:
        print("return args.num_threads")
        return args.num_threads

if __name__ == '__main__':
    train()
<commit_msg>add ctr ce<commit_after>from __future__ import print_function

import argparse
import logging
import os
import time

import numpy as np

import paddle
import paddle.fluid as fluid

import reader
from network_conf import ctr_dnn_model
from multiprocessing import cpu_count

# disable gpu training for this example
os.environ["CUDA_VISIBLE_DEVICES"] = ""

logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger("fluid")
logger.setLevel(logging.INFO)

def parse_args():
    parser = argparse.ArgumentParser(description="PaddlePaddle CTR example")
    parser.add_argument(
        '--train_data_path',
        type=str,
        default='./data/raw/train.txt',
        help="The path of training dataset")
    parser.add_argument(
        '--test_data_path',
        type=str,
        default='./data/raw/valid.txt',
        help="The path of testing dataset")
    parser.add_argument(
        '--batch_size',
        type=int,
        default=1000,
        help="The size of mini-batch (default:1000)")
    parser.add_argument(
        '--embedding_size',
        type=int,
        default=10,
        help="The size for embedding layer (default:10)")
    parser.add_argument(
        '--num_passes',
        type=int,
        default=10,
        help="The number of passes to train (default: 10)")
    parser.add_argument(
        '--model_output_dir',
        type=str,
        default='models',
        help='The path for model to store (default: models)')
    parser.add_argument(
        '--sparse_feature_dim',
        type=int,
        default=1000001,
        help='sparse feature hashing space for index processing')
    parser.add_argument(
        '--is_local',
        type=int,
        default=1,
        help='Local train or distributed train (default: 1)')
    parser.add_argument(
        '--cloud_train',
        type=int,
        default=0,
        help='Local train or distributed train on paddlecloud (default: 0)')
    parser.add_argument(
        '--async_mode',
        action='store_true',
        default=False,
        help='Whether start pserver in async mode to support ASGD')
    parser.add_argument(
        '--no_split_var',
        action='store_true',
        default=False,
        help='Whether split variables into blocks when update_method is pserver')
    # the following arguments is used for distributed train, if is_local == false, then you should set them
    parser.add_argument(
        '--role',
        type=str,
        default='pserver', # trainer or pserver
        help='The path for model to store (default: models)')
    parser.add_argument(
        '--endpoints',
        type=str,
        default='127.0.0.1:6000',
        help='The pserver endpoints, like: 127.0.0.1:6000,127.0.0.1:6001')
    parser.add_argument(
        '--current_endpoint',
        type=str,
        default='127.0.0.1:6000',
        help='The path for model to store (default: 127.0.0.1:6000)')
    parser.add_argument(
        '--trainer_id',
        type=int,
        default=0,
        help='The path for model to store (default: models)')
    parser.add_argument(
        '--trainers',
        type=int,
        default=1,
        help='The num of trianers, (default: 1)')
    parser.add_argument(
        '--enable_ce',
        action='store_true',
        help='If set, run the task with continuous evaluation logs.')
    parser.add_argument(
        '--num_threads',
        type=int,
        default=1,
        help='The num of threads, (default: 1)')

    return parser.parse_args()

def train_loop(args, train_program, py_reader, loss, auc_var, batch_auc_var,
               trainer_num, trainer_id):

    if args.enable_ce:
        SEED = 102
        train_program.random_seed = SEED
        fluid.default_startup_program().random_seed = SEED

    dataset = reader.CriteoDataset(args.sparse_feature_dim)
    train_reader = paddle.batch(
        paddle.reader.shuffle(
            dataset.train([args.train_data_path], trainer_num, trainer_id),
            buf_size=args.batch_size * 100),
        batch_size=args.batch_size)

    py_reader.decorate_paddle_reader(train_reader)
    data_name_list = []

    place = fluid.CPUPlace()
    exe = fluid.Executor(place)

    exec_strategy = fluid.ExecutionStrategy()
    build_strategy = fluid.BuildStrategy()

    if os.getenv("NUM_THREADS", ""):
        exec_strategy.num_threads = int(os.getenv("NUM_THREADS"))

    cpu_num = int(os.environ.get('CPU_NUM', cpu_count()))
    build_strategy.reduce_strategy = \
        fluid.BuildStrategy.ReduceStrategy.Reduce if cpu_num > 1 \
            else fluid.BuildStrategy.ReduceStrategy.AllReduce

    pe = fluid.ParallelExecutor(
        use_cuda=False,
        loss_name=loss.name,
        main_program=train_program,
        build_strategy=build_strategy,
        exec_strategy=exec_strategy)

    exe.run(fluid.default_startup_program())

    total_time = 0
    for pass_id in range(args.num_passes):
        pass_start = time.time()
        batch_id = 0
        py_reader.start()

        try:
            while True:
                loss_val, auc_val, batch_auc_val = pe.run(fetch_list=[loss.name, auc_var.name, batch_auc_var.name])
                loss_val = np.mean(loss_val)
                auc_val = np.mean(auc_val)
                batch_auc_val = np.mean(batch_auc_val)

                logger.info("TRAIN --> pass: {} batch: {} loss: {} auc: {}, batch_auc: {}"
                      .format(pass_id, batch_id, loss_val/args.batch_size, auc_val, batch_auc_val))
                if batch_id % 1000 == 0 and batch_id != 0:
                    model_dir = args.model_output_dir + '/batch-' + str(batch_id)
                    if args.trainer_id == 0:
                        fluid.io.save_inference_model(model_dir, data_name_list, [loss, auc_var], exe)
                batch_id += 1
        except fluid.core.EOFException:
            py_reader.reset()
        print("pass_id: %d, pass_time_cost: %f" % (pass_id, time.time() - pass_start))

        total_time += time.time() - pass_start

        model_dir = args.model_output_dir + '/pass-' + str(pass_id)
        if args.trainer_id == 0:
            fluid.io.save_inference_model(model_dir, data_name_list, [loss, auc_var], exe)

    # only for ce
    if args.enable_ce:
        cpu_num = get_cards(args)
        epoch_idx = args.num_passes
        print("kpis\teach_pass_duration_card%s\t%s" %
                (cpu_num, total_time / epoch_idx))
        print("kpis\ttrain_loss_card%s\t%s" %
                (cpu_num, loss_val/args.batch_size))
        print("kpis\ttrain_auc_val_card%s\t%s" %
                (cpu_num, auc_val))
        print("kpis\ttrain_batch_auc_val_card%s\t%s" %
                (cpu_num, batch_auc_val))

def train():
    args = parse_args()

    if not os.path.isdir(args.model_output_dir):
        os.mkdir(args.model_output_dir)

    loss, auc_var, batch_auc_var, py_reader = ctr_dnn_model(args.embedding_size, args.sparse_feature_dim)
    optimizer = fluid.optimizer.Adam(learning_rate=1e-4)
    optimizer.minimize(loss)
    if args.cloud_train:
        # the port of all pservers, needed by both trainer and pserver
        port = os.getenv("PADDLE_PORT", "6174")
        # comma separated ips of all pservers, needed by trainer and
        pserver_ips = os.getenv("PADDLE_PSERVERS", "")
        eplist = []
        for ip in pserver_ips.split(","):
            eplist.append(':'.join([ip, port]))
        args.endpoints = ",".join(eplist)
        args.trainers = int(os.getenv("PADDLE_TRAINERS_NUM", "1"))
        args.current_endpoint = os.getenv("POD_IP", "localhost") + ":" + port
        args.role = os.getenv("TRAINING_ROLE", "TRAINER")
        args.trainer_id = int(os.getenv("PADDLE_TRAINER_ID", "0"))
        args.is_local = bool(int(os.getenv("PADDLE_IS_LOCAL", 0)))

    if args.is_local:
        logger.info("run local training")
        main_program = fluid.default_main_program()
        train_loop(args, main_program, py_reader, loss, auc_var, batch_auc_var, 1, 0)
    else:
        logger.info("run dist training")
        t = fluid.DistributeTranspiler()
        t.transpile(args.trainer_id, pservers=args.endpoints, trainers=args.trainers)
        if args.role == "pserver" or args.role == "PSERVER":
            logger.info("run pserver")
            prog = t.get_pserver_program(args.current_endpoint)
            startup = t.get_startup_program(args.current_endpoint, pserver_program=prog)
            exe = fluid.Executor(fluid.CPUPlace())
            exe.run(startup)
            exe.run(prog)
        elif args.role == "trainer" or args.role == "TRAINER":
            logger.info("run trainer")
            train_prog = t.get_trainer_program()
            train_loop(args, train_prog, py_reader, loss, auc_var, batch_auc_var,
                       args.trainers, args.trainer_id)
        else:
            raise ValueError(
                'PADDLE_TRAINING_ROLE environment variable must be either TRAINER or PSERVER'
            )

def get_cards(args):
    if args.enable_ce:
        cards = os.environ.get('NUM_THREADS', 1)
        return int(cards)
    else:
        return args.num_threads

if __name__ == '__main__':
    train()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Copyright (c) 2015 ObjectLabs Corporation
# Distributed under the MIT license - http://opensource.org/licenses/MIT

from pymongo import MongoClient  # pymongo>=3.0
import os

"""
PyMongo Production Connection Example
The following example uses PyMongo to connect to a MongoDB deployment using the options we have found most appropriate to
production applications. This example supports both replica set and SSL connections.
For demonstration purposes, configuration is hard-coded; in practice, configuration should be externalized e.g. into a file or
environment variables.
  ##########################################################################################################################
  WARNING: This example requires PyMongo 3.0 or later to work correctly. In particular, if you are connecting over SSL
           with an earlier driver version additional configuration will be required for it to properly authenticate the
           server.
  ##########################################################################################################################
"""

# Your deployment's URI in the standard format (http://docs.mongodb.org/manual/reference/connection-string/).
#
# The URI can be found via the MongoLab management portal (http://docs.mongolab.com/connecting/#connect-string).
#
# If you are using a PaaS add-on integration e.g. via Heroku, the URI is usually available via an environment variable, often
# named "MONGOLAB_URI". Consult the documentation for the add-on you are using.
mongolab_uri = os.environ['MONGOLAB_URI']

# Pass the following keyword arguments to ensure proper production behavior:
#
#   connectTimeoutMS  30 s to allow for PaaS warm-up; adjust down as needed for faster failures. For more, see docs:
#                     http://docs.mongolab.com/timeouts/#connection-timeout
#
#   socketTimeoutMS   No timeout (None) to allow for long-running operations
#                     (http://docs.mongolab.com/timeouts/#socket-timeout).
#
#   socketKeepAlive   Enabled (True) to ensure idle connections are kept alive in the presence of a firewall.
#
# See PyMongo docs for more details about the connection options:
#
#   https://api.mongodb.org/python/3.0/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient
#

class SmartHomeDB:

	def __init__(self):
		self.client = client = MongoClient(mongolab_uri,
                     connectTimeoutMS=30000,
                     socketTimeoutMS=None,
                     socketKeepAlive=True)
		self.db = client.get_default_database()
		self.temperature_collection = self.db['temperature']
		self.cur_stats=self.db['smart_home_stats']

	def getTemperaturePoints(self):
		count = max(self.temperature_collection.count()-48,0)
		record = self.temperature_collection.find({},{'timestamp':1, 'temperature':1,'humidity':1,'device_name':1,'_id':0}).skip(count)
		return list(record)

	def insertTemperaturePoint(self,tempData):
		record = {};
		record['temperature'] = tempData['temperature']
		record['humidity'] = tempData['humidity'];
		record['timestamp'] = tempData['timestamp']
		record['device_name'] = tempData['device_name']
		self.temperature_collection.insert_one(record)

	def updateCurrentStats(self,stats):
		record = {}
		record['cur_temperature'] = stats['cur_temperature']
		record['cur_humidity'] = stats['cur_humidity']
		record['device_name'] = stats['device_name']
		record['timestamp'] = stats['timestamp']
		self.cur_stats.update({"device_name":{"$eq":stats['device_name']}}, record,upsert=True)

	def getCurrentStats(self,device_name):
		record = self.cur_stats.find_one({"device_name":{"$eq":device_name}},{ 'timestamp':1, 'cur_temperature':1,'cur_humidity':1,'device_name':1,'_id':0})
		if record is not None:
			record_info = dict(record)
			try:
				record_info['temperature'] = format(float(record_info['temperature']),'.2f')
				record_info['humidity'] = format(float(record_info['humidity']),'.2f')
			except e:
				print(e)
			return record_info

		return dict()

def run_tests():
	smartDB = SmartHomeDB()
	smartDB.updateCurrentStats({'cur_temperature':12.0,'cur_humidity':23.0,'device_name':'raspberrypi','timestamp':123})
	record = smartDB.getCurrentStats('raspberrypi')
	print(record)

if __name__=='__main__':
	run_tests()

<commit_msg>fixing format issues<commit_after>#!/usr/bin/env python3

# Copyright (c) 2015 ObjectLabs Corporation
# Distributed under the MIT license - http://opensource.org/licenses/MIT

from pymongo import MongoClient  # pymongo>=3.0
import os

"""
PyMongo Production Connection Example
The following example uses PyMongo to connect to a MongoDB deployment using the options we have found most appropriate to
production applications. This example supports both replica set and SSL connections.
For demonstration purposes, configuration is hard-coded; in practice, configuration should be externalized e.g. into a file or
environment variables.
  ##########################################################################################################################
  WARNING: This example requires PyMongo 3.0 or later to work correctly. In particular, if you are connecting over SSL
           with an earlier driver version additional configuration will be required for it to properly authenticate the
           server.
  ##########################################################################################################################
"""

# Your deployment's URI in the standard format (http://docs.mongodb.org/manual/reference/connection-string/).
#
# The URI can be found via the MongoLab management portal (http://docs.mongolab.com/connecting/#connect-string).
#
# If you are using a PaaS add-on integration e.g. via Heroku, the URI is usually available via an environment variable, often
# named "MONGOLAB_URI". Consult the documentation for the add-on you are using.
mongolab_uri = os.environ['MONGOLAB_URI']

# Pass the following keyword arguments to ensure proper production behavior:
#
#   connectTimeoutMS  30 s to allow for PaaS warm-up; adjust down as needed for faster failures. For more, see docs:
#                     http://docs.mongolab.com/timeouts/#connection-timeout
#
#   socketTimeoutMS   No timeout (None) to allow for long-running operations
#                     (http://docs.mongolab.com/timeouts/#socket-timeout).
#
#   socketKeepAlive   Enabled (True) to ensure idle connections are kept alive in the presence of a firewall.
#
# See PyMongo docs for more details about the connection options:
#
#   https://api.mongodb.org/python/3.0/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient
#

class SmartHomeDB:

	def __init__(self):
		self.client = client = MongoClient(mongolab_uri,
                     connectTimeoutMS=30000,
                     socketTimeoutMS=None,
                     socketKeepAlive=True)
		self.db = client.get_default_database()
		self.temperature_collection = self.db['temperature']
		self.cur_stats=self.db['smart_home_stats']

	def getTemperaturePoints(self):
		count = max(self.temperature_collection.count()-48,0)
		record = self.temperature_collection.find({},{'timestamp':1, 'temperature':1,'humidity':1,'device_name':1,'_id':0}).skip(count)
		return list(record)

	def insertTemperaturePoint(self,tempData):
		record = {};
		record['temperature'] = tempData['temperature']
		record['humidity'] = tempData['humidity'];
		record['timestamp'] = tempData['timestamp']
		record['device_name'] = tempData['device_name']
		self.temperature_collection.insert_one(record)

	def updateCurrentStats(self,stats):
		record = {}
		record['cur_temperature'] = stats['cur_temperature']
		record['cur_humidity'] = stats['cur_humidity']
		record['device_name'] = stats['device_name']
		record['timestamp'] = stats['timestamp']
		self.cur_stats.update({"device_name":{"$eq":stats['device_name']}}, record,upsert=True)

	def getCurrentStats(self,device_name):
		record = self.cur_stats.find_one({"device_name":{"$eq":device_name}},{ 'timestamp':1, 'cur_temperature':1,'cur_humidity':1,'device_name':1,'_id':0})
		if record is not None:
			record_info = dict(record)
			# try:
			# 	record_info['temperature'] = format(float(record_info['temperature']),'.2f')
			# 	record_info['humidity'] = format(float(record_info['humidity']),'.2f')
			# except e:
			# 	print(e)
			return record_info

		return dict()

def run_tests():
	smartDB = SmartHomeDB()
	smartDB.updateCurrentStats({'cur_temperature':12.0,'cur_humidity':23.0,'device_name':'raspberrypi','timestamp':123})
	record = smartDB.getCurrentStats('raspberrypi')
	print(record)

if __name__=='__main__':
	run_tests()

<|endoftext|>


<commit_before>'''
sphinx_cython.py

This module monkeypatches sphinx autodoc to support Cython generated
function signatures in the first line of the docstring of functions
implemented as C extensions.

Copyright (C) Nikolaus Rath <Nikolaus@rath.org>

This file is part of LLFUSE (http://python-llfuse.googlecode.com).
LLFUSE can be distributed under the terms of the GNU LGPL.
'''

import sphinx.ext.autodoc as SphinxAutodoc
from sphinx.util.docstrings import prepare_docstring
import inspect
import re
from sphinx.util import force_decode

TYPE_RE = re.compile(r'(?:int|char)(?:\s+\*?\s*|\s*\*?\s+)([a-zA-Z_].*)')

class MyDocumenter(SphinxAutodoc.Documenter):
    '''
    Overwrites `get_doc()` to remove function and
    method signatures and `format_args` to parse and give
    precedence to function signatures in the first line
    of the docstring.
    '''

    def get_doc(self, encoding=None):
        docstr = self.get_attr(self.object, '__doc__', None)

        myname = self.fullname[len(self.modname)+1:]
        if myname.endswith('()'):
            myname = myname[:-2]

        if (docstr
            and docstr.startswith(myname + '(')
            and '\n' in docstr
            and docstr[docstr.index('\n')-1] == ')'):
            docstr = docstr[docstr.index('\n')+1:]

        if docstr:
            # make sure we have Unicode docstrings, then sanitize and split
            # into lines
            return [prepare_docstring(force_decode(docstr, encoding))]
        return []

    def format_args(self):

        myname = self.fullname[len(self.modname)+1:]
        if myname.endswith('()'):
            myname = myname[:-2]

        # Try to parse docstring
        docstr = self.get_attr(self.object, '__doc__', None)
        if (docstr
            and docstr.startswith(myname + '(')
            and '\n' in docstr
            and docstr[docstr.index('\n')-1] == ')'):
            args = docstr[len(myname)+1:docstr.index('\n')-1]

            # Get rid of Cython style types declarations
            argl = []
            for arg in [ x.strip() for x in args.split(',') ]:
                if (arg in ('cls', 'self')
                    and isinstance(self, SphinxAutodoc.MethodDocumenter)):
                    continue
                hit = TYPE_RE.match(arg)
                if hit:
                    argl.append(hit.group(1))
                else:
                    argl.append(arg)
            args = '(%s)' % ', '.join(argl)

        else:
            return super(self.__class__, self).format_args()

        # escape backslashes for reST
        args = args.replace('\\', '\\\\')
        return args

class MyFunctionDocumenter(MyDocumenter, SphinxAutodoc.FunctionDocumenter):
    pass

class MyMethodDocumenter(MyDocumenter, SphinxAutodoc.MethodDocumenter):
    pass

SphinxAutodoc.MethodDocumenter = MyMethodDocumenter
SphinxAutodoc.FunctionDocumenter = MyFunctionDocumenter
__<commit_msg>Remove bogus line.<commit_after>'''
sphinx_cython.py

This module monkeypatches sphinx autodoc to support Cython generated
function signatures in the first line of the docstring of functions
implemented as C extensions.

Copyright (C) Nikolaus Rath <Nikolaus@rath.org>

This file is part of LLFUSE (http://python-llfuse.googlecode.com).
LLFUSE can be distributed under the terms of the GNU LGPL.
'''

import sphinx.ext.autodoc as SphinxAutodoc
from sphinx.util.docstrings import prepare_docstring
import inspect
import re
from sphinx.util import force_decode

TYPE_RE = re.compile(r'(?:int|char)(?:\s+\*?\s*|\s*\*?\s+)([a-zA-Z_].*)')

class MyDocumenter(SphinxAutodoc.Documenter):
    '''
    Overwrites `get_doc()` to remove function and
    method signatures and `format_args` to parse and give
    precedence to function signatures in the first line
    of the docstring.
    '''

    def get_doc(self, encoding=None):
        docstr = self.get_attr(self.object, '__doc__', None)

        myname = self.fullname[len(self.modname)+1:]
        if myname.endswith('()'):
            myname = myname[:-2]

        if (docstr
            and docstr.startswith(myname + '(')
            and '\n' in docstr
            and docstr[docstr.index('\n')-1] == ')'):
            docstr = docstr[docstr.index('\n')+1:]

        if docstr:
            # make sure we have Unicode docstrings, then sanitize and split
            # into lines
            return [prepare_docstring(force_decode(docstr, encoding))]
        return []

    def format_args(self):

        myname = self.fullname[len(self.modname)+1:]
        if myname.endswith('()'):
            myname = myname[:-2]

        # Try to parse docstring
        docstr = self.get_attr(self.object, '__doc__', None)
        if (docstr
            and docstr.startswith(myname + '(')
            and '\n' in docstr
            and docstr[docstr.index('\n')-1] == ')'):
            args = docstr[len(myname)+1:docstr.index('\n')-1]

            # Get rid of Cython style types declarations
            argl = []
            for arg in [ x.strip() for x in args.split(',') ]:
                if (arg in ('cls', 'self')
                    and isinstance(self, SphinxAutodoc.MethodDocumenter)):
                    continue
                hit = TYPE_RE.match(arg)
                if hit:
                    argl.append(hit.group(1))
                else:
                    argl.append(arg)
            args = '(%s)' % ', '.join(argl)

        else:
            return super(self.__class__, self).format_args()

        # escape backslashes for reST
        args = args.replace('\\', '\\\\')
        return args

class MyFunctionDocumenter(MyDocumenter, SphinxAutodoc.FunctionDocumenter):
    pass

class MyMethodDocumenter(MyDocumenter, SphinxAutodoc.MethodDocumenter):
    pass

SphinxAutodoc.MethodDocumenter = MyMethodDocumenter
SphinxAutodoc.FunctionDocumenter = MyFunctionDocumenter
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# (c) 2014 Mike Lewis

import os
import unittest

import foursquare

if 'CLIENT_ID' in os.environ and 'CLIENT_SECRET' in os.environ and 'ACCESS_TOKEN' in os.environ:
    CLIENT_ID = os.environ['CLIENT_ID']
    CLIENT_SECRET = os.environ['CLIENT_SECRET']
    ACCESS_TOKEN = os.environ['ACCESS_TOKEN']
else:
    try:
        from foursquare.tests._creds import *
    except ImportError:
        print("Please create a creds.py file in this package, based upon creds.example.py")

TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), 'testdata')

class BaseEndpointTestCase(unittest.TestCase):
    default_geo = u'40.7,-74.0'
    default_geo_radius = 100
    default_userid = u'1070527'
    default_venueid = u'40a55d80f964a52020f31ee3'
    default_tipid = u'4b5e662a70c603bba7d790b4'
    default_listid = u'32/tips'
    default_photoid = u'4d0fb8162d39a340637dc42b'
    default_settingid = u'receivePings'
    default_specialid = u'4e0debea922e6f94b1410bb7'
    default_special_venueid = u'4e0deab3922e6f94b1410af3'
    default_eventid = u'4e173d2cbd412187aabb3c04'
    default_pageid = u'1070527'

class BaseAuthenticationTestCase(BaseEndpointTestCase):
    def setUp(self):
        self.api = foursquare.Foursquare(
            client_id=CLIENT_ID,
            client_secret=CLIENT_SECRET,
            redirect_uri='http://example.org'
        )

class BaseAuthenticatedEndpointTestCase(BaseEndpointTestCase):
    def setUp(self):
        self.api = foursquare.Foursquare(
#            client_id=CLIENT_ID,
#            client_secret=CLIENT_SECRET,
            access_token=ACCESS_TOKEN
        )

class BaseUserlessEndpointTestCase(BaseEndpointTestCase):
    def setUp(self):
        self.api = foursquare.Foursquare(
            client_id=CLIENT_ID,
            client_secret=CLIENT_SECRET
        )

class MultilangEndpointTestCase(BaseEndpointTestCase):
    def setUp(self):
        self.apis = []
        for lang in ('es', 'fr', 'de', 'it', 'ja', 'th', 'ko', 'ru', 'pt', 'id'):
            self.apis.append(
                foursquare.Foursquare(
                    client_id=CLIENT_ID,
                    client_secret=CLIENT_SECRET,
                    lang=lang
                )
            )<commit_msg>updated tip id for tests<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# (c) 2014 Mike Lewis

import os
import unittest

import foursquare

if 'CLIENT_ID' in os.environ and 'CLIENT_SECRET' in os.environ and 'ACCESS_TOKEN' in os.environ:
    CLIENT_ID = os.environ['CLIENT_ID']
    CLIENT_SECRET = os.environ['CLIENT_SECRET']
    ACCESS_TOKEN = os.environ['ACCESS_TOKEN']
else:
    try:
        from foursquare.tests._creds import *
    except ImportError:
        print("Please create a creds.py file in this package, based upon creds.example.py")

TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), 'testdata')

class BaseEndpointTestCase(unittest.TestCase):
    default_geo = u'40.7,-74.0'
    default_geo_radius = 100
    default_userid = u'1070527'
    default_venueid = u'40a55d80f964a52020f31ee3'
    default_tipid = u'534c20ea498e7b69ab15e269'
    default_listid = u'32/tips'
    default_photoid = u'4d0fb8162d39a340637dc42b'
    default_settingid = u'receivePings'
    default_specialid = u'4e0debea922e6f94b1410bb7'
    default_special_venueid = u'4e0deab3922e6f94b1410af3'
    default_eventid = u'4e173d2cbd412187aabb3c04'
    default_pageid = u'1070527'

class BaseAuthenticationTestCase(BaseEndpointTestCase):
    def setUp(self):
        self.api = foursquare.Foursquare(
            client_id=CLIENT_ID,
            client_secret=CLIENT_SECRET,
            redirect_uri='http://example.org'
        )

class BaseAuthenticatedEndpointTestCase(BaseEndpointTestCase):
    def setUp(self):
        self.api = foursquare.Foursquare(
#            client_id=CLIENT_ID,
#            client_secret=CLIENT_SECRET,
            access_token=ACCESS_TOKEN
        )

class BaseUserlessEndpointTestCase(BaseEndpointTestCase):
    def setUp(self):
        self.api = foursquare.Foursquare(
            client_id=CLIENT_ID,
            client_secret=CLIENT_SECRET
        )

class MultilangEndpointTestCase(BaseEndpointTestCase):
    def setUp(self):
        self.apis = []
        for lang in ('es', 'fr', 'de', 'it', 'ja', 'th', 'ko', 'ru', 'pt', 'id'):
            self.apis.append(
                foursquare.Foursquare(
                    client_id=CLIENT_ID,
                    client_secret=CLIENT_SECRET,
                    lang=lang
                )
            )
<|endoftext|>


<commit_before>#!/usr/bin/env libtbx.python
from __future__ import division
from scitbx.array_family import flex

def source(length=10000, f_hz=100):
  import math
  import random

  data = flex.double(length, 0.0)

  for j in range(length):
    data[j] = 10 * math.sin(2 * math.pi * j * (f_hz / length))
    data[j] += random.random()

  return data

def fft(d):
  from scitbx import fftpack
  f = fftpack.real_to_complex(d.size())
  _d = flex.double(f.m_real(), 0.0)
  for j in range(d.size()):
    _d[j] = d[j]
  t = f.forward(_d)
  p = flex.abs(t) ** 2
  p[0] = 0.0
  return p

def test():
  f_hz = [100, 200, 500, 100, 50]
  length = [10000, 10000, 10000, 1000, 2000]
  for l, f in zip(length, f_hz):
    d = source(length=l, f_hz=f)
    d = fft(d)
    assert(flex.max(d) == d[f])

if __name__ == '__main__':
  test()
<commit_msg>Proper test, evaluation of scaling factor etc.<commit_after>#!/usr/bin/env libtbx.python
from __future__ import division
from scitbx.array_family import flex

def source(length=10000, sample_rate_hz=100, f_hz=10):
  import math
  import random

  data = flex.double(length, 0.0)

  for j in range(length):
    data[j] = 10 * math.sin(2 * math.pi * j * (f_hz / sample_rate_hz))
    data[j] += random.random()

  return data

def fft(d):
  from scitbx import fftpack
  f = fftpack.real_to_complex(d.size())
  _d = flex.double(f.m_real(), 0.0)
  for j in range(d.size()):
    _d[j] = d[j]
  t = f.forward(_d)
  p = flex.abs(t) ** 2

  # remove the DC component
  p[0] = 0.0

  return p

def test():
  sample_rate_hz = [100, 200, 500, 100, 50]
  length = [10000, 10000, 10000, 1000, 2000]
  f_hz = [5, 10, 20, 20, 4]
  for l, s, f in zip(length, sample_rate_hz, f_hz):
    f_scale = s / l
    d = source(length=l, sample_rate_hz=s, f_hz=f)
    d = fft(d)
    assert(flex.max(d) == d[int(round(f / f_scale))])
  print 'OK'

if __name__ == '__main__':
  test()
<|endoftext|>