<commit_before># This file is part of LiSE, a framework for life simulation games.
# Copyright (c) Zachary Spector,  zacharyspector@gmail.com
from kivy.clock import Clock
from kivy.properties import (
    DictProperty,
    NumericProperty,
    StringProperty,
    ReferenceListProperty,
    ObjectProperty,
    OptionProperty
)
from kivy.lang import Builder
from kivy.uix.widget import Widget
from kivy.uix.screenmanager import Screen
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.button import Button
from kivy.uix.dropdown import DropDown
from kivy.uix.textinput import TextInput
from .statlist import AbstractStatListView

control_txt = {
    'readout': 'Readout',
    'textinput': 'Text input',
    'togglebutton': 'Toggle button',
    'slider': 'Slider'
}

class FloatInput(TextInput):
    def insert_text(self, s, from_undo=False):
        return super().insert_text(
            ''.join(c for c in s if c in '0123456789.'),
            from_undo
        )

class ControlTypePicker(Button):
    app = ObjectProperty()
    key = ObjectProperty()
    mainbutton = ObjectProperty()
    dropdown = ObjectProperty()
    sett = ObjectProperty()

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.build()

    def set_value(self, k, v):
        if v is None:
            del self.app.selected_remote[k]
        else:
            self.app.selected_remote[k] = v

    def selected(self, v):
        self.sett(self.key, v)
        self.text = str(v)

    def build(self, *args):
        if None in (
                self.key,
                self.setter
        ):
            Clock.schedule_once(self.build, 0)
            return
        self.mainbutton = None
        self.dropdown = None
        self.dropdown = DropDown()
        self.dropdown.bind(on_select=lambda instance, x: self.selected(x))
        readoutbut = Button(
            text='Readout',
            size_hint_y=None,
            height=self.height
        )
        readoutbut.bind(on_press=lambda instance: self.dropdown.select('readout'))
        self.dropdown.add_widget(readoutbut)
        textinbut = Button(
            text='Text input',
            size_hint_y=None,
            height=self.height
        )
        textinbut.bind(on_press=lambda instance: self.dropdown.select('textinput'))
        self.dropdown.add_widget(textinbut)
        togbut = Button(
            text='Toggle button',
            size_hint_y=None,
            height=self.height
        )
        togbut.bind(on_press=lambda instance: self.dropdown.select('togglebutton'))
        self.dropdown.add_widget(togbut)
        sliderbut = Button(
            text='Slider',
            size_hint_y=None,
            height=self.height
        )
        sliderbut.bind(on_press=lambda instance: self.dropdown.select('slider'))
        self.dropdown.add_widget(sliderbut)
        self.bind(on_press=self.dropdown.open)

class ConfigListItemToggleButton(BoxLayout):
    config = ObjectProperty()

class ConfigListItemSlider(BoxLayout):
    config = ObjectProperty()

class ConfigListItemCustomizer(Widget):
    control = ObjectProperty()
    config = ObjectProperty()

    def on_control(self, *args):
        self.clear_widgets()
        if self.control == 'togglebutton':
            wid = ConfigListItemToggleButton(config=self.config)
            self.add_widget(wid)
        elif self.control == 'slider':
            wid = ConfigListItemSlider(config=self.config)
            self.add_widget(wid)

class ConfigListItem(BoxLayout):
    key = ObjectProperty()
    config = DictProperty()
    sett = ObjectProperty()
    deleter = ObjectProperty()
    control = OptionProperty('readout', options=['readout', 'textinput', 'togglebutton', 'slider'])

class StatListViewConfigurator(AbstractStatListView):
    statlist = ObjectProperty()
    _key_cfg_setters = DictProperty()
    _val_text_setters = DictProperty()
    _control_wids = DictProperty()

    def set_config(self, key, option, value):
        super().set_config(key, option, value)
        self.statlist.set_config(key, option, value)

    def set_configs(self, key, d):
        super().set_configs(key, d)
        self.statlist.config[key] = d

    def inst_set_configs(self, inst, val):
        self.set_configs(inst.key, val)

    def set_control(self, key, control):
        super().set_control(key, control)
        self.statlist.set_control(key, control)

    def inst_set_control(self, inst, val):
        self.set_control(inst.key, val)

    def del_key(self, key):
        del self.mirror[key]
        self.statlist.del_key(key)

    def munge(self, k, v):
        ret = super().munge(k, v)
        ret['on_control'] = self.inst_set_control
        ret['on_config'] = self.inst_set_configs
        ret['deleter'] = self.del_key
        return ret

class StatScreen(Screen):
    statlist = ObjectProperty()
    statcfg = ObjectProperty()
    toggle = ObjectProperty()
    engine = ObjectProperty()
    branch = StringProperty('master')
    tick = NumericProperty(0)
    time = ReferenceListProperty(branch, tick)
    remote = ObjectProperty()

    def new_stat(self):
        """Look at the key and value that the user has entered into the stat
        configurator, and set them on the currently selected
        entity.

        """
        key = self.ids.newstatkey.text
        value = self.ids.newstatval.text
        if not (key and value):
            # TODO implement some feedback to the effect that
            # you need to enter things
            return
        try:
            self.remote[key] = self.statlist.mirror[key] = self.statcfg.mirror[key] = self.engine.json_load(value)
        except (TypeError, ValueError):
            self.remote[key] = self.statlist.mirror[key] = self.statcfg.mirror[key] = value
        self.ids.newstatkey.text = ''
        self.ids.newstatval.text = ''

Builder.load_string("""
<ConfigListItem>:
    height: 30
    Button:
        text: 'del'
        on_press: root.deleter(root.key)
    Label:
        text: str(root.key)
    ControlTypePicker:
        key: root.key
        sett: root.sett
        control: root.control
    ConfigListItemCustomizer:
        config: root.config
        control: root.control
<StatScreen>:
    name: 'statcfg'
    statcfg: cfg
    BoxLayout:
        orientation: 'vertical'
        StatListViewConfigurator:
            viewclass: 'ConfigListItem'
            id: cfg
            engine: root.engine
            remote: root.remote
            branch: root.branch
            tick: root.tick
            statlist: root.statlist
            size_hint_y: 0.95
            RecycleBoxLayout:
                default_size: None, dp(56)
                default_size_hint: 1, None
                size_hint_y: None
                height: self.minimum_height
                orientation: 'vertical'
        BoxLayout:
            orientation: 'horizontal'
            size_hint_y: 0.05
            TextInput:
                id: newstatkey
                multiline: False
                write_tab: False
                hint_text: 'New stat'
            TextInput:
                id: newstatval
                multiline: False
                write_tab: False
                hint_text: 'Value'
            Button:
                id: newstatbut
                text: '+'
                on_press: root.new_stat()
            Button:
                id: closer
                text: 'Close'
                on_press: root.toggle()
""")
<commit_msg>Change the text on the input type dropdown in kv, not manually<commit_after># This file is part of LiSE, a framework for life simulation games.
# Copyright (c) Zachary Spector,  zacharyspector@gmail.com
from kivy.clock import Clock
from kivy.properties import (
    DictProperty,
    NumericProperty,
    StringProperty,
    ReferenceListProperty,
    ObjectProperty,
    OptionProperty
)
from kivy.lang import Builder
from kivy.uix.widget import Widget
from kivy.uix.screenmanager import Screen
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.button import Button
from kivy.uix.dropdown import DropDown
from kivy.uix.textinput import TextInput
from .statlist import AbstractStatListView

control_txt = {
    'readout': 'Readout',
    'textinput': 'Text input',
    'togglebutton': 'Toggle button',
    'slider': 'Slider'
}

class FloatInput(TextInput):
    def insert_text(self, s, from_undo=False):
        return super().insert_text(
            ''.join(c for c in s if c in '0123456789.'),
            from_undo
        )

class ControlTypePicker(Button):
    app = ObjectProperty()
    key = ObjectProperty()
    mainbutton = ObjectProperty()
    dropdown = ObjectProperty()
    sett = ObjectProperty()

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.build()

    def set_value(self, k, v):
        if v is None:
            del self.app.selected_remote[k]
        else:
            self.app.selected_remote[k] = v

    def build(self, *args):
        if None in (
                self.key,
                self.setter
        ):
            Clock.schedule_once(self.build, 0)
            return
        self.mainbutton = None
        self.dropdown = None
        self.dropdown = DropDown()
        self.dropdown.bind(on_select=lambda instance, x: self.sett(self.key, x))
        readoutbut = Button(
            text='Readout',
            size_hint_y=None,
            height=self.height
        )
        readoutbut.bind(on_press=lambda instance: self.dropdown.select('readout'))
        self.dropdown.add_widget(readoutbut)
        textinbut = Button(
            text='Text input',
            size_hint_y=None,
            height=self.height
        )
        textinbut.bind(on_press=lambda instance: self.dropdown.select('textinput'))
        self.dropdown.add_widget(textinbut)
        togbut = Button(
            text='Toggle button',
            size_hint_y=None,
            height=self.height
        )
        togbut.bind(on_press=lambda instance: self.dropdown.select('togglebutton'))
        self.dropdown.add_widget(togbut)
        sliderbut = Button(
            text='Slider',
            size_hint_y=None,
            height=self.height
        )
        sliderbut.bind(on_press=lambda instance: self.dropdown.select('slider'))
        self.dropdown.add_widget(sliderbut)
        self.bind(on_press=self.dropdown.open)

class ConfigListItemToggleButton(BoxLayout):
    config = ObjectProperty()

class ConfigListItemSlider(BoxLayout):
    config = ObjectProperty()

class ConfigListItemCustomizer(Widget):
    control = ObjectProperty()
    config = ObjectProperty()

    def on_control(self, *args):
        self.clear_widgets()
        if self.control == 'togglebutton':
            wid = ConfigListItemToggleButton(config=self.config)
            self.add_widget(wid)
        elif self.control == 'slider':
            wid = ConfigListItemSlider(config=self.config)
            self.add_widget(wid)

class ConfigListItem(BoxLayout):
    key = ObjectProperty()
    config = DictProperty()
    sett = ObjectProperty()
    deleter = ObjectProperty()
    control = OptionProperty('readout', options=['readout', 'textinput', 'togglebutton', 'slider'])

class StatListViewConfigurator(AbstractStatListView):
    statlist = ObjectProperty()
    _key_cfg_setters = DictProperty()
    _val_text_setters = DictProperty()
    _control_wids = DictProperty()

    def set_config(self, key, option, value):
        super().set_config(key, option, value)
        self.statlist.set_config(key, option, value)

    def set_configs(self, key, d):
        super().set_configs(key, d)
        self.statlist.config[key] = d

    def inst_set_configs(self, inst, val):
        self.set_configs(inst.key, val)

    def set_control(self, key, control):
        super().set_control(key, control)
        self.statlist.set_control(key, control)

    def inst_set_control(self, inst, val):
        self.set_control(inst.key, val)

    def del_key(self, key):
        del self.mirror[key]
        self.statlist.del_key(key)

    def munge(self, k, v):
        ret = super().munge(k, v)
        ret['on_control'] = self.inst_set_control
        ret['on_config'] = self.inst_set_configs
        ret['deleter'] = self.del_key
        return ret

class StatScreen(Screen):
    statlist = ObjectProperty()
    statcfg = ObjectProperty()
    toggle = ObjectProperty()
    engine = ObjectProperty()
    branch = StringProperty('master')
    tick = NumericProperty(0)
    time = ReferenceListProperty(branch, tick)
    remote = ObjectProperty()

    def new_stat(self):
        """Look at the key and value that the user has entered into the stat
        configurator, and set them on the currently selected
        entity.

        """
        key = self.ids.newstatkey.text
        value = self.ids.newstatval.text
        if not (key and value):
            # TODO implement some feedback to the effect that
            # you need to enter things
            return
        try:
            self.remote[key] = self.statlist.mirror[key] = self.statcfg.mirror[key] = self.engine.json_load(value)
        except (TypeError, ValueError):
            self.remote[key] = self.statlist.mirror[key] = self.statcfg.mirror[key] = value
        self.ids.newstatkey.text = ''
        self.ids.newstatval.text = ''

Builder.load_string("""
<ConfigListItem>:
    height: 30
    Button:
        text: 'del'
        on_press: root.deleter(root.key)
    Label:
        text: str(root.key)
    ControlTypePicker:
        key: root.key
        sett: root.sett
        control: root.control
        text: 'Text input' if self.control == 'texttinput' else 'Slider' if self.control == 'slider' else 'Toggle button' if self.control == 'togglebutton' else 'Readout'
    ConfigListItemCustomizer:
        config: root.config
        control: root.control
<StatScreen>:
    name: 'statcfg'
    statcfg: cfg
    BoxLayout:
        orientation: 'vertical'
        StatListViewConfigurator:
            viewclass: 'ConfigListItem'
            id: cfg
            engine: root.engine
            remote: root.remote
            branch: root.branch
            tick: root.tick
            statlist: root.statlist
            size_hint_y: 0.95
            RecycleBoxLayout:
                default_size: None, dp(56)
                default_size_hint: 1, None
                size_hint_y: None
                height: self.minimum_height
                orientation: 'vertical'
        BoxLayout:
            orientation: 'horizontal'
            size_hint_y: 0.05
            TextInput:
                id: newstatkey
                multiline: False
                write_tab: False
                hint_text: 'New stat'
            TextInput:
                id: newstatval
                multiline: False
                write_tab: False
                hint_text: 'Value'
            Button:
                id: newstatbut
                text: '+'
                on_press: root.new_stat()
            Button:
                id: closer
                text: 'Close'
                on_press: root.toggle()
""")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
github3.repos.hook
==================

This module contains only the Hook object for GitHub's Hook API.

"""
from __future__ import unicode_literals

from json import dumps
from ..decorators import requires_auth
from ..models import GitHubCore

class Hook(GitHubCore):
    """The :class:`Hook <Hook>` object. This handles the information returned
    by GitHub about hooks set on a repository.

    Two hook instances can be checked like so::

        h1 == h2
        h1 != h2

    And is equivalent to::

        h1.id == h2.id
        h1.id != h2.id

    See also: http://developer.github.com/v3/repos/hooks/
    """
    def _update_attributes(self, hook, session=None):
        self._api = self._get_attribute(hook, 'url')

        #: datetime object representing when this hook was last updated.
        self.updated_at = self._strptime_attribute(hook, 'updated_at')

        #: datetime object representing the date the hook was created.
        self.created_at = self._strptime_attribute(hook, 'created_at')

        #: The name of the hook.
        self.name = self._get_attribute(hook, 'name')

        #: Events which trigger the hook.
        self.events = self._get_attribute(hook, 'events')

        #: Whether or not this Hook is marked as active on GitHub
        self.active = self._get_attribute(hook, 'active')

        #: Dictionary containing the configuration for the Hook.
        self.config = self._get_attribute(hook, 'config')

        #: Unique id of the hook.
        self.id = self._get_attribute(hook, 'id')

    def _repr(self):
        return '<Hook [{0}]>'.format(self.name)

    @requires_auth
    def delete(self):
        """Delete this hook.

        :returns: bool
        """
        return self._boolean(self._delete(self._api), 204, 404)

    @requires_auth
    def edit(self, config={}, events=[], add_events=[], rm_events=[],
             active=True):
        """Edit this hook.

        :param dict config: (optional), key-value pairs of settings for this
            hook
        :param list events: (optional), which events should this be triggered
            for
        :param list add_events: (optional), events to be added to the list of
           events that this hook triggers for
        :param list rm_events: (optional), events to be removed from the list
            of events that this hook triggers for
        :param bool active: (optional), should this event be active
        :returns: bool
        """
        data = {'config': config, 'active': active}
        if events:
            data['events'] = events

        if add_events:
            data['add_events'] = add_events

        if rm_events:
            data['remove_events'] = rm_events

        json = self._json(self._patch(self._api, data=dumps(data)), 200)

        if json:
            self._update_attributes(json)
            return True

        return False

    @requires_auth
    def ping(self):
        """Ping this hook.

        :returns: bool
        """
        url = self._build_url('pings', base_url=self._api)
        return self._boolean(self._post(url), 204, 404)

    @requires_auth
    def test(self):
        """Test this hook

        :returns: bool
        """
        url = self._build_url('tests', base_url=self._api)
        return self._boolean(self._post(url), 204, 404)
<commit_msg>Update github3.repos.hook for consistency<commit_after># -*- coding: utf-8 -*-
"""This module contains only the Hook object for GitHub's Hook API."""
from __future__ import unicode_literals

from json import dumps
from ..decorators import requires_auth
from ..models import GitHubCore

class Hook(GitHubCore):
    """The representation of a hook on a repository.

    See also: http://developer.github.com/v3/repos/hooks/

    This object has the following attributes:

    .. attribute:: active

        A boolean attribute describing whether the hook is active or not.

    .. attribute:: config

        A dictionary containing the configuration for this hook.

    .. attribute:: created_at

        A :class:`~datetime.datetime` object representing the date and time
        when this hook was created.

    .. attribute:: events

        The list of events which trigger this hook.

    .. attribute:: id

        The unique identifier for this hook.

    .. attribute:: name

        The name provided to this hook.

    .. attribute:: updated_at

        A :class:`~datetime.datetime` object representing the date and time
        when this hook was updated.
    """

    def _update_attributes(self, hook, session=None):
        self._api = hook['url']
        self.active = hook['active']
        self.config = hook['config']
        self.created_at = self._strptime(hook['created_at'])
        self.events = hook['events']
        self.id = hook['id']
        self.name = hook['name']
        self.updated_at = self._strptime(hook['updated_at'])

    def _repr(self):
        return '<Hook [{0}]>'.format(self.name)

    @requires_auth
    def delete(self):
        """Delete this hook.

        :returns:
            True if successful, False otherwise
        :rtype:
            bool
        """
        return self._boolean(self._delete(self._api), 204, 404)

    @requires_auth
    def edit(self, config={}, events=[], add_events=[], rm_events=[],
             active=True):
        """Edit this hook.

        :param dict config:
            (optional), key-value pairs of settings for this hook
        :param list events:
            (optional), which events should this be triggered for
        :param list add_events:
            (optional), events to be added to the list of events that this hook
            triggers for
        :param list rm_events:
            (optional), events to be removed from the list of events that this
            hook triggers for
        :param bool active:
            (optional), should this event be active
        :returns:
            True if successful, False otherwise
        :rtype:
            bool
        """
        data = {'config': config, 'active': active}
        if events:
            data['events'] = events

        if add_events:
            data['add_events'] = add_events

        if rm_events:
            data['remove_events'] = rm_events

        json = self._json(self._patch(self._api, data=dumps(data)), 200)

        if json:
            self._update_attributes(json)
            return True

        return False

    @requires_auth
    def ping(self):
        """Ping this hook.

        :returns:
            True if successful, False otherwise
        :rtype:
            bool
        """
        url = self._build_url('pings', base_url=self._api)
        return self._boolean(self._post(url), 204, 404)

    @requires_auth
    def test(self):
        """Test this hook.

        :returns:
            True if successful, False otherwise
        :rtype:
            bool
        """
        url = self._build_url('tests', base_url=self._api)
        return self._boolean(self._post(url), 204, 404)
<|endoftext|>


<commit_before>DEBUG = True
import time
import os
import os.path
import sys

import pickle
import logging
import logging.handlers
from logging.handlers import TimedRotatingFileHandler
import SocketServer
import struct
import socket
from multiprocessing import Process, Event
import threading
import sys
import signal

import static
import boto
import boto.utils
from boto.s3.key import Key

import select
import errno

class TimeTracker:
    def __init__(self):
        self._wait_tick = time.time()
        self._work_tick = time.time()
        self._waiting = 0.0
        self._working = 0.0

    def start_work(self):
        self._work_tick= time.time()

    def end_work(self):
        self._working += time.time() - self._work_tick
        self._work_tick = time.time()

    def start_wait(self):
        self._wait_tick= time.time()

    def end_wait(self):
        self._waiting += time.time() - self._wait_tick
        self._wait_tick = time.time()

    def print_stats(self):
        print
        print "Waiting Time:", self._waiting
        print "Working Time:", self._working
        print "working/waiting", self._working/self._waiting
        print
"""
Below is lightly modified from http://docs.python.org/2/howto/logging-cookbook.html#logging-cookbook
"""
#global variable to kill stream handler from server
damn_global = 0
class LogRecordStreamHandler(SocketServer.StreamRequestHandler):
    """Handler for a streaming logging request.
    This basically logs the record using whatever logging policy is
    configured locally.
    """
    def handle(self):
        """
        Handle multiple requests - each expected to be a 4-byte length,
        followed by the LogRecord in pickle format. Logs the record
        according to whatever policy is configured locally.
        """
        global damn_global
        while not damn_global:
            chunk = self.connection.recv(4)
            if len(chunk) < 4:
                break
            slen = struct.unpack('>L', chunk)[0]
            chunk = self.connection.recv(slen)
            while len(chunk) < slen:
                chunk = chunk + self.connection.recv(slen - len(chunk))
            obj = self.unPickle(chunk)
            record = logging.makeLogRecord(obj)
            self.handleLogRecord(record)

    def unPickle(self, data):
        return pickle.loads(data)

    def handleLogRecord(self, record):
        # if a name is specified, we use the named logger rather than the one
        # implied by the record.
        if self.server.logname is not None:
            name = self.server.logname
        else:
            name = record.name
        logger = logging.getLogger(name)
        # N.B. EVERY record gets logged. This is because Logger.handle
        # is normally called AFTER logger-level filtering. If you want
        # to do filtering, do it at the client end to save wasting
        # cycles and network bandwidth!
        logger.handle(record)

class LogRecordSocketReceiver(SocketServer.ThreadingTCPServer):
    """
    Simple TCP socket-based logging receiver suitable for testing.
    """
    allow_reuse_address = 1
    def __init__(self, host='localhost', port=logging.handlers.DEFAULT_TCP_LOGGING_PORT, handler=LogRecordStreamHandler,doneEvent=None):
        SocketServer.ThreadingTCPServer.__init__(self, (host, port), handler)
        self.abort = 0
        self.timeout = 1
        self.logname = None
        self.doneEvent=doneEvent

    def serve_until_stopped(self):
        import select
        cont = self.doneEvent.is_set()
        while cont:
            try:
                rd, wr, ex = select.select([self.socket.fileno()], [], [],  self.timeout)
                cont = self.doneEvent.is_set()
                if rd:
                    self.handle_request()
            except select.error,v:
                #interrupt during select, ignore
                if v[0] != errno.EINTR: raise
                else: break
        #set global variable to end any running handlers
        #I am not proud of this
        global damn_global
        damn_global = 1
        for handler in logging.getLogger('').handlers:
            if isinstance( handler, S3TimedRotatatingFileHandler):
                handler.doRollover()
        self.doneEvent.set()

class S3TimedRotatatingFileHandler(TimedRotatingFileHandler):
    def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, bucket='diraclog'):
        TimedRotatingFileHandler. __init__(self, filename, when, interval, backupCount, encoding, delay, utc)
        self.bucket = bucket

    def doRollover(self):
        """
        do a rollover; in this case, a date/time stamp is appended to the filename
        when the rollover happens.  However, you want the file to be named for the
        start of the interval, not the current time.  If there is a backup count,
        then we have to get a list of matching filenames, sort them and remove
        the one with the oldest suffix.
        """
        if self.stream:
            self.stream.close()
            self.stream = None
        # get the time that this sequence started at and make it a TimeTuple
        currentTime = int(time.time())
        dstNow = time.localtime(currentTime)[-1]
        t = self.rolloverAt - self.interval
        if self.utc:
            timeTuple = time.gmtime(t)
        else:
            timeTuple = time.localtime(t)
            dstThen = timeTuple[-1]
            if dstNow != dstThen:
                if dstNow:
                    addend = 3600
                else:
                    addend = -3600
                timeTuple = time.localtime(t + addend)
        dfn = self.baseFilename + "." + time.strftime(self.suffix, timeTuple)
        if os.path.exists(dfn):
            os.remove(dfn)
        # Issue 18940: A file may not have been created if delay is True.
        if os.path.exists(self.baseFilename):
            os.rename(self.baseFilename, dfn)
        if self.backupCount > 0:
            for s in self.getFilesToDelete():
                os.remove(s)
        #if not self.delay:
        self.stream = self._open()
        newRolloverAt = self.computeRollover(currentTime)
        while newRolloverAt <= currentTime:
            newRolloverAt = newRolloverAt + self.interval
        #If DST changes and midnight or weekly rollover, adjust for this.
        if (self.when == 'MIDNIGHT' or self.when.startswith('W')) and not self.utc:
            dstAtRollover = time.localtime(newRolloverAt)[-1]
            if dstNow != dstAtRollover:
                if not dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
                    addend = -3600
                else:           # DST bows out before next rollover, so we need to add an hour
                    addend = 3600
                newRolloverAt += addend
        self.rolloverAt = newRolloverAt
        #this is the new stuff
        #copy to s3 the old file
        self.pushToS3( dfn )

    def pushToS3(self, filename):
        conn = boto.connect_s3()
        bucket = conn.get_bucket(self.bucket)
        k = Key(bucket)
        k.key = os.path.split(filename)[1]
        k.set_contents_from_filename(filename, reduced_redundancy=True)

def startLogger():
    import os, os.path
    import logging
    import masterdirac.models.systemdefaults as sys_def
    ls_settings = sys_def.get_system_defaults(component='Dual GPU',
            setting_name='logserver')
    log_dir = ls_settings['directory']
    LOG_FILENAME = ls_settings['log_filename']
    if LOG_FILENAME == 'None':
        md =  boto.utils.get_instance_metadata()
        LOG_FILENAME = md['instance-id'] + '.log'
    bucket = ls_settings[ 'bucket']
    interval_type = ls_settings[ 'interval_type' ]
    interval = int(ls_settings[ 'interval'])
    log_format = ls_settings[ 'log_format' ]
    port = ls_settings[ 'port' ]
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)
    handler = S3TimedRotatatingFileHandler(os.path.join(log_dir,LOG_FILENAME),
                    when=interval_type, interval = interval, bucket=bucket)

    doneEvent = threading.Event()
    doneEvent.set()
    tcpserver = LogRecordSocketReceiver(doneEvent=doneEvent, port=int(port))
    def shutdownHandler(msg,evt):
        logging.getLogger('logging.SIGHANDLER').critical("Shutdown handler activated")
        if evt.is_set():#only want to do this once, if it is clear then it is shutting down
            evt.clear()
        sys.exit(0)

    def terminate(signal,frame):
        t = threading.Thread(target = shutdownHandler,
                args = ('SIGTERM received',doneEvent))
        t.start()
        t.join()
    for s in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP, signal.SIGQUIT]:
        signal.signal(s, terminate)
    handler.setFormatter(logging.Formatter(log_format))
    logging.getLogger('').addHandler(handler)
    try:
        conn = boto.connect_s3()
        conn.get_bucket(bucket)
    except:
        logging.getLogger('logging').exception("Creating s3://%s"%bucket)
        conn = boto.connect_s3()
        conn.create_bucket(bucket)
    tcpserver.serve_until_stopped()

def initLogging():
    import masterdirac.models.systemdefaults as sys_def
    log_settings = sys_def.get_system_defaults(component='Dual GPU',
            setting_name='logging')

    log_format = log_settings['log_format']
    es_name = log_settings['external_server_name']
    es_port = log_settings['external_server_port']
    es_level = log_settings['external_server_level']

    is_name = log_settings[ 'internal_server_name']
    is_port =  log_settings['internal_server_port']
    is_level = log_settings['internal_server_level']

    boto_level =  log_settings['boto_level']
    stdout_level = log_settings['stdout_level']

    botoLogger = logging.getLogger('boto')
    botoLogger.setLevel(int(boto_level))

    formatter = logging.Formatter(log_format)
    rootLogger = logging.getLogger('')
    rootLogger.setLevel(logging.DEBUG)

    if es_name !='None':
        print "In external server", es_level
        server = es_name
        port = es_port
        server_level = es_level
        socketHandler = logging.handlers.SocketHandler(server, int(port))
        socketHandler.setLevel(int(server_level))
        socketHandler.setFormatter(formatter)
        rootLogger.addHandler(socketHandler)
    if is_name !='None':
        print "In Internal server", is_level
        server = is_name
        port = is_port
        server_level = is_level
        socketHandler = logging.handlers.SocketHandler(server, int(port))
        socketHandler.setLevel(int(server_level))
        socketHandler.setFormatter(formatter)
        rootLogger.addHandler(socketHandler)
    if stdout_level != 'None':
        print "in stdout", stdout_level
        ch = logging.StreamHandler(sys.stdout)
        ch.setLevel(int(stdout_level))
        ch.setFormatter(formatter)
        rootLogger.addHandler(ch)

if __name__ == "__main__":
    startLogger()
<commit_msg>Get to create bucket<commit_after>DEBUG = True
import time
import os
import os.path
import sys

import pickle
import logging
import logging.handlers
from logging.handlers import TimedRotatingFileHandler
import SocketServer
import struct
import socket
from multiprocessing import Process, Event
import threading
import sys
import signal

import static
import boto
import boto.utils
from boto.s3.key import Key

import select
import errno

class TimeTracker:
    def __init__(self):
        self._wait_tick = time.time()
        self._work_tick = time.time()
        self._waiting = 0.0
        self._working = 0.0

    def start_work(self):
        self._work_tick= time.time()

    def end_work(self):
        self._working += time.time() - self._work_tick
        self._work_tick = time.time()

    def start_wait(self):
        self._wait_tick= time.time()

    def end_wait(self):
        self._waiting += time.time() - self._wait_tick
        self._wait_tick = time.time()

    def print_stats(self):
        print
        print "Waiting Time:", self._waiting
        print "Working Time:", self._working
        print "working/waiting", self._working/self._waiting
        print
"""
Below is lightly modified from http://docs.python.org/2/howto/logging-cookbook.html#logging-cookbook
"""
#global variable to kill stream handler from server
damn_global = 0
class LogRecordStreamHandler(SocketServer.StreamRequestHandler):
    """Handler for a streaming logging request.
    This basically logs the record using whatever logging policy is
    configured locally.
    """
    def handle(self):
        """
        Handle multiple requests - each expected to be a 4-byte length,
        followed by the LogRecord in pickle format. Logs the record
        according to whatever policy is configured locally.
        """
        global damn_global
        while not damn_global:
            chunk = self.connection.recv(4)
            if len(chunk) < 4:
                break
            slen = struct.unpack('>L', chunk)[0]
            chunk = self.connection.recv(slen)
            while len(chunk) < slen:
                chunk = chunk + self.connection.recv(slen - len(chunk))
            obj = self.unPickle(chunk)
            record = logging.makeLogRecord(obj)
            self.handleLogRecord(record)

    def unPickle(self, data):
        return pickle.loads(data)

    def handleLogRecord(self, record):
        # if a name is specified, we use the named logger rather than the one
        # implied by the record.
        if self.server.logname is not None:
            name = self.server.logname
        else:
            name = record.name
        logger = logging.getLogger(name)
        # N.B. EVERY record gets logged. This is because Logger.handle
        # is normally called AFTER logger-level filtering. If you want
        # to do filtering, do it at the client end to save wasting
        # cycles and network bandwidth!
        logger.handle(record)

class LogRecordSocketReceiver(SocketServer.ThreadingTCPServer):
    """
    Simple TCP socket-based logging receiver suitable for testing.
    """
    allow_reuse_address = 1
    def __init__(self, host='localhost', port=logging.handlers.DEFAULT_TCP_LOGGING_PORT, handler=LogRecordStreamHandler,doneEvent=None):
        SocketServer.ThreadingTCPServer.__init__(self, (host, port), handler)
        self.abort = 0
        self.timeout = 1
        self.logname = None
        self.doneEvent=doneEvent

    def serve_until_stopped(self):
        import select
        cont = self.doneEvent.is_set()
        while cont:
            try:
                rd, wr, ex = select.select([self.socket.fileno()], [], [],  self.timeout)
                cont = self.doneEvent.is_set()
                if rd:
                    self.handle_request()
            except select.error,v:
                #interrupt during select, ignore
                if v[0] != errno.EINTR: raise
                else: break
        #set global variable to end any running handlers
        #I am not proud of this
        global damn_global
        damn_global = 1
        for handler in logging.getLogger('').handlers:
            if isinstance( handler, S3TimedRotatatingFileHandler):
                handler.doRollover()
        self.doneEvent.set()

class S3TimedRotatatingFileHandler(TimedRotatingFileHandler):
    def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, bucket='diraclog'):
        TimedRotatingFileHandler. __init__(self, filename, when, interval, backupCount, encoding, delay, utc)
        self.bucket = bucket

    def doRollover(self):
        """
        do a rollover; in this case, a date/time stamp is appended to the filename
        when the rollover happens.  However, you want the file to be named for the
        start of the interval, not the current time.  If there is a backup count,
        then we have to get a list of matching filenames, sort them and remove
        the one with the oldest suffix.
        """
        if self.stream:
            self.stream.close()
            self.stream = None
        # get the time that this sequence started at and make it a TimeTuple
        currentTime = int(time.time())
        dstNow = time.localtime(currentTime)[-1]
        t = self.rolloverAt - self.interval
        if self.utc:
            timeTuple = time.gmtime(t)
        else:
            timeTuple = time.localtime(t)
            dstThen = timeTuple[-1]
            if dstNow != dstThen:
                if dstNow:
                    addend = 3600
                else:
                    addend = -3600
                timeTuple = time.localtime(t + addend)
        dfn = self.baseFilename + "." + time.strftime(self.suffix, timeTuple)
        if os.path.exists(dfn):
            os.remove(dfn)
        # Issue 18940: A file may not have been created if delay is True.
        if os.path.exists(self.baseFilename):
            os.rename(self.baseFilename, dfn)
        if self.backupCount > 0:
            for s in self.getFilesToDelete():
                os.remove(s)
        #if not self.delay:
        self.stream = self._open()
        newRolloverAt = self.computeRollover(currentTime)
        while newRolloverAt <= currentTime:
            newRolloverAt = newRolloverAt + self.interval
        #If DST changes and midnight or weekly rollover, adjust for this.
        if (self.when == 'MIDNIGHT' or self.when.startswith('W')) and not self.utc:
            dstAtRollover = time.localtime(newRolloverAt)[-1]
            if dstNow != dstAtRollover:
                if not dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
                    addend = -3600
                else:           # DST bows out before next rollover, so we need to add an hour
                    addend = 3600
                newRolloverAt += addend
        self.rolloverAt = newRolloverAt
        #this is the new stuff
        #copy to s3 the old file
        self.pushToS3( dfn )

    def pushToS3(self, filename):
        conn = boto.connect_s3()
        bucket = conn.create_bucket(self.bucket)
        k = Key(bucket)
        k.key = os.path.split(filename)[1]
        k.set_contents_from_filename(filename, reduced_redundancy=True)

def startLogger():
    import os, os.path
    import logging
    import masterdirac.models.systemdefaults as sys_def
    ls_settings = sys_def.get_system_defaults(component='Dual GPU',
            setting_name='logserver')
    log_dir = ls_settings['directory']
    LOG_FILENAME = ls_settings['log_filename']
    if LOG_FILENAME == 'None':
        md =  boto.utils.get_instance_metadata()
        LOG_FILENAME = md['instance-id'] + '.log'
    bucket = ls_settings[ 'bucket']
    interval_type = ls_settings[ 'interval_type' ]
    interval = int(ls_settings[ 'interval'])
    log_format = ls_settings[ 'log_format' ]
    port = ls_settings[ 'port' ]
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)
    handler = S3TimedRotatatingFileHandler(os.path.join(log_dir,LOG_FILENAME),
                    when=interval_type, interval = interval, bucket=bucket)

    doneEvent = threading.Event()
    doneEvent.set()
    tcpserver = LogRecordSocketReceiver(doneEvent=doneEvent, port=int(port))
    def shutdownHandler(msg,evt):
        logging.getLogger('logging.SIGHANDLER').critical("Shutdown handler activated")
        if evt.is_set():#only want to do this once, if it is clear then it is shutting down
            evt.clear()
        sys.exit(0)

    def terminate(signal,frame):
        t = threading.Thread(target = shutdownHandler,
                args = ('SIGTERM received',doneEvent))
        t.start()
        t.join()
    for s in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP, signal.SIGQUIT]:
        signal.signal(s, terminate)
    handler.setFormatter(logging.Formatter(log_format))
    logging.getLogger('').addHandler(handler)
    try:
        conn = boto.connect_s3()
        conn.create_bucket(bucket)
    except:
        logging.getLogger('logging').exception("Creating s3://%s"%bucket)
        conn = boto.connect_s3()
        conn.create_bucket(bucket)
    tcpserver.serve_until_stopped()

def initLogging():
    import masterdirac.models.systemdefaults as sys_def
    log_settings = sys_def.get_system_defaults(component='Dual GPU',
            setting_name='logging')

    log_format = log_settings['log_format']
    es_name = log_settings['external_server_name']
    es_port = log_settings['external_server_port']
    es_level = log_settings['external_server_level']

    is_name = log_settings[ 'internal_server_name']
    is_port =  log_settings['internal_server_port']
    is_level = log_settings['internal_server_level']

    boto_level =  log_settings['boto_level']
    stdout_level = log_settings['stdout_level']

    botoLogger = logging.getLogger('boto')
    botoLogger.setLevel(int(boto_level))

    formatter = logging.Formatter(log_format)
    rootLogger = logging.getLogger('')
    rootLogger.setLevel(logging.DEBUG)

    if es_name !='None':
        print "In external server", es_level
        server = es_name
        port = es_port
        server_level = es_level
        socketHandler = logging.handlers.SocketHandler(server, int(port))
        socketHandler.setLevel(int(server_level))
        socketHandler.setFormatter(formatter)
        rootLogger.addHandler(socketHandler)
    if is_name !='None':
        print "In Internal server", is_level
        server = is_name
        port = is_port
        server_level = is_level
        socketHandler = logging.handlers.SocketHandler(server, int(port))
        socketHandler.setLevel(int(server_level))
        socketHandler.setFormatter(formatter)
        rootLogger.addHandler(socketHandler)
    if stdout_level != 'None':
        print "in stdout", stdout_level
        ch = logging.StreamHandler(sys.stdout)
        ch.setLevel(int(stdout_level))
        ch.setFormatter(formatter)
        rootLogger.addHandler(ch)

if __name__ == "__main__":
    startLogger()
<|endoftext|>


<commit_before>#----------------------------------------------------------------------
# Copyright (c) 2011 Raytheon BBN Technologies
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and/or hardware specification (the "Work") to
# deal in the Work without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Work, and to permit persons to whom the Work
# is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Work.
#
# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
# IN THE WORK.
#----------------------------------------------------------------------
import sys
import logging
import os
import M2Crypto.SSL
from omnilib.util.paths import getAbsPath
from omnilib.util import OmniError

class Framework_Base():
    """
    Framework_Base is an abstract class that identifies the minimal set of functions
    that must be implemented in order to add a control framework to omni.

    Instructions for adding a new framework:

    Create "framework_X" in the frameworks directory, where X is your control framework.

    Create a Framework class in the file that inherits "Framework_Base" and fill out each of the functions.

    Edit the sample "omni_config" file and add a section for your framework, giving the section
    the same name as X used in framework_X.  For instance, 'sfa' or 'gcf'.  Your framework's section
    of the omni config *MUST* have a cert and key entry, which omni will use when talking to
    the GENI Aggregate managers.

    """

    def __init__(self, config):
        self.cert = getAbsPath(config['cert'])
        if not os.path.exists(self.cert):
            sys.exit("Frameworks certfile %s doesn't exist" % self.cert)

        self.key = getAbsPath(config['key'])
        if not os.path.exists(self.key):
            sys.exit("Frameworks keyfile %s doesn't exist" % self.key)
        self.sslctx = None

    def get_user_cred(self):
        """
        Returns a user credential from the control framework as a string.
        """
        raise NotImplementedError('get_user_cred')

    def get_slice_cred(self, urn):
        """
        Retrieve a slice with the given urn and returns the signed credential as a string.
        """
        raise NotImplementedError('get_slice_cred')

    def create_slice(self, urn):
        """
        If the slice already exists in the framework, it returns that.  Otherwise it creates the slice
        and returns the new slice as a string.
        """
        raise NotImplementedError('create_slice')

    def delete_slice(self, urn):
        """
        Removes the slice from the control framework.
        """
        raise NotImplementedError('delete_slice')

    def list_aggregates(self):
        """
        Get a list of available GENI Aggregates from the control framework.
        Returns: a dictionary where keys are urns and values are aggregate urls
        """
        raise NotImplementedError('list_aggregates')

    def list_my_slices(self, username):
        """
        Get a list of slices for this user.
        Returns: a list of slice names
        """
        raise NotImplementedError('list_my_slices')

    def slice_name_to_urn(self, name):
        """Convert a slice name to a slice urn."""
        # Default implementation just converts to generic URN.
        raise NotImplementedError('slice_name_to_urn')

    def renew_slice(self, urn, requested_expiration):
        """Renew a slice.

        urn is framework urn, already converted via slice_name_to_urn.
        requested_expiration is a datetime object.

        Returns the expiration date as a datetime. If there is an error,
        print it and return None.
        """
        raise NotImplementedError('renew_slice')

    def ssl_context(self, retries=2):
        """Returns an SSL Context or an exception is raised."""
        if hasattr(self, 'logger'):
            logger = self.logger
        else:
            logger = logging.getLogger("omni.framework")
        if not self.sslctx:
            # Initialize the M2Crypto SSL Context
            attempts = 0
            while attempts <= retries:
                sslctx = M2Crypto.SSL.Context()
                try:
                    sslctx.load_cert(self.cert, self.key)
                    self.sslctx = sslctx
                    break
                except M2Crypto.SSL.SSLError, err:
                    logger.error('Wrong pass phrase for private key.')
                    attempts = attempts + 1
                    if attempts > retries:
                        logger.error("Wrong pass phrase after %d tries.",
                                     attempts)
                        raise OmniError(err)
                    else:
                        logger.info('.... please retry.')
        return self.sslctx
<commit_msg>use load_cert_chain not just load_cert when creating m2crypto context<commit_after>#----------------------------------------------------------------------
# Copyright (c) 2011 Raytheon BBN Technologies
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and/or hardware specification (the "Work") to
# deal in the Work without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Work, and to permit persons to whom the Work
# is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Work.
#
# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
# IN THE WORK.
#----------------------------------------------------------------------
import sys
import logging
import os
import M2Crypto.SSL
from omnilib.util.paths import getAbsPath
from omnilib.util import OmniError

class Framework_Base():
    """
    Framework_Base is an abstract class that identifies the minimal set of functions
    that must be implemented in order to add a control framework to omni.

    Instructions for adding a new framework:

    Create "framework_X" in the frameworks directory, where X is your control framework.

    Create a Framework class in the file that inherits "Framework_Base" and fill out each of the functions.

    Edit the sample "omni_config" file and add a section for your framework, giving the section
    the same name as X used in framework_X.  For instance, 'sfa' or 'gcf'.  Your framework's section
    of the omni config *MUST* have a cert and key entry, which omni will use when talking to
    the GENI Aggregate managers.

    """

    def __init__(self, config):
        self.cert = getAbsPath(config['cert'])
        if not os.path.exists(self.cert):
            sys.exit("Frameworks certfile %s doesn't exist" % self.cert)

        self.key = getAbsPath(config['key'])
        if not os.path.exists(self.key):
            sys.exit("Frameworks keyfile %s doesn't exist" % self.key)
        self.sslctx = None

    def get_user_cred(self):
        """
        Returns a user credential from the control framework as a string.
        """
        raise NotImplementedError('get_user_cred')

    def get_slice_cred(self, urn):
        """
        Retrieve a slice with the given urn and returns the signed credential as a string.
        """
        raise NotImplementedError('get_slice_cred')

    def create_slice(self, urn):
        """
        If the slice already exists in the framework, it returns that.  Otherwise it creates the slice
        and returns the new slice as a string.
        """
        raise NotImplementedError('create_slice')

    def delete_slice(self, urn):
        """
        Removes the slice from the control framework.
        """
        raise NotImplementedError('delete_slice')

    def list_aggregates(self):
        """
        Get a list of available GENI Aggregates from the control framework.
        Returns: a dictionary where keys are urns and values are aggregate urls
        """
        raise NotImplementedError('list_aggregates')

    def list_my_slices(self, username):
        """
        Get a list of slices for this user.
        Returns: a list of slice names
        """
        raise NotImplementedError('list_my_slices')

    def slice_name_to_urn(self, name):
        """Convert a slice name to a slice urn."""
        # Default implementation just converts to generic URN.
        raise NotImplementedError('slice_name_to_urn')

    def renew_slice(self, urn, requested_expiration):
        """Renew a slice.

        urn is framework urn, already converted via slice_name_to_urn.
        requested_expiration is a datetime object.

        Returns the expiration date as a datetime. If there is an error,
        print it and return None.
        """
        raise NotImplementedError('renew_slice')

    def ssl_context(self, retries=2):
        """Returns an SSL Context or an exception is raised."""
        if hasattr(self, 'logger'):
            logger = self.logger
        else:
            logger = logging.getLogger("omni.framework")
        if not self.sslctx:
            # Initialize the M2Crypto SSL Context
            attempts = 0
            while attempts <= retries:
                sslctx = M2Crypto.SSL.Context()
                try:
                    sslctx.load_cert_chain(self.cert, self.key)
                    self.sslctx = sslctx
                    break
                except M2Crypto.SSL.SSLError, err:
                    logger.error('Wrong pass phrase for private key.')
                    attempts = attempts + 1
                    if attempts > retries:
                        logger.error("Wrong pass phrase after %d tries.",
                                     attempts)
                        raise OmniError(err)
                    else:
                        logger.info('.... please retry.')
        return self.sslctx
<|endoftext|>


<commit_before>from __future__ import absolute_import

import re

from . import defaults
from ..dataset import Dataset
from ..geojson import get_encoded_data, get_bounds

try:
    import geopandas
    HAS_GEOPANDAS = True
except ImportError:
    HAS_GEOPANDAS = False

class SourceType:
    QUERY = 'Query'
    GEOJSON = 'GeoJSON'

class Source(object):
    """Source

    Args:
        data (str, :py:class:`GeoFrame <geopandas.GeoDataFrame>`,
          :py:class:`Dataset <cartoframes.viz.Dataset>` ): a table name, SQL query
          ,GeoJSON file, GeoFrame object or Dataset object.
        context (:py:class:`Context <cartoframes.Context>`):
          A Conext instance. If not provided the context will be automatically
          obtained from the default context.
        bounds (dict or list): a dict with `east`,`north`,`west`,`south`
          properties, or a list of floats in the following order: [west,
          south, east, north]. If not provided the bounds will be automatically
          calculated to fit all features.

    Example:

        Table name.

        .. code::

            from cartoframes.auth import set_default_context
            from cartoframes.viz import Source

            set_default_context(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )

            Source('table_name')

        SQL query.

        .. code::

            from cartoframes.auth import set_default_context
            from cartoframes.viz import Source

            set_default_context(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )

            Source('SELECT * FROM table_name')

        GeoJSON file.

        .. code::

            from cartoframes.auth import set_default_context
            from cartoframes.viz import Source

            set_default_context(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )

            Source('path/to/file.geojson')

        Dataset object.

        .. code::

            from cartoframes.auth import set_default_context
            from cartoframes.viz import Source
            from cartoframes import Dataset

            set_default_context(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )

            ds = Dataset.from_table('table_name')

            Source(ds)

        Setting the context.

        .. code::

            from cartoframes.auth import Context
            from cartoframes.viz import Source

            context = Context(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )

            Source('table_name', context)

        Setting the bounds.

        .. code::

            from cartoframes.auth import set_default_context
            from cartoframes.viz import Source

            set_default_context(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )

            bounds = {
                'west': -10,
                'east': 10,
                'north': -10,
                'south': 10
            }

            Source('table_name', bounds=bounds)
    """

    def __init__(self, data, context=None, bounds=None, schema=None):
        self._init_source(data, context, bounds, schema)

        self.context = self.dataset.cc
        self.credentials = _get_credentials(self.context)
        self.geom_type = _get_geom_type(self.dataset)

    def _init_source(self, data, context, bounds, schema):
        if isinstance(data, str):
            from cartoframes.auth import _default_context
            context = context or _default_context

            if _check_sql_query(data):
                self._init_source_query(data, context, bounds)

            elif _check_geojson_file(data):
                self._init_source_geojson(data, bounds)

            elif _check_table_name(data):
                schema = schema or context.get_default_schema()
                self._init_source_query(_format_query(data, schema), context, bounds)

        elif HAS_GEOPANDAS and isinstance(data, (list, dict, geopandas.GeoDataFrame)):
            self._init_source_geojson(data, bounds)

        elif isinstance(data, Dataset):
            self._init_source_dataset(data, bounds)

        else:
            raise ValueError('Wrong source input')

    def _init_source_query(self, data, context, bounds):
        self.dataset = Dataset.from_query(data, context)
        self.type = SourceType.QUERY
        self.query = self.dataset.query
        self.bounds = bounds

    def _init_source_geojson(self, data, bounds):
        self.dataset = Dataset.from_geojson(data)
        self.type = SourceType.GEOJSON
        self.query = get_encoded_data(self.dataset.gdf)
        self.bounds = bounds or get_bounds(self.dataset.gdf)

    def _init_source_dataset(self, data, bounds):
        self.dataset = data
        self.type = _map_dataset_state(self.dataset.state)

        if self.dataset.state == Dataset.STATE_REMOTE:
            self.bounds = bounds
            if self.dataset.query:
                self.query = self.dataset.query
            else:
                self.query = _format_query(self.dataset.table_name, self.dataset.schema)
        elif self.dataset.state == Dataset.STATE_LOCAL:
            if self.dataset.gdf:
                self.query = get_encoded_data(self.dataset.gdf)
                self.bounds = bounds or get_bounds(self.dataset.gdf)
            else:
                # TODO: Dataframe
                pass

def _check_table_name(data):
    return True

def _check_sql_query(data):
    return re.match(r'^\s*(WITH|SELECT)\s+', data, re.IGNORECASE)

def _check_geojson_file(data):
    return re.match(r'^.*\.geojson\s*$', data, re.IGNORECASE)

def _format_query(table_name, schema='public'):
    return 'SELECT * FROM "{0}"."{1}"'.format(schema, table_name)

def _map_dataset_state(state):
    return {
        Dataset.STATE_REMOTE: SourceType.QUERY,
        Dataset.STATE_LOCAL: SourceType.GEOJSON
    }[state]

def _get_credentials(context):
    if context and context.creds:
        return {
            'username': context.creds.username(),
            'api_key': context.creds.key(),
            'base_url': context.creds.base_url()
        }
    else:
        return defaults.CREDENTIALS

def _get_geom_type(dataset):
    return dataset.compute_geom_type() or Dataset.GEOM_TYPE_POINT
<commit_msg>extract and fix schema generation<commit_after>from __future__ import absolute_import

import re

from . import defaults
from ..dataset import Dataset
from ..geojson import get_encoded_data, get_bounds

try:
    import geopandas
    HAS_GEOPANDAS = True
except ImportError:
    HAS_GEOPANDAS = False

class SourceType:
    QUERY = 'Query'
    GEOJSON = 'GeoJSON'

class Source(object):
    """Source

    Args:
        data (str, :py:class:`GeoFrame <geopandas.GeoDataFrame>`,
          :py:class:`Dataset <cartoframes.viz.Dataset>` ): a table name, SQL query
          ,GeoJSON file, GeoFrame object or Dataset object.
        context (:py:class:`Context <cartoframes.Context>`):
          A Conext instance. If not provided the context will be automatically
          obtained from the default context.
        bounds (dict or list): a dict with `east`,`north`,`west`,`south`
          properties, or a list of floats in the following order: [west,
          south, east, north]. If not provided the bounds will be automatically
          calculated to fit all features.

    Example:

        Table name.

        .. code::

            from cartoframes.auth import set_default_context
            from cartoframes.viz import Source

            set_default_context(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )

            Source('table_name')

        SQL query.

        .. code::

            from cartoframes.auth import set_default_context
            from cartoframes.viz import Source

            set_default_context(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )

            Source('SELECT * FROM table_name')

        GeoJSON file.

        .. code::

            from cartoframes.auth import set_default_context
            from cartoframes.viz import Source

            set_default_context(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )

            Source('path/to/file.geojson')

        Dataset object.

        .. code::

            from cartoframes.auth import set_default_context
            from cartoframes.viz import Source
            from cartoframes import Dataset

            set_default_context(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )

            ds = Dataset.from_table('table_name')

            Source(ds)

        Setting the context.

        .. code::

            from cartoframes.auth import Context
            from cartoframes.viz import Source

            context = Context(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )

            Source('table_name', context)

        Setting the bounds.

        .. code::

            from cartoframes.auth import set_default_context
            from cartoframes.viz import Source

            set_default_context(
                base_url='https://your_user_name.carto.com',
                api_key='your api key'
            )

            bounds = {
                'west': -10,
                'east': 10,
                'north': -10,
                'south': 10
            }

            Source('table_name', bounds=bounds)
    """

    def __init__(self, data, context=None, bounds=None, schema=None):
        self._init_source(data, context, bounds, schema)

        self.context = self.dataset.cc
        self.credentials = _get_credentials(self.context)
        self.geom_type = _get_geom_type(self.dataset)

    def _init_source(self, data, context, bounds, schema):
        if isinstance(data, str):
            if _check_sql_query(data):
                self._init_source_query(data, context, bounds)

            elif _check_geojson_file(data):
                self._init_source_geojson(data, bounds)

            elif _check_table_name(data):
                schema = schema or _get_schema(context)
                self._init_source_query(_format_query(data, schema), context, bounds)

        elif HAS_GEOPANDAS and isinstance(data, (list, dict, geopandas.GeoDataFrame)):
            self._init_source_geojson(data, bounds)

        elif isinstance(data, Dataset):
            self._init_source_dataset(data, bounds)

        else:
            raise ValueError('Wrong source input')

    def _init_source_query(self, data, context, bounds):
        self.dataset = Dataset.from_query(data, context)
        self.type = SourceType.QUERY
        self.query = self.dataset.query
        self.bounds = bounds

    def _init_source_geojson(self, data, bounds):
        self.dataset = Dataset.from_geojson(data)
        self.type = SourceType.GEOJSON
        self.query = get_encoded_data(self.dataset.gdf)
        self.bounds = bounds or get_bounds(self.dataset.gdf)

    def _init_source_dataset(self, data, bounds):
        self.dataset = data
        self.type = _map_dataset_state(self.dataset.state)

        if self.dataset.state == Dataset.STATE_REMOTE:
            self.bounds = bounds
            if self.dataset.query:
                self.query = self.dataset.query
            else:
                self.query = _format_query(self.dataset.table_name, self.dataset.schema)
        elif self.dataset.state == Dataset.STATE_LOCAL:
            if self.dataset.gdf:
                self.query = get_encoded_data(self.dataset.gdf)
                self.bounds = bounds or get_bounds(self.dataset.gdf)
            else:
                # TODO: Dataframe
                pass

def _check_table_name(data):
    return True

def _check_sql_query(data):
    return re.match(r'^\s*(WITH|SELECT)\s+', data, re.IGNORECASE)

def _check_geojson_file(data):
    return re.match(r'^.*\.geojson\s*$', data, re.IGNORECASE)

def _format_query(table_name, schema='public'):
    return 'SELECT * FROM "{0}"."{1}"'.format(schema, table_name)

def _map_dataset_state(state):
    return {
        Dataset.STATE_REMOTE: SourceType.QUERY,
        Dataset.STATE_LOCAL: SourceType.GEOJSON
    }[state]

def _get_credentials(context):
    if context and context.creds:
        return {
            'username': context.creds.username(),
            'api_key': context.creds.key(),
            'base_url': context.creds.base_url()
        }
    else:
        return defaults.CREDENTIALS

def _get_geom_type(dataset):
    return dataset.compute_geom_type() or Dataset.GEOM_TYPE_POINT

def _get_schema(context):
    from cartoframes.auth import _default_context
    context = context or _default_context

    if context:
        return context.get_default_schema()
    else:
        return 'public'
<|endoftext|>


<commit_before>"""
Django settings for icforum project.

Generated by 'django-admin startproject' using Django 1.9.8.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os
from django.utils.translation import ugettext_lazy as _

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '_=413j3u@tnjo0gr27jghwu0(*f8(nfmththi9qsf$^!)9%(n*'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django_tables2',
    'bootstrap3',
    'users',
    'forum',
    'files',
]

MIDDLEWARE_CLASSES = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'icforum.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(BASE_DIR, 'templates'),],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
                'django.core.context_processors.request'
            ],
        },
    },
]

WSGI_APPLICATION = 'icforum.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

LANGUAGES = (
    ('en', _('English')),
    ('fr', _('French')),
)

LOCALE_PATHS = (
    os.path.join(BASE_DIR, 'locale'),
)

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'

STATICFILES_DIRS = (
    os.path.join(BASE_DIR, 'static'),
)

# Media files (avatars..)

MEDIA_URL = '/media/'
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')

# Forum settings

IC_FORUM_VERSION = '0.0.1-dev'

CONTENT_TYPES = ['image']
MAX_UPLOAD_SIZE = '524288'
<commit_msg>Version jump to 0.0.2-dev<commit_after>"""
Django settings for icforum project.

Generated by 'django-admin startproject' using Django 1.9.8.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os
from django.utils.translation import ugettext_lazy as _

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '_=413j3u@tnjo0gr27jghwu0(*f8(nfmththi9qsf$^!)9%(n*'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django_tables2',
    'bootstrap3',
    'users',
    'forum',
    'files',
]

MIDDLEWARE_CLASSES = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'icforum.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(BASE_DIR, 'templates'),],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
                'django.core.context_processors.request'
            ],
        },
    },
]

WSGI_APPLICATION = 'icforum.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

LANGUAGES = (
    ('en', _('English')),
    ('fr', _('French')),
)

LOCALE_PATHS = (
    os.path.join(BASE_DIR, 'locale'),
)

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'

STATICFILES_DIRS = (
    os.path.join(BASE_DIR, 'static'),
)

# Media files (avatars..)

MEDIA_URL = '/media/'
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')

# Forum settings

IC_FORUM_VERSION = '0.0.2-dev'

CONTENT_TYPES = ['image']
MAX_UPLOAD_SIZE = '524288'
<|endoftext|>