<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""SchedulerServer base."""

from uuid import UUID

import time
import random
import traceback
import logging

from heapq import heappush, heappop
from twisted.internet import reactor, task
from twisted.web import server
from twisted.internet.defer import inlineCallbacks
from txamqp.content import Content
from .base import BaseServer, LOGGER
from .mixins import MySQLMixin, AMQPMixin

logger = logging.getLogger(__name__)

from twisted.web.resource import Resource

class SchedulerServer(BaseServer, AMQPMixin, MySQLMixin):

    heap = []
    # TODO: make this a set()
    unscheduled_items = []
    enqueueCallLater = None
    enqueueloop = None

    def __init__(self, config, port=None):
        super(SchedulerServer, self).__init__(config)
        self.setupAMQP(config)
        self.setupMySQL(config)
        # HTTP interface
        resource = Resource()
        self.function_resource = Resource()
        resource.putChild("function", self.function_resource)
        if port is None:
            port = config["scheduler_server_port"]
        self.site_port = reactor.listenTCP(port, server.Site(resource))
        # Logging, etc
        self.expose(self.remoteRemoveFromHeap)
        self.expose(self.remoteAddToHeap)
        self.expose(self.enqueueUUID)

    def start(self):
        start_deferred = super(SchedulerServer, self).start()
        start_deferred.addCallback(self._schedulerStart)
        return start_deferred

    @inlineCallbacks
    def _schedulerStart(self, started=None):
        yield self.startJobQueue()
        yield self._loadFromMySQL()
        self.enqueueloop = task.LoopingCall(self.enqueue)
        self.enqueueloop.start(1)

    @inlineCallbacks
    def _loadFromMySQL(self):
        data = []
        start = 0
        while len(data) >= 100000 or start == 0:
            sql = """SELECT uuid, type
                     FROM spider_service
                     ORDER BY id LIMIT %s, 100000
                  """ % start
            start += 100000
            data = yield self.mysql.runQuery(sql)
            for row in data:
                self.addToHeap(row["uuid"], row["type"])

    @inlineCallbacks
    def shutdown(self):
        self.enqueueloop.stop()
        try:
            self.enqueueCallLater.cancel()
        except:
            pass
        yield self.stopJobQueue()
        yield super(SchedulerServer, self).shutdown()

    def enqueue(self):
        now = int(time.time())
        # Compare the heap min timestamp with now().
        # If it's time for the item to be queued, pop it, update the
        # timestamp and add it back to the heap for the next go round.
        queued_items = 0
        if self.amqp_queue_size < 100000:
            LOGGER.debug("%s:%s" % (self.heap[0][0], now))
            while self.heap[0][0] < now and queued_items < 1000:
                job = heappop(self.heap)
                uuid = UUID(bytes=job[1][0])
                if not uuid.hex in self.unscheduled_items:
                    queued_items += 1
                    self.chan.basic_publish(
                        exchange=self.amqp_exchange,
                        content=Content(job[1][0]))
                    heappush(self.heap, (now + job[1][1], job[1]))
                else:
                    self.unscheduled_items.remove(uuid.hex)
        else:
            LOGGER.critical('AMQP queue is at or beyond max limit (%d/100000)'
                % self.amqp_queue_size)

    def enqueueUUID(self, uuid):
        LOGGER.debug('enqueueUUID: uuid=%s' % uuid)
        self.chan.basic_publish(
            exchange=self.amqp_exchange,
            content=Content(UUID(uuid).bytes))
        return uuid

    def remoteAddToHeap(self, uuid=None, type=None):
        if uuid and type:
            LOGGER.debug('remoteAddToHeap: uuid=%s, type=%s' % (uuid, type))
            self.addToHeap(uuid, type)
            return {}
        else:
            LOGGER.error('Required parameters are uuid and type')
            return {'error': 'Required parameters are uuid and type'}

    def remoteRemoveFromHeap(self, uuid):
        LOGGER.debug('remoteRemoveFromHeap: uuid=%s' % uuid)
        self.removeFromHeap(uuid)

    def addToHeap(self, uuid, type):
        # lookup if type is in the service_mapping, if it is
        # then rewrite type to the proper resource
        if not uuid in self.unscheduled_items:
            if self.service_mapping and type in self.service_mapping:
                LOGGER.info('Remapping resource %s to %s'
                    % (type, self.service_mapping[type]))
                type = self.service_mapping[type]
            try:
                # Make sure the uuid is in bytes
                uuid_bytes = UUID(uuid).bytes
            except ValueError:
                LOGGER.error('Cound not turn UUID into bytes using string: "%s" with type of "%s"'
                    % (uuid, type))
                return
            if type in self.functions and 'interval' in self.functions[type]:
                interval = int(self.functions[type]['interval'])
            else:
                LOGGER.error('Could not find interval for type %s' % type)
                return
            # Enqueue randomly over the interval so it doesn't
            # flood the server at the interval time. only if an interval is defined
            if interval:
                enqueue_time = int(time.time() + random.randint(0, interval))
                # Add a UUID to the heap.
                LOGGER.debug('Adding %s to heap with time %s and interval of %s'
                    % (uuid, enqueue_time, interval))
                heappush(self.heap, (enqueue_time, (uuid_bytes, interval)))
        else:
            LOGGER.info('Unscheduling %s' % uuid)
            self.unscheduled_items.remove(uuid)

    def removeFromHeap(self, uuid):
        LOGGER.info('Removing %s from heap' % uuid)
        self.unscheduled_items.append(uuid)

    @inlineCallbacks
    def executeReservation(self, function_name, **kwargs):
        if not isinstance(function_name, str):
            for key in self.functions:
                if self.functions[key]["function"] == function_name:
                    function_name = key
                    break
        if function_name not in self.functions:
            raise Exception("Function %s does not exist." % function_name)
        function = self.functions[function_name]
        logger.debug("Calling function %s with kwargs %s" % (function_name, kwargs))
        try:
            data = yield self.executeFunction(function_name, **kwargs)
        except Exception, e:
            tb = traceback.format_exc()
            logger.error("function %s failed with args %s:\n%s" % (
                function_name, kwargs, tb))
        returnValue(data)

<commit_msg>redo scheduler's logging<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""SchedulerServer base."""

from uuid import UUID

import time
import random
import traceback
import logging

from heapq import heappush, heappop
from twisted.internet import reactor, task
from twisted.web import server
from twisted.internet.defer import inlineCallbacks
from txamqp.content import Content
from .base import BaseServer
from .mixins import MySQLMixin, AMQPMixin

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

from twisted.web.resource import Resource

class SchedulerServer(BaseServer, AMQPMixin, MySQLMixin):

    heap = []
    # TODO: make this a set()
    unscheduled_items = []
    enqueueCallLater = None
    enqueueloop = None

    def __init__(self, config, port=None):
        super(SchedulerServer, self).__init__(config)
        self.setupAMQP(config)
        self.setupMySQL(config)
        # HTTP interface
        resource = Resource()
        self.function_resource = Resource()
        resource.putChild("function", self.function_resource)
        if port is None:
            port = config["scheduler_server_port"]
        self.site_port = reactor.listenTCP(port, server.Site(resource))
        # Logging, etc
        self.expose(self.remoteRemoveFromHeap)
        self.expose(self.remoteAddToHeap)
        self.expose(self.enqueueUUID)

    def start(self):
        start_deferred = super(SchedulerServer, self).start()
        start_deferred.addCallback(self._schedulerStart)
        return start_deferred

    @inlineCallbacks
    def _schedulerStart(self, started=None):
        yield self.startJobQueue()
        yield self._loadFromMySQL()
        self.enqueueloop = task.LoopingCall(self.enqueue)
        self.enqueueloop.start(1)

    @inlineCallbacks
    def _loadFromMySQL(self):
        data = []
        start = 0
        while len(data) >= 100000 or start == 0:
            sql = """SELECT uuid, type
                     FROM spider_service
                     ORDER BY id LIMIT %s, 100000
                  """ % start
            start += 100000
            data = yield self.mysql.runQuery(sql)
            for row in data:
                self.addToHeap(row["uuid"], row["type"])

    @inlineCallbacks
    def shutdown(self):
        self.enqueueloop.stop()
        try:
            self.enqueueCallLater.cancel()
        except:
            pass
        yield self.stopJobQueue()
        yield super(SchedulerServer, self).shutdown()

    def enqueue(self):
        now = int(time.time())
        # Compare the heap min timestamp with now().
        # If it's time for the item to be queued, pop it, update the
        # timestamp and add it back to the heap for the next go round.
        queued_items = 0
        if self.amqp_queue_size < 100000:
            logger.debug("%s:%s" % (self.heap[0][0], now))
            while self.heap[0][0] < now and queued_items < 1000:
                job = heappop(self.heap)
                uuid = UUID(bytes=job[1][0])
                if not uuid.hex in self.unscheduled_items:
                    queued_items += 1
                    self.chan.basic_publish(
                        exchange=self.amqp_exchange,
                        content=Content(job[1][0]))
                    heappush(self.heap, (now + job[1][1], job[1]))
                else:
                    self.unscheduled_items.remove(uuid.hex)
        else:
            logger.critical('AMQP queue is at or beyond max limit (%d/100000)'
                % self.amqp_queue_size)

    def enqueueUUID(self, uuid):
        logger.debug('enqueueUUID: uuid=%s' % uuid)
        self.chan.basic_publish(
            exchange=self.amqp_exchange,
            content=Content(UUID(uuid).bytes))
        return uuid

    def remoteAddToHeap(self, uuid=None, type=None):
        if uuid and type:
            logger.debug('remoteAddToHeap: uuid=%s, type=%s' % (uuid, type))
            self.addToHeap(uuid, type)
            return {}
        else:
            logger.error('Required parameters are uuid and type')
            return {'error': 'Required parameters are uuid and type'}

    def remoteRemoveFromHeap(self, uuid):
        logger.debug('remoteRemoveFromHeap: uuid=%s' % uuid)
        self.removeFromHeap(uuid)

    def addToHeap(self, uuid, type):
        # lookup if type is in the service_mapping, if it is
        # then rewrite type to the proper resource
        if not uuid in self.unscheduled_items:
            if self.service_mapping and type in self.service_mapping:
                logger.info('Remapping resource %s to %s'
                    % (type, self.service_mapping[type]))
                type = self.service_mapping[type]
            try:
                # Make sure the uuid is in bytes
                uuid_bytes = UUID(uuid).bytes
            except ValueError:
                logger.error('Cound not turn UUID into bytes using string: "%s" with type of "%s"'
                    % (uuid, type))
                return
            if type in self.functions and 'interval' in self.functions[type]:
                interval = int(self.functions[type]['interval'])
            else:
                logger.error('Could not find interval for type %s' % type)
                return
            # Enqueue randomly over the interval so it doesn't
            # flood the server at the interval time. only if an interval is defined
            if interval:
                enqueue_time = int(time.time() + random.randint(0, interval))
                # Add a UUID to the heap.
                logger.debug('Adding %s to heap with time %s and interval of %s'
                    % (uuid, enqueue_time, interval))
                heappush(self.heap, (enqueue_time, (uuid_bytes, interval)))
        else:
            logger.info('Unscheduling %s' % uuid)
            self.unscheduled_items.remove(uuid)

    def removeFromHeap(self, uuid):
        logger.info('Removing %s from heap' % uuid)
        self.unscheduled_items.append(uuid)

    @inlineCallbacks
    def executeReservation(self, function_name, **kwargs):
        if not isinstance(function_name, str):
            for key in self.functions:
                if self.functions[key]["function"] == function_name:
                    function_name = key
                    break
        if function_name not in self.functions:
            raise Exception("Function %s does not exist." % function_name)
        function = self.functions[function_name]
        logger.error("Calling function %s with kwargs %s" % (function_name, kwargs))
        try:
            data = yield self.executeFunction(function_name, **kwargs)
        except Exception, e:
            tb = traceback.format_exc()
            logger.error("function %s failed with args %s:\n%s" % (
                function_name, kwargs, tb))
        returnValue(data)

<|endoftext|>


<commit_before>import subprocess
import os

# install packages for coding
print("pakete installieren")
install_req = "apt-get install -y aptitude vim-nox build-essentials python3-pip code gimp inkscape".split()
sudo = subprocess.Popen(["sudo", "-S"] + install_req, stdout=subprocess.PIPE)
print(sudo.stdout.read())
<commit_msg>vundle installation<commit_after>import subprocess
import os

# install packages for coding
print("pakete installieren")
install_req = "apt-get install -y aptitude vim-nox build-essential python3-pip code gimp inkscape".split()
sudo = subprocess.Popen(["sudo", "-S"] + install_req, stdout=subprocess.PIPE)
print(sudo.stdout.read())

# install vundle
print("vundle installieren")
install_req = f"git clone https://github.com/VundleVim/Vundle.vim.git {os.path.expanduser('~/.vim/bundle/Vundle.vim')}".split()
cmd = subprocess.Popen(install_req, stdout=subprocess.PIPE)
print(cmd.stdout.read())

<|endoftext|>


<commit_before>import cherrypy
import json
import sys

# This function defines the structure of a service response.  Each service
# module should import this function from this package.
#
# The payload is contained in the 'result' field, while the 'error' field can
# indicate that something went wrong.  Posibly more fields could be added in the
# future.
def empty_response():
    return {'result' : None,
            'error' : None}

# 'current_dir' is used by the CherryPy config file to set the root for static
# file service.
import os.path
current_dir = os.path.dirname(os.path.abspath(__file__))

class Server(object):
    @cherrypy.expose
    def index(self):
        # Redirect the user to the actual front page.
        raise cherrypy.HTTPRedirect("/index.html")

    @cherrypy.expose
    def service(self, module, *pargs, **kwargs):
        # TODO(choudhury): This method should attempt to load the named module, then invoke it
        # with the given arguments.  However, if the named module is "config" or
        # something similar, the method should instead launch a special "config"
        # app, which lists the available app modules, along with docstrings or
        # similar.  It should also allow the user to add/delete search paths for
        # other modules.
        cherrypy.response.headers['Content-type'] = 'text/plain'

        # Construct a response container for reporting possible errors, as the
        # service modules themselves do.
        response = empty_response()

        # Construct import statement.
        import_string = "import modules.%s" % (module)
        try:
            exec(import_string)
        except ImportError:
            response['error'] = "xdataweb: error: no such module '%s'" % (module)
            return json.dumps(response)

        # Report an error if the module has no Handler object.
        m = eval("modules.%s" % (module))
        if 'Handler' not in dir(m):
            response['error'] = "xdataweb: error: no Handler class defined in module '%s'" % (module)
            return json.dumps(response)

        # Construct a Handler object from the imported module.
        handler = m.Handler()

        # Report an error if the Handler object has no go() method.
        if 'go' not in dir(handler):
            response['error'] = "xdataweb: error: no method go() defined in class 'Handler' in module '%s'" % (module)
            return json.dumps(response)

        # Call the go() method of the handler object, passing it the positional
        # and keyword args that came into this method.
        try:
            return handler.go(*pargs, **kwargs)
        except Exception as e:
            response['error'] = "xdataweb: error: %s: %s" % (e.__class__.__name__, e.message)
            return json.dumps(response)

    @cherrypy.expose
    def config(self, module, *pargs, **kwargs):
        cherrypy.response.headers['Content-type'] = 'text/html'

        # Redirect to the appropriately named configuration page, or to a "no
        # configuration" information page if it doesn't exist.
        try:
            # Test whether the requested config app file exists.
            target = "/config/%s.html" % (module)
            os.stat(current_dir + "/web" + target)

            # Redirect to the config app page.
            raise cherrypy.HTTPRedirect(target)
        except OSError:
            # This code path means the configuration webpage doesn't exist.
            #
            # TODO(choudhury): a templating engine would be useful here; look
            # into Jinja2 (recommended by DJ Deo).
            notfound_text = open(current_dir + "/config/notfound.html").read().replace("[APPNAME]", module)
            return notfound_text
<commit_msg>Updated new directory structure for notfound.html error page.<commit_after>import cherrypy
import json
import sys

# This function defines the structure of a service response.  Each service
# module should import this function from this package.
#
# The payload is contained in the 'result' field, while the 'error' field can
# indicate that something went wrong.  Posibly more fields could be added in the
# future.
def empty_response():
    return {'result' : None,
            'error' : None}

# 'current_dir' is used by the CherryPy config file to set the root for static
# file service.
import os.path
current_dir = os.path.dirname(os.path.abspath(__file__))

class Server(object):
    @cherrypy.expose
    def index(self):
        # Redirect the user to the actual front page.
        raise cherrypy.HTTPRedirect("/index.html")

    @cherrypy.expose
    def service(self, module, *pargs, **kwargs):
        # TODO(choudhury): This method should attempt to load the named module, then invoke it
        # with the given arguments.  However, if the named module is "config" or
        # something similar, the method should instead launch a special "config"
        # app, which lists the available app modules, along with docstrings or
        # similar.  It should also allow the user to add/delete search paths for
        # other modules.
        cherrypy.response.headers['Content-type'] = 'text/plain'

        # Construct a response container for reporting possible errors, as the
        # service modules themselves do.
        response = empty_response()

        # Construct import statement.
        import_string = "import modules.%s" % (module)
        try:
            exec(import_string)
        except ImportError:
            response['error'] = "xdataweb: error: no such module '%s'" % (module)
            return json.dumps(response)

        # Report an error if the module has no Handler object.
        m = eval("modules.%s" % (module))
        if 'Handler' not in dir(m):
            response['error'] = "xdataweb: error: no Handler class defined in module '%s'" % (module)
            return json.dumps(response)

        # Construct a Handler object from the imported module.
        handler = m.Handler()

        # Report an error if the Handler object has no go() method.
        if 'go' not in dir(handler):
            response['error'] = "xdataweb: error: no method go() defined in class 'Handler' in module '%s'" % (module)
            return json.dumps(response)

        # Call the go() method of the handler object, passing it the positional
        # and keyword args that came into this method.
        try:
            return handler.go(*pargs, **kwargs)
        except Exception as e:
            response['error'] = "xdataweb: error: %s: %s" % (e.__class__.__name__, e.message)
            return json.dumps(response)

    @cherrypy.expose
    def config(self, module, *pargs, **kwargs):
        cherrypy.response.headers['Content-type'] = 'text/html'

        # Redirect to the appropriately named configuration page, or to a "no
        # configuration" information page if it doesn't exist.
        try:
            # Test whether the requested config app file exists.
            target = "/config/%s.html" % (module)
            os.stat(current_dir + "/web" + target)

            # Redirect to the config app page.
            raise cherrypy.HTTPRedirect(target)
        except OSError:
            # This code path means the configuration webpage doesn't exist.
            #
            # TODO(choudhury): a templating engine would be useful here; look
            # into Jinja2 (recommended by DJ Deo).
            notfound_text = open(current_dir + "/web/config/notfound.html").read().replace("[APPNAME]", module)
            return notfound_text
<|endoftext|>


<commit_before>from mpf.core.case_insensitive_dict import CaseInsensitiveDict
from mpf.core.utility_functions import Util
from mpfmc.core.config_collection import ConfigCollection
from mpfmc.uix.slide_frame import SlideFrame
from mpfmc.widgets.image import ImageWidget
from mpfmc.widgets.text import Text
from mpfmc.widgets.video import VideoWidget
from mpfmc.widgets.line import Line
from mpfmc.widgets.triangle import Triangle
from mpfmc.widgets.quad import Quad
from mpfmc.widgets.rectangle import Rectangle
from mpfmc.widgets.ellipse import Ellipse
from mpfmc.widgets.bezier import Bezier
from mpfmc.widgets.point import Point
from mpfmc.widgets.dmd import Dmd, ColorDmd
from mpfmc.widgets.text_input import MpfTextInput

class Widget(ConfigCollection):

    config_section = 'widgets'
    collection = 'widgets'
    class_label = 'WidgetConfig'

    type_map = CaseInsensitiveDict(text=Text,
                                   image=ImageWidget,
                                   video=VideoWidget,
                                   slide_frame=SlideFrame,
                                   bezier=Bezier,
                                   ellipse=Ellipse,
                                   line=Line,
                                   point=Point,
                                   points=Point,
                                   quad=Quad,
                                   rectangle=Rectangle,
                                   triangle=Triangle,
                                   dmd=Dmd,
                                   color_dmd=ColorDmd,
                                   text_input=MpfTextInput)

    def process_config(self, config):
        # config is localized to a specific widget section
        if isinstance(config, dict):
            config = [config]

        config.reverse()

        widget_list = list()

        for widget in config:
            widget_list.append(self.process_widget(widget))

        return widget_list

    def process_widget(self, config):
        # config is localized widget settings

        try:
            widget_cls = Widget.type_map[config['type']]
        except KeyError:
            raise ValueError('"{}" is not a valid MPF display widget type'
                             .format(config['type']))

        config['_default_settings'] = list()

        for default_setting_name in widget_cls.merge_settings:
            if default_setting_name in config:
                config['_default_settings'].append(default_setting_name)

        self.mc.config_validator.validate_config('widgets:{}'.format(
            config['type']).lower(), config, base_spec='widgets:common')

        if 'animations' in config:
            config['animations'] = (
                self.process_animations(config['animations']))

        else:
            config['animations'] = None

        return config

    def process_animations(self, config):
        # config is localized to the slide's 'animations' section

        for event_name, event_settings in config.items():

            # make sure the event_name is registered as a trigger event so MPF
            # will send those events as triggers via BCP
            if event_name != 'entrance':
                self.mc.bcp_processor.send(bcp_command='register_trigger',
                                           event=event_name)

            # str means it's a list of named animations
            if type(event_settings) is str:
                event_settings = Util.string_to_list(event_settings)

            # dict means it's a single set of settings for one animation step
            elif isinstance(event_settings, dict):
                event_settings = [event_settings]

            # ultimately we're producing a list of dicts, so build that list
            # as we iterate
            new_list = list()
            for settings in event_settings:
                new_list.append(self.mc.animations.process_animation(settings))

            config[event_name] = new_list

        return config

collection_cls = Widget
<commit_msg>better error for mal-formed widget config<commit_after>from mpf.core.case_insensitive_dict import CaseInsensitiveDict
from mpf.core.utility_functions import Util
from mpfmc.core.config_collection import ConfigCollection
from mpfmc.uix.slide_frame import SlideFrame
from mpfmc.widgets.image import ImageWidget
from mpfmc.widgets.text import Text
from mpfmc.widgets.video import VideoWidget
from mpfmc.widgets.line import Line
from mpfmc.widgets.triangle import Triangle
from mpfmc.widgets.quad import Quad
from mpfmc.widgets.rectangle import Rectangle
from mpfmc.widgets.ellipse import Ellipse
from mpfmc.widgets.bezier import Bezier
from mpfmc.widgets.point import Point
from mpfmc.widgets.dmd import Dmd, ColorDmd
from mpfmc.widgets.text_input import MpfTextInput

class Widget(ConfigCollection):

    config_section = 'widgets'
    collection = 'widgets'
    class_label = 'WidgetConfig'

    type_map = CaseInsensitiveDict(text=Text,
                                   image=ImageWidget,
                                   video=VideoWidget,
                                   slide_frame=SlideFrame,
                                   bezier=Bezier,
                                   ellipse=Ellipse,
                                   line=Line,
                                   point=Point,
                                   points=Point,
                                   quad=Quad,
                                   rectangle=Rectangle,
                                   triangle=Triangle,
                                   dmd=Dmd,
                                   color_dmd=ColorDmd,
                                   text_input=MpfTextInput)

    def process_config(self, config):
        # config is localized to a specific widget section
        if isinstance(config, dict):
            config = [config]

        config.reverse()

        widget_list = list()

        for widget in config:
            widget_list.append(self.process_widget(widget))

        return widget_list

    def process_widget(self, config):
        # config is localized widget settings
        try:
            widget_cls = Widget.type_map[config['type']]
        except KeyError:
            try:
                raise ValueError('"{}" is not a valid MPF display widget type'
                                 .format(config['type']))
            except KeyError:
                raise ValueError("Invalid widget config: {}".format(config))

        config['_default_settings'] = list()

        for default_setting_name in widget_cls.merge_settings:
            if default_setting_name in config:
                config['_default_settings'].append(default_setting_name)

        self.mc.config_validator.validate_config('widgets:{}'.format(
            config['type']).lower(), config, base_spec='widgets:common')

        if 'animations' in config:
            config['animations'] = (
                self.process_animations(config['animations']))

        else:
            config['animations'] = None

        return config

    def process_animations(self, config):
        # config is localized to the slide's 'animations' section

        for event_name, event_settings in config.items():

            # make sure the event_name is registered as a trigger event so MPF
            # will send those events as triggers via BCP
            if event_name != 'entrance':
                self.mc.bcp_processor.send(bcp_command='register_trigger',
                                           event=event_name)

            # str means it's a list of named animations
            if type(event_settings) is str:
                event_settings = Util.string_to_list(event_settings)

            # dict means it's a single set of settings for one animation step
            elif isinstance(event_settings, dict):
                event_settings = [event_settings]

            # ultimately we're producing a list of dicts, so build that list
            # as we iterate
            new_list = list()
            for settings in event_settings:
                new_list.append(self.mc.animations.process_animation(settings))

            config[event_name] = new_list

        return config

collection_cls = Widget
<|endoftext|>


<commit_before>import os
import time
import glob
import fcntl
from AdaptivePELE.utilities import utilities

try:
    ProcessLookupError
except NameError:
    ProcessLookupError = OSError

class ProcessesManager:
    """
        Object that sinchronizes multiple adaptivePELE instances, designed to
        be able to use multiple nodes of a gpu cluster
    """
    RUNNING = "RUNNING"
    WAITING = "WAITING"
    INIT = "INIT"

    def __init__(self, output_path, num_replicas):
        self.syncFolder =  os.path.join(os.path.abspath(output_path), "synchronization")
        utilities.makeFolder(self.syncFolder)
        self.lockFile = os.path.join(self.syncFolder,  "syncFile.lock")
        self.pid = os.getpid()
        self.nReplicas = num_replicas
        self.id = None
        self.lockInfo = {}
        self.stateStack = {}
        self.status = self.INIT
        self.sleepTime = 0.5
        self.createLockFile()
        self.writeProcessInfo()
        self.initLockFile()
        self.syncStep = 0

    def __len__(self):
        # define the size of the ProcessesManager object as the number of
        # processes
        return len(self.lockInfo)

    def writeProcessInfo(self):
        with open(os.path.join(self.syncFolder, "%d.proc" % self.pid), "w") as fw:
            fw.write("%d\n" % self.pid)

    def createLockFile(self):
        """
            Create the lock file
        """
        try:
            # whith the flags set, if the file exists the open will fail
            fd = os.open(self.lockFile, os.O_CREAT | os.O_EXCL)
            os.close(fd)
        except OSError:
            return
        file_lock = open(self.lockFile, "w")
        file_lock.write("0\n")
        file_lock.close()

    def initLockFile(self):
        """
            Initialize and write the information for the current process

        """
        while True:
            processes = glob.glob(os.path.join(self.syncFolder, "*.proc"))
            processes.sort()
            if len(processes) > self.nReplicas:
                raise utilities.ImproperParameterValueException("More processors files than replicas found, this could be due to wrong number of replicas chosen in the control file or files remaining from previous that were not clean properly")
            if len(processes) != self.nReplicas:
                time.sleep(self.sleepTime)
                continue
            # only reach this block if all processes have written their own
            # files
            for i, process_file in enumerate(processes):
                process = int(os.path.splitext(os.path.split(process_file)[1])[0])
                self.lockInfo[process] = (i, self.status)
                if process == self.pid:
                    self.id = i
            break
        file_lock = open(self.lockFile, "r+")
        while True:
            fcntl.lockf(file_lock, fcntl.LOCK_EX)
            if self.isMaster():
                self.writeLockInfo(file_lock)
                fcntl.lockf(file_lock, fcntl.LOCK_UN)
                self.createStateStack()
                file_lock.close()
                return
            else:
                lock_info = self.getLockInfo(file_lock)
                fcntl.lockf(file_lock, fcntl.LOCK_UN)
                # ensure that all process are created before continuing
                if len(lock_info) == len(self.lockInfo):
                    self.createStateStack()
                    file_lock.close()
                    return

    def createStateStack(self):
        """
            Create a stack with all states that each replica has gone through
        """
        for pid in self.lockInfo:
            self.stateStack[pid] = set([self.lockInfo[pid][1]])

    def updateStateStack(self):
        """
            Update the stack with all states that each replica has gone through
        """
        for pid in self.lockInfo:
            self.stateStack[pid].add(self.lockInfo[pid][1])

    def getLockInfo(self, file_descriptor):
        """
            Return the info stored in the lock file

            :param file_descriptor: File object of the lock file
            :type file_descriptor: file

            :returns: dict -- A dictonary containing the information of the
            different process managers initialized
        """
        info = {}
        file_descriptor.seek(0)
        for line in file_descriptor:
            if line == "0\n":
                break
            pid, id_num, label = line.rstrip().split(":")
            info[int(pid)] = (int(id_num), label)
        return info

    def writeLockInfo(self, file_descriptor):
        """
            Write the lock info to the lock file

            :param file_descriptor: File object of the lock file
            :type file_descriptor: file
        """
        file_descriptor.seek(0)
        for pid, data in self.lockInfo.items():
            file_descriptor.write("%d:%d:%s\n" % ((pid,)+data))
        file_descriptor.truncate()

    def isMaster(self):
        """
            Return wether the current process is the master process
        """
        return self.id == 0

    def setStatus(self, status):
        """
            Set the current status of the process

            :param status: Status of the process (INIT, WAITING or RUNNING)
            :type status: str
        """
        self.status = status
        file_lock = open(self.lockFile, "r+")
        fcntl.lockf(file_lock, fcntl.LOCK_EX)
        self.lockInfo = self.getLockInfo(file_lock)
        self.lockInfo[self.pid] = (self.id, self.status)
        self.updateStateStack()
        self.writeLockInfo(file_lock)
        fcntl.lockf(file_lock, fcntl.LOCK_UN)
        file_lock.close()

    def getStatus(self):
        """
            Return the current status of the process
        """
        return self.status

    def isSynchronized(self, status):
        """
            Return wether all processes are synchronized, that is, they all
            have the same status

            :param status: Status of the process (INIT, WAITING or RUNNING)
            :type status: str

            :returns: bool -- Whether all processes are synchronized
        """
        assert len(self.lockInfo) > 0, "No processes found in lockInfo!!!"
        for pid in self.stateStack:
            if status not in self.lockInfo[pid]:
                return False
        return True

    def synchronize(self, status):
        """
            Create a barrier-like situation to wait for all processes to finish
        """
        while True:
            if self.isSynchronized(status):
                # wait for some time before returning to avoid deadlocks caused
                # by lock reaquisition by the same process due to poor
                # load-balance between different syncronize calls
                time.sleep(self.sleepTime)
                return
            file_lock = open(self.lockFile, "r+")
            fcntl.lockf(file_lock, fcntl.LOCK_EX)
            self.lockInfo = self.getLockInfo(file_lock)
            self.updateStateStack()
            fcntl.lockf(file_lock, fcntl.LOCK_UN)
            file_lock.close()

    def allRunning(self):
        """
            Check if all processes are still running
        """
        for pid in self.lockInfo:
            try:
                os.kill(pid, 0)
            except ProcessLookupError:
                print("Process %d not found!!!" % pid)
                return False
        return True

    def writeEquilibrationStructures(self, path, structures):
        """
            Write the equilibration structures for the current replica

            :param path: Path where to write the structures
            :type path: str
            :param structures: Filename with the structures
            :type structures: list
        """
        outfile = os.path.join(path, "structures_equilibration_%d.txt" % self.id)
        with open(outfile, "w") as fw:
            fw.write(",".join(structures))

    def readEquilibrationStructures(self, path):
        """
            Read the equilibration structures for all replicas

            :param path: Path from where to read the structures
            :type path: str
        """
        files = glob.glob(os.path.join(path, "structures_equilibration_*.txt"))
        assert len(files) == self.__len__(), "Missing files for some of the replicas"
        structures = []
        for i in range(self.__len__()):
            with open(os.path.join(path, "structures_equilibration_%d.txt" % i)) as fr:
                structure_partial = fr.read().rstrip().split(",")
            if structure_partial != [""]:
                structures.extend(structure_partial)
        return structures

    def getStructureListPerReplica(self, initialStructures, trajsPerReplica):
        """
            Filter the list of initial structures to select only the ones
            corresponding to the current replica

            :param initialStructures: Name of the initial structures to copy
            :type initialStructures: list of str
            :param trajsPerReplica: Number of trajectories that each replica has to calculate
            :type trajsPerReplica: int

            :returns: list -- List with a tuple containing the initial structures of the replica and their indices
        """
        n_structures = len(initialStructures)
        end = min(n_structures, (self.id+1)*trajsPerReplica)
        return [(i, initialStructures[i]) for i in range(self.id*trajsPerReplica, end)]

    def getBarrierName(self):
        """
            Create a unique status name so that every time we synchronize we do
            it under a different name
        """
        self.syncStep += 1
        return "%s-%d" % (self.WAITING, self.syncStep)

    def barrier(self):
        """
            Create a barrier
        """
        status = self.getBarrierName()
        self.setStatus(status)
        self.synchronize(status)
<commit_msg>Now lockInfo holds all the status of each replica<commit_after>import os
import time
import glob
import fcntl
from AdaptivePELE.utilities import utilities

try:
    ProcessLookupError
except NameError:
    ProcessLookupError = OSError

class ProcessesManager:
    """
        Object that sinchronizes multiple adaptivePELE instances, designed to
        be able to use multiple nodes of a gpu cluster
    """
    RUNNING = "RUNNING"
    WAITING = "WAITING"
    INIT = "INIT"

    def __init__(self, output_path, num_replicas):
        self.syncFolder =  os.path.join(os.path.abspath(output_path), "synchronization")
        utilities.makeFolder(self.syncFolder)
        self.lockFile = os.path.join(self.syncFolder,  "syncFile.lock")
        self.pid = os.getpid()
        self.nReplicas = num_replicas
        self.id = None
        self.lockInfo = {}
        self.status = self.INIT
        self.sleepTime = 0.5
        self.createLockFile()
        self.writeProcessInfo()
        self.initLockFile()
        self.syncStep = 0

    def __len__(self):
        # define the size of the ProcessesManager object as the number of
        # processes
        return len(self.lockInfo)

    def writeProcessInfo(self):
        with open(os.path.join(self.syncFolder, "%d.proc" % self.pid), "w") as fw:
            fw.write("%d\n" % self.pid)

    def createLockFile(self):
        """
            Create the lock file
        """
        try:
            # whith the flags set, if the file exists the open will fail
            fd = os.open(self.lockFile, os.O_CREAT | os.O_EXCL)
            os.close(fd)
        except OSError:
            return
        file_lock = open(self.lockFile, "w")
        file_lock.write("0\n")
        file_lock.close()

    def initLockFile(self):
        """
            Initialize and write the information for the current process

        """
        while True:
            processes = glob.glob(os.path.join(self.syncFolder, "*.proc"))
            processes.sort()
            if len(processes) > self.nReplicas:
                raise utilities.ImproperParameterValueException("More processors files than replicas found, this could be due to wrong number of replicas chosen in the control file or files remaining from previous that were not clean properly")
            if len(processes) != self.nReplicas:
                time.sleep(self.sleepTime)
                continue
            # only reach this block if all processes have written their own
            # files
            for i, process_file in enumerate(processes):
                process = int(os.path.splitext(os.path.split(process_file)[1])[0])
                self.lockInfo[process] = (i, set([self.status]))
                if process == self.pid:
                    self.id = i
            break
        file_lock = open(self.lockFile, "r+")
        while True:
            fcntl.lockf(file_lock, fcntl.LOCK_EX)
            if self.isMaster():
                self.writeLockInfo(file_lock)
                fcntl.lockf(file_lock, fcntl.LOCK_UN)
                file_lock.close()
                return
            else:
                lock_info = self.getLockInfo(file_lock)
                fcntl.lockf(file_lock, fcntl.LOCK_UN)
                # ensure that all process are created before continuing
                if len(lock_info) == len(self.lockInfo):
                    file_lock.close()
                    return

    def getLockInfo(self, file_descriptor):
        """
            Return the info stored in the lock file

            :param file_descriptor: File object of the lock file
            :type file_descriptor: file

            :returns: dict -- A dictonary containing the information of the
            different process managers initialized
        """
        info = {}
        file_descriptor.seek(0)
        for line in file_descriptor:
            if line == "0\n":
                break
            pid, id_num, label = line.rstrip().split(":")
            info[int(pid)] = (int(id_num), set(label.split(";")))
        return info

    def writeLockInfo(self, file_descriptor):
        """
            Write the lock info to the lock file

            :param file_descriptor: File object of the lock file
            :type file_descriptor: file
        """
        file_descriptor.seek(0)
        for pid, data in self.lockInfo.items():
            id_num, status_set = data
            file_descriptor.write("%d:%d:%s\n" % (pid, id_num, ";".join(status_set)))
        file_descriptor.truncate()

    def isMaster(self):
        """
            Return wether the current process is the master process
        """
        return self.id == 0

    def setStatus(self, status):
        """
            Set the current status of the process

            :param status: Status of the process (INIT, WAITING or RUNNING)
            :type status: str
        """
        self.status = status
        file_lock = open(self.lockFile, "r+")
        fcntl.lockf(file_lock, fcntl.LOCK_EX)
        self.lockInfo = self.getLockInfo(file_lock)
        self.lockInfo[self.pid][1].add(self.status)
        self.writeLockInfo(file_lock)
        fcntl.lockf(file_lock, fcntl.LOCK_UN)
        file_lock.close()

    def getStatus(self):
        """
            Return the current status of the process
        """
        return self.status

    def isSynchronized(self, status):
        """
            Return wether all processes are synchronized, that is, they all
            have the same status

            :param status: Status of the process (INIT, WAITING or RUNNING)
            :type status: str

            :returns: bool -- Whether all processes are synchronized
        """
        assert len(self.lockInfo) > 0, "No processes found in lockInfo!!!"
        for pid in self.lockInfo:
            if status not in self.lockInfo[pid][1]:
                return False
        return True

    def synchronize(self, status):
        """
            Create a barrier-like situation to wait for all processes to finish
        """
        while True:
            if self.isSynchronized(status):
                return
            file_lock = open(self.lockFile, "r+")
            fcntl.lockf(file_lock, fcntl.LOCK_EX)
            self.lockInfo = self.getLockInfo(file_lock)
            fcntl.lockf(file_lock, fcntl.LOCK_UN)
            file_lock.close()

    def allRunning(self):
        """
            Check if all processes are still running
        """
        for pid in self.lockInfo:
            try:
                os.kill(pid, 0)
            except ProcessLookupError:
                print("Process %d not found!!!" % pid)
                return False
        return True

    def writeEquilibrationStructures(self, path, structures):
        """
            Write the equilibration structures for the current replica

            :param path: Path where to write the structures
            :type path: str
            :param structures: Filename with the structures
            :type structures: list
        """
        outfile = os.path.join(path, "structures_equilibration_%d.txt" % self.id)
        with open(outfile, "w") as fw:
            fw.write(",".join(structures))

    def readEquilibrationStructures(self, path):
        """
            Read the equilibration structures for all replicas

            :param path: Path from where to read the structures
            :type path: str
        """
        files = glob.glob(os.path.join(path, "structures_equilibration_*.txt"))
        assert len(files) == self.__len__(), "Missing files for some of the replicas"
        structures = []
        for i in range(self.__len__()):
            with open(os.path.join(path, "structures_equilibration_%d.txt" % i)) as fr:
                structure_partial = fr.read().rstrip().split(",")
            if structure_partial != [""]:
                structures.extend(structure_partial)
        return structures

    def getStructureListPerReplica(self, initialStructures, trajsPerReplica):
        """
            Filter the list of initial structures to select only the ones
            corresponding to the current replica

            :param initialStructures: Name of the initial structures to copy
            :type initialStructures: list of str
            :param trajsPerReplica: Number of trajectories that each replica has to calculate
            :type trajsPerReplica: int

            :returns: list -- List with a tuple containing the initial structures of the replica and their indices
        """
        n_structures = len(initialStructures)
        end = min(n_structures, (self.id+1)*trajsPerReplica)
        return [(i, initialStructures[i]) for i in range(self.id*trajsPerReplica, end)]

    def getBarrierName(self):
        """
            Create a unique status name so that every time we synchronize we do
            it under a different name
        """
        self.syncStep += 1
        return "%s-%d" % (self.WAITING, self.syncStep)

    def barrier(self):
        """
            Create a barrier
        """
        status = self.getBarrierName()
        self.setStatus(status)
        self.synchronize(status)
<|endoftext|>


<commit_before># __BEGIN_LICENSE__
# Copyright (c) 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
# The xGDS platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
# __END_LICENSE__
from django.contrib.auth.decorators import login_required
from django.shortcuts import render_to_response,  get_object_or_404, redirect
from django.core.urlresolvers import reverse
from django.http import HttpResponseBadRequest, HttpResponse
from django.template import RequestContext
from django.forms.formsets import formset_factory
from django.conf import settings
from django.contrib import messages
from django.db.models import Max
from django.contrib.auth.models import User
from PyPDF2 import PdfFileMerger, PdfFileReader

import json
import os
import re

from geocamUtil.loader import getClassByName, LazyGetModelByName
from forms import SampleForm
from xgds_data.forms import SearchForm, SpecializedForm
from xgds_sample.models import SampleType, Region, SampleLabelSize
from xgds_core.views import get_handlebars_templates

from geocamUtil.datetimeJsonEncoder import DatetimeJsonEncoder
from django.views.static import serve
from xgds_sample.labels import *

from django.http import HttpResponse
from StringIO import StringIO

SAMPLE_MODEL = LazyGetModelByName(settings.XGDS_SAMPLE_SAMPLE_MODEL)
LABEL_MODEL = LazyGetModelByName(settings.XGDS_SAMPLE_LABEL_MODEL)

TRACK_MODEL = LazyGetModelByName(settings.GEOCAM_TRACK_TRACK_MODEL)
POSITION_MODEL = LazyGetModelByName(settings.GEOCAM_TRACK_PAST_POSITION_MODEL)
RESOURCE_MODEL = LazyGetModelByName(settings.GEOCAM_TRACK_RESOURCE_MODEL)

XGDS_SAMPLE_TEMPLATE_LIST = list(settings.XGDS_MAP_SERVER_HANDLEBARS_DIRS)
XGDS_SAMPLE_TEMPLATE_LIST = XGDS_SAMPLE_TEMPLATE_LIST + settings.XGDS_CORE_TEMPLATE_DIRS[settings.XGDS_SAMPLE_SAMPLE_MODEL]

# get all user names (string)
def getUserNames():
    allUsers = [user.first_name + ' ' + user.last_name  for user in User.objects.all()]
    allUsers = [str(x) for x in allUsers if x.strip()]
    return allUsers

def deleteLabelAndSample(request, labelNum):
    label = LABEL_MODEL.get().objects.get(number=labelNum)
    message = 'Deleted label %d' % label.number
    try:
        message = message + ' and sample %s ' % label.sample.name
        label.sample.delete()
    except:
        pass
    label.delete()
    messages.error(request, message)
    return render_to_response('xgds_sample/recordSample.html',
                              RequestContext(request, {}))

@login_required
def editSample(request, samplePK, form=None):
    if not form:
        sample = SAMPLE_MODEL.get().objects.get(pk=samplePK)
        form = SampleForm(instance=sample)
    return render_to_response('xgds_sample/sampleEdit.html',
                              RequestContext(request, {'form': form,
                                                       'users': getUserNames(),
                                                       'modelName': settings.XGDS_SAMPLE_SAMPLE_KEY,
                                                       'templates': get_handlebars_templates(list(settings.XGDS_MAP_SERVER_HANDLEBARS_DIRS), 'XGDS_MAP_SERVER_HANDLEBARS_DIRS')})
                              )

@login_required
def viewSampleByLabel(request, labelNum):
    try:
        label, create = LABEL_MODEL.get().objects.get_or_create(number=labelNum)
        if create:
            return editSample(request, label.sample.pk)
        else:
            return redirect(reverse('search_map_single_object', kwargs={'modelPK':label.sample.pk,
                                                                        'modelName': settings.XGDS_SAMPLE_SAMPLE_KEY}))
    except:
        return createSample(request, labelNum)

def createSample(request, labelNum, label=None):
    """
    Create a label and/or sample based on the requested label number
    """
    if not label:
        label, create = LABEL_MODEL.get().objects.get_or_create(number=labelNum)
    sample, sample_create = SAMPLE_MODEL.get().objects.get_or_create(label=label)
    if sample_create:
        label.url = reverse('search_map_single_object', kwargs={'modelName':settings.XGDS_SAMPLE_SAMPLE_KEY,
                                                                'modelPK':sample.pk})
        label.save()
        # save sample location from resource's track.
        if sample.resource is not None:
            resrc = sample.resource
        else:
            collectorName = settings.XGDS_SAMPLE_DEFAULT_COLLECTOR
            resrc = RESOURCE_MODEL.get().objects.get(name=collectorName)
            track = TRACK_MODEL.get().objects.get(resource=resrc)
            latest_position = track.getPositions().latest('timestamp')
            sample.track_position = latest_position
            sample.save()
    form = SampleForm(instance=sample)
    return editSample(request, sample.pk, form)

def setSampleCustomFields(form, sample):
    # set custom field values with existing data.
    positionDict = sample.getPositionDict()
    form.fields['latitude'].initial = positionDict['lat']
    form.fields['longitude'].initial = positionDict['lon']
    if 'altitude' in positionDict:
        form.fields['altitude'].initial = positionDict['altitude']
    if sample.collection_time:
        form.fields['collection_time'].initial = sample.collection_time
    if sample.collector:
        form.fields['collector'].initial = sample.collector.first_name + ' ' + sample.collector.last_name
    return form

@login_required
def getSampleEditPage(request):
    label = None
    if request.POST:
        numberOrName = request.POST['label_num_or_sample_name']
        if not numberOrName:
            messages.error(request, 'Please enter a valid sample name or label number')
            return render_to_response('xgds_sample/recordSample.html',
                                       RequestContext(request, {}))
        labelNum = None
        label = None
        sample = None
        try:
            labelNum = int(numberOrName)
        except:
            try:
                sample = SAMPLE_MODEL.get().objects.get(name=numberOrName)
                label = sample.label
            except:
                messages.error(request, 'Could not find sample %s; start with a label. ' % numberOrName,
                               extra_tags='safe')
                return render_to_response('xgds_sample/recordSample.html',
                                          RequestContext(request, {}))

        if labelNum and not label:
            try:
                label = LABEL_MODEL.get().objects.get(number=labelNum)
                sample = label.sample
            except:
                return createSample(request, labelNum, label)
        form = SampleForm(instance=sample)
        return editSample(request, sample.pk, form)

@login_required
def editSampleByLabel(request, labelNum):
    """ make changes to a sample based on form inputs and save,
    OR open the edit page
    """
    # get all user names (first last). Needed for autocompleting collector field.
    try:
        label, create = LABEL_MODEL.get().objects.get_or_create(number=labelNum)
        sample = label.sample
    except:
        return createSample(request, labelNum, label)
    # if is updating the sample info from edit form
    if request.method == "POST":
        # swap the user id
        form = SampleForm(request.POST, instance=sample)
        if form.is_valid():
            form.save()
            if form.errors:
                for key, msg in form.errors.items():
                    if key == 'warning':
                        messages.warning(request, msg)
                    elif key == 'error':
                        messages.error(request, msg)
            else:
                messages.success(request, 'Sample data successfully updated.')
            return redirect(reverse('search_map_single_object', kwargs={'modelPK': form.instance.pk,
                                                                        'modelName': settings.XGDS_SAMPLE_SAMPLE_KEY}))
        else:
            messages.error(request, 'The form is not valid')
            return editSample(request, sample.pk, form)

    # edit page opened via edit/<label number>
    elif request.method == "GET":
        form = SampleForm(instance=sample)
        return editSample(request, sample.pk, form)

@login_required
def getSampleLabelsPage(request):
    labels = LABEL_MODEL.get().objects.all()
    return render_to_response('xgds_sample/sampleLabels.html',
                              RequestContext(request,
                                             {'labels': labels,
                                              'file_url': ""}))

def chunks(l, n):
    n = max(1, n)
    return [l[i:i + n] for i in range(0, len(l), n)]

def printSampleLabels(request):
    data = {'file_url': ""}
    if request.method == 'POST':
        if 'label_quantity' in request.POST:
            quantity = int(request.POST['label_quantity'])
            labels = LABEL_MODEL.get().objects.all()
            data['labels'] = labels
            aggregate = labels.aggregate(Max('number'))
            if aggregate['number__max']:
                startNum = aggregate['number__max'] + 1
            else:
                startNum = 1
            labelNumbers = range(startNum, startNum + quantity)
            labelsToPrint = []
            for labelNum in labelNumbers:
                label, create = LABEL_MODEL.get().objects.get_or_create(number=int(labelNum))
                sample, sample_create = SAMPLE_MODEL.get().objects.get_or_create(label=label)
                sample.save()
                if sample_create:
                    label.url = reverse('search_map_single_object', kwargs={'modelName':settings.XGDS_SAMPLE_SAMPLE_KEY,
                                                                            'modelPK':sample.pk})
                    label.save()
                labelsToPrint.append(label)
            if quantity <=0:
                messages.error(request, "Quantity must be an integer greater than 0.")
            else:
                messages.error(request, "")
        else:
            labelIds = request.POST.getlist('label_checkbox')
            labelsToPrint = [LABEL_MODEL.get().objects.get(id=int(labelId)) for labelId in labelIds]
        if labelsToPrint:
            size = SampleLabelSize.objects.get(name="small")
            labelChunks = chunks(labelsToPrint, 10)  # labels in chunks of 10. (list of lists)
            pdfFileNames = []
            index = 0
            for labelChunk in labelChunks:
                pdfFileName = generateMultiPDF(labelChunk, size, index)
                pdfFileNames.append(pdfFileName)
                index = index + 1

            # merge the single sheet of pdf files into one to create multiple page pdf.
            merger = PdfFileMerger()
            outputFileName = re.sub(r"_temp\d+", "", pdfFileNames[0])
            for filename in pdfFileNames:
                merger.append(PdfFileReader(file(filename, 'rb')))
                # delete the temporary file
                os.remove(filename)
            merger.write(outputFileName)  # merge them into one file, and take the first file's name

            outputFileName = outputFileName.replace(settings.DATA_ROOT, settings.DATA_URL)
            messages.success(request, "Labels successfully generated.")
            data['file_url'] = outputFileName
    return render_to_response('xgds_sample/sampleLabels.html',
                               RequestContext(request, data))

def getSampleHelpPage(request):
    image1Url = settings.STATIC_URL + 'xgds_sample/images/Slide1.png'
    image2Url = settings.STATIC_URL + 'xgds_sample/images/Slide2.png'
    image3Url = settings.STATIC_URL + 'xgds_sample/images//Slide3.png'
    data = {'image1': image1Url,
            'image2': image2Url,
            'image3': image3Url}
    return render_to_response('xgds_sample/sampleHelp.html',
                              RequestContext(request, data))
    <commit_msg>sample position update - copied from xgds_image<commit_after># __BEGIN_LICENSE__
# Copyright (c) 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
# The xGDS platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
# __END_LICENSE__
from django.contrib.auth.decorators import login_required
from django.shortcuts import render_to_response,  get_object_or_404, redirect
from django.core.urlresolvers import reverse
from django.http import HttpResponseBadRequest, HttpResponse
from django.template import RequestContext
from django.forms.formsets import formset_factory
from django.conf import settings
from django.contrib import messages
from django.db.models import Max
from django.contrib.auth.models import User
from PyPDF2 import PdfFileMerger, PdfFileReader

from datetime import datetime
import json
import os
import re

from geocamUtil.loader import getClassByName, LazyGetModelByName
from forms import SampleForm
from xgds_data.forms import SearchForm, SpecializedForm
from xgds_sample.models import SampleType, Region, SampleLabelSize
from xgds_core.views import get_handlebars_templates
from geocamTrack.utils import getClosestPosition

from geocamUtil.datetimeJsonEncoder import DatetimeJsonEncoder
from django.views.static import serve
from xgds_sample.labels import *

from django.http import HttpResponse
from StringIO import StringIO

SAMPLE_MODEL = LazyGetModelByName(settings.XGDS_SAMPLE_SAMPLE_MODEL)
LABEL_MODEL = LazyGetModelByName(settings.XGDS_SAMPLE_LABEL_MODEL)

TRACK_MODEL = LazyGetModelByName(settings.GEOCAM_TRACK_TRACK_MODEL)
POSITION_MODEL = LazyGetModelByName(settings.GEOCAM_TRACK_PAST_POSITION_MODEL)
RESOURCE_MODEL = LazyGetModelByName(settings.GEOCAM_TRACK_RESOURCE_MODEL)

XGDS_SAMPLE_TEMPLATE_LIST = list(settings.XGDS_MAP_SERVER_HANDLEBARS_DIRS)
XGDS_SAMPLE_TEMPLATE_LIST = XGDS_SAMPLE_TEMPLATE_LIST + settings.XGDS_CORE_TEMPLATE_DIRS[settings.XGDS_SAMPLE_SAMPLE_MODEL]

# get all user names (string)
def getUserNames():
    allUsers = [user.first_name + ' ' + user.last_name  for user in User.objects.all()]
    allUsers = [str(x) for x in allUsers if x.strip()]
    return allUsers

def deleteLabelAndSample(request, labelNum):
    label = LABEL_MODEL.get().objects.get(number=labelNum)
    message = 'Deleted label %d' % label.number
    try:
        message = message + ' and sample %s ' % label.sample.name
        label.sample.delete()
    except:
        pass
    label.delete()
    messages.error(request, message)
    return render_to_response('xgds_sample/recordSample.html',
                              RequestContext(request, {}))

def getTrackPosition(timestamp, resource):
    '''
    Look up and return the closest tracked position if there is one.
    '''
    return getClosestPosition(timestamp=timestamp, resource=resource)

@login_required
def editSample(request, samplePK, form=None):
    if not form:
        sample = SAMPLE_MODEL.get().objects.get(pk=samplePK)
        form = SampleForm(instance=sample)
    return render_to_response('xgds_sample/sampleEdit.html',
                              RequestContext(request, {'form': form,
                                                       'users': getUserNames(),
                                                       'modelName': settings.XGDS_SAMPLE_SAMPLE_KEY,
                                                       'templates': get_handlebars_templates(list(settings.XGDS_MAP_SERVER_HANDLEBARS_DIRS), 'XGDS_MAP_SERVER_HANDLEBARS_DIRS')})
                              )

@login_required
def viewSampleByLabel(request, labelNum):
    try:
        label, create = LABEL_MODEL.get().objects.get_or_create(number=labelNum)
        if create:
            return editSample(request, label.sample.pk)
        else:
            return redirect(reverse('search_map_single_object', kwargs={'modelPK':label.sample.pk,
                                                                        'modelName': settings.XGDS_SAMPLE_SAMPLE_KEY}))
    except:
        return createSample(request, labelNum)

def createSample(request, labelNum, label=None):
    """
    Create a label and/or sample based on the requested label number
    """
    if not label:
        label, create = LABEL_MODEL.get().objects.get_or_create(number=labelNum)
    sample, sample_create = SAMPLE_MODEL.get().objects.get_or_create(label=label)
    if sample_create:
        label.url = reverse('search_map_single_object', kwargs={'modelName':settings.XGDS_SAMPLE_SAMPLE_KEY,
                                                                'modelPK':sample.pk})
        label.save()
        # save sample location from resource's track.
        if sample.resource is not None:
            resrc = sample.resource
        else:
            collectorName = settings.XGDS_SAMPLE_DEFAULT_COLLECTOR
            resrc = RESOURCE_MODEL.get().objects.get(name=collectorName)
            sample.track_position = getTrackPosition(datetime.utcnow(), resrc)
            sample.save()
    form = SampleForm(instance=sample)
    return editSample(request, sample.pk, form)

def setSampleCustomFields(form, sample):
    # set custom field values with existing data.
    positionDict = sample.getPositionDict()
    form.fields['latitude'].initial = positionDict['lat']
    form.fields['longitude'].initial = positionDict['lon']
    if 'altitude' in positionDict:
        form.fields['altitude'].initial = positionDict['altitude']
    if sample.collection_time:
        form.fields['collection_time'].initial = sample.collection_time
    if sample.collector:
        form.fields['collector'].initial = sample.collector.first_name + ' ' + sample.collector.last_name
    return form

@login_required
def getSampleEditPage(request):
    label = None
    if request.POST:
        numberOrName = request.POST['label_num_or_sample_name']
        if not numberOrName:
            messages.error(request, 'Please enter a valid sample name or label number')
            return render_to_response('xgds_sample/recordSample.html',
                                       RequestContext(request, {}))
        labelNum = None
        label = None
        sample = None
        try:
            labelNum = int(numberOrName)
        except:
            try:
                sample = SAMPLE_MODEL.get().objects.get(name=numberOrName)
                label = sample.label
            except:
                messages.error(request, 'Could not find sample %s; start with a label. ' % numberOrName,
                               extra_tags='safe')
                return render_to_response('xgds_sample/recordSample.html',
                                          RequestContext(request, {}))

        if labelNum and not label:
            try:
                label = LABEL_MODEL.get().objects.get(number=labelNum)
                sample = label.sample
            except:
                return createSample(request, labelNum, label)
        form = SampleForm(instance=sample)
        return editSample(request, sample.pk, form)

@login_required
def editSampleByLabel(request, labelNum):
    """ make changes to a sample based on form inputs and save,
    OR open the edit page
    """
    # get all user names (first last). Needed for autocompleting collector field.
    try:
        label, create = LABEL_MODEL.get().objects.get_or_create(number=labelNum)
        sample = label.sample
    except:
        return createSample(request, labelNum, label)
    # if is updating the sample info from edit form
    if request.method == "POST":
        # swap the user id
        form = SampleForm(request.POST, instance=sample)
        if form.is_valid():
            form.save()
            if form.errors:
                for key, msg in form.errors.items():
                    if key == 'warning':
                        messages.warning(request, msg)
                    elif key == 'error':
                        messages.error(request, msg)
            else:
                messages.success(request, 'Sample data successfully updated.')
            return redirect(reverse('search_map_single_object', kwargs={'modelPK': form.instance.pk,
                                                                        'modelName': settings.XGDS_SAMPLE_SAMPLE_KEY}))
        else:
            messages.error(request, 'The form is not valid')
            return editSample(request, sample.pk, form)

    # edit page opened via edit/<label number>
    elif request.method == "GET":
        form = SampleForm(instance=sample)
        return editSample(request, sample.pk, form)

@login_required
def getSampleLabelsPage(request):
    labels = LABEL_MODEL.get().objects.all()
    return render_to_response('xgds_sample/sampleLabels.html',
                              RequestContext(request,
                                             {'labels': labels,
                                              'file_url': ""}))

def chunks(l, n):
    n = max(1, n)
    return [l[i:i + n] for i in range(0, len(l), n)]

def printSampleLabels(request):
    data = {'file_url': ""}
    if request.method == 'POST':
        if 'label_quantity' in request.POST:
            quantity = int(request.POST['label_quantity'])
            labels = LABEL_MODEL.get().objects.all()
            data['labels'] = labels
            aggregate = labels.aggregate(Max('number'))
            if aggregate['number__max']:
                startNum = aggregate['number__max'] + 1
            else:
                startNum = 1
            labelNumbers = range(startNum, startNum + quantity)
            labelsToPrint = []
            for labelNum in labelNumbers:
                label, create = LABEL_MODEL.get().objects.get_or_create(number=int(labelNum))
                sample, sample_create = SAMPLE_MODEL.get().objects.get_or_create(label=label)
                sample.save()
                if sample_create:
                    label.url = reverse('search_map_single_object', kwargs={'modelName':settings.XGDS_SAMPLE_SAMPLE_KEY,
                                                                            'modelPK':sample.pk})
                    label.save()
                labelsToPrint.append(label)
            if quantity <=0:
                messages.error(request, "Quantity must be an integer greater than 0.")
            else:
                messages.error(request, "")
        else:
            labelIds = request.POST.getlist('label_checkbox')
            labelsToPrint = [LABEL_MODEL.get().objects.get(id=int(labelId)) for labelId in labelIds]
        if labelsToPrint:
            size = SampleLabelSize.objects.get(name="small")
            labelChunks = chunks(labelsToPrint, 10)  # labels in chunks of 10. (list of lists)
            pdfFileNames = []
            index = 0
            for labelChunk in labelChunks:
                pdfFileName = generateMultiPDF(labelChunk, size, index)
                pdfFileNames.append(pdfFileName)
                index = index + 1

            # merge the single sheet of pdf files into one to create multiple page pdf.
            merger = PdfFileMerger()
            outputFileName = re.sub(r"_temp\d+", "", pdfFileNames[0])
            for filename in pdfFileNames:
                merger.append(PdfFileReader(file(filename, 'rb')))
                # delete the temporary file
                os.remove(filename)
            merger.write(outputFileName)  # merge them into one file, and take the first file's name

            outputFileName = outputFileName.replace(settings.DATA_ROOT, settings.DATA_URL)
            messages.success(request, "Labels successfully generated.")
            data['file_url'] = outputFileName
    return render_to_response('xgds_sample/sampleLabels.html',
                               RequestContext(request, data))

def getSampleHelpPage(request):
    image1Url = settings.STATIC_URL + 'xgds_sample/images/Slide1.png'
    image2Url = settings.STATIC_URL + 'xgds_sample/images/Slide2.png'
    image3Url = settings.STATIC_URL + 'xgds_sample/images//Slide3.png'
    data = {'image1': image1Url,
            'image2': image2Url,
            'image3': image3Url}
    return render_to_response('xgds_sample/sampleHelp.html',
                              RequestContext(request, data))
    <|endoftext|>


<commit_before># -*- coding: cp1252 -*-

from xmppd import filter
from xmpp import *
from spade import *
from spade.ACLMessage import *

import os

class ACC(filter.Filter):

	def __init__(self,router):

		filter.Filter.__init__(self,router)

		if os.name == "posix":
			configfile = "/etc/spade/spade.xml"
		else:
			configfile = "etc/spade.xml"

		parser = SpadeConfigParser.ConfigParser()

		config = parser.parse(configfile)

		path = str(config.platform.path)

		try:
			mod = __import__(u"/usr/share/spade/mtp/simba")
		except Exception, e:
			print "ACC: HARDCODE IMPORT FAILED"

		self.mtps = {}
		for name,mtp in config.acc.items():
			#self.mtps[mtp.protocol] = mtp.instance(name)
			try:
				#mod = __import__(str(path + os.sep + "mtp" + os.sep + name))
				mod = __import__("/usr/share/spade/mtp/simba")
			except Exception, e:
				print "PETO AL IMPORTAR " + str(path + os.sep + "mtp" + os.sep + name) + ":" + str(e)
				print os.path.realpath(os.path.curdir)
			try:
				self.mtps[mtp['protocol']] = mod.INSTANCE(name,config)
			except:
				print "Y MI INSTANCIA? " + str(name)

	def test(self,stanza):

            	children = stanza.getChildren()
		#self._router.DEBUG("ACC PLUGIN: analyzing message structure", "info")
            	for child in children:
                	if (child.getNamespace() == "jabber:x:fipa") or (child.getNamespace() == u"jabber:x:fipa"):
				self.envelope = child.getData()
				return True

		return False

		"""
		simple_to = str(stanza['to'])
		if not('@' in simple_to):  # Component name
			# SPADE CASE: message directed to the ACC component
			if simple_to[0:4] == "acc.":
				#self._router.DEBUG("Message for the ACC",'ok')
				return True
		#self._router.DEBUG("Message NOT for the ACC",'error')
		return False
		"""

	def filter(self,session,stanza):

		xmpp_processed = False

		if (stanza.getError() == None):
			envxml = self.envelope
			#payload = stanza.getBody()
			payload = stanza.getTag("body")
			xc = XMLCodec.XMLCodec()
			self.envelope = xc.parse(str(envxml))

			for aid in self.envelope.getTo():
				for addr in aid.getAddresses():
					protocol = addr.split("://")[0]

					if protocol == "xmpp":
						#FIXME: only supports 1 xmpp sender
						receivers = self.getRealTo(stanza)
						to = str(receivers[0])  # FIX THIS TO ALLOW MULTIPLE RECEIVERS
						# We do NOT raise the NodeProcessed exception,
						# the message follows its normal course from here
						stanza.setTo(to)
						xmpp_processed = True

					elif protocol in self.mtps.keys():
						print ">>>ACC: Message for external MTP: " + str(protocol)
						mtp = self.mtps[protocol]
						print ">>>ACC: MTP found: " + str(mtp)
						#envelope is in Envelope format
						#payload is in string format (with escaped characters)
						mtp.send(self.envelope,payload)
						print ">>>ACC: External MTP message sent"

					else:
						print ">>>ACC: Could not find suitable MTP for protocol: " + str(protocol)

		if xmpp_processed: return stanza
		else: return None

	def getRealTo(self, mess):
		#return the real JID of the receiver of a "jabber:x:fipa" message

		receiver_names = []
		for aid in self.envelope.getTo():
			receiver_names.append(str(aid.getName()))

		return receiver_names

<commit_msg>packaging<commit_after># -*- coding: cp1252 -*-

from xmppd import filter
from xmpp import *
from spade import *
from spade.ACLMessage import *

import os, os.path

class ACC(filter.Filter):

	def __init__(self,router):

		filter.Filter.__init__(self,router)

		if os.name == "posix":
			configfile = "/etc/spade/spade.xml"
		else:
			configfile = "etc/spade.xml"

		parser = SpadeConfigParser.ConfigParser()

		config = parser.parse(configfile)

		path = str(config.platform.path)

		self.mtps = {}
		curdir = os.path.curdir
		os.chdir(path)
		for name,mtp in config.acc.items():
			#self.mtps[mtp.protocol] = mtp.instance(name)
			try:
				mod = __import__(str(name))
			except Exception, e:
				print "PETO AL IMPORTAR " + str(name) + ":" + str(e)
				print os.path.realpath(os.path.curdir)
			try:
				self.mtps[mtp['protocol']] = mod.INSTANCE(name,config)
			except:
				print "Y MI INSTANCIA? " + str(name)
		os.chdir(curdir)

	def test(self,stanza):

            	children = stanza.getChildren()
		#self._router.DEBUG("ACC PLUGIN: analyzing message structure", "info")
            	for child in children:
                	if (child.getNamespace() == "jabber:x:fipa") or (child.getNamespace() == u"jabber:x:fipa"):
				self.envelope = child.getData()
				return True

		return False

		"""
		simple_to = str(stanza['to'])
		if not('@' in simple_to):  # Component name
			# SPADE CASE: message directed to the ACC component
			if simple_to[0:4] == "acc.":
				#self._router.DEBUG("Message for the ACC",'ok')
				return True
		#self._router.DEBUG("Message NOT for the ACC",'error')
		return False
		"""

	def filter(self,session,stanza):

		xmpp_processed = False

		if (stanza.getError() == None):
			envxml = self.envelope
			#payload = stanza.getBody()
			payload = stanza.getTag("body")
			xc = XMLCodec.XMLCodec()
			self.envelope = xc.parse(str(envxml))

			for aid in self.envelope.getTo():
				for addr in aid.getAddresses():
					protocol = addr.split("://")[0]

					if protocol == "xmpp":
						#FIXME: only supports 1 xmpp sender
						receivers = self.getRealTo(stanza)
						to = str(receivers[0])  # FIX THIS TO ALLOW MULTIPLE RECEIVERS
						# We do NOT raise the NodeProcessed exception,
						# the message follows its normal course from here
						stanza.setTo(to)
						xmpp_processed = True

					elif protocol in self.mtps.keys():
						print ">>>ACC: Message for external MTP: " + str(protocol)
						mtp = self.mtps[protocol]
						print ">>>ACC: MTP found: " + str(mtp)
						#envelope is in Envelope format
						#payload is in string format (with escaped characters)
						mtp.send(self.envelope,payload)
						print ">>>ACC: External MTP message sent"

					else:
						print ">>>ACC: Could not find suitable MTP for protocol: " + str(protocol)

		if xmpp_processed: return stanza
		else: return None

	def getRealTo(self, mess):
		#return the real JID of the receiver of a "jabber:x:fipa" message

		receiver_names = []
		for aid in self.envelope.getTo():
			receiver_names.append(str(aid.getName()))

		return receiver_names

<|endoftext|>


<commit_before>import os
import csv
import requests
import pandas as pd
import numpy as np
from _decorators import *

def read_api_dict():
    """Creates a dictionary of Yahoo Finance API parameters from a csv file.

    Arguments:
        takes no arguments
    File specs:
        csv with columns 'parameter, description'
    Returns:
        a dictionary consisting of parameters and their descriptions
    """
    api_dict_csv_path = os.path.join(os.path.dirname(__file__), 'yahoo_api_dict.csv')
    with open(api_dict_csv_path) as api:
        reader = csv.DictReader(api)
        api_dict = {row['parameter']: row['description'] for row in reader}

    return api_dict

def get_ticker_list_from_file(tickers_csv_path):
    """Creates a list of tickers from a csv file listing tickers one per line.

    Arguments:
        tickers_csv_path --> string containing path to tickers csv file
    File specs:
        csv with column 'ticker'
    Returns:
        a list of ticker strings
    """
    with open(tickers_csv_path) as tickers_file:
        reader = csv.DictReader(tickers_file)
        return [row['ticker'] for row in reader]

def get_ticker_string_from_list(ticker_list):
    """Returns a string of tickers separated by commas joined from a list."""

    ticker_string = ','.join(ticker_list)
    return ticker_string

def get_param_list_from_api_dict(api_dict):
    """Returns the keys of the API dictionary."""
    return api_dict.keys()

def get_param_string_from_list(param_list):
    """Returns a string of parameters with no separators joined from a list."""
    return ''.join(param_list)

@timeit
def get_current_answer_string(ticker_string, param_string):
    """Queries Yahoo Finance API, returns a CSV string with the response.

    Arguments:
        ticker_string --> comma-separated string of tickers
        param_string --> comm-separated string of Yahoo API parameters
    Returns:
        response string, CSV formatted
    """
    base_url = 'http://finance.yahoo.com/d/quotes.csv'
    params = {'s': ticker_string, 'f': param_string}

    while True:
        try:
            response = requests.get(base_url, params)
        except requests.exceptions.ConnectionError:
            print 'Connection error, trying again...'
        else:
            answer_string = response.text
            return answer_string

def get_date_components(date_object):
    """Extracts components dates for a historical data request.

    Arguments:
        date_object --> string date, Python datetime, NumPy datetime64 or pandas Timestamp
    Returns:
        (m, d, y) --> month, day, year (integers)
    """
    # TODO implement datetime, datetime64 and Timestamp handling
    if type(date_object) == str:
        date_string = date_object
        y, m, d = [int(i) for i in date_string.split('-')]
        m -= 1  # Need to decrement, Yahoo has 0-indexing for months

        return m, d, y

class NoHistoricalDataError(Exception):
    def __init__(self, message):
        self.message = message

@timeit
def get_historical_answer_string(ticker, from_date=None, to_date=None):

    base_url = 'http://real-chart.finance.yahoo.com/table.csv'

    params = {'s': ticker}
    if from_date is not None:
        params['a'], params['b'], params['c'] = get_date_components(from_date)
    if to_date is not None:
        params['d'], params['e'], params['f'] = get_date_components(to_date)

    while True:
        try:
            response = requests.get(base_url, params)
            if response.status_code == 404:
                raise Exception('No historical data for ticker ' + params['s'])
        except requests.exceptions.ConnectionError:
            print 'Connection error, trying again...'
        except NoHistoricalDataError:
            return None
        else:
            answer_string = response.text
            return answer_string

def get_header_list(param_list, api_dict):
    """Creates a header for a CSV data file.

    Arguments:
        param_list --> list of Yahoo Finance API parameters formed by comma-splitting param_string
        api_dict --> dictionary with Yahoo Finance API parameter descriptions mapped to parameters
    Returns:
        list of parameter descriptions retrieved from the API dictionary.
    """
    return [api_dict[param] for param in param_list]

def get_answer_list_from_string(answer_string):
    """Creates a list from the response string.

    Arguments:
        answer_string --> text content of Yahoo Finance API request response
    Returns:
        list generated from the string by comma-splitting
    """
    csv_rows_list = answer_string.splitlines()
    reader = csv.reader(csv_rows_list)
    answer_list = list(reader)
    return answer_list

def save_raw_answer_to_file(answer_path, answer_string):
    """Saves the response string to a specified path.

    Arguments:
        answer_path --> path to target
        answer_string --> text content of Yahoo Finance API request response
    Returns:
        nothing
    """
    with open(answer_path, 'wb') as raw_data:
        raw_data.write(answer_string)
    print 'Raw response csv file saved to ' + answer_path

def construct_row(k, header_list, answer_list):
    """Creates a row of data for a CSV file (to be used with csv.DictWriter).

    Arguments:
        k --> position of observation sublist on the response list
        header_list --> list of parameter names except 'ticker'
        answer_list --> list formed by comma-splitting answer_string
    Returns:
        row of data in form of a dictionary
    """
    return {header_list[i]: answer_list[k][i] for i in xrange(len(header_list))}

def current_pd_dataframe(api_dict, answer_list, param_list, ticker_list):
    """Constructs a pandas DataFrame from a data dictionary and cleans it.

    Arguments:
        api_dict --> dictionary containing Yahoo Finance API parameters and their definitions
        answer_list --> list of lists containing rows for future DataFrame
        param_list --> list of Yahoo Finance API parameters that were used to request data
        ticker_list --> list of tickers for which data was requested

    Returns:
        pandas_dataframe --> configuration: rows are companies, columns are parameter definitions
                             wrangling:
                                        - 'N/A's are replaced with NumPy NaNs
                                        - all-NaN columns are dropped
                                        - where possible, columns are converted to numeric
    """

    dict_for_pandas = {api_dict[param_list[index]]: item for index, item in enumerate(zip(*answer_list))}

    pandas_dataframe = pd.DataFrame(dict_for_pandas)
    pandas_dataframe.index = ticker_list
    pandas_dataframe = pandas_dataframe.replace(to_replace='N/A', value=np.nan)

    for item in pandas_dataframe:
        if pandas_dataframe[item].count() == 0:
            del pandas_dataframe[item]

    for colname in pandas_dataframe:
        try:
            pandas_dataframe[colname] = pd.to_numeric(pandas_dataframe[colname])
        except ValueError:
            print colname, 'could not be converted.'

    # TODO Convert columns to datetimes if possible
    # TODO Parse values with M, B for million/billion

    return pandas_dataframe

def historical_pd_dataframe(answer_list):

    columns, data = answer_list[0], answer_list[1:]

    dict_for_pandas = {columns[index]: item for index, item in enumerate(zip(*data))}

    pandas_dataframe = pd.DataFrame(dict_for_pandas)
    pandas_dataframe.index = pd.to_datetime(pandas_dataframe['Date'])
    pandas_dataframe = pandas_dataframe[columns]
    del pandas_dataframe['Date']

    for colname in pandas_dataframe:
        try:
            pandas_dataframe[colname] = pd.to_numeric(pandas_dataframe[colname])
        except ValueError:
            print colname, 'could not be converted.'

    return pandas_dataframe

def save_formatted_csv(result_csv_path, header_list, ticker_list, answer_list):
    """DEPRECATED: use write_to_csv option instead.
    Writes the data from the response list to a CSV file using csv.DictWriter.

    Arguments:
        result_csv_path -- > path to target csv file (existing file will be overwritten)
        header_list --> list of parameter descriptions
        ticker_list --> list of tickers for which data is to be retrieved
        answer_list --> list formed by comma-splitting answer_string
    Returns:
        nothing
    """
    with open(result_csv_path, 'wb') as data:
        fieldnames = ['ticker'] + header_list
        writer = csv.DictWriter(data, fieldnames, quoting=csv.QUOTE_NONE, escapechar='\b')
        writer.writeheader()
        for k in range(len(ticker_list)):
            row = construct_row(k, header_list, answer_list)
            row['ticker'] = ticker_list[k]
            writer.writerow(row)
<commit_msg>error fix<commit_after>import os
import csv
import requests
import pandas as pd
import numpy as np
from _decorators import *

def read_api_dict():
    """Creates a dictionary of Yahoo Finance API parameters from a csv file.

    Arguments:
        takes no arguments
    File specs:
        csv with columns 'parameter, description'
    Returns:
        a dictionary consisting of parameters and their descriptions
    """
    api_dict_csv_path = os.path.join(os.path.dirname(__file__), 'yahoo_api_dict.csv')
    with open(api_dict_csv_path) as api:
        reader = csv.DictReader(api)
        api_dict = {row['parameter']: row['description'] for row in reader}

    return api_dict

def get_ticker_list_from_file(tickers_csv_path):
    """Creates a list of tickers from a csv file listing tickers one per line.

    Arguments:
        tickers_csv_path --> string containing path to tickers csv file
    File specs:
        csv with column 'ticker'
    Returns:
        a list of ticker strings
    """
    with open(tickers_csv_path) as tickers_file:
        reader = csv.DictReader(tickers_file)
        return [row['ticker'] for row in reader]

def get_ticker_string_from_list(ticker_list):
    """Returns a string of tickers separated by commas joined from a list."""

    ticker_string = ','.join(ticker_list)
    return ticker_string

def get_param_list_from_api_dict(api_dict):
    """Returns the keys of the API dictionary."""
    return api_dict.keys()

def get_param_string_from_list(param_list):
    """Returns a string of parameters with no separators joined from a list."""
    return ''.join(param_list)

@timeit
def get_current_answer_string(ticker_string, param_string):
    """Queries Yahoo Finance API, returns a CSV string with the response.

    Arguments:
        ticker_string --> comma-separated string of tickers
        param_string --> comm-separated string of Yahoo API parameters
    Returns:
        response string, CSV formatted
    """
    base_url = 'http://finance.yahoo.com/d/quotes.csv'
    params = {'s': ticker_string, 'f': param_string}

    while True:
        try:
            response = requests.get(base_url, params)
        except requests.exceptions.ConnectionError:
            print 'Connection error, trying again...'
        else:
            answer_string = response.text
            return answer_string

def get_date_components(date_object):
    """Extracts components dates for a historical data request.

    Arguments:
        date_object --> string date, Python datetime, NumPy datetime64 or pandas Timestamp
    Returns:
        (m, d, y) --> month, day, year (integers)
    """
    # TODO implement datetime, datetime64 and Timestamp handling
    if type(date_object) == str:
        date_string = date_object
        y, m, d = [int(i) for i in date_string.split('-')]
        m -= 1  # Need to decrement, Yahoo has 0-indexing for months

        return m, d, y

class NoHistoricalDataError(Exception):
    def __init__(self, message):
        self.message = message

@timeit
def get_historical_answer_string(ticker, from_date=None, to_date=None):

    base_url = 'http://real-chart.finance.yahoo.com/table.csv'

    params = {'s': ticker}
    if from_date is not None:
        params['a'], params['b'], params['c'] = get_date_components(from_date)
    if to_date is not None:
        params['d'], params['e'], params['f'] = get_date_components(to_date)

    while True:
        try:
            response = requests.get(base_url, params)
            if response.status_code == 404:
                raise NoHistoricalDataError('No historical data for ticker ' + params['s'])
        except requests.exceptions.ConnectionError:
            print 'Connection error, trying again...'
        except NoHistoricalDataError:
            return None
        else:
            answer_string = response.text
            return answer_string

def get_header_list(param_list, api_dict):
    """Creates a header for a CSV data file.

    Arguments:
        param_list --> list of Yahoo Finance API parameters formed by comma-splitting param_string
        api_dict --> dictionary with Yahoo Finance API parameter descriptions mapped to parameters
    Returns:
        list of parameter descriptions retrieved from the API dictionary.
    """
    return [api_dict[param] for param in param_list]

def get_answer_list_from_string(answer_string):
    """Creates a list from the response string.

    Arguments:
        answer_string --> text content of Yahoo Finance API request response
    Returns:
        list generated from the string by comma-splitting
    """
    csv_rows_list = answer_string.splitlines()
    reader = csv.reader(csv_rows_list)
    answer_list = list(reader)
    return answer_list

def save_raw_answer_to_file(answer_path, answer_string):
    """Saves the response string to a specified path.

    Arguments:
        answer_path --> path to target
        answer_string --> text content of Yahoo Finance API request response
    Returns:
        nothing
    """
    with open(answer_path, 'wb') as raw_data:
        raw_data.write(answer_string)
    print 'Raw response csv file saved to ' + answer_path

def construct_row(k, header_list, answer_list):
    """Creates a row of data for a CSV file (to be used with csv.DictWriter).

    Arguments:
        k --> position of observation sublist on the response list
        header_list --> list of parameter names except 'ticker'
        answer_list --> list formed by comma-splitting answer_string
    Returns:
        row of data in form of a dictionary
    """
    return {header_list[i]: answer_list[k][i] for i in xrange(len(header_list))}

def current_pd_dataframe(api_dict, answer_list, param_list, ticker_list):
    """Constructs a pandas DataFrame from a data dictionary and cleans it.

    Arguments:
        api_dict --> dictionary containing Yahoo Finance API parameters and their definitions
        answer_list --> list of lists containing rows for future DataFrame
        param_list --> list of Yahoo Finance API parameters that were used to request data
        ticker_list --> list of tickers for which data was requested

    Returns:
        pandas_dataframe --> configuration: rows are companies, columns are parameter definitions
                             wrangling:
                                        - 'N/A's are replaced with NumPy NaNs
                                        - all-NaN columns are dropped
                                        - where possible, columns are converted to numeric
    """

    dict_for_pandas = {api_dict[param_list[index]]: item for index, item in enumerate(zip(*answer_list))}

    pandas_dataframe = pd.DataFrame(dict_for_pandas)
    pandas_dataframe.index = ticker_list
    pandas_dataframe = pandas_dataframe.replace(to_replace='N/A', value=np.nan)

    for item in pandas_dataframe:
        if pandas_dataframe[item].count() == 0:
            del pandas_dataframe[item]

    for colname in pandas_dataframe:
        try:
            pandas_dataframe[colname] = pd.to_numeric(pandas_dataframe[colname])
        except ValueError:
            print colname, 'could not be converted.'

    # TODO Convert columns to datetimes if possible
    # TODO Parse values with M, B for million/billion

    return pandas_dataframe

def historical_pd_dataframe(answer_list):

    columns, data = answer_list[0], answer_list[1:]

    dict_for_pandas = {columns[index]: item for index, item in enumerate(zip(*data))}

    pandas_dataframe = pd.DataFrame(dict_for_pandas)
    pandas_dataframe.index = pd.to_datetime(pandas_dataframe['Date'])
    pandas_dataframe = pandas_dataframe[columns]
    del pandas_dataframe['Date']

    for colname in pandas_dataframe:
        try:
            pandas_dataframe[colname] = pd.to_numeric(pandas_dataframe[colname])
        except ValueError:
            print colname, 'could not be converted.'

    return pandas_dataframe

def save_formatted_csv(result_csv_path, header_list, ticker_list, answer_list):
    """DEPRECATED: use write_to_csv option instead.
    Writes the data from the response list to a CSV file using csv.DictWriter.

    Arguments:
        result_csv_path -- > path to target csv file (existing file will be overwritten)
        header_list --> list of parameter descriptions
        ticker_list --> list of tickers for which data is to be retrieved
        answer_list --> list formed by comma-splitting answer_string
    Returns:
        nothing
    """
    with open(result_csv_path, 'wb') as data:
        fieldnames = ['ticker'] + header_list
        writer = csv.DictWriter(data, fieldnames, quoting=csv.QUOTE_NONE, escapechar='\b')
        writer.writeheader()
        for k in range(len(ticker_list)):
            row = construct_row(k, header_list, answer_list)
            row['ticker'] = ticker_list[k]
            writer.writerow(row)
<|endoftext|>