<commit_before>
import clusto
from clusto.schema import select, and_, ATTR_TABLE, Attribute, func, Counter
from clusto.drivers.base import Driver, ClustoMeta
from clusto.exceptions import ResourceTypeException, ResourceNotAvailableException, ResourceException

class ResourceManager(Driver):
    """The ResourceManager driver should be subclassed by a driver that will
    manage a resource such as IP allocation, MAC Address lists, etc.

    This base class just allocates unique integers.

    Resources are attributes on Entities that are managed by a ResourceManger.
    The implementation has the following properties:

    1. The Entity being assigned the resource gets an attribute who's key is
    defined by the resource manager, a number assigned by the resource manager
    (sometimes related to the resource being allocated), and a value which is
    a representation of the resource.

    2. The Entity gets an additional attribute who's key, and number match the
    allocated resource, but with subkey='manager', and value that is a
    reference to the resource manager assigning the resource.

    Any additional attributes with same attribute key and number are
    considered part of the resource and can be managed by the resource
    manager.
    """

    _clusto_type = "resourcemanager"
    _driver_name = "resourcemanager"

    _attr_name = "resource"
    _record_allocations = True

    def allocator(self, thing=None):
        """return an unused resource from this resource manager"""

        raise NotImplemented("No allocator implemented for %s you must explicitly specify a resource."
                             % self.name)

    def ensure_type(self, resource, number=True, thing=None):
        """checks the type of a given resourece

        if the resource is valid return it and optionally convert it to
        another format.  The format it returns has to be compatible with
        attribute naming
        """
        return (resource, number)

    def get_resource_number(self, thing, resource):
        """Get the number for a resource on a given entity."""

        resource, number = self.ensure_type(resource, thing=thing)

        attrs = thing.attrs(self._attr_name, value=resource)

        if attrs:
            return attrs[0].number
        else:
            raise ResourceException("%s isn't assigned resource %s"
                                    % (thing.name, str(resource)))

    def get_resource_attr_values(self, thing, resource, key, number=True):
        """Get the value for the attrs on the resource assigned to a given entity matching the given key."""

        return [x.value for x in self.get_resource_attrs(thing, resource,
                                                         key, number)]

    def get_resource_attrs(self, thing, resource, key=(), number=True):
        """Get the Attributes for the attrs on the resource assigned to a given enttiy matching the given key."""

        resource, number = self.ensure_type(resource, number, thing=thing)

        return thing.attrs(self._attr_name, number=number, subkey=key)

    def add_resource_attr(self, thing, resource, key, value, number=True):
        """Add an Attribute for the resource assigned to a given entity setting the given key and value"""

        resource, number = self.ensure_type(resource, number, thing=thing)

        attr = thing.add_attr(self._attr_name, number=number, subkey=key, value=value)
        return attr

    def set_resource_attr(self, thing, resource, key, value, number=True):
        """Set an Attribute for the resource assigned to a given entity with the given key and value"""

        resource, number = self.ensure_type(resource, number, thing=thing)
        attr = thing.set_attr(self._attr_name, number=number, subkey=key, value=value)

        return attr

    def del_resource_attr(self, thing, resource, key, value=(), number=True):
        """Delete an Attribute for the resource assigned to a given entity matching the given key and value"""

        resource, number = self.ensure_type(resource, number, thing=thing)
        thing.del_attrs(self._attr_name, number=number, subkey=key, value=value)

    def additional_attrs(self, thing, resource, number):
        pass

    def post_automatic_allocation(self, thing, resource, number):
        pass

    def post_allocation(self, thing, resource, number):
        pass

    def allocate(self, thing, resource=(), number=True, force=False):
        """allocates a resource element to the given thing.

        resource - is passed as an argument it will be checked
                   before assignment.

        refattr - the attribute name on the entity that will refer back
                  this resource manager.

        returns the resource that was either passed in and processed
        or generated.
        """

        try:
            clusto.begin_transaction()
            if not isinstance(thing, Driver):
                raise TypeError("thing is not of type Driver")

            if resource is ():
                # allocate a new resource
                resource, number = self.allocator(thing)
                auto_allocated = True
            else:
                auto_allocated = False
                resource, number = self.ensure_type(resource, number, thing)
                if not force and not self.available(resource, number, thing):
                    raise ResourceException("Requested resource is not available.")

            if self._record_allocations:
                if number == True:
                    c = Counter.get(ClustoMeta().entity, self._attr_name)
                    attr = thing.add_attr(self._attr_name,
                                          resource,
                                          number=c.value
                                          )
                    c.next()
                else:
                    attr = thing.add_attr(self._attr_name, resource, number=number)

                clusto.flush()

                a=thing.add_attr(self._attr_name,
                            self.entity,
                            number=attr.number,
                            subkey='manager',
                            )

                clusto.flush()
                self.additional_attrs(thing, resource, attr.number)

            else:
                attr = None
            clusto.commit()
        except Exception, x:
            clusto.rollback_transaction()
            raise x

        if auto_allocated:
            self.post_automatic_allocation(thing, resource, number)

        self.post_allocation(thing, resource, number)

        return attr #resource

    def deallocate(self, thing, resource=(), number=True):
        """deallocates a resource from the given thing."""

        clusto.begin_transaction()
        try:
            if resource is ():
                for res in self.resources(thing):
                    thing.del_attrs(self._attr_name, number=res.number)

            elif resource and not self.available(resource, number):
                resource, number = self.ensure_type(resource, number)

                res = thing.attrs(self._attr_name, self, subkey='manager', number=number)
                for a in res:
                    thing.del_attrs(self._attr_name, number=a.number)

            clusto.commit()
        except Exception, x:
            clusto.rollback_transaction()
            raise x

    def available(self, resource, number=True, thing=None):
        """return True if resource is available, False otherwise.
        """

        resource, number = self.ensure_type(resource, number)

        if self.owners(resource, number):
            return False

        return True

    def owners(self, resource, number=True):
        """return a list of driver objects for the owners of a given resource.
        """

        resource, number = self.ensure_type(resource, number)

        return Driver.get_by_attr(self._attr_name, resource, number=number)

    @classmethod
    def get_resource_manager(cls, resource_attr):
        """Return the resource manager for a given resource_attr"""

        thing = Driver(resource_attr.entity)

        return thing.attr_value(key=resource_attr.key,
                                subkey='manager',
                                number=resource_attr.number)

    @classmethod
    def resources(cls, thing):
        """return a list of resources from the resource manager that is
        associated with the given thing.

        A resource is a resource attribute in a resource manager.
        """

        attrs = [x for x in thing.attrs(cls._attr_name, subkey='manager')
                 if isinstance(Driver(x.value), cls)]

        res = []

        for attr in attrs:
            t=thing.attrs(cls._attr_name, number=attr.number, subkey=None)
            res.extend(t)

        return res

    @property
    def count(self):
        """Return the number of resources used."""

        return len(self.references(self._attr_name, self, subkey='manager'))

<commit_msg>Don't re-raise exceptions; allow existing exception to propagate up so that the caller can more easily debug.<commit_after>
import clusto
from clusto.schema import select, and_, ATTR_TABLE, Attribute, func, Counter
from clusto.drivers.base import Driver, ClustoMeta
from clusto.exceptions import ResourceTypeException, ResourceNotAvailableException, ResourceException

class ResourceManager(Driver):
    """The ResourceManager driver should be subclassed by a driver that will
    manage a resource such as IP allocation, MAC Address lists, etc.

    This base class just allocates unique integers.

    Resources are attributes on Entities that are managed by a ResourceManger.
    The implementation has the following properties:

    1. The Entity being assigned the resource gets an attribute who's key is
    defined by the resource manager, a number assigned by the resource manager
    (sometimes related to the resource being allocated), and a value which is
    a representation of the resource.

    2. The Entity gets an additional attribute who's key, and number match the
    allocated resource, but with subkey='manager', and value that is a
    reference to the resource manager assigning the resource.

    Any additional attributes with same attribute key and number are
    considered part of the resource and can be managed by the resource
    manager.
    """

    _clusto_type = "resourcemanager"
    _driver_name = "resourcemanager"

    _attr_name = "resource"
    _record_allocations = True

    def allocator(self, thing=None):
        """return an unused resource from this resource manager"""

        raise NotImplemented("No allocator implemented for %s you must explicitly specify a resource."
                             % self.name)

    def ensure_type(self, resource, number=True, thing=None):
        """checks the type of a given resourece

        if the resource is valid return it and optionally convert it to
        another format.  The format it returns has to be compatible with
        attribute naming
        """
        return (resource, number)

    def get_resource_number(self, thing, resource):
        """Get the number for a resource on a given entity."""

        resource, number = self.ensure_type(resource, thing=thing)

        attrs = thing.attrs(self._attr_name, value=resource)

        if attrs:
            return attrs[0].number
        else:
            raise ResourceException("%s isn't assigned resource %s"
                                    % (thing.name, str(resource)))

    def get_resource_attr_values(self, thing, resource, key, number=True):
        """Get the value for the attrs on the resource assigned to a given entity matching the given key."""

        return [x.value for x in self.get_resource_attrs(thing, resource,
                                                         key, number)]

    def get_resource_attrs(self, thing, resource, key=(), number=True):
        """Get the Attributes for the attrs on the resource assigned to a given enttiy matching the given key."""

        resource, number = self.ensure_type(resource, number, thing=thing)

        return thing.attrs(self._attr_name, number=number, subkey=key)

    def add_resource_attr(self, thing, resource, key, value, number=True):
        """Add an Attribute for the resource assigned to a given entity setting the given key and value"""

        resource, number = self.ensure_type(resource, number, thing=thing)

        attr = thing.add_attr(self._attr_name, number=number, subkey=key, value=value)
        return attr

    def set_resource_attr(self, thing, resource, key, value, number=True):
        """Set an Attribute for the resource assigned to a given entity with the given key and value"""

        resource, number = self.ensure_type(resource, number, thing=thing)
        attr = thing.set_attr(self._attr_name, number=number, subkey=key, value=value)

        return attr

    def del_resource_attr(self, thing, resource, key, value=(), number=True):
        """Delete an Attribute for the resource assigned to a given entity matching the given key and value"""

        resource, number = self.ensure_type(resource, number, thing=thing)
        thing.del_attrs(self._attr_name, number=number, subkey=key, value=value)

    def additional_attrs(self, thing, resource, number):
        pass

    def post_automatic_allocation(self, thing, resource, number):
        pass

    def post_allocation(self, thing, resource, number):
        pass

    def allocate(self, thing, resource=(), number=True, force=False):
        """allocates a resource element to the given thing.

        resource - is passed as an argument it will be checked
                   before assignment.

        refattr - the attribute name on the entity that will refer back
                  this resource manager.

        returns the resource that was either passed in and processed
        or generated.
        """

        try:
            clusto.begin_transaction()
            if not isinstance(thing, Driver):
                raise TypeError("thing is not of type Driver")

            if resource is ():
                # allocate a new resource
                resource, number = self.allocator(thing)
                auto_allocated = True
            else:
                auto_allocated = False
                resource, number = self.ensure_type(resource, number, thing)
                if not force and not self.available(resource, number, thing):
                    raise ResourceException("Requested resource is not available.")

            if self._record_allocations:
                if number == True:
                    c = Counter.get(ClustoMeta().entity, self._attr_name)
                    attr = thing.add_attr(self._attr_name,
                                          resource,
                                          number=c.value
                                          )
                    c.next()
                else:
                    attr = thing.add_attr(self._attr_name, resource, number=number)

                clusto.flush()

                a=thing.add_attr(self._attr_name,
                            self.entity,
                            number=attr.number,
                            subkey='manager',
                            )

                clusto.flush()
                self.additional_attrs(thing, resource, attr.number)

            else:
                attr = None
            clusto.commit()
        except Exception, x:
            clusto.rollback_transaction()
            raise

        if auto_allocated:
            self.post_automatic_allocation(thing, resource, number)

        self.post_allocation(thing, resource, number)

        return attr #resource

    def deallocate(self, thing, resource=(), number=True):
        """deallocates a resource from the given thing."""

        clusto.begin_transaction()
        try:
            if resource is ():
                for res in self.resources(thing):
                    thing.del_attrs(self._attr_name, number=res.number)

            elif resource and not self.available(resource, number):
                resource, number = self.ensure_type(resource, number)

                res = thing.attrs(self._attr_name, self, subkey='manager', number=number)
                for a in res:
                    thing.del_attrs(self._attr_name, number=a.number)

            clusto.commit()
        except Exception, x:
            clusto.rollback_transaction()
            raise

    def available(self, resource, number=True, thing=None):
        """return True if resource is available, False otherwise.
        """

        resource, number = self.ensure_type(resource, number)

        if self.owners(resource, number):
            return False

        return True

    def owners(self, resource, number=True):
        """return a list of driver objects for the owners of a given resource.
        """

        resource, number = self.ensure_type(resource, number)

        return Driver.get_by_attr(self._attr_name, resource, number=number)

    @classmethod
    def get_resource_manager(cls, resource_attr):
        """Return the resource manager for a given resource_attr"""

        thing = Driver(resource_attr.entity)

        return thing.attr_value(key=resource_attr.key,
                                subkey='manager',
                                number=resource_attr.number)

    @classmethod
    def resources(cls, thing):
        """return a list of resources from the resource manager that is
        associated with the given thing.

        A resource is a resource attribute in a resource manager.
        """

        attrs = [x for x in thing.attrs(cls._attr_name, subkey='manager')
                 if isinstance(Driver(x.value), cls)]

        res = []

        for attr in attrs:
            t=thing.attrs(cls._attr_name, number=attr.number, subkey=None)
            res.extend(t)

        return res

    @property
    def count(self):
        """Return the number of resources used."""

        return len(self.references(self._attr_name, self, subkey='manager'))

<|endoftext|>


<commit_before># Copyright (c) 2012 Spotify AB
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

import unittest
import luigi
import luigi.lock
import luigi.notifications
import tempfile
import os
import hashlib
import subprocess

luigi.notifications.DEBUG = True

class TestCmd(unittest.TestCase):
    def test_getpcmd(self):
        p = subprocess.Popen(["sleep", "1"])
        self.assertEquals(
            luigi.lock.getpcmd(p.pid),
            "sleep 1"
        )
        p.kill()

class LockTest(unittest.TestCase):

    def setUp(self):
        self.pid_dir = tempfile.mkdtemp()

    def tearDown(self):
        my_pid = os.getpid()
        my_cmd = luigi.lock.getpcmd(my_pid)
        pidfile = os.path.join(self.pid_dir, hashlib.md5(my_cmd).hexdigest()) + '.pid'

        os.remove(pidfile)
        os.rmdir(self.pid_dir)

    def test_acquiring_free_lock(self):
        acquired = luigi.lock.acquire_for(self.pid_dir)
        self.assertTrue(acquired)

    def test_acquiring_taken_lock(self):
        my_pid = os.getpid()
        my_cmd = luigi.lock.getpcmd(my_pid)

        pidfile = os.path.join(self.pid_dir, hashlib.md5(my_cmd).hexdigest()) + '.pid'

        f = open(pidfile, 'w')
        f.write('%d\n' % (my_pid, ))
        f.close()

        acquired = luigi.lock.acquire_for(self.pid_dir)
        self.assertFalse(acquired)
<commit_msg>Adds tests for multiple lock acquisitions<commit_after># Copyright (c) 2012 Spotify AB
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

import unittest
import luigi
import luigi.lock
import luigi.notifications
import tempfile
import os
import hashlib
import subprocess

luigi.notifications.DEBUG = True

class TestCmd(unittest.TestCase):
    def test_getpcmd(self):
        p = subprocess.Popen(["sleep", "1"])
        self.assertEquals(
            luigi.lock.getpcmd(p.pid),
            "sleep 1"
        )
        p.kill()

class LockTest(unittest.TestCase):

    def setUp(self):
        self.pid_dir = tempfile.mkdtemp()
        self.pid = os.getpid()
        self.cmd = luigi.lock.getpcmd(self.pid)
        self.pidfile = os.path.join(self.pid_dir, hashlib.md5(self.cmd).hexdigest()) + '.pid'

    def tearDown(self):

        os.remove(self.pidfile)
        os.rmdir(self.pid_dir)

    def test_acquiring_free_lock(self):
        acquired = luigi.lock.acquire_for(self.pid_dir)
        self.assertTrue(acquired)

    def test_acquiring_taken_lock(self):
        with open(self.pidfile, 'w') as f:
            f.write('%d\n' % (self.pid, ))

        acquired = luigi.lock.acquire_for(self.pid_dir)
        self.assertFalse(acquired)

    def test_acquiring_partially_taken_lock(self):
        with open(self.pidfile, 'w') as f:
            f.write('%d\n' % (self.pid, ))

        acquired = luigi.lock.acquire_for(self.pid_dir, 2)
        self.assertTrue(acquired)

    def test_acquiring_lock_from_missing_process(self):
        with open(self.pidfile, 'w') as f:
            f.write('%d\n' % (self.pid + 1, ))

        acquired = luigi.lock.acquire_for(self.pid_dir)
        self.assertTrue(acquired)
<|endoftext|>


<commit_before>
import sys
from PyQt5.QtWidgets import QMainWindow, QWidget, QApplication, QHBoxLayout, QVBoxLayout, QLabel, QGroupBox,\
     QFormLayout, QLineEdit
from PyQt5.QtGui import QIntValidator
from PyQt5.QtCore import Qt

class Receiving(QMainWindow):

    def __init__(self):
        super().__init__()

        self.initUI()

    def initUI(self):

        centralWidget = QWidget()
        self.setCentralWidget(centralWidget)

        self.createTopLabels()
        self.createMainInputs()
        self.createMainTable()

        mainLayout = QVBoxLayout()
        mainLayout.addWidget(self.horizontalLabels)
        mainLayout.addLayout(self.inputLayout)

        centralWidget.setLayout(mainLayout)

        self.setWindowTitle('Vendor Receiving')
        self.show()

    def createTopLabels(self):

        self.horizontalLabels = QGroupBox('Vendor Receiving Entry')
        self.horizontalLabels.setAlignment(Qt.AlignHCenter)
        layout = QHBoxLayout()

        idLabel = QLabel('Clerk ID:[           ]')
        passLabel = QLabel('Password:[********* ]')
        nameLabel = QLabel('Name: <Michie, Jon T.   >')

        layout.addWidget(idLabel)
        layout.addWidget(passLabel)
        layout.addWidget(nameLabel)

        self.horizontalLabels.setLayout(layout)

    def createMainInputs(self):

        self.inputLayout = QHBoxLayout()
        leftForm = QFormLayout()
        rightForm = QFormLayout()

        intValidator = QIntValidator()

        poLine = QLineEdit()
        poLine.setValidator(intValidator)
        vendorLine = QLineEdit()
        vendorNameLabel = QLabel('<Example Vendor Name>')
        addrLabel = QLabel('<Example Address>')
        ctstLabel = QLabel('<Example City/State>')
        zipLabel = QLabel('<Example Zip Code>')

        leftForm.addRow('&P.O.# ', poLine)
        leftForm.addRow('Vendor ', vendorLine)
        leftForm.addRow('Name ', vendorNameLabel)
        leftForm.addRow('Address ', addrLabel)
        leftForm.addRow('City, St ', ctstLabel)
        leftForm.addRow('Zip ', zipLabel)

        bolLine = QLineEdit()
        pacSlipLine = QLineEdit()
        totUnitsLine = QLineEdit()
        totUnitsLine.setValidator(intValidator)
        psUnitsLine = QLineEdit()
        psUnitsLine.setValidator(intValidator)
        depLabel = QLabel('<D1>')
        coordNumLine = QLineEdit()
        coordNumLine.setValidator(intValidator)

        rightForm.addRow('Bill of Landing ', bolLine)
        rightForm.addRow('Packing Slip ', pacSlipLine)
        rightForm.addRow('Total Units ', totUnitsLine)
        rightForm.addRow('PS Units ', psUnitsLine)
        rightForm.addRow('Department ', depLabel)
        rightForm.addRow('Receiving Coordinator ', coordNumLine)

        self.inputLayout.addLayout(leftForm)
        self.inputLayout.addLayout(rightForm)

    def createMainTable(self):

        pass

if __name__ == '__main__':

    app = QApplication(sys.argv)
    receiver = Receiving()
    sys.exit(app.exec_())
<commit_msg>Added main table<commit_after>
import sys
from PyQt5.QtWidgets import QMainWindow, QWidget, QApplication, QHBoxLayout, QVBoxLayout, QLabel, QGroupBox,\
     QFormLayout, QLineEdit, QTableWidget, QTableWidgetItem
from PyQt5.QtGui import QIntValidator, QFont
from PyQt5.QtCore import Qt

class Receiving(QMainWindow):

    def __init__(self):
        super().__init__()

        self.initUI()

    def initUI(self):

        centralWidget = QWidget()
        self.setCentralWidget(centralWidget)

        self.createTopLabels()
        self.createMainInputs()
        self.createMainTable()

        mainLayout = QVBoxLayout()
        mainLayout.addWidget(self.horizontalLabels)
        mainLayout.addLayout(self.inputLayout)
        mainLayout.addWidget(self.mainTable)

        centralWidget.setLayout(mainLayout)

        self.setWindowTitle('Vendor Receiving')
        self.show()

    def createTopLabels(self):

        self.horizontalLabels = QGroupBox('Vendor Receiving Entry')
        self.horizontalLabels.setAlignment(Qt.AlignHCenter)
        layout = QHBoxLayout()

        idLabel = QLabel('Clerk ID:[           ]')
        passLabel = QLabel('Password:[********* ]')
        nameLabel = QLabel('Name: <Michie, Jon T.   >')

        layout.addWidget(idLabel)
        layout.addWidget(passLabel)
        layout.addWidget(nameLabel)

        self.horizontalLabels.setLayout(layout)

    def createMainInputs(self):

        self.inputLayout = QHBoxLayout()
        leftForm = QFormLayout()
        rightForm = QFormLayout()

        intValidator = QIntValidator()

        poLine = QLineEdit()
        poLine.setValidator(intValidator)
        vendorLine = QLineEdit()
        vendorNameLabel = QLabel('<Example Vendor Name>')
        addrLabel = QLabel('<Example Address>')
        ctstLabel = QLabel('<Example City/State>')
        zipLabel = QLabel('<Example Zip Code>')

        leftForm.addRow('&P.O.# ', poLine)
        leftForm.addRow('Vendor ', vendorLine)
        leftForm.addRow('Name ', vendorNameLabel)
        leftForm.addRow('Address ', addrLabel)
        leftForm.addRow('City, St ', ctstLabel)
        leftForm.addRow('Zip ', zipLabel)

        bolLine = QLineEdit()
        pacSlipLine = QLineEdit()
        totUnitsLine = QLineEdit()
        totUnitsLine.setValidator(intValidator)

        psUnitsLabel = QLabel('PS &Units')
        psUnitsLine = QLineEdit()
        psUnitsLine.setValidator(intValidator)
        psUnitsLabel.setBuddy(psUnitsLine)
        totUnitsLabel = QLabel('Total <   0>')

        psHorizontalLayout = QHBoxLayout()
        psHorizontalLayout.addWidget(psUnitsLabel)
        psHorizontalLayout.addWidget(psUnitsLine)
        psHorizontalLayout.addWidget(totUnitsLabel)

        storeLocation = QLabel('Location < 22>')
        depLabel = QLabel('Department <D1>')

        locHorizontalLayout = QHBoxLayout()
        locHorizontalLayout.addWidget(storeLocation)
        locHorizontalLayout.addWidget(depLabel)

        coordNumLine = QLineEdit()
        coordNumLine.setValidator(intValidator)

        rightForm.addRow('&Bill of Landing ', bolLine)
        rightForm.addRow('Packing &Slip ', pacSlipLine)
        rightForm.addRow('&Total Units Received', totUnitsLine)
        rightForm.addRow(psHorizontalLayout)
        rightForm.addRow(locHorizontalLayout)
        rightForm.addRow('Receiving &Coordinator ', coordNumLine)

        self.inputLayout.addLayout(leftForm)
        self.inputLayout.addLayout(rightForm)

    def setupTable(self, qtable):

        columns = (2, 5)
        rows = qtable.rowCount()

        for column in columns:
            for row in range(rows):

                qtable.setRowHeight(row, 20)
                placeholderCell = QTableWidgetItem('###')
                placeholderCell.setFlags(placeholderCell.flags() ^ Qt.ItemIsEditable)
                qtable.setItem(row, column, placeholderCell)

        headers = ['UPC', 'PLU', 'DESCRIPTION', 'UNITS RECEIVED', 'PS UNITS', 'VENDOR MODEL']
        qtable.setHorizontalHeaderLabels(headers)

        font = qtable.font()
        font.setPointSize(20)

        qtable.setColumnWidth(0, 100)
        qtable.setShowGrid(False)

    def createMainTable(self):

        self.mainTable = QTableWidget(50, 6)
        self.setupTable(self.mainTable)

if __name__ == '__main__':

    app = QApplication(sys.argv)
    receiver = Receiving()
    sys.exit(app.exec_())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
redi.core
~~~~~~~~~

This module is the core of redi. Simple, eh?

"""

from models import *
from .config import init, redis

def value(key, r=redis):
    return RedisValue(key, r=r)

def list(key, r=redis):
    return RedisList(key, r=r)<commit_msg>document all the things<commit_after># -*- coding: utf-8 -*-

"""
redi.core
~~~~~~~~~

This module is the core of redi. Simple, eh?

"""

from models import *
from .config import init, redis

def value(key, r=redis):
    """Return RedisValue instance for given key.
    Optional `r` keyword argument sets Redis instance.
    """

    return RedisValue(key, r=r)

def list(key, r=redis):
    """Return RedisList instance for given key.
    Optional `r` keyword argument sets Redis instance.
    """
    return RedisList(key, r=r)<|endoftext|>


<commit_before>from djangorestframework.renderers import (BaseRenderer,
     JSONRenderer, JSONPRenderer, XMLRenderer)
from djangorestframework import status
from pystache.renderer import Renderer as Mustache
from django.conf import settings
from wq.db.util import get_config, get_ct, get_id, user_dict

dirs = []
for d in settings.TEMPLATE_DIRS:
    dirs.append(d)
    dirs.append(d + '/partials')

class RedirectRenderer(BaseRenderer):
    media_type = 'text/html'
    def render(self, obj=None, accept=None):
        path = self.view.request.path
        query = self.view.request.GET.urlencode()
        if query:
            path += "?" + query
        self.view.response.status = status.HTTP_302_FOUND
        self.view.response.headers['Location'] = '/#' + path
        return 'Redirecting...'

class MustacheRenderer(BaseRenderer):
    media_type = 'text/html'
    file_extension = 'html'
    def render(self, obj=None, accept=None):
        user   = self.view.request.user
        config = get_config(user)
        template = self.view.template
        if user.is_authenticated() and obj:
            obj['user'] = user_dict(user)
        if 'list' in obj and 'pages' in obj:
            obj['multiple'] = (obj['pages'] > 1)

        if not hasattr(self, '_mustache'):
            self._mustache = Mustache(
                file_extension = self.file_extension,
                search_dirs    = dirs
            )

        template = self._mustache.load_template(self.view.template)
        return self._mustache.render(template, obj)

class AMDRenderer(JSONPRenderer):
    media_type = 'application/javascript'
    format = 'js'
    callback_parameter = 'define'

if getattr(settings, 'RENDER_ON_SERVER', False):
    HTMLRenderer = MustacheRenderer
else:
    HTMLRenderer = RedirectRenderer
<commit_msg>utilize existing redirect path<commit_after>from djangorestframework.renderers import (BaseRenderer,
     JSONRenderer, JSONPRenderer, XMLRenderer)
from djangorestframework import status
from pystache.renderer import Renderer as Mustache
from django.conf import settings
from wq.db.util import get_config, get_ct, get_id, user_dict

dirs = []
for d in settings.TEMPLATE_DIRS:
    dirs.append(d)
    dirs.append(d + '/partials')

class RedirectRenderer(BaseRenderer):
    media_type = 'text/html'
    def render(self, obj=None, accept=None):
        path = self.view.response.headers.get('Location', self.view.request.path)
        query = self.view.request.GET.urlencode()
        if query:
            path += "?" + query
        self.view.response.status = status.HTTP_302_FOUND
        self.view.response.headers['Location'] = '/#' + path
        return 'Redirecting...'

class MustacheRenderer(BaseRenderer):
    media_type = 'text/html'
    file_extension = 'html'
    def render(self, obj=None, accept=None):
        user   = self.view.request.user
        config = get_config(user)
        template = self.view.template
        if user.is_authenticated() and obj:
            obj['user'] = user_dict(user)
        if 'list' in obj and 'pages' in obj:
            obj['multiple'] = (obj['pages'] > 1)

        if not hasattr(self, '_mustache'):
            self._mustache = Mustache(
                file_extension = self.file_extension,
                search_dirs    = dirs
            )

        template = self._mustache.load_template(self.view.template)
        return self._mustache.render(template, obj)

class AMDRenderer(JSONPRenderer):
    media_type = 'application/javascript'
    format = 'js'
    callback_parameter = 'define'

if getattr(settings, 'RENDER_ON_SERVER', False):
    HTMLRenderer = MustacheRenderer
else:
    HTMLRenderer = RedirectRenderer
<|endoftext|>


<commit_before>#!/usr/bin/python2.4
#
# Copyright (c) 2004-2005 rPath, Inc.
#

from time import sleep
import testsuite
testsuite.setup()

from mint_rephelp import MintRepositoryHelper
import recipes

from conary.conarycfg import ConaryConfiguration
from conary.conaryclient import ConaryClient
from conary import repository
from conary import versions

testRecipe = """
class TestCase(PackageRecipe):
    name = "testcase"
    version = "1.0"

    def setup(r):
        r.Create("/temp/foo")
"""

testGroup = """
class GroupTest(GroupRecipe):
    name = "group-test"
    version = "1.0"

    def setup(r):
        r.add('testcase')
"""

class RepositoryTest(MintRepositoryHelper):
    def testCommitStats(self):
        client, userId = self.quickMintUser("testuser", "testpass")
        projectId = self.newProject(client)

        client.server.registerCommit('test.rpath.local', 'testuser', 'mytrove:source', '/test.rpath.local@rpl:devel/1.0-1')
        project = client.getProject(projectId)
        assert(project.getCommits() == [('mytrove:source', '1.0-1')])

        # using a bogus username should not fail
        client.server.registerCommit('test.rpath.local', 'nonexistentuser', 'mytrove:source', '/test.rpath.local@rpl:devel/1.0-1')

    def testBasicRepository(self):
        client, userId = self.quickMintUser("testuser", "testpass")
        projectId = self.newProject(client)

        self.makeSourceTrove("testcase", testRecipe)
        project = client.getProject(projectId)

        cfg = project.getConaryConfig()
        nc = ConaryClient(cfg).getRepos()

        # test that the source trove landed properly
        troveNames = nc.troveNames(versions.Label("test.rpath.local@rpl:devel"))
        assert(troveNames == ["testcase:source"])

        # test that the commits table was updated
        # give some time for the commit action to run
        iters = 0
        while True:
            sleep(0.1)
            iters += 1
            if project.getCommits() != []:
                break
            if iters > 50:
                self.fail("commits didn't show up")

        assert(project.getCommits() == [('testcase:source', '1.0-1')])

    def testHooksResponse(self):
        cfg = ConaryConfiguration()
        cfg.installLabelPath = ['notfound.rpath.local@rpl:devel']
        cfg.repositoryMap = {'notfound.rpath.local': 'http://test.rpath.local:%d/repos/notfound/' % self.getPort()}

        repos = ConaryClient(cfg).getRepos()

        try:
            repos.troveNames('notfound.rpath.local')
        except repository.errors.OpenError, e:
            assert "404 Not Found" in str(e), "accessing a non-existent repository did not return a 404 Not Found error"
            pass
        else:
            self.fail("accessing a non-existent repository did not return an error")

    def testCook(self):
        client, userId = self.quickMintUser("testuser", "testpass")
        projectId = self.newProject(client)

        project = client.getProject(projectId)
        self.makeSourceTrove("testcase", testRecipe)
        self.cookFromRepository("testcase",
            versions.Label("test.rpath.local@rpl:devel"),
            ignoreDeps = True)

        self.makeSourceTrove("group-test", testGroup)
        self.cookFromRepository("group-test",
            versions.Label("test.rpath.local@rpl:devel"))

        cfg = project.getConaryConfig()
        nc = ConaryClient(cfg).getRepos()

        troveNames = nc.troveNames(versions.Label("test.rpath.local@rpl:devel"))
        assert(troveNames == ['testcase', 'testcase:runtime', 'group-test',
                              'group-test:source', 'testcase:source'])

        groupTroves = client.server.getGroupTroves(projectId)
        assert(groupTroves == {'test.rpath.local@rpl:devel': ['group-test']})

if __name__ == "__main__":
    testsuite.main()
<commit_msg>use in-memory database<commit_after>#!/usr/bin/python2.4
#
# Copyright (c) 2004-2005 rPath, Inc.
#

from time import sleep
import testsuite
testsuite.setup()

from mint_rephelp import MintRepositoryHelper
import recipes

from conary.conarycfg import ConaryConfiguration
from conary.conaryclient import ConaryClient
from conary import repository
from conary import versions

testRecipe = """
class TestCase(PackageRecipe):
    name = "testcase"
    version = "1.0"

    def setup(r):
        r.Create("/temp/foo")
"""

testGroup = """
class GroupTest(GroupRecipe):
    name = "group-test"
    version = "1.0"

    def setup(r):
        r.add('testcase')
"""

class RepositoryTest(MintRepositoryHelper):
    def testCommitStats(self):
        client, userId = self.quickMintUser("testuser", "testpass")
        projectId = self.newProject(client)

        client.server.registerCommit('test.rpath.local', 'testuser', 'mytrove:source', '/test.rpath.local@rpl:devel/1.0-1')
        project = client.getProject(projectId)
        assert(project.getCommits() == [('mytrove:source', '1.0-1')])

        # using a bogus username should not fail
        client.server.registerCommit('test.rpath.local', 'nonexistentuser', 'mytrove:source', '/test.rpath.local@rpl:devel/1.0-1')

    def testBasicRepository(self):
        client, userId = self.quickMintUser("testuser", "testpass")
        projectId = self.newProject(client)

        self.makeSourceTrove("testcase", testRecipe)
        project = client.getProject(projectId)

        cfg = project.getConaryConfig()
        nc = ConaryClient(cfg).getRepos()

        # test that the source trove landed properly
        troveNames = nc.troveNames(versions.Label("test.rpath.local@rpl:devel"))
        assert(troveNames == ["testcase:source"])

        # test that the commits table was updated
        # give some time for the commit action to run
        iters = 0
        while True:
            sleep(0.1)
            iters += 1
            if project.getCommits() != []:
                break
            if iters > 50:
                self.fail("commits didn't show up")

        assert(project.getCommits() == [('testcase:source', '1.0-1')])

    def testHooksResponse(self):
        cfg = ConaryConfiguration(readConfigFiles = False)
        cfg.installLabelPath = ['notfound.rpath.local@rpl:devel']
        cfg.repositoryMap = {'notfound.rpath.local': 'http://test.rpath.local:%d/repos/notfound/' % self.getPort()}
        cfg.root = ':memory:'
        cfg.dbPath = ':memory:'

        repos = ConaryClient(cfg).getRepos()

        try:
            repos.troveNames('notfound.rpath.local')
        except repository.errors.OpenError, e:
            assert "404 Not Found" in str(e), "accessing a non-existent repository did not return a 404 Not Found error"
            pass
        else:
            self.fail("accessing a non-existent repository did not return an error")

    def testCook(self):
        client, userId = self.quickMintUser("testuser", "testpass")
        projectId = self.newProject(client)

        project = client.getProject(projectId)
        self.makeSourceTrove("testcase", testRecipe)
        self.cookFromRepository("testcase",
            versions.Label("test.rpath.local@rpl:devel"),
            ignoreDeps = True)

        self.makeSourceTrove("group-test", testGroup)
        self.cookFromRepository("group-test",
            versions.Label("test.rpath.local@rpl:devel"))

        cfg = project.getConaryConfig()
        nc = ConaryClient(cfg).getRepos()

        troveNames = nc.troveNames(versions.Label("test.rpath.local@rpl:devel"))
        assert(troveNames == ['testcase', 'testcase:runtime', 'group-test',
                              'group-test:source', 'testcase:source'])

        groupTroves = client.server.getGroupTroves(projectId)
        assert(groupTroves == {'test.rpath.local@rpl:devel': ['group-test']})

if __name__ == "__main__":
    testsuite.main()
<|endoftext|>


<commit_before># Copyright (C) 2015 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: anze@reciprocitylabs.com
# Maintained By: anze@reciprocitylabs.com

from ggrc_basic_permissions.roles.Creator import _is_owner

scope = "System"
description = """
  This role grants a user basic, read-only, access permission to a gGRC
  instance.
  """
permissions = {
    "read": [
        "Audit",
        "Categorization",
        "Category",
        "ControlCategory",
        "ControlAssertion",
        "Control",
        "ControlAssessment",
        "Issue",
        "ControlControl",
        "DataAsset",
        "Directive",
        "Contract",
        "Policy",
        "Regulation",
        "Standard",
        "Document",
        "Facility",
        "Help",
        "Market",
        "Objective",
        "ObjectControl",
        "ObjectDocument",
        "ObjectObjective",
        "ObjectPerson",
        "Option",
        "OrgGroup",
        "Vendor",
        "PopulationSample",
        "Product",
        "ProgramControl",
        "ProgramDirective",
        "Project",
        "Relationship",
        "RelationshipType",
        "SectionBase",
        "Section",
        "Clause",
        "SystemOrProcess",
        "System",
        "Process",
        "SystemControl",
        "SystemSystem",
        "ObjectOwner",
        "Person",
        "Program",
        "Role",
        "UserRole",
        "Context",
        {
            "type": "BackgroundTask",
            "terms": {
                "property_name": "modified_by",
                "value": "$current_user"
            },
            "condition": "is"
        },
    ],
    "create": [
        "Workflow"
        "Categorization",
        "Category",
        "ControlCategory",
        "ControlAssertion",
        "Control",
        "ControlAssessment",
        "Issue",
        "DataAsset",
        "Directive",
        "Contract",
        "Policy",
        "Regulation",
        "Standard",
        "Document",
        "Facility",
        "Help",
        "Market",
        "Objective",
        "ObjectDocument",
        "ObjectPerson",
        "Option",
        "OrgGroup",
        "Vendor",
        "PopulationSample",
        "Product",
        "Project",
        "Relationship",
        "RelationshipType",
        "SectionBase",
        "Section",
        "Clause",
        "SystemOrProcess",
        "System",
        "Process",
        {
            "type": "ObjectOwner",
            "terms": {
                "property_name": "ownable.modified_by",
                "value": "$current_user"
            },
            "condition": "is"
        },
        "Person",
        "Program",
        "Role",
        "UserRole",
        "Request",
        "Response",
        "Context",
        "BackgroundTask",
    ],
    "view_object_page": [
        "__GGRC_ALL__"
    ],
    "update": _is_owner,
    "delete": _is_owner
}
<commit_msg>Add relationship check for Reader system wide role<commit_after># Copyright (C) 2015 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: anze@reciprocitylabs.com
# Maintained By: anze@reciprocitylabs.com

from ggrc_basic_permissions.roles.Creator import _is_owner

scope = "System"
description = """
  This role grants a user basic, read-only, access permission to a gGRC
  instance.
  """
permissions = {
    "read": [
        "Audit",
        "Categorization",
        "Category",
        "ControlCategory",
        "ControlAssertion",
        "Control",
        "ControlAssessment",
        "Issue",
        "ControlControl",
        "DataAsset",
        "Directive",
        "Contract",
        "Policy",
        "Regulation",
        "Standard",
        "Document",
        "Facility",
        "Help",
        "Market",
        "Objective",
        "ObjectControl",
        "ObjectDocument",
        "ObjectObjective",
        "ObjectPerson",
        "Option",
        "OrgGroup",
        "Vendor",
        "PopulationSample",
        "Product",
        "ProgramControl",
        "ProgramDirective",
        "Project",
        {
            "type": "Relationship",
            "terms": {
                "property_name": "source,destination",
                "action": "update"
            },
            "condition": "relationship",
        },
        "RelationshipType",
        "SectionBase",
        "Section",
        "Clause",
        "SystemOrProcess",
        "System",
        "Process",
        "SystemControl",
        "SystemSystem",
        "ObjectOwner",
        "Person",
        "Program",
        "Role",
        "UserRole",
        "Context",
        {
            "type": "BackgroundTask",
            "terms": {
                "property_name": "modified_by",
                "value": "$current_user"
            },
            "condition": "is"
        },
    ],
    "create": [
        "Workflow"
        "Categorization",
        "Category",
        "ControlCategory",
        "ControlAssertion",
        "Control",
        "ControlAssessment",
        "Issue",
        "DataAsset",
        "Directive",
        "Contract",
        "Policy",
        "Regulation",
        "Standard",
        "Document",
        "Facility",
        "Help",
        "Market",
        "Objective",
        "ObjectDocument",
        "ObjectPerson",
        "Option",
        "OrgGroup",
        "Vendor",
        "PopulationSample",
        "Product",
        "Project",
        "Relationship",
        "RelationshipType",
        "SectionBase",
        "Section",
        "Clause",
        "SystemOrProcess",
        "System",
        "Process",
        {
            "type": "ObjectOwner",
            "terms": {
                "property_name": "ownable.modified_by",
                "value": "$current_user"
            },
            "condition": "is"
        },
        "Person",
        "Program",
        "Role",
        "UserRole",
        "Request",
        "Response",
        "Context",
        "BackgroundTask",
    ],
    "view_object_page": [
        "__GGRC_ALL__"
    ],
    "update": _is_owner,
    "delete": _is_owner
}
<|endoftext|>


<commit_before>#!/usr/bin/env python
############################################################################
#
#   Copyright (C) 2012, 2013 PX4 Development Team. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
# 3. Neither the name PX4 nor the names of its contributors may be
#    used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
############################################################################

#
# PX4 firmware image generator
#
# The PX4 firmware file is a JSON-encoded Python object, containing
# metadata fields and a zlib-compressed base64-encoded firmware image.
#

import argparse
import json
import base64
import zlib
import time
import subprocess

#
# Construct a basic firmware description
#
def mkdesc():
	proto = {}
	proto['magic']		= "PX4FWv1"
	proto['board_id']	= 0
	proto['board_revision']	= 0
	proto['version']	= ""
	proto['summary']	= ""
	proto['description']	= ""
	proto['git_identity']	= ""
	proto['build_time']	= 0
	proto['image']		= bytes()
	proto['image_size']	= 0
	return proto

# Parse commandline
parser = argparse.ArgumentParser(description="Firmware generator for the PX autopilot system.")
parser.add_argument("--prototype",	action="store", help="read a prototype description from a file")
parser.add_argument("--board_id",	action="store", help="set the board ID required")
parser.add_argument("--board_revision",	action="store", help="set the board revision required")
parser.add_argument("--version",	action="store", help="set a version string")
parser.add_argument("--summary",	action="store", help="set a brief description")
parser.add_argument("--description",	action="store", help="set a longer description")
parser.add_argument("--git_identity",	action="store", help="the working directory to check for git identity")
parser.add_argument("--parameter_xml",	action="store", help="the parameters.xml file")
parser.add_argument("--airframe_xml",	action="store", help="the airframes.xml file")
parser.add_argument("--image",		action="store", help="the firmware image")
args = parser.parse_args()

# Fetch the firmware descriptor prototype if specified
if args.prototype != None:
	f = open(args.prototype,"r")
	desc = json.load(f)
	f.close()
else:
	desc = mkdesc()

desc['build_time'] 		= int(time.time())

if args.board_id != None:
	desc['board_id']	= int(args.board_id)
if args.board_revision != None:
	desc['board_revision']	= int(args.board_revision)
if args.version != None:
	desc['version']		= str(args.version)
if args.summary != None:
	desc['summary']		= str(args.summary)
if args.description != None:
	desc['description']	= str(args.description)
if args.git_identity != None:
	cmd = " ".join(["git", "--git-dir", args.git_identity + "/.git", "describe", "--always", "--dirty"])
	p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout
	desc['git_identity']	= str(p.read().strip())
	p.close()
if args.parameter_xml != None:
	f = open(args.parameter_xml, "rb")
	bytes = f.read()
	desc['parameter_xml_size'] = len(bytes)
	desc['parameter_xml'] = base64.b64encode(zlib.compress(bytes,9)).decode('utf-8')
	desc['mav_autopilot'] = 12 # 12 = MAV_AUTOPILOT_PX4
if args.airframe_xml != None:
	f = open(args.airframe_xml, "rb")
	bytes = f.read()
	desc['airframe_xml_size'] = len(bytes)
	desc['airframe_xml'] = base64.b64encode(zlib.compress(bytes,9)).decode('utf-8')
if args.image != None:
	f = open(args.image, "rb")
	bytes = f.read()
	desc['image_size'] = len(bytes)
	desc['image'] = base64.b64encode(zlib.compress(bytes,9)).decode('utf-8')

print(json.dumps(desc, indent=4))
<commit_msg>Tools/px_mkfw.py: use the same args to get the git version as px_update_git_header.py<commit_after>#!/usr/bin/env python
############################################################################
#
#   Copyright (C) 2012, 2013 PX4 Development Team. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
# 3. Neither the name PX4 nor the names of its contributors may be
#    used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
############################################################################

#
# PX4 firmware image generator
#
# The PX4 firmware file is a JSON-encoded Python object, containing
# metadata fields and a zlib-compressed base64-encoded firmware image.
#

import argparse
import json
import base64
import zlib
import time
import subprocess

#
# Construct a basic firmware description
#
def mkdesc():
	proto = {}
	proto['magic']		= "PX4FWv1"
	proto['board_id']	= 0
	proto['board_revision']	= 0
	proto['version']	= ""
	proto['summary']	= ""
	proto['description']	= ""
	proto['git_identity']	= ""
	proto['build_time']	= 0
	proto['image']		= bytes()
	proto['image_size']	= 0
	return proto

# Parse commandline
parser = argparse.ArgumentParser(description="Firmware generator for the PX autopilot system.")
parser.add_argument("--prototype",	action="store", help="read a prototype description from a file")
parser.add_argument("--board_id",	action="store", help="set the board ID required")
parser.add_argument("--board_revision",	action="store", help="set the board revision required")
parser.add_argument("--version",	action="store", help="set a version string")
parser.add_argument("--summary",	action="store", help="set a brief description")
parser.add_argument("--description",	action="store", help="set a longer description")
parser.add_argument("--git_identity",	action="store", help="the working directory to check for git identity")
parser.add_argument("--parameter_xml",	action="store", help="the parameters.xml file")
parser.add_argument("--airframe_xml",	action="store", help="the airframes.xml file")
parser.add_argument("--image",		action="store", help="the firmware image")
args = parser.parse_args()

# Fetch the firmware descriptor prototype if specified
if args.prototype != None:
	f = open(args.prototype,"r")
	desc = json.load(f)
	f.close()
else:
	desc = mkdesc()

desc['build_time'] 		= int(time.time())

if args.board_id != None:
	desc['board_id']	= int(args.board_id)
if args.board_revision != None:
	desc['board_revision']	= int(args.board_revision)
if args.version != None:
	desc['version']		= str(args.version)
if args.summary != None:
	desc['summary']		= str(args.summary)
if args.description != None:
	desc['description']	= str(args.description)
if args.git_identity != None:
	cmd = " ".join(["git", "--git-dir", args.git_identity + "/.git", "describe", "--always", "--tags"])
	p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout
	desc['git_identity']	= str(p.read().strip())
	p.close()
if args.parameter_xml != None:
	f = open(args.parameter_xml, "rb")
	bytes = f.read()
	desc['parameter_xml_size'] = len(bytes)
	desc['parameter_xml'] = base64.b64encode(zlib.compress(bytes,9)).decode('utf-8')
	desc['mav_autopilot'] = 12 # 12 = MAV_AUTOPILOT_PX4
if args.airframe_xml != None:
	f = open(args.airframe_xml, "rb")
	bytes = f.read()
	desc['airframe_xml_size'] = len(bytes)
	desc['airframe_xml'] = base64.b64encode(zlib.compress(bytes,9)).decode('utf-8')
if args.image != None:
	f = open(args.image, "rb")
	bytes = f.read()
	desc['image_size'] = len(bytes)
	desc['image'] = base64.b64encode(zlib.compress(bytes,9)).decode('utf-8')

print(json.dumps(desc, indent=4))
<|endoftext|>


<commit_before>from importlib import import_module
import sys
import traceback

from rhc.log import logmsg
from rhc.tcpsocket import Server
from rhc.resthandler import RESTMapper, RESTHandler

SERVER = Server()

def _import(item_path):
    path, function = item_path.rsplit('.', 1)
    module = import_module(path)
    return getattr(module, function)

def _load(f):

    result = dict(
        routes=[],
        context=None,
        port=None,
    )

    kwargs = None
    for lnum, l in enumerate((ll for ll in (l.split('#', 1)[0].strip() for l in f.readlines()) if len(ll)), start=1):
        if l.find(' ') == -1:
            raise Exception("Line %d doesn't contain a space" % lnum)
        rectyp, recval = l.split(' ', 1)
        rectyp = rectyp.upper()
        recval = recval.strip()

        if rectyp == 'ROUTE':
            kwargs = {}
            result['routes'].append((recval.strip(), kwargs))
        elif rectyp in ('GET', 'PUT', 'POST', 'DELETE'):
            if kwargs is None:
                raise Exception("Line %d contains a %s that doesn't belong to a ROUTE" % (lnum, rectyp))
            kwargs[rectyp.lower()] = _import(recval)
        elif rectyp == 'CONTEXT':
            kwargs = None
            # result['context'] = type('context', (object,), _import(recval)())()  # import recval, evaluate as function returning dict, wrap in object
            result['context'] = _import(recval)()
        elif rectyp == 'PORT':
            kwargs = None
            result['port'] = int(recval)
        else:
            raise Exception("Line %d is an invalid record type: %s" % (lnum, rectyp))

    return result

class MicroRESTHandler(RESTHandler):

    def on_open(self):
        logmsg(102, self.full_address())

    def on_close(self):
        logmsg(103, self.full_address())

    def on_rest_data(self, request, *groups):
        logmsg(104, request.http_method, request.http_resource, request.http_query_string, groups)

    def on_rest_exception(self, exception_type, value, trace):
        data = traceback.format_exe(trace)
        logmsg(105, data)
        return data

if __name__ == '__main__':
    from StringIO import StringIO
    from rhc.log import LOG

    LOG.setup(StringIO('''
        MESSAGE 100
        LOG     INFO
        DISPLAY ALWAYS
        TEXT Server listening on port %s

        MESSAGE 101
        LOG     INFO
        DISPLAY ALWAYS
        TEXT Received shutdown command from keyboard

        MESSAGE 102
        LOG     INFO
        DISPLAY ALWAYS
        TEXT open: %s

        MESSAGE 103
        LOG     INFO
        DISPLAY ALWAYS
        TEXT close: %s

        MESSAGE 104
        LOG     INFO
        DISPLAY ALWAYS
        TEXT request method=%s, resource=%s, query=%s, groups=%s

        MESSAGE 105
        LOG     WARNING
        DISPLAY ALWAYS
        TEXT exception encountered: %s

    '''), stdout=True)

    f = sys.stdin if len(sys.argv) < 2 else open(sys.argv[1])
    config = _load(f)

    m = RESTMapper(context=config['context'])
    for pattern, kwargs in config['routes']:
        m.add(pattern, **kwargs)

    SERVER.add_server(config['port'], MicroRESTHandler, m)
    try:
        while True:
            SERVER.service(.1)
    except KeyboardInterrupt:
        print 'done'
<commit_msg>micro checkpoint<commit_after>from importlib import import_module
import sys
import traceback

from rhc.log import logmsg
from rhc.tcpsocket import Server
from rhc.resthandler import RESTMapper, RESTHandler

SERVER = Server()

def _import(item_path):
    path, function = item_path.rsplit('.', 1)
    module = import_module(path)
    return getattr(module, function)

def _load(f):

    result = dict(
        routes=[],
        context=None,
        port=None,
    )

    kwargs = None
    for lnum, l in enumerate((ll for ll in (l.split('#', 1)[0].strip() for l in f.readlines()) if len(ll)), start=1):
        if l.find(' ') == -1:
            raise Exception("Line %d doesn't contain a space" % lnum)
        rectyp, recval = l.split(' ', 1)
        rectyp = rectyp.upper()
        recval = recval.strip()

        if rectyp == 'ROUTE':
            kwargs = {}
            result['routes'].append((recval.strip(), kwargs))
        elif rectyp in ('GET', 'PUT', 'POST', 'DELETE'):
            if kwargs is None:
                raise Exception("Line %d contains a %s that doesn't belong to a ROUTE" % (lnum, rectyp))
            kwargs[rectyp.lower()] = _import(recval)
        elif rectyp == 'CONTEXT':
            kwargs = None
            result['context'] = _import(recval)()
        elif rectyp == 'PORT':
            kwargs = None
            result['port'] = int(recval)
        else:
            raise Exception("Line %d is an invalid record type: %s" % (lnum, rectyp))

    return result

class MicroRESTHandler(RESTHandler):

    NEXT_ID = 0
    NEXT_REQUEST_ID = 0

    def on_open(self):
        self.id = MicroRESTHandler.NEXT_ID = MicroRESTHandler.NEXT_ID + 1
        logmsg(102, self.id, self.full_address())

    def on_close(self):
        logmsg(103, self.id, self.full_address())

    def on_rest_data(self, request, *groups):
        request.id = MicroRESTHandler.NEXT_REQUEST_ID = MicroRESTHandler.NEXT_REQUEST_ID + 1
        logmsg(104, self.id, request.id, request.http_method, request.http_resource, request.http_query_string, groups)

    def on_rest_exception(self, exception_type, value, trace):
        data = traceback.format_exc(trace)
        logmsg(105, data)
        return data

if __name__ == '__main__':
    from StringIO import StringIO
    from rhc.log import LOG

    LOG.setup(StringIO('''
        MESSAGE 100
        LOG     INFO
        DISPLAY ALWAYS
        TEXT Server listening on port %s

        MESSAGE 101
        LOG     INFO
        DISPLAY ALWAYS
        TEXT Received shutdown command from keyboard

        MESSAGE 102
        LOG     INFO
        DISPLAY ALWAYS
        TEXT open: cid=%d, %s

        MESSAGE 103
        LOG     INFO
        DISPLAY ALWAYS
        TEXT close: cid=%d, %s

        MESSAGE 104
        LOG     INFO
        DISPLAY ALWAYS
        TEXT request cid=%d, rid=%d, method=%s, resource=%s, query=%s, groups=%s

        MESSAGE 105
        LOG     WARNING
        DISPLAY ALWAYS
        TEXT exception encountered: %s

    '''), stdout=True)

    f = sys.stdin if len(sys.argv) < 2 else open(sys.argv[1])
    config = _load(f)

    m = RESTMapper(context=config['context'])
    for pattern, kwargs in config['routes']:
        m.add(pattern, **kwargs)

    SERVER.add_server(config['port'], MicroRESTHandler, m)
    logmsg(100, config['port'])
    try:
        while True:
            SERVER.service(.1)
    except KeyboardInterrupt:
        print 'done'
<|endoftext|>


<commit_before>import json
import os
import time
import errno
import md5
import uuid
from dateutil import parser
import calendar

from TwitterAPI import TwitterAPI
from WookieDb import WookieDb
from StreamMessage import StreamMessage
import _mysql_exceptions
from helpers import save_data, convert_to_unix

class TrendAnalyser:

    def __init__(self, config_location="conf.json", load_api=True, load_db=True):
        '''Initialises the TrendAnalyser class object'''
        self.load_conf(config_location)
        self.api = None
        self.db = None

        if load_db:
            self.connect_to_db()
            self.load_db_conf()

        if load_api:
            self.load_api()

    def connect_to_db(self):
        '''Connects to the database using the details in conf'''
        self.db = WookieDb(self.conf['database_host'],
                           self.conf['database_username'],
                           self.conf['database_password'],
                           self.conf['database_schema'],
                           select_type='dict', charset="utf8")

    def load_conf(self, config_location):
        '''Loads the config from the given conf file location'''
        self.conf = {"debug" : False,
                     "offline": False}
        self.conf.update(json.load(open(config_location)))

    def load_db_conf(self):
        '''Accesses the DB and loads all the config details into the object'''
        db_options = self.db.select("options", "*")
        for option in db_options:
            value = option["value"]
            if value == "true":
                value = True
            elif value == "false":
                value = False

            self.conf[option['key']] = value

    def load_api(self):
        '''creates an instance of the TwitterAPI class to use'''
        details = json.load(open(self.conf['twitter_key_location']))

        self.api = TwitterAPI(details['consumer_key'],
                              details['consumer_secret'],
                              details['access_token_key'],
                              details['access_token_secret'])

    def save_trend_data(self, json_data, woeid):
        '''Saves Twitter Trend data to a json file'''
        filelocation = os.path.join(self.conf['save_data_location'],
                                "trends/place/weoid_" + str(woeid) + "_" +
                                str(int(time.time())) + ".json")
        save_data(json_data, filelocation)

    def save_twitter_sample_data(self, json_data):
        '''Saves data from the sample API to json files'''
        time_folder = int(time.time() / 300) * 300

        filelocation = os.path.join(self.conf['save_data_location'],
                                "statuses/sample", str(time_folder),
                                str(int(time.time())) + "_" +
                                md5.md5(json.dumps(json_data)).hexdigest() +
                                ".json")
        save_data(json_data, filelocation)

    def save_sample_data_db(self, json_data):
        '''Saves as little as possible from the sample API to the database'''
        msg = StreamMessage(json_data)

        if msg.get_type() == "tweet":
            if msg.data['entities']['hashtags'] != [] or msg.data['entities']['user_mentions'] != []:
                words = map(json.dumps, msg.data['text'].split(" "))
                where_clause = "WHERE word in (" + ",".join(words) + ");"

                positive = self.db.select("words_positive", "count(*)", where_clause)[0]["count(*)"]
                negative = self.db.select("words_negative", "count(*)", where_clause)[0]["count(*)"]

                tweet_details = {"tweetId" : msg.data['id'],
                                 "positive" : positive,
                                 "negative" : negative,
                                 "created_at" : convert_to_unix(msg.data['created_at'])}
                try:
                    self.db.insert("tweet_details", tweet_details)
                except _mysql_exceptions.IntegrityError as err:
                    if err[0] == 1062: #Duplicate error, ignore as twitter has resent the hashtag id
                        return
                    else:
                        raise

                for mention in msg.data['entities']['user_mentions']:
                    mention_details = {"tweetId": msg.data["id"],
                                       "user_id": mention["id"],
                                       "name" : mention["name"],
                                       "screen_name" : mention["screen_name"]}
                    try:
                        self.db.insert("tweet_mentions", mention_details)
                    except _mysql_exceptions.IntegrityError as err:
                        if err[0] == 1062: #Duplicate error, ignore it as they ahve put multiple mentions in the message
                            pass
                        else:
                            raise

                for hashtag in msg.get_hashtags():
                    tweet_hashtags = {"tweetId" : msg.data['id'], "hashtag" : hashtag}
                    try:
                        self.db.insert("tweet_hashtags", tweet_hashtags)
                    except _mysql_exceptions.IntegrityError as err:
                        if err[0] == 1062: #Duplicate error, ignore it as they have put multiple hashtags in the message
                            pass
                        else:
                            raise

                self.db.commit()

    def save_twitter_filter_data(self, json_data):
        '''Saves the given json data into the filter directory'''
        time_folder = int(time.time() / 300) * 300

        filelocation = os.path.join(self.conf['save_data_location'],
                                "statuses/filter", str(time_folder),
                                str(int(time.time())) + "_" +
                                uuid.uuid4().hex +
                                ".json")
        save_data(json_data, filelocation)

    def start_stream_filter(self):
        '''Gets the filter terms to search for them starts downloading them'''
        terms = self._get_filter_keywords()

        print "Loading terms from database"
        print "Terms to download data for:"
        for term in terms:
            print term
        conjoined_terms = ','.join(terms)

        print "Starting to download data"
        response = self.api.request("statuses/filter", {"track" : conjoined_terms})

        items = 0
        item_data = []
        for item in response.get_iterator():
            items += 1
            item_data.append(item)

            if items > 1000:
                self.save_twitter_filter_data(item_data)
                item_data = []
                items = 0

    def _get_filter_keywords(self):
        '''Get all the filter keywords from the database'''
        terms = self.db.select("filter_status_terms", "*")
        combined_terms = []
        for term in terms:
            combined_terms.append(term['term_name'])

        return combined_terms

    def _get_latest_trends(self):
        '''Gets the latest trending topics from each woeid'''
        woeids = self.db.select("trend_top_list", "distinct(woeid)")
        latest_trends = {}
        for row in woeids:
            trend_details = self.db.select("trend_top_list t "+
                                           "left join woeid_data w on t.woeid = w.woeid" ,
                                           "t.*, w.name",
                                           "WHERE t.woeid = '" + str(row['woeid']) + "' " +
                                           "order by t.trend_top_list_id desc limit 1")
            trend_details = trend_details[0]

            trends = self.db.select("trend_top_list_trends",
                                    "*", "WHERE trend_top_list_id = '" + str(trend_details['trend_top_list_id']) + "'" )

            trend_details['trends'] = []
            for trend in trends:
                trend_details['trends'].append(trend)

            latest_trends[row['woeid']] = trend_details

        return latest_trends

    def _get_trending_details(self, search_term):
        '''Gets all trending information for a specific search term'''
        details = self.db.select("trend_top_list l "+
                                 "left join trend_top_list_trends t on t.trend_top_list_id = l.trend_top_list_id " +
                                 "left join woeid_data w on l.woeid = w.woeid",
                                 "l.*, t.events, t.promoted_content, w.name as woeid_name",
                                 "WHERE t.name = '"+ search_term+"'")
        search_details = {'trends' : details, 'search_term' : search_term}
        return search_details

    def _get_hashtag_details(self, search_term):
        '''Gets the details about a specific hashtag from the database'''
        details = self.db.select("tweet_hashtags h left join tweet_details d on h.tweetId = d.tweetId", "d.*",
                                 "WHERE hashtag = '"+ search_term +"'")
        search_details = {'hashtags' : details, 'hashtag' : search_term}
        return search_details

    def _get_hashtag_frequency(self, search_term):
        '''Returns data about a hashtag's popularity over time'''
        time_period = 86400# seconds in a day

        details = self._get_hashtag_details(search_term)
        tweet_spikes = {}
        for hashtag in details['hashtags']:
            if hashtag['created_at'] is None:
                continue
            created_at = hashtag['created_at']

            if created_at / time_period in tweet_spikes:
                tweet_spikes[created_at / time_period] += 1
            else:
                tweet_spikes[created_at / time_period] = 1

        if not tweet_spikes:
            return []

        largest_value = max(tweet_spikes.keys())
        smallest_value = min(tweet_spikes.keys())
        for i in xrange(largest_value - smallest_value):
            if not smallest_value + i in tweet_spikes:
                tweet_spikes[smallest_value + i] = 0

        data = []
        for spike in tweet_spikes:
            data.append({"x": int(spike), "y": tweet_spikes[spike]})

        data.sort(key=lambda x : x['x'])

        return data

    def _get_mention_details(self, search_term):
        '''Returns data about a specific mention screenname'''
        details = self.db.select("tweet_mentions m left join tweet_details d on m.tweetId = d.tweetId",
                                 "d.*",
                                 "WHERE screen_name = '" + search_term +"'")
        search_details = {'mentions' : details, 'screen_name' : search_term}
        return search_details

    def _get_mention_frequency(self, search_term):
        '''Returns data about a mention's popularity over time'''
        time_period = 86400# seconds in a day

        details = self._get_mention_details(search_term)
        tweet_spikes = {}
        for mention in details['mentions']:
            if mention['created_at'] is None:
                continue
            created_at = mention['created_at']

            if created_at / time_period in tweet_spikes:
                tweet_spikes[created_at / time_period] += 1
            else:
                tweet_spikes[created_at / time_period] = 1

        if not tweet_spikes:
            return []

        largest_value = max(tweet_spikes.keys())
        smallest_value = min(tweet_spikes.keys())
        for i in xrange(largest_value - smallest_value):
            if not smallest_value + i in tweet_spikes:
                tweet_spikes[smallest_value + i] = 0

        data = []
        for spike in tweet_spikes:
            data.append({"x": int(spike), "y": tweet_spikes[spike]})

        data.sort(key=lambda x : x['x'])
        return data

    def _get_trending_woeids_downloading(self):
        '''Gets the details of woeids you want to download from the database'''
        return self.db.select("woeids_download d left join woeid_data w on d.woeid = w.woeid", "d.*, w.name")

    def download_trends(self):
        '''Downloads all the woeid data from the ones you want to download'''
        woeids = self.db.select("woeids_download", "*")
        for woeid in woeids:
            self.download_trend_list(woeid['woeid'])

    def download_trend_list(self, woeid):
        '''Saves one woeid data to the data'''
        response = self.api.request("trends/place", {"id" : woeid})
        response_json = json.loads(response.text)[0]

        trend_top_list_data = {
            'woeid' : response_json['locations'][0]['woeid'],
            'as_of' : convert_to_unix(response_json['as_of']),
            'created_at' : convert_to_unix(response_json['created_at'])
        }

        self.db.insert("trend_top_list", trend_top_list_data)
        trend_top_list_id = self.db.get_last_autoincrement()

        for trend in response_json['trends']:
            trend_data = {
                'trend_top_list_id' : trend_top_list_id,
                'name' : trend['name'],
                'events' : trend['events'],
                'promoted_content' : trend['promoted_content']
            }
            self.db.insert("trend_top_list_trends", trend_data)

        self.db.connection.commit()

    def _update_woeid_data(self):
        '''Downloads all the avalible woeid data from Twitter and stores it'''
        response = self.api.request("trends/available")
        response_json = json.loads(response.text)

        for res in response_json:
            data = {"woeid" : res['woeid'],
                    "country" : res['country'],
                    "countryCode" : res['countryCode'],
                    "name" : res['name'],
                    "parentWoeid" : res['parentid'],
                    "placeCode" : res['placeType']['code'],
                    "placeName" : res['placeType']['name'],
                    "url" : res['url']}
            self.db.delete("woeid_data", "WHERE woeid = " + str(data['woeid']))
            self.db.insert("woeid_data", data)

        self.db.commit()

    def _get_woeid_data(self):
        '''Gets all woeid data from the database'''
        details = self.db.select("woeid_data", "*")
        return details

    def _get_words_positive(self):
        '''Gets all the positive words from the database'''
        details = self.db.select("words_positive", "*")
        return details

    def _get_words_negative(self):
        '''Gets all the negative words from the database'''
        details = self.db.select("words_negative", "*")
        return details

    def set_option(self, key, value):
        '''Allows you to set database options'''
        existing = self.db.select("options", "*", " WHERE options.key = '" + key + "' LIMIT 1;")
        if existing:
            self.db.update("options", {"value" : value}, "WHERE options.key = '" + key + "' LIMIT 1;")
        else:
            self.db.insert("options", {"options.key" : key, "value" : value})
        self.db.commit()
<commit_msg>Added conf variable to __init__<commit_after>import json
import os
import time
import errno
import md5
import uuid
from dateutil import parser
import calendar

from TwitterAPI import TwitterAPI
from WookieDb import WookieDb
from StreamMessage import StreamMessage
import _mysql_exceptions
from helpers import save_data, convert_to_unix

class TrendAnalyser:

    def __init__(self, config_location="conf.json", load_api=True, load_db=True):
        '''Initialises the TrendAnalyser class object'''
        self.load_conf(config_location)
        self.api = None
        self.db = None
        self.conf = None

        if load_db:
            self.connect_to_db()
            self.load_db_conf()

        if load_api:
            self.load_api()

    def connect_to_db(self):
        '''Connects to the database using the details in conf'''
        self.db = WookieDb(self.conf['database_host'],
                           self.conf['database_username'],
                           self.conf['database_password'],
                           self.conf['database_schema'],
                           select_type='dict', charset="utf8")

    def load_conf(self, config_location):
        '''Loads the config from the given conf file location'''
        self.conf = {"debug" : False,
                     "offline": False}
        self.conf.update(json.load(open(config_location)))

    def load_db_conf(self):
        '''Accesses the DB and loads all the config details into the object'''
        db_options = self.db.select("options", "*")
        for option in db_options:
            value = option["value"]
            if value == "true":
                value = True
            elif value == "false":
                value = False

            self.conf[option['key']] = value

    def load_api(self):
        '''creates an instance of the TwitterAPI class to use'''
        details = json.load(open(self.conf['twitter_key_location']))

        self.api = TwitterAPI(details['consumer_key'],
                              details['consumer_secret'],
                              details['access_token_key'],
                              details['access_token_secret'])

    def save_trend_data(self, json_data, woeid):
        '''Saves Twitter Trend data to a json file'''
        filelocation = os.path.join(self.conf['save_data_location'],
                                "trends/place/weoid_" + str(woeid) + "_" +
                                str(int(time.time())) + ".json")
        save_data(json_data, filelocation)

    def save_twitter_sample_data(self, json_data):
        '''Saves data from the sample API to json files'''
        time_folder = int(time.time() / 300) * 300

        filelocation = os.path.join(self.conf['save_data_location'],
                                "statuses/sample", str(time_folder),
                                str(int(time.time())) + "_" +
                                md5.md5(json.dumps(json_data)).hexdigest() +
                                ".json")
        save_data(json_data, filelocation)

    def save_sample_data_db(self, json_data):
        '''Saves as little as possible from the sample API to the database'''
        msg = StreamMessage(json_data)

        if msg.get_type() == "tweet":
            if msg.data['entities']['hashtags'] != [] or msg.data['entities']['user_mentions'] != []:
                words = map(json.dumps, msg.data['text'].split(" "))
                where_clause = "WHERE word in (" + ",".join(words) + ");"

                positive = self.db.select("words_positive", "count(*)", where_clause)[0]["count(*)"]
                negative = self.db.select("words_negative", "count(*)", where_clause)[0]["count(*)"]

                tweet_details = {"tweetId" : msg.data['id'],
                                 "positive" : positive,
                                 "negative" : negative,
                                 "created_at" : convert_to_unix(msg.data['created_at'])}
                try:
                    self.db.insert("tweet_details", tweet_details)
                except _mysql_exceptions.IntegrityError as err:
                    if err[0] == 1062: #Duplicate error, ignore as twitter has resent the hashtag id
                        return
                    else:
                        raise

                for mention in msg.data['entities']['user_mentions']:
                    mention_details = {"tweetId": msg.data["id"],
                                       "user_id": mention["id"],
                                       "name" : mention["name"],
                                       "screen_name" : mention["screen_name"]}
                    try:
                        self.db.insert("tweet_mentions", mention_details)
                    except _mysql_exceptions.IntegrityError as err:
                        if err[0] == 1062: #Duplicate error, ignore it as they ahve put multiple mentions in the message
                            pass
                        else:
                            raise

                for hashtag in msg.get_hashtags():
                    tweet_hashtags = {"tweetId" : msg.data['id'], "hashtag" : hashtag}
                    try:
                        self.db.insert("tweet_hashtags", tweet_hashtags)
                    except _mysql_exceptions.IntegrityError as err:
                        if err[0] == 1062: #Duplicate error, ignore it as they have put multiple hashtags in the message
                            pass
                        else:
                            raise

                self.db.commit()

    def save_twitter_filter_data(self, json_data):
        '''Saves the given json data into the filter directory'''
        time_folder = int(time.time() / 300) * 300

        filelocation = os.path.join(self.conf['save_data_location'],
                                "statuses/filter", str(time_folder),
                                str(int(time.time())) + "_" +
                                uuid.uuid4().hex +
                                ".json")
        save_data(json_data, filelocation)

    def start_stream_filter(self):
        '''Gets the filter terms to search for them starts downloading them'''
        terms = self._get_filter_keywords()

        print "Loading terms from database"
        print "Terms to download data for:"
        for term in terms:
            print term
        conjoined_terms = ','.join(terms)

        print "Starting to download data"
        response = self.api.request("statuses/filter", {"track" : conjoined_terms})

        items = 0
        item_data = []
        for item in response.get_iterator():
            items += 1
            item_data.append(item)

            if items > 1000:
                self.save_twitter_filter_data(item_data)
                item_data = []
                items = 0

    def _get_filter_keywords(self):
        '''Get all the filter keywords from the database'''
        terms = self.db.select("filter_status_terms", "*")
        combined_terms = []
        for term in terms:
            combined_terms.append(term['term_name'])

        return combined_terms

    def _get_latest_trends(self):
        '''Gets the latest trending topics from each woeid'''
        woeids = self.db.select("trend_top_list", "distinct(woeid)")
        latest_trends = {}
        for row in woeids:
            trend_details = self.db.select("trend_top_list t "+
                                           "left join woeid_data w on t.woeid = w.woeid" ,
                                           "t.*, w.name",
                                           "WHERE t.woeid = '" + str(row['woeid']) + "' " +
                                           "order by t.trend_top_list_id desc limit 1")
            trend_details = trend_details[0]

            trends = self.db.select("trend_top_list_trends",
                                    "*", "WHERE trend_top_list_id = '" + str(trend_details['trend_top_list_id']) + "'" )

            trend_details['trends'] = []
            for trend in trends:
                trend_details['trends'].append(trend)

            latest_trends[row['woeid']] = trend_details

        return latest_trends

    def _get_trending_details(self, search_term):
        '''Gets all trending information for a specific search term'''
        details = self.db.select("trend_top_list l "+
                                 "left join trend_top_list_trends t on t.trend_top_list_id = l.trend_top_list_id " +
                                 "left join woeid_data w on l.woeid = w.woeid",
                                 "l.*, t.events, t.promoted_content, w.name as woeid_name",
                                 "WHERE t.name = '"+ search_term+"'")
        search_details = {'trends' : details, 'search_term' : search_term}
        return search_details

    def _get_hashtag_details(self, search_term):
        '''Gets the details about a specific hashtag from the database'''
        details = self.db.select("tweet_hashtags h left join tweet_details d on h.tweetId = d.tweetId", "d.*",
                                 "WHERE hashtag = '"+ search_term +"'")
        search_details = {'hashtags' : details, 'hashtag' : search_term}
        return search_details

    def _get_hashtag_frequency(self, search_term):
        '''Returns data about a hashtag's popularity over time'''
        time_period = 86400# seconds in a day

        details = self._get_hashtag_details(search_term)
        tweet_spikes = {}
        for hashtag in details['hashtags']:
            if hashtag['created_at'] is None:
                continue
            created_at = hashtag['created_at']

            if created_at / time_period in tweet_spikes:
                tweet_spikes[created_at / time_period] += 1
            else:
                tweet_spikes[created_at / time_period] = 1

        if not tweet_spikes:
            return []

        largest_value = max(tweet_spikes.keys())
        smallest_value = min(tweet_spikes.keys())
        for i in xrange(largest_value - smallest_value):
            if not smallest_value + i in tweet_spikes:
                tweet_spikes[smallest_value + i] = 0

        data = []
        for spike in tweet_spikes:
            data.append({"x": int(spike), "y": tweet_spikes[spike]})

        data.sort(key=lambda x : x['x'])

        return data

    def _get_mention_details(self, search_term):
        '''Returns data about a specific mention screenname'''
        details = self.db.select("tweet_mentions m left join tweet_details d on m.tweetId = d.tweetId",
                                 "d.*",
                                 "WHERE screen_name = '" + search_term +"'")
        search_details = {'mentions' : details, 'screen_name' : search_term}
        return search_details

    def _get_mention_frequency(self, search_term):
        '''Returns data about a mention's popularity over time'''
        time_period = 86400# seconds in a day

        details = self._get_mention_details(search_term)
        tweet_spikes = {}
        for mention in details['mentions']:
            if mention['created_at'] is None:
                continue
            created_at = mention['created_at']

            if created_at / time_period in tweet_spikes:
                tweet_spikes[created_at / time_period] += 1
            else:
                tweet_spikes[created_at / time_period] = 1

        if not tweet_spikes:
            return []

        largest_value = max(tweet_spikes.keys())
        smallest_value = min(tweet_spikes.keys())
        for i in xrange(largest_value - smallest_value):
            if not smallest_value + i in tweet_spikes:
                tweet_spikes[smallest_value + i] = 0

        data = []
        for spike in tweet_spikes:
            data.append({"x": int(spike), "y": tweet_spikes[spike]})

        data.sort(key=lambda x : x['x'])
        return data

    def _get_trending_woeids_downloading(self):
        '''Gets the details of woeids you want to download from the database'''
        return self.db.select("woeids_download d left join woeid_data w on d.woeid = w.woeid", "d.*, w.name")

    def download_trends(self):
        '''Downloads all the woeid data from the ones you want to download'''
        woeids = self.db.select("woeids_download", "*")
        for woeid in woeids:
            self.download_trend_list(woeid['woeid'])

    def download_trend_list(self, woeid):
        '''Saves one woeid data to the data'''
        response = self.api.request("trends/place", {"id" : woeid})
        response_json = json.loads(response.text)[0]

        trend_top_list_data = {
            'woeid' : response_json['locations'][0]['woeid'],
            'as_of' : convert_to_unix(response_json['as_of']),
            'created_at' : convert_to_unix(response_json['created_at'])
        }

        self.db.insert("trend_top_list", trend_top_list_data)
        trend_top_list_id = self.db.get_last_autoincrement()

        for trend in response_json['trends']:
            trend_data = {
                'trend_top_list_id' : trend_top_list_id,
                'name' : trend['name'],
                'events' : trend['events'],
                'promoted_content' : trend['promoted_content']
            }
            self.db.insert("trend_top_list_trends", trend_data)

        self.db.connection.commit()

    def _update_woeid_data(self):
        '''Downloads all the avalible woeid data from Twitter and stores it'''
        response = self.api.request("trends/available")
        response_json = json.loads(response.text)

        for res in response_json:
            data = {"woeid" : res['woeid'],
                    "country" : res['country'],
                    "countryCode" : res['countryCode'],
                    "name" : res['name'],
                    "parentWoeid" : res['parentid'],
                    "placeCode" : res['placeType']['code'],
                    "placeName" : res['placeType']['name'],
                    "url" : res['url']}
            self.db.delete("woeid_data", "WHERE woeid = " + str(data['woeid']))
            self.db.insert("woeid_data", data)

        self.db.commit()

    def _get_woeid_data(self):
        '''Gets all woeid data from the database'''
        details = self.db.select("woeid_data", "*")
        return details

    def _get_words_positive(self):
        '''Gets all the positive words from the database'''
        details = self.db.select("words_positive", "*")
        return details

    def _get_words_negative(self):
        '''Gets all the negative words from the database'''
        details = self.db.select("words_negative", "*")
        return details

    def set_option(self, key, value):
        '''Allows you to set database options'''
        existing = self.db.select("options", "*", " WHERE options.key = '" + key + "' LIMIT 1;")
        if existing:
            self.db.update("options", {"value" : value}, "WHERE options.key = '" + key + "' LIMIT 1;")
        else:
            self.db.insert("options", {"options.key" : key, "value" : value})
        self.db.commit()
<|endoftext|>