<commit_before>import time
import random
import configparser
import queue
import threading
import twitter

import neopixel as np
import LightControl as lc

class Wall:

    def __init__(self, lights):

        self.lights = lights
        self.queued_words = queue.Queue()
        self.running = True
        self.inputs_required = True
        self.MAX_WORD_LENGTH = 20
        self.ALLOWED_CHARACTERS = set("ABCDEFGHIJKLMNOPQRSTUVWXYZ ")

        white = np.Color(255, 255, 255)
        dark_blue = np.Color(0, 0, 170)
        purple = np.Color(100, 0, 100)
        cyan = np.Color(50, 220, 220)
        light_blue = np.Color(0, 150, 200)
        yellow = np.Color(255, 200, 0)
        pink = np.Color(255, 50, 255)

        self.LETTER_COLOUR = {"A" : white, "B" : dark_blue, "C" : purple, "D" : cyan, "E" : light_blue, "F" : yellow,
                              "G" : pink, "H" : cyan,
                              "I" : cyan, "J" : pink, "K" : light_blue, "L" : cyan, "M" : yellow, "N" : pink,
                              "O" : purple, "P" : cyan, "Q" : pink,
                              "R" : cyan, "S" : white, "T" : yellow, "U" : light_blue, "V" : pink, "W" : light_blue,
                              "X" : yellow, "Y" : pink, "Z" : pink}

        config = configparser.ConfigParser()
        config.read("wall.ini")

        self.LETTER_LED = {}

        #file = open("leds.txt", "r")
        #file_output = file.readlines()
        options = config.options("LED Numbers")
        for option in options:
            letter = option
            number = config.get("LED Numbers", option)
            self.LETTER_LED[letter] = number

        self.words = []

        #file = open("words.txt", "r")
        #file_output = file.readlines()
        words = config.get("Words", "words").split(",")
        for word in words:
            word = word.upper().rstrip()
            #print(line)
            #print(self.word_is_ok(line))
            if self.word_is_ok(word):
                self.words.append(word)

        #Twitter setup
        consumer_key = config.get("Twitter", "consumer_key")
        consumer_secret = config.get("Twitter", "consumer_secret")
        access_token = config.get("Twitter", "access_token")
        access_token_secret = config.get("Twitter", "access_token_secret")
        self.api = twitter.Api(consumer_key=consumer_key,
                               consumer_secret=consumer_secret,
                               access_token_key=access_token,
                               access_token_secret=access_token_secret,
                               sleep_on_rate_limit=True)

    def run(self):
        inputs = threading.Thread(target=self.get_console_inputs)
        inputs.start()
        while self.running:
            if self.queued_words.empty():
                self.queue_random_word()
            self.display_queued_word()
            self.flicker()

    def get_console_inputs(self):
        while self.inputs_required:
            word = input("> ")
            if self.word_is_ok(word):
                print("Word added to queue")
                self.queued_words.put(word)
            else:
                print("Sorry, not a valid input")

    def get_twitter_inputs(self):
        latest_id = None
        while True:
            results = self.api.GetHomeTimeline(since_id=latest_id)
            for status in results:
                latest_id = 0
                word = status["text"]
                if self.word_is_ok(word):
                    print("Word added to queue")
                    self.queued_words.put(word)
                else:
                    print("Sorry, not a valid input")
                latest_id = max(latest_id, status["id"])
            time.sleep(60)

    def word_is_ok(self, word):
        word = word.upper()
        #print(len(word) < self.MAX_WORD_LENGTH)
        #print(set(word).issubset(self.ALLOWED_CHARACTERS))
        return set(word).issubset(self.ALLOWED_CHARACTERS) and len(word) < self.MAX_WORD_LENGTH

    def light_letter(self, letter):
        led = self.LETTER_LED.get(letter)
        colour = self.LETTER_COLOUR.get(letter)
        self.lights.activate(led, colour)

    def display_word(self, word):
        letters = list(word.upper())
        for letter in letters:
            if not(letter == " "):
                self.light_letter(letter)
                self.lights.show()
            time.sleep(1)
            self.lights.turn_all_off()
            time.sleep(0.5)

    def display_queued_word(self):
        word = self.queued_words.get()
        self.display_word(word)

    def queue_random_word(self):
        self.queued_words.put(random.choice(self.words))

    def flicker(self, repetitions=10):
        for i in range(repetitions):
            self.turn_letters_on()
            time.sleep(0.2)
            self.lights.turn_all_off()
            time.sleep(0.2)

    def turn_letters_on(self):
        for letter in self.LETTER_LED:
            self.light_letter(letter)
        self.lights.show()

    def turn_all_off(self):
        self.lights.turn_all_off()

def main():
    try:
        lights = lc.LEDString(count=50)
        wall = Wall(lights)
        wall.run()
    finally:
        wall.running = False
        wall.inputs_required = False
        wall.turn_all_off()

if __name__ == "__main__":
    main()<commit_msg>Added word print statement<commit_after>import time
import random
import configparser
import queue
import threading
import twitter

import neopixel as np
import LightControl as lc

class Wall:

    def __init__(self, lights):

        self.lights = lights
        self.queued_words = queue.Queue()
        self.running = True
        self.inputs_required = True
        self.MAX_WORD_LENGTH = 20
        self.ALLOWED_CHARACTERS = set("ABCDEFGHIJKLMNOPQRSTUVWXYZ ")

        white = np.Color(255, 255, 255)
        dark_blue = np.Color(0, 0, 170)
        purple = np.Color(100, 0, 100)
        cyan = np.Color(50, 220, 220)
        light_blue = np.Color(0, 150, 200)
        yellow = np.Color(255, 200, 0)
        pink = np.Color(255, 50, 255)

        self.LETTER_COLOUR = {"A" : white, "B" : dark_blue, "C" : purple, "D" : cyan, "E" : light_blue, "F" : yellow,
                              "G" : pink, "H" : cyan,
                              "I" : cyan, "J" : pink, "K" : light_blue, "L" : cyan, "M" : yellow, "N" : pink,
                              "O" : purple, "P" : cyan, "Q" : pink,
                              "R" : cyan, "S" : white, "T" : yellow, "U" : light_blue, "V" : pink, "W" : light_blue,
                              "X" : yellow, "Y" : pink, "Z" : pink}

        config = configparser.ConfigParser()
        config.read("wall.ini")

        self.LETTER_LED = {}

        #file = open("leds.txt", "r")
        #file_output = file.readlines()
        options = config.options("LED Numbers")
        for option in options:
            letter = option
            number = config.get("LED Numbers", option)
            self.LETTER_LED[letter] = number

        self.words = []

        #file = open("words.txt", "r")
        #file_output = file.readlines()
        words = config.get("Words", "words").split(",")
        for word in words:
            word = word.upper().rstrip()
            #print(line)
            #print(self.word_is_ok(line))
            print(word)
            if self.word_is_ok(word):
                self.words.append(word)

        #Twitter setup
        consumer_key = config.get("Twitter", "consumer_key")
        consumer_secret = config.get("Twitter", "consumer_secret")
        access_token = config.get("Twitter", "access_token")
        access_token_secret = config.get("Twitter", "access_token_secret")
        self.api = twitter.Api(consumer_key=consumer_key,
                               consumer_secret=consumer_secret,
                               access_token_key=access_token,
                               access_token_secret=access_token_secret,
                               sleep_on_rate_limit=True)

    def run(self):
        inputs = threading.Thread(target=self.get_console_inputs)
        inputs.start()
        while self.running:
            if self.queued_words.empty():
                self.queue_random_word()
            self.display_queued_word()
            self.flicker()

    def get_console_inputs(self):
        while self.inputs_required:
            word = input("> ")
            if self.word_is_ok(word):
                print("Word added to queue")
                self.queued_words.put(word)
            else:
                print("Sorry, not a valid input")

    def get_twitter_inputs(self):
        latest_id = None
        while True:
            results = self.api.GetHomeTimeline(since_id=latest_id)
            for status in results:
                latest_id = 0
                word = status["text"]
                if self.word_is_ok(word):
                    print("Word added to queue")
                    self.queued_words.put(word)
                else:
                    print("Sorry, not a valid input")
                latest_id = max(latest_id, status["id"])
            time.sleep(60)

    def word_is_ok(self, word):
        word = word.upper()
        #print(len(word) < self.MAX_WORD_LENGTH)
        #print(set(word).issubset(self.ALLOWED_CHARACTERS))
        return set(word).issubset(self.ALLOWED_CHARACTERS) and len(word) < self.MAX_WORD_LENGTH

    def light_letter(self, letter):
        led = self.LETTER_LED.get(letter)
        colour = self.LETTER_COLOUR.get(letter)
        self.lights.activate(led, colour)

    def display_word(self, word):
        letters = list(word.upper())
        for letter in letters:
            if not(letter == " "):
                self.light_letter(letter)
                self.lights.show()
            time.sleep(1)
            self.lights.turn_all_off()
            time.sleep(0.5)

    def display_queued_word(self):
        word = self.queued_words.get()
        self.display_word(word)

    def queue_random_word(self):
        self.queued_words.put(random.choice(self.words))

    def flicker(self, repetitions=10):
        for i in range(repetitions):
            self.turn_letters_on()
            time.sleep(0.2)
            self.lights.turn_all_off()
            time.sleep(0.2)

    def turn_letters_on(self):
        for letter in self.LETTER_LED:
            self.light_letter(letter)
        self.lights.show()

    def turn_all_off(self):
        self.lights.turn_all_off()

def main():
    try:
        lights = lc.LEDString(count=50)
        wall = Wall(lights)
        wall.run()
    finally:
        wall.running = False
        wall.inputs_required = False
        wall.turn_all_off()

if __name__ == "__main__":
    main()<|endoftext|>


<commit_before>#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2014-2016  Wojtek Porczyk <woju@invisiblethingslab.com>
# Copyright (C) 2016       Marek Marczykowski <marmarek@invisiblethingslab.com>)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, see <https://www.gnu.org/licenses/>.
#

''' A disposable vm implementation '''

import asyncio

import qubes.vm.qubesvm
import qubes.vm.appvm
import qubes.config

def _setter_template(self, prop, value):
    if not getattr(value, 'template_for_dispvms', False):
        raise qubes.exc.QubesPropertyValueError(self, prop, value,
            'template for DispVM must have template_for_dispvms=True')
    return value

class DispVM(qubes.vm.qubesvm.QubesVM):
    '''Disposable VM'''

    template = qubes.VMProperty('template',
                                load_stage=4,
                                setter=_setter_template,
                                doc='AppVM, on which this DispVM is based.')

    dispid = qubes.property('dispid', type=int, write_once=True,
        clone=False,
        doc='''Internal, persistent identifier of particular DispVM.''')

    auto_cleanup = qubes.property('auto_cleanup', type=bool, default=False,
        doc='automatically remove this VM upon shutdown')

    include_in_backups = qubes.property('include_in_backups', type=bool,
        default=(lambda self: not self.auto_cleanup),
        doc='If this domain is to be included in default backup.')

    default_dispvm = qubes.VMProperty('default_dispvm',
        load_stage=4,
        allow_none=True,
        default=(lambda self: self.template),
        doc='Default VM to be used as Disposable VM for service calls.')

    def __init__(self, app, xml, *args, **kwargs):
        self.volume_config = {
            'root': {
                'name': 'root',
                'snap_on_start': True,
                'save_on_stop': False,
                'rw': True,
                'source': None,
            },
            'private': {
                'name': 'private',
                'snap_on_start': True,
                'save_on_stop': False,
                'rw': True,
                'source': None,
            },
            'volatile': {
                'name': 'volatile',
                'snap_on_start': False,
                'save_on_stop': False,
                'rw': True,
                'size': qubes.config.defaults['root_img_size'] +
                        qubes.config.defaults['private_img_size'],
            },
            'kernel': {
                'name': 'kernel',
                'snap_on_start': False,
                'save_on_stop': False,
                'rw': False,
            }
        }

        template = kwargs.get('template', None)

        if xml is None:
            assert template is not None

            if not getattr(template, 'template_for_dispvms', False):
                raise qubes.exc.QubesValueError(
                    'template for DispVM ({}) needs to be an AppVM with '
                    'template_for_dispvms=True'.format(template.name))

            if 'dispid' not in kwargs:
                kwargs['dispid'] = app.domains.get_new_unused_dispid()
            if 'name' not in kwargs:
                kwargs['name'] = 'disp' + str(kwargs['dispid'])

        if template is not None:
            # template is only passed if the AppVM is created, in other cases we
            # don't need to patch the volume_config because the config is
            # coming from XML, already as we need it
            for name, config in template.volume_config.items():
                # in case the template vm has more volumes add them to own
                # config
                if name not in self.volume_config:
                    self.volume_config[name] = config.copy()
                    if 'vid' in self.volume_config[name]:
                        del self.volume_config[name]['vid']
                # copy pool setting from base AppVM; root and private would be
                # in the same pool anyway (because of snap_on_start),
                # but not volatile, which could be surprising
                elif 'pool' not in self.volume_config[name] \
                        and 'pool' in config:
                    self.volume_config[name]['pool'] = config['pool']

        super().__init__(app, xml, *args, **kwargs)

        if xml is None:
            # by default inherit properties from the DispVM template
            proplist = [prop.__name__ for prop in template.property_list()
                if prop.clone and prop.__name__ not in ['template']]
            # Do not overwrite properties that have already been set to a
            # non-default value.
            self_props = [prop.__name__ for prop in self.property_list()
                if self.property_is_default(prop)]
            self.clone_properties(template, set(proplist).intersection(
                self_props))

            self.firewall.clone(template.firewall)
            self.features.update(template.features)
            self.tags.update(template.tags)

    @qubes.events.handler('domain-load')
    def on_domain_loaded(self, event):
        ''' When domain is loaded assert that this vm has a template.
        '''  # pylint: disable=unused-argument
        assert self.template

    @qubes.events.handler('property-pre-reset:template')
    def on_property_pre_reset_template(self, event, name, oldvalue=None):
        '''Forbid deleting template of running VM
        '''  # pylint: disable=unused-argument,no-self-use
        raise qubes.exc.QubesValueError('Cannot unset template')

    @qubes.events.handler('property-pre-set:template')
    def on_property_pre_set_template(self, event, name, newvalue,
            oldvalue=None):
        '''Forbid changing template of running VM
        '''  # pylint: disable=unused-argument
        if not self.is_halted():
            raise qubes.exc.QubesVMNotHaltedError(self,
                'Cannot change template while qube is running')

    @qubes.events.handler('property-set:template')
    def on_property_set_template(self, event, name, newvalue, oldvalue=None):
        ''' Adjust root (and possibly other snap_on_start=True) volume
        on template change.
        '''  # pylint: disable=unused-argument
        qubes.vm.appvm.template_changed_update_storage(self, DispVM.default_volume_config)

    @qubes.events.handler('domain-shutdown')
    @asyncio.coroutine
    def on_domain_shutdown(self, _event, **_kwargs):
        yield from self._auto_cleanup()

    @asyncio.coroutine
    def _auto_cleanup(self):
        '''Do auto cleanup if enabled'''
        if self.auto_cleanup and self in self.app.domains:
            del self.app.domains[self]
            yield from self.remove_from_disk()
            self.app.save()

    @classmethod
    @asyncio.coroutine
    def from_appvm(cls, appvm, **kwargs):
        '''Create a new instance from given AppVM

        :param qubes.vm.appvm.AppVM appvm: template from which the VM should \
            be created
        :returns: new disposable vm

        *kwargs* are passed to the newly created VM

        >>> import qubes.vm.dispvm.DispVM
        >>> dispvm = qubes.vm.dispvm.DispVM.from_appvm(appvm).start()
        >>> dispvm.run_service('qubes.VMShell', input='firefox')
        >>> dispvm.cleanup()

        This method modifies :file:`qubes.xml` file.
        The qube returned is not started.
        '''
        if not appvm.template_for_dispvms:
            raise qubes.exc.QubesException(
                'Refusing to create DispVM out of this AppVM, because '
                'template_for_dispvms=False')
        app = appvm.app
        dispvm = app.add_new_vm(
            cls,
            template=appvm,
            auto_cleanup=True,
            **kwargs)
        yield from dispvm.create_on_disk()
        app.save()
        return dispvm

    @asyncio.coroutine
    def cleanup(self):
        '''Clean up after the DispVM

        This stops the disposable qube and removes it from the store.
        This method modifies :file:`qubes.xml` file.
        '''
        try:
            # pylint: disable=not-an-iterable
            yield from self.kill()
        except qubes.exc.QubesVMNotStartedError:
            pass
        # if auto_cleanup is set, this will be done automatically
        if not self.auto_cleanup:
            del self.app.domains[self]
            yield from self.remove_from_disk()
            self.app.save()

    @asyncio.coroutine
    def start(self, **kwargs):
        # pylint: disable=arguments-differ

        try:
            # sanity check, if template_for_dispvm got changed in the meantime
            if not self.template.template_for_dispvms:
                raise qubes.exc.QubesException(
                    'template for DispVM ({}) needs to have '
                    'template_for_dispvms=True'.format(self.template.name))

            yield from super().start(**kwargs)
        except:
            # Cleanup also on failed startup
            yield from self._auto_cleanup()
            raise

    def create_qdb_entries(self):
        super().create_qdb_entries()
        self.untrusted_qdb.write('/qubes-vm-persistence', 'none')
<commit_msg>DispVM has no “default_volume_config” property<commit_after>#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2014-2016  Wojtek Porczyk <woju@invisiblethingslab.com>
# Copyright (C) 2016       Marek Marczykowski <marmarek@invisiblethingslab.com>)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, see <https://www.gnu.org/licenses/>.
#

''' A disposable vm implementation '''

import asyncio

import qubes.vm.qubesvm
import qubes.vm.appvm
import qubes.config

def _setter_template(self, prop, value):
    if not getattr(value, 'template_for_dispvms', False):
        raise qubes.exc.QubesPropertyValueError(self, prop, value,
            'template for DispVM must have template_for_dispvms=True')
    return value

class DispVM(qubes.vm.qubesvm.QubesVM):
    '''Disposable VM'''

    template = qubes.VMProperty('template',
                                load_stage=4,
                                setter=_setter_template,
                                doc='AppVM, on which this DispVM is based.')

    dispid = qubes.property('dispid', type=int, write_once=True,
        clone=False,
        doc='''Internal, persistent identifier of particular DispVM.''')

    auto_cleanup = qubes.property('auto_cleanup', type=bool, default=False,
        doc='automatically remove this VM upon shutdown')

    include_in_backups = qubes.property('include_in_backups', type=bool,
        default=(lambda self: not self.auto_cleanup),
        doc='If this domain is to be included in default backup.')

    default_dispvm = qubes.VMProperty('default_dispvm',
        load_stage=4,
        allow_none=True,
        default=(lambda self: self.template),
        doc='Default VM to be used as Disposable VM for service calls.')

    def __init__(self, app, xml, *args, **kwargs):
        self.volume_config = {
            'root': {
                'name': 'root',
                'snap_on_start': True,
                'save_on_stop': False,
                'rw': True,
                'source': None,
            },
            'private': {
                'name': 'private',
                'snap_on_start': True,
                'save_on_stop': False,
                'rw': True,
                'source': None,
            },
            'volatile': {
                'name': 'volatile',
                'snap_on_start': False,
                'save_on_stop': False,
                'rw': True,
                'size': qubes.config.defaults['root_img_size'] +
                        qubes.config.defaults['private_img_size'],
            },
            'kernel': {
                'name': 'kernel',
                'snap_on_start': False,
                'save_on_stop': False,
                'rw': False,
            }
        }

        template = kwargs.get('template', None)

        if xml is None:
            assert template is not None

            if not getattr(template, 'template_for_dispvms', False):
                raise qubes.exc.QubesValueError(
                    'template for DispVM ({}) needs to be an AppVM with '
                    'template_for_dispvms=True'.format(template.name))

            if 'dispid' not in kwargs:
                kwargs['dispid'] = app.domains.get_new_unused_dispid()
            if 'name' not in kwargs:
                kwargs['name'] = 'disp' + str(kwargs['dispid'])

        if template is not None:
            # template is only passed if the AppVM is created, in other cases we
            # don't need to patch the volume_config because the config is
            # coming from XML, already as we need it
            for name, config in template.volume_config.items():
                # in case the template vm has more volumes add them to own
                # config
                if name not in self.volume_config:
                    self.volume_config[name] = config.copy()
                    if 'vid' in self.volume_config[name]:
                        del self.volume_config[name]['vid']
                # copy pool setting from base AppVM; root and private would be
                # in the same pool anyway (because of snap_on_start),
                # but not volatile, which could be surprising
                elif 'pool' not in self.volume_config[name] \
                        and 'pool' in config:
                    self.volume_config[name]['pool'] = config['pool']

        super().__init__(app, xml, *args, **kwargs)

        if xml is None:
            # by default inherit properties from the DispVM template
            proplist = [prop.__name__ for prop in template.property_list()
                if prop.clone and prop.__name__ not in ['template']]
            # Do not overwrite properties that have already been set to a
            # non-default value.
            self_props = [prop.__name__ for prop in self.property_list()
                if self.property_is_default(prop)]
            self.clone_properties(template, set(proplist).intersection(
                self_props))

            self.firewall.clone(template.firewall)
            self.features.update(template.features)
            self.tags.update(template.tags)

    @qubes.events.handler('domain-load')
    def on_domain_loaded(self, event):
        ''' When domain is loaded assert that this vm has a template.
        '''  # pylint: disable=unused-argument
        assert self.template

    @qubes.events.handler('property-pre-reset:template')
    def on_property_pre_reset_template(self, event, name, oldvalue=None):
        '''Forbid deleting template of running VM
        '''  # pylint: disable=unused-argument,no-self-use
        raise qubes.exc.QubesValueError('Cannot unset template')

    @qubes.events.handler('property-pre-set:template')
    def on_property_pre_set_template(self, event, name, newvalue,
            oldvalue=None):
        '''Forbid changing template of running VM
        '''  # pylint: disable=unused-argument
        if not self.is_halted():
            raise qubes.exc.QubesVMNotHaltedError(self,
                'Cannot change template while qube is running')

    @qubes.events.handler('property-set:template')
    def on_property_set_template(self, event, name, newvalue, oldvalue=None):
        ''' Adjust root (and possibly other snap_on_start=True) volume
        on template change.
        '''  # pylint: disable=unused-argument
        qubes.vm.appvm.template_changed_update_storage(self, self.volume_config)

    @qubes.events.handler('domain-shutdown')
    @asyncio.coroutine
    def on_domain_shutdown(self, _event, **_kwargs):
        yield from self._auto_cleanup()

    @asyncio.coroutine
    def _auto_cleanup(self):
        '''Do auto cleanup if enabled'''
        if self.auto_cleanup and self in self.app.domains:
            del self.app.domains[self]
            yield from self.remove_from_disk()
            self.app.save()

    @classmethod
    @asyncio.coroutine
    def from_appvm(cls, appvm, **kwargs):
        '''Create a new instance from given AppVM

        :param qubes.vm.appvm.AppVM appvm: template from which the VM should \
            be created
        :returns: new disposable vm

        *kwargs* are passed to the newly created VM

        >>> import qubes.vm.dispvm.DispVM
        >>> dispvm = qubes.vm.dispvm.DispVM.from_appvm(appvm).start()
        >>> dispvm.run_service('qubes.VMShell', input='firefox')
        >>> dispvm.cleanup()

        This method modifies :file:`qubes.xml` file.
        The qube returned is not started.
        '''
        if not appvm.template_for_dispvms:
            raise qubes.exc.QubesException(
                'Refusing to create DispVM out of this AppVM, because '
                'template_for_dispvms=False')
        app = appvm.app
        dispvm = app.add_new_vm(
            cls,
            template=appvm,
            auto_cleanup=True,
            **kwargs)
        yield from dispvm.create_on_disk()
        app.save()
        return dispvm

    @asyncio.coroutine
    def cleanup(self):
        '''Clean up after the DispVM

        This stops the disposable qube and removes it from the store.
        This method modifies :file:`qubes.xml` file.
        '''
        try:
            # pylint: disable=not-an-iterable
            yield from self.kill()
        except qubes.exc.QubesVMNotStartedError:
            pass
        # if auto_cleanup is set, this will be done automatically
        if not self.auto_cleanup:
            del self.app.domains[self]
            yield from self.remove_from_disk()
            self.app.save()

    @asyncio.coroutine
    def start(self, **kwargs):
        # pylint: disable=arguments-differ

        try:
            # sanity check, if template_for_dispvm got changed in the meantime
            if not self.template.template_for_dispvms:
                raise qubes.exc.QubesException(
                    'template for DispVM ({}) needs to have '
                    'template_for_dispvms=True'.format(self.template.name))

            yield from super().start(**kwargs)
        except:
            # Cleanup also on failed startup
            yield from self._auto_cleanup()
            raise

    def create_qdb_entries(self):
        super().create_qdb_entries()
        self.untrusted_qdb.write('/qubes-vm-persistence', 'none')
<|endoftext|>


<commit_before>#!/usr/bin/python2 -O
# vim: fileencoding=utf-8
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2014-2016  Wojtek Porczyk <woju@invisiblethingslab.com>
# Copyright (C) 2016       Marek Marczykowski <marmarek@invisiblethingslab.com>)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#

''' A disposable vm implementation '''
import copy

import qubes.vm.qubesvm
import qubes.vm.appvm
import qubes.config

class DispVM(qubes.vm.qubesvm.QubesVM):
    '''Disposable VM'''

    template = qubes.VMProperty('template',
                                load_stage=4,
                                vmclass=qubes.vm.appvm.AppVM,
                                ls_width=31,
                                doc='AppVM, on which this DispVM is based.')

    dispid = qubes.property('dispid', type=int, write_once=True,
        clone=False,
        ls_width=3,
        doc='''Internal, persistent identifier of particular DispVM.''')

    def __init__(self, *args, **kwargs):
        self.volume_config = {
            'root': {
                'name': 'root',
                'pool': 'default',
                'snap_on_start': True,
                'save_on_stop': False,
                'rw': False,
                'internal': True
            },
            'private': {
                'name': 'private',
                'pool': 'default',
                'snap_on_start': True,
                'save_on_stop': False,
                'internal': True,
                'rw': True,
            },
            'volatile': {
                'name': 'volatile',
                'pool': 'default',
                'internal': True,
                'rw': True,
                'size': qubes.config.defaults['root_img_size'] +
                        qubes.config.defaults['private_img_size'],
            },
            'kernel': {
                'name': 'kernel',
                'pool': 'linux-kernel',
                'snap_on_start': True,
                'rw': False,
                'internal': True
            }
        }
        if 'name' not in kwargs and 'dispid' in kwargs:
            kwargs['name'] = 'disp' + str(kwargs['dispid'])
        template = kwargs.get('template', None)

        if template is not None:
            # template is only passed if the AppVM is created, in other cases we
            # don't need to patch the volume_config because the config is
            # coming from XML, already as we need it

            for name, conf in self.volume_config.items():
                tpl_volume = template.volumes[name]

                conf['size'] = tpl_volume.size
                conf['pool'] = tpl_volume.pool

                has_source = ('source' in conf and conf['source'] is not None)
                is_snapshot = 'snap_on_start' in conf and conf['snap_on_start']
                if is_snapshot and not has_source:
                    if tpl_volume.source is not None:
                        conf['source'] = tpl_volume.source
                    else:
                        conf['source'] = tpl_volume.vid

            for name, config in template.volume_config.items():
                # in case the template vm has more volumes add them to own
                # config
                if name not in self.volume_config:
                    self.volume_config[name] = copy.deepcopy(config)
                    if 'vid' in self.volume_config[name]:
                        del self.volume_config[name]['vid']

            # by default inherit label from the DispVM template
            if 'label' not in kwargs:
                kwargs['label'] = template.label

        super(DispVM, self).__init__(*args, **kwargs)

    @qubes.events.handler('domain-load')
    def on_domain_loaded(self, event):
        ''' When domain is loaded assert that this vm has a template.
        '''  # pylint: disable=unused-argument
        assert self.template

    @classmethod
    def from_appvm(cls, appvm, **kwargs):
        '''Create a new instance from given AppVM

        :param qubes.vm.appvm.AppVM appvm: template from which the VM should \
            be created (could also be name or qid)
        :returns: new disposable vm

        *kwargs* are passed to the newly created VM

        >>> import qubes.vm.dispvm.DispVM
        >>> dispvm = qubes.vm.dispvm.DispVM.from_appvm(appvm).start()
        >>> dispvm.run_service('qubes.VMShell', input='firefox')
        >>> dispvm.cleanup()

        This method modifies :file:`qubes.xml` file. In fact, the newly created
        vm belongs to other :py:class:`qubes.Qubes` instance than the *app*.
        The qube returned is not started.
        '''
        store = appvm.app.store if isinstance(appvm, qubes.vm.BaseVM) else None
        app = qubes.Qubes(store)
        dispvm = app.add_new_vm(
            cls,
            dispid=app.domains.get_new_unused_dispid(),
            template=app.domains[appvm],
            **kwargs)
        dispvm.clone_properties(app.domains[appvm])
        dispvm.create_on_disk()
        app.save()
        return dispvm

    def cleanup(self):
        '''Clean up after the DispVM

        This stops the disposable qube and removes it from the store.

        This method modifies :file:`qubes.xml` file.
        '''
        app = qubes.Qubes(self.app.store)
        self = app.domains[self.uuid]
        self.force_shutdown()
        self.remove_from_disk()
        del app.domains[self]
        app.save()
<commit_msg>qubes/dispvm: do not clone 'template' property from base VM<commit_after>#!/usr/bin/python2 -O
# vim: fileencoding=utf-8
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2014-2016  Wojtek Porczyk <woju@invisiblethingslab.com>
# Copyright (C) 2016       Marek Marczykowski <marmarek@invisiblethingslab.com>)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#

''' A disposable vm implementation '''
import copy

import qubes.vm.qubesvm
import qubes.vm.appvm
import qubes.config

class DispVM(qubes.vm.qubesvm.QubesVM):
    '''Disposable VM'''

    template = qubes.VMProperty('template',
                                load_stage=4,
                                vmclass=qubes.vm.appvm.AppVM,
                                ls_width=31,
                                doc='AppVM, on which this DispVM is based.')

    dispid = qubes.property('dispid', type=int, write_once=True,
        clone=False,
        ls_width=3,
        doc='''Internal, persistent identifier of particular DispVM.''')

    def __init__(self, *args, **kwargs):
        self.volume_config = {
            'root': {
                'name': 'root',
                'pool': 'default',
                'snap_on_start': True,
                'save_on_stop': False,
                'rw': False,
                'internal': True
            },
            'private': {
                'name': 'private',
                'pool': 'default',
                'snap_on_start': True,
                'save_on_stop': False,
                'internal': True,
                'rw': True,
            },
            'volatile': {
                'name': 'volatile',
                'pool': 'default',
                'internal': True,
                'rw': True,
                'size': qubes.config.defaults['root_img_size'] +
                        qubes.config.defaults['private_img_size'],
            },
            'kernel': {
                'name': 'kernel',
                'pool': 'linux-kernel',
                'snap_on_start': True,
                'rw': False,
                'internal': True
            }
        }
        if 'name' not in kwargs and 'dispid' in kwargs:
            kwargs['name'] = 'disp' + str(kwargs['dispid'])
        template = kwargs.get('template', None)

        if template is not None:
            # template is only passed if the AppVM is created, in other cases we
            # don't need to patch the volume_config because the config is
            # coming from XML, already as we need it

            for name, conf in self.volume_config.items():
                tpl_volume = template.volumes[name]

                conf['size'] = tpl_volume.size
                conf['pool'] = tpl_volume.pool

                has_source = ('source' in conf and conf['source'] is not None)
                is_snapshot = 'snap_on_start' in conf and conf['snap_on_start']
                if is_snapshot and not has_source:
                    if tpl_volume.source is not None:
                        conf['source'] = tpl_volume.source
                    else:
                        conf['source'] = tpl_volume.vid

            for name, config in template.volume_config.items():
                # in case the template vm has more volumes add them to own
                # config
                if name not in self.volume_config:
                    self.volume_config[name] = copy.deepcopy(config)
                    if 'vid' in self.volume_config[name]:
                        del self.volume_config[name]['vid']

            # by default inherit label from the DispVM template
            if 'label' not in kwargs:
                kwargs['label'] = template.label

        super(DispVM, self).__init__(*args, **kwargs)

    @qubes.events.handler('domain-load')
    def on_domain_loaded(self, event):
        ''' When domain is loaded assert that this vm has a template.
        '''  # pylint: disable=unused-argument
        assert self.template

    @classmethod
    def from_appvm(cls, appvm, **kwargs):
        '''Create a new instance from given AppVM

        :param qubes.vm.appvm.AppVM appvm: template from which the VM should \
            be created (could also be name or qid)
        :returns: new disposable vm

        *kwargs* are passed to the newly created VM

        >>> import qubes.vm.dispvm.DispVM
        >>> dispvm = qubes.vm.dispvm.DispVM.from_appvm(appvm).start()
        >>> dispvm.run_service('qubes.VMShell', input='firefox')
        >>> dispvm.cleanup()

        This method modifies :file:`qubes.xml` file. In fact, the newly created
        vm belongs to other :py:class:`qubes.Qubes` instance than the *app*.
        The qube returned is not started.
        '''
        store = appvm.app.store if isinstance(appvm, qubes.vm.BaseVM) else None
        app = qubes.Qubes(store)
        dispvm = app.add_new_vm(
            cls,
            dispid=app.domains.get_new_unused_dispid(),
            template=app.domains[appvm],
            **kwargs)
        # exclude template
        proplist = [prop for prop in dispvm.property_list()
            if prop.clone and prop.__name__ not in ['template']]
        dispvm.clone_properties(app.domains[appvm], proplist=proplist)
        dispvm.create_on_disk()
        app.save()
        return dispvm

    def cleanup(self):
        '''Clean up after the DispVM

        This stops the disposable qube and removes it from the store.

        This method modifies :file:`qubes.xml` file.
        '''
        app = qubes.Qubes(self.app.store)
        self = app.domains[self.uuid]
        self.force_shutdown()
        self.remove_from_disk()
        del app.domains[self]
        app.save()
<|endoftext|>


<commit_before>"""
PyAlge

Provides pattern matching through the "Case" class and (minimal support of)
algebraic data type (ADT).

ADT are implemented as namedtuple.

"""

from __future__ import print_function, absolute_import
from collections import namedtuple
import functools
import tokenize

class MissingCaseError(ValueError):
    pass

class PatternSyntaxError(ValueError):
    pass

class NoMatch(object):
    """Used internally to indicate where a case does not match.
    """
    pass

class Case(object):
    """A case-class is used to describe a pattern matching dispatch logic.
    Each pattern is described as a method with the `of` decorator.

    Details
    -------
    The `__new__` method of Case is overridden to not return an instance of
    Case.  One will use a subclass of Case as a function.
    """

    def __new__(cls, value, state=None):
        """
        Args
        ----
        value:
            object being matched
        state:
            mutable internal state accessible as `self.state`
        """
        if not hasattr(cls, "_case_ofs"):
            # First time running.
            # Prepare dispatch table.
            cls.__prepare()
        obj = object.__new__(cls)
        obj.state = state
        return obj.__process(value)

    @classmethod
    def __prepare(cls):
        """Insert "_case_ofs" attribute to the class
        """
        ofs = []
        for fd in dir(cls):
            if not fd.startswith('_') and fd != 'otherwise':
                fn = getattr(cls, fd)
                firstline = fn._inner.func_code.co_firstlineno
                ofs.append((firstline, fn))
                # Order cases by lineno
        cls._case_ofs = zip(*sorted(ofs))[1]

    def __process(self, value, state=None):
        """The actual matching/dispatch.
        Returns the result of the match.
        """
        ofs = self._case_ofs
        for case in ofs:
            res = case(self, value)
            if res is not NoMatch:
                # Matches
                return res
                # Run default
        return self.otherwise(value)

    def otherwise(self, value):
        """Default is to raise MissingCaseError exception with `value` as
        argument.

        Can be overridden.
        """
        raise MissingCaseError(value)

class _TypePattern(namedtuple("_TypePattern", ["name", "body"])):
    def match(self, match, input):
        if type(input).__name__ == self.name:
            if len(self.body) != len(input):
                # Insufficient fields
                return
            # Try to match the fields
            for field, fdvalue in zip(self.body, input):
                m = field.match(_Match(), fdvalue)
                if m is not None:
                    match.bindings.update(m.bindings)
                else:
                    return
            return match

class _Binding(namedtuple("Binding", ["name"])):
    def match(self, match, input):
        assert self.name not in match.bindings, "duplicated binding"
        match.bindings[self.name] = input
        return match

class _Ignored(object):
    def match(self, match, input):
        return match

class _PatternParser(object):
    """Pattern parser

    Overhead of the library will almost likely be in parsing.  Luckily,
    we parse once at class creation.
    """

    def __init__(self, pat):
        self.pat = pat
        self.bindings = set()

        lines = iter(self.pat.split())
        tokens = tokenize.generate_tokens(lambda: next(lines))
        self._nexttoken = lambda: next(tokens)
        self.pbtok = None
        self.result = None

    def get_token(self):
        if self.pbtok:
            token, self.pbtok = self.pbtok, None
        else:
            token = self._nexttoken()
        return token

    def putback(self, token):
        assert self.pbtok is None, "internal error: lookahead(1)"
        self.pbtok = token

    def parse(self):
        typename = self.expect_name()
        self.result = self.parse_typebody(typename)

    def parse_typebody(self, typename):
        if not typename[0].isupper():
            raise PatternSyntaxError("type name must start with uppercase "
                                     "letter")
        self.expect_lparen()
        body = []
        while True:
            body.append(self.expect_type_or_binding())
            if not self.is_comma():
                break
        self.expect_rparen()
        return _TypePattern(typename, body)

    def expect_type_or_binding(self):
        name = self.expect_name()
        if name[0].isupper():
            # is type
            return self.parse_typebody(name)
        else:
            # is binding
            return self.parse_binding(name)

    def parse_binding(self, name):
        ignored = name[0].startswith('_')
        if not name[0].islower() and not ignored:
            raise PatternSyntaxError("binding name must start with lowercase "
                                     "letter")
        if ignored:
            return _Ignored()
        else:
            return _Binding(name)

    def expect_name(self):
        token = self.get_token()
        tokty, tokstr = token[:2]
        if tokty != tokenize.NAME:
            raise PatternSyntaxError("expected name: %r" % tokstr)
        return tokstr

    def expect_lparen(self):
        token = self.get_token()
        tokstr = token[1]
        if tokstr != '(':
            raise PatternSyntaxError("expected '('; got %r" % tokstr)

    def expect_rparen(self):
        token = self.get_token()
        tokstr = token[1]
        if tokstr != ')':
            raise PatternSyntaxError("expected ')'; got %r" % tokstr)

    def is_comma(self):
        token = self.get_token()
        tokstr = token[1]
        if tokstr != ',':
            self.putback(token)
            return False
        else:
            return True

class _Match(object):
    def __init__(self):
        self.bindings = {}

def of(pat):
    """Decorator for methods of Case to describe the pattern.

    Args
    ----
    pat: str

    Patterns are like writing tuples (of tuples (of ...)) for the type
    structure to match against.  Names starting with a lowercase letter are
    used as binding slots that the matcher will capture and used as argument
    to the action function, the function being decorated.  Names that starts
    with a underscore '_' is ignored.  Names starting with a uppercase letter
    are type names.
    """
    parser = _PatternParser(pat)
    parser.parse()
    matcher = parser.result

    def decor(fn):
        @functools.wraps(fn)
        def closure(self, value):
            match = matcher.match(_Match(), value)
            if match is not None:
                self.value = value
                self.match = match
                return fn(self, **match.bindings)
            else:
                return NoMatch

        closure._inner = fn
        return closure

    return decor

# TODO: it seems we don't really need this
class Data(object):
    """Algebraic Data Type
    """
    pass

def datatype(name, fields):
    """Constructor for algebraic data type.
    Used like `collections.namedtuple`.

    Args
    ----
    name: str
        Type name.  Must start with a uppercase letter.
    fields: sequence of str
        Sequence of field names.
    """
    assert name[0].isupper(), "Type name must start with uppercase letter."
    tupcls = namedtuple(name, fields)
    return type(name, (tupcls, Data), {})

<commit_msg>Match type by isinstance() instead of by name so that user can alias typenames<commit_after>"""
PyAlge

Provides pattern matching through the "Case" class and (minimal support of)
algebraic data type (ADT).

ADT are implemented as namedtuple.

"""

from __future__ import print_function, absolute_import
from collections import namedtuple
import functools
import tokenize
import inspect

class MissingCaseError(ValueError):
    pass

class PatternSyntaxError(ValueError):
    pass

class PatternContextError(ValueError):
    pass

class NoMatch(object):
    """Used internally to indicate where a case does not match.
    """
    pass

class Case(object):
    """A case-class is used to describe a pattern matching dispatch logic.
    Each pattern is described as a method with the `of` decorator.

    Details
    -------
    The `__new__` method of Case is overridden to not return an instance of
    Case.  One will use a subclass of Case as a function.
    """

    def __new__(cls, value, state=None):
        """
        Args
        ----
        value:
            object being matched
        state:
            mutable internal state accessible as `self.state`
        """
        if not hasattr(cls, "_case_ofs"):
            # First time running.
            # Prepare dispatch table.
            cls.__prepare()
        obj = object.__new__(cls)
        obj.state = state
        return obj.__process(value)

    @classmethod
    def __prepare(cls):
        """Insert "_case_ofs" attribute to the class
        """
        ofs = []
        for fd in dir(cls):
            if not fd.startswith('_') and fd != 'otherwise':
                fn = getattr(cls, fd)
                firstline = fn._inner.func_code.co_firstlineno
                ofs.append((firstline, fn))
                # Order cases by lineno
        cls._case_ofs = zip(*sorted(ofs))[1]

    def __process(self, value, state=None):
        """The actual matching/dispatch.
        Returns the result of the match.
        """
        ofs = self._case_ofs
        for case in ofs:
            res = case(self, value)
            if res is not NoMatch:
                # Matches
                return res
                # Run default
        return self.otherwise(value)

    def otherwise(self, value):
        """Default is to raise MissingCaseError exception with `value` as
        argument.

        Can be overridden.
        """
        raise MissingCaseError(value)

class _TypePattern(namedtuple("_TypePattern", ["typ", "body"])):
    def match(self, match, input):
        if isinstance(input, self.typ):
            if len(self.body) != len(input):
                # Insufficient fields
                return
            # Try to match the fields
            for field, fdvalue in zip(self.body, input):
                m = field.match(_Match(), fdvalue)
                if m is not None:
                    match.bindings.update(m.bindings)
                else:
                    return
            return match

class _Binding(namedtuple("Binding", ["name"])):
    def match(self, match, input):
        assert self.name not in match.bindings, "duplicated binding"
        match.bindings[self.name] = input
        return match

class _Ignored(object):
    def match(self, match, input):
        return match

class _PatternParser(object):
    """Pattern parser

    Overhead of the library will almost likely be in parsing.  Luckily,
    we parse once at class creation.
    """

    def __init__(self, pat, env):
        self.pat = pat
        self.bindings = set()
        self.env = env

        lines = iter(self.pat.split())
        tokens = tokenize.generate_tokens(lambda: next(lines))
        self._nexttoken = lambda: next(tokens)
        self.pbtok = None
        self.result = None

    def get_token(self):
        if self.pbtok:
            token, self.pbtok = self.pbtok, None
        else:
            token = self._nexttoken()
        return token

    def putback(self, token):
        assert self.pbtok is None, "internal error: lookahead(1)"
        self.pbtok = token

    def parse(self):
        typename = self.expect_name()
        self.result = self.parse_typebody(typename)

    def parse_typebody(self, typename):
        if not typename[0].isupper():
            raise PatternSyntaxError("type name must start with uppercase "
                                     "letter")
        self.expect_lparen()
        body = []
        while True:
            body.append(self.expect_type_or_binding())
            if not self.is_comma():
                break
        self.expect_rparen()

        try:
            typcls = self.env[typename]
        except KeyError:
            raise PatternContextError("unknown type reference: %s" % typename)
        else:
            return _TypePattern(typcls, body)

    def expect_type_or_binding(self):
        name = self.expect_name()
        if name[0].isupper():
            # is type
            return self.parse_typebody(name)
        else:
            # is binding
            return self.parse_binding(name)

    def parse_binding(self, name):
        ignored = name[0].startswith('_')
        if not name[0].islower() and not ignored:
            raise PatternSyntaxError("binding name must start with lowercase "
                                     "letter")
        if ignored:
            return _Ignored()
        else:
            return _Binding(name)

    def expect_name(self):
        token = self.get_token()
        tokty, tokstr = token[:2]
        if tokty != tokenize.NAME:
            raise PatternSyntaxError("expected name: %r" % tokstr)
        return tokstr

    def expect_lparen(self):
        token = self.get_token()
        tokstr = token[1]
        if tokstr != '(':
            raise PatternSyntaxError("expected '('; got %r" % tokstr)

    def expect_rparen(self):
        token = self.get_token()
        tokstr = token[1]
        if tokstr != ')':
            raise PatternSyntaxError("expected ')'; got %r" % tokstr)

    def is_comma(self):
        token = self.get_token()
        tokstr = token[1]
        if tokstr != ',':
            self.putback(token)
            return False
        else:
            return True

class _Match(object):
    def __init__(self):
        self.bindings = {}

def of(pat):
    """Decorator for methods of Case to describe the pattern.

    Args
    ----
    pat: str

    Patterns are like writing tuples (of tuples (of ...)) for the type
    structure to match against.  Names starting with a lowercase letter are
    used as binding slots that the matcher will capture and used as argument
    to the action function, the function being decorated.  Names that starts
    with a underscore '_' is ignored.  Names starting with a uppercase letter
    are type names.
    """

    # Get globals from caller's frame
    glbls = inspect.currentframe().f_back.f_globals
    # Parse pattern
    parser = _PatternParser(pat, glbls)
    parser.parse()
    matcher = parser.result

    def decor(fn):
        @functools.wraps(fn)
        def closure(self, value):
            match = matcher.match(_Match(), value)
            if match is not None:
                self.value = value
                self.match = match
                return fn(self, **match.bindings)
            else:
                return NoMatch

        closure._inner = fn
        return closure

    return decor

# TODO: it seems we don't really need this
class Data(object):
    """Algebraic Data Type
    """
    pass

def datatype(name, fields):
    """Constructor for algebraic data type.
    Used like `collections.namedtuple`.

    Args
    ----
    name: str
        Type name.  Must start with a uppercase letter.
    fields: sequence of str
        Sequence of field names.
    """
    assert name[0].isupper(), "Type name must start with uppercase letter."
    tupcls = namedtuple(name, fields)
    return type(name, (tupcls, Data), {})

<|endoftext|>


<commit_before>from binascii import a2b_hex, b2a_hex
from datetime import datetime
from socket import socket, AF_INET, SOCK_STREAM
from struct import pack, unpack

try:
    from ssl import wrap_socket
except ImportError:
    from socket import ssl

try:
    import json
except ImportError:
    import simplejson as json

MAX_PAYLOAD_LENGTH = 256

class APNs(object):
    """A class representing an Apple Push Notification service connection"""

    def __init__(self, use_sandbox=False, cert_file=None, key_file=None):
        """
        Set use_sandbox to True to use the sandbox (test) APNs servers.
        Default is False.
        """
        super(APNs, self).__init__()
        self.use_sandbox = use_sandbox
        self.cert_file = cert_file
        self.key_file = key_file
        self._feedback_connection = None
        self._gateway_connection = None

    @staticmethod
    def packed_ushort_big_endian(num):
        """
        Returns an unsigned short in packed big-endian (network) form
        """
        return pack('>H', num)

    @staticmethod
    def unpacked_ushort_big_endian(bytes):
        """
        Returns an unsigned short from a packed big-endian (network) byte
        array
        """
        return unpack('>H', bytes)[0]

    @staticmethod
    def packed_uint_big_endian(num):
        """
        Returns an unsigned int in packed big-endian (network) form
        """
        return pack('>I', num)

    @staticmethod
    def unpacked_uint_big_endian(bytes):
        """
        Returns an unsigned int from a packed big-endian (network) byte array
        """
        return unpack('>I', bytes)[0]

    @property
    def feedback_server(self):
        if not self._feedback_connection:
            self._feedback_connection = FeedbackConnection(
                use_sandbox = self.use_sandbox,
                cert_file = self.cert_file,
                key_file = self.key_file
            )
        return self._feedback_connection

    @property
    def gateway_server(self):
        if not self._gateway_connection:
            self._gateway_connection = GatewayConnection(
                use_sandbox = self.use_sandbox,
                cert_file = self.cert_file,
                key_file = self.key_file
            )
        return self._gateway_connection

class APNsConnection(object):
    """
    A generic connection class for communicating with the APNs
    """
    def __init__(self, cert_file=None, key_file=None):
        super(APNsConnection, self).__init__()
        self.cert_file = cert_file
        self.key_file = key_file
        self._socket = None
        self._ssl = None

    def __del__(self):
        self._disconnect();

    def _connect(self):
        # Establish an SSL connection
        self._socket = socket(AF_INET, SOCK_STREAM)
        self._socket.connect((self.server, self.port))
        if wrap_socket:
            self._ssl = wrap_socket(self._socket, self.key_file, self.cert_file)
        else:
            self._ssl = ssl(self._socket, self.key_file, self.cert_file)

    def _disconnect(self):
        if self._socket:
            self._socket.close()

    def _connection(self):
        if not self._ssl:
            self._connect()
        return self._ssl

    def read(self, n=None):
        return self._connection().read(n)

    def write(self, string):
        return self._connection().write(string)

class PayloadAlert(object):
    def __init__(self, body, action_loc_key=None, loc_key=None,
                 loc_args=None, launch_image=None):
        super(PayloadAlert, self).__init__()
        self.body = body
        self.action_loc_key = action_loc_key
        self.loc_key = loc_key
        self.loc_args = loc_args
        self.launch_image = launch_image

    def dict(self):
        d = { 'body': self.body }
        if self.action_loc_key:
            d['action-loc-key'] = self.action_loc_key
        if self.loc_key:
            d['loc-key'] = self.loc_key
        if self.loc_args:
            d['loc-args'] = self.loc_args
        if self.launch_image:
            d['launch-image'] = self.launch_image
        return d

class PayloadTooLargeError(Exception):
    def __init__(self):
        super(PayloadTooLargeError, self).__init__()

class Payload(object):
    """A class representing an APNs message payload"""
    def __init__(self, alert=None, badge=None, sound=None, custom={}):
        super(Payload, self).__init__()
        self.alert = alert
        self.badge = badge
        self.sound = sound
        self.custom = custom
        self._check_size()

    def dict(self):
        """Returns the payload as a regular Python dictionary"""
        d = {}
        if self.alert:
            # Alert can be either a string or a PayloadAlert
            # object
            if isinstance(self.alert, PayloadAlert):
                d['alert'] = self.alert.dict()
            else:
                d['alert'] = self.alert
        if self.sound:
            d['sound'] = self.sound
        if self.badge:
            d['badge'] = int(self.badge)

        d = { 'aps': d }
        d.update(self.custom)
        return d

    def json(self):
        return json.dumps(self.dict(), separators=(',',':'))

    def _check_size(self):
        if len(self.json()) > MAX_PAYLOAD_LENGTH:
            raise PayloadTooLargeError()

    def __repr__(self):
        attrs = ("alert", "badge", "sound", "custom")
        args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs])
        return "%s(%s)" % (self.__class__.__name__, args)

class FeedbackConnection(APNsConnection):
    """
    A class representing a connection to the APNs Feedback server
    """
    def __init__(self, use_sandbox=False, **kwargs):
        super(FeedbackConnection, self).__init__(**kwargs)
        self.server = (
            'feedback.push.apple.com',
            'feedback.sandbox.push.apple.com')[use_sandbox]
        self.port = 2196

    def _chunks(self):
        BUF_SIZE = 4096
        while 1:
            data = self.read(BUF_SIZE)
            yield data
            if not data:
                break

    def items(self):
        """
        A generator that yields (token_hex, fail_time) pairs retrieved from
        the APNs feedback server
        """
        buff = ''
        for chunk in self._chunks():
            buff += chunk

            # Quit if there's no more data to read
            if not buff:
                break

            # Sanity check: after a socket read we should always have at least
            # 6 bytes in the buffer
            if len(buff) < 6:
                break

            while len(buff) > 6:
                token_length = APNs.unpacked_ushort_big_endian(buff[4:6])
                bytes_to_read = 6 + token_length
                if len(buff) >= bytes_to_read:
                    fail_time_unix = APNs.unpacked_uint_big_endian(buff[0:4])
                    fail_time = datetime.utcfromtimestamp(fail_time_unix)
                    token = b2a_hex(buff[6:bytes_to_read])

                    yield (token, fail_time)

                    # Remove data for current token from buffer
                    buff = buff[bytes_to_read:]
                else:
                    # break out of inner while loop - i.e. go and fetch
                    # some more data and append to buffer
                    break

class GatewayConnection(APNsConnection):
    """
    A class that represents a connection to the APNs gateway server
    """
    def __init__(self, use_sandbox=False, **kwargs):
        super(GatewayConnection, self).__init__(**kwargs)
        self.server = (
            'gateway.push.apple.com',
            'gateway.sandbox.push.apple.com')[use_sandbox]
        self.port = 2195

    def _get_notification(self, token_hex, payload):
        """
        Takes a token as a hex string and a payload as a Python dict and sends
        the notification
        """
        token_bin = a2b_hex(token_hex)
        token_length_bin = APNs.packed_ushort_big_endian(len(token_bin))
        payload_json = payload.json()
        payload_length_bin = APNs.packed_ushort_big_endian(len(payload_json))

        notification = ('\0' + token_length_bin + token_bin
            + payload_length_bin + payload_json)

        return notification

    def send_notification(self, token_hex, payload):
        self.write(self._get_notification(token_hex, payload))

<commit_msg>Added MIT license text to top of apns.py<commit_after># PyAPNs was developed by Simon Whitaker <simon@goosoftware.co.uk>
# Source available at https://github.com/simonwhitaker/PyAPNs
#
# PyAPNs is distributed under the terms of the MIT license.
#
# Copyright (c) 2011 Goo Software Ltd
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from binascii import a2b_hex, b2a_hex
from datetime import datetime
from socket import socket, AF_INET, SOCK_STREAM
from struct import pack, unpack

try:
    from ssl import wrap_socket
except ImportError:
    from socket import ssl

try:
    import json
except ImportError:
    import simplejson as json

MAX_PAYLOAD_LENGTH = 256

class APNs(object):
    """A class representing an Apple Push Notification service connection"""

    def __init__(self, use_sandbox=False, cert_file=None, key_file=None):
        """
        Set use_sandbox to True to use the sandbox (test) APNs servers.
        Default is False.
        """
        super(APNs, self).__init__()
        self.use_sandbox = use_sandbox
        self.cert_file = cert_file
        self.key_file = key_file
        self._feedback_connection = None
        self._gateway_connection = None

    @staticmethod
    def packed_ushort_big_endian(num):
        """
        Returns an unsigned short in packed big-endian (network) form
        """
        return pack('>H', num)

    @staticmethod
    def unpacked_ushort_big_endian(bytes):
        """
        Returns an unsigned short from a packed big-endian (network) byte
        array
        """
        return unpack('>H', bytes)[0]

    @staticmethod
    def packed_uint_big_endian(num):
        """
        Returns an unsigned int in packed big-endian (network) form
        """
        return pack('>I', num)

    @staticmethod
    def unpacked_uint_big_endian(bytes):
        """
        Returns an unsigned int from a packed big-endian (network) byte array
        """
        return unpack('>I', bytes)[0]

    @property
    def feedback_server(self):
        if not self._feedback_connection:
            self._feedback_connection = FeedbackConnection(
                use_sandbox = self.use_sandbox,
                cert_file = self.cert_file,
                key_file = self.key_file
            )
        return self._feedback_connection

    @property
    def gateway_server(self):
        if not self._gateway_connection:
            self._gateway_connection = GatewayConnection(
                use_sandbox = self.use_sandbox,
                cert_file = self.cert_file,
                key_file = self.key_file
            )
        return self._gateway_connection

class APNsConnection(object):
    """
    A generic connection class for communicating with the APNs
    """
    def __init__(self, cert_file=None, key_file=None):
        super(APNsConnection, self).__init__()
        self.cert_file = cert_file
        self.key_file = key_file
        self._socket = None
        self._ssl = None

    def __del__(self):
        self._disconnect();

    def _connect(self):
        # Establish an SSL connection
        self._socket = socket(AF_INET, SOCK_STREAM)
        self._socket.connect((self.server, self.port))
        if wrap_socket:
            self._ssl = wrap_socket(self._socket, self.key_file, self.cert_file)
        else:
            self._ssl = ssl(self._socket, self.key_file, self.cert_file)

    def _disconnect(self):
        if self._socket:
            self._socket.close()

    def _connection(self):
        if not self._ssl:
            self._connect()
        return self._ssl

    def read(self, n=None):
        return self._connection().read(n)

    def write(self, string):
        return self._connection().write(string)

class PayloadAlert(object):
    def __init__(self, body, action_loc_key=None, loc_key=None,
                 loc_args=None, launch_image=None):
        super(PayloadAlert, self).__init__()
        self.body = body
        self.action_loc_key = action_loc_key
        self.loc_key = loc_key
        self.loc_args = loc_args
        self.launch_image = launch_image

    def dict(self):
        d = { 'body': self.body }
        if self.action_loc_key:
            d['action-loc-key'] = self.action_loc_key
        if self.loc_key:
            d['loc-key'] = self.loc_key
        if self.loc_args:
            d['loc-args'] = self.loc_args
        if self.launch_image:
            d['launch-image'] = self.launch_image
        return d

class PayloadTooLargeError(Exception):
    def __init__(self):
        super(PayloadTooLargeError, self).__init__()

class Payload(object):
    """A class representing an APNs message payload"""
    def __init__(self, alert=None, badge=None, sound=None, custom={}):
        super(Payload, self).__init__()
        self.alert = alert
        self.badge = badge
        self.sound = sound
        self.custom = custom
        self._check_size()

    def dict(self):
        """Returns the payload as a regular Python dictionary"""
        d = {}
        if self.alert:
            # Alert can be either a string or a PayloadAlert
            # object
            if isinstance(self.alert, PayloadAlert):
                d['alert'] = self.alert.dict()
            else:
                d['alert'] = self.alert
        if self.sound:
            d['sound'] = self.sound
        if self.badge:
            d['badge'] = int(self.badge)

        d = { 'aps': d }
        d.update(self.custom)
        return d

    def json(self):
        return json.dumps(self.dict(), separators=(',',':'))

    def _check_size(self):
        if len(self.json()) > MAX_PAYLOAD_LENGTH:
            raise PayloadTooLargeError()

    def __repr__(self):
        attrs = ("alert", "badge", "sound", "custom")
        args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs])
        return "%s(%s)" % (self.__class__.__name__, args)

class FeedbackConnection(APNsConnection):
    """
    A class representing a connection to the APNs Feedback server
    """
    def __init__(self, use_sandbox=False, **kwargs):
        super(FeedbackConnection, self).__init__(**kwargs)
        self.server = (
            'feedback.push.apple.com',
            'feedback.sandbox.push.apple.com')[use_sandbox]
        self.port = 2196

    def _chunks(self):
        BUF_SIZE = 4096
        while 1:
            data = self.read(BUF_SIZE)
            yield data
            if not data:
                break

    def items(self):
        """
        A generator that yields (token_hex, fail_time) pairs retrieved from
        the APNs feedback server
        """
        buff = ''
        for chunk in self._chunks():
            buff += chunk

            # Quit if there's no more data to read
            if not buff:
                break

            # Sanity check: after a socket read we should always have at least
            # 6 bytes in the buffer
            if len(buff) < 6:
                break

            while len(buff) > 6:
                token_length = APNs.unpacked_ushort_big_endian(buff[4:6])
                bytes_to_read = 6 + token_length
                if len(buff) >= bytes_to_read:
                    fail_time_unix = APNs.unpacked_uint_big_endian(buff[0:4])
                    fail_time = datetime.utcfromtimestamp(fail_time_unix)
                    token = b2a_hex(buff[6:bytes_to_read])

                    yield (token, fail_time)

                    # Remove data for current token from buffer
                    buff = buff[bytes_to_read:]
                else:
                    # break out of inner while loop - i.e. go and fetch
                    # some more data and append to buffer
                    break

class GatewayConnection(APNsConnection):
    """
    A class that represents a connection to the APNs gateway server
    """
    def __init__(self, use_sandbox=False, **kwargs):
        super(GatewayConnection, self).__init__(**kwargs)
        self.server = (
            'gateway.push.apple.com',
            'gateway.sandbox.push.apple.com')[use_sandbox]
        self.port = 2195

    def _get_notification(self, token_hex, payload):
        """
        Takes a token as a hex string and a payload as a Python dict and sends
        the notification
        """
        token_bin = a2b_hex(token_hex)
        token_length_bin = APNs.packed_ushort_big_endian(len(token_bin))
        payload_json = payload.json()
        payload_length_bin = APNs.packed_ushort_big_endian(len(payload_json))

        notification = ('\0' + token_length_bin + token_bin
            + payload_length_bin + payload_json)

        return notification

    def send_notification(self, token_hex, payload):
        self.write(self._get_notification(token_hex, payload))

<|endoftext|>


<commit_before>"""
Auto-download of ancillary files.

It allows Operations to specify a serious of source locations (http/ftp/rss URLs)
and destination locations to download to.

This is intended to replace Operations maintenance of many diverse and
complicated scripts with a single, central configuration file.
"""
import fcntl
import logging
import os
import stat
import sys
import heapq
import time
import multiprocessing
import signal
from croniter import croniter
from setproctitle import setproctitle

from . import DataSource, FetchReporter
from onreceipt.fetch.load import load_config

_log = logging.getLogger(__name__)

class _PrintReporter(FetchReporter):
    """
    Print events to the log.
    """

    def file_complete(self, uri, name, path):
        """
        :type uri: str
        :type name: str
        :type path: str
        """
        _log.info('Completed %r: %r -> %r', name, uri, path)

    def file_error(self, uri, message):
        """
        :type uri: str
        :type message: str
        """
        _log.info('Error (%r): %r)', uri, message)

def _schedule_item(schedule, now, item):
    """

    :type schedule: list of (float, ScheduledItem)
    :param now: float
    :param item: ScheduledItem
    :return:
    """
    next_trigger = croniter(item.cron_pattern, start_time=now).get_next()
    heapq.heappush(schedule, (next_trigger, item))
    return next_trigger

def _build_schedule(items):
    """
    :type items: list of ScheduledItem
    """
    scheduled = []
    now = time.time()
    for scheduled_item in items:
        _schedule_item(scheduled, now, scheduled_item)

    return scheduled

def _can_lock(lock_file):
    """
    Use the given file as a lock.

    Return true if successful.

    :type lock_file: str
    :rtype: bool
    """
    umask_original = os.umask(0)
    try:
        fp = os.open(lock_file, os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
    finally:
        os.umask(umask_original)

    try:
        fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except IOError:
        return False

    return True

def _redirect_output(log_file):
    """
    Redirect all output to the given file.

    :type log_file: str
    """
    output = open(log_file, 'w')
    sys.stdout = output
    sys.stderr = output
    logging.getLogger().removeHandler(_log_handler)
    handler = logging.StreamHandler(stream=output)
    handler.setFormatter(_log_formatter)
    logging.getLogger().addHandler(handler)

def _run_module(reporter, name, module, scheduled_time, log_directory, lock_directory):
    """
    Run the given module in a subprocess
    :type reporter: FetchReporter
    :type name: str
    :type module: DataSource
    :rtype: multiprocessing.Process
    """
    def _run_proc(reporter, readable_name, module, log_file, lock_file):
        """
        (from a new process), run the given module.
        :param reporter:
        :param readable_name:
        :param module:
        :return:
        """
        _redirect_output(log_file)

        if not _can_lock(lock_file):
            _log.debug('Lock is activated. Skipping run. %r', readable_name)
            sys.exit(0)

        setproctitle(readable_name)
        _init_signals()
        _log.debug('Triggering %s: %r', readable_name, module)
        module.trigger(reporter)

    _id = _sanitize_for_filename(name)
    lock_file = os.path.join(
        lock_directory,
        '{id}.lck'.format(id=_id)
    )
    scheduled_time_st = time.strftime('%H%M', time.localtime(scheduled_time))
    log_file = os.path.join(
        log_directory,
        '{time}-{id}.log'.format(
            id=_id,
            time=scheduled_time_st
        )
    )
    readable_name = 'fetch {} {}'.format(scheduled_time_st, name)

    _log.info('Spawning %r. Log %r, Lock %r', readable_name, log_file, lock_file)
    _log.debug('Module info %r', module)
    p = multiprocessing.Process(
        target=_run_proc,
        name=readable_name,
        args=(reporter, readable_name, module, log_file, lock_file)
    )
    p.start()
    return p

def _init_signals(trigger_exit=None, trigger_reload=None):
    """
    Set signal handlers
    :param trigger_reload: Handler for reload
    :type trigger_exit: Handler for exit
    """
    # For a SIGINT signal (Ctrl-C) or SIGTERM signal (`kill <pid>` command), we start a graceful shutdown.
    signal.signal(signal.SIGINT, trigger_exit if trigger_exit else signal.SIG_DFL)
    signal.signal(signal.SIGTERM, trigger_exit if trigger_exit else signal.SIG_DFL)

    # SIGHUP triggers a reload of config (following conventions of many daemons).
    signal.signal(signal.SIGHUP, trigger_reload if trigger_reload else signal.SIG_DFL)

def _on_child_finish(child):
    """
    Handle child process cleanup: Check for errors.

    :type child: multiprocessing.Process
    """
    exit_code = child.exitcode
    if exit_code is None:
        _log.warn('Child not finished %s %s', child.name, child.pid)
        return

    _log.debug('Child finished %r %r', child.name, child.pid)

    # TODO: Send mail, alert or something?
    if exit_code != 0:
        _log.error('Error return code %s from %r', exit_code, child.name)

def _filter_finished_children(running_children):
    """
    Filter and check the exit codes of finished children.

    :type running_children: set of multiprocessing.Process
    :rtype: set of multiprocessing.Process
    """
    still_running = set()

    for child in running_children:
        exit_code = child.exitcode
        if exit_code is None:
            still_running.add(child)
            continue

        _on_child_finish(child)

    return still_running

def _sanitize_for_filename(filename):
    """
    :type filename: str
    :rtype: str
    >>> _sanitize_for_filename('some one')
    'some-one'
    >>> _sanitize_for_filename('s@me one')
    's-me-one'
    """
    return "".join([x if x.isalnum() else "-" for x in filename])

def get_day_log_dir(log_directory, time_secs):
    """
    Get log directory for this day.
    :type log_directory: str
    :type time_secs: float
    :return:
    """
    # We use localtime because the cron scheduling uses localtime.
    t = time.localtime(time_secs)
    day_log_dir = os.path.join(
        log_directory,
        time.strftime('%Y', t),
        time.strftime('%m-%d', t)
    )
    if not os.path.exists(day_log_dir):
        os.makedirs(day_log_dir)

    return day_log_dir

def run_loop():
    """
    Main loop
    """

    class RunState(object):
        """
        Workaround for python 2's lack of 'nonlocal'.
        Contains vars that we change within signal handlers.
        """

        def __init__(self):
            self.exiting = False
            self.schedule = []
            self.base_directory = None

    o = RunState()

    def _reload_config():
        """Reload configuration."""
        _log.info('Reloading configuration')
        config = load_config()
        o.schedule = _build_schedule(config.schedule)
        o.base_directory = config.directory
        _log.debug('%s rules loaded', len(o.schedule))

    def trigger_exit(signal_, frame_):
        """Start a graceful shutdown"""
        o.exiting = True

    def trigger_reload(signal_, frame_):
        """Handle signal to reload config"""
        _reload_config()

    _reload_config()

    if not os.path.exists(o.base_directory):
        raise ValueError('Configured base folder does not exist: %r' % o.base_directory)

    # Cannot change these values 'live' (config reload).
    lock_directory = os.path.join(o.base_directory, 'lock')
    if not os.path.exists(lock_directory):
        os.makedirs(lock_directory)
    log_directory = os.path.join(o.base_directory, 'log')
    if not os.path.exists(log_directory):
        os.makedirs(log_directory)

    _init_signals(trigger_exit=trigger_exit, trigger_reload=trigger_reload)

    reporter = _PrintReporter()
    running_children = set()

    while not o.exiting:
        running_children = _filter_finished_children(running_children)

        # active_children() also cleans up zombie subprocesses.
        child_count = len(multiprocessing.active_children())

        _log.debug('%r recorded children, %r total children', len(running_children), child_count)

        if not o.schedule:
            _log.info('No scheduled items. Sleeping.')
            time.sleep(500)
            continue

        now = time.time()
        # : :type: (int, ScheduledItem)
        next_time, next_item = o.schedule[0]

        if next_time < now:
            # Trigger time has passed, so let's run it.

            # : :type: (int, ScheduledItem)
            next_time, next_item = heapq.heappop(o.schedule)

            day_log_dir = get_day_log_dir(log_directory, next_time)

            p = _run_module(
                reporter,
                next_item.name,
                next_item.module,
                scheduled_time=next_time,
                log_directory=day_log_dir,
                lock_directory=lock_directory
            )
            running_children.add(p)

            # Schedule next run for this module
            next_trigger = _schedule_item(o.schedule, now, next_item)

            _log.debug('Next trigger in %.1f minutes', next_trigger - now)
        else:
            # Sleep until next action is ready.
            sleep_seconds = (next_time - now) + 0.1
            _log.debug('Sleeping for %.1f minutes until action %r', sleep_seconds / 60.0, next_item.name)
            time.sleep(sleep_seconds)

    # Shut down -- Join all children.
    all_children = running_children.union(multiprocessing.active_children())
    _log.info('Shutting down. Joining %r children', len(all_children))
    for p in all_children:
        p.join()
        _on_child_finish(p)

_log_handler = logging.StreamHandler(stream=sys.stderr)
_log_formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s")
_log_handler.setFormatter(_log_formatter)

if __name__ == '__main__':
    logging.getLogger().addHandler(_log_handler)
    logging.getLogger().setLevel(logging.WARNING)

    _log.setLevel(logging.DEBUG)
    logging.getLogger('onreceipt').setLevel(logging.INFO)

    run_loop()
<commit_msg>Add skeleton log level settings.<commit_after>"""
Auto-download of ancillary files.

It allows Operations to specify a serious of source locations (http/ftp/rss URLs)
and destination locations to download to.

This is intended to replace Operations maintenance of many diverse and
complicated scripts with a single, central configuration file.
"""
import fcntl
import logging
import os
import stat
import sys
import heapq
import time
import multiprocessing
import signal
from croniter import croniter
from setproctitle import setproctitle

from . import DataSource, FetchReporter
from onreceipt.fetch.load import load_config

_log = logging.getLogger(__name__)

class _PrintReporter(FetchReporter):
    """
    Print events to the log.
    """

    def file_complete(self, uri, name, path):
        """
        :type uri: str
        :type name: str
        :type path: str
        """
        _log.info('Completed %r: %r -> %r', name, uri, path)

    def file_error(self, uri, message):
        """
        :type uri: str
        :type message: str
        """
        _log.info('Error (%r): %r)', uri, message)

def _schedule_item(schedule, now, item):
    """

    :type schedule: list of (float, ScheduledItem)
    :param now: float
    :param item: ScheduledItem
    :return:
    """
    next_trigger = croniter(item.cron_pattern, start_time=now).get_next()
    heapq.heappush(schedule, (next_trigger, item))
    return next_trigger

def _build_schedule(items):
    """
    :type items: list of ScheduledItem
    """
    scheduled = []
    now = time.time()
    for scheduled_item in items:
        _schedule_item(scheduled, now, scheduled_item)

    return scheduled

def _can_lock(lock_file):
    """
    Use the given file as a lock.

    Return true if successful.

    :type lock_file: str
    :rtype: bool
    """
    umask_original = os.umask(0)
    try:
        fp = os.open(lock_file, os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
    finally:
        os.umask(umask_original)

    try:
        fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except IOError:
        return False

    return True

def _redirect_output(log_file):
    """
    Redirect all output to the given file.

    :type log_file: str
    """
    output = open(log_file, 'w')
    sys.stdout = output
    sys.stderr = output
    logging.getLogger().removeHandler(_log_handler)
    handler = logging.StreamHandler(stream=output)
    handler.setFormatter(_log_formatter)
    logging.getLogger().addHandler(handler)

def _run_module(reporter, name, module, scheduled_time, log_directory, lock_directory):
    """
    Run the given module in a subprocess
    :type reporter: FetchReporter
    :type name: str
    :type module: DataSource
    :rtype: multiprocessing.Process
    """
    def _run_proc(reporter, readable_name, module, log_file, lock_file):
        """
        (from a new process), run the given module.
        :param reporter:
        :param readable_name:
        :param module:
        :return:
        """
        _redirect_output(log_file)

        if not _can_lock(lock_file):
            _log.debug('Lock is activated. Skipping run. %r', readable_name)
            sys.exit(0)

        setproctitle(readable_name)
        _init_signals()
        _log.debug('Triggering %s: %r', readable_name, module)
        module.trigger(reporter)

    _id = _sanitize_for_filename(name)
    lock_file = os.path.join(
        lock_directory,
        '{id}.lck'.format(id=_id)
    )
    scheduled_time_st = time.strftime('%H%M', time.localtime(scheduled_time))
    log_file = os.path.join(
        log_directory,
        '{time}-{id}.log'.format(
            id=_id,
            time=scheduled_time_st
        )
    )
    readable_name = 'fetch {} {}'.format(scheduled_time_st, name)

    _log.info('Spawning %r. Log %r, Lock %r', readable_name, log_file, lock_file)
    _log.debug('Module info %r', module)
    p = multiprocessing.Process(
        target=_run_proc,
        name=readable_name,
        args=(reporter, readable_name, module, log_file, lock_file)
    )
    p.start()
    return p

def _init_signals(trigger_exit=None, trigger_reload=None):
    """
    Set signal handlers
    :param trigger_reload: Handler for reload
    :type trigger_exit: Handler for exit
    """
    # For a SIGINT signal (Ctrl-C) or SIGTERM signal (`kill <pid>` command), we start a graceful shutdown.
    signal.signal(signal.SIGINT, trigger_exit if trigger_exit else signal.SIG_DFL)
    signal.signal(signal.SIGTERM, trigger_exit if trigger_exit else signal.SIG_DFL)

    # SIGHUP triggers a reload of config (following conventions of many daemons).
    signal.signal(signal.SIGHUP, trigger_reload if trigger_reload else signal.SIG_DFL)

def _on_child_finish(child):
    """
    Handle child process cleanup: Check for errors.

    :type child: multiprocessing.Process
    """
    exit_code = child.exitcode
    if exit_code is None:
        _log.warn('Child not finished %s %s', child.name, child.pid)
        return

    _log.debug('Child finished %r %r', child.name, child.pid)

    # TODO: Send mail, alert or something?
    if exit_code != 0:
        _log.error('Error return code %s from %r', exit_code, child.name)

def _filter_finished_children(running_children):
    """
    Filter and check the exit codes of finished children.

    :type running_children: set of multiprocessing.Process
    :rtype: set of multiprocessing.Process
    """
    still_running = set()

    for child in running_children:
        exit_code = child.exitcode
        if exit_code is None:
            still_running.add(child)
            continue

        _on_child_finish(child)

    return still_running

def _sanitize_for_filename(filename):
    """
    :type filename: str
    :rtype: str
    >>> _sanitize_for_filename('some one')
    'some-one'
    >>> _sanitize_for_filename('s@me one')
    's-me-one'
    """
    return "".join([x if x.isalnum() else "-" for x in filename])

def get_day_log_dir(log_directory, time_secs):
    """
    Get log directory for this day.
    :type log_directory: str
    :type time_secs: float
    :return:
    """
    # We use localtime because the cron scheduling uses localtime.
    t = time.localtime(time_secs)
    day_log_dir = os.path.join(
        log_directory,
        time.strftime('%Y', t),
        time.strftime('%m-%d', t)
    )
    if not os.path.exists(day_log_dir):
        os.makedirs(day_log_dir)

    return day_log_dir

def run_loop():
    """
    Main loop
    """

    class RunState(object):
        """
        Workaround for python 2's lack of 'nonlocal'.
        Contains vars that we change within signal handlers.
        """

        def __init__(self):
            self.exiting = False
            self.schedule = []
            self.base_directory = None

    o = RunState()

    def _reload_config():
        """Reload configuration."""
        _log.info('Reloading configuration')
        config = load_config()
        o.schedule = _build_schedule(config.schedule)
        o.base_directory = config.directory
        _log.debug('%s rules loaded', len(o.schedule))

    def trigger_exit(signal_, frame_):
        """Start a graceful shutdown"""
        o.exiting = True

    def trigger_reload(signal_, frame_):
        """Handle signal to reload config"""
        _reload_config()

    _reload_config()

    if not os.path.exists(o.base_directory):
        raise ValueError('Configured base folder does not exist: %r' % o.base_directory)

    # Cannot change these values 'live' (config reload).
    lock_directory = os.path.join(o.base_directory, 'lock')
    if not os.path.exists(lock_directory):
        os.makedirs(lock_directory)
    log_directory = os.path.join(o.base_directory, 'log')
    if not os.path.exists(log_directory):
        os.makedirs(log_directory)

    _init_signals(trigger_exit=trigger_exit, trigger_reload=trigger_reload)

    reporter = _PrintReporter()
    running_children = set()

    while not o.exiting:
        running_children = _filter_finished_children(running_children)

        # active_children() also cleans up zombie subprocesses.
        child_count = len(multiprocessing.active_children())

        _log.debug('%r recorded children, %r total children', len(running_children), child_count)

        if not o.schedule:
            _log.info('No scheduled items. Sleeping.')
            time.sleep(500)
            continue

        now = time.time()
        # : :type: (int, ScheduledItem)
        next_time, next_item = o.schedule[0]

        if next_time < now:
            # Trigger time has passed, so let's run it.

            # : :type: (int, ScheduledItem)
            next_time, next_item = heapq.heappop(o.schedule)

            day_log_dir = get_day_log_dir(log_directory, next_time)

            p = _run_module(
                reporter,
                next_item.name,
                next_item.module,
                scheduled_time=next_time,
                log_directory=day_log_dir,
                lock_directory=lock_directory
            )
            running_children.add(p)

            # Schedule next run for this module
            next_trigger = _schedule_item(o.schedule, now, next_item)

            _log.debug('Next trigger in %.1f minutes', next_trigger - now)
        else:
            # Sleep until next action is ready.
            sleep_seconds = (next_time - now) + 0.1
            _log.debug('Sleeping for %.1f minutes until action %r', sleep_seconds / 60.0, next_item.name)
            time.sleep(sleep_seconds)

    # Shut down -- Join all children.
    all_children = running_children.union(multiprocessing.active_children())
    _log.info('Shutting down. Joining %r children', len(all_children))
    for p in all_children:
        p.join()
        _on_child_finish(p)

_log_handler = logging.StreamHandler(stream=sys.stderr)
_log_formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s")
_log_handler.setFormatter(_log_formatter)

if __name__ == '__main__':
    logging.getLogger().addHandler(_log_handler)
    logging.getLogger().setLevel(logging.WARNING)

    logging.getLogger('onreceipt').setLevel(logging.INFO)
    logging.getLogger('onreceipt.fetch').setLevel(logging.INFO)
    _log.setLevel(logging.DEBUG)

    run_loop()
<|endoftext|>


<commit_before>import urllib,re,sys

class BlipVideo:
    def __init__(self):
        pass

    def get_rss(self,url):
        urlR=re.compile('http:\/\/blip\.tv/([^/]+)/([^/]+)-(\d+)')
        print(urlR.findall(url))
        pass

    def check(self):
        pass

if __name__=="__main__":
    blip=BlipVideo()
    print get_rss()
<commit_msg>added basic features<commit_after>#!/usr/bin/python2
# -*- coding: utf-8 -*-
import urllib,re,sys

class BlipVideo:
    def __init__(self,url=""):
        if re.match('.*blip.tv/.*',url):
            pass

    def get_info(self,url):
        pass

    def get_rss(self,url):
        urlR=re.compile('http:\/\/blip\.tv/([^/]+)/([^/]+)-(\d+)')
        (channel,name,vid)=urlR.findall(url)[0]
        return 'http://blip.tv/rss/flash/%s' % vid

    def check(self):
        pass

if __name__=="__main__":
    blip=BlipVideo()
<|endoftext|>