<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (c) 2007-2011 Parisson
# Copyright (c) 2007 Olivier Guilyardi <olivier@samalyse.com>
# Copyright (c) 2007-2011 Guillaume Pellerin <pellerin@parisson.com>
# Copyright (c) 2010-2011 Paul Brossier <piem@piem.org>
#
# This file is part of TimeSide.

# TimeSide is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.

# TimeSide is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with TimeSide.  If not, see <http://www.gnu.org/licenses/>.

# Authors: Paul Brossier <piem@piem.org>
#          Guilaume Pellerin <yomguy@parisson.com>

from timeside.core import Processor, implements, interfacedoc
from timeside.api import IDecoder
from timeside.encoder.gstutils import *

import Queue

GST_APPSINK_MAX_BUFFERS = 10
QUEUE_SIZE = 10

class FileDecoder(Processor):
    """ gstreamer-based decoder """
    implements(IDecoder)

    # duration ms, before discovery process times out
    MAX_DISCOVERY_TIME = 3000

    mimetype = ''
    output_nframes    = 8*1024
    output_samplerate = 44100
    output_channels   = 1

    was_discovered    = False
    pipeline          = None
    mainloopthread    = None
    read_error        = None

    # IProcessor methods

    @staticmethod
    @interfacedoc
    def id():
        return "gstreamerdec"

    def __init__(self, uri):
        # is this a file?
        import os.path
        if os.path.exists(uri):
            # get the absolute path
            uri = os.path.abspath(uri)
            # first run the file/uri through the discover pipeline
            self.discover(uri)
            if not self.was_discovered:
                raise Exception('discovered was not run')
            # and make a uri of it
            from urllib import quote
            self.uri = 'file://'+quote(uri)
        else:
            self.uri = uri

    def setup(self, channels = None, samplerate = None, nframes = None):
        # the output data format we want
        if nframes:    self.output_nframes    = nframes
        if samplerate: self.output_samplerate = int(samplerate)
        if channels:   self.output_channels   = int(channels)
        uri = self.uri

        self.pipe = ''' uridecodebin name=uridecodebin uri=%(uri)s
                  ! audioconvert name=audioconvert
                  ! audioresample
                  ! appsink name=sink sync=False async=True
                  ''' % locals()
        self.pipeline = gst.parse_launch(self.pipe)

        sink_caps = gst.Caps("""audio/x-raw-float,
            endianness=(int)1234,
            channels=(int)%s,
            width=(int)32,
            rate=(int)%d""" % (int(self.output_channels), int(self.output_samplerate)))

        self.decodebin = self.pipeline.get_by_name('uridecodebin')
        self.decodebin.connect("pad-added", self._pad_added_cb)
        self.decodebin.connect("no-more-pads", self._no_more_pads_cb)
        self.decodebin.connect("unknown-type", self._unknown_type_cb)

        self.conv = self.pipeline.get_by_name('audioconvert')
        self.conv.get_pad("sink").connect("notify::caps", self._notify_caps_cb)

        self.sink = self.pipeline.get_by_name('sink')
        self.sink.set_property("caps", sink_caps)
        self.sink.set_property('max-buffers', GST_APPSINK_MAX_BUFFERS)
        self.sink.set_property("drop", False)
        self.sink.set_property('emit-signals', True)
        self.sink.connect("new-buffer", self._on_new_buffer_cb)

        self.bus = self.pipeline.get_bus()
        self.bus.add_signal_watch()
        self.bus.connect('message', self._on_message_cb)

        self.queue = Queue.Queue(QUEUE_SIZE)

        import threading
        class MainloopThread(threading.Thread):
            def __init__(self, mainloop):
                threading.Thread.__init__(self)
                self.mainloop = mainloop

            def run(self):
                self.mainloop.run()
        self.mainloopthread = MainloopThread(self.mainloop)
        self.mainloopthread.start()
        #self.mainloopthread = get_loop_thread()
        ##self.mainloop = self.mainloopthread.mainloop

        self.eod = False

        self.last_buffer = None

        self._pad_found = False

        # start pipeline
        self.pipeline.set_state(gst.STATE_PLAYING)

        if self.read_error:
            self.release()
            raise self.read_error

    def _pad_added_cb(self, decodebin, pad):
        caps = pad.get_caps()
        if caps.to_string().startswith('audio'):
            if not self.conv.get_pad('sink').is_linked():
                self._pad_found = True
                pad.link(self.conv.get_pad('sink'))

    def _no_more_pads_cb(self, decodebin):
        self.pipeline.info("no more pads")
        if not self._pad_found:
            self.read_exc = Exception("no audio stream found")

    def _unknown_type_cb(self, decodebin, pad, caps):
        self.pipeline.debug("unknown type : %s" % caps.to_string())
        if not caps.to_string().startswith('audio'):
            return
        self.read_error = Exception("no known audio stream found")

    def _notify_caps_cb(self, pad, args):
        caps = pad.get_negotiated_caps()
        if not caps:
            pad.info("no negotiated caps available")
            return
        # the caps are fixed
        # We now get the total length of that stream
        q = gst.query_new_duration(gst.FORMAT_TIME)
        pad.info("sending duration query")
        if pad.get_peer().query(q):
            format, length = q.parse_duration()
            if format == gst.FORMAT_TIME:
                pad.info("got duration (time) : %s" % (gst.TIME_ARGS(length),))
            else:
                pad.info("got duration : %d [format:%d]" % (length, format))
        else:
            length = -1
            gst.warning("duration query failed")

        # We store the caps and length in the proper location
        if "audio" in caps.to_string():
            self.input_samplerate = caps[0]["rate"]
            self.input_channels = caps[0]["channels"]
            self.input_duration = length / 1.e9
            self.input_total_frames = int(self.input_duration * self.input_samplerate)
            if "x-raw-float" in caps.to_string():
                self.input_width = caps[0]["width"]
            else:
                self.input_width = caps[0]["depth"]

    def _on_message_cb(self, bus, message):
        t = message.type
        if t == gst.MESSAGE_EOS:
            self.pipeline.set_state(gst.STATE_NULL)
            self.queue.put(gst.MESSAGE_EOS)
            self.mainloop.quit()
        elif t == gst.MESSAGE_ERROR:
            self.pipeline.set_state(gst.STATE_NULL)
            err, debug = message.parse_error()
            self.mainloop.quit()
            print "Error: %s" % err, debug
        elif t == gst.MESSAGE_TAG:
            # TODO
            # msg.parse_tags()
            pass

    def _on_new_buffer_cb(self, sink):
        from numpy import concatenate
        buf = sink.emit('pull-buffer')
        new_array = gst_buffer_to_numpy_array(buf, self.output_channels)
        #print 'processing new buffer', new_array.shape
        if self.last_buffer is None:
            self.last_buffer = new_array
        else:
            self.last_buffer = concatenate((self.last_buffer, new_array), axis=0)
        while self.last_buffer.shape[0] >= self.output_nframes:
            new_block = self.last_buffer[:self.output_nframes]
            self.last_buffer = self.last_buffer[self.output_nframes:]
            #print 'queueing', new_block.shape, 'remaining', self.last_buffer.shape
            self.queue.put( [new_block, False ] )

    @interfacedoc
    def process(self, frames = None, eod = False):
        buf = self.queue.get()
        if buf == gst.MESSAGE_EOS:
            return self.last_buffer, True
        frames, eod = buf
        return frames, eod

    @interfacedoc
    def channels(self):
        return  self.output_channels

    @interfacedoc
    def samplerate(self):
        return self.output_samplerate

    @interfacedoc
    def nframes(self):
        return self.input_total_frames

    @interfacedoc
    def release(self):
        if self.pipeline: self.pipeline.set_state(gst.STATE_NULL)
        if self.mainloopthread: self.mainloopthread.join()

    def __del__(self):
        self.release()

    ## IDecoder methods

    @interfacedoc
    @interfacedoc
    def format(self):
        # TODO check
        if self.mimetype == 'application/x-id3':
            self.mimetype = 'audio/mpeg'
        return self.mimetype

    @interfacedoc
    def encoding(self):
        # TODO check
        return self.mimetype.split('/')[-1]

    @interfacedoc
    def resolution(self):
        # TODO check: width or depth?
        return self.audiowidth

    @interfacedoc
    def metadata(self):
        # TODO check
        return self.tags

    def duration(self):
        return self.duration

    ## gst.extend discoverer

    def discover(self, path):
        """ gstreamer based helper function to get file attributes """
        from gst.extend import discoverer
        d = discoverer.Discoverer(path, timeout = self.MAX_DISCOVERY_TIME)
        d.connect('discovered', self.discovered)

        # start discover pipeline
        self.mainloop = gobject.MainLoop()
        d.discover()
        self.mainloop.run()
        #d.print_info()

    def discovered(self, d, is_media):
        """ gstreamer based helper executed upon discover() completion """
        from math import ceil
        if is_media and d.is_audio:
            # copy the discoverer attributes to self
            self.mimetype= d.mimetype
            self.input_samplerate = d.audiorate
            self.input_channels = d.audiochannels
            self.input_width = d.audiowidth
            # conversion from time in nanoseconds to seconds
            self.input_duration = d.audiolength * 1.e-9
            # conversion from time in nanoseconds to frames
            self.input_total_frames = int ( ceil (d.audiorate * d.audiolength * 1.e-9) )
            self.input_total_frames = d.audiorate * d.audiolength * 1.e-9
            # copy tags
            self.tags = d.tags
        elif not d.is_audio:
            print "error, no audio found!"
        else:
            print "fail", path
        self.mainloop.quit()
        self.was_discovered = True
<commit_msg>timeside/decoder/core.py: remove discover, triggering bug in graphers<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (c) 2007-2011 Parisson
# Copyright (c) 2007 Olivier Guilyardi <olivier@samalyse.com>
# Copyright (c) 2007-2011 Guillaume Pellerin <pellerin@parisson.com>
# Copyright (c) 2010-2011 Paul Brossier <piem@piem.org>
#
# This file is part of TimeSide.

# TimeSide is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.

# TimeSide is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with TimeSide.  If not, see <http://www.gnu.org/licenses/>.

# Authors: Paul Brossier <piem@piem.org>
#          Guilaume Pellerin <yomguy@parisson.com>

from timeside.core import Processor, implements, interfacedoc
from timeside.api import IDecoder
from timeside.encoder.gstutils import *

import Queue

GST_APPSINK_MAX_BUFFERS = 10
QUEUE_SIZE = 10

class FileDecoder(Processor):
    """ gstreamer-based decoder """
    implements(IDecoder)

    mimetype = ''
    output_nframes    = 8*1024
    output_samplerate = 44100
    output_channels   = 1

    pipeline          = None
    mainloopthread    = None
    read_error        = None

    # IProcessor methods

    @staticmethod
    @interfacedoc
    def id():
        return "gstreamerdec"

    def __init__(self, uri):
        # is this a file?
        import os.path
        if os.path.exists(uri):
            # get the absolute path
            uri = os.path.abspath(uri)
            # and make a uri of it
            from urllib import quote
            self.uri = 'file://'+quote(uri)
        else:
            self.uri = uri

    def setup(self, channels = None, samplerate = None, nframes = None):
        # the output data format we want
        if nframes:    self.output_nframes    = nframes
        if samplerate: self.output_samplerate = int(samplerate)
        if channels:   self.output_channels   = int(channels)
        uri = self.uri

        self.pipe = ''' uridecodebin name=uridecodebin uri=%(uri)s
                  ! audioconvert name=audioconvert
                  ! audioresample
                  ! appsink name=sink sync=False async=True
                  ''' % locals()
        self.pipeline = gst.parse_launch(self.pipe)

        sink_caps = gst.Caps("""audio/x-raw-float,
            endianness=(int)1234,
            channels=(int)%s,
            width=(int)32,
            rate=(int)%d""" % (int(self.output_channels), int(self.output_samplerate)))

        self.decodebin = self.pipeline.get_by_name('uridecodebin')
        self.decodebin.connect("pad-added", self._pad_added_cb)
        self.decodebin.connect("no-more-pads", self._no_more_pads_cb)
        self.decodebin.connect("unknown-type", self._unknown_type_cb)

        self.conv = self.pipeline.get_by_name('audioconvert')
        self.conv.get_pad("sink").connect("notify::caps", self._notify_caps_cb)

        self.sink = self.pipeline.get_by_name('sink')
        self.sink.set_property("caps", sink_caps)
        self.sink.set_property('max-buffers', GST_APPSINK_MAX_BUFFERS)
        self.sink.set_property("drop", False)
        self.sink.set_property('emit-signals', True)
        self.sink.connect("new-buffer", self._on_new_buffer_cb)

        self.bus = self.pipeline.get_bus()
        self.bus.add_signal_watch()
        self.bus.connect('message', self._on_message_cb)

        self.queue = Queue.Queue(QUEUE_SIZE)

        import threading
        class MainloopThread(threading.Thread):
            def __init__(self, mainloop):
                threading.Thread.__init__(self)
                self.mainloop = mainloop

            def run(self):
                self.mainloop.run()
        self.mainloop = gobject.MainLoop()
        self.mainloopthread = MainloopThread(self.mainloop)
        self.mainloopthread.start()
        #self.mainloopthread = get_loop_thread()
        ##self.mainloop = self.mainloopthread.mainloop

        self.eod = False

        self.last_buffer = None

        self._pad_found = False

        # start pipeline
        self.pipeline.set_state(gst.STATE_PLAYING)

        if self.read_error:
            self.release()
            raise self.read_error

    def _pad_added_cb(self, decodebin, pad):
        caps = pad.get_caps()
        if caps.to_string().startswith('audio'):
            if not self.conv.get_pad('sink').is_linked():
                self._pad_found = True
                pad.link(self.conv.get_pad('sink'))

    def _no_more_pads_cb(self, decodebin):
        self.pipeline.info("no more pads")
        if not self._pad_found:
            self.read_exc = Exception("no audio stream found")

    def _unknown_type_cb(self, decodebin, pad, caps):
        self.pipeline.debug("unknown type : %s" % caps.to_string())
        if not caps.to_string().startswith('audio'):
            return
        self.read_error = Exception("no known audio stream found")

    def _notify_caps_cb(self, pad, args):
        caps = pad.get_negotiated_caps()
        if not caps:
            pad.info("no negotiated caps available")
            return
        # the caps are fixed
        # We now get the total length of that stream
        q = gst.query_new_duration(gst.FORMAT_TIME)
        pad.info("sending duration query")
        if pad.get_peer().query(q):
            format, length = q.parse_duration()
            if format == gst.FORMAT_TIME:
                pad.info("got duration (time) : %s" % (gst.TIME_ARGS(length),))
            else:
                pad.info("got duration : %d [format:%d]" % (length, format))
        else:
            length = -1
            gst.warning("duration query failed")

        # We store the caps and length in the proper location
        if "audio" in caps.to_string():
            self.input_samplerate = caps[0]["rate"]
            self.input_channels = caps[0]["channels"]
            self.input_duration = length / 1.e9
            self.input_total_frames = int(self.input_duration * self.input_samplerate)
            if "x-raw-float" in caps.to_string():
                self.input_width = caps[0]["width"]
            else:
                self.input_width = caps[0]["depth"]

    def _on_message_cb(self, bus, message):
        t = message.type
        if t == gst.MESSAGE_EOS:
            self.pipeline.set_state(gst.STATE_NULL)
            self.queue.put(gst.MESSAGE_EOS)
            self.mainloop.quit()
        elif t == gst.MESSAGE_ERROR:
            self.pipeline.set_state(gst.STATE_NULL)
            err, debug = message.parse_error()
            self.mainloop.quit()
            print "Error: %s" % err, debug
        elif t == gst.MESSAGE_TAG:
            # TODO
            # msg.parse_tags()
            pass

    def _on_new_buffer_cb(self, sink):
        from numpy import concatenate
        buf = sink.emit('pull-buffer')
        new_array = gst_buffer_to_numpy_array(buf, self.output_channels)
        #print 'processing new buffer', new_array.shape
        if self.last_buffer is None:
            self.last_buffer = new_array
        else:
            self.last_buffer = concatenate((self.last_buffer, new_array), axis=0)
        while self.last_buffer.shape[0] >= self.output_nframes:
            new_block = self.last_buffer[:self.output_nframes]
            self.last_buffer = self.last_buffer[self.output_nframes:]
            #print 'queueing', new_block.shape, 'remaining', self.last_buffer.shape
            self.queue.put( [new_block, False ] )

    @interfacedoc
    def process(self, frames = None, eod = False):
        buf = self.queue.get()
        if buf == gst.MESSAGE_EOS:
            return self.last_buffer, True
        frames, eod = buf
        return frames, eod

    @interfacedoc
    def channels(self):
        return  self.output_channels

    @interfacedoc
    def samplerate(self):
        return self.output_samplerate

    @interfacedoc
    def nframes(self):
        return self.input_total_frames

    @interfacedoc
    def release(self):
        if self.pipeline: self.pipeline.set_state(gst.STATE_NULL)
        if self.mainloopthread: self.mainloopthread.join()

    def __del__(self):
        self.release()

    ## IDecoder methods

    @interfacedoc
    @interfacedoc
    def format(self):
        # TODO check
        if self.mimetype == 'application/x-id3':
            self.mimetype = 'audio/mpeg'
        return self.mimetype

    @interfacedoc
    def encoding(self):
        # TODO check
        return self.mimetype.split('/')[-1]

    @interfacedoc
    def resolution(self):
        # TODO check: width or depth?
        return self.audiowidth

    @interfacedoc
    def metadata(self):
        # TODO check
        return self.tags

    def duration(self):
        return self.duration
<|endoftext|>


<commit_before>import datetime
import pytz

from django.db import models
from django.utils.encoding import smart_unicode, smart_str

class TimeZoneField(models.CharField):

    description = "A datetime.tzinfo object"

    __metaclass__ = models.SubfieldBase

    CHOICES = tuple(zip(pytz.all_timezones, pytz.all_timezones))

    def __init__(self, *args, **kwargs):
        defaults = {
            'max_length': 63,
            'choices': TimeZoneField.CHOICES,
        }
        defaults.update(kwargs)
        super(TimeZoneField, self).__init__(*args, **defaults)

    def validate(self, value, model_instance):
        # coerce value back to a string to validate correctly
        return super(TimeZoneField, self).validate(smart_str(value), model_instance)

    def run_validators(self, value):
        # coerce value back to a string to validate correctly
        return super(TimeZoneField, self).run_validators(smart_str(value))

    def to_python(self, value):
        "Returns pytz.tzinfo objects"
        # inspriation from django's Datetime field
        if not value:
            return None
        if isinstance(value, datetime.tzinfo):
            return value
        return pytz.timezone(smart_unicode(value))

    def get_prep_value(self, value):
        "Accepts both a pytz.info object or a string representing a timezone"
        # inspriation from django's Datetime field
        value = self.to_python(value)
        if not value:
            return None
        return smart_unicode(value)

    def flatten_data(self, follow, obj=None):
        value = self._get_val_from_obj(obj)
        if value is None:
            value = ''
        return {self.attname: smart_unicode(value)}
<commit_msg>Rework of validation<commit_after>import pytz

from django.core.exceptions import ValidationError
from django.db import models
from django.utils.encoding import smart_unicode

class TimeZoneField(models.CharField):
    """
    A TimeZoneField stores pytz DstTzInfo objects to the database.

    If you choose to add validators at runtime, they need to accept
    pytz.tzinfo objects as input.
    """

    description = "A pytz.tzinfo.DstTzInfo object"

    __metaclass__ = models.SubfieldBase

    CHOICES = tuple(zip(pytz.all_timezones, pytz.all_timezones))

    def __init__(self, validators=[], **kwargs):
        defaults = {
            'max_length': 63,
            'choices': TimeZoneField.CHOICES,
        }
        defaults.update(kwargs)

        super(TimeZoneField, self).__init__(**defaults)

        # validators our parent (CharField) register aren't going to
        # work right out of the box. They expect a string, while our
        # python type is a pytz.tzinfo
        # So, we'll wrap them in a small conversion object.

        class ValidateTimeZoneAsString(object):
            def __init__(self, org_validator):
                self.org_validator = org_validator
            def __call__(self, timezone):
                self.org_validator(smart_unicode(timezone))

        self.validators = [
            ValidateTimeZoneAsString(validator)
            for validator in self.validators
        ]

        self.validators += validators

    def validate(self, value, model_instance):
        # ensure we can consume the value
        value = self.to_python(value)
        # parent validation works on strings
        str_value = smart_unicode(value)
        return super(TimeZoneField, self).validate(str_value, model_instance)

    def to_python(self, value):
        "Returns pytz.tzinfo objects"
        # inspriation from django's Datetime field
        if not value:
            return None
        if isinstance(value, pytz.tzinfo.DstTzInfo):
            return value
        try:
            return pytz.timezone(smart_unicode(value))
        except pytz.UnknownTimeZoneError:
            raise ValidationError("Invalid timezone '{}'".format(value))

    def get_prep_value(self, value):
        "Accepts both a pytz.info object or a string representing a timezone"
        # inspriation from django's Datetime field
        value = self.to_python(value)
        if not value:
            return None
        return smart_unicode(value)

    def value_to_string(self, value):
        return self.get_prep_value(value)
<|endoftext|>


<commit_before># Copyright (C) 2007-2010 Samuel Abels.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
"""
Represents an activity within an order.
"""
import sys
from datetime import datetime
from lxml import etree
from Exscriptd.DBObject import DBObject
from Exscript.util.event import Event

class Task(DBObject):
    def __init__(self, order_id, name, queue_name, func):
        DBObject.__init__(self)
        self.id            = None
        self.order_id      = order_id
        self.name          = name
        self.queue_name    = queue_name
        self.status        = 'new'
        self.progress      = .0
        self.started       = datetime.utcnow()
        self.closed        = None
        self.logfile       = None
        self.tracefile     = None
        self.vars          = {}
        self.go_event      = Event()
        self.closed_event  = Event()
        self.changed_event = Event()
        if hasattr(func, '__call__'):
            self.func_name = func.__name__
        else:
            self.func_name = func

    @staticmethod
    def from_etree(task_node):
        """
        Creates a new instance by parsing the given XML.

        @type  task_node: lxml.etree.Element
        @param task_node: The task node of an etree.
        @rtype:  Task
        @return: A new instance of an task.
        """
        # Parse required attributes.
        name           = task_node.find('name').text
        task           = Task(name)
        task.id        = int(task_node.get('id'))
        task.status    = task_node.find('status').text
        task.progress  = float(task_node.find('progress').text)
        started_node   = task_node.find('started')
        closed_node    = task_node.find('closed')
        logfile_node   = task_node.find('logfile')
        tracefile_node = task_node.find('tracefile')
        if started_node is not None:
            started = started_node.text.split('.', 1)[0]
            started = datetime.strptime(started, "%Y-%m-%d %H:%M:%S")
            task.started = started
        if closed_node is not None:
            closed = closed_node.text.split('.', 1)[0]
            closed = datetime.strptime(closed, "%Y-%m-%d %H:%M:%S")
            task.closed = closed
        if logfile_node is not None:
            task.logfile = logfile_node.text
        if tracefile_node is not None:
            task.tracefile = tracefile_node.text
        return task

    @staticmethod
    def from_xml(xml):
        """
        Creates a new instance by parsing the given XML.

        @type  xml: str
        @param xml: A string containing an XML formatted task.
        @rtype:  Task
        @return: A new instance of an task.
        """
        xml = etree.fromstring(xml)
        return Task.from_etree(xml.find('task'))

    def toetree(self):
        """
        Returns the task as an lxml etree.

        @rtype:  lxml.etree
        @return: The resulting tree.
        """
        task = etree.Element('task', id = str(self.id))
        etree.SubElement(task, 'name').text     = str(self.name)
        etree.SubElement(task, 'status').text   = str(self.status)
        etree.SubElement(task, 'progress').text = str(self.progress)
        if self.started:
            etree.SubElement(task, 'started').text = str(self.started)
        if self.closed:
            etree.SubElement(task, 'closed').text = str(self.closed)
        if self.logfile:
            etree.SubElement(task, 'logfile').text = str(self.logfile)
        if self.tracefile:
            etree.SubElement(task, 'tracefile').text = str(self.tracefile)
        return task

    def toxml(self, pretty = True):
        """
        Returns the task as an XML formatted string.

        @type  pretty: bool
        @param pretty: Whether to format the XML in a human readable way.
        @rtype:  str
        @return: The XML representing the task.
        """
        xml  = etree.Element('xml')
        task = self.toetree()
        xml.append(task)
        return etree.tostring(xml, pretty_print = pretty)

    def todict(self):
        return dict(order_id  = self.order_id,
                    name      = self.get_name(),
                    status    = self.get_status(),
                    progress  = self.get_progress(),
                    started   = self.get_started_timestamp(),
                    closed    = self.get_closed_timestamp(),
                    logfile   = self.get_logfile(),
                    tracefile = self.get_tracefile(),
                    queue     = self.queue_name,
                    function  = self.func_name,
                    vars      = self.vars)

    def get_id(self):
        """
        Returns the task id.

        @rtype:  str
        @return: The id of the task.
        """
        return self.id

    def set_name(self, name):
        """
        Change the task name.

        @type  name: string
        @param name: A human readable name.
        """
        self.touch()
        self.name = name

    def get_name(self):
        """
        Returns the current name as a string.

        @rtype:  string
        @return: A human readable name.
        """
        return self.name

    def set_status(self, status):
        """
        Change the current status.

        @type  status: string
        @param status: A human readable status.
        """
        self.touch()
        self.status = status
        self.changed_event(self)

    def get_status(self):
        """
        Returns the current status as a string.

        @rtype:  string
        @return: A human readable status.
        """
        return self.status

    def set_progress(self, progress):
        """
        Change the current progress.

        @type  progress: float
        @param progress: The new progress.
        """
        self.touch()
        self.progress = progress

    def get_progress(self):
        """
        Returns the progress as a float between 0.0 and 1.0.

        @rtype:  float
        @return: The progress.
        """
        return self.progress

    def get_progress_percent(self):
        """
        Returns the progress as a string, in percent.

        @rtype:  str
        @return: The progress in percent.
        """
        return '%.1f' % (self.progress * 100.0)

    def get_started_timestamp(self):
        """
        Returns the time at which the task was started.

        @rtype:  datetime.datetime
        @return: The timestamp.
        """
        return self.started

    def go(self):
        """
        Marks the task as being ready for execution.
        """
        self.touch()
        self.set_status('go')
        self.go_event(self)

    def close(self, status = None):
        """
        Marks the task closed.

        @type  status: string
        @param status: A human readable status, or None to leave unchanged.
        """
        self.touch()
        self.closed = datetime.utcnow()
        if status:
            self.set_status(status)
        self.closed_event(self)

    def completed(self):
        """
        Like close(), but sets the status to 'completed' and the progress
        to 100%.
        """
        self.close('completed')
        self.set_progress(1.0)

    def get_closed_timestamp(self):
        """
        Returns the time at which the task was closed, or None if the
        task is still open.

        @rtype:  datetime.datetime|None
        @return: The timestamp or None.
        """
        return self.closed

    def set_logfile(self, logfile):
        """
        Set the name of the logfile.

        @type  logfile: string
        @param logfile: A filename.
        """
        self.touch()
        self.logfile = logfile

    def get_logfile(self):
        """
        Returns the name of the logfile as a string.

        @rtype:  string|None
        @return: A filename, or None.
        """
        return self.logfile

    def set_tracefile(self, tracefile):
        """
        Set the name of the tracefile.

        @type  tracefile: string
        @param tracefile: A filename.
        """
        self.touch()
        self.tracefile = tracefile

    def get_tracefile(self):
        """
        Returns the name of the tracefile as a string.

        @rtype:  string|None
        @return: A filename, or None.
        """
        return self.tracefile

    def set(self, key, value):
        """
        Defines a variable that is carried along with the task.
        The value *must* be pickleable.
        """
        self.vars[key] = value

    def get(self, key, default = None):
        """
        Returns the value as previously defined by L{Task.set()}.
        """
        return self.vars.get(key, default)
<commit_msg>Exscriptd: Task.set_logfile() defines a default tracefile.<commit_after># Copyright (C) 2007-2010 Samuel Abels.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
"""
Represents an activity within an order.
"""
import sys
from datetime import datetime
from lxml import etree
from Exscriptd.DBObject import DBObject
from Exscript.util.event import Event

class Task(DBObject):
    def __init__(self, order_id, name, queue_name, func):
        DBObject.__init__(self)
        self.id            = None
        self.order_id      = order_id
        self.name          = name
        self.queue_name    = queue_name
        self.status        = 'new'
        self.progress      = .0
        self.started       = datetime.utcnow()
        self.closed        = None
        self.logfile       = None
        self.tracefile     = None
        self.vars          = {}
        self.go_event      = Event()
        self.closed_event  = Event()
        self.changed_event = Event()
        if hasattr(func, '__call__'):
            self.func_name = func.__name__
        else:
            self.func_name = func

    @staticmethod
    def from_etree(task_node):
        """
        Creates a new instance by parsing the given XML.

        @type  task_node: lxml.etree.Element
        @param task_node: The task node of an etree.
        @rtype:  Task
        @return: A new instance of an task.
        """
        # Parse required attributes.
        name           = task_node.find('name').text
        task           = Task(name)
        task.id        = int(task_node.get('id'))
        task.status    = task_node.find('status').text
        task.progress  = float(task_node.find('progress').text)
        started_node   = task_node.find('started')
        closed_node    = task_node.find('closed')
        logfile_node   = task_node.find('logfile')
        tracefile_node = task_node.find('tracefile')
        if started_node is not None:
            started = started_node.text.split('.', 1)[0]
            started = datetime.strptime(started, "%Y-%m-%d %H:%M:%S")
            task.started = started
        if closed_node is not None:
            closed = closed_node.text.split('.', 1)[0]
            closed = datetime.strptime(closed, "%Y-%m-%d %H:%M:%S")
            task.closed = closed
        if logfile_node is not None:
            task.logfile = logfile_node.text
        if tracefile_node is not None:
            task.tracefile = tracefile_node.text
        return task

    @staticmethod
    def from_xml(xml):
        """
        Creates a new instance by parsing the given XML.

        @type  xml: str
        @param xml: A string containing an XML formatted task.
        @rtype:  Task
        @return: A new instance of an task.
        """
        xml = etree.fromstring(xml)
        return Task.from_etree(xml.find('task'))

    def toetree(self):
        """
        Returns the task as an lxml etree.

        @rtype:  lxml.etree
        @return: The resulting tree.
        """
        task = etree.Element('task', id = str(self.id))
        etree.SubElement(task, 'name').text     = str(self.name)
        etree.SubElement(task, 'status').text   = str(self.status)
        etree.SubElement(task, 'progress').text = str(self.progress)
        if self.started:
            etree.SubElement(task, 'started').text = str(self.started)
        if self.closed:
            etree.SubElement(task, 'closed').text = str(self.closed)
        if self.logfile:
            etree.SubElement(task, 'logfile').text = str(self.logfile)
        if self.tracefile:
            etree.SubElement(task, 'tracefile').text = str(self.tracefile)
        return task

    def toxml(self, pretty = True):
        """
        Returns the task as an XML formatted string.

        @type  pretty: bool
        @param pretty: Whether to format the XML in a human readable way.
        @rtype:  str
        @return: The XML representing the task.
        """
        xml  = etree.Element('xml')
        task = self.toetree()
        xml.append(task)
        return etree.tostring(xml, pretty_print = pretty)

    def todict(self):
        return dict(order_id  = self.order_id,
                    name      = self.get_name(),
                    status    = self.get_status(),
                    progress  = self.get_progress(),
                    started   = self.get_started_timestamp(),
                    closed    = self.get_closed_timestamp(),
                    logfile   = self.get_logfile(),
                    tracefile = self.get_tracefile(),
                    queue     = self.queue_name,
                    function  = self.func_name,
                    vars      = self.vars)

    def get_id(self):
        """
        Returns the task id.

        @rtype:  str
        @return: The id of the task.
        """
        return self.id

    def set_name(self, name):
        """
        Change the task name.

        @type  name: string
        @param name: A human readable name.
        """
        self.touch()
        self.name = name

    def get_name(self):
        """
        Returns the current name as a string.

        @rtype:  string
        @return: A human readable name.
        """
        return self.name

    def set_status(self, status):
        """
        Change the current status.

        @type  status: string
        @param status: A human readable status.
        """
        self.touch()
        self.status = status
        self.changed_event(self)

    def get_status(self):
        """
        Returns the current status as a string.

        @rtype:  string
        @return: A human readable status.
        """
        return self.status

    def set_progress(self, progress):
        """
        Change the current progress.

        @type  progress: float
        @param progress: The new progress.
        """
        self.touch()
        self.progress = progress

    def get_progress(self):
        """
        Returns the progress as a float between 0.0 and 1.0.

        @rtype:  float
        @return: The progress.
        """
        return self.progress

    def get_progress_percent(self):
        """
        Returns the progress as a string, in percent.

        @rtype:  str
        @return: The progress in percent.
        """
        return '%.1f' % (self.progress * 100.0)

    def get_started_timestamp(self):
        """
        Returns the time at which the task was started.

        @rtype:  datetime.datetime
        @return: The timestamp.
        """
        return self.started

    def go(self):
        """
        Marks the task as being ready for execution.
        """
        self.touch()
        self.set_status('go')
        self.go_event(self)

    def close(self, status = None):
        """
        Marks the task closed.

        @type  status: string
        @param status: A human readable status, or None to leave unchanged.
        """
        self.touch()
        self.closed = datetime.utcnow()
        if status:
            self.set_status(status)
        self.closed_event(self)

    def completed(self):
        """
        Like close(), but sets the status to 'completed' and the progress
        to 100%.
        """
        self.close('completed')
        self.set_progress(1.0)

    def get_closed_timestamp(self):
        """
        Returns the time at which the task was closed, or None if the
        task is still open.

        @rtype:  datetime.datetime|None
        @return: The timestamp or None.
        """
        return self.closed

    def set_logfile(self, logfile):
        """
        Set the name of the logfile, and set the name of the tracefile
        to the same name with '.error' appended.

        @type  logfile: string
        @param logfile: A filename.
        """
        self.touch()
        self.logfile   = logfile
        self.tracefile = logfile + '.error'

    def get_logfile(self):
        """
        Returns the name of the logfile as a string.

        @rtype:  string|None
        @return: A filename, or None.
        """
        return self.logfile

    def set_tracefile(self, tracefile):
        """
        Set the name of the tracefile.

        @type  tracefile: string
        @param tracefile: A filename.
        """
        self.touch()
        self.tracefile = tracefile

    def get_tracefile(self):
        """
        Returns the name of the tracefile as a string.

        @rtype:  string|None
        @return: A filename, or None.
        """
        return self.tracefile

    def set(self, key, value):
        """
        Defines a variable that is carried along with the task.
        The value *must* be pickleable.
        """
        self.vars[key] = value

    def get(self, key, default = None):
        """
        Returns the value as previously defined by L{Task.set()}.
        """
        return self.vars.get(key, default)
<|endoftext|>


<commit_before>import attr
import pandas as pd
import re
import sys
from random import Random

from .generators import BaseGenerator, SeedGenerator
from .csv_formatter import CSVFormatter
from .csv_formatter_v1 import CSVFormatterV1

__all__ = ["CustomGenerator"]

def make_item_class(clsname, attr_names):
    """
    Parameters
    ----------
    clsname: string
        Name of the class to be created

    attr_names: list of strings
        Names of the attributes of the class to be created
    """

    item_cls = attr.make_class(clsname, {name: attr.ib() for name in attr_names}, repr=False, cmp=True)

    def new_repr(self):
        all_fields = ', '.join([f'{name}={repr(value)}' for name, value in attr.asdict(self).items()])
        return f'{clsname}({all_fields})'

    orig_eq = item_cls.__eq__
    def new_eq(self, other):
        # allow comparisons with tuples and dicts, too (mostly for convenience in testing)
        if isinstance(other, self.__class__):
            return orig_eq(self, other)
        else:
            if isinstance(other, tuple):
                return attr.astuple(self) == other
            elif isinstance(other, dict):
                return attr.asdict(self) == other
            else:
                return NotImplemented

    item_cls.__repr__ = new_repr
    item_cls.__eq__ = new_eq
    item_cls.keys = lambda self: attr_names
    item_cls.__getitem__ = lambda self, key: getattr(self, key)
    item_cls.as_dict = lambda self: attr.asdict(self)
    item_cls.as_series = lambda self: pd.Series(attr.asdict(self))

    return item_cls

class CustomGeneratorMeta(type):
    def __new__(metacls, cg_name, bases, clsdict):
        gen_cls = super(CustomGeneratorMeta, metacls).__new__(metacls, cg_name, bases, clsdict)
        orig_init = gen_cls.__init__

        def gen_init(self, *args, **kwargs):
            seed = kwargs.pop('seed', None)

            # Call original __init__ function to make sure all generator attributes are defined
            orig_init(self, *args, **kwargs)

            self.field_gens = self._calculate_field_gens()
            self.fmt_dict = {name: "${" + name + "}" for name in self.field_gens.keys()}
            self.csvformatter = CSVFormatterV1(self.fmt_dict)
            self.item_cls = make_item_class(self.get_item_class_name(), self.field_gens.keys())
            self.seed_generator = SeedGenerator()
            self.reset(seed)

        gen_cls.__init__ = gen_init
        return gen_cls

class CustomGenerator(BaseGenerator, metaclass=CustomGeneratorMeta):

    def get_item_class_name(self):
        """
        Return the first part of the class name of this custom generator.
        This will be used for the class name of the items produced by this
        generator.

        Examples:
            FoobarGenerator -> Foobar
            QuuxGenerator   -> Quux
        """
        return re.match('^(.*)Generator$', self.__class__.__name__).group(1)

    def reset(self, seed=None):
        """
        Reset generator using the given seed (unless seed is None, in which case this is a no-op).
        """
        # Reset the seed generator
        if seed is not None:
            self.seed_generator.reset(seed)

            # Reset each constituent generator with a new seed
            # produced by the seed generator.
            for g, x in zip(self.field_gens.values(), self.seed_generator):
                g.reset(x)

    def _calculate_field_gens(self):
        clsdict = self.__class__.__dict__
        instdict = self.__dict__
        return {name: gen._spawn() for name, gen in dict(**clsdict, **instdict).items() if isinstance(gen, BaseGenerator)}

    def __next__(self):
        field_values = [next(g) for g in self.field_gens.values()]
        return self.item_cls(*field_values)

    def to_csv(self, path_or_buf=None, *, N, seed=None, fields=None, fmt_str=None, header=None, progressbar=True):
        """
        Generate N items and return the resulting CSV string or output to a file.

        Parameters
        ----------
        path_or_buf: string or file handle, default None
            File path or object. If None is provided the result
            is returned as a string.
        N: integer
            Number of items to generate.
        seed: integer (optional)
            Seed with which to initialise random generator.
        progressbar: boolean
            Whether to display a progressbar during item generation.

        The remaining arguments `fields`, `fmt_str`, `header`
        are passed on to CSVFormatter.
        """
        if fmt_str is None:
            fmt_str = getattr(self, 'CSV_FMT_STR', None)
        if fields is None and fmt_str is None:
            fields = getattr(self, 'CSV_FIELDS', self.fmt_dict)
        if header is None:
            header = getattr(self, 'CSV_HEADER', None)

        formatter = CSVFormatter(fmt_str=fmt_str, fields=fields, header=header)
        return formatter.to_csv(self.generate_OLD(N, seed=seed, progressbar=progressbar), path_or_buf=path_or_buf)

    def to_df(self, *, N, seed=None):
        """
        Return pandas DataFrame containing N items produced by this generator.
        """
        return self.generate_OLD(N, seed=seed).to_df()

    def to_psql(self, url, table_name, N, *, if_exists='fail', seed=None):
        """
        Generate N items and export them as rows in a PostgreSQL table.

        Parameters
        ----------

        url: string
            Connection string to connect to the database.
            Example: "postgresql://postgres@127.0.0.1:5432/testdb"

        table_name: string
            Name of the database table.

        N: integer
            Number of items to export.

        if_exists : {'fail', 'replace', 'append'}, default 'fail'
            - fail: If table exists, raise an error.
            - replace: If table exists, drop it, recreate it, and insert data.
            - append: If table exists, insert data. Create if does not exist.

        seed: integer (optional)
            Seed with which to initialise random generator.
        """
        self.generate_OLD(N, seed=seed).to_psql(url, table_name, if_exists=if_exists)
<commit_msg>Allow spawning of custom generators<commit_after>import attr
import pandas as pd
import re
import sys
from random import Random

from .generators import BaseGenerator, SeedGenerator
from .csv_formatter import CSVFormatter
from .csv_formatter_v1 import CSVFormatterV1

__all__ = ["CustomGenerator"]

def make_item_class(clsname, attr_names):
    """
    Parameters
    ----------
    clsname: string
        Name of the class to be created

    attr_names: list of strings
        Names of the attributes of the class to be created
    """

    item_cls = attr.make_class(clsname, {name: attr.ib() for name in attr_names}, repr=False, cmp=True)

    def new_repr(self):
        all_fields = ', '.join([f'{name}={repr(value)}' for name, value in attr.asdict(self).items()])
        return f'{clsname}({all_fields})'

    orig_eq = item_cls.__eq__
    def new_eq(self, other):
        # allow comparisons with tuples and dicts, too (mostly for convenience in testing)
        if isinstance(other, self.__class__):
            return orig_eq(self, other)
        else:
            if isinstance(other, tuple):
                return attr.astuple(self) == other
            elif isinstance(other, dict):
                return attr.asdict(self) == other
            else:
                return NotImplemented

    item_cls.__repr__ = new_repr
    item_cls.__eq__ = new_eq
    item_cls.keys = lambda self: attr_names
    item_cls.__getitem__ = lambda self, key: getattr(self, key)
    item_cls.as_dict = lambda self: attr.asdict(self)
    item_cls.as_series = lambda self: pd.Series(attr.asdict(self))

    return item_cls

class CustomGeneratorMeta(type):
    def __new__(metacls, cg_name, bases, clsdict):
        gen_cls = super(CustomGeneratorMeta, metacls).__new__(metacls, cg_name, bases, clsdict)
        orig_init = gen_cls.__init__

        def gen_init(self, *args, **kwargs):
            seed = kwargs.pop('seed', None)

            # Call original __init__ function to make sure all generator attributes are defined
            orig_init(self, *args, **kwargs)

            self.field_gens = self._calculate_field_gens()
            self.fmt_dict = {name: "${" + name + "}" for name in self.field_gens.keys()}
            self.csvformatter = CSVFormatterV1(self.fmt_dict)
            self.item_cls = make_item_class(self.get_item_class_name(), self.field_gens.keys())
            self.seed_generator = SeedGenerator()
            self.reset(seed)

        gen_cls.__init__ = gen_init
        return gen_cls

class CustomGenerator(BaseGenerator, metaclass=CustomGeneratorMeta):

    def get_item_class_name(self):
        """
        Return the first part of the class name of this custom generator.
        This will be used for the class name of the items produced by this
        generator.

        Examples:
            FoobarGenerator -> Foobar
            QuuxGenerator   -> Quux
        """
        return re.match('^(.*)Generator$', self.__class__.__name__).group(1)

    def reset(self, seed=None):
        """
        Reset generator using the given seed (unless seed is None, in which case this is a no-op).
        """
        # Reset the seed generator
        if seed is not None:
            self.seed_generator.reset(seed)

            # Reset each constituent generator with a new seed
            # produced by the seed generator.
            for g, x in zip(self.field_gens.values(), self.seed_generator):
                g.reset(x)

    def _calculate_field_gens(self):
        clsdict = self.__class__.__dict__
        instdict = self.__dict__
        return {name: gen._spawn() for name, gen in dict(**clsdict, **instdict).items() if isinstance(gen, BaseGenerator)}

    def _spawn(self):
        # TODO/FIXME: Check that this does the right thing:
        # (i) the spawned generator is independent of the original one (i.e. they can be reset independently without altering the other's behaviour)
        # (ii) ensure that it also works if this custom generator's __init__ requires additional arguments
        return self.__class__()

    def __next__(self):
        field_values = [next(g) for g in self.field_gens.values()]
        return self.item_cls(*field_values)

    def to_csv(self, path_or_buf=None, *, N, seed=None, fields=None, fmt_str=None, header=None, progressbar=True):
        """
        Generate N items and return the resulting CSV string or output to a file.

        Parameters
        ----------
        path_or_buf: string or file handle, default None
            File path or object. If None is provided the result
            is returned as a string.
        N: integer
            Number of items to generate.
        seed: integer (optional)
            Seed with which to initialise random generator.
        progressbar: boolean
            Whether to display a progressbar during item generation.

        The remaining arguments `fields`, `fmt_str`, `header`
        are passed on to CSVFormatter.
        """
        if fmt_str is None:
            fmt_str = getattr(self, 'CSV_FMT_STR', None)
        if fields is None and fmt_str is None:
            fields = getattr(self, 'CSV_FIELDS', self.fmt_dict)
        if header is None:
            header = getattr(self, 'CSV_HEADER', None)

        formatter = CSVFormatter(fmt_str=fmt_str, fields=fields, header=header)
        return formatter.to_csv(self.generate_OLD(N, seed=seed, progressbar=progressbar), path_or_buf=path_or_buf)

    def to_df(self, *, N, seed=None):
        """
        Return pandas DataFrame containing N items produced by this generator.
        """
        return self.generate_OLD(N, seed=seed).to_df()

    def to_psql(self, url, table_name, N, *, if_exists='fail', seed=None):
        """
        Generate N items and export them as rows in a PostgreSQL table.

        Parameters
        ----------

        url: string
            Connection string to connect to the database.
            Example: "postgresql://postgres@127.0.0.1:5432/testdb"

        table_name: string
            Name of the database table.

        N: integer
            Number of items to export.

        if_exists : {'fail', 'replace', 'append'}, default 'fail'
            - fail: If table exists, raise an error.
            - replace: If table exists, drop it, recreate it, and insert data.
            - append: If table exists, insert data. Create if does not exist.

        seed: integer (optional)
            Seed with which to initialise random generator.
        """
        self.generate_OLD(N, seed=seed).to_psql(url, table_name, if_exists=if_exists)
<|endoftext|>


<commit_before>import optparse, sys, StringIO
sys.path.append(".")  # Allow tests to be called from parent directory with Python 2.6
sys.path.append("..")
import ctypesgencore

"""ctypesgentest is a simple module for testing ctypesgen on various C constructs. It consists of a
single function, test(). test() takes a string that represents a C header file, along with some
keyword arguments representing options. It processes the header using ctypesgen and returns a tuple
containing the resulting module object and the output that ctypesgen produced."""

def test(header, **more_options):

	assert isinstance(header, str)
	file("temp.h","w").write(header)

	options = ctypesgencore.options.get_default_options()
	options.headers = ["temp.h"]
	for opt in more_options:
		setattr(options, opt, more_options[opt])

	# Redirect output
	sys.stdout = StringIO.StringIO()

	# Step 1: Parse
	descriptions=ctypesgencore.parser.parse(options.headers,options)

	# Step 2: Process
	ctypesgencore.processor.process(descriptions,options)

	# Step 3: Print
	ctypesgencore.printer.WrapperPrinter("temp.py",options,descriptions)

	# Un-redirect output
	output = sys.stdout.getvalue()
	sys.stdout.close()
	sys.stdout = sys.__stdout__

	# Load the module we have just produced
	module = __import__("temp")

	return module, output<commit_msg>Added code to test/ctypesgentest.py so that stdout can be left alone, this is useful if using a console passed debugger.<commit_after>import optparse, sys, StringIO
sys.path.append(".")  # Allow tests to be called from parent directory with Python 2.6
sys.path.append("..")
import ctypesgencore

"""ctypesgentest is a simple module for testing ctypesgen on various C constructs. It consists of a
single function, test(). test() takes a string that represents a C header file, along with some
keyword arguments representing options. It processes the header using ctypesgen and returns a tuple
containing the resulting module object and the output that ctypesgen produced."""

# set redirect_stdout to False if using console based debugger like pdb
redirect_stdout = True

def test(header, **more_options):

	assert isinstance(header, str)
	file("temp.h","w").write(header)

	options = ctypesgencore.options.get_default_options()
	options.headers = ["temp.h"]
	for opt in more_options:
		setattr(options, opt, more_options[opt])

	if redirect_stdout:
		# Redirect output
		sys.stdout = StringIO.StringIO()

	# Step 1: Parse
	descriptions=ctypesgencore.parser.parse(options.headers,options)

	# Step 2: Process
	ctypesgencore.processor.process(descriptions,options)

	# Step 3: Print
	ctypesgencore.printer.WrapperPrinter("temp.py",options,descriptions)

	if redirect_stdout:
		# Un-redirect output
		output = sys.stdout.getvalue()
		sys.stdout.close()
		sys.stdout = sys.__stdout__
	else:
		output = ''

	# Load the module we have just produced
	module = __import__("temp")

	return module, output<|endoftext|>


<commit_before>#! /usr/bin/env python
# -.- coding: utf-8 -.-

# Zeitgeist - Insert random events into the database
#
# Copyright © 2012 Canonical Ltd.
#             By Siegfried-A. Gevatter <siegfried.gevatter@collabora.co.uk>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# #############################################################################
# WARNING: make sure you launch Zeitgeist with ZEITGEIST_DATA_PATH set if
#          you don't want to fill your real database!
# #############################################################################

import os
import sys
import time
import random
from collections import deque
from gi.repository import GLib, GObject

from zeitgeist import mimetypes
from zeitgeist.datamodel import *
from zeitgeist.client import ZeitgeistDBusInterface

class EventGenerator:

    NUM_WORDS = 1000
    NUM_SIMULTANEOUS_URIS = 1000
    MAX_EVENT_AGE = 366*24*3600*1000

    _words = None
    _mimetypes = None
    _desktop_files = None
    _schemas = None
    _uri_table = None

    def __init__(self):
        # Initialize a pool of random words for use in URIs, etc.
        dictionary_words = map(str.strip,
            open('/usr/share/dict/words').readlines())
        dictionary_words = filter(lambda x: '\'s' not in x, dictionary_words)
        self._words = random.sample(dictionary_words, self.NUM_WORDS)

        # Initialize a pool of MIME-Types
        self._mimetypes = mimetypes.MIMES.keys()

        # Initialize a pool of application names
        self._desktop_files =  filter(lambda actor: actor.endswith('.desktop'),
            os.listdir('/usr/share/applications'))

        # Initialize a list of URI schemas
        self._schemas = ('application', 'davs', 'http', 'https', 'ftp')

        # Initialize a cache of URIs
        self._uri_table = deque(maxlen=self.NUM_SIMULTANEOUS_URIS)

    def get_word(self):
        # FIXME: add numbers and stuff?
        return random.choice(self._words)

    def get_extension(self):
        if random.random() < 0.8:
            extensions = [
                'odt', 'odp', 'doc',
                'oga', 'ogv', 'mp3'
                'png', 'jpg', 'gif', 'tiff'
                'html', 'xml', 'txt'
                'py', 'c', 'cpp', 'js', 'vala'
            ]
        else:
            extensions = self._words
        return filter(str.isalpha, random.choice(extensions))

    def get_path(self, force_directory=False):
        path = ''
        num_parts = 1 + abs(int(random.gauss(3, 3)))
        for i in range(num_parts):
            path += '/%s' % self.get_word()
        if random.random() < 0.9 and not force_directory:
            path += '.%s' % self.get_extension()
        return path

    def get_schema(self):
        rand = random.random()
        if rand < 0.005:
            return '%s://' % random.choice(self._words)
        elif rand < 0.4:
            return '%s://' % random.choice(self._schemas)
        else:
            return 'file:///'

    def generate_uri(self):
        file_uri = GLib.filename_to_uri(self.get_path(), None)
        return self.get_schema() + file_uri[8:]

    def get_uri(self):
        """
        We keep a cache of NUM_SIMULATENOUS_URIS uris for reuse. Every access
        has a 1% chance of replacing a URI in the table with a new one.
        """
        index = random.randint(0, self.NUM_SIMULTANEOUS_URIS)
        if index >= len(self._uri_table):
            # The URI table isn't fully initialized yet...
            uri = self.generate_uri()
            self._uri_table.append(uri)
            return uri
        if random.random() < 0.01:
            # Generate a new URI
            self._uri_table[index] = self.generate_uri()
        return self._uri_table[index]

    def get_text(self):
        num_words = abs(int(random.gauss(4, 3)))
        return ' '.join(self.get_word() for i in range(num_words))

    def get_subject_origin(self, uri):
        scheme = GLib.uri_parse_scheme(uri)
        if scheme == 'file':
            return GLib.path_get_dirname(uri)
        elif scheme in ('http', 'https'):
            scheme, domain = uri.split('://', 1)
            return '%s://%s' % (scheme, domain.split('/', 1)[0])
        else:
            return GLib.filename_to_uri(
                self.get_path(force_directory=True), None)

    def get_event_origin(self):
        if random.random() < 0.005:
            return self.get_uri()
        return ''

    def get_actor(self):
        return 'application://%s' % random.choice(self._desktop_files)

    def get_timestamp(self):
        current_time = int(time.time() * 1000)
        return random.randint(current_time - self.MAX_EVENT_AGE, current_time)

    def get_event_interpretation(self):
        interpretations = Interpretation.EVENT_INTERPRETATION.get_children()
        return random.choice(list(interpretations))

    def get_subject_interpretation(self):
        ev_interp = Interpretation.EVENT_INTERPRETATION.get_children()
        subj_interp = set(Interpretation.get_children())
        subj_interp.difference_update(ev_interp)
        return random.choice(list(subj_interp))

    def get_event_manifestation(self):
        if random.random() < 0.3:
            manifestations = Manifestation.EVENT_MANIFESTATION.get_children()
            return random.choice(list(manifestations))
        else:
            return Manifestation.USER_ACTIVITY

    def get_subject_manifestation(self):
        ev_manif = Manifestation.EVENT_MANIFESTATION.get_children()
        subj_manif = set(Interpretation.get_children())
        subj_manif.difference_update(ev_manif)
        return random.choice(list(subj_manif))

    def get_subject(self, event_interpretation):
        uri = self.get_uri()

        subject = Subject.new_for_values(
            uri            = uri,
            current_uri    = uri,
            interpretation = self.get_subject_interpretation(),
            manifestation  = self.get_subject_manifestation(),
            origin         = self.get_subject_origin(uri),
            mimetype       = random.choice(self._mimetypes),
            text           = self.get_text(),
            storage        = "")

        if event_interpretation == Interpretation.MOVE_EVENT:
            while subject.uri == subject.current_uri:
                subject.current_uri = self.get_uri()

        return subject

    def get_event(self):
        event_interpretation = self.get_event_interpretation()
        event = Event.new_for_values(
            timestamp      = self.get_timestamp(),
            interpretation = event_interpretation,
            manifestation  = self.get_event_manifestation(),
            actor          = self.get_actor(),
            origin         = self.get_event_origin())

        num_subjects = max(1, abs(int(random.gauss(1, 1))))
        while len(event.subjects) < num_subjects:
            subject = self.get_subject(event_interpretation)
            if subject.uri not in (x.uri for x in event.get_subjects()):
                # events with two subjects having the same URI aren't supported
                event.append_subject(subject)

        return event

class EventInserter():

    BUFFER_SIZE = 1000

    _log = None
    _buffer = None
    _events_inserted = None

    def __init__(self):
        self._log = ZeitgeistDBusInterface()
        self._buffer = []
        self._events_inserted = 0

    def insert(self, event):
        buffer_full = len(self._buffer) >= self.BUFFER_SIZE
        if buffer_full:
            self.flush()
        self._buffer.append(event)
        return buffer_full

    def flush(self):
        if self._buffer:
            self._log.InsertEvents(self._buffer)
            self._events_inserted += len(self._buffer)
            self._buffer = []

    def get_insertion_count(self):
        return self._events_inserted

def main():
    limit = '10000000' if len(sys.argv) < 2 else sys.argv[1]
    if len(sys.argv) > 2 or not limit.isdigit():
        print "Usage: %s [<num_events>]" % sys.argv[0]
        sys.exit(1)
    limit = int(limit)

    event_inserter = EventInserter()
    try:
        generator = EventGenerator()
        for i in xrange(limit):
            event = generator.get_event()
            event.payload = 'generate_events.py'
            if event_inserter.insert(event):
                print "Inserted %d events." % i
    except KeyboardInterrupt:
        pass
    event_inserter.flush()
    print "Inserted %d events. Done." % event_inserter.get_insertion_count()

if __name__ == '__main__':
    main()
<commit_msg>generate_events.py: insert events in more of a sequential order<commit_after>#! /usr/bin/env python
# -.- coding: utf-8 -.-

# Zeitgeist - Insert random events into the database
#
# Copyright © 2012 Canonical Ltd.
#             By Siegfried-A. Gevatter <siegfried.gevatter@collabora.co.uk>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# #############################################################################
# WARNING: make sure you launch Zeitgeist with ZEITGEIST_DATA_PATH set if
#          you don't want to fill your real database!
# #############################################################################

import os
import sys
import time
import random
from collections import deque
from gi.repository import GLib, GObject

from zeitgeist import mimetypes
from zeitgeist.datamodel import *
from zeitgeist.client import ZeitgeistDBusInterface

class EventGenerator:

    NUM_WORDS = 1000
    NUM_SIMULTANEOUS_URIS = 1000

    _words = None
    _mimetypes = None
    _desktop_files = None
    _schemas = None
    _uri_table = None
    _timestamp_generator = None

    def __init__(self):
        # Initialize a pool of random words for use in URIs, etc.
        dictionary_words = map(str.strip,
            open('/usr/share/dict/words').readlines())
        dictionary_words = filter(lambda x: '\'s' not in x, dictionary_words)
        self._words = random.sample(dictionary_words, self.NUM_WORDS)

        # Initialize timestamp generator
        self._timestamp_generator = TimestampGenerator()

        # Initialize a pool of MIME-Types
        self._mimetypes = mimetypes.MIMES.keys()

        # Initialize a pool of application names
        self._desktop_files =  filter(lambda actor: actor.endswith('.desktop'),
            os.listdir('/usr/share/applications'))

        # Initialize a list of URI schemas
        self._schemas = ('application', 'davs', 'http', 'https', 'ftp')

        # Initialize a cache of URIs
        self._uri_table = deque(maxlen=self.NUM_SIMULTANEOUS_URIS)

    def get_word(self):
        # FIXME: add numbers and stuff?
        return random.choice(self._words)

    def get_extension(self):
        if random.random() < 0.8:
            extensions = [
                'odt', 'odp', 'doc',
                'oga', 'ogv', 'mp3'
                'png', 'jpg', 'gif', 'tiff'
                'html', 'xml', 'txt'
                'py', 'c', 'cpp', 'js', 'vala'
            ]
        else:
            extensions = self._words
        return filter(str.isalpha, random.choice(extensions))

    def get_path(self, force_directory=False):
        path = ''
        num_parts = 1 + abs(int(random.gauss(3, 3)))
        for i in range(num_parts):
            path += '/%s' % self.get_word()
        if random.random() < 0.9 and not force_directory:
            path += '.%s' % self.get_extension()
        return path

    def get_schema(self):
        rand = random.random()
        if rand < 0.005:
            return '%s://' % random.choice(self._words)
        elif rand < 0.4:
            return '%s://' % random.choice(self._schemas)
        else:
            return 'file:///'

    def generate_uri(self):
        file_uri = GLib.filename_to_uri(self.get_path(), None)
        return self.get_schema() + file_uri[8:]

    def get_uri(self):
        """
        We keep a cache of NUM_SIMULATENOUS_URIS uris for reuse. Every access
        has a 1% chance of replacing a URI in the table with a new one.
        """
        index = random.randint(0, self.NUM_SIMULTANEOUS_URIS)
        if index >= len(self._uri_table):
            # The URI table isn't fully initialized yet...
            uri = self.generate_uri()
            self._uri_table.append(uri)
            return uri
        if random.random() < 0.01:
            # Generate a new URI
            self._uri_table[index] = self.generate_uri()
        return self._uri_table[index]

    def get_text(self):
        num_words = abs(int(random.gauss(4, 3)))
        return ' '.join(self.get_word() for i in range(num_words))

    def get_subject_origin(self, uri):
        scheme = GLib.uri_parse_scheme(uri)
        if scheme == 'file':
            return GLib.path_get_dirname(uri)
        elif scheme in ('http', 'https'):
            scheme, domain = uri.split('://', 1)
            return '%s://%s' % (scheme, domain.split('/', 1)[0])
        else:
            return GLib.filename_to_uri(
                self.get_path(force_directory=True), None)

    def get_event_origin(self):
        if random.random() < 0.005:
            return self.get_uri()
        return ''

    def get_actor(self):
        return 'application://%s' % random.choice(self._desktop_files)

    def get_timestamp(self):
        return self._timestamp_generator.next()

    def get_event_interpretation(self):
        interpretations = Interpretation.EVENT_INTERPRETATION.get_children()
        return random.choice(list(interpretations))

    def get_subject_interpretation(self):
        ev_interp = Interpretation.EVENT_INTERPRETATION.get_children()
        subj_interp = set(Interpretation.get_children())
        subj_interp.difference_update(ev_interp)
        return random.choice(list(subj_interp))

    def get_event_manifestation(self):
        if random.random() < 0.3:
            manifestations = Manifestation.EVENT_MANIFESTATION.get_children()
            return random.choice(list(manifestations))
        else:
            return Manifestation.USER_ACTIVITY

    def get_subject_manifestation(self):
        ev_manif = Manifestation.EVENT_MANIFESTATION.get_children()
        subj_manif = set(Interpretation.get_children())
        subj_manif.difference_update(ev_manif)
        return random.choice(list(subj_manif))

    def get_subject(self, event_interpretation):
        uri = self.get_uri()

        subject = Subject.new_for_values(
            uri            = uri,
            current_uri    = uri,
            interpretation = self.get_subject_interpretation(),
            manifestation  = self.get_subject_manifestation(),
            origin         = self.get_subject_origin(uri),
            mimetype       = random.choice(self._mimetypes),
            text           = self.get_text(),
            storage        = "")

        if event_interpretation == Interpretation.MOVE_EVENT:
            while subject.uri == subject.current_uri:
                subject.current_uri = self.get_uri()

        return subject

    def get_event(self):
        event_interpretation = self.get_event_interpretation()
        event = Event.new_for_values(
            timestamp      = self.get_timestamp(),
            interpretation = event_interpretation,
            manifestation  = self.get_event_manifestation(),
            actor          = self.get_actor(),
            origin         = self.get_event_origin())

        num_subjects = max(1, abs(int(random.gauss(1, 1))))
        while len(event.subjects) < num_subjects:
            subject = self.get_subject(event_interpretation)
            if subject.uri not in (x.uri for x in event.get_subjects()):
                # events with two subjects having the same URI aren't supported
                event.append_subject(subject)

        return event

class TimestampGenerator():

    MAX_EVENT_AGE = 366*24*3600*1000

    _start_time = None
    _lowest_limit = None

    _next_time = None

    def __init__(self):
        self._start_time = self.current_time() - (7*24*3600*1000)
        self._lowest_time = self._start_time - self.MAX_EVENT_AGE
        self._next_time = self._start_time

    def next(self):
        if random.random() < 0.005:
            return random.randint(self._lowest_time, self.current_time())
        return_time = self._next_time
        self._next_time += abs(int(random.gauss(1000, 5000)))
        return return_time

    @staticmethod
    def current_time():
        return int(time.time() * 1000)

class EventInserter():

    BUFFER_SIZE = 1000

    _log = None
    _buffer = None
    _events_inserted = None

    def __init__(self):
        self._log = ZeitgeistDBusInterface()
        self._buffer = []
        self._events_inserted = 0

    def insert(self, event):
        buffer_full = len(self._buffer) >= self.BUFFER_SIZE
        if buffer_full:
            self.flush()
        self._buffer.append(event)
        return buffer_full

    def flush(self):
        if self._buffer:
            self._log.InsertEvents(self._buffer)
            self._events_inserted += len(self._buffer)
            self._buffer = []

    def get_insertion_count(self):
        return self._events_inserted

def main():
    limit = '10000000' if len(sys.argv) < 2 else sys.argv[1]
    if len(sys.argv) > 2 or not limit.isdigit():
        print "Usage: %s [<num_events>]" % sys.argv[0]
        sys.exit(1)
    limit = int(limit)

    event_inserter = EventInserter()
    try:
        generator = EventGenerator()
        for i in xrange(limit):
            event = generator.get_event()
            event.payload = 'generate_events.py'
            if event_inserter.insert(event):
                print "Inserted %d events." % i
    except KeyboardInterrupt:
        pass
    event_inserter.flush()
    print "Inserted %d events. Done." % event_inserter.get_insertion_count()

if __name__ == '__main__':
    main()
<|endoftext|>