<commit_before>import logging
import urlparse
import xml.sax
import posixpath
import feedparser
import urllib2
import httplib
import socket
from flexget.feed import Entry
from flexget.plugin import register_plugin, internet, PluginError
from flexget.utils.cached_input import cached
from flexget.utils.tools import urlopener

log = logging.getLogger('rss')

class InputRSS(object):
    """
        Parses RSS feed.

        Hazzlefree configuration for public rss feeds:

        rss: <url>

        Configuration with basic http authentication:

        rss:
          url: <url>
          username: <name>
          password: <password>

        Advanced usages:

        You may wish to clean up the entry by stripping out all non-ascii characters.
        This can be done by setting ascii value to yes.

        Example:

        rss:
          url: <url>
          ascii: yes

        Incase RSS-feed uses some nonstandard field for urls and automatic detection fails
        you can configure plugin to use url from any feedparser entry attribute.

        Example:

        rss:
          url: <url>
          link: guid

        If you want to keep information in another rss field attached to the flexget entry, you can use the other_fields option.

        Example:

        rss:
          url: <url>
          other_fields: [date]

        You can disable few possibly annoying warnings by setting silent value to
        yes on feeds where there are frequently invalid items.

        Example:

        rss:
          url: <url>
          silent: yes

        You can group all the links of an item, to make the download plugin tolerant
        to broken urls: it will try to download each url until one works.
        Links are enclosures plus item fields given by the link value, in that order.
        The value to set is "group_links".

        Example:

        rss:
          url: <url>
          group_links: yes
    """

    def validator(self):
        from flexget import validator
        root = validator.factory()
        root.accept('url')
        root.accept('file')
        advanced = root.accept('dict')
        advanced.accept('url', key='url', required=True)
        advanced.accept('file', key='url')
        advanced.accept('text', key='username')
        advanced.accept('text', key='password')
        advanced.accept('text', key='title')
        advanced.accept('text', key='link')
        advanced.accept('list', key='link').accept('text')
        advanced.accept('list', key='other_fields').accept('text')
        advanced.accept('boolean', key='silent')
        advanced.accept('boolean', key='ascii')
        advanced.accept('boolean', key='filename')
        advanced.accept('boolean', key='group_links')
        return root

    def build_config(self, config):
        """Set default values to config"""
        if isinstance(config, basestring):
            config = {'url': config}
        # set the default link value to 'auto'
        config.setdefault('link', 'auto')
        # Replace : with _ and lower case other fields so they can be found in rss
        if config.get('other_fields'):
            config['other_fields'] = [field.replace(':', '_').lower() for field in config['other_fields']]
        # set default value for group_links as deactivated
        config.setdefault('group_links', False)
        # use basic auth when needed
        if 'username' in config and 'password' in config:
            config['url'] = self.passwordize(config['url'], config['username'], config['password'])
        return config

    def passwordize(self, url, user, password):
        """Add username and password to url"""
        parts = list(urlparse.urlsplit(url))
        parts[1] = user + ':' + password + '@' + parts[1]
        url = urlparse.urlunsplit(parts)
        return url

    def process_invalid_content(self, feed, url):
        """If feedparser reports error, save the received data and log error."""
        log.critical('Invalid XML received from feed %s' % feed.name)
        try:
            req = urlopener(url, log)
        except ValueError, exc:
            log.debug('invalid url `%s` due to %s (ok for a file)' % (url, exc))
            return
        data = req.read()
        req.close()
        ext = 'xml'
        if '<html>' in data.lower():
            log.critical('Received content is HTML page, not an RSS feed')
            ext = 'html'
        if 'login' in data.lower() or 'username' in data.lower():
            log.critical('Received content looks a bit like login page')
        if 'error' in data.lower():
            log.critical('Received content looks a bit like error page')
        import os
        received = os.path.join(feed.manager.config_base, 'received')
        if not os.path.isdir(received):
            os.mkdir(received)
        filename = os.path.join(received, '%s.%s' % (feed.name, ext))
        f = open(filename, 'w')
        f.write(data)
        f.close()
        log.critical('I have saved the invalid content to %s for you to view' % filename)

    def add_enclosure_info(self, entry, enclosure, filename=True, multiple=False):
        """Stores information from an rss enclosure into an Entry."""
        entry['url'] = enclosure['href']
        # get optional meta-data
        if 'length' in enclosure:
            try:
                entry['size'] = int(enclosure['length'])
            except:
                entry['size'] = 0
        if 'type' in enclosure:
            entry['type'] = enclosure['type']
        # TODO: better and perhaps join/in download plugin?
        # Parse filename from enclosure url
        basename = posixpath.basename(urlparse.urlsplit(entry['url']).path)
        # If enclosure has size OR there are multiple enclosures use filename from url
        if (entry.get('size') or multiple and basename) and filename:
            entry['filename'] = basename
            log.trace('filename `%s` from enclosure' % entry['filename'])

    @cached('rss')
    @internet(log)
    def on_feed_input(self, feed, config):
        config = self.build_config(config)

        log.debug('Requesting feed `%s` url `%s`' % (feed.name, config['url']))

        # check etags and last modified -headers
        # let's not, flexget works better when feed contains all entries all the time ?
        etag = None
        modified = None
        """
        etag = feed.cache.get('etag', None)
        if etag:
            log.debug('Sending etag %s for feed %s' % (etag, feed.name))
        modified = feed.cache.get('modified', None)
        if modified:
            log.debug('Sending last-modified %s for feed %s' % (etag, feed.name))
        """

        # set timeout to one minute
        orig_timout = socket.getdefaulttimeout()
        try:
            socket.setdefaulttimeout(60)

            # get the feed & parse
            if urllib2._opener:
                rss = feedparser.parse(config['url'], etag=etag, modified=modified, handlers=urllib2._opener.handlers)
            else:
                rss = feedparser.parse(config['url'], etag=etag, modified=modified)
        except LookupError, e:
            raise PluginError('Unable to parse the RSS: %s' % e)
        finally:
            # restore original timeout
            socket.setdefaulttimeout(orig_timout)

        # status checks
        status = rss.get('status', False)
        if not status:
            log.debug('RSS does not have status (normal if processing a file)')
        elif status == 304:
            log.debug('Feed %s hasn\'t changed, skipping' % feed.name)
            return
        elif status == 401:
            raise PluginError('Authentication needed for feed %s: %s' % \
                (feed.name, rss.headers['www-authenticate']), log)
        elif status == 404:
            raise PluginError('RSS Feed %s not found' % feed.name, log)
        elif status == 500:
            raise PluginError('Internal server exception on feed %s' % feed.name, log)

        # check for bozo
        ex = rss.get('bozo_exception', False)
        ignore = False
        if ex:
            if isinstance(ex, feedparser.NonXMLContentType):
                # see: http://www.feedparser.org/docs/character-encoding.html#advanced.encoding.nonxml
                log.debug('ignoring feedparser.NonXMLContentType')
                ignore = True
            elif isinstance(ex, feedparser.CharacterEncodingOverride):
                # see: ticket 88
                log.debug('ignoring feedparser.CharacterEncodingOverride')
                ignore = True
            elif isinstance(ex, UnicodeEncodeError):
                if rss.entries:
                    log.info('Feed has UnicodeEncodeError but seems to produce entries, ignoring the error ...')
                    ignore = True
            elif isinstance(ex, xml.sax._exceptions.SAXParseException):
                if not rss.entries:
                    # save invalid data for review, this is a bit ugly but users seem to really confused when
                    # html pages (login pages) are received
                    self.process_invalid_content(feed, config['url'])
                    if feed.manager.options.debug:
                        log.exception(ex)
                    raise PluginError('Received invalid RSS content')
                else:
                    msg = ('Invalid XML received (%s). However feedparser still produced entries.'
                        ' Ignoring the error...' % str(ex).replace('<unknown>:', 'line '))
                    if not config.get('silent', False):
                        log.info(msg)
                    else:
                        log.debug(msg)
                    ignore = True
            elif isinstance(ex, httplib.BadStatusLine) or \
                 isinstance(ex, IOError):
                raise ex # let the @internet decorator handle
            else:
                # all other bozo errors
                if not rss.entries:
                    self.process_invalid_content(feed, config['url'])
                    raise PluginError('Unhandled bozo_exception. Type: %s (feed: %s)' % \
                        (ex.__class__.__name__, feed.name), log)
                else:
                    msg = 'Invalid RSS received. However feedparser still produced entries. Ignoring the error ...'
                    if not config.get('silent', False):
                        log.info(msg)
                    else:
                        log.debug(msg)

        if 'bozo' in rss:
            if rss.bozo and not ignore:
                log.error(rss)
                log.error('Bozo exception %s on feed %s' % (type(ex), feed.name))
                return
        else:
            log.warn('feedparser bozo bit missing, feedparser bug? (FlexGet ticket #721)')

        log.debug('encoding %s' % rss.encoding)

        # update etag, use last modified if no etag exists
        """
        if 'etag' in rss and type(rss['etag']) != feedparser.types.NoneType:
            etag = rss.etag.replace("'", '').replace('"', '')
            feed.cache.store('etag', etag, 90)
            log.debug('etag %s saved for feed %s' % (etag, feed.name))
        elif hasattr(rss, 'headers'):
            if 'last-modified' in rss.headers:
                feed.cache.store('modified', rss.modified, 90)
                log.debug('last modified saved for feed %s', feed.name)
        """

        # new entries to be created
        entries = []

        # field name for url can be configured by setting link.
        # default value is auto but for example guid is used in some feeds
        ignored = 0
        for entry in rss.entries:

            # Check if title field is overridden in config
            title_field = config.get('title', 'title')
            # ignore entries without title
            if not entry.get(title_field):
                log.debug('skipping entry without title')
                ignored += 1
                continue

            # Set the title from the source field
            entry.title = entry[title_field]

            # convert title to ascii (cleanup)
            if config.get('ascii', False):
                entry.title = entry.title.encode('ascii', 'ignore')

            # remove annoying zero width spaces
            entry.title = entry.title.replace(u'\u200B', u'')

            # helper
            # TODO: confusing? refactor into class member ...

            def add_entry(ea):
                from flexget.utils.tools import decode_html
                ea['title'] = entry.title

                # Dict with fields to grab mapping from rss field name to FlexGet field name
                fields = {'guid': 'guid',
                          'author': 'author',
                          'description': 'description',
                          'infohash': 'torrent_info_hash'}
                # extend the dict of fields to grab with other_fields list in config
                for field in config.get('other_fields', []):
                    fields[field] = field

                for rss_field, flexget_field in fields.iteritems():
                    if rss_field in entry:
                        if not isinstance(getattr(entry, rss_field), basestring):
                            # Error if this field is not a string
                            log.error('Cannot grab non text field `%s` from rss.' % rss_field)
                            # Remove field from list of fields to avoid repeated error
                            config['other_fields'].remove(rss_field)
                            continue
                        try:
                            ea[flexget_field] = decode_html(entry[rss_field])
                            if rss_field in config.get('other_fields', []):
                                # Print a debug message for custom added fields
                                log.debug('Field `%s` set to `%s` for `%s`' % (rss_field, ea[rss_field], ea['title']))
                        except UnicodeDecodeError:
                            log.warning('Failed to decode entry `%s` field `%s`' % (ea['title'], rss_field))

                # store basic auth info
                if 'username' in config and 'password' in config:
                    ea['basic_auth_username'] = config['username']
                    ea['basic_auth_password'] = config['password']
                entries.append(ea)

            # create from enclosures if present
            enclosures = entry.get('enclosures', [])

            if len(enclosures) > 1 and not config.get('group_links'):
                # There is more than 1 enclosure, create an Entry for each of them
                log.debug('adding %i entries from enclosures' % len(enclosures))
                for enclosure in enclosures:
                    if not 'href' in enclosure:
                        log.debug('RSS-entry `%s` enclosure does not have URL' % entry.title)
                        continue
                    # There is a valid url for this enclosure, create an Entry for it
                    ee = Entry()
                    self.add_enclosure_info(ee, enclosure, config.get('filename', True), True)
                    add_entry(ee)
                # If we created entries for enclosures, we should not create an Entry for the main rss item
                continue

            # create flexget entry
            e = Entry()
            urls = []

            if not isinstance(config.get('link'), list):
                # If the link field is not a list, search for first valid url
                if config['link'] == 'auto':
                    # Auto mode, check for a single enclosure url first
                    if len(entry.get('enclosures', [])) == 1 and entry['enclosures'][0].get('href'):
                        self.add_enclosure_info(e, entry['enclosures'][0], config.get('filename', True))
                    else:
                        # If there is no enclosure url, check link, then guid field for urls
                        for field in ['link', 'guid']:
                            if entry.get(field):
                                e['url'] = entry[field]
                                break
                else:
                    if entry.get(config['link']):
                        e['url'] = entry[config['link']]
            else:
                # If link was passed as a list, we create a list of urls
                for field in config['link']:
                    if entry.get(field):
                        e.setdefault('url', entry[field])
                        if entry[field] not in e.setdefault('urls', []):
                            e['urls'].append(entry[field])

            if config.get('group_links'):
                # Append a list of urls from enclosures to the urls field if group_links is enabled
                e.setdefault('urls', [e['url']]).extend(
                        [enc.href for enc in entry.get('enclosures', []) if enc.get('href') not in e['urls']])

            if not e.get('url'):
                log.debug('%s does not have link (%s) or enclosure' % (entry.title, config['link']))
                ignored += 1
                continue

            add_entry(e)

        if ignored:
            if not config.get('silent'):
                log.warning('Skipped %s RSS-entries without required information (title, link or enclosures)' % ignored)

        return entries

register_plugin(InputRSS, 'rss', api_ver=2)
<commit_msg>rss plugin now provides the rss_pubdate field.<commit_after>import logging
import urlparse
import xml.sax
import posixpath
import urllib2
import httplib
import socket
from datetime import datetime
import feedparser
from flexget.feed import Entry
from flexget.plugin import register_plugin, internet, PluginError
from flexget.utils.cached_input import cached
from flexget.utils.tools import urlopener

log = logging.getLogger('rss')

class InputRSS(object):
    """
        Parses RSS feed.

        Hazzlefree configuration for public rss feeds:

        rss: <url>

        Configuration with basic http authentication:

        rss:
          url: <url>
          username: <name>
          password: <password>

        Advanced usages:

        You may wish to clean up the entry by stripping out all non-ascii characters.
        This can be done by setting ascii value to yes.

        Example:

        rss:
          url: <url>
          ascii: yes

        Incase RSS-feed uses some nonstandard field for urls and automatic detection fails
        you can configure plugin to use url from any feedparser entry attribute.

        Example:

        rss:
          url: <url>
          link: guid

        If you want to keep information in another rss field attached to the flexget entry, you can use the other_fields option.

        Example:

        rss:
          url: <url>
          other_fields: [date]

        You can disable few possibly annoying warnings by setting silent value to
        yes on feeds where there are frequently invalid items.

        Example:

        rss:
          url: <url>
          silent: yes

        You can group all the links of an item, to make the download plugin tolerant
        to broken urls: it will try to download each url until one works.
        Links are enclosures plus item fields given by the link value, in that order.
        The value to set is "group_links".

        Example:

        rss:
          url: <url>
          group_links: yes
    """

    def validator(self):
        from flexget import validator
        root = validator.factory()
        root.accept('url')
        root.accept('file')
        advanced = root.accept('dict')
        advanced.accept('url', key='url', required=True)
        advanced.accept('file', key='url')
        advanced.accept('text', key='username')
        advanced.accept('text', key='password')
        advanced.accept('text', key='title')
        advanced.accept('text', key='link')
        advanced.accept('list', key='link').accept('text')
        advanced.accept('list', key='other_fields').accept('text')
        advanced.accept('boolean', key='silent')
        advanced.accept('boolean', key='ascii')
        advanced.accept('boolean', key='filename')
        advanced.accept('boolean', key='group_links')
        return root

    def build_config(self, config):
        """Set default values to config"""
        if isinstance(config, basestring):
            config = {'url': config}
        # set the default link value to 'auto'
        config.setdefault('link', 'auto')
        # Replace : with _ and lower case other fields so they can be found in rss
        if config.get('other_fields'):
            config['other_fields'] = [field.replace(':', '_').lower() for field in config['other_fields']]
        # set default value for group_links as deactivated
        config.setdefault('group_links', False)
        # use basic auth when needed
        if 'username' in config and 'password' in config:
            config['url'] = self.passwordize(config['url'], config['username'], config['password'])
        return config

    def passwordize(self, url, user, password):
        """Add username and password to url"""
        parts = list(urlparse.urlsplit(url))
        parts[1] = user + ':' + password + '@' + parts[1]
        url = urlparse.urlunsplit(parts)
        return url

    def process_invalid_content(self, feed, url):
        """If feedparser reports error, save the received data and log error."""
        log.critical('Invalid XML received from feed %s' % feed.name)
        try:
            req = urlopener(url, log)
        except ValueError, exc:
            log.debug('invalid url `%s` due to %s (ok for a file)' % (url, exc))
            return
        data = req.read()
        req.close()
        ext = 'xml'
        if '<html>' in data.lower():
            log.critical('Received content is HTML page, not an RSS feed')
            ext = 'html'
        if 'login' in data.lower() or 'username' in data.lower():
            log.critical('Received content looks a bit like login page')
        if 'error' in data.lower():
            log.critical('Received content looks a bit like error page')
        import os
        received = os.path.join(feed.manager.config_base, 'received')
        if not os.path.isdir(received):
            os.mkdir(received)
        filename = os.path.join(received, '%s.%s' % (feed.name, ext))
        f = open(filename, 'w')
        f.write(data)
        f.close()
        log.critical('I have saved the invalid content to %s for you to view' % filename)

    def add_enclosure_info(self, entry, enclosure, filename=True, multiple=False):
        """Stores information from an rss enclosure into an Entry."""
        entry['url'] = enclosure['href']
        # get optional meta-data
        if 'length' in enclosure:
            try:
                entry['size'] = int(enclosure['length'])
            except:
                entry['size'] = 0
        if 'type' in enclosure:
            entry['type'] = enclosure['type']
        # TODO: better and perhaps join/in download plugin?
        # Parse filename from enclosure url
        basename = posixpath.basename(urlparse.urlsplit(entry['url']).path)
        # If enclosure has size OR there are multiple enclosures use filename from url
        if (entry.get('size') or multiple and basename) and filename:
            entry['filename'] = basename
            log.trace('filename `%s` from enclosure' % entry['filename'])

    @cached('rss')
    @internet(log)
    def on_feed_input(self, feed, config):
        config = self.build_config(config)

        log.debug('Requesting feed `%s` url `%s`' % (feed.name, config['url']))

        # check etags and last modified -headers
        # let's not, flexget works better when feed contains all entries all the time ?
        etag = None
        modified = None
        """
        etag = feed.cache.get('etag', None)
        if etag:
            log.debug('Sending etag %s for feed %s' % (etag, feed.name))
        modified = feed.cache.get('modified', None)
        if modified:
            log.debug('Sending last-modified %s for feed %s' % (etag, feed.name))
        """

        # set timeout to one minute
        orig_timout = socket.getdefaulttimeout()
        try:
            socket.setdefaulttimeout(60)

            # get the feed & parse
            if urllib2._opener:
                rss = feedparser.parse(config['url'], etag=etag, modified=modified, handlers=urllib2._opener.handlers)
            else:
                rss = feedparser.parse(config['url'], etag=etag, modified=modified)
        except LookupError, e:
            raise PluginError('Unable to parse the RSS: %s' % e)
        finally:
            # restore original timeout
            socket.setdefaulttimeout(orig_timout)

        # status checks
        status = rss.get('status', False)
        if not status:
            log.debug('RSS does not have status (normal if processing a file)')
        elif status == 304:
            log.debug('Feed %s hasn\'t changed, skipping' % feed.name)
            return
        elif status == 401:
            raise PluginError('Authentication needed for feed %s: %s' % \
                (feed.name, rss.headers['www-authenticate']), log)
        elif status == 404:
            raise PluginError('RSS Feed %s not found' % feed.name, log)
        elif status == 500:
            raise PluginError('Internal server exception on feed %s' % feed.name, log)

        # check for bozo
        ex = rss.get('bozo_exception', False)
        ignore = False
        if ex:
            if isinstance(ex, feedparser.NonXMLContentType):
                # see: http://www.feedparser.org/docs/character-encoding.html#advanced.encoding.nonxml
                log.debug('ignoring feedparser.NonXMLContentType')
                ignore = True
            elif isinstance(ex, feedparser.CharacterEncodingOverride):
                # see: ticket 88
                log.debug('ignoring feedparser.CharacterEncodingOverride')
                ignore = True
            elif isinstance(ex, UnicodeEncodeError):
                if rss.entries:
                    log.info('Feed has UnicodeEncodeError but seems to produce entries, ignoring the error ...')
                    ignore = True
            elif isinstance(ex, xml.sax._exceptions.SAXParseException):
                if not rss.entries:
                    # save invalid data for review, this is a bit ugly but users seem to really confused when
                    # html pages (login pages) are received
                    self.process_invalid_content(feed, config['url'])
                    if feed.manager.options.debug:
                        log.exception(ex)
                    raise PluginError('Received invalid RSS content')
                else:
                    msg = ('Invalid XML received (%s). However feedparser still produced entries.'
                        ' Ignoring the error...' % str(ex).replace('<unknown>:', 'line '))
                    if not config.get('silent', False):
                        log.info(msg)
                    else:
                        log.debug(msg)
                    ignore = True
            elif isinstance(ex, httplib.BadStatusLine) or \
                 isinstance(ex, IOError):
                raise ex # let the @internet decorator handle
            else:
                # all other bozo errors
                if not rss.entries:
                    self.process_invalid_content(feed, config['url'])
                    raise PluginError('Unhandled bozo_exception. Type: %s (feed: %s)' % \
                        (ex.__class__.__name__, feed.name), log)
                else:
                    msg = 'Invalid RSS received. However feedparser still produced entries. Ignoring the error ...'
                    if not config.get('silent', False):
                        log.info(msg)
                    else:
                        log.debug(msg)

        if 'bozo' in rss:
            if rss.bozo and not ignore:
                log.error(rss)
                log.error('Bozo exception %s on feed %s' % (type(ex), feed.name))
                return
        else:
            log.warn('feedparser bozo bit missing, feedparser bug? (FlexGet ticket #721)')

        log.debug('encoding %s' % rss.encoding)

        # update etag, use last modified if no etag exists
        """
        if 'etag' in rss and type(rss['etag']) != feedparser.types.NoneType:
            etag = rss.etag.replace("'", '').replace('"', '')
            feed.cache.store('etag', etag, 90)
            log.debug('etag %s saved for feed %s' % (etag, feed.name))
        elif hasattr(rss, 'headers'):
            if 'last-modified' in rss.headers:
                feed.cache.store('modified', rss.modified, 90)
                log.debug('last modified saved for feed %s', feed.name)
        """

        # new entries to be created
        entries = []

        # field name for url can be configured by setting link.
        # default value is auto but for example guid is used in some feeds
        ignored = 0
        for entry in rss.entries:

            # Check if title field is overridden in config
            title_field = config.get('title', 'title')
            # ignore entries without title
            if not entry.get(title_field):
                log.debug('skipping entry without title')
                ignored += 1
                continue

            # Set the title from the source field
            entry.title = entry[title_field]

            # convert title to ascii (cleanup)
            if config.get('ascii', False):
                entry.title = entry.title.encode('ascii', 'ignore')

            # remove annoying zero width spaces
            entry.title = entry.title.replace(u'\u200B', u'')

            # helper
            # TODO: confusing? refactor into class member ...

            def add_entry(ea):
                from flexget.utils.tools import decode_html
                ea['title'] = entry.title

                # Dict with fields to grab mapping from rss field name to FlexGet field name
                fields = {'guid': 'guid',
                          'author': 'author',
                          'description': 'description',
                          'infohash': 'torrent_info_hash'}
                # extend the dict of fields to grab with other_fields list in config
                for field in config.get('other_fields', []):
                    fields[field] = field

                for rss_field, flexget_field in fields.iteritems():
                    if rss_field in entry:
                        if not isinstance(getattr(entry, rss_field), basestring):
                            # Error if this field is not a string
                            log.error('Cannot grab non text field `%s` from rss.' % rss_field)
                            # Remove field from list of fields to avoid repeated error
                            config['other_fields'].remove(rss_field)
                            continue
                        try:
                            ea[flexget_field] = decode_html(entry[rss_field])
                            if rss_field in config.get('other_fields', []):
                                # Print a debug message for custom added fields
                                log.debug('Field `%s` set to `%s` for `%s`' % (rss_field, ea[rss_field], ea['title']))
                        except UnicodeDecodeError:
                            log.warning('Failed to decode entry `%s` field `%s`' % (ea['title'], rss_field))
                # Also grab pubdate if available
                if hasattr(entry, 'date_parsed'):
                    ea['rss_pubdate'] = datetime(*entry.date_parsed[:6])
                # store basic auth info
                if 'username' in config and 'password' in config:
                    ea['basic_auth_username'] = config['username']
                    ea['basic_auth_password'] = config['password']
                entries.append(ea)

            # create from enclosures if present
            enclosures = entry.get('enclosures', [])

            if len(enclosures) > 1 and not config.get('group_links'):
                # There is more than 1 enclosure, create an Entry for each of them
                log.debug('adding %i entries from enclosures' % len(enclosures))
                for enclosure in enclosures:
                    if not 'href' in enclosure:
                        log.debug('RSS-entry `%s` enclosure does not have URL' % entry.title)
                        continue
                    # There is a valid url for this enclosure, create an Entry for it
                    ee = Entry()
                    self.add_enclosure_info(ee, enclosure, config.get('filename', True), True)
                    add_entry(ee)
                # If we created entries for enclosures, we should not create an Entry for the main rss item
                continue

            # create flexget entry
            e = Entry()
            urls = []

            if not isinstance(config.get('link'), list):
                # If the link field is not a list, search for first valid url
                if config['link'] == 'auto':
                    # Auto mode, check for a single enclosure url first
                    if len(entry.get('enclosures', [])) == 1 and entry['enclosures'][0].get('href'):
                        self.add_enclosure_info(e, entry['enclosures'][0], config.get('filename', True))
                    else:
                        # If there is no enclosure url, check link, then guid field for urls
                        for field in ['link', 'guid']:
                            if entry.get(field):
                                e['url'] = entry[field]
                                break
                else:
                    if entry.get(config['link']):
                        e['url'] = entry[config['link']]
            else:
                # If link was passed as a list, we create a list of urls
                for field in config['link']:
                    if entry.get(field):
                        e.setdefault('url', entry[field])
                        if entry[field] not in e.setdefault('urls', []):
                            e['urls'].append(entry[field])

            if config.get('group_links'):
                # Append a list of urls from enclosures to the urls field if group_links is enabled
                e.setdefault('urls', [e['url']]).extend(
                        [enc.href for enc in entry.get('enclosures', []) if enc.get('href') not in e['urls']])

            if not e.get('url'):
                log.debug('%s does not have link (%s) or enclosure' % (entry.title, config['link']))
                ignored += 1
                continue

            add_entry(e)

        if ignored:
            if not config.get('silent'):
                log.warning('Skipped %s RSS-entries without required information (title, link or enclosures)' % ignored)

        return entries

register_plugin(InputRSS, 'rss', api_ver=2)
<|endoftext|>


<commit_before>from haystack import indexes
from models import BaseImageModel
import datetime

class BaseIndex(indexes.RealTimeSearchIndex):
#class BaseIndex(indexes.SearchIndex):
    text = indexes.CharField(document=True, use_template=True)
    is_live = indexes.BooleanField()
    def prepare_is_live(self, instance):
        if hasattr(instance, 'publication_date'):
            return instance.is_live and (getattr(instance, 'publication_date', None) <= datetime.date.today())
        else:
            return instance.is_live

    name = indexes.CharField(model_attr='name')
    url = indexes.CharField(model_attr='get_absolute_url')
    image = indexes.CharField()
    def prepare_image(self, instance):
        image = instance.primary_image()
        if image:
            if isinstance(image, BaseImageModel):
                return unicode(image.file)
            else:
                return unicode(image)
        else:
            return ''

    class Meta:
        abstract = True
<commit_msg>remove is_live default from search indexes<commit_after>from haystack import indexes
from models import BaseImageModel
import datetime

class BaseIndex(indexes.RealTimeSearchIndex):
#class BaseIndex(indexes.SearchIndex):
    text = indexes.CharField(document=True, use_template=True)

    name = indexes.CharField(model_attr='name')
    url = indexes.CharField(model_attr='get_absolute_url')
    image = indexes.CharField()
    def prepare_image(self, instance):
        image = instance.primary_image()
        if image:
            if isinstance(image, BaseImageModel):
                return unicode(image.file)
            else:
                return unicode(image)
        else:
            return ''

    def get_queryset(self):
        if hasattr(self.model, 'live'):
            return self.model.live
        else:
            return self.model.objects

    class Meta:
        abstract = True
<|endoftext|>


<commit_before># -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4
#
# Flumotion - a video streaming server
#
# flumotion/admin/videotest.py: MVC for video test producer
#
# Copyright (C) 2004 Fluendo
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.

# FIXME: moving this down causes errors
import flumotion.utils.log

# johan says modules should not do this, only apps
import pygtk
pygtk.require('2.0')

import gobject
gobject.threads_init()
import gst
import gst.interfaces
import gtk
import gtk.glade

import flumotion.config

import os
import string

if gtk.pygtk_version < (2,3,96):
   raise SystemExit, "PyGTK 2.3.96 or higher required"

def _debug(*args):
    flumotion.utils.log.debug('videotest', ' '.join(args))

# only Controller is to be shown in epydoc
__all__ = ('Controller', )

class Controller(gobject.GObject):
    '''
    Controller for a video test producer, used to generate a video test feed.
    The controller's model produces a raw video test feed using videotestsrc.
    '''
# FIXME: decide on a good name for prepared that says "you can do stuff with me
# now"
    __gsignals__ = {
        'prepared': (gobject.SIGNAL_RUN_FIRST, None, ()),
    }
    def __init__(self):
        """
        Create a new video test controller.
        The spyglass controller needs to be prepared.
        """
        self.__gobject_init__()
        self.view = View()
        self.model = Model()

    ### public methods

    def prepare(self):
        """
        Prepares the controller.
        Returns immediately.
        Emits 'prepared' signal when it is done preparing.
        """
        # the view's prepare is synchronous for now
        self.view.prepare()
        self.view.connect('width-changed', self.view_width_changed_cb)
        self.view.connect('height-changed', self.view_height_changed_cb)
        self.view.connect('framerate-changed', self.view_framerate_changed_cb)
        self.view.connect('pattern-changed', self.view_pattern_changed_cb)
        # the model doesn't currently have a prepare
        self.view.add_pattern("SMPTE")
        self.view.add_pattern("Snow")
        self.view.add_pattern("0% Black")
        self.view.set_pattern(1)

    ### callbacks
    def view_width_changed_cb(self, view, width):
        _debug("width changed to %d" % width)
        self.model.set_width(width)
    def view_height_changed_cb(self, view, height):
        _debug("height changed to %d" % height)
        self.model.set_height(height)
    def view_framerate_changed_cb(self, view, framerate):
        _debug("framerate changed to %f" % framerate)
        self.model.set_framerate(framerate)
    def view_pattern_changed_cb(self, view, index):
        _debug("pattern changed to index %d" % index)
        self.model.set_pattern(index)

class View(gobject.GObject):
    __gsignals__ = {
        'width-changed': (gobject.SIGNAL_RUN_FIRST, None, (int, )),
        'height-changed': (gobject.SIGNAL_RUN_FIRST, None, (int, )),
        'framerate-changed': (gobject.SIGNAL_RUN_FIRST, None, (float, )),
        'pattern-changed': (gobject.SIGNAL_RUN_FIRST, None, (int, )),
    }

    def __init__(self):
        """
        Construct a new videotest View.
        """
        self.__gobject_init__()
        self._gladefile = os.path.join(flumotion.config.uidir, 'videotest.glade')
        self._glade = gtk.glade.XML(self._gladefile, "videotest-widget")
        self._widget = self._glade.get_widget("videotest-widget")

    def prepare(self):
        # connect callbacks
        w = self._glade.get_widget("width-button")
        w.connect("value-changed", self.width_changed_cb)
        w = self._glade.get_widget("height-button")
        w.connect("value-changed", self.height_changed_cb)
        w = self._glade.get_widget("framerate-button")
        w.connect("value-changed", self.framerate_changed_cb)
        w = self._glade.get_widget("pattern-combo")
        self._pattern_combo = w
        self._pattern_model = gtk.ListStore(str)
        w.set_model(self._pattern_model)
        w.connect("changed", self.pattern_changed_cb)

    def get_widget(self):
        return self._widget

    def add_pattern(self, description):
        'add a pattern description to the pattern combobox'
        # FIXME: for now we don't even store enum keys, which we might want
        # to do if they're added in a different order or not always start
        # at 0
        self._pattern_model.append((description, ))

    def set_pattern(self, index):
        self._pattern_combo.set_active(index)

    ### callbacks

    def width_changed_cb(self, widget):
        width = widget.get_value()
        self.emit('width-changed', width)
    def height_changed_cb(self, widget):
        height = widget.get_value()
        self.emit('height-changed', height)
    def framerate_changed_cb(self, widget):
        framerate = widget.get_value()
        self.emit('framerate-changed', framerate)
    def pattern_changed_cb(self, widget):
        index = widget.get_active()
        self.emit('pattern-changed', index)

class Model:
    def __init__(self):
        self._src = gst.Element('videotestsrc')
        self._src.set_property('sync', gtk.TRUE)
        self._src.get_pad('src').connect("notify::caps", self.have_caps_cb)

    def get_element(self):
        'Gets the element we should link and put in our main bin'
        return self._src

    def get_caps(self):
        'Gets the caps that should be used as filter'
        return self._caps

    def set_width(self, width):
        if not self._caps: return
        self._structure['width'] = width
        _debug("set_width, caps now %s" % self._caps)
        self._relink()

    def set_height(self, height):
        if not self._caps: return
        self._structure['height'] = height
        _debug("set_height, caps now %s" % self._caps)
        self._relink()

    def set_framerate(self, framerate):
        if not self._caps: return
        self._structure['framerate'] = framerate
        _debug("set_framerate,caps now %s" % self._caps)
        self._relink()

    def set_pattern(self, pattern):
        'Sets the pattern enum on videotestsrc to the given pattern enum value'
        self._src.set_property("pattern", pattern)

    def have_caps_cb(self, pad, dunno):
        caps = pad.get_negotiated_caps()
        _debug("HAVE_CAPS: pad %s, caps %s" % (pad, caps))
        self._caps = caps
        self._structure = self._caps.get_structure(0)

    def _relink(self):
        'try a relink of our pad with our caps'
        'if no caps, or no peer, then just return'
        if not self._caps:
            print "ERROR: don't have caps"
            return
        pad = self._src.get_pad('src')
        peer = pad.get_peer()
        if peer:
            # we pause our parent so we can link and unlink
            # normally we'd have relink_filtered do that for us if it worked.
            parent = self._src.get_parent()
            parent.set_state(gst.STATE_PAUSED)
            pad.unlink(peer)
            if not pad.link_filtered(peer, self._caps):
                print "ERROR: could not link %s and %s with caps %s" % (pad, peer, self._caps)
            if not parent.set_state(gst.STATE_PLAYING):
                print "ERROR: could not set parent %s to playing" % parent
        _debug("gst caps of pad now %s" % pad.get_negotiated_caps())

# register our types globally
gobject.type_register(View)
gobject.type_register(Controller)

if __name__ == '__main__':
    exposed_cb_id = -1
    def area_exposed_cb(widget, event, thread, sink):
        'drawing area shown, get xid and start streaming'
        xid = widget.window.xid
        _debug("area exposed, xid is %d" % xid)
        sink.set_xwindow_id(xid)
        thread.set_state(gst.STATE_PLAYING)
        widget.disconnect(exposed_cb_id)

    def controller_prepared_cb(controller, thread):
        # we can set stuff to playing now
        _debug("setting thread to PLAYING")
        thread.set_state(gst.STATE_PLAYING)

    def controller_focus_changed_cb(controller, key, src):
        _debug("focus changed to key %s" % key)
        src.set_property('pattern', key)

    _debug("testing")

    # create fake toplevel model
    thread = gst.Thread()
    csp = gst.Element('ffmpegcolorspace')
    sink = gst.Element('ximagesink')

    # create our controller
    controller = Controller()
    controller.connect("prepared", controller_prepared_cb, thread)
    controller.prepare()

    # embed the view in our fake toplevel view
    window = gtk.Window()
    window.connect('destroy', gtk.main_quit)
    box = gtk.HBox(gtk.FALSE, 2)
    box.add(controller.view.get_widget())
    area = gtk.DrawingArea()
    area.set_size_request(320, 240)
    exposed_cb_id = area.connect('expose-event', area_exposed_cb, thread, sink)
    box.add(area)
    window.add(box)
    window.show_all()

    # embed the model in our fake toplevel model
    src = controller.model.get_element()
    prev = None
    thread.add_many(src, csp, sink)
    src.link(csp)
    csp.link(sink)

    _debug("going into gtk.main")
    gtk.main()
<commit_msg>add stuff to change format, needs fixing for fourcc's<commit_after># -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4
#
# Flumotion - a video streaming server
#
# flumotion/admin/videotest.py: MVC for video test producer
#
# Copyright (C) 2004 Fluendo
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.

# FIXME: moving this down causes errors
import flumotion.utils.log

# johan says modules should not do this, only apps
import pygtk
pygtk.require('2.0')

import gobject
gobject.threads_init()
import gst
import gst.interfaces
import gtk
import gtk.glade

import flumotion.config

import os
import string

if gtk.pygtk_version < (2,3,96):
   raise SystemExit, "PyGTK 2.3.96 or higher required"

def _debug(*args):
    flumotion.utils.log.debug('videotest', ' '.join(args))

# only Controller is to be shown in epydoc
__all__ = ('Controller', )

class Controller(gobject.GObject):
    '''
    Controller for a video test producer, used to generate a video test feed.
    The controller's model produces a raw video test feed using videotestsrc.
    '''
# FIXME: decide on a good name for prepared that says "you can do stuff with me
# now"
    __gsignals__ = {
        'prepared': (gobject.SIGNAL_RUN_FIRST, None, ()),
    }
    def __init__(self):
        """
        Create a new video test controller.
        The spyglass controller needs to be prepared.
        """
        self.__gobject_init__()
        self.view = View()
        self.model = Model()

    ### public methods

    def prepare(self):
        """
        Prepares the controller.
        Returns immediately.
        Emits 'prepared' signal when it is done preparing.
        """
        # the view's prepare is synchronous for now
        self.view.prepare()

        # add possible patterns
        self.view.add_pattern("SMPTE")
        self.view.add_pattern("Snow")
        self.view.add_pattern("0% Black")
        self.view.set_pattern(1)

        # add possible formats
        self.view.add_format("RGB")
        self.view.add_format("YUY2")
        self.view.add_format("I420")
        self.view.set_format(0)

        self.view.connect('width-changed', self.view_width_changed_cb)
        self.view.connect('height-changed', self.view_height_changed_cb)
        self.view.connect('framerate-changed', self.view_framerate_changed_cb)
        self.view.connect('format-changed', self.view_format_changed_cb)
        self.view.connect('pattern-changed', self.view_pattern_changed_cb)
        # the model doesn't currently have a prepare

    ### callbacks
    def view_width_changed_cb(self, view, width):
        _debug("width changed to %d" % width)
        self.model.set_width(width)
    def view_height_changed_cb(self, view, height):
        _debug("height changed to %d" % height)
        self.model.set_height(height)
    def view_framerate_changed_cb(self, view, framerate):
        _debug("framerate changed to %f" % framerate)
        self.model.set_framerate(framerate)
    def view_format_changed_cb(self, view, format):
        _debug("format changed to %f" % format)
        self.model.set_format(format)
    def view_pattern_changed_cb(self, view, index):
        _debug("pattern changed to index %d" % index)
        self.model.set_pattern(index)

class View(gobject.GObject):
    __gsignals__ = {
        'width-changed': (gobject.SIGNAL_RUN_FIRST, None, (int, )),
        'height-changed': (gobject.SIGNAL_RUN_FIRST, None, (int, )),
        'framerate-changed': (gobject.SIGNAL_RUN_FIRST, None, (float, )),
        'format-changed': (gobject.SIGNAL_RUN_FIRST, None, (int, )),
        'pattern-changed': (gobject.SIGNAL_RUN_FIRST, None, (int, )),
    }
    latency = 100 # latency for timeouts on config changes

    def __init__(self):
        """
        Construct a new videotest View.
        """
        self.__gobject_init__()
        self._gladefile = os.path.join(flumotion.config.uidir, 'videotest.glade')
        self._glade = gtk.glade.XML(self._gladefile, "videotest-widget")
        self._widget = self._glade.get_widget("videotest-widget")
        self._width_timeout = 0
        self._height_timeout = 0
        self._framerate_timeout = 0
        self._format_timeout = 0

    def prepare(self):
        # connect callbacks
        w = self._glade.get_widget("width-button")
        w.connect("value-changed", self.width_changed_cb)
        w = self._glade.get_widget("height-button")
        w.connect("value-changed", self.height_changed_cb)
        w = self._glade.get_widget("framerate-button")
        w.connect("value-changed", self.framerate_changed_cb)
        w = self._glade.get_widget("pattern-combo")
        self._pattern_combo = w
        self._pattern_model = gtk.ListStore(str)
        w.set_model(self._pattern_model)
        w.connect("changed", self.pattern_changed_cb)
        w = self._glade.get_widget("format-combo")
        self._format_combo = w
        self._format_model = gtk.ListStore(str)
        w.set_model(self._format_model)
        w.connect("changed", self.format_changed_cb)

    def get_widget(self):
        return self._widget

    def add_pattern(self, description):
        'add a pattern description to the pattern combobox'
        # FIXME: for now we don't even store enum keys, which we might want
        # to do if they're added in a different order or not always start
        # at 0
        self._pattern_model.append((description, ))

    def set_pattern(self, index):
        self._pattern_combo.set_active(index)

    def add_format(self, description):
        'add a format description to the format combobox'
        # FIXME: for now we don't even store enum keys, which we might want
        # to do if they're added in a different order or not always start
        # at 0
        self._format_model.append((description, ))

    def set_format(self, index):
        self._format_combo.set_active(index)

    ### timeouts
    def view_width_timeout(self, widget):
        width = widget.get_value()
        self.emit('width-changed', width)
        self._width_timeout = 0
        return gtk.FALSE
    def view_height_timeout(self, widget):
        height = widget.get_value()
        self.emit('height-changed', height)
        self._height_timeout = 0
        return gtk.FALSE
    def view_framerate_timeout(self, widget):
        framerate = widget.get_value()
        self.emit('framerate-changed', framerate)
        self._framerate_timeout = 0
        return gtk.FALSE
    def view_format_timeout(self, widget):
        format = widget.get_active()
        self.emit('format-changed', format)
        self._format_timeout = 0
        return gtk.FALSE

    ### callbacks
    def width_changed_cb(self, widget):
        if not self._width_timeout:
            id = gobject.timeout_add(View.latency, self.view_width_timeout, widget)
            self._width_timeout = id
    def height_changed_cb(self, widget):
        if not self._height_timeout:
            id = gobject.timeout_add(View.latency, self.view_height_timeout, widget)
            self._height_timeout = id
    def framerate_changed_cb(self, widget):
        if not self._framerate_timeout:
            id = gobject.timeout_add(View.latency, self.view_framerate_timeout, widget)
            self._framerate_timeout = id
    def format_changed_cb(self, widget):
        if not self._format_timeout:
            id = gobject.timeout_add(View.latency, self.view_format_timeout, widget)
            self._format_timeout = id
    def pattern_changed_cb(self, widget):
        index = widget.get_active()
        self.emit('pattern-changed', index)

class Model:
    def __init__(self):
        self._src = gst.Element('videotestsrc')
        self._src.set_property('sync', gtk.TRUE)
        self._src.get_pad('src').connect("notify::caps", self.have_caps_cb)
        self._caps = None

    def get_element(self):
        'Gets the element we should link and put in our main bin'
        return self._src

    def get_caps(self):
        'Gets the caps that should be used as filter'
        return self._caps

    def set_width(self, width):
        if not self._caps: return
        self._structure['width'] = width
        _debug("set_width, caps now %s" % self._caps)
        self._relink()

    def set_height(self, height):
        if not self._caps: return
        self._structure['height'] = height
        _debug("set_height, caps now %s" % self._caps)
        self._relink()

    def set_framerate(self, framerate):
        if not self._caps: return
        self._structure['framerate'] = framerate
        _debug("set_framerate,caps now %s" % self._caps)
        self._relink()

    # FIXME: use lookup table
    # FIXME: setting 'format' fourcc's doesn't work yet
    def set_format(self, format):
        print "SET FORMAT"
        if not self._caps: return
        if format > 2: return
        if format == 0:
            # RGB
            self._structure.set_name('video/x-raw-rgb')
            del self._structure['format']
        else:
            # YUV
            self._structure.set_name('video/x-raw-yuv')
            del self._structure['blue_mask']
            del self._structure['red_mask']
            del self._structure['green_mask']
            del self._structure['depth']
            del self._structure['bpp']
            del self._structure['endianness']
            if format == 1:
                self._structure['format'] = '(fourcc)YUY2'
            #elif format == 2:
            #    self._structure['format'] = 'I420'

        print("set_format,caps now %s" % self._caps)
        self._relink()

    def set_pattern(self, pattern):
        'Sets the pattern enum on videotestsrc to the given pattern enum value'
        self._src.set_property("pattern", pattern)

    def have_caps_cb(self, pad, dunno):
        caps = pad.get_negotiated_caps()
        _debug("HAVE_CAPS: pad %s, caps %s" % (pad, caps))
        self._caps = caps
        self._structure = self._caps.get_structure(0)

    def _relink(self):
        'try a relink of our pad with our caps'
        'if no caps, or no peer, then just return'
        if not self._caps:
            print "ERROR: don't have caps"
            return
        pad = self._src.get_pad('src')
        peer = pad.get_peer()
        if peer:
            # we pause our parent so we can link and unlink
            # normally we'd have relink_filtered do that for us if it worked.
            parent = self._src.get_parent()
            parent.set_state(gst.STATE_PAUSED)
            pad.unlink(peer)
            if not pad.link_filtered(peer, self._caps):
                print "ERROR: could not link %s and %s with caps %s" % (pad, peer, self._caps)
            if not parent.set_state(gst.STATE_PLAYING):
                print "ERROR: could not set parent %s to playing" % parent
        _debug("gst caps of pad now %s" % pad.get_negotiated_caps())

# register our types globally
gobject.type_register(View)
gobject.type_register(Controller)

if __name__ == '__main__':
    exposed_cb_id = -1
    width = 320
    height = 240
    def area_exposed_cb(widget, event, thread, sink):
        'drawing area shown, get xid and start streaming'
        xid = widget.window.xid
        _debug("area exposed, xid is %d" % xid)
        sink.set_xwindow_id(xid)
        thread.set_state(gst.STATE_PLAYING)
        widget.disconnect(exposed_cb_id)

    def controller_prepared_cb(controller, thread):
        # we can set stuff to playing now
        _debug("setting thread to PLAYING")
        thread.set_state(gst.STATE_PLAYING)

    def controller_focus_changed_cb(controller, key, src):
        _debug("focus changed to key %s" % key)
        src.set_property('pattern', key)

    _debug("testing")

    def view_width_changed_cb(widget, value, area):
        width = value
        area.set_size_request(width, height)
    def view_height_changed_cb(widget, value, area):
        height = value
        area.set_size_request(width, height)

    # create fake toplevel model
    thread = gst.Thread()
    csp = gst.Element('ffmpegcolorspace')
    sink = gst.Element('ximagesink')

    # create our controller
    controller = Controller()
    controller.connect("prepared", controller_prepared_cb, thread)
    controller.prepare()

    # embed the view in our fake toplevel view
    window = gtk.Window()
    window.connect('destroy', gtk.main_quit)
    box = gtk.HBox(gtk.FALSE, 2)
    box.add(controller.view.get_widget())
    area = gtk.DrawingArea()
    area.set_size_request(width, height)
    # FIXME: changing the area's size request this way makes it flash a lot
    controller.view.connect('width-changed', view_width_changed_cb, area)
    controller.view.connect('height-changed', view_height_changed_cb, area)
    exposed_cb_id = area.connect('expose-event', area_exposed_cb, thread, sink)
    box.add(area)
    window.add(box)
    window.show_all()

    # embed the model in our fake toplevel model
    src = controller.model.get_element()
    prev = None
    thread.add_many(src, csp, sink)
    src.link(csp)
    csp.link(sink)

    _debug("going into gtk.main")
    gtk.main()
<|endoftext|>


<commit_before># Copyright 2016 Tecnativa - Antonio Espinosa
# Copyright 2016 Tecnativa - Sergio Teruel
# Copyright 2016-2018 Tecnativa - Pedro M. Baeza
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

from datetime import datetime

from dateutil.relativedelta import relativedelta

from odoo import _, api, fields, models
from odoo.exceptions import UserError

class AccountAnalyticLine(models.Model):
    _inherit = "account.analytic.line"
    _order = "date_time desc"

    date_time = fields.Datetime(
        string="Start Time", default=fields.Datetime.now, copy=False
    )
    date_time_end = fields.Datetime(
        string="End Time",
        compute="_compute_date_time_end",
        inverse="_inverse_date_time_end",
    )
    show_time_control = fields.Selection(
        selection=[("resume", "Resume"), ("stop", "Stop")],
        compute="_compute_show_time_control",
        help="Indicate which time control button to show, if any.",
    )

    @api.depends("date_time", "unit_amount", "product_uom_id")
    def _compute_date_time_end(self):
        hour_uom = self.env.ref("uom.product_uom_hour")
        for record in self:
            if (
                record.product_uom_id == hour_uom
                and record.date_time
                and record.unit_amount
            ):
                record.date_time_end = record.date_time + relativedelta(
                    hours=record.unit_amount
                )
            else:
                record.date_time_end = record.date_time_end

    def _inverse_date_time_end(self):
        hour_uom = self.env.ref("uom.product_uom_hour")
        for record in self.filtered(lambda x: x.date_time and x.date_time_end):
            if record.product_uom_id == hour_uom:
                record.unit_amount = (
                    record.date_time_end - record.date_time
                ).seconds / 3600

    @api.model
    def _eval_date(self, vals):
        if vals.get("date_time"):
            return dict(vals, date=self._convert_datetime_to_date(vals["date_time"]))
        return vals

    def _convert_datetime_to_date(self, datetime_):
        if isinstance(datetime_, str):
            datetime_ = fields.Datetime.from_string(datetime_)
        return fields.Date.context_today(self, datetime_)

    @api.model
    def _running_domain(self):
        """Domain to find running timesheet lines."""
        return [
            ("date_time", "!=", False),
            ("employee_id", "in", self.env.user.employee_ids.ids),
            ("project_id.allow_timesheets", "=", True),
            ("unit_amount", "=", 0),
        ]

    @api.model
    def _duration(self, start, end):
        """Compute float duration between start and end."""
        try:
            return (end - start).total_seconds() / 3600
        except TypeError:
            return 0

    @api.depends("employee_id", "unit_amount")
    def _compute_show_time_control(self):
        """Decide when to show time controls."""
        for one in self:
            if one.employee_id not in self.env.user.employee_ids:
                one.show_time_control = False
            elif one.unit_amount or not one.date_time:
                one.show_time_control = "resume"
            else:
                one.show_time_control = "stop"

    @api.model_create_multi
    def create(self, vals_list):
        return super().create(list(map(self._eval_date, vals_list)))

    def write(self, vals):
        return super().write(self._eval_date(vals))

    def button_resume_work(self):
        """Create a new record starting now, with a running timer."""
        return {
            "name": _("Resume work"),
            "res_model": "hr.timesheet.switch",
            "target": "new",
            "type": "ir.actions.act_window",
            "view_mode": "form",
            "view_type": "form",
        }

    def button_end_work(self):
        end = fields.Datetime.to_datetime(
            self.env.context.get("stop_dt", datetime.now())
        )
        for line in self:
            if line.unit_amount:
                raise UserError(
                    _(
                        "Cannot stop timer %d because it is not running. "
                        "Refresh the page and check again."
                    )
                    % line.id
                )
            line.unit_amount = line._duration(line.date_time, end)
        return True
<commit_msg>FIX project_timesheet_time_control when not using employees<commit_after># Copyright 2016 Tecnativa - Antonio Espinosa
# Copyright 2016 Tecnativa - Sergio Teruel
# Copyright 2016-2018 Tecnativa - Pedro M. Baeza
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

from datetime import datetime

from dateutil.relativedelta import relativedelta

from odoo import _, api, fields, models
from odoo.exceptions import UserError

class AccountAnalyticLine(models.Model):
    _inherit = "account.analytic.line"
    _order = "date_time desc"

    date_time = fields.Datetime(
        string="Start Time", default=fields.Datetime.now, copy=False
    )
    date_time_end = fields.Datetime(
        string="End Time",
        compute="_compute_date_time_end",
        inverse="_inverse_date_time_end",
    )
    show_time_control = fields.Selection(
        selection=[("resume", "Resume"), ("stop", "Stop")],
        compute="_compute_show_time_control",
        help="Indicate which time control button to show, if any.",
    )

    @api.depends("date_time", "unit_amount", "product_uom_id")
    def _compute_date_time_end(self):
        hour_uom = self.env.ref("uom.product_uom_hour")
        for record in self:
            if (
                record.product_uom_id == hour_uom
                and record.date_time
                and record.unit_amount
            ):
                record.date_time_end = record.date_time + relativedelta(
                    hours=record.unit_amount
                )
            else:
                record.date_time_end = record.date_time_end

    def _inverse_date_time_end(self):
        hour_uom = self.env.ref("uom.product_uom_hour")
        for record in self.filtered(lambda x: x.date_time and x.date_time_end):
            if record.product_uom_id == hour_uom:
                record.unit_amount = (
                    record.date_time_end - record.date_time
                ).seconds / 3600

    @api.model
    def _eval_date(self, vals):
        if vals.get("date_time"):
            return dict(vals, date=self._convert_datetime_to_date(vals["date_time"]))
        return vals

    def _convert_datetime_to_date(self, datetime_):
        if isinstance(datetime_, str):
            datetime_ = fields.Datetime.from_string(datetime_)
        return fields.Date.context_today(self, datetime_)

    @api.model
    def _running_domain(self):
        """Domain to find running timesheet lines."""
        return [
            ("date_time", "!=", False),
            ("user_id", "=", self.env.user.id),
            ("project_id.allow_timesheets", "=", True),
            ("unit_amount", "=", 0),
        ]

    @api.model
    def _duration(self, start, end):
        """Compute float duration between start and end."""
        try:
            return (end - start).total_seconds() / 3600
        except TypeError:
            return 0

    @api.depends("employee_id", "unit_amount")
    def _compute_show_time_control(self):
        """Decide when to show time controls."""
        for one in self:
            if one.employee_id not in self.env.user.employee_ids:
                one.show_time_control = False
            elif one.unit_amount or not one.date_time:
                one.show_time_control = "resume"
            else:
                one.show_time_control = "stop"

    @api.model_create_multi
    def create(self, vals_list):
        return super().create(list(map(self._eval_date, vals_list)))

    def write(self, vals):
        return super().write(self._eval_date(vals))

    def button_resume_work(self):
        """Create a new record starting now, with a running timer."""
        return {
            "name": _("Resume work"),
            "res_model": "hr.timesheet.switch",
            "target": "new",
            "type": "ir.actions.act_window",
            "view_mode": "form",
            "view_type": "form",
        }

    def button_end_work(self):
        end = fields.Datetime.to_datetime(
            self.env.context.get("stop_dt", datetime.now())
        )
        for line in self:
            if line.unit_amount:
                raise UserError(
                    _(
                        "Cannot stop timer %d because it is not running. "
                        "Refresh the page and check again."
                    )
                    % line.id
                )
            line.unit_amount = line._duration(line.date_time, end)
        return True
<|endoftext|>


<commit_before>import json
import logging
import os
import subprocess
import time
from abc import ABCMeta
from datetime import timedelta
from enum import Enum
from typing import Any, List, Dict, Optional

from baton._baton._constants import BATON_ERROR_MESSAGE_KEY, IRODS_ERROR_USER_FILE_DOES_NOT_EXIST, BATON_ERROR_PROPERTY,\
    BATON_ERROR_CODE_KEY, IRODS_ERROR_CATALOG_ALREADY_HAS_ITEM_BY_THAT_NAME, IRODS_ERROR_CAT_SUCCESS_BUT_WITH_NO_INFO, \
    IRODS_ERROR_CAT_INVALID_ARGUMENT

class BatonBinary(Enum):
    """
    Names of baton binaries.
    """
    BATON = "baton"
    BATON_METAQUERY = "baton-metaquery"
    BATON_LIST = "baton-list"
    BATON_SPECIFIC_QUERY = "baton-specificquery"
    BATON_GET = "baton-get"
    BATON_METAMOD = "baton-metamod"
    BATON_CHMOD = "baton-chmod"

class BatonRunner(metaclass=ABCMeta):
    """
    Baton query runner.
    """
    @staticmethod
    def validate_baton_binaries_location(baton_binaries_directory: str) -> Optional[Exception]:
        """
        Validates that the given directory contains the baton binaries required to use the runner.
        :param baton_binaries_directory: the directory to check
        :return: exception that describes the issue else `None` if no issues
        """
        if os.path.isfile(baton_binaries_directory):
            return ValueError("The given baton binary directory (%s) is not a directory! Be sure to provide the path "
                              "to the directory containing the baton binaries, not the binary named `baton`")

        for baton_binary in BatonBinary:
            binary_location = os.path.join(baton_binaries_directory, baton_binary.value)
            if not (os.path.isfile(binary_location) and os.access(binary_location, os.X_OK)):
                return ValueError("The given baton binary directory (%s) did not contain all of the required binaries "
                                  "with executable permissions (%s)"
                                  % (baton_binaries_directory, [name.value for name in BatonBinary]))

        return None

    @staticmethod
    def _raise_any_errors_given_in_baton_out(baton_out_as_json: List[Dict]):
        """
        Raises any errors that baton has expressed in its output.
        :param baton_out_as_json: the output baton gave as parsed serialization
        """
        if not isinstance(baton_out_as_json, list):
            baton_out_as_json = [baton_out_as_json]

        for baton_item_as_json in baton_out_as_json:
            if BATON_ERROR_PROPERTY in baton_item_as_json:
                error = baton_item_as_json[BATON_ERROR_PROPERTY]
                error_message = error[BATON_ERROR_MESSAGE_KEY]
                error_code = error[BATON_ERROR_CODE_KEY]

                # Working around baton issue: https://github.com/wtsi-npg/baton/issues/155
                if error_code == IRODS_ERROR_USER_FILE_DOES_NOT_EXIST or \
                        (error_code == IRODS_ERROR_CAT_INVALID_ARGUMENT and "Failed to modify permissions" in error_message):
                    raise FileNotFoundError(error_message)
                elif error_code == IRODS_ERROR_CATALOG_ALREADY_HAS_ITEM_BY_THAT_NAME \
                        or error_code == IRODS_ERROR_CAT_SUCCESS_BUT_WITH_NO_INFO:
                    raise KeyError(error_message)
                else:
                    raise RuntimeError(error_message)

    def __init__(self, baton_binaries_directory: str, skip_baton_binaries_validation: bool=False,
                 timeout_queries_after: timedelta=None):
        """
        Constructor.
        :param baton_binaries_directory: the host of baton's binaries
        :param irods_query_zone: the iRODS zone to query
        :param skip_baton_binaries_validation: skips validation of baton binaries (intending for testing only)
        """
        if not skip_baton_binaries_validation:
            exception = BatonRunner.validate_baton_binaries_location(baton_binaries_directory)
            if exception is not None:
                raise exception

        self._baton_binaries_directory = baton_binaries_directory
        self.timeout_queries_after = timeout_queries_after

    def run_baton_query(self, baton_binary: BatonBinary, program_arguments: List[str]=None, input_data: Any=None) \
            -> List[Dict]:
        """
        Runs a baton query.
        :param baton_binary: the baton binary to use
        :param program_arguments: arguments to give to the baton binary
        :param input_data: input data to the baton binary
        :return: parsed serialization returned by baton
        """
        if program_arguments is None:
            program_arguments = []

        baton_binary_location = os.path.join(self._baton_binaries_directory, baton_binary.value)
        program_arguments = [baton_binary_location] + program_arguments

        logging.info("Running baton command: '%s' with data '%s'" % (program_arguments, input_data))
        start_at = time.monotonic()
        baton_out = self._run_command(program_arguments, input_data=input_data)
        time_taken_to_run_query = time.monotonic() - start_at
        logging.debug("baton output (took %s seconds, wall time): %s" % (time_taken_to_run_query, baton_out))

        if len(baton_out) == 0:
            return []
        if len(baton_out) > 0 and baton_out[0] != '[':
            # If information about multiple files is returned, baton does not return valid JSON - it returns a line
            # separated list of JSON, where each line corresponds to a different file
            baton_out = "[%s]" % baton_out.replace('\n', ',')

        baton_out_as_json = json.loads(baton_out)
        BatonRunner._raise_any_errors_given_in_baton_out(baton_out_as_json)

        return baton_out_as_json

    def _run_command(self, arguments: List[str], input_data: Any=None, output_encoding: str="utf-8") -> str:
        """
        Run a command as a subprocess.

        Ignores errors given over stderr if there is output on stdout (this is the case where baton has been run
        correctly and has expressed the error in it's JSON out, which can be handled more appropriately upstream to this
        method.)
        :param arguments: the arguments to run
        :param input_data: the input data to pass to the subprocess
        :param output_encoding: optional specification of the output encoding to expect
        :return: the process' standard out
        """
        process = subprocess.Popen(arguments, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)

        if isinstance(input_data, List):
            for to_write in input_data:
                to_write_as_json = json.dumps(to_write)
                process.stdin.write(str.encode(to_write_as_json))
            input_data = None
        else:
            input_data = str.encode(json.dumps(input_data))

        timeout_in_seconds = self.timeout_queries_after.total_seconds() if self.timeout_queries_after is not None \
            else None
        out, error = process.communicate(input=input_data, timeout=timeout_in_seconds)
        if len(out) == 0 and len(error) > 0:
            raise RuntimeError(error)

        return out.decode(output_encoding).rstrip()
<commit_msg>Fixes #39.<commit_after>import json
import logging
import os
import subprocess
import time
from abc import ABCMeta
from datetime import timedelta
from enum import Enum
from typing import Any, List, Dict, Optional

from baton._baton._constants import BATON_ERROR_MESSAGE_KEY, IRODS_ERROR_USER_FILE_DOES_NOT_EXIST, BATON_ERROR_PROPERTY,\
    BATON_ERROR_CODE_KEY, IRODS_ERROR_CATALOG_ALREADY_HAS_ITEM_BY_THAT_NAME, IRODS_ERROR_CAT_SUCCESS_BUT_WITH_NO_INFO, \
    IRODS_ERROR_CAT_INVALID_ARGUMENT

_logger = logging.getLogger(__name__)

class BatonBinary(Enum):
    """
    Names of baton binaries.
    """
    BATON = "baton"
    BATON_METAQUERY = "baton-metaquery"
    BATON_LIST = "baton-list"
    BATON_SPECIFIC_QUERY = "baton-specificquery"
    BATON_GET = "baton-get"
    BATON_METAMOD = "baton-metamod"
    BATON_CHMOD = "baton-chmod"

class BatonRunner(metaclass=ABCMeta):
    """
    Baton query runner.
    """
    @staticmethod
    def validate_baton_binaries_location(baton_binaries_directory: str) -> Optional[Exception]:
        """
        Validates that the given directory contains the baton binaries required to use the runner.
        :param baton_binaries_directory: the directory to check
        :return: exception that describes the issue else `None` if no issues
        """
        if os.path.isfile(baton_binaries_directory):
            return ValueError("The given baton binary directory (%s) is not a directory! Be sure to provide the path "
                              "to the directory containing the baton binaries, not the binary named `baton`")

        for baton_binary in BatonBinary:
            binary_location = os.path.join(baton_binaries_directory, baton_binary.value)
            if not (os.path.isfile(binary_location) and os.access(binary_location, os.X_OK)):
                return ValueError("The given baton binary directory (%s) did not contain all of the required binaries "
                                  "with executable permissions (%s)"
                                  % (baton_binaries_directory, [name.value for name in BatonBinary]))

        return None

    @staticmethod
    def _raise_any_errors_given_in_baton_out(baton_out_as_json: List[Dict]):
        """
        Raises any errors that baton has expressed in its output.
        :param baton_out_as_json: the output baton gave as parsed serialization
        """
        if not isinstance(baton_out_as_json, list):
            baton_out_as_json = [baton_out_as_json]

        for baton_item_as_json in baton_out_as_json:
            if BATON_ERROR_PROPERTY in baton_item_as_json:
                error = baton_item_as_json[BATON_ERROR_PROPERTY]
                error_message = error[BATON_ERROR_MESSAGE_KEY]
                error_code = error[BATON_ERROR_CODE_KEY]

                # Working around baton issue: https://github.com/wtsi-npg/baton/issues/155
                if error_code == IRODS_ERROR_USER_FILE_DOES_NOT_EXIST or \
                        (error_code == IRODS_ERROR_CAT_INVALID_ARGUMENT and "Failed to modify permissions" in error_message):
                    raise FileNotFoundError(error_message)
                elif error_code == IRODS_ERROR_CATALOG_ALREADY_HAS_ITEM_BY_THAT_NAME \
                        or error_code == IRODS_ERROR_CAT_SUCCESS_BUT_WITH_NO_INFO:
                    raise KeyError(error_message)
                else:
                    raise RuntimeError(error_message)

    def __init__(self, baton_binaries_directory: str, skip_baton_binaries_validation: bool=False,
                 timeout_queries_after: timedelta=None):
        """
        Constructor.
        :param baton_binaries_directory: the host of baton's binaries
        :param irods_query_zone: the iRODS zone to query
        :param skip_baton_binaries_validation: skips validation of baton binaries (intending for testing only)
        """
        if not skip_baton_binaries_validation:
            exception = BatonRunner.validate_baton_binaries_location(baton_binaries_directory)
            if exception is not None:
                raise exception

        self._baton_binaries_directory = baton_binaries_directory
        self.timeout_queries_after = timeout_queries_after

    def run_baton_query(self, baton_binary: BatonBinary, program_arguments: List[str]=None, input_data: Any=None) \
            -> List[Dict]:
        """
        Runs a baton query.
        :param baton_binary: the baton binary to use
        :param program_arguments: arguments to give to the baton binary
        :param input_data: input data to the baton binary
        :return: parsed serialization returned by baton
        """
        if program_arguments is None:
            program_arguments = []

        baton_binary_location = os.path.join(self._baton_binaries_directory, baton_binary.value)
        program_arguments = [baton_binary_location] + program_arguments

        _logger.info("Running baton command: '%s' with data '%s'" % (program_arguments, input_data))
        start_at = time.monotonic()
        baton_out = self._run_command(program_arguments, input_data=input_data)
        time_taken_to_run_query = time.monotonic() - start_at
        _logger.debug("baton output (took %s seconds, wall time): %s" % (time_taken_to_run_query, baton_out))

        if len(baton_out) == 0:
            return []
        if len(baton_out) > 0 and baton_out[0] != '[':
            # If information about multiple files is returned, baton does not return valid JSON - it returns a line
            # separated list of JSON, where each line corresponds to a different file
            baton_out = "[%s]" % baton_out.replace('\n', ',')

        baton_out_as_json = json.loads(baton_out)
        BatonRunner._raise_any_errors_given_in_baton_out(baton_out_as_json)

        return baton_out_as_json

    def _run_command(self, arguments: List[str], input_data: Any=None, output_encoding: str="utf-8") -> str:
        """
        Run a command as a subprocess.

        Ignores errors given over stderr if there is output on stdout (this is the case where baton has been run
        correctly and has expressed the error in it's JSON out, which can be handled more appropriately upstream to this
        method.)
        :param arguments: the arguments to run
        :param input_data: the input data to pass to the subprocess
        :param output_encoding: optional specification of the output encoding to expect
        :return: the process' standard out
        """
        process = subprocess.Popen(arguments, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)

        if isinstance(input_data, List):
            for to_write in input_data:
                to_write_as_json = json.dumps(to_write)
                process.stdin.write(str.encode(to_write_as_json))
            input_data = None
        else:
            input_data = str.encode(json.dumps(input_data))

        timeout_in_seconds = self.timeout_queries_after.total_seconds() if self.timeout_queries_after is not None \
            else None
        out, error = process.communicate(input=input_data, timeout=timeout_in_seconds)
        if len(out) == 0 and len(error) > 0:
            raise RuntimeError(error)

        return out.decode(output_encoding).rstrip()
<|endoftext|>


<commit_before>from itertools import product
from unittest import TestCase, main

def has_nested_combos(input_types):
    ''' returns True if any object in the type collection is not a type '''
    assert isinstance(input_types, (list, tuple, set)), input_types
    return any(not isinstance(i, type) for i in input_types)

def flatten_types(input_types):
    ''' returns every type from a nested structure '''
    assert isinstance(input_types, (list, tuple, set, type)), input_types
    if isinstance(input_types, type):
        yield input_types
    else:
        for i in input_types:
            yield from flatten_types(i)

class IllegalTypeComboSettings(ValueError):
    '''raised if illegal combination of type_combos are entered with a give arg_count'''

def type_combos(input_types, arg_count):
    ''' expands all combinations generated from the user's given settings '''
    assert isinstance(input_types, (list, tuple, set)), input_types
    assert isinstance(arg_count, int), arg_count
    assert arg_count > 0, arg_count
    if arg_count == 1:
        # flatten out the types and yield the unique product
        yield from product(set(flatten_types(input_types)))
    elif has_nested_combos(input_types):
        if len(input_types) == arg_count:
            # turn every input type into a tuple
            pipe = (
                (i,) if isinstance(i, type) else i
                for i in input_types
            )
            # yield out the product of those tuples
            yield from product(*pipe)
        else:
            raise IllegalTypeComboSettings(f'{locals()}')
    else:
        # yield out the product of every type for each arg
        yield from product(input_types, repeat=arg_count)

class Test_input_type_combos(TestCase):
    def test_basic_one_type_one_arg(self):
        self.assertEqual(set(input_type_combos((int, ), 1)), {(int, )})

    def test_basic_two_types_one_arg(self):
        self.assertEqual(set(input_type_combos((int, bool), 1)),
                         {(int, )(bool, )})

    def test_basic_one_type_two_args(self):
        self.assertEqual(set(input_type_combos((int, ), 2)), {(int, int)})

    def test_basic_two_types_two_args(self):
        self.assertEqual(set(input_type_combos((int, bool), 2)), {(int, bool)})

    def test_one_nested_two_args(self):
        self.assertEqual(set(input_type_combos((int, (bool, str)), 2)),
                         {(int, bool), (int, str)})

    def test_two_nested_three_args(self):
        self.assertEqual(
            set(input_type_combos(((int, float), bool, (bool, str)), 2)), {
                (float, bool, bool),
                (float, bool, str),
                (int, bool, bool),
                (int, bool, str)
            })

    def test_two_nested_three_args_different_sizes(self):
        self.assertEqual(
            set(input_type_combos(((int, float), bool, (bool, str, int)), 2)),
            {
                (float, bool, bool),
                (float, bool, str),
                (float, bool, str),
                (int, bool, bool),
                (int, bool, str),
                (int, bool, str)
            })

    def test_three_nested_three_args_different_sizes(self):
        self.assertEqual(
            set(
                input_type_combos(
                    ((int, float), (bool, int), (bool, str, int)), 2)), {
                        (float, bool, bool),
                        (float, bool, str),
                        (float, bool, str),
                        (float, int, bool),
                        (float, int, str),
                        (float, int, str),
                        (int, bool, bool),
                        (int, bool, str),
                        (int, bool, str),
                        (int, int, bool),
                        (int, int, str),
                        (int, int, str)
                    })

    def test_no_nested_single_combos(self):
        with self.assertRaises(ValueError):
            set(input_type_combos((int, float, (str, bool)), 1))

    def test_not_enough_args_for_types(self):
        with self.assertRaises(ValueError):
            set(
                input_type_combos(
                    (
                        int,
                        float,
                        (
                            str, bool
                        )  # no nested cases here since everything will apply to every arg when types outnumbers args
                    ),
                    2))

    def test_multiply_types_for_not_enough_args(self):
        self.assertEqual(
            set(input_type_combos((
                int,
                float,
                str
            ), 2)), {(int, int), (int, float), (int, str), (float, int),
                     (float, float), (float, str), (str, int), (str, float),
                     (str, str)})

if __name__ == '__main__':
    main(verbosity=2)
<commit_msg>renamed type_combos to input_type_combos<commit_after>from itertools import product
from unittest import TestCase, main

def has_nested_combos(input_types):
    ''' returns True if any object in the type collection is not a type '''
    assert isinstance(input_types, (list, tuple, set)), input_types
    return any(not isinstance(i, type) for i in input_types)

def flatten_types(input_types):
    ''' returns every type from a nested structure '''
    assert isinstance(input_types, (list, tuple, set, type)), input_types
    if isinstance(input_types, type):
        yield input_types
    else:
        for i in input_types:
            yield from flatten_types(i)

class IllegalTypeComboSettings(ValueError):
    '''raised if illegal combination of type_combos are entered with a give arg_count'''

def input_type_combos(input_types, arg_count):
    ''' expands all combinations generated from the user's given settings '''
    assert isinstance(input_types, (list, tuple, set)), input_types
    assert isinstance(arg_count, int), arg_count
    assert arg_count > 0, arg_count
    if arg_count == 1:
        # flatten out the types and yield the unique product
        yield from product(set(flatten_types(input_types)))
    elif has_nested_combos(input_types):
        if len(input_types) == arg_count:
            # turn every input type into a tuple
            pipe = (
                (i,) if isinstance(i, type) else i
                for i in input_types
            )
            # yield out the product of those tuples
            yield from product(*pipe)
        else:
            raise IllegalTypeComboSettings(f'{locals()}')
    else:
        # yield out the product of every type for each arg
        yield from product(input_types, repeat=arg_count)

class Test_input_type_combos(TestCase):
    def test_basic_one_type_one_arg(self):
        self.assertEqual(set(input_type_combos((int, ), 1)), {(int, )})

    def test_basic_two_types_one_arg(self):
        self.assertEqual(set(input_type_combos((int, bool), 1)),
                         {(int, )(bool, )})

    def test_basic_one_type_two_args(self):
        self.assertEqual(set(input_type_combos((int, ), 2)), {(int, int)})

    def test_basic_two_types_two_args(self):
        self.assertEqual(set(input_type_combos((int, bool), 2)), {(int, bool)})

    def test_one_nested_two_args(self):
        self.assertEqual(set(input_type_combos((int, (bool, str)), 2)),
                         {(int, bool), (int, str)})

    def test_two_nested_three_args(self):
        self.assertEqual(
            set(input_type_combos(((int, float), bool, (bool, str)), 2)), {
                (float, bool, bool),
                (float, bool, str),
                (int, bool, bool),
                (int, bool, str)
            })

    def test_two_nested_three_args_different_sizes(self):
        self.assertEqual(
            set(input_type_combos(((int, float), bool, (bool, str, int)), 2)),
            {
                (float, bool, bool),
                (float, bool, str),
                (float, bool, str),
                (int, bool, bool),
                (int, bool, str),
                (int, bool, str)
            })

    def test_three_nested_three_args_different_sizes(self):
        self.assertEqual(
            set(
                input_type_combos(
                    ((int, float), (bool, int), (bool, str, int)), 2)), {
                        (float, bool, bool),
                        (float, bool, str),
                        (float, bool, str),
                        (float, int, bool),
                        (float, int, str),
                        (float, int, str),
                        (int, bool, bool),
                        (int, bool, str),
                        (int, bool, str),
                        (int, int, bool),
                        (int, int, str),
                        (int, int, str)
                    })

    def test_no_nested_single_combos(self):
        with self.assertRaises(ValueError):
            set(input_type_combos((int, float, (str, bool)), 1))

    def test_not_enough_args_for_types(self):
        with self.assertRaises(ValueError):
            set(
                input_type_combos(
                    (
                        int,
                        float,
                        (
                            str, bool
                        )  # no nested cases here since everything will apply to every arg when types outnumbers args
                    ),
                    2))

    def test_multiply_types_for_not_enough_args(self):
        self.assertEqual(
            set(input_type_combos((
                int,
                float,
                str
            ), 2)), {(int, int), (int, float), (int, str), (float, int),
                     (float, float), (float, str), (str, int), (str, float),
                     (str, str)})

if __name__ == '__main__':
    main(verbosity=2)
<|endoftext|>


<commit_before>import sublime, sublime_plugin
import subprocess

class CompileCommand(sublime_plugin.TextCommand):

	def run(self, edit):

		coffeeStr = getSelection(self.view)

		compiledJS = compile(coffeeStr)
		showResult(compiledJS)

class CompileAndRunCommand(sublime_plugin.TextCommand):

	def run(self, edit):

		view = self.view
		coffeeStr = getSelection(view)
		compiledCode = compile(coffeeStr)
		output = run()
		showResult(output)

		return output

class EnterLiveSession(sublime_plugin.TextCommand):

	def run(self, edit):

		toggledValue = not self.view.settings().get("isLiveRunnerSession", False)
		self.view.settings().set("isLiveRunnerSession", toggledValue)

class ModifyListener(sublime_plugin.EventListener):

	def on_modified_async(self, view):

		if view.settings().get("isLiveRunnerSession", False):
			coffeeStr = view.substr(sublime.Region(0, view.size()))
			compiledJS = compile(coffeeStr)

			if compiledJS:
				showResult(compiledJS)
			else:
				markFaultyCode()

			print("compiledJS", compiledJS)

class TestEvalPrinterCommand(sublime_plugin.TextCommand):

	def run(self, edit, action, codeStr):

		if action == "compile":
			output = compile(codeStr)
		else:
			compile(codeStr)
			output = run()

		self.view.run_command("append", {"characters": output.decode("ascii")})

def getSelection(view):

	codeParts = []

	for region in view.sel():

		if region.a == region.b:
			region = view.line(region)

		code = view.substr(region)
		codeParts.append(code)

	return "\n".join(codeParts)

def showResult(resultStr):

	myOutput = sublime.active_window().create_output_panel("myOutput")
	sublime.active_window().run_command("show_panel", {"panel": "output.myOutput"})

	myOutput.run_command("append", {"characters": resultStr})
	myOutput.set_syntax_file("Packages/JavaScript/JavaScript.tmLanguage")

def markFaultyCode():

	pass

def executeCommand(cmd):

	startupinfo = None
	if sublime.platform() == "windows":
	    startupinfo = subprocess.STARTUPINFO()
	    startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW

	sp = subprocess.Popen(cmd,
		startupinfo=startupinfo,
		shell=True,
	# 	stderr=subprocess.PIPE,
		stdout=subprocess.PIPE)

	out, err = sp.communicate()
	return out

def compile(coffeeStr):

	with open("code.coffee", "wt") as out_file:
		out_file.write(coffeeStr)

	cmd = "coffee -p -b code.coffee"
	compiledJS = executeCommand(cmd)

	return compiledJS

def run():

	return executeCommand("coffee -p -b code.coffee | node -p")
<commit_msg>output error if error occurs<commit_after>import sublime, sublime_plugin
import subprocess

class CompileCommand(sublime_plugin.TextCommand):

	def run(self, edit):

		coffeeStr = getSelection(self.view)

		compiledJS = compile(coffeeStr)
		showResult(compiledJS)

class CompileAndRunCommand(sublime_plugin.TextCommand):

	def run(self, edit):

		view = self.view
		coffeeStr = getSelection(view)
		compiledCode = compile(coffeeStr)
		output = run()
		showResult(output)

		return output

class EnterLiveSession(sublime_plugin.TextCommand):

	def run(self, edit):

		toggledValue = not self.view.settings().get("isLiveRunnerSession", False)
		self.view.settings().set("isLiveRunnerSession", toggledValue)

class ModifyListener(sublime_plugin.EventListener):

	def on_modified_async(self, view):

		if view.settings().get("isLiveRunnerSession", False):
			coffeeStr = view.substr(sublime.Region(0, view.size()))
			compiledJS = compile(coffeeStr)

			if compiledJS:
				showResult(compiledJS)
			else:
				markFaultyCode()

			print("compiledJS", compiledJS)

class TestEvalPrinterCommand(sublime_plugin.TextCommand):

	def run(self, edit, action, codeStr):

		if action == "compile":
			output = compile(codeStr)
		else:
			compile(codeStr)
			output = run()

		self.view.run_command("append", {"characters": output.decode("ascii")})

def getSelection(view):

	codeParts = []

	for region in view.sel():

		if region.a == region.b:
			region = view.line(region)

		code = view.substr(region)
		codeParts.append(code)

	return "\n".join(codeParts)

def showResult(resultStr):

	myOutput = sublime.active_window().create_output_panel("myOutput")
	sublime.active_window().run_command("show_panel", {"panel": "output.myOutput"})

	myOutput.run_command("append", {"characters": resultStr})
	myOutput.set_syntax_file("Packages/JavaScript/JavaScript.tmLanguage")

def markFaultyCode():

	pass

def executeCommand(cmd):

	startupinfo = None
	if sublime.platform() == "windows":
	    startupinfo = subprocess.STARTUPINFO()
	    startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW

	sp = subprocess.Popen(cmd,
		startupinfo=startupinfo,
		shell=True,
		stderr=subprocess.PIPE,
		stdout=subprocess.PIPE)

	out, err = sp.communicate()
	if err:
		# raise Exception("Couldn't execute command " + cmd, err)
		return err

	return out

def compile(coffeeStr):

	with open("code.coffee", "wt") as out_file:
		out_file.write(coffeeStr)

	cmd = "coffee -p -b code.coffee"
	compiledJS = executeCommand(cmd)

	return compiledJS

def run():

	return executeCommand("coffee -p -b code.coffee | node -p")
<|endoftext|>