<commit_before>#!/usr/bin/env python
from __future__ import print_function
from __future__ import division
from wcwidth import wcwidth, wcwidth_cjk
from blessed import Terminal
from functools import partial
import unicodedata
import string
import signal

# BEGIN, python 2.6 through 3.4 compatibilities,

echo = partial(print, end='')

try:
    flushout = partial(print, end='', flush=True)
    flushout('')
except TypeError as err:
    assert "'flush' is an invalid keyword argument" in err.args[0]

    def flushout():
        import sys
        try:
            sys.stdout.flush()
        except BrokenPipeError:
            pass

try:
    _ = unichr(0)
except NameError as err:
    if err.args[0] == "name 'unichr' is not defined":
        unichr = chr
    else:
        raise

try:
    _ = xrange(0)
except NameError as err:
    if err.args[0] == "name 'xrange' is not defined":
        xrange = range
    else:
        raise

# END, python 2.6 - 3.3 compatibilities

# some poor python builds (apple, etc.) are narrow, presumably
# for smaller memory footprint of character strings.
try:
    _ = unichr(0x10000)
    LIMIT_UCS = 0x3fffd
except ValueError as err:
    assert 'narrow Python build' in err.args[0], err.args
    LIMIT_UCS = 0x10000
    import time
    warnings.warn('narrow Python build, only a small subset of '
                  'characters may be represented.')
    time.sleep(3)

class WcWideCharacterGenerator(object):
    " generates unicode characters of the presumed terminal width "
    def __init__(self, width=2, cjk=False):
        """
        ``cjk``
          Treat ambiguous-width characters as double-width
        """
        self.characters = (unichr(idx)
                           for idx in xrange(LIMIT_UCS)
                           if cjk and wcwidth_cjk(unichr(idx)) == width
                           or not cjk and wcwidth(unichr(idx)) == width
                           and not unicodedata.combining(unichr(idx))
                           )

    def __next__(self):
        while True:
            ucs = next(self.characters)
            try:
                name = string.capwords(unicodedata.name(ucs))
            except ValueError:
                continue
            return (ucs, name)

    # python 2.6 - 3.3 compatibility
    next = __next__

class Style(object):
    " decorator for screen output "
    heading = lambda self, text: text
    hint = lambda self, text: text
    delimiter = u'|'
    continuation = u' ..'
    header_hint = u'-'
    header_fill = u'='
    name_len = 25
    alignment = 'left'
    loading = ' ... loading ... '

    def __init__(self, **kwargs):
        for key, val in kwargs.items():
            setattr(self, key, val)

class Screen(object):
    " represents terminal and data dimensions "
    intro_msg_fmt = (u'Characters {wide} terminal cells wide. '
                     u'{ambiguity}'
                     u'Delimiters ({delim}) should align.')

    def __init__(self, term, style, wide=2, cjk=False):
        self.term = term
        self.style = style
        self.wide = wide
        self.cjk = cjk

    @property
    def header(self):
        return self.head_item * self.num_columns

    @property
    def hint_width(self):
        return sum((len(self.style.delimiter),
                    self.wide,
                    len(self.style.delimiter),
                    len(u' '),
                    len(u'0xZZZZZ'),
                    len(u' '),
                    self.style.name_len,))

    @property
    def head_item(self):
        heading = (u'{delimiter}{hint}{delimiter}'
                   .format(delimiter=self.style.hint(self.style.delimiter),
                           hint=self.style.header_hint * self.wide))
        alignment = lambda *args: (
            self.term.rjust(*args) if self.style.alignment == 'right' else
            self.term.ljust(*args))
        txt = alignment(heading, self.hint_width, self.style.header_fill)
        return self.style.heading(txt)

    @property
    def msg_intro(self):
        delim = self.style.hint(self.style.delimiter)
        wide = self.style.heading('{}'.format(self.wide))
        ambiguity = u'' if not self.cjk else (
            u'Ambiguous-width characters are double-width. ')
        return self.term.wrap(self.intro_msg_fmt.format(
            wide=wide, ambiguity=ambiguity, delim=delim))

    @property
    def row_ends(self):
        return self.term.height - 1

    @property
    def num_columns(self):
        return self.term.width // self.hint_width

    @property
    def num_rows(self):
        return self.row_ends - self.row_begins - 1

    @property
    def row_begins(self):
        return len(self.msg_intro) + 1

    @property
    def page_size(self):
        return self.num_rows * self.num_columns

class Pager(object):
    " a specialized varient of less(1), you can pipe output to less, too. "
    def __init__(self, term, screen, character_factory):
        self.term = term
        self.screen = screen
        self.character_factory = character_factory
        self.character_generator = self.character_factory(self.screen.wide)
        self._page_data = dict()
        self.on_resize(None, None)
        self._idx = -1
        self._offset = 0

    def on_resize(self, sig, action):
        assert self.term.width >= self.screen.hint_width, (
            'Screen to small {}, must be at least {}'.format(
                self.term.width, self.screen.hint_width))
        self.dirty = 2
        self._page_data.clear()
        self.last_page = None
        self.character_generator = self.character_factory(
            self.screen.wide, self.screen.cjk)

    def page_data(self, idx, offset):
        size = self.screen.page_size
        while offset < 0 and idx:
            offset += size
            idx -= 1
        offset = max(0, offset)
        while offset > size:
            offset -= size
            idx += 1
        if idx == self.last_page:
            offset = 0
        _idx = -1
        while _idx <= idx:
            # retrieve new page data
            _idx += 1
            if not _idx in self._page_data:
                pgdata = list()
                for count in xrange(size):
                    try:
                        pgdata.append(next(self.character_generator))
                    except StopIteration:
                        break
                self._page_data[_idx] = pgdata
                if not pgdata:
                    break

        last = max(self._page_data.keys())
        if len(self._page_data[last]) == 0:
            del self._page_data[last]
            last = max(self._page_data.keys())
        self.last_page = last

        nxt = max(0, min(idx, _idx, self.last_page or float('inf')))
        page_data = (self._page_data[nxt][offset:] +
                     self._page_data.get(nxt + 1, list())[:offset])
        return (nxt, offset), page_data

    def run(self, writer, reader):
        page_idx = page_offset = 0
        if not self.term.is_a_tty:
            while True:
                npage_idx, offset = self.draw(writer, page_idx + 1, page_offset)
                if npage_idx == page_idx:
                    break
                page_idx = npage_idx
            return
        while True:
            if self.dirty:
                page_idx, page_offset = self.draw(writer,
                                                  page_idx,
                                                  page_offset)
                self.dirty = False
            nxt, noff = self.process_keystroke(reader(timeout=0.25),
                                               page_idx,
                                               page_offset)
            if not self.dirty:
                self.dirty = nxt != page_idx or noff != page_offset
            page_idx, page_offset = nxt, noff
            if page_idx == -1:
                return

    def process_keystroke(self, inp, idx, offset):
        term = self.term
        # exit
        if inp.lower() in (u'q', u'Q'):
            return (-1, -1)
        elif inp in (u'1', u'2', u'a'):
            if inp == u'a':
                self.screen.cjk = not self.screen.cjk
                self.on_resize(None, None)
            elif int(inp) != self.screen.wide:
                self.screen.wide = int(inp)
                self.on_resize(None, None)
        elif inp in (u'_', u'-'):
            # adjust name length -1
            nlen = max(1, self.screen.style.name_len - 2)
            if nlen != self.screen.style.name_len:
                self.screen.style.name_len = nlen
                self.on_resize(None, None)
        elif inp in (u'+', u'='):
            # adjust name length +1
            nlen = max(1, self.screen.style.name_len + 2)
            if nlen != self.screen.style.name_len:
                self.screen.style.name_len = nlen
                self.on_resize(None, None)
        elif inp == u'2' and self.wide != 2:
            # change 2 or 1-cell wide view
            self.screen.wide = 2
            self.on_resize(None, None)
        elif inp in (u'y', u'k') or inp.code in (term.KEY_UP,):
            # scroll backward 1 line
            return (idx, offset - self.screen.num_columns)
        elif inp in (u'e', u'j') or inp.code in (term.KEY_ENTER,
                                                 term.KEY_DOWN,):
            # scroll forward 1 line
            return (idx, offset + self.screen.num_columns)
        elif inp in (u'f', u' ') or inp.code in (term.KEY_PGDOWN,):
            # scroll forward 1 page
            return (idx + 1, offset)
        elif inp == u'b' or inp.code in (term.KEY_PGUP,):
            # scroll backward 1 page
            return (max(0, idx - 1), offset)
        elif inp.code in (term.KEY_SDOWN,):
            # scroll forward 10 pages
            return (max(0, idx + 10), offset)
        elif inp.code in (term.KEY_SUP,):
            # scroll forward 10 pages
            return (max(0, idx - 10), offset)
        elif inp.code == term.KEY_HOME:
            # top
            return (0, 0)
        elif inp.code == term.KEY_END:
            # bottom
            return (float('inf'), 0)
        # offset w/return
        return idx, offset

    def draw(self, writer, idx, offset):
        # as our screen can be resized while we're mid-calculation,
        # our self.dirty flag can become re-toggled; because we are
        # not reflowing our pagination, we must begin over again.
        while self.dirty:
            if not self.draw_heading(writer):
                self.draw_loading(writer, idx)
            self.dirty = False
            (idx, offset), data = self.page_data(idx, offset)
            for txt in self.page_view(data):
                writer(txt)
        self.draw_status(writer, idx)
        flushout()
        return idx, offset

    def draw_heading(self, writer):
        if self.dirty == 2 and self.term.is_a_tty:
            writer(self.term.clear)
            writer(self.term.move(0, 0))
            writer('\n'.join(self.screen.msg_intro))
            writer('\n')
            writer(self.screen.header)
            return True

    def draw_loading(self, writer, idx):
        if self.term.is_a_tty:
            if not idx in self._page_data:
                writer(u' ' + self.screen.style.loading)
            else:
                writer(u' +')
            flushout()

    def draw_status(self, writer, idx):
        if self.term.is_a_tty:
            writer(self.term.move(self.term.height - 1))
            end=u' (END)' if idx == self.last_page else u''
            writer('Page {idx}{end}. q to quit, [keys: kjfb12-=]'
                   .format(idx=idx, end=end))

    def page_view(self, data):
        if self.term.is_a_tty:
            yield self.term.move(self.screen.row_begins, 0)
        clear_eol = self.term.clear_eol
        clear_eos = self.term.clear_eos
        num_cols = self.screen.num_columns

        col = row = 0
        for ucs, name in data:
            val = self.text_entry(ucs, name)
            col += 1
            if col == num_cols:
                col = 0
                row += 1
                val = u''.join((val, clear_eol, u'\n'))
            yield val
        if self.term.is_a_tty:
            yield u''.join((clear_eol, u'\n', clear_eos))

    def text_entry(self, ucs, name):
        style = self.screen.style
        if len(name) > style.name_len:
            idx = max(0, style.name_len - len(style.continuation))
            name = u''.join((name[:idx], style.continuation if idx else u''))
        if style.alignment == 'right':
            fmt = u' '.join(('{name:{name_len}s}',
                             '{delimiter}{ucs}{delimiter}'
                             '0x{value:05X}'))
        else:
            fmt = u' '.join(('{delimiter}{ucs}{delimiter}',
                             '0x{value:05X}',
                             '{name:{name_len}s}'))
        delimiter = style.hint(style.delimiter)
        return fmt.format(name_len=style.name_len,
                          delimiter=delimiter,
                          name=name, ucs=ucs,
                          value=ord(ucs))

def main():
    term = Terminal()
    style = Style(heading=term.magenta,
                  hint=term.bright_cyan,
                  delimiter=u'|',
                  ) if term.number_of_colors else Style()
    screen = Screen(term, style)
    character_factory = WcWideCharacterGenerator
    pager = Pager(term, screen, character_factory)
    signal.signal(signal.SIGWINCH, pager.on_resize)
    with term.location(), term.cbreak(), term.fullscreen():
        pager.run(writer=echo, reader=term.inkey)

if __name__ == '__main__':
    main()
<commit_msg>remove wcwidth_cjk testing<commit_after>#!/usr/bin/env python

# standard imports
from __future__ import print_function
from __future__ import division
from functools import partial
import unicodedata
import string
import signal

# local imports
from wcwidth import wcwidth

# 3rd party imports
from blessed import Terminal

# BEGIN, python 2.6 through 3.4 compatibilities,

echo = partial(print, end='')

try:
    flushout = partial(print, end='', flush=True)
    flushout('')
except TypeError as err:
    assert "'flush' is an invalid keyword argument" in err.args[0]

    def flushout():
        import sys
        try:
            sys.stdout.flush()
        except BrokenPipeError:
            pass

try:
    _ = unichr(0)
except NameError as err:
    if err.args[0] == "name 'unichr' is not defined":
        unichr = chr
    else:
        raise

try:
    _ = xrange(0)
except NameError as err:
    if err.args[0] == "name 'xrange' is not defined":
        xrange = range
    else:
        raise

# END, python 2.6 - 3.3 compatibilities

# some poor python builds (apple, etc.) are narrow, presumably
# for smaller memory footprint of character strings.
try:
    _ = unichr(0x10000)
    LIMIT_UCS = 0x3fffd
except ValueError as err:
    assert 'narrow Python build' in err.args[0], err.args
    LIMIT_UCS = 0x10000
    import time
    warnings.warn('narrow Python build, only a small subset of '
                  'characters may be represented.')
    time.sleep(3)

class WcWideCharacterGenerator(object):
    " generates unicode characters of the presumed terminal width "
    def __init__(self, width=2):
        """
        ``cjk``
          Treat ambiguous-width characters as double-width
        """
        self.characters = (unichr(idx)
                           for idx in xrange(LIMIT_UCS)
                           if wcwidth(unichr(idx)) == width
                           )

    def __next__(self):
        while True:
            ucs = next(self.characters)
            try:
                name = string.capwords(unicodedata.name(ucs))
            except ValueError:
                continue
            return (ucs, name)

    # python 2.6 - 3.3 compatibility
    next = __next__

class Style(object):
    " decorator for screen output "
    heading = lambda self, text: text
    hint = lambda self, text: text
    delimiter = u'|'
    continuation = u' ..'
    header_hint = u'-'
    header_fill = u'='
    name_len = 25
    alignment = 'left'
    loading = ' ... loading ... '

    def __init__(self, **kwargs):
        for key, val in kwargs.items():
            setattr(self, key, val)

class Screen(object):
    " represents terminal and data dimensions "
    intro_msg_fmt = (u'Characters {wide} terminal cells wide. '
                     u'Delimiters ({delim}) should align.')

    def __init__(self, term, style, wide=2, cjk=False):
        self.term = term
        self.style = style
        self.wide = wide

    @property
    def header(self):
        return self.head_item * self.num_columns

    @property
    def hint_width(self):
        return sum((len(self.style.delimiter),
                    self.wide,
                    len(self.style.delimiter),
                    len(u' '),
                    len(u'0xZZZZZ'),
                    len(u' '),
                    self.style.name_len,))

    @property
    def head_item(self):
        heading = (u'{delimiter}{hint}{delimiter}'
                   .format(delimiter=self.style.hint(self.style.delimiter),
                           hint=self.style.header_hint * self.wide))
        alignment = lambda *args: (
            self.term.rjust(*args) if self.style.alignment == 'right' else
            self.term.ljust(*args))
        txt = alignment(heading, self.hint_width, self.style.header_fill)
        return self.style.heading(txt)

    @property
    def msg_intro(self):
        delim = self.style.hint(self.style.delimiter)
        wide = self.style.heading('{}'.format(self.wide))
        return self.term.wrap(self.intro_msg_fmt.format(
            wide=wide, delim=delim))

    @property
    def row_ends(self):
        return self.term.height - 1

    @property
    def num_columns(self):
        if self.term.is_a_tty:
            return self.term.width // self.hint_width
        return 1

    @property
    def num_rows(self):
        return self.row_ends - self.row_begins - 1

    @property
    def row_begins(self):
        return len(self.msg_intro) + 1

    @property
    def page_size(self):
        return self.num_rows * self.num_columns

class Pager(object):
    " a specialized varient of less(1), you can pipe output to less, too. "
    def __init__(self, term, screen, character_factory):
        self.term = term
        self.screen = screen
        self.character_factory = character_factory
        self.character_generator = self.character_factory(self.screen.wide)
        self._page_data = dict()
        self.on_resize(None, None)
        self._idx = -1
        self._offset = 0

    def on_resize(self, sig, action):
        assert self.term.width >= self.screen.hint_width, (
            'Screen to small {}, must be at least {}'.format(
                self.term.width, self.screen.hint_width))
        self.dirty = 2
        self._page_data.clear()
        self.last_page = None
        self.character_generator = self.character_factory(self.screen.wide)

    def page_data(self, idx, offset):
        size = self.screen.page_size
        while offset < 0 and idx:
            offset += size
            idx -= 1
        offset = max(0, offset)
        while offset > size:
            offset -= size
            idx += 1
        if idx == self.last_page:
            offset = 0
        _idx = -1
        while _idx <= idx:
            # retrieve new page data
            _idx += 1
            if not _idx in self._page_data:
                pgdata = list()
                for count in xrange(size):
                    try:
                        pgdata.append(next(self.character_generator))
                    except StopIteration:
                        break
                self._page_data[_idx] = pgdata
                if not pgdata:
                    break

        last = max(self._page_data.keys())
        if len(self._page_data[last]) == 0:
            del self._page_data[last]
            last = max(self._page_data.keys())
        self.last_page = last

        nxt = max(0, min(idx, _idx, self.last_page or float('inf')))
        page_data = (self._page_data[nxt][offset:] +
                     self._page_data.get(nxt + 1, list())[:offset])
        return (nxt, offset), page_data

    def run(self, writer, reader):
        page_idx = page_offset = 0
        if not self.term.is_a_tty:
            while True:
                npage_idx, offset = self.draw(writer,
                                              page_idx + 1,
                                              page_offset)
                if npage_idx == page_idx:
                    break
                page_idx = npage_idx
                self.dirty = True
            return
        while True:
            if self.dirty:
                page_idx, page_offset = self.draw(writer,
                                                  page_idx,
                                                  page_offset)
                self.dirty = False
            nxt, noff = self.process_keystroke(reader(timeout=0.25),
                                               page_idx,
                                               page_offset)
            if not self.dirty:
                self.dirty = nxt != page_idx or noff != page_offset
            page_idx, page_offset = nxt, noff
            if page_idx == -1:
                return

    def process_keystroke(self, inp, idx, offset):
        term = self.term
        # exit
        if inp.lower() in (u'q', u'Q'):
            return (-1, -1)
        elif inp in (u'1', u'2'):
            if int(inp) != self.screen.wide:
                self.screen.wide = int(inp)
                self.on_resize(None, None)
        elif inp in (u'_', u'-'):
            # adjust name length -1
            nlen = max(1, self.screen.style.name_len - 2)
            if nlen != self.screen.style.name_len:
                self.screen.style.name_len = nlen
                self.on_resize(None, None)
        elif inp in (u'+', u'='):
            # adjust name length +1
            nlen = max(1, self.screen.style.name_len + 2)
            if nlen != self.screen.style.name_len:
                self.screen.style.name_len = nlen
                self.on_resize(None, None)
        elif inp == u'2' and self.wide != 2:
            # change 2 or 1-cell wide view
            self.screen.wide = 2
            self.on_resize(None, None)
        elif inp in (u'y', u'k') or inp.code in (term.KEY_UP,):
            # scroll backward 1 line
            return (idx, offset - self.screen.num_columns)
        elif inp in (u'e', u'j') or inp.code in (term.KEY_ENTER,
                                                 term.KEY_DOWN,):
            # scroll forward 1 line
            return (idx, offset + self.screen.num_columns)
        elif inp in (u'f', u' ') or inp.code in (term.KEY_PGDOWN,):
            # scroll forward 1 page
            return (idx + 1, offset)
        elif inp == u'b' or inp.code in (term.KEY_PGUP,):
            # scroll backward 1 page
            return (max(0, idx - 1), offset)
        elif inp.code in (term.KEY_SDOWN,):
            # scroll forward 10 pages
            return (max(0, idx + 10), offset)
        elif inp.code in (term.KEY_SUP,):
            # scroll forward 10 pages
            return (max(0, idx - 10), offset)
        elif inp.code == term.KEY_HOME:
            # top
            return (0, 0)
        elif inp.code == term.KEY_END:
            # bottom
            return (float('inf'), 0)
        # offset w/return
        return idx, offset

    def draw(self, writer, idx, offset):
        # as our screen can be resized while we're mid-calculation,
        # our self.dirty flag can become re-toggled; because we are
        # not reflowing our pagination, we must begin over again.
        while self.dirty:
            if not self.draw_heading(writer):
                self.draw_loading(writer, idx)
            self.dirty = False
            (idx, offset), data = self.page_data(idx, offset)
            for txt in self.page_view(data):
                writer(txt)
        self.draw_status(writer, idx)
        flushout()
        return idx, offset

    def draw_heading(self, writer):
        if self.dirty == 2:  # and self.term.is_a_tty:
            writer(self.term.clear)
            writer(self.term.move(0, 0))
            writer('\n'.join(self.screen.msg_intro))
            writer('\n')
            writer(self.screen.header)
            writer('\n')
            return True

    def draw_loading(self, writer, idx):
        if self.term.is_a_tty:
            if not idx in self._page_data:
                writer(u' ' + self.screen.style.loading)
            else:
                writer(u' +')
            flushout()

    def draw_status(self, writer, idx):
        if self.term.is_a_tty:
            writer(self.term.move(self.term.height - 1))
            end=u' (END)' if idx == self.last_page else u''
            writer('Page {idx}{end}. q to quit, [keys: kjfb12-=]'
                   .format(idx=idx, end=end))

    def page_view(self, data):
        if self.term.is_a_tty:
            yield self.term.move(self.screen.row_begins, 0)
        clear_eol = self.term.clear_eol
        clear_eos = self.term.clear_eos
        num_cols = self.screen.num_columns

        col = row = 0
        for ucs, name in data:
            val = self.text_entry(ucs, name)
            col += 1
            if col == num_cols:
                col = 0
                row += 1
                if self.term.is_a_tty:
                    val = u''.join((val, clear_eol, u'\n'))
                else:
                    val = u''.join((val.rstrip(), u'\n'))
            yield val
        if self.term.is_a_tty:
            yield u''.join((clear_eol, u'\n', clear_eos))

    def text_entry(self, ucs, name):
        style = self.screen.style
        if len(name) > style.name_len:
            idx = max(0, style.name_len - len(style.continuation))
            name = u''.join((name[:idx], style.continuation if idx else u''))
        if style.alignment == 'right':
            fmt = u' '.join(('{name:{name_len}s}',
                             '{delimiter}{ucs}{delimiter}'
                             '0x{value:x}'))
        else:
            fmt = u' '.join(('{delimiter}{ucs}{delimiter}',
                             '0x{value:x}',
                             '{name:{name_len}s}'))
        delimiter = style.hint(style.delimiter)
        return fmt.format(name_len=style.name_len,
                          delimiter=delimiter,
                          name=name, ucs=ucs,
                          value=ord(ucs))

def main():
    term = Terminal()
    style = Style(heading=term.magenta,
                  hint=term.bright_cyan,
                  delimiter=u'|',
                  ) if term.number_of_colors else Style()
    screen = Screen(term, style)
    character_factory = WcWideCharacterGenerator
    pager = Pager(term, screen, character_factory)
    if term.is_a_tty:
        signal.signal(signal.SIGWINCH, pager.on_resize)
    else:
        screen.style.name_len = 80 - 15
        pager.dirty = 2
    with term.location(), term.cbreak(), term.fullscreen():
        pager.run(writer=echo, reader=term.inkey)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""The base Controller API

Provides the BaseController class for subclassing.
"""
from time import time, gmtime, strftime

from pylons.controllers import WSGIController
from pylons.templating import literal, pylons_globals
from pylons import tmpl_context as c, request, response, config, app_globals, session
from pylons.controllers.util import abort
from genshi.filters import HTMLFormFiller

from openspending import model
from openspending.ui import i18n
from openspending.plugins.core import PluginImplementations
from openspending.plugins.interfaces import IGenshiStreamFilter, IRequest

import logging
log = logging.getLogger(__name__)

ACCEPT_MIMETYPES = {
    "application/json": "json",
    "text/javascript": "json",
    "application/javascript": "json",
    "text/csv": "csv"
    }

def render(template_name,
           extra_vars=None,
           form_fill=None, form_errors={},
           cache_expire=3600, cache_private=False,
           method='xhtml'):

    _setup_cache(cache_expire, cache_private)

    # Pull in extra vars if needed
    globs = extra_vars or {}

    # Second, get the globals
    globs.update(pylons_globals())
    globs['g'] = app_globals
    globs['_form_errors'] = form_errors

    # Grab a template reference
    template = globs['app_globals'].genshi_loader.load(template_name)

    stream = template.generate(**globs)

    if form_fill is not None:
        filler = HTMLFormFiller(data=form_fill)
        stream = stream | filler

    for item in PluginImplementations(IGenshiStreamFilter):
        stream = item.filter(stream)

    return literal(stream.render(method=method, encoding=None))

def _setup_cache(cache_expire, cache_private):
    del response.headers["Pragma"]

    if app_globals.debug or request.method not in ('HEAD', 'GET'):
        response.headers["Cache-Control"] = 'no-cache, no-store, max-age=0'
    else:
        response.headers["Last-Modified"] = strftime("%a, %d %b %Y %H:%M:%S GMT", gmtime())

        if cache_private:
            response.headers["Cache-Control"] = 'private, max-age=%d' % cache_expire
        else:
            response.headers["Cache-Control"] = 'public, max-age=0, s-maxage=%d' % cache_expire

class BaseController(WSGIController):

    items = PluginImplementations(IRequest)

    def __call__(self, environ, start_response):
        """Invoke the Controller"""
        # WSGIController.__call__ dispatches to the Controller method
        # the request is routed to. This routing information is
        # available in environ['pylons.routes_dict']
        begin = time()
        try:
            return WSGIController.__call__(self, environ, start_response)
        finally:
            log.debug("Request to %s took %sms" % (request.path,
               int((time() - begin) * 1000)))

    def __before__(self, action, **params):
        account_name = request.environ.get('REMOTE_USER', None)
        if account_name:
            c.account = model.Account.by_name(account_name)
        else:
            c.account = None

        i18n.handle_request(request, c)
        c.state = session.get('state', {})

        c.q = ''
        c.items_per_page = int(request.params.get('items_per_page', 20))
        c.datasets = model.meta.session.query(model.Dataset).all()
        c.dataset = None
        self._detect_dataset_subdomain()

        for item in self.items:
            item.before(request, c)

    def __after__(self):
        if session.get('state', {}) != c.state:
            session['state'] = c.state
            session.save()
        for item in self.items:
            item.after(request, c)

    def _detect_dataset_subdomain(self):
        http_host = request.environ.get('HTTP_HOST').lower()
        if http_host.startswith('www.'):
            http_host = http_host[len('www.'):]
        if not '.' in http_host:
            return
        dataset_name, domain = http_host.split('.', 1)
        for dataset in c.datasets:
            if dataset.name.lower() == dataset_name:
                c.dataset = dataset

    def _detect_format(self, format):
        for mimetype, mimeformat in self.accept_mimetypes.items():
            if format == mimeformat or \
                    mimetype in request.headers.get("Accept", ""):
                return mimeformat
        return "html"

    def _get_dataset(self, dataset):
        c.dataset = model.Dataset.by_name(dataset)
        if not c.dataset:
            abort(404, _('Sorry, there is no dataset named %r') % dataset)
<commit_msg>add missing i18n import<commit_after>"""The base Controller API

Provides the BaseController class for subclassing.
"""
from time import time, gmtime, strftime

from pylons.controllers import WSGIController
from pylons.templating import literal, pylons_globals
from pylons import tmpl_context as c, request, response, config, app_globals, session
from pylons.controllers.util import abort
from genshi.filters import HTMLFormFiller
from pylons.i18n import _

from openspending import model
from openspending.ui import i18n
from openspending.plugins.core import PluginImplementations
from openspending.plugins.interfaces import IGenshiStreamFilter, IRequest

import logging
log = logging.getLogger(__name__)

ACCEPT_MIMETYPES = {
    "application/json": "json",
    "text/javascript": "json",
    "application/javascript": "json",
    "text/csv": "csv"
    }

def render(template_name,
           extra_vars=None,
           form_fill=None, form_errors={},
           cache_expire=3600, cache_private=False,
           method='xhtml'):

    _setup_cache(cache_expire, cache_private)

    # Pull in extra vars if needed
    globs = extra_vars or {}

    # Second, get the globals
    globs.update(pylons_globals())
    globs['g'] = app_globals
    globs['_form_errors'] = form_errors

    # Grab a template reference
    template = globs['app_globals'].genshi_loader.load(template_name)

    stream = template.generate(**globs)

    if form_fill is not None:
        filler = HTMLFormFiller(data=form_fill)
        stream = stream | filler

    for item in PluginImplementations(IGenshiStreamFilter):
        stream = item.filter(stream)

    return literal(stream.render(method=method, encoding=None))

def _setup_cache(cache_expire, cache_private):
    del response.headers["Pragma"]

    if app_globals.debug or request.method not in ('HEAD', 'GET'):
        response.headers["Cache-Control"] = 'no-cache, no-store, max-age=0'
    else:
        response.headers["Last-Modified"] = strftime("%a, %d %b %Y %H:%M:%S GMT", gmtime())

        if cache_private:
            response.headers["Cache-Control"] = 'private, max-age=%d' % cache_expire
        else:
            response.headers["Cache-Control"] = 'public, max-age=0, s-maxage=%d' % cache_expire

class BaseController(WSGIController):

    items = PluginImplementations(IRequest)

    def __call__(self, environ, start_response):
        """Invoke the Controller"""
        # WSGIController.__call__ dispatches to the Controller method
        # the request is routed to. This routing information is
        # available in environ['pylons.routes_dict']
        begin = time()
        try:
            return WSGIController.__call__(self, environ, start_response)
        finally:
            log.debug("Request to %s took %sms" % (request.path,
               int((time() - begin) * 1000)))

    def __before__(self, action, **params):
        account_name = request.environ.get('REMOTE_USER', None)
        if account_name:
            c.account = model.Account.by_name(account_name)
        else:
            c.account = None

        i18n.handle_request(request, c)
        c.state = session.get('state', {})

        c.q = ''
        c.items_per_page = int(request.params.get('items_per_page', 20))
        c.datasets = model.meta.session.query(model.Dataset).all()
        c.dataset = None
        self._detect_dataset_subdomain()

        for item in self.items:
            item.before(request, c)

    def __after__(self):
        if session.get('state', {}) != c.state:
            session['state'] = c.state
            session.save()
        for item in self.items:
            item.after(request, c)

    def _detect_dataset_subdomain(self):
        http_host = request.environ.get('HTTP_HOST').lower()
        if http_host.startswith('www.'):
            http_host = http_host[len('www.'):]
        if not '.' in http_host:
            return
        dataset_name, domain = http_host.split('.', 1)
        for dataset in c.datasets:
            if dataset.name.lower() == dataset_name:
                c.dataset = dataset

    def _detect_format(self, format):
        for mimetype, mimeformat in self.accept_mimetypes.items():
            if format == mimeformat or \
                    mimetype in request.headers.get("Accept", ""):
                return mimeformat
        return "html"

    def _get_dataset(self, dataset):
        c.dataset = model.Dataset.by_name(dataset)
        if not c.dataset:
            abort(404, _('Sorry, there is no dataset named %r') % dataset)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Module for demonstating salt-reduce
'''

# Import Python libs
import os
import sys
import time
import traceback
import random

# Import Salt libs
import salt
import salt.version
import salt.loader

# __proxyenabled__ = ['*']

from resource import getrusage as resource_usage, RUSAGE_SELF
from time import time as timestamp

try:
    # ...a hack used to ignore this class when used as a salt
    # execution module
    # TODO: implement this using decorator:
    # http://docs.saltstack.com/en/latest/ref/modules/#useful-decorators-for-modules
    #
    from lib.mapper import *

    class _mapper(mapper):

        sum = 0
        times = {}

        def __init__(self, module_name=None):
            if module_name == 'adder.add':
                # redirect "map" method to "partial_result"
                self.module_name = 'adder.partial_result'
            # self.module_name = module_name

        class partializer():
            part_size = 100000000

            def __init__(self, upper):
                self.upper = int(upper[0]) # need to cast this to int because "salt-call" does not
                self.x = 0

            def next(self):
                ret = self.x
                if ret >= self.upper:
                    raise StopIteration
                remainder = self.upper - ret
                if remainder >= self.part_size:
                    remainder = self.part_size
                self.x += remainder
                return [ret, remainder]

        def reducer(self, n):
            self.sum += n[0]
            for k, v in n[1].iteritems():
                try:
                    self.times[k] += v
                except KeyError:
                    self.times[k] = 0

            return self.sum

        def statit(self):
            return self.sum, self.times

except:
    pass

def sum_nums(upper):
    '''
    Return the sum of the sequence of numbers from 1 to [upper], and the
    time it took to compute in seconds. Useful for validating the mapreduce runner

    CLI Example:

    .. code-block:: bash

        salt '*' mapit.sum_nums 10

    '''

    upper = int(upper)
    start = time.time()

    sum = 0

    for a in xrange(0, upper):
        sum += a

    print "sum = " + str(sum)
    return sum

def partial_result(lower, count):
    '''
    Return the sum of the sequence of numbers in the range [lower, lower+count], and the
    time it took to compute in seconds.

    CLI Example:

    .. code-block:: bash

        salt '*' mapit.sum_nums_partial 10 20

    '''
    start_time, start_resources = timestamp(), resource_usage(RUSAGE_SELF)
    lower = int(lower)
    sum = 0

    for a in xrange(lower, lower + count):
        sum += a

    end_resources, end_time = resource_usage(RUSAGE_SELF), timestamp()

    return sum, {'real': end_time - start_time,
                 'sys': end_resources.ru_stime - start_resources.ru_stime,
                 'user': end_resources.ru_utime - start_resources.ru_utime}

<commit_msg>change "mapit" in comments to "adder"<commit_after># -*- coding: utf-8 -*-
'''
Module for demonstating salt-reduce
'''

# Import Python libs
import os
import sys
import time
import traceback
import random

# Import Salt libs
import salt
import salt.version
import salt.loader

# __proxyenabled__ = ['*']

from resource import getrusage as resource_usage, RUSAGE_SELF
from time import time as timestamp

try:
    # ...a hack used to ignore this class when used as a salt
    # execution module
    # TODO: implement this using decorator:
    # http://docs.saltstack.com/en/latest/ref/modules/#useful-decorators-for-modules
    #
    from lib.mapper import *

    class _mapper(mapper):

        sum = 0
        times = {}

        def __init__(self, module_name=None):
            if module_name == 'adder.add':
                # redirect "map" method to "partial_result"
                self.module_name = 'adder.partial_result'
            # self.module_name = module_name

        class partializer():
            part_size = 100000000

            def __init__(self, upper):
                self.upper = int(upper[0]) # need to cast this to int because "salt-call" does not
                self.x = 0

            def next(self):
                ret = self.x
                if ret >= self.upper:
                    raise StopIteration
                remainder = self.upper - ret
                if remainder >= self.part_size:
                    remainder = self.part_size
                self.x += remainder
                return [ret, remainder]

        def reducer(self, n):
            self.sum += n[0]
            for k, v in n[1].iteritems():
                try:
                    self.times[k] += v
                except KeyError:
                    self.times[k] = 0

            return self.sum

        def statit(self):
            return self.sum, self.times

except:
    pass

def sum_nums(upper):
    '''
    Return the sum of the sequence of numbers from 1 to [upper], and the
    time it took to compute in seconds. Useful for validating the mapreduce runner

    CLI Example:

    .. code-block:: bash

        salt '*' adder.sum_nums 10

    '''

    upper = int(upper)
    start = time.time()

    sum = 0

    for a in xrange(0, upper):
        sum += a

    print "sum = " + str(sum)
    return sum

def partial_result(lower, count):
    '''
    Return the sum of the sequence of numbers in the range [lower, lower+count], and the
    time it took to compute in seconds.

    CLI Example:

    .. code-block:: bash

        salt '*' adder.sum_nums_partial 10 20

    '''
    start_time, start_resources = timestamp(), resource_usage(RUSAGE_SELF)
    lower = int(lower)
    sum = 0

    for a in xrange(lower, lower + count):
        sum += a

    end_resources, end_time = resource_usage(RUSAGE_SELF), timestamp()

    return sum, {'real': end_time - start_time,
                 'sys': end_resources.ru_stime - start_resources.ru_stime,
                 'user': end_resources.ru_utime - start_resources.ru_utime}

<|endoftext|>


<commit_before>from functools import wraps
import hashlib
import logging
import re
from django.core.cache import cache

LOGGER = logging.getLogger('django.request')

def disable_for_loaddata(signal_handler):
    @wraps(signal_handler)
    def wrapper(*args, **kwargs):
        if kwargs.get('raw'):
            return
        signal_handler(*args, **kwargs)

    return wrapper

def cache_pure(f, expiration=60 * 60 * 24 * 30):
    """ Cache decorator for functions taking one or more arguments. """

    @wraps(f)
    def wrapper(*args, **kwargs):
        if len(args) > 0 and re.match(r"<.+ object at \w+>", repr(args[0])) is not None:
            key_args = [args[0].__class__] + list(args[1:])
        else:
            key_args = args

        key = "{}:args:{}-kwargs:{}".format(f.__name__, repr(key_args), repr(kwargs))
        hash = hashlib.sha1(key).hexdigest()
        if hash in cache:
            value = cache.get(hash)
            LOGGER.debug("loaded function result (%s) form CACHE; key: %s, hash %s", value, key, hash)
            return value

        value = f(*args, **kwargs)
        LOGGER.debug("saved function result (%s) to CACHE; key: %s, hash %s", value, key, hash)
        cache.set(hash, value, expiration)

        return value

    return wrapper
<commit_msg>shorten debug message for too long results in cache decorator<commit_after>from functools import wraps
import hashlib
import logging
import re
from django.core.cache import cache

LOGGER = logging.getLogger('django.request')

def disable_for_loaddata(signal_handler):
    @wraps(signal_handler)
    def wrapper(*args, **kwargs):
        if kwargs.get('raw'):
            return
        signal_handler(*args, **kwargs)

    return wrapper

def cache_pure(f, expiration=60 * 60 * 24 * 30):
    """ Cache decorator for functions taking one or more arguments. """

    @wraps(f)
    def wrapper(*args, **kwargs):
        if len(args) > 0 and re.match(r"<.+ object at \w+>", repr(args[0])) is not None:
            key_args = [args[0].__class__] + list(args[1:])
        else:
            key_args = args

        key = "{}:args:{}-kwargs:{}".format(f.__name__, repr(key_args), repr(kwargs))
        hash = hashlib.sha1(key).hexdigest()
        if hash in cache:
            value = cache.get(hash)
            LOGGER.debug("loaded function result (%s...) form CACHE; key: %s, hash %s", str(value)[:300], key, hash)
            return value

        value = f(*args, **kwargs)
        LOGGER.debug("saved function result (%s...) to CACHE; key: %s, hash %s", str(value)[:300], key, hash)
        cache.set(hash, value, expiration)

        return value

    return wrapper
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2011 Justin Santa Barbara
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Generic Node base class for all workers that run on hosts."""

import errno
import os
import random
import signal
import sys
import time

import eventlet
import greenlet
import logging as std_logging

from openstack.common import cfg
from openstack.common import eventlet_backdoor
from openstack.common.gettextutils import _
from openstack.common import log as logging
from openstack.common import threadgroup

try:
    from openstack.common import rpc
except ImportError:
    rpc = None

CONF = cfg.CONF
LOG = logging.getLogger(__name__)

class Launcher(object):
    """Launch one or more services and wait for them to complete."""

    def __init__(self):
        """Initialize the service launcher.

        :returns: None

        """
        self._services = []
        eventlet_backdoor.initialize_if_enabled()

    @staticmethod
    def run_service(service):
        """Start and wait for a service to finish.

        :param service: service to run and wait for.
        :returns: None

        """
        service.start()
        service.wait()

    def launch_service(self, service):
        """Load and start the given service.

        :param service: The service you would like to start.
        :returns: None

        """
        gt = eventlet.spawn(self.run_service, service)
        self._services.append(gt)

    def stop(self):
        """Stop all services which are currently running.

        :returns: None

        """
        for service in self._services:
            service.kill()

    def wait(self):
        """Waits until all services have been stopped, and then returns.

        :returns: None

        """
        for service in self._services:
            try:
                service.wait()
            except greenlet.GreenletExit:
                pass

class SignalExit(SystemExit):
    def __init__(self, signo, exccode=1):
        super(SignalExit, self).__init__(exccode)
        self.signo = signo

class ServiceLauncher(Launcher):
    def _handle_signal(self, signo, frame):
        # Allow the process to be killed again and die from natural causes
        signal.signal(signal.SIGTERM, signal.SIG_DFL)
        signal.signal(signal.SIGINT, signal.SIG_DFL)

        raise SignalExit(signo)

    def wait(self):
        signal.signal(signal.SIGTERM, self._handle_signal)
        signal.signal(signal.SIGINT, self._handle_signal)

        LOG.debug(_('Full set of CONF:'))
        CONF.log_opt_values(LOG, std_logging.DEBUG)

        status = None
        try:
            super(ServiceLauncher, self).wait()
        except SignalExit as exc:
            signame = {signal.SIGTERM: 'SIGTERM',
                       signal.SIGINT: 'SIGINT'}[exc.signo]
            LOG.info(_('Caught %s, exiting'), signame)
            status = exc.code
        except SystemExit as exc:
            status = exc.code
        finally:
            self.stop()
            if rpc:
                rpc.cleanup()
        return status

class ServiceWrapper(object):
    def __init__(self, service, workers):
        self.service = service
        self.workers = workers
        self.children = set()
        self.forktimes = []

class ProcessLauncher(object):
    def __init__(self):
        self.children = {}
        self.sigcaught = None
        self.running = True
        rfd, self.writepipe = os.pipe()
        self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r')

        signal.signal(signal.SIGTERM, self._handle_signal)
        signal.signal(signal.SIGINT, self._handle_signal)

    def _handle_signal(self, signo, frame):
        self.sigcaught = signo
        self.running = False

        # Allow the process to be killed again and die from natural causes
        signal.signal(signal.SIGTERM, signal.SIG_DFL)
        signal.signal(signal.SIGINT, signal.SIG_DFL)

    def _pipe_watcher(self):
        # This will block until the write end is closed when the parent
        # dies unexpectedly
        self.readpipe.read()

        LOG.info(_('Parent process has died unexpectedly, exiting'))

        sys.exit(1)

    def _child_process(self, service):
        # Setup child signal handlers differently
        def _sigterm(*args):
            signal.signal(signal.SIGTERM, signal.SIG_DFL)
            raise SignalExit(signal.SIGTERM)

        signal.signal(signal.SIGTERM, _sigterm)
        # Block SIGINT and let the parent send us a SIGTERM
        signal.signal(signal.SIGINT, signal.SIG_IGN)

        # Reopen the eventlet hub to make sure we don't share an epoll
        # fd with parent and/or siblings, which would be bad
        eventlet.hubs.use_hub()

        # Close write to ensure only parent has it open
        os.close(self.writepipe)
        # Create greenthread to watch for parent to close pipe
        eventlet.spawn_n(self._pipe_watcher)

        # Reseed random number generator
        random.seed()

        launcher = Launcher()
        launcher.run_service(service)

    def _start_child(self, wrap):
        if len(wrap.forktimes) > wrap.workers:
            # Limit ourselves to one process a second (over the period of
            # number of workers * 1 second). This will allow workers to
            # start up quickly but ensure we don't fork off children that
            # die instantly too quickly.
            if time.time() - wrap.forktimes[0] < wrap.workers:
                LOG.info(_('Forking too fast, sleeping'))
                time.sleep(1)

            wrap.forktimes.pop(0)

        wrap.forktimes.append(time.time())

        pid = os.fork()
        if pid == 0:
            # NOTE(johannes): All exceptions are caught to ensure this
            # doesn't fallback into the loop spawning children. It would
            # be bad for a child to spawn more children.
            status = 0
            try:
                self._child_process(wrap.service)
            except SignalExit as exc:
                signame = {signal.SIGTERM: 'SIGTERM',
                           signal.SIGINT: 'SIGINT'}[exc.signo]
                LOG.info(_('Caught %s, exiting'), signame)
                status = exc.code
            except SystemExit as exc:
                status = exc.code
            except BaseException:
                LOG.exception(_('Unhandled exception'))
                status = 2
            finally:
                wrap.service.stop()

            os._exit(status)

        LOG.info(_('Started child %d'), pid)

        wrap.children.add(pid)
        self.children[pid] = wrap

        return pid

    def launch_service(self, service, workers=1):
        wrap = ServiceWrapper(service, workers)

        LOG.info(_('Starting %d workers'), wrap.workers)
        while self.running and len(wrap.children) < wrap.workers:
            self._start_child(wrap)

    def _wait_child(self):
        try:
            pid, status = os.wait()
        except OSError as exc:
            if exc.errno not in (errno.EINTR, errno.ECHILD):
                raise
            return None

        if os.WIFSIGNALED(status):
            sig = os.WTERMSIG(status)
            LOG.info(_('Child %(pid)d killed by signal %(sig)d'), locals())
        else:
            code = os.WEXITSTATUS(status)
            LOG.info(_('Child %(pid)d exited with status %(code)d'), locals())

        if pid not in self.children:
            LOG.warning(_('pid %d not in child list'), pid)
            return None

        wrap = self.children.pop(pid)
        wrap.children.remove(pid)
        return wrap

    def wait(self):
        """Loop waiting on children to die and respawning as necessary"""

        LOG.debug(_('Full set of CONF:'))
        CONF.log_opt_values(LOG, std_logging.DEBUG)

        while self.running:
            wrap = self._wait_child()
            if not wrap:
                continue

            while self.running and len(wrap.children) < wrap.workers:
                self._start_child(wrap)

        if self.sigcaught:
            signame = {signal.SIGTERM: 'SIGTERM',
                       signal.SIGINT: 'SIGINT'}[self.sigcaught]
            LOG.info(_('Caught %s, stopping children'), signame)

        for pid in self.children:
            try:
                os.kill(pid, signal.SIGTERM)
            except OSError as exc:
                if exc.errno != errno.ESRCH:
                    raise

        # Wait for children to die
        if self.children:
            LOG.info(_('Waiting on %d children to exit'), len(self.children))
            while self.children:
                self._wait_child()

class Service(object):
    """Service object for binaries running on hosts."""

    def __init__(self):
        self.tg = threadgroup.ThreadGroup('service')

    def start(self):
        pass

    def stop(self):
        self.tg.stop()

    def wait(self):
        self.tg.wait()

def launch(service, workers=None):
    if workers:
        launcher = ProcessLauncher()
        launcher.launch_service(service, workers=workers)
    else:
        launcher = ServiceLauncher()
        launcher.launch_service(service)
    return launcher
<commit_msg>Change RPC cleanup ordering<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2011 Justin Santa Barbara
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Generic Node base class for all workers that run on hosts."""

import errno
import os
import random
import signal
import sys
import time

import eventlet
import greenlet
import logging as std_logging

from openstack.common import cfg
from openstack.common import eventlet_backdoor
from openstack.common.gettextutils import _
from openstack.common import log as logging
from openstack.common import threadgroup

try:
    from openstack.common import rpc
except ImportError:
    rpc = None

CONF = cfg.CONF
LOG = logging.getLogger(__name__)

class Launcher(object):
    """Launch one or more services and wait for them to complete."""

    def __init__(self):
        """Initialize the service launcher.

        :returns: None

        """
        self._services = []
        eventlet_backdoor.initialize_if_enabled()

    @staticmethod
    def run_service(service):
        """Start and wait for a service to finish.

        :param service: service to run and wait for.
        :returns: None

        """
        service.start()
        service.wait()

    def launch_service(self, service):
        """Load and start the given service.

        :param service: The service you would like to start.
        :returns: None

        """
        gt = eventlet.spawn(self.run_service, service)
        self._services.append(gt)

    def stop(self):
        """Stop all services which are currently running.

        :returns: None

        """
        for service in self._services:
            service.kill()

    def wait(self):
        """Waits until all services have been stopped, and then returns.

        :returns: None

        """
        for service in self._services:
            try:
                service.wait()
            except greenlet.GreenletExit:
                pass

class SignalExit(SystemExit):
    def __init__(self, signo, exccode=1):
        super(SignalExit, self).__init__(exccode)
        self.signo = signo

class ServiceLauncher(Launcher):
    def _handle_signal(self, signo, frame):
        # Allow the process to be killed again and die from natural causes
        signal.signal(signal.SIGTERM, signal.SIG_DFL)
        signal.signal(signal.SIGINT, signal.SIG_DFL)

        raise SignalExit(signo)

    def wait(self):
        signal.signal(signal.SIGTERM, self._handle_signal)
        signal.signal(signal.SIGINT, self._handle_signal)

        LOG.debug(_('Full set of CONF:'))
        CONF.log_opt_values(LOG, std_logging.DEBUG)

        status = None
        try:
            super(ServiceLauncher, self).wait()
        except SignalExit as exc:
            signame = {signal.SIGTERM: 'SIGTERM',
                       signal.SIGINT: 'SIGINT'}[exc.signo]
            LOG.info(_('Caught %s, exiting'), signame)
            status = exc.code
        except SystemExit as exc:
            status = exc.code
        finally:
            if rpc:
                rpc.cleanup()
            self.stop()
        return status

class ServiceWrapper(object):
    def __init__(self, service, workers):
        self.service = service
        self.workers = workers
        self.children = set()
        self.forktimes = []

class ProcessLauncher(object):
    def __init__(self):
        self.children = {}
        self.sigcaught = None
        self.running = True
        rfd, self.writepipe = os.pipe()
        self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r')

        signal.signal(signal.SIGTERM, self._handle_signal)
        signal.signal(signal.SIGINT, self._handle_signal)

    def _handle_signal(self, signo, frame):
        self.sigcaught = signo
        self.running = False

        # Allow the process to be killed again and die from natural causes
        signal.signal(signal.SIGTERM, signal.SIG_DFL)
        signal.signal(signal.SIGINT, signal.SIG_DFL)

    def _pipe_watcher(self):
        # This will block until the write end is closed when the parent
        # dies unexpectedly
        self.readpipe.read()

        LOG.info(_('Parent process has died unexpectedly, exiting'))

        sys.exit(1)

    def _child_process(self, service):
        # Setup child signal handlers differently
        def _sigterm(*args):
            signal.signal(signal.SIGTERM, signal.SIG_DFL)
            raise SignalExit(signal.SIGTERM)

        signal.signal(signal.SIGTERM, _sigterm)
        # Block SIGINT and let the parent send us a SIGTERM
        signal.signal(signal.SIGINT, signal.SIG_IGN)

        # Reopen the eventlet hub to make sure we don't share an epoll
        # fd with parent and/or siblings, which would be bad
        eventlet.hubs.use_hub()

        # Close write to ensure only parent has it open
        os.close(self.writepipe)
        # Create greenthread to watch for parent to close pipe
        eventlet.spawn_n(self._pipe_watcher)

        # Reseed random number generator
        random.seed()

        launcher = Launcher()
        launcher.run_service(service)

    def _start_child(self, wrap):
        if len(wrap.forktimes) > wrap.workers:
            # Limit ourselves to one process a second (over the period of
            # number of workers * 1 second). This will allow workers to
            # start up quickly but ensure we don't fork off children that
            # die instantly too quickly.
            if time.time() - wrap.forktimes[0] < wrap.workers:
                LOG.info(_('Forking too fast, sleeping'))
                time.sleep(1)

            wrap.forktimes.pop(0)

        wrap.forktimes.append(time.time())

        pid = os.fork()
        if pid == 0:
            # NOTE(johannes): All exceptions are caught to ensure this
            # doesn't fallback into the loop spawning children. It would
            # be bad for a child to spawn more children.
            status = 0
            try:
                self._child_process(wrap.service)
            except SignalExit as exc:
                signame = {signal.SIGTERM: 'SIGTERM',
                           signal.SIGINT: 'SIGINT'}[exc.signo]
                LOG.info(_('Caught %s, exiting'), signame)
                status = exc.code
            except SystemExit as exc:
                status = exc.code
            except BaseException:
                LOG.exception(_('Unhandled exception'))
                status = 2
            finally:
                wrap.service.stop()

            os._exit(status)

        LOG.info(_('Started child %d'), pid)

        wrap.children.add(pid)
        self.children[pid] = wrap

        return pid

    def launch_service(self, service, workers=1):
        wrap = ServiceWrapper(service, workers)

        LOG.info(_('Starting %d workers'), wrap.workers)
        while self.running and len(wrap.children) < wrap.workers:
            self._start_child(wrap)

    def _wait_child(self):
        try:
            pid, status = os.wait()
        except OSError as exc:
            if exc.errno not in (errno.EINTR, errno.ECHILD):
                raise
            return None

        if os.WIFSIGNALED(status):
            sig = os.WTERMSIG(status)
            LOG.info(_('Child %(pid)d killed by signal %(sig)d'), locals())
        else:
            code = os.WEXITSTATUS(status)
            LOG.info(_('Child %(pid)d exited with status %(code)d'), locals())

        if pid not in self.children:
            LOG.warning(_('pid %d not in child list'), pid)
            return None

        wrap = self.children.pop(pid)
        wrap.children.remove(pid)
        return wrap

    def wait(self):
        """Loop waiting on children to die and respawning as necessary"""

        LOG.debug(_('Full set of CONF:'))
        CONF.log_opt_values(LOG, std_logging.DEBUG)

        while self.running:
            wrap = self._wait_child()
            if not wrap:
                continue

            while self.running and len(wrap.children) < wrap.workers:
                self._start_child(wrap)

        if self.sigcaught:
            signame = {signal.SIGTERM: 'SIGTERM',
                       signal.SIGINT: 'SIGINT'}[self.sigcaught]
            LOG.info(_('Caught %s, stopping children'), signame)

        for pid in self.children:
            try:
                os.kill(pid, signal.SIGTERM)
            except OSError as exc:
                if exc.errno != errno.ESRCH:
                    raise

        # Wait for children to die
        if self.children:
            LOG.info(_('Waiting on %d children to exit'), len(self.children))
            while self.children:
                self._wait_child()

class Service(object):
    """Service object for binaries running on hosts."""

    def __init__(self):
        self.tg = threadgroup.ThreadGroup('service')

    def start(self):
        pass

    def stop(self):
        self.tg.stop()

    def wait(self):
        self.tg.wait()

def launch(service, workers=None):
    if workers:
        launcher = ProcessLauncher()
        launcher.launch_service(service, workers=workers)
    else:
        launcher = ServiceLauncher()
        launcher.launch_service(service)
    return launcher
<|endoftext|>


<commit_before># Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import abc
import collections

from blazar.db import api as db_api
from blazar.db import utils as db_utils
from blazar.i18n import _
from blazar.utils.openstack import keystone
from oslo_config import cfg
from oslo_log import log as logging
import six

LOG = logging.getLogger(__name__)
CONF = cfg.CONF

@six.add_metaclass(abc.ABCMeta)
class BasePlugin(object):

    resource_type = 'none'
    title = None
    description = None
    monitor = None
    query_options = None

    def get_plugin_opts(self):
        """Plugin can expose some options that should be specified in conf file

        For example:

            def get_plugin_opts(self):
            return [
                cfg.StrOpt('mandatory-conf', required=True),
                cfg.StrOpt('optional_conf', default="42"),
            ]
        """
        return []

    def setup(self, conf):
        """Plugin initialization

        :param conf: plugin-specific configurations
        """
        pass

    def to_dict(self):
        return {
            'resource_type': self.resource_type,
            'title': self.title,
            'description': self.description,
        }

    @abc.abstractmethod
    def get(self, resource_id):
        """Get resource by id"""
        pass

    @abc.abstractmethod
    def reserve_resource(self, reservation_id, values):
        """Reserve resource."""
        pass

    @abc.abstractmethod
    def list_allocations(self, query, detail=False):
        """List resource allocations."""
        pass

    @abc.abstractmethod
    def query_allocations(self, resource_id_list, lease_id=None,
                          reservation_id=None):
        """List resource allocations."""
        pass

    @abc.abstractmethod
    def allocation_candidates(self, lease_values):
        """Get candidates for reservation allocation."""
        pass

    @abc.abstractmethod
    def update_reservation(self, reservation_id, values):
        """Update reservation."""
        pass

    @abc.abstractmethod
    def on_end(self, resource_id, lease=None):
        """Delete resource."""
        pass

    @abc.abstractmethod
    def on_start(self, resource_id, lease=None):
        """Wake up resource."""
        pass

    def list_resource_properties(self, query):
        detail = False if not query else query.get('detail', False)
        resource_properties = collections.defaultdict(list)

        for name, private, value in db_api.resource_properties_list(
                self.resource_type):

            if not private:
                resource_properties[name].append(value)

        if detail:
            return [
                dict(property=k, private=False, values=v)
                for k, v in resource_properties.items()]
        else:
            return [dict(property=k) for k, v in resource_properties.items()]

    def update_resource_property(self, property_name, values):
        return db_api.resource_property_update(
            self.resource_type, property_name, values)

    def before_end(self, resource_id, lease=None):
        """Take actions before the end of a lease"""
        pass

    def heal_reservations(self, failed_resources, interval_begin,
                          interval_end):
        """Heal reservations which suffer from resource failures.

        :param failed_resources: failed resources
        :param interval_begin: start date of the period to heal.
        :param interval_end: end date of the period to heal.
        :return: a dictionary of {reservation id: flags to update}
                 e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
                       {'missing_resources': True}}
        """
        raise NotImplementedError

    def get_query_options(self, params, index_type):
        options = {k: params[k] for k in params
                   if k in self.query_options[index_type]}
        unsupported = set(params) - set(options)
        if unsupported:
            LOG.debug('Unsupported query key is specified in API request: %s',
                      unsupported)
        return options

    def is_project_allowed(self, project_id, resource):
        # If this resource has the extra capability "authorized_projects"
        if "authorized_projects" in resource and \
                isinstance(resource["authorized_projects"], str):
            # Parse the field as a CSV, and check the resulting list
            authorized_projects = resource["authorized_projects"].split(",")
            return project_id in authorized_projects
        return True

    def add_extra_allocation_info(self, resource_allocations):
        """Add extra information to allocations (to show in calendar)"""
        extras = CONF.manager.extras
        for allocs in resource_allocations.values():
            for alloc in allocs:
                alloc["extras"] = []
        if "name" in extras:
            ids = []
            for allocations in resource_allocations.values():
                for alloc in allocations:
                    ids.append(alloc["lease_id"])
            items = db_utils.get_user_ids_for_lease_ids(ids)
            keystoneclient = keystone.BlazarKeystoneClient()
            lease_to_name = dict()
            for lease_id, user_id in items:
                user = keystoneclient.users.get(user_id)
                lease_to_name[lease_id] = user.name

            for allocations in resource_allocations.values():
                for alloc in allocations:
                    alloc["extras"] = [
                        (_("Reserved by"), lease_to_name[alloc["lease_id"]]),
                    ]

@six.add_metaclass(abc.ABCMeta)
class BaseMonitorPlugin():
    """Base class of monitor plugin."""
    @abc.abstractmethod
    def is_notification_enabled(self):
        """Check if the notification monitor is enabled."""
        pass

    @abc.abstractmethod
    def get_notification_event_types(self):
        """Get a list of event types of messages to handle."""
        pass

    @abc.abstractmethod
    def get_notification_topics(self):
        """Get a list of topics of notification to subscribe to."""
        pass

    @abc.abstractmethod
    def notification_callback(self, event_type, payload):
        """Handle a notification message.

        It is used as a callback of a notification based resource monitor.

        :param event_type: an event type of a notification.
        :param payload: a payload of a notification.
        :return: a dictionary of {reservation id: flags to update}
                 e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
                       {'missing_resources': True}}
        """
        pass

    @abc.abstractmethod
    def is_polling_enabled(self):
        """Check if the polling monitor is enabled."""
        pass

    @abc.abstractmethod
    def get_polling_interval(self):
        """Get an interval of polling in seconds."""
        pass

    @abc.abstractmethod
    def poll(self):
        """Check health of resources.

        :return: a dictionary of {reservation id: flags to update}
                 e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
                       {'missing_resources': True}}
        """
        pass

    @abc.abstractmethod
    def get_healing_interval(self):
        """Get interval of reservation healing in minutes."""
        pass

    @abc.abstractmethod
    def heal(self):
        """Heal suffering reservations.

        :return: a dictionary of {reservation id: flags to update}
        """
<commit_msg>Gather KS users all at once<commit_after># Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import abc
import collections

from blazar.db import api as db_api
from blazar.db import utils as db_utils
from blazar.utils.openstack import keystone
from oslo_config import cfg
from oslo_log import log as logging
import six

LOG = logging.getLogger(__name__)
CONF = cfg.CONF

@six.add_metaclass(abc.ABCMeta)
class BasePlugin(object):

    resource_type = 'none'
    title = None
    description = None
    monitor = None
    query_options = None

    def get_plugin_opts(self):
        """Plugin can expose some options that should be specified in conf file

        For example:

            def get_plugin_opts(self):
            return [
                cfg.StrOpt('mandatory-conf', required=True),
                cfg.StrOpt('optional_conf', default="42"),
            ]
        """
        return []

    def setup(self, conf):
        """Plugin initialization

        :param conf: plugin-specific configurations
        """
        pass

    def to_dict(self):
        return {
            'resource_type': self.resource_type,
            'title': self.title,
            'description': self.description,
        }

    @abc.abstractmethod
    def get(self, resource_id):
        """Get resource by id"""
        pass

    @abc.abstractmethod
    def reserve_resource(self, reservation_id, values):
        """Reserve resource."""
        pass

    @abc.abstractmethod
    def list_allocations(self, query, detail=False):
        """List resource allocations."""
        pass

    @abc.abstractmethod
    def query_allocations(self, resource_id_list, lease_id=None,
                          reservation_id=None):
        """List resource allocations."""
        pass

    @abc.abstractmethod
    def allocation_candidates(self, lease_values):
        """Get candidates for reservation allocation."""
        pass

    @abc.abstractmethod
    def update_reservation(self, reservation_id, values):
        """Update reservation."""
        pass

    @abc.abstractmethod
    def on_end(self, resource_id, lease=None):
        """Delete resource."""
        pass

    @abc.abstractmethod
    def on_start(self, resource_id, lease=None):
        """Wake up resource."""
        pass

    def list_resource_properties(self, query):
        detail = False if not query else query.get('detail', False)
        resource_properties = collections.defaultdict(list)

        for name, private, value in db_api.resource_properties_list(
                self.resource_type):

            if not private:
                resource_properties[name].append(value)

        if detail:
            return [
                dict(property=k, private=False, values=v)
                for k, v in resource_properties.items()]
        else:
            return [dict(property=k) for k, v in resource_properties.items()]

    def update_resource_property(self, property_name, values):
        return db_api.resource_property_update(
            self.resource_type, property_name, values)

    def before_end(self, resource_id, lease=None):
        """Take actions before the end of a lease"""
        pass

    def heal_reservations(self, failed_resources, interval_begin,
                          interval_end):
        """Heal reservations which suffer from resource failures.

        :param failed_resources: failed resources
        :param interval_begin: start date of the period to heal.
        :param interval_end: end date of the period to heal.
        :return: a dictionary of {reservation id: flags to update}
                 e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
                       {'missing_resources': True}}
        """
        raise NotImplementedError

    def get_query_options(self, params, index_type):
        options = {k: params[k] for k in params
                   if k in self.query_options[index_type]}
        unsupported = set(params) - set(options)
        if unsupported:
            LOG.debug('Unsupported query key is specified in API request: %s',
                      unsupported)
        return options

    def is_project_allowed(self, project_id, resource):
        # If this resource has the extra capability "authorized_projects"
        if "authorized_projects" in resource and \
                isinstance(resource["authorized_projects"], str):
            # Parse the field as a CSV, and check the resulting list
            authorized_projects = resource["authorized_projects"].split(",")
            return project_id in authorized_projects
        return True

    def add_extra_allocation_info(self, resource_allocations):
        """Add extra information to allocations (to show in calendar)"""
        extras = CONF.manager.extras
        for allocs in resource_allocations.values():
            for alloc in allocs:
                alloc["extras"] = {}
        if "name" in extras:
            ids = []
            for allocations in resource_allocations.values():
                for alloc in allocations:
                    ids.append(alloc["lease_id"])
            items = db_utils.get_user_ids_for_lease_ids(ids)
            keystoneclient = keystone.BlazarKeystoneClient()
            users = keystoneclient.users.list()
            user_map = { user.id: user for user in users }
            lease_to_name = dict()
            for lease_id, user_id in items:
                user = user_map[user_id]
                lease_to_name[lease_id] = user.name

            for allocations in resource_allocations.values():
                for alloc in allocations:
                    alloc["extras"]["name"] = lease_to_name[alloc["lease_id"]]

@six.add_metaclass(abc.ABCMeta)
class BaseMonitorPlugin():
    """Base class of monitor plugin."""
    @abc.abstractmethod
    def is_notification_enabled(self):
        """Check if the notification monitor is enabled."""
        pass

    @abc.abstractmethod
    def get_notification_event_types(self):
        """Get a list of event types of messages to handle."""
        pass

    @abc.abstractmethod
    def get_notification_topics(self):
        """Get a list of topics of notification to subscribe to."""
        pass

    @abc.abstractmethod
    def notification_callback(self, event_type, payload):
        """Handle a notification message.

        It is used as a callback of a notification based resource monitor.

        :param event_type: an event type of a notification.
        :param payload: a payload of a notification.
        :return: a dictionary of {reservation id: flags to update}
                 e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
                       {'missing_resources': True}}
        """
        pass

    @abc.abstractmethod
    def is_polling_enabled(self):
        """Check if the polling monitor is enabled."""
        pass

    @abc.abstractmethod
    def get_polling_interval(self):
        """Get an interval of polling in seconds."""
        pass

    @abc.abstractmethod
    def poll(self):
        """Check health of resources.

        :return: a dictionary of {reservation id: flags to update}
                 e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657':
                       {'missing_resources': True}}
        """
        pass

    @abc.abstractmethod
    def get_healing_interval(self):
        """Get interval of reservation healing in minutes."""
        pass

    @abc.abstractmethod
    def heal(self):
        """Heal suffering reservations.

        :return: a dictionary of {reservation id: flags to update}
        """
<|endoftext|>


<commit_before>#
# Copyright 2011, Blender Foundation.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#

# <pep8 compliant>

from . import engine

devices = (
    ('CPU', "CPU", "Use CPU for rendering"),
    ('GPU', "GPU Compute", "Use GPU compute device for rendering, configured in user preferences"))

feature_set = (
    ('SUPPORTED', "Supported", "Only use finished and supported features"),
    ('EXPERIMENTAL', "Experimental", "Use experimental and incomplete features that might be broken or change in the future"),
    )

shading_systems = (
    ('GPU_COMPATIBLE', "GPU Compatible", "Restricted shading system compatible with GPU rendering"),
    ('OSL', "Open Shading Language", "Open Shading Language shading system that only runs on the CPU"),
    )

displacement_methods = (
    ('BUMP', "Bump", "Bump mapping to simulate the appearance of displacement"),
    ('TRUE', "True", "Use true displacement only, requires fine subdivision"),
    ('BOTH', "Both", "Combination of displacement and bump mapping"),
    )

bvh_types = (
    ('DYNAMIC_BVH', "Dynamic BVH", "Objects can be individually updated, at the cost of slower render time"),
    ('STATIC_BVH', "Static BVH", "Any object modification requires a complete BVH rebuild, but renders faster"),
    )

filter_types = (
    ('BOX', "Box", "Box filter"),
    ('GAUSSIAN', "Gaussian", "Gaussian filter"),
    )

aperture_types = (
    ('RADIUS', "Radius", "Directly change the size of the aperture"),
    ('FSTOP', "F/stop", "Change the size of the aperture by f/stops"),
    )

panorama_types = (
    ('EQUIRECTANGULAR', "Equirectangular", "Render the scene with a spherical camera, also known as Lat Long panorama"),
    ('FISHEYE_EQUIDISTANT', "Fisheye Equidistant", "Ideal for fulldomes, ignore the sensor dimensions"),
    ('FISHEYE_EQUISOLID', "Fisheye Equisolid", "Similar to most fisheye modern lens, take sensor dimensions into consideration"),
    )

<commit_msg>update fisheye tooltip - it was wrongly suggesting equidistant for fulldomes<commit_after>#
# Copyright 2011, Blender Foundation.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#

# <pep8 compliant>

from . import engine

devices = (
    ('CPU', "CPU", "Use CPU for rendering"),
    ('GPU', "GPU Compute", "Use GPU compute device for rendering, configured in user preferences"))

feature_set = (
    ('SUPPORTED', "Supported", "Only use finished and supported features"),
    ('EXPERIMENTAL', "Experimental", "Use experimental and incomplete features that might be broken or change in the future"),
    )

shading_systems = (
    ('GPU_COMPATIBLE', "GPU Compatible", "Restricted shading system compatible with GPU rendering"),
    ('OSL', "Open Shading Language", "Open Shading Language shading system that only runs on the CPU"),
    )

displacement_methods = (
    ('BUMP', "Bump", "Bump mapping to simulate the appearance of displacement"),
    ('TRUE', "True", "Use true displacement only, requires fine subdivision"),
    ('BOTH', "Both", "Combination of displacement and bump mapping"),
    )

bvh_types = (
    ('DYNAMIC_BVH', "Dynamic BVH", "Objects can be individually updated, at the cost of slower render time"),
    ('STATIC_BVH', "Static BVH", "Any object modification requires a complete BVH rebuild, but renders faster"),
    )

filter_types = (
    ('BOX', "Box", "Box filter"),
    ('GAUSSIAN', "Gaussian", "Gaussian filter"),
    )

aperture_types = (
    ('RADIUS', "Radius", "Directly change the size of the aperture"),
    ('FSTOP', "F/stop", "Change the size of the aperture by f/stops"),
    )

panorama_types = (
    ('EQUIRECTANGULAR', "Equirectangular", "Render the scene with a spherical camera, also known as Lat Long panorama"),
    ('FISHEYE_EQUIDISTANT', "Fisheye Equidistant", "Ignore the sensor dimensions"),
    ('FISHEYE_EQUISOLID', "Fisheye Equisolid", "Similar to most fisheye modern lens, take sensor dimensions into consideration. For fulldomes use it with a square sensor ratio",
    )

<|endoftext|>


<commit_before>import os
import subprocess
import zipfile
import hashlib
import re
import glob
from zipfile import ZipFile
from StringIO import StringIO

class RainbowBuilder(object):
    VERSIONS = {
        'css': '1.0.1',
        'generic': '1.0.1',
        'html': '1.0',
        'javascript': '1.0.1',
        'php': '1.0',
        'python': '1.0',
        'ruby': '1.0.1',
        'shell': '1.0.2'
    }

    def __init__(self, js_path, closure_path, theme_path=None):
        self.js_path = js_path
        self.closure_path = closure_path
        self.js_files_to_include = []
        self.file_name = ""
        self.theme_path = theme_path

    def getPathForLanguage(self, language):
        return os.path.join(self.js_path, 'language/' + language + '.js')

    def getRainbowPath(self):
        return os.path.join(self.js_path, 'rainbow.js')

    def verifyPaths(self):
        if not os.path.exists(self.js_path):
            raise Exception('directory does not exist at: %s' % self.js_path)

        if not os.path.isfile(self.closure_path):
            raise Exception('closure compiler does not exist at: %s' % self.closure_path)

    def getZipForLanguages(self, languages, path=None):
        self.verifyPaths()

        # strip out any duplicates
        languages = list(set(languages))

        write_to = StringIO() if path is None else path
        zip_file = ZipFile(write_to, 'w')
        zip_file.write(self.getRainbowPath(), 'rainbow.js', zipfile.ZIP_DEFLATED)

        # include minimized version even when downloading the dev version
        zip_file.write(self.getRainbowPath().replace('.js', '.min.js'), 'rainbow.min.js', zipfile.ZIP_DEFLATED)

        # include themes as well
        if self.theme_path:
            files = glob.glob(self.theme_path + '/*.css')
            for file_name in files:
                zip_file.write(file_name, os.path.join('themes', os.path.basename(file_name)), zipfile.ZIP_DEFLATED)

        for language in languages:
            zip_file.write(self.getPathForLanguage(language), os.path.join('language', language + '.js'), zipfile.ZIP_DEFLATED)

        zip_file.close()

        return write_to

    def openFile(self, path):
        file = open(path, "r")
        content = file.read()
        file.close()
        return content

    def writeFile(self, path, content):
        file = open(path, "w")
        file.write(content)
        file.close()

    def getVersion(self):
        contents = self.openFile(self.getRainbowPath())
        match = re.search(r'@version\s(.*)\s+?', contents)
        return match.group(1)

    def getLanguageVersions(self, languages):
        groups = []
        for language in languages:
            if language in RainbowBuilder.VERSIONS:
                groups.append(language + ' v' + RainbowBuilder.VERSIONS[language])

        return ', '.join(groups)

    def getFileForLanguages(self, languages, cache=None):
        self.verifyPaths()

        # strip out any duplicates
        languages = list(set(languages))

        self.js_files_to_include = [self.getRainbowPath()]
        for language in languages:
            path = self.getPathForLanguage(language)
            if not os.path.exists(path):
                continue

            self.js_files_to_include.append(path)

        self.file_name = 'rainbow' + ('-custom' if len(languages) else '') + '.min.js'

        if cache is not None:
            version = self.getVersion()
            cache_key = 'rainbow_' + hashlib.md5(self.getLanguageVersions(languages)).hexdigest() + '_' + version
            cached_version = cache.get(cache_key)
            if cached_version:
                return cached_version

        command = ['java', '-jar', self.closure_path, '--compilation_level', 'ADVANCED_OPTIMIZATIONS'] + self.js_files_to_include
        proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        output, err = proc.communicate()

        lines = output.splitlines()
        comments = lines[0:4]
        version = comments[1].replace(' @version ', '')
        url = comments[2].replace(' @url ', '')
        new_comment = '/* Rainbow v' + version + ' ' + url

        if len(languages):
            new_comment += ' | included languages: ' + ', '.join(languages)

        new_comment += ' */'

        output = new_comment + '\n' + '\n'.join(lines[4:])

        if cache is not None:
            cache.set(cache_key, output, 14400)  # 4 hours

        return output
<commit_msg>Increase versions<commit_after>import os
import subprocess
import zipfile
import hashlib
import re
import glob
from zipfile import ZipFile
from StringIO import StringIO

class RainbowBuilder(object):
    VERSIONS = {
        'css': '1.0.1',
        'generic': '1.0.1',
        'html': '1.0',
        'javascript': '1.0.2',
        'php': '1.0',
        'python': '1.0',
        'ruby': '1.0.2',
        'shell': '1.0.2'
    }

    def __init__(self, js_path, closure_path, theme_path=None):
        self.js_path = js_path
        self.closure_path = closure_path
        self.js_files_to_include = []
        self.file_name = ""
        self.theme_path = theme_path

    def getPathForLanguage(self, language):
        return os.path.join(self.js_path, 'language/' + language + '.js')

    def getRainbowPath(self):
        return os.path.join(self.js_path, 'rainbow.js')

    def verifyPaths(self):
        if not os.path.exists(self.js_path):
            raise Exception('directory does not exist at: %s' % self.js_path)

        if not os.path.isfile(self.closure_path):
            raise Exception('closure compiler does not exist at: %s' % self.closure_path)

    def getZipForLanguages(self, languages, path=None):
        self.verifyPaths()

        # strip out any duplicates
        languages = list(set(languages))

        write_to = StringIO() if path is None else path
        zip_file = ZipFile(write_to, 'w')
        zip_file.write(self.getRainbowPath(), 'rainbow.js', zipfile.ZIP_DEFLATED)

        # include minimized version even when downloading the dev version
        zip_file.write(self.getRainbowPath().replace('.js', '.min.js'), 'rainbow.min.js', zipfile.ZIP_DEFLATED)

        # include themes as well
        if self.theme_path:
            files = glob.glob(self.theme_path + '/*.css')
            for file_name in files:
                zip_file.write(file_name, os.path.join('themes', os.path.basename(file_name)), zipfile.ZIP_DEFLATED)

        for language in languages:
            zip_file.write(self.getPathForLanguage(language), os.path.join('language', language + '.js'), zipfile.ZIP_DEFLATED)

        zip_file.close()

        return write_to

    def openFile(self, path):
        file = open(path, "r")
        content = file.read()
        file.close()
        return content

    def writeFile(self, path, content):
        file = open(path, "w")
        file.write(content)
        file.close()

    def getVersion(self):
        contents = self.openFile(self.getRainbowPath())
        match = re.search(r'@version\s(.*)\s+?', contents)
        return match.group(1)

    def getLanguageVersions(self, languages):
        groups = []
        for language in languages:
            if language in RainbowBuilder.VERSIONS:
                groups.append(language + ' v' + RainbowBuilder.VERSIONS[language])

        return ', '.join(groups)

    def getFileForLanguages(self, languages, cache=None):
        self.verifyPaths()

        # strip out any duplicates
        languages = list(set(languages))

        self.js_files_to_include = [self.getRainbowPath()]
        for language in languages:
            path = self.getPathForLanguage(language)
            if not os.path.exists(path):
                continue

            self.js_files_to_include.append(path)

        self.file_name = 'rainbow' + ('-custom' if len(languages) else '') + '.min.js'

        if cache is not None:
            version = self.getVersion()
            cache_key = 'rainbow_' + hashlib.md5(self.getLanguageVersions(languages)).hexdigest() + '_' + version
            cached_version = cache.get(cache_key)
            if cached_version:
                return cached_version

        command = ['java', '-jar', self.closure_path, '--compilation_level', 'ADVANCED_OPTIMIZATIONS'] + self.js_files_to_include
        proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        output, err = proc.communicate()

        lines = output.splitlines()
        comments = lines[0:4]
        version = comments[1].replace(' @version ', '')
        url = comments[2].replace(' @url ', '')
        new_comment = '/* Rainbow v' + version + ' ' + url

        if len(languages):
            new_comment += ' | included languages: ' + ', '.join(languages)

        new_comment += ' */'

        output = new_comment + '\n' + '\n'.join(lines[4:])

        if cache is not None:
            cache.set(cache_key, output, 14400)  # 4 hours

        return output
<|endoftext|>