<commit_before>import random # pragma: no cover
import io # pragma: no cover
from collections import Counter # pragma: no cover
import os.path # pragma: no cover
import csv # pragma: no cover
import numpy
from pathlib import Path
import json

from ._vendorized.keras_data_utils import get_file # pragma: no cover
from ..neural.util import partition
from ..neural.util import to_categorical

try:
    basestring
except NameError:
    basestring = str

GITHUB = 'https://github.com/UniversalDependencies/' # pragma: no cover
ANCORA_1_4_ZIP = '{github}/{ancora}/archive/r1.4.zip'.format(
    github=GITHUB, ancora='UD_Spanish-AnCora') # pragma: no cover
EWTB_1_4_ZIP = '{github}/{ewtb}/archive/r1.4.zip'.format(
    github=GITHUB, ewtb='UD_English') # pragma: no cover

SNLI_URL = 'http://nlp.stanford.edu/projects/snli/snli_1.0.zip'
QUORA_QUESTIONS_URL = 'http://qim.ec.quoracdn.net/quora_duplicate_questions.tsv'
IMDB_URL = 'http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz'

def ancora_pos_tags(encode_words=False): # pragma: no cover
    data_dir = get_file('UD_Spanish-AnCora-r1.4', ANCORA_1_4_ZIP,
                        unzip=True)
    train_loc = os.path.join(data_dir, 'es_ancora-ud-train.conllu')
    dev_loc = os.path.join(data_dir, 'es_ancora-ud-dev.conllu')
    return ud_pos_tags(train_loc, dev_loc, encode_words=encode_words)

def ewtb_pos_tags(encode_tags=False, encode_words=False): # pragma: no cover
    data_dir = get_file('UD_English-r1.4', EWTB_1_4_ZIP, unzip=True)
    train_loc = os.path.join(data_dir, 'en-ud-train.conllu')
    dev_loc = os.path.join(data_dir, 'en-ud-dev.conllu')
    return ud_pos_tags(train_loc, dev_loc,
        encode_tags=encode_tags, encode_words=encode_words)

def ud_pos_tags(train_loc, dev_loc, encode_tags=True, encode_words=True): # pragma: no cover
    train_sents = list(read_conll(train_loc))
    dev_sents = list(read_conll(dev_loc))
    tagmap = {}
    freqs = Counter()
    for words, tags in train_sents:
        for tag in tags:
            tagmap.setdefault(tag, len(tagmap))
        for word in words:
            freqs[word] += 1
    vocab = {word: i for i, (word, freq) in enumerate(freqs.most_common())
             if (freq >= 5)}

    def _encode(sents):
        X = []
        y = []
        for words, tags  in sents:
            if encode_words:
                X.append(
                    numpy.asarray(
                        [vocab.get(word, len(vocab)) for word in words],
                        dtype='uint64'))
            else:
                X.append(words)
            if encode_tags:
                y.append(numpy.asarray(
                    [tagmap[tag] for tag in tags],
                    dtype='int32'))
            else:
                y.append(tags)
        return zip(X, y)

    return _encode(train_sents), _encode(dev_sents), len(tagmap)

def imdb(loc=None):
    if loc is None:
        loc = get_file('aclImdb', IMDB_URL, untar=True, unzip=True)
    train_loc = Path(loc) / 'train'
    test_loc = Path(loc) / 'test'
    return read_imdb(train_loc), read_imdb(test_loc)

def read_wikiner(file_, tagmap=None):
    Xs = []
    ys = []
    for line in file_:
        if not line.strip():
            continue
        tokens = [t.rsplit('|', 2) for t in line.split()]
        words, _, tags = zip(*tokens)
        if tagmap is not None:
            tags = [tagmap.setdefault(tag, len(tagmap)) for tag in tags]
        Xs.append(words)
        ys.append(tags)
    return zip(Xs, ys)

def read_imdb(data_dir, limit=0):
    examples = []
    for subdir, label in (('pos', 1), ('neg', 0)):
        for filename in (data_dir / subdir).iterdir():
            with filename.open() as file_:
                text = file_.read()
            if text.strip():
                examples.append((text, label))
    random.shuffle(examples)
    if limit >= 1:
        examples = examples[:limit]
    return examples

def read_conll(loc): # pragma: no cover
    n = 0
    with io.open(loc, encoding='utf8') as file_:
        sent_strs = file_.read().strip().split('\n\n')
    for sent_str in sent_strs:
        lines = [line.split() for line in sent_str.split('\n')
                 if not line.startswith('#')]
        words = []
        tags = []
        for i, pieces in enumerate(lines):
            if len(pieces) == 4:
                word, pos, head, label = pieces
            else:
                idx, word, lemma, pos1, pos, morph, head, label, _, _2 = pieces
            if '-' in idx:
                continue
            words.append(word)
            tags.append(pos)
        yield words, tags

def mnist(): # pragma: no cover
    from ._vendorized.keras_datasets import load_mnist

    # the data, shuffled and split between tran and test sets
    (X_train, y_train), (X_test, y_test) = load_mnist()

    X_train = X_train.reshape(60000, 784)
    X_test = X_test.reshape(10000, 784)
    X_train = X_train.astype('float32')
    X_test = X_test.astype('float32')

    X_train /= 255.
    X_test /= 255.
    train_data = list(zip(X_train, y_train))
    nr_train = X_train.shape[0]
    random.shuffle(train_data)
    heldout_data = train_data[:int(nr_train * 0.1)]
    train_data = train_data[len(heldout_data):]
    test_data = list(zip(X_test, y_test))
    return train_data, heldout_data, test_data

def reuters(): # pragma: no cover
    from ._vendorized.keras_datasets import load_reuters
    (X_train, y_train), (X_test, y_test) = load_reuters()
    return (X_train, y_train), (X_test, y_test)

def quora_questions(loc=None):
    if loc is None:
        loc = get_file('quora_similarity.tsv', QUORA_QUESTIONS_URL)
    if isinstance(loc, basestring):
        loc = Path(loc)
    is_header = True
    lines = []
    with loc.open('r') as file_:
        for row in csv.reader(file_, delimiter='\t'):
            if is_header:
                is_header = False
                continue
            id_, qid1, qid2, sent1, sent2, is_duplicate = row
            sent1 = sent1.decode('utf8').strip()
            sent2 = sent2.decode('utf8').strip()
            if sent1 and sent2:
                lines.append(((sent1, sent2), int(is_duplicate)))
    train, dev = partition(lines, 0.9)
    return train, dev

THREE_LABELS = {'entailment': 2, 'contradiction': 1, 'neutral': 0}
TWO_LABELS = {'entailment': 1, 'contradiction': 0, 'neutral': 0}
def snli(loc=None, ternary=False):
    label_scheme = THREE_LABELS if ternary else TWO_LABELS
    if loc is None:
        loc = get_file('snli_1.0', SNLI_URL, unzip=True)
    if isinstance(loc, basestring):
        loc = Path(loc)

    train = read_snli(Path(loc) / 'snli_1.0_train.jsonl', label_scheme)
    dev = read_snli(Path(loc) / 'snli_1.0_dev.jsonl', label_scheme)
    return train, dev

def stack_exchange(loc=None):
    if loc is None:
        raise ValueError("No default path for Stack Exchange yet")
    rows = []
    with loc.open() as file_:
        for line in file_:
            eg = json.loads(line)
            rows.append(((eg['text1'], eg['text2']), int(eg['label'])))
    train, dev = partition(rows, 0.7)
    return train, dev

def read_snli(loc, label_scheme):
    rows = []
    with loc.open() as file_:
        for line in file_:
            eg = json.loads(line)
            label = eg['gold_label']
            if label == '-':
                continue
            rows.append(((eg['sentence1'], eg['sentence2']), label_scheme[label]))
    return rows

def get_word_index(path='reuters_word_index.pkl'): # pragma: no cover
    path = get_file(path, origin='https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl')
    f = open(path, 'rb')

    if sys.version_info < (3,):
        data = cPickle.load(f)
    else:
        data = cPickle.load(f, encoding='latin1')

    f.close()
    return data

<commit_msg>Support file limit in imdb data<commit_after>import random # pragma: no cover
import io # pragma: no cover
from collections import Counter # pragma: no cover
import os.path # pragma: no cover
import csv # pragma: no cover
import numpy
from pathlib import Path
import json

from ._vendorized.keras_data_utils import get_file # pragma: no cover
from ..neural.util import partition
from ..neural.util import to_categorical

try:
    basestring
except NameError:
    basestring = str

GITHUB = 'https://github.com/UniversalDependencies/' # pragma: no cover
ANCORA_1_4_ZIP = '{github}/{ancora}/archive/r1.4.zip'.format(
    github=GITHUB, ancora='UD_Spanish-AnCora') # pragma: no cover
EWTB_1_4_ZIP = '{github}/{ewtb}/archive/r1.4.zip'.format(
    github=GITHUB, ewtb='UD_English') # pragma: no cover

SNLI_URL = 'http://nlp.stanford.edu/projects/snli/snli_1.0.zip'
QUORA_QUESTIONS_URL = 'http://qim.ec.quoracdn.net/quora_duplicate_questions.tsv'
IMDB_URL = 'http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz'

def ancora_pos_tags(encode_words=False): # pragma: no cover
    data_dir = get_file('UD_Spanish-AnCora-r1.4', ANCORA_1_4_ZIP,
                        unzip=True)
    train_loc = os.path.join(data_dir, 'es_ancora-ud-train.conllu')
    dev_loc = os.path.join(data_dir, 'es_ancora-ud-dev.conllu')
    return ud_pos_tags(train_loc, dev_loc, encode_words=encode_words)

def ewtb_pos_tags(encode_tags=False, encode_words=False): # pragma: no cover
    data_dir = get_file('UD_English-r1.4', EWTB_1_4_ZIP, unzip=True)
    train_loc = os.path.join(data_dir, 'en-ud-train.conllu')
    dev_loc = os.path.join(data_dir, 'en-ud-dev.conllu')
    return ud_pos_tags(train_loc, dev_loc,
        encode_tags=encode_tags, encode_words=encode_words)

def ud_pos_tags(train_loc, dev_loc, encode_tags=True, encode_words=True): # pragma: no cover
    train_sents = list(read_conll(train_loc))
    dev_sents = list(read_conll(dev_loc))
    tagmap = {}
    freqs = Counter()
    for words, tags in train_sents:
        for tag in tags:
            tagmap.setdefault(tag, len(tagmap))
        for word in words:
            freqs[word] += 1
    vocab = {word: i for i, (word, freq) in enumerate(freqs.most_common())
             if (freq >= 5)}

    def _encode(sents):
        X = []
        y = []
        for words, tags  in sents:
            if encode_words:
                X.append(
                    numpy.asarray(
                        [vocab.get(word, len(vocab)) for word in words],
                        dtype='uint64'))
            else:
                X.append(words)
            if encode_tags:
                y.append(numpy.asarray(
                    [tagmap[tag] for tag in tags],
                    dtype='int32'))
            else:
                y.append(tags)
        return zip(X, y)

    return _encode(train_sents), _encode(dev_sents), len(tagmap)

def imdb(loc=None, limit=0):
    if loc is None:
        loc = get_file('aclImdb', IMDB_URL, untar=True, unzip=True)
    train_loc = Path(loc) / 'train'
    test_loc = Path(loc) / 'test'
    return read_imdb(train_loc, limit=limit), read_imdb(test_loc, limit=limit)

def read_wikiner(file_, tagmap=None):
    Xs = []
    ys = []
    for line in file_:
        if not line.strip():
            continue
        tokens = [t.rsplit('|', 2) for t in line.split()]
        words, _, tags = zip(*tokens)
        if tagmap is not None:
            tags = [tagmap.setdefault(tag, len(tagmap)) for tag in tags]
        Xs.append(words)
        ys.append(tags)
    return zip(Xs, ys)

def read_imdb(data_dir, limit=0):
    examples = []
    for subdir, label in (('pos', 1), ('neg', 0)):
        for filename in (data_dir / subdir).iterdir():
            with filename.open() as file_:
                text = file_.read()
            if text.strip():
                examples.append((text, label))
    random.shuffle(examples)
    if limit >= 1:
        examples = examples[:limit]
    return examples

def read_conll(loc): # pragma: no cover
    n = 0
    with io.open(loc, encoding='utf8') as file_:
        sent_strs = file_.read().strip().split('\n\n')
    for sent_str in sent_strs:
        lines = [line.split() for line in sent_str.split('\n')
                 if not line.startswith('#')]
        words = []
        tags = []
        for i, pieces in enumerate(lines):
            if len(pieces) == 4:
                word, pos, head, label = pieces
            else:
                idx, word, lemma, pos1, pos, morph, head, label, _, _2 = pieces
            if '-' in idx:
                continue
            words.append(word)
            tags.append(pos)
        yield words, tags

def mnist(): # pragma: no cover
    from ._vendorized.keras_datasets import load_mnist

    # the data, shuffled and split between tran and test sets
    (X_train, y_train), (X_test, y_test) = load_mnist()

    X_train = X_train.reshape(60000, 784)
    X_test = X_test.reshape(10000, 784)
    X_train = X_train.astype('float32')
    X_test = X_test.astype('float32')

    X_train /= 255.
    X_test /= 255.
    train_data = list(zip(X_train, y_train))
    nr_train = X_train.shape[0]
    random.shuffle(train_data)
    heldout_data = train_data[:int(nr_train * 0.1)]
    train_data = train_data[len(heldout_data):]
    test_data = list(zip(X_test, y_test))
    return train_data, heldout_data, test_data

def reuters(): # pragma: no cover
    from ._vendorized.keras_datasets import load_reuters
    (X_train, y_train), (X_test, y_test) = load_reuters()
    return (X_train, y_train), (X_test, y_test)

def quora_questions(loc=None):
    if loc is None:
        loc = get_file('quora_similarity.tsv', QUORA_QUESTIONS_URL)
    if isinstance(loc, basestring):
        loc = Path(loc)
    is_header = True
    lines = []
    with loc.open('r') as file_:
        for row in csv.reader(file_, delimiter='\t'):
            if is_header:
                is_header = False
                continue
            id_, qid1, qid2, sent1, sent2, is_duplicate = row
            sent1 = sent1.decode('utf8').strip()
            sent2 = sent2.decode('utf8').strip()
            if sent1 and sent2:
                lines.append(((sent1, sent2), int(is_duplicate)))
    train, dev = partition(lines, 0.9)
    return train, dev

THREE_LABELS = {'entailment': 2, 'contradiction': 1, 'neutral': 0}
TWO_LABELS = {'entailment': 1, 'contradiction': 0, 'neutral': 0}
def snli(loc=None, ternary=False):
    label_scheme = THREE_LABELS if ternary else TWO_LABELS
    if loc is None:
        loc = get_file('snli_1.0', SNLI_URL, unzip=True)
    if isinstance(loc, basestring):
        loc = Path(loc)

    train = read_snli(Path(loc) / 'snli_1.0_train.jsonl', label_scheme)
    dev = read_snli(Path(loc) / 'snli_1.0_dev.jsonl', label_scheme)
    return train, dev

def stack_exchange(loc=None):
    if loc is None:
        raise ValueError("No default path for Stack Exchange yet")
    rows = []
    with loc.open() as file_:
        for line in file_:
            eg = json.loads(line)
            rows.append(((eg['text1'], eg['text2']), int(eg['label'])))
    train, dev = partition(rows, 0.7)
    return train, dev

def read_snli(loc, label_scheme):
    rows = []
    with loc.open() as file_:
        for line in file_:
            eg = json.loads(line)
            label = eg['gold_label']
            if label == '-':
                continue
            rows.append(((eg['sentence1'], eg['sentence2']), label_scheme[label]))
    return rows

def get_word_index(path='reuters_word_index.pkl'): # pragma: no cover
    path = get_file(path, origin='https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl')
    f = open(path, 'rb')

    if sys.version_info < (3,):
        data = cPickle.load(f)
    else:
        data = cPickle.load(f, encoding='latin1')

    f.close()
    return data

<|endoftext|>


<commit_before><commit_msg>add addBone Function to skinSetterUI<commit_after><|endoftext|>


<commit_before>from os.path import join, dirname, abspath

from qtpy.QtCore import Qt, QMetaObject, Signal, Slot, QEvent
from qtpy.QtWidgets import (QWidget, QVBoxLayout, QHBoxLayout, QToolButton,
                            QLabel, QSizePolicy)

from ._utils import QT_VERSION

_FL_STYLESHEET = join(dirname(abspath(__file__)), 'resources/frameless.qss')
""" str: Frameless window stylesheet. """

class WindowDragger(QWidget):
    """ Window dragger.

        Args:
            window (QWidget): Associated window.
            parent (QWidget, optional): Parent widget.
    """

    doubleClicked = Signal()

    def __init__(self, window, parent=None):
        QWidget.__init__(self, parent)

        self._window = window
        self._mousePressed = False

    def mousePressEvent(self, event):
        self._mousePressed = True
        self._mousePos = event.globalPos()
        self._windowPos = self._window.pos()

    def mouseMoveEvent(self, event):
        if self._mousePressed:
            self._window.move(self._windowPos +
                              (event.globalPos() - self._mousePos))

    def mouseReleaseEvent(self, event):
        self._mousePressed = False

    def mouseDoubleClickEvent(self, event):
        self.doubleClicked.emit()

class ModernWindow(QWidget):
    """ Modern window.

        Args:
            w (QWidget): Main widget.
            parent (QWidget, optional): Parent widget.
    """

    def __init__(self, w, parent=None):
        QWidget.__init__(self, parent)

        self._w = w
        self.setupUi()

        contentLayout = QHBoxLayout()
        contentLayout.setContentsMargins(0, 0, 0, 0)
        contentLayout.addWidget(w)

        self.windowContent.setLayout(contentLayout)

        self.setWindowTitle(w.windowTitle())
        self.setGeometry(w.geometry())

        self.installEventFilter(self)

    def setupUi(self):
        # create title bar, content
        self.vboxWindow = QVBoxLayout(self)
        self.vboxWindow.setContentsMargins(0, 0, 0, 0)

        self.windowFrame = QWidget(self)
        self.windowFrame.setObjectName('windowFrame')

        self.vboxFrame = QVBoxLayout(self.windowFrame)
        self.vboxFrame.setContentsMargins(0, 0, 0, 0)

        self.titleBar = WindowDragger(self, self.windowFrame)
        self.titleBar.setObjectName('titleBar')
        self.titleBar.setSizePolicy(QSizePolicy(QSizePolicy.Preferred,
                                                QSizePolicy.Fixed))

        self.hboxTitle = QHBoxLayout(self.titleBar)
        self.hboxTitle.setContentsMargins(0, 0, 0, 0)
        self.hboxTitle.setSpacing(0)

        self.lblTitle = QLabel('Title')
        self.lblTitle.setObjectName('lblTitle')
        self.lblTitle.setAlignment(Qt.AlignCenter)
        self.hboxTitle.addWidget(self.lblTitle)

        spButtons = QSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)

        self.btnMinimize = QToolButton(self.titleBar)
        self.btnMinimize.setObjectName('btnMinimize')
        self.btnMinimize.setSizePolicy(spButtons)
        self.hboxTitle.addWidget(self.btnMinimize)

        self.btnRestore = QToolButton(self.titleBar)
        self.btnRestore.setObjectName('btnRestore')
        self.btnRestore.setSizePolicy(spButtons)
        self.btnRestore.setVisible(False)
        self.hboxTitle.addWidget(self.btnRestore)

        self.btnMaximize = QToolButton(self.titleBar)
        self.btnMaximize.setObjectName('btnMaximize')
        self.btnMaximize.setSizePolicy(spButtons)
        self.hboxTitle.addWidget(self.btnMaximize)

        self.btnClose = QToolButton(self.titleBar)
        self.btnClose.setObjectName('btnClose')
        self.btnClose.setSizePolicy(spButtons)
        self.hboxTitle.addWidget(self.btnClose)

        self.vboxFrame.addWidget(self.titleBar)

        self.windowContent = QWidget(self.windowFrame)
        self.vboxFrame.addWidget(self.windowContent)

        self.vboxWindow.addWidget(self.windowFrame)

        # set window flags
        self.setWindowFlags(
                Qt.Window | Qt.FramelessWindowHint | Qt.WindowSystemMenuHint)

        if QT_VERSION >= (5,):
            self.setAttribute(Qt.WA_TranslucentBackground)

        # set stylesheet
        with open(_FL_STYLESHEET) as stylesheet:
            self.setStyleSheet(stylesheet.read())

        # automatically connect slots
        QMetaObject.connectSlotsByName(self)

    def eventFilter(self, source, event):
        if event.type() == QEvent.Close:
            return self._w.close()

        return QWidget.eventFilter(self, source, event)

    def setWindowTitle(self, title):
        """ Set window title.

            Args:
                title (str): Title.
        """

        super(ModernWindow, self).setWindowTitle(title)
        self.lblTitle.setText(title)

    @Slot()
    def on_btnMinimize_clicked(self):
        self.setWindowState(Qt.WindowMinimized)

    @Slot()
    def on_btnRestore_clicked(self):
        self.btnRestore.setVisible(False)
        self.btnMaximize.setVisible(True)

        self.setWindowState(Qt.WindowNoState)

    @Slot()
    def on_btnMaximize_clicked(self):
        self.btnRestore.setVisible(True)
        self.btnMaximize.setVisible(False)

        self.setWindowState(Qt.WindowMaximized)

    @Slot()
    def on_btnClose_clicked(self):
        self.close()

    @Slot()
    def on_titleBar_doubleClicked(self):
        if self.btnMaximize.isVisible():
            self.on_btnMaximize_clicked()
        else:
            self.on_btnRestore_clicked()
<commit_msg>Clean up parent window on child delete/close<commit_after>from os.path import join, dirname, abspath

from qtpy.QtCore import Qt, QMetaObject, Signal, Slot, QEvent
from qtpy.QtWidgets import (QWidget, QVBoxLayout, QHBoxLayout, QToolButton,
                            QLabel, QSizePolicy)

from ._utils import QT_VERSION

_FL_STYLESHEET = join(dirname(abspath(__file__)), 'resources/frameless.qss')
""" str: Frameless window stylesheet. """

class WindowDragger(QWidget):
    """ Window dragger.

        Args:
            window (QWidget): Associated window.
            parent (QWidget, optional): Parent widget.
    """

    doubleClicked = Signal()

    def __init__(self, window, parent=None):
        QWidget.__init__(self, parent)

        self._window = window
        self._mousePressed = False

    def mousePressEvent(self, event):
        self._mousePressed = True
        self._mousePos = event.globalPos()
        self._windowPos = self._window.pos()

    def mouseMoveEvent(self, event):
        if self._mousePressed:
            self._window.move(self._windowPos +
                              (event.globalPos() - self._mousePos))

    def mouseReleaseEvent(self, event):
        self._mousePressed = False

    def mouseDoubleClickEvent(self, event):
        self.doubleClicked.emit()

class ModernWindow(QWidget):
    """ Modern window.

        Args:
            w (QWidget): Main widget.
            parent (QWidget, optional): Parent widget.
    """

    def __init__(self, w, parent=None):
        QWidget.__init__(self, parent)

        self._w = w
        self.setupUi()

        contentLayout = QHBoxLayout()
        contentLayout.setContentsMargins(0, 0, 0, 0)
        contentLayout.addWidget(w)

        self.windowContent.setLayout(contentLayout)

        self.setWindowTitle(w.windowTitle())
        self.setGeometry(w.geometry())

        self.installEventFilter(self)

        # Adding attribute to clean up the parent window when the child is closed
        self._w.setAttribute(Qt.WA_DeleteOnClose, True)
        self._w.destroyed.connect(self.__child_was_closed)

    def setupUi(self):
        # create title bar, content
        self.vboxWindow = QVBoxLayout(self)
        self.vboxWindow.setContentsMargins(0, 0, 0, 0)

        self.windowFrame = QWidget(self)
        self.windowFrame.setObjectName('windowFrame')

        self.vboxFrame = QVBoxLayout(self.windowFrame)
        self.vboxFrame.setContentsMargins(0, 0, 0, 0)

        self.titleBar = WindowDragger(self, self.windowFrame)
        self.titleBar.setObjectName('titleBar')
        self.titleBar.setSizePolicy(QSizePolicy(QSizePolicy.Preferred,
                                                QSizePolicy.Fixed))

        self.hboxTitle = QHBoxLayout(self.titleBar)
        self.hboxTitle.setContentsMargins(0, 0, 0, 0)
        self.hboxTitle.setSpacing(0)

        self.lblTitle = QLabel('Title')
        self.lblTitle.setObjectName('lblTitle')
        self.lblTitle.setAlignment(Qt.AlignCenter)
        self.hboxTitle.addWidget(self.lblTitle)

        spButtons = QSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)

        self.btnMinimize = QToolButton(self.titleBar)
        self.btnMinimize.setObjectName('btnMinimize')
        self.btnMinimize.setSizePolicy(spButtons)
        self.hboxTitle.addWidget(self.btnMinimize)

        self.btnRestore = QToolButton(self.titleBar)
        self.btnRestore.setObjectName('btnRestore')
        self.btnRestore.setSizePolicy(spButtons)
        self.btnRestore.setVisible(False)
        self.hboxTitle.addWidget(self.btnRestore)

        self.btnMaximize = QToolButton(self.titleBar)
        self.btnMaximize.setObjectName('btnMaximize')
        self.btnMaximize.setSizePolicy(spButtons)
        self.hboxTitle.addWidget(self.btnMaximize)

        self.btnClose = QToolButton(self.titleBar)
        self.btnClose.setObjectName('btnClose')
        self.btnClose.setSizePolicy(spButtons)
        self.hboxTitle.addWidget(self.btnClose)

        self.vboxFrame.addWidget(self.titleBar)

        self.windowContent = QWidget(self.windowFrame)
        self.vboxFrame.addWidget(self.windowContent)

        self.vboxWindow.addWidget(self.windowFrame)

        # set window flags
        self.setWindowFlags(
                Qt.Window | Qt.FramelessWindowHint | Qt.WindowSystemMenuHint)

        if QT_VERSION >= (5,):
            self.setAttribute(Qt.WA_TranslucentBackground)

        # set stylesheet
        with open(_FL_STYLESHEET) as stylesheet:
            self.setStyleSheet(stylesheet.read())

        # automatically connect slots
        QMetaObject.connectSlotsByName(self)

    def __child_was_closed(self):
        self._w = None  # The child was deleted, remove the reference to it and close the parent window
        self.close()

    def eventFilter(self, source, event):
        if event.type() == QEvent.Close:
            if not self._w:
                return True
            return self._w.close()

        return QWidget.eventFilter(self, source, event)

    def setWindowTitle(self, title):
        """ Set window title.

            Args:
                title (str): Title.
        """

        super(ModernWindow, self).setWindowTitle(title)
        self.lblTitle.setText(title)

    @Slot()
    def on_btnMinimize_clicked(self):
        self.setWindowState(Qt.WindowMinimized)

    @Slot()
    def on_btnRestore_clicked(self):
        self.btnRestore.setVisible(False)
        self.btnMaximize.setVisible(True)

        self.setWindowState(Qt.WindowNoState)

    @Slot()
    def on_btnMaximize_clicked(self):
        self.btnRestore.setVisible(True)
        self.btnMaximize.setVisible(False)

        self.setWindowState(Qt.WindowMaximized)

    @Slot()
    def on_btnClose_clicked(self):
        self.close()

    @Slot()
    def on_titleBar_doubleClicked(self):
        if self.btnMaximize.isVisible():
            self.on_btnMaximize_clicked()
        else:
            self.on_btnRestore_clicked()
<|endoftext|>


<commit_before><commit_msg>Use a per API instance requests Session<commit_after><|endoftext|>


<commit_before>import saliweb.backend
import logging, os
import shutil
from random import randint,choice
import string, re
import subprocess

class Stage(object):
    """Keep track of where we are in a multi-stage job.
       The current stage of the job is stored in a file in the job directory."""

    filename = 'stage.out'

    @classmethod
    def read(cls):
        """Get the current job stage. Can be a string or None."""
        if os.path.exists(cls.filename):
            with open(cls.filename) as data:
                return data.readline().strip()

    @classmethod
    def write(cls, val):
        """Set the current job stage. Can be a string or None."""
        if val is None:
            if os.path.exists(cls.filename):
                os.unlink(cls.filename)
        else:
            with open(cls.filename, 'w') as out:
                out.write(val)

class Job(saliweb.backend.Job):

    runnercls = saliweb.backend.SGERunner

    def preprocess(self):
        # Clean up from any previous runs (e.g. a failed run being resubmitted)
        Stage.write(None)

    def _set_random(self):
        """Determine and return a random file name"""
        rint = randint(100,999)
        rfil = ''.join([choice(string.letters)
                       for nl in range(3)]) + '%3i' % rint
        with open('random.out', 'w') as outran:
            outran.write(rfil)
        return rfil

    def _get_random(self):
        """Return the previously-determined random file name"""
        with open('random.out') as outran:
            return outran.read().strip()

    def run(self):
        # Determine which stage we're at
        stages = {None: self.run_first,
                  'pre-AllosMod': self.run_pre_allosmod,
                  'AllosMod': self.run_allosmod,
                  'AllosMod-bmi': self.run_allosmod_bmi}
        return stages[Stage.read()]()

    def run_pre_allosmod(self):
        rfil = self._get_random()
        os.system('zip -r %s.zip XXX' % rfil)
        ### - submit to AllosMod
        r = saliweb.backend.SaliWebServiceRunner('http://modbase.compbio.ucsf.edu/allosmod/job',
                         ['name=%s' % rfil, 'jobemail=peterc@salilab.org', 'zip=@%s.zip' % rfil])
        Stage.write('AllosMod')
        return r

    def run_allosmod(self):
        rfil = self._get_random()
        self.logger.info("Post-processing AllosMod results -bmi")

        ### - setup the SGE script #1
        script = """
export TMPDIR=/scratch
export MYTMP=`mktemp -d`
cd $MYTMP

TT=%s
ARRAY=(/scrapp/$TT/pred_dECALCrAS1000/*)
DRIN=${ARRAY[$SGE_TASK_ID - 1]}
DIN=`echo ${DRIN} | cut -d '/' -f 5`
DROUT=`echo ${DRIN} | cut -d '/' -f 4,5`
DRT="./$DROUT"
mkdir -p $DRT
PDB=`echo $DIN | cut -d \. -f 1`

echo $ARRAY
echo $DRIN
echo $DROUT
echo $DRT
echo $PDB

module load cryptosite
cp -r /netapp/sali/peterc/cryptosite/src_multichain/analysis/for_peter/* .
cp $DRIN/pm.pdb.B*.pdb $DRIN/$PDB_mdl.pdb .

wc *.pdb

cp /netapp/sali/peterc/cryptosite/src_multichain/AM_BMI.py .
cp /netapp/sali/peterc/cryptosite/src_multichain/CHASA.py .

cryptosite soap

cat SnapList.txt

sleep 20

## - pockets
cryptosite pockets

mkdir -p /scrapp/AM/$TT/$DROUT

cp pockets.out /scrapp/AM/$TT/$DROUT

sleep 20

## - AM features
cryptosite am_bmi

cp am_features.out /scrapp/AM/$TT/$DROUT

## - copy energy.dat
ls -l

pwd

cp $DRIN/* $DRT/

echo "AM ANALYSIS"
pwd
echo "./pred_dECALCrAS1000" > list
bash /netapp/sali/peterc/cryptosite/src_multichain/analysis/allosmod_analysis.sh

cp $DRT/*.dat /scrapp/AM/$TT/$DROUT

cat check_runs.out

ls -l

rm -rf $MYTMP
date
""" % (rfil,)

        r = self.runnercls(script)
        r.set_sge_options('-l arch=linux-x64 -l scratch=2G -l mem_free=6G -t 1-25')
        self.logger.info("Calculated pockets for AllosMod results")

        Stage.write('AllosMod-bmi')
        return r

    def run_allosmod_bmi(self):
        self.logger.info("Applying SVM model")

        ### - setup the SGE script #1
        script = """
module load cryptosite
cryptosite predict XXX
date"""

        r = self.runnercls(script)
        r.set_sge_options('-l arch=linux-x64 -l scratch=2G -l mem_free=2G')

        self.logger.info("Prediction DONE!")

        Stage.write('DONE')

        return r

    def run_first(self):
        ### - set logging file
        self.logger.setLevel(logging.INFO)
        self.logger.info("Beginning preprocess() for job %s " %self.name)

        with open('param.txt') as params_file:
            pdb_file, chainid = [i.strip() for i in params_file.readlines()]

        ### - set random number
        rfil = self._set_random()

        ### - setup the SGE script #1
        script = """
ls -lt

pwd

echo "working on"

module load cryptosite

cryptosite setup --short %s %s

# Put AllosMod outputs on /scrapp
echo "SCRAPP=True" >> XXX/input.dat

echo "pre-AllosMod" > stage.out

date

""" % (pdb_file, chainid)

        r = self.runnercls(script)
        r.set_sge_options('-l arch=linux-x64 -l diva1=1G -l scratch=2G -l mem_free=2G')

        self.logger.info("Calculated bioinformatics features for job: %s" % rfil)
        self.logger.info("\n\nSubmitting to AllosMod")
        return r

    def postprocess(self, results=None):
        stages = {'pre-AllosMod': self.reschedule_run,
                  'AllosMod': self.reschedule_run,
                  'AllosMod-bmi': self.postprocess_allosmod_bmi,
                  'DONE': self.postprocess_final}
        return stages[Stage.read()]()

    def postprocess_allosmod_bmi(self):
        """Gather results from processing AllosMod output"""
        rfil = self._get_random()

        ### - gather the AM data
        self.logger.info("Gathering AllosMod results")
        subprocess.check_call("module load cryptosite && cryptosite gather /scrapp/AM/%s" % rfil)

        ### - run SVM
        ## TO DELETE
        Stage.write('AllosMod-bmi')
        ##
        self.reschedule_run()

    def postprocess_final(self):
        ### - make chimera session file
        self.logger.info("Completing the job: writing a Chimera session file.")
        shutil.copy('XXX.pol.pred', 'cryptosite.pol.pred')
        shutil.copy('XXX.pol.pred.pdb', 'cryptosite.pol.pred.pdb')

        with open('script.chimerax') as data:
            chimeraSession = data.read()

        (filepath, jobdir) = os.path.split(self.url)
        pdb_temp = "/cryptosite.pol.pred.pdb?"
        pdb_url = re.sub('\?', pdb_temp, jobdir)
        pdburl = filepath + "/" + pdb_url
        urlAddress = self.url

        ftr_temp = "/cryptosite.pol.pred?"
        ftr_url = re.sub('\?', ftr_temp, jobdir)
        ftrurl = filepath + "/" + ftr_url

        chimeraSession += """
open_files("%s", "%s")
]]>
</py_cmd>
</ChimeraPuppet>""" % (pdburl,ftrurl)

        with open('cryptosite.chimerax', 'w') as out:
            out.write(chimeraSession)

        self.logger.info("Job completed! Results available at: %s" % urlAddress)
        subprocess.check_call(['zip', 'chimera.zip', 'cryptosite.chimerax'])

def get_web_service(config_file):
    db = saliweb.backend.Database(Job)
    config = saliweb.backend.Config(config_file)
    return saliweb.backend.WebService(config, db)
<commit_msg>Run through reindent.py.<commit_after>import saliweb.backend
import logging, os
import shutil
from random import randint,choice
import string, re
import subprocess

class Stage(object):
    """Keep track of where we are in a multi-stage job.
       The current stage of the job is stored in a file in the job directory."""

    filename = 'stage.out'

    @classmethod
    def read(cls):
        """Get the current job stage. Can be a string or None."""
        if os.path.exists(cls.filename):
            with open(cls.filename) as data:
                return data.readline().strip()

    @classmethod
    def write(cls, val):
        """Set the current job stage. Can be a string or None."""
        if val is None:
            if os.path.exists(cls.filename):
                os.unlink(cls.filename)
        else:
            with open(cls.filename, 'w') as out:
                out.write(val)

class Job(saliweb.backend.Job):

    runnercls = saliweb.backend.SGERunner

    def preprocess(self):
        # Clean up from any previous runs (e.g. a failed run being resubmitted)
        Stage.write(None)

    def _set_random(self):
        """Determine and return a random file name"""
        rint = randint(100,999)
        rfil = ''.join([choice(string.letters)
                       for nl in range(3)]) + '%3i' % rint
        with open('random.out', 'w') as outran:
            outran.write(rfil)
        return rfil

    def _get_random(self):
        """Return the previously-determined random file name"""
        with open('random.out') as outran:
            return outran.read().strip()

    def run(self):
        # Determine which stage we're at
        stages = {None: self.run_first,
                  'pre-AllosMod': self.run_pre_allosmod,
                  'AllosMod': self.run_allosmod,
                  'AllosMod-bmi': self.run_allosmod_bmi}
        return stages[Stage.read()]()

    def run_pre_allosmod(self):
        rfil = self._get_random()
        os.system('zip -r %s.zip XXX' % rfil)
        ### - submit to AllosMod
        r = saliweb.backend.SaliWebServiceRunner('http://modbase.compbio.ucsf.edu/allosmod/job',
                         ['name=%s' % rfil, 'jobemail=peterc@salilab.org', 'zip=@%s.zip' % rfil])
        Stage.write('AllosMod')
        return r

    def run_allosmod(self):
        rfil = self._get_random()
        self.logger.info("Post-processing AllosMod results -bmi")

        ### - setup the SGE script #1
        script = """
export TMPDIR=/scratch
export MYTMP=`mktemp -d`
cd $MYTMP

TT=%s
ARRAY=(/scrapp/$TT/pred_dECALCrAS1000/*)
DRIN=${ARRAY[$SGE_TASK_ID - 1]}
DIN=`echo ${DRIN} | cut -d '/' -f 5`
DROUT=`echo ${DRIN} | cut -d '/' -f 4,5`
DRT="./$DROUT"
mkdir -p $DRT
PDB=`echo $DIN | cut -d \. -f 1`

echo $ARRAY
echo $DRIN
echo $DROUT
echo $DRT
echo $PDB

module load cryptosite
cp -r /netapp/sali/peterc/cryptosite/src_multichain/analysis/for_peter/* .
cp $DRIN/pm.pdb.B*.pdb $DRIN/$PDB_mdl.pdb .

wc *.pdb

cp /netapp/sali/peterc/cryptosite/src_multichain/AM_BMI.py .
cp /netapp/sali/peterc/cryptosite/src_multichain/CHASA.py .

cryptosite soap

cat SnapList.txt

sleep 20

## - pockets
cryptosite pockets

mkdir -p /scrapp/AM/$TT/$DROUT

cp pockets.out /scrapp/AM/$TT/$DROUT

sleep 20

## - AM features
cryptosite am_bmi

cp am_features.out /scrapp/AM/$TT/$DROUT

## - copy energy.dat
ls -l

pwd

cp $DRIN/* $DRT/

echo "AM ANALYSIS"
pwd
echo "./pred_dECALCrAS1000" > list
bash /netapp/sali/peterc/cryptosite/src_multichain/analysis/allosmod_analysis.sh

cp $DRT/*.dat /scrapp/AM/$TT/$DROUT

cat check_runs.out

ls -l

rm -rf $MYTMP
date
""" % (rfil,)

        r = self.runnercls(script)
        r.set_sge_options('-l arch=linux-x64 -l scratch=2G -l mem_free=6G -t 1-25')
        self.logger.info("Calculated pockets for AllosMod results")

        Stage.write('AllosMod-bmi')
        return r

    def run_allosmod_bmi(self):
        self.logger.info("Applying SVM model")

        ### - setup the SGE script #1
        script = """
module load cryptosite
cryptosite predict XXX
date"""

        r = self.runnercls(script)
        r.set_sge_options('-l arch=linux-x64 -l scratch=2G -l mem_free=2G')

        self.logger.info("Prediction DONE!")

        Stage.write('DONE')

        return r

    def run_first(self):
        ### - set logging file
        self.logger.setLevel(logging.INFO)
        self.logger.info("Beginning preprocess() for job %s " %self.name)

        with open('param.txt') as params_file:
            pdb_file, chainid = [i.strip() for i in params_file.readlines()]

        ### - set random number
        rfil = self._set_random()

        ### - setup the SGE script #1
        script = """
ls -lt

pwd

echo "working on"

module load cryptosite

cryptosite setup --short %s %s

# Put AllosMod outputs on /scrapp
echo "SCRAPP=True" >> XXX/input.dat

echo "pre-AllosMod" > stage.out

date

""" % (pdb_file, chainid)

        r = self.runnercls(script)
        r.set_sge_options('-l arch=linux-x64 -l diva1=1G -l scratch=2G -l mem_free=2G')

        self.logger.info("Calculated bioinformatics features for job: %s" % rfil)
        self.logger.info("\n\nSubmitting to AllosMod")
        return r

    def postprocess(self, results=None):
        stages = {'pre-AllosMod': self.reschedule_run,
                  'AllosMod': self.reschedule_run,
                  'AllosMod-bmi': self.postprocess_allosmod_bmi,
                  'DONE': self.postprocess_final}
        return stages[Stage.read()]()

    def postprocess_allosmod_bmi(self):
        """Gather results from processing AllosMod output"""
        rfil = self._get_random()

        ### - gather the AM data
        self.logger.info("Gathering AllosMod results")
        subprocess.check_call("module load cryptosite && cryptosite gather /scrapp/AM/%s" % rfil)

        ### - run SVM
        ## TO DELETE
        Stage.write('AllosMod-bmi')
        ##
        self.reschedule_run()

    def postprocess_final(self):
        ### - make chimera session file
        self.logger.info("Completing the job: writing a Chimera session file.")
        shutil.copy('XXX.pol.pred', 'cryptosite.pol.pred')
        shutil.copy('XXX.pol.pred.pdb', 'cryptosite.pol.pred.pdb')

        with open('script.chimerax') as data:
            chimeraSession = data.read()

        (filepath, jobdir) = os.path.split(self.url)
        pdb_temp = "/cryptosite.pol.pred.pdb?"
        pdb_url = re.sub('\?', pdb_temp, jobdir)
        pdburl = filepath + "/" + pdb_url
        urlAddress = self.url

        ftr_temp = "/cryptosite.pol.pred?"
        ftr_url = re.sub('\?', ftr_temp, jobdir)
        ftrurl = filepath + "/" + ftr_url

        chimeraSession += """
open_files("%s", "%s")
]]>
</py_cmd>
</ChimeraPuppet>""" % (pdburl,ftrurl)

        with open('cryptosite.chimerax', 'w') as out:
            out.write(chimeraSession)

        self.logger.info("Job completed! Results available at: %s" % urlAddress)
        subprocess.check_call(['zip', 'chimera.zip', 'cryptosite.chimerax'])

def get_web_service(config_file):
    db = saliweb.backend.Database(Job)
    config = saliweb.backend.Config(config_file)
    return saliweb.backend.WebService(config, db)
<|endoftext|>


<commit_before>"""
Filename: kalman.py
Reference: http://quant-econ.net/py/kalman.html

Implements the Kalman filter for a linear Gaussian state space model.

"""
from textwrap import dedent
import numpy as np
from numpy import dot
from scipy.linalg import inv
from quantecon.matrix_eqn import solve_discrete_riccati

class Kalman(object):
    r"""
    Implements the Kalman filter for the Gaussian state space model

        x_{t+1} = A x_t + C w_{t+1}
        y_t = G x_t + H v_t.

    Here x_t is the hidden state and y_t is the measurement. The shocks
    w_t and v_t are iid standard normals.  Below we use the notation

        Q := CC'
        R := HH'

    Parameters
    -----------
    ss : instance of LinearStateSpace
        An instance of the quantecon.lss.LinearStateSpace class
    x_hat : scalar(float) or array_like(float), optional(default=None)
        An n x 1 array representing the mean x_hat and covariance
        matrix Sigma of the prior/predictive density.  Set to zero if
        not supplied.
    Sigma : scalar(float) or array_like(float), optional(default=None)
        An n x n array representing the covariance matrix Sigma of
        the prior/predictive density.  Must be positive definite.
        Set to the identity if not supplied.

    Attributes
    ----------
    Sigma, x_hat : as above
    Sigma_infinity : array_like or scalar(float)
        The infinite limit of Sigma_t
    K_infinity : array_like or scalar(float)
        The stationary Kalman gain.

    References
    ----------

    http://quant-econ.net/py/kalman.html

    """

    def __init__(self, ss, x_hat=None, Sigma=None):
        self.ss = ss
        self.set_state(x_hat, Sigma)
        self.K_infinity = None
        self.Sigma_infinity = None

    def set_state(self, x_hat, Sigma):
        if Sigma is None:
            Sigma = np.identity(self.ss.n)
        else:
            self.Sigma = np.atleast_2d(Sigma)
        if x_hat is None:
            x_hat = np.zeros((self.ss.n, 1))
        else:
            self.x_hat = np.atleast_2d(x_hat)
            self.x_hat.shape = self.ss.n, 1

    def __repr__(self):
        return self.__str__()

    def __str__(self):
        m = """\
        Kalman filter:
          - dimension of state space          : {n}
          - dimension of observation equation : {k}
        """
        return dedent(m.format(n=self.ss.n, k=self.ss.k))

    def prior_to_filtered(self, y):
        r"""
        Updates the moments (x_hat, Sigma) of the time t prior to the
        time t filtering distribution, using current measurement y_t.

        The updates are according to

            x_{hat}^F = x_{hat} + Sigma G' (G Sigma G' + R)^{-1}
                (y - G x_{hat})
            Sigma^F = Sigma - Sigma G' (G Sigma G' + R)^{-1} G
                Sigma

        Parameters
        ----------
        y : scalar or array_like(float)
            The current measurement

        """
        # === simplify notation === #
        G, H = self.ss.G, self.ss.H
        R = np.dot(H, H.T)

        # === and then update === #
        y = np.atleast_2d(y)
        y.shape = self.ss.k, 1
        E = dot(self.Sigma, G.T)
        F = dot(dot(G, self.Sigma), G.T) + R
        M = dot(E, inv(F))
        self.x_hat = self.x_hat + dot(M, (y - dot(G, self.x_hat)))
        self.Sigma = self.Sigma - dot(M, dot(G,  self.Sigma))

    def filtered_to_forecast(self):
        """
        Updates the moments of the time t filtering distribution to the
        moments of the predictive distribution, which becomes the time
        t+1 prior

        """
        # === simplify notation === #
        A, C = self.ss.A, self.ss.C
        Q = np.dot(C, C.T)

        # === and then update === #
        self.x_hat = dot(A, self.x_hat)
        self.Sigma = dot(A, dot(self.Sigma, A.T)) + Q

    def update(self, y):
        """
        Updates x_hat and Sigma given k x 1 ndarray y.  The full
        update, from one period to the next

        Parameters
        ----------
        y : np.ndarray
            A k x 1 ndarray y representing the current measurement

        """
        self.prior_to_filtered(y)
        self.filtered_to_forecast()

    def stationary_values(self):
        """
        Computes the limit of Sigma_t as t  goes to infinity by
        solving the associated Riccati equation.  Computation is via the
        doubling algorithm (see the documentation in
        `matrix_eqn.solve_discrete_riccati`).

        Returns
        -------
        Sigma_infinity : array_like or scalar(float)
            The infinite limit of Sigma_t
        K_infinity : array_like or scalar(float)
            The stationary Kalman gain.

        """
        # === simplify notation === #
        A, C, G, H = self.ss.A, self.ss.C, self.ss.G, self.ss.H
        Q, R = np.dot(C, C.T), np.dot(H, H.T)

        # === solve Riccati equation, obtain Kalman gain === #
        Sigma_infinity = solve_discrete_riccati(A.T, G.T, Q, R)
        temp1 = dot(dot(A, Sigma_infinity), G.T)
        temp2 = inv(dot(G, dot(Sigma_infinity, G.T)) + R)
        K_infinity = dot(temp1, temp2)

        # == record as attributes and return == #
        self.Sigma_infinity, self.K_infinity = Sigma_infinity, K_infinity
        return Sigma_infinity, K_infinity

    def stationary_coefficients(self, j, coeff_type='ma'):
        """
        Wold representation moving average or VAR coefficients for the
        steady state Kalman filter.

        Parameters
        ----------
        j : int
            The lag length
        coeff_type : string, either 'ma' or 'var' (default='ma')
            The type of coefficent sequence to compute.  Either 'ma' for
            moving average or 'var' for VAR.
        """
        # == simplify notation == #
        A, G = self.ss.A, self.ss.G
        K_infinity = self.K_infinity
        # == make sure that K_infinity has actually been computed == #
        if K_infinity is None:
            S, K_infinity = self.stationary_values()
        # == compute and return coefficients == #
        coeffs = []
        i = 1
        if coeff_type == 'ma':
            coeffs.append(np.identity(self.ss.k))
            P_mat = A
            P = np.identity(self.ss.n)  # Create a copy
        elif coeff_type == 'var':
            coeffs.append(dot(G, K_infinity))
            P_mat = A - dot(K_infinity, G)
            P = np.copy(P_mat)  # Create a copy
        else:
            raise ValueError("Unknown coefficient type")
        while i <= j:
            coeffs.append(dot(dot(G, P), K_infinity))
            P = dot(P, P_mat)
            i += 1
        return coeffs

    def stationary_innovation_covar(self):
        # == simplify notation == #
        H, G = self.ss.H, self.ss.G
        R = np.dot(H, H.T)
        Sigma_infinity = self.Sigma_infinity

        # == make sure that Sigma_infinity has been computed == #
        if Sigma_infinity is None:
            Sigma_infinity, K = self.stationary_values()
        return dot(G, dot(Sigma_infinity, G.T)) + R
<commit_msg>Add a whitener method to Kalman. Add LSS.<commit_after>"""
Filename: kalman.py
Reference: http://quant-econ.net/py/kalman.html

Implements the Kalman filter for a linear Gaussian state space model.

"""
from textwrap import dedent
import numpy as np
from numpy import dot
from scipy.linalg import inv
from quantecon.lss import LinearStateSpace
from quantecon.matrix_eqn import solve_discrete_riccati

class Kalman(object):
    r"""
    Implements the Kalman filter for the Gaussian state space model

        x_{t+1} = A x_t + C w_{t+1}
        y_t = G x_t + H v_t.

    Here x_t is the hidden state and y_t is the measurement. The shocks
    w_t and v_t are iid standard normals.  Below we use the notation

        Q := CC'
        R := HH'

    Parameters
    -----------
    ss : instance of LinearStateSpace
        An instance of the quantecon.lss.LinearStateSpace class
    x_hat : scalar(float) or array_like(float), optional(default=None)
        An n x 1 array representing the mean x_hat and covariance
        matrix Sigma of the prior/predictive density.  Set to zero if
        not supplied.
    Sigma : scalar(float) or array_like(float), optional(default=None)
        An n x n array representing the covariance matrix Sigma of
        the prior/predictive density.  Must be positive definite.
        Set to the identity if not supplied.

    Attributes
    ----------
    Sigma, x_hat : as above
    Sigma_infinity : array_like or scalar(float)
        The infinite limit of Sigma_t
    K_infinity : array_like or scalar(float)
        The stationary Kalman gain.

    References
    ----------

    http://quant-econ.net/py/kalman.html

    """

    def __init__(self, ss, x_hat=None, Sigma=None):
        self.ss = ss
        self.set_state(x_hat, Sigma)
        self.K_infinity = None
        self.Sigma_infinity = None

    def set_state(self, x_hat, Sigma):
        if Sigma is None:
            Sigma = np.identity(self.ss.n)
        else:
            self.Sigma = np.atleast_2d(Sigma)
        if x_hat is None:
            x_hat = np.zeros((self.ss.n, 1))
        else:
            self.x_hat = np.atleast_2d(x_hat)
            self.x_hat.shape = self.ss.n, 1

    def __repr__(self):
        return self.__str__()

    def __str__(self):
        m = """\
        Kalman filter:
          - dimension of state space          : {n}
          - dimension of observation equation : {k}
        """
        return dedent(m.format(n=self.ss.n, k=self.ss.k))

    def whitener_lss(self):
        r"""
        This function takes the linear state space system
        that is an input to the Kalman class and it converts
        that system to the time-invariant whitener represenation
        given by (NOTE: FIX NOTATION LATER)

            x_{t+1}^* = A x + C v
            a = G x

        where

            x_t = [x+{t}, \hat{x}_{t}, v_{t}]

        and

            A = [A  0    0
                 KG A-KG KH
                 0  0    0]

            C = [C 0
                 0 0
                 0 I]

            G = [G -G H]
        """
        # Check for steady state Sigma and K
        if self.K_infinity is None:
            Sig, K = self.stationary_values()
            self.Sigma_infinity = Sig
            self.K_infinity = K
        else:
            K = self.K_infinity

        # Get the matrix sizes
        n, k, m, l = self.ss.n, self.ss.k, self.ss.m, self.ss.l
        A, C, G, H = self.ss.A, self.ss.C, self.ss.G, self.ss.H

        Atil = np.vstack([np.hstack([A, np.zeros((n, n)), np.zeros((n, l))]),
                          np.hstack([dot(K, G), A-dot(K, G), dot(K, H)]),
                          np.zeros((l, 2*n + l))])

        Ctil = np.vstack([np.hstack([C, np.zeros((n, l))]),
                          np.zeros((n, m+l)),
                          np.hstack([np.zeros((l, m)), np.eye(l)])])

        Gtil = np.hstack([G, -G, H])

        whitened_lss = LinearStateSpace(Atil, Ctil, Gtil)
        self.whitened_lss = whitened_lss

        return whitened_lss

    def prior_to_filtered(self, y):
        r"""
        Updates the moments (x_hat, Sigma) of the time t prior to the
        time t filtering distribution, using current measurement y_t.

        The updates are according to

            x_{hat}^F = x_{hat} + Sigma G' (G Sigma G' + R)^{-1}
                (y - G x_{hat})
            Sigma^F = Sigma - Sigma G' (G Sigma G' + R)^{-1} G
                Sigma

        Parameters
        ----------
        y : scalar or array_like(float)
            The current measurement

        """
        # === simplify notation === #
        G, H = self.ss.G, self.ss.H
        R = np.dot(H, H.T)

        # === and then update === #
        y = np.atleast_2d(y)
        y.shape = self.ss.k, 1
        E = dot(self.Sigma, G.T)
        F = dot(dot(G, self.Sigma), G.T) + R
        M = dot(E, inv(F))
        self.x_hat = self.x_hat + dot(M, (y - dot(G, self.x_hat)))
        self.Sigma = self.Sigma - dot(M, dot(G,  self.Sigma))

    def filtered_to_forecast(self):
        """
        Updates the moments of the time t filtering distribution to the
        moments of the predictive distribution, which becomes the time
        t+1 prior

        """
        # === simplify notation === #
        A, C = self.ss.A, self.ss.C
        Q = np.dot(C, C.T)

        # === and then update === #
        self.x_hat = dot(A, self.x_hat)
        self.Sigma = dot(A, dot(self.Sigma, A.T)) + Q

    def update(self, y):
        """
        Updates x_hat and Sigma given k x 1 ndarray y.  The full
        update, from one period to the next

        Parameters
        ----------
        y : np.ndarray
            A k x 1 ndarray y representing the current measurement

        """
        self.prior_to_filtered(y)
        self.filtered_to_forecast()

    def stationary_values(self):
        """
        Computes the limit of Sigma_t as t  goes to infinity by
        solving the associated Riccati equation.  Computation is via the
        doubling algorithm (see the documentation in
        `matrix_eqn.solve_discrete_riccati`).

        Returns
        -------
        Sigma_infinity : array_like or scalar(float)
            The infinite limit of Sigma_t
        K_infinity : array_like or scalar(float)
            The stationary Kalman gain.

        """
        # === simplify notation === #
        A, C, G, H = self.ss.A, self.ss.C, self.ss.G, self.ss.H
        Q, R = np.dot(C, C.T), np.dot(H, H.T)

        # === solve Riccati equation, obtain Kalman gain === #
        Sigma_infinity = solve_discrete_riccati(A.T, G.T, Q, R)
        temp1 = dot(dot(A, Sigma_infinity), G.T)
        temp2 = inv(dot(G, dot(Sigma_infinity, G.T)) + R)
        K_infinity = dot(temp1, temp2)

        # == record as attributes and return == #
        self.Sigma_infinity, self.K_infinity = Sigma_infinity, K_infinity
        return Sigma_infinity, K_infinity

    def stationary_coefficients(self, j, coeff_type='ma'):
        """
        Wold representation moving average or VAR coefficients for the
        steady state Kalman filter.

        Parameters
        ----------
        j : int
            The lag length
        coeff_type : string, either 'ma' or 'var' (default='ma')
            The type of coefficent sequence to compute.  Either 'ma' for
            moving average or 'var' for VAR.
        """
        # == simplify notation == #
        A, G = self.ss.A, self.ss.G
        K_infinity = self.K_infinity
        # == make sure that K_infinity has actually been computed == #
        if K_infinity is None:
            S, K_infinity = self.stationary_values()
        # == compute and return coefficients == #
        coeffs = []
        i = 1
        if coeff_type == 'ma':
            coeffs.append(np.identity(self.ss.k))
            P_mat = A
            P = np.identity(self.ss.n)  # Create a copy
        elif coeff_type == 'var':
            coeffs.append(dot(G, K_infinity))
            P_mat = A - dot(K_infinity, G)
            P = np.copy(P_mat)  # Create a copy
        else:
            raise ValueError("Unknown coefficient type")
        while i <= j:
            coeffs.append(dot(dot(G, P), K_infinity))
            P = dot(P, P_mat)
            i += 1
        return coeffs

    def stationary_innovation_covar(self):
        # == simplify notation == #
        H, G = self.ss.H, self.ss.G
        R = np.dot(H, H.T)
        Sigma_infinity = self.Sigma_infinity

        # == make sure that Sigma_infinity has been computed == #
        if Sigma_infinity is None:
            Sigma_infinity, K = self.stationary_values()
        return dot(G, dot(Sigma_infinity, G.T)) + R
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/globocom/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com timehome@corp.globo.com

import sys
import optparse

from thumbor import __version__
from libthumbor import CryptoURL
from thumbor.config import Config

def main(arguments=None):
    '''Converts a given url with the specified arguments.'''
    if arguments is None:
        arguments = sys.argv[1:]

    parser = optparse.OptionParser(usage='thumbor-url [options] imageurl or type thumbor-url -h (--help) for help', description=__doc__, version=__version__)

    parser.add_option('-l', '--key_file', dest='key_file', default=None, help='The file to read the security key from [default: %default].')
    parser.add_option('-k', '--key', dest='key', default=None, help='The security key to encrypt the url with [default: %default].')
    parser.add_option('-w', '--width', dest='width', type='int', default=0, help='The target width for the image [default: %default].')
    parser.add_option('-e', '--height', dest='height', type='int', default=0, help='The target height for the image [default: %default].')
    parser.add_option('-n', '--fitin', dest='fitin', action='store_true', default=False, help='Indicates that fit-in resizing should be performed.')
    parser.add_option('-m', '--meta', dest='meta', action='store_true', default=False, help='Indicates that meta information should be retrieved.')
    parser.add_option('', '--adaptive', action='store_true', dest='adaptive', default=False, help='Indicates that adaptive fit-in cropping should be used.')
    parser.add_option('', '--full', action='store_true', dest='full', default=False, help='Indicates that fit-full cropping should be used.')
    parser.add_option('-s', '--smart', action='store_true', dest='smart', default=False, help='Indicates that smart cropping should be used.')
    parser.add_option('-t', '--trim', action='store_true', default=False, help='Indicate that surrounding whitespace should be trimmed.')
    parser.add_option('-f', '--horizontal-flip', action='store_true', dest='horizontal_flip', default=False, help='Indicates that the image should be horizontally flipped.')
    parser.add_option('-v', '--vertical-flip', action='store_true', dest='vertical_flip', default=False, help='Indicates that the image should be vertically flipped.')
    parser.add_option('-a', '--halign', dest='halign', default='center', help='The horizontal alignment to use for cropping [default: %default].')
    parser.add_option('-i', '--valign', dest='valign', default='middle', help='The vertical alignment to use for cropping [default: %default].')
    parser.add_option('', '--filters', dest='filters', default='', help='Filters to be applied to the image, e.g. brightness(10) [default: %default].')
    parser.add_option('-o', '--old-format', dest='old', action='store_true', default=False, help='Indicates that thumbor should generate old-format urls [default: %default].')

    parser.add_option('-c', '--crop', dest='crop', default=None, help='The coordinates of the points to manual cropping in the format leftxtop:rightxbottom (100x200:400x500) [default: %default].')

    (parsed_options, arguments) = parser.parse_args(arguments)

    if not arguments:
        print 'Error: The image argument is mandatory. For more information type thumbor-url -h'
        return

    image_url = arguments[0]
    if image_url.startswith('/'):
        image_url = image_url[1:]

    try:
        config = Config.load(None)
    except:
        config = None

    if not parsed_options.key and not config:
        print 'Error: The -k or --key argument is mandatory. For more information type thumbor-url -h'
        return

    if parsed_options.key_file:
        f = open(parsed_options.key_file)
        security_key = f.read().strip()
        f.close()
    else:
        security_key = config.SECURITY_KEY if not parsed_options.key else parsed_options.key

    crop_left = crop_top = crop_right = crop_bottom = 0
    if parsed_options.crop:
        crops = parsed_options.crop.split(':')
        crop_left, crop_top = crops[0].split('x')
        crop_right, crop_bottom = crops[1].split('x')

    options = {
        'old': parsed_options.old,
        'width': parsed_options.width,
        'height': parsed_options.height,
        'smart': parsed_options.smart,
        'meta': parsed_options.meta,
        'horizontal_flip': parsed_options.horizontal_flip,
        'vertical_flip': parsed_options.vertical_flip,
        'halign': parsed_options.halign,
        'valign': parsed_options.valign,
        'trim': parsed_options.trim,
        'crop_left': crop_left,
        'crop_top': crop_top,
        'crop_right': crop_right,
        'crop_bottom': crop_bottom,
        'filters': parsed_options.filters,
        'image_url': image_url,
    }

    if parsed_options.fitin and parsed_options.full and parsed_options.adaptive:
        options['adaptive_full_fit_in'] = True
    elif parsed_options.fitin and parsed_options.full:
        options['full_fit_in'] = True
    elif parsed_options.fitin and parsed_options.adaptive:
        options['adaptive_fit_in'] = True
    elif parsed_options.fitin:
        options['fit_in'] = True
    print options
    crypto = CryptoURL(key=security_key)
    url = crypto.generate(**options)
    print 'URL:'

    print url
    return url

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>Fixing thumbor-url filters<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/globocom/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com timehome@corp.globo.com

import sys
import optparse

from thumbor import __version__
from libthumbor import CryptoURL
from thumbor.config import Config

def main(arguments=None):
    '''Converts a given url with the specified arguments.'''
    if arguments is None:
        arguments = sys.argv[1:]

    parser = optparse.OptionParser(usage='thumbor-url [options] imageurl or type thumbor-url -h (--help) for help', description=__doc__, version=__version__)

    parser.add_option('-l', '--key_file', dest='key_file', default=None, help='The file to read the security key from [default: %default].')
    parser.add_option('-k', '--key', dest='key', default=None, help='The security key to encrypt the url with [default: %default].')
    parser.add_option('-w', '--width', dest='width', type='int', default=0, help='The target width for the image [default: %default].')
    parser.add_option('-e', '--height', dest='height', type='int', default=0, help='The target height for the image [default: %default].')
    parser.add_option('-n', '--fitin', dest='fitin', action='store_true', default=False, help='Indicates that fit-in resizing should be performed.')
    parser.add_option('-m', '--meta', dest='meta', action='store_true', default=False, help='Indicates that meta information should be retrieved.')
    parser.add_option('', '--adaptive', action='store_true', dest='adaptive', default=False, help='Indicates that adaptive fit-in cropping should be used.')
    parser.add_option('', '--full', action='store_true', dest='full', default=False, help='Indicates that fit-full cropping should be used.')
    parser.add_option('-s', '--smart', action='store_true', dest='smart', default=False, help='Indicates that smart cropping should be used.')
    parser.add_option('-t', '--trim', action='store_true', default=False, help='Indicate that surrounding whitespace should be trimmed.')
    parser.add_option('-f', '--horizontal-flip', action='store_true', dest='horizontal_flip', default=False, help='Indicates that the image should be horizontally flipped.')
    parser.add_option('-v', '--vertical-flip', action='store_true', dest='vertical_flip', default=False, help='Indicates that the image should be vertically flipped.')
    parser.add_option('-a', '--halign', dest='halign', default='center', help='The horizontal alignment to use for cropping [default: %default].')
    parser.add_option('-i', '--valign', dest='valign', default='middle', help='The vertical alignment to use for cropping [default: %default].')
    parser.add_option('', '--filters', dest='filters', action='append', help='Filters to be applied to the image, e.g. brightness(10).')
    parser.add_option('-o', '--old-format', dest='old', action='store_true', default=False, help='Indicates that thumbor should generate old-format urls [default: %default].')

    parser.add_option('-c', '--crop', dest='crop', default=None, help='The coordinates of the points to manual cropping in the format leftxtop:rightxbottom (100x200:400x500) [default: %default].')

    (parsed_options, arguments) = parser.parse_args(arguments)

    if not arguments:
        print 'Error: The image argument is mandatory. For more information type thumbor-url -h'
        return

    image_url = arguments[0]
    if image_url.startswith('/'):
        image_url = image_url[1:]

    try:
        config = Config.load(None)
    except:
        config = None

    if not parsed_options.key and not config:
        print 'Error: The -k or --key argument is mandatory. For more information type thumbor-url -h'
        return

    if parsed_options.key_file:
        f = open(parsed_options.key_file)
        security_key = f.read().strip()
        f.close()
    else:
        security_key = config.SECURITY_KEY if not parsed_options.key else parsed_options.key

    crop_left = crop_top = crop_right = crop_bottom = 0
    if parsed_options.crop:
        crops = parsed_options.crop.split(':')
        crop_left, crop_top = crops[0].split('x')
        crop_right, crop_bottom = crops[1].split('x')

    options = {
        'old': parsed_options.old,
        'width': parsed_options.width,
        'height': parsed_options.height,
        'smart': parsed_options.smart,
        'meta': parsed_options.meta,
        'horizontal_flip': parsed_options.horizontal_flip,
        'vertical_flip': parsed_options.vertical_flip,
        'halign': parsed_options.halign,
        'valign': parsed_options.valign,
        'trim': parsed_options.trim,
        'crop_left': crop_left,
        'crop_top': crop_top,
        'crop_right': crop_right,
        'crop_bottom': crop_bottom,
        'filters': parsed_options.filters,
        'image_url': image_url,
    }

    if parsed_options.fitin and parsed_options.full and parsed_options.adaptive:
        options['adaptive_full_fit_in'] = True
    elif parsed_options.fitin and parsed_options.full:
        options['full_fit_in'] = True
    elif parsed_options.fitin and parsed_options.adaptive:
        options['adaptive_fit_in'] = True
    elif parsed_options.fitin:
        options['fit_in'] = True

    crypto = CryptoURL(key=security_key)
    url = crypto.generate(**options)
    print 'URL:'

    print url
    return url

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>import ast

from restroom import API, RestroomError
from mock import Mock
from sure import expect
from django.conf.urls import url, patterns
from django.http import QueryDict

def transform_to_attrs_dict(url_patterns):
    """
    Transforms url patterns to dictionaries of
    their attributes (regex, view and name)
    """
    return [{'regex': pattern._regex,
             'view': pattern._callback_str,
             'name': pattern.name}
            for pattern in url_patterns]

def test_validate_registration_invalid_fields_raises_error():
    api = API()

    model_class = Mock()
    fields = [
        Mock(attname='id'),
        Mock(attname='text'),
        Mock(attname='status'),
    ]
    model_class._meta = Mock(fields=fields, object_name='MyModel')
    options = {
        'fields': ['id', 'timestamp']
    }
    (api.register
     .when.called_with(model_class, options)
     .should
     .throw(RestroomError, "timestamp is not a valid field of MyModel"))

def test_validate_fields_with_valid_fields():
    api = API()

    (expect(api.validate_fields(['id', 'address'],
     ['id', 'address', 'city', 'ZIP']))
     .to.equal((True, None)))

def test_validate_fields_with_invalid_fields():
    api = API()

    (expect(api.validate_fields(
     ['id', 'name'],
     ['id', 'address', 'city', 'ZIP']))
     .to.equal((False, 'name')))

def test_validate_allowed_methods_with_valid_methods():
    api = API()

    (expect(api.validate_allowed_methods(
     ['GET', 'POST']))
     .to.equal((True, None)))

def test_validate_allowed_methods_with_invalid_methods():
    api = API()

    (expect(api.validate_allowed_methods(
     ['GET', 'RANDOMMETHOD']))
     .to.equal((False, 'RANDOMMETHOD')))

def test_validate_registration_invalid_http_methods_raises_error():
    api = API()

    model_class = Mock()
    fields = [
        Mock(attname='id'),
        Mock(attname='text'),
        Mock(attname='status'),
    ]
    model_class._meta = Mock(fields=fields, object_name='MyModel')
    options = {
        'fields': ['id', 'text'],
        'allowed_methods': ['GET', 'HELLO']
    }
    (api.register
     .when.called_with(model_class, options)
     .should
     .throw(RestroomError, "HELLO is not a valid allowable HTTP method"))

def test_get_url_data():
    # Given a restroom API instance
    api = API()

    api.generate_list_view = Mock()
    api.generate_single_item_view = Mock()
    api.generate_list_view.return_value.as_view.return_value = 'list view'
    (api.generate_single_item_view
     .return_value
     .as_view
     .return_value) = 'single item view'

    # When I get the url_data for the table `table_model`
    # with data {}
    url_data = api.get_url_data('table_model', {})

    # I should get back a dictionary containing
    # the URL Regex, the View string and the URL name
    expected_data = {
        'list_regex': r'^table_model/$',
        'list_view': 'list view',
        'list_name': 'table_model_list_api',
        'single_item_regex': r'^table_model/(?P<_id>[\d]+)/$',
        'single_item_view': 'single item view',
        'single_item_name': 'table_model_single_item_api',
    }
    expect(url_data).to.equal(expected_data)

def test_api_url_data_property():
    # Given a restroom API instance
    api = API()

    # And we have two models, MyModel and YourModel
    MyModel = Mock()
    MyModel._meta.fields = [Mock(attname='id'), Mock(attname='text')]
    MyModel._meta.db_table = "table_mymodel"

    YourModel = Mock()
    YourModel._meta.fields = [Mock(attname='id'), Mock(attname='name')]
    YourModel._meta.db_table = "table_yourmodel"

    # And we register these models to the api
    api.register(MyModel)
    api.register(YourModel)

    api.generate_list_view = Mock()
    api.generate_single_item_view = Mock()
    api.generate_list_view.return_value.as_view.return_value = 'list view'
    (api.generate_single_item_view
     .return_value
     .as_view
     .return_value) = 'single item view'

    # We should get back a list of dictionaries containing
    # the url data for each of these models
    # when we call url_data
    expected_data = [
        {
            'list_regex': r'^table_mymodel/$',
            'list_view': 'list view',
            'list_name': 'table_mymodel_list_api',
            'single_item_regex': r'^table_mymodel/(?P<_id>[\d]+)/$',
            'single_item_view': 'single item view',
            'single_item_name': 'table_mymodel_single_item_api',
        },
        {
            'list_regex': r'^table_yourmodel/$',
            'list_view': 'list view',
            'list_name': 'table_yourmodel_list_api',
            'single_item_regex': r'^table_yourmodel/(?P<_id>[\d]+)/$',
            'single_item_view': 'single item view',
            'single_item_name': 'table_yourmodel_single_item_api',
        },
    ]
    expect(api.url_data).to.equal(expected_data)

def test_api_construct_url_pattern():
    # Given a restroom API instance
    api = API()

    url_data = [{
        'list_regex': r'^table_mymodel/$',
        'list_view': 'list view',
        'list_name': 'table_mymodel_list_api',
        'single_item_regex': r'^table_mymodel/(?P<_id>[\d]+)/$',
        'single_item_view': 'single item view',
        'single_item_name': 'table_mymodel_single_item_api',
    }]

    url_patterns = api.construct_url_patterns(url_data)
    expected_patterns = patterns('',
                                 url(r'^table_mymodel/$',
                                     'list view',
                                     name='table_mymodel_list_api'),
                                 url(r'^table_mymodel/(?P<_id>[\d]+)/$',
                                     'single item view',
                                     name='table_mymodel_single_item_api'),
                                 )

    (expect(transform_to_attrs_dict(url_patterns))
     .to.equal(transform_to_attrs_dict(expected_patterns)))

def test_url_patterns():
    # Given a restroom API instance
    api = API()

    # And we have two models, MyModel and YourModel
    MyModel = Mock()
    MyModel._meta.fields = [Mock(attname='id'), Mock(attname='text')]
    MyModel._meta.db_table = "table_mymodel"

    YourModel = Mock()
    YourModel._meta.fields = [Mock(attname='id'), Mock(attname='name')]
    YourModel._meta.db_table = "table_yourmodel"

    # And we register these models to the api
    api.register(MyModel)
    api.register(YourModel)

    api.generate_list_view = Mock()
    api.generate_single_item_view = Mock()
    api.generate_list_view.return_value.as_view.return_value = 'list view'
    (api.generate_single_item_view
     .return_value
     .as_view
     .return_value) = 'single item view'

    expected_patterns = patterns('',
                                 url(r'^table_mymodel/$',
                                     'list view',
                                     name='table_mymodel_list_api'),
                                 url(r'^table_mymodel/(?P<_id>[\d]+)/$',
                                     'single item view',
                                     name='table_mymodel_single_item_api'),
                                 url(r'^table_yourmodel/$',
                                     'list view',
                                     name='table_yourmodel_list_api'),
                                 url(r'^table_yourmodel/(?P<_id>[\d]+)/$',
                                     'single item view',
                                     name='table_yourmodel_single_item_api'),
                                 )
    url_patterns = api.url_patterns

    (expect(transform_to_attrs_dict(url_patterns))
     .to.equal(transform_to_attrs_dict(expected_patterns)))

def test_generate_list_view_for_get():
    # Given a restroom API instance
    api = API()

    # And we have a model
    MyModel = Mock()
    MyModel._meta.fields = [Mock(attname='id'), Mock(attname='text')]
    MyModel._meta.db_table = "table_mymodel"

    MyModel.objects.values.return_value = [{'id': 1, 'text': 'Shalom'}]
    # And we register this model to the api
    api.register(MyModel)

    # The api view that is generated by api.generate_view
    # should return the expected json data when requested
    # via GET
    request = Mock(method='GET')
    response_from_GET = (api.generate_list_view("table_mymodel")
                         .as_view()(request))
    expected_response_json = [{'id': 1, 'text': 'Shalom'}]
    response_json = ast.literal_eval(response_from_GET.content)
    expect(response_json).to.equal(expected_response_json)

def test_generate_list_view_for_post():
    # Given a restroom API instance
    api = API()

    # And we have a model
    MyModel = Mock()
    MyModel._meta.fields = [Mock(attname='id'), Mock(attname='text')]
    MyModel._meta.db_table = "table_mymodel"

    # We mock the return of Django .create
    # to be an object with id 1 and text=Shalom
    # whose __class__._meta.db_table is `table_mymodel`
    returned_obj = Mock()
    returned_obj.id = 1
    returned_obj.text = 'Shalom'
    _meta = Mock(db_table='table_mymodel')
    returned_obj.__class__ = Mock(_meta=_meta)
    MyModel.objects.create.return_value = returned_obj

    # And we register this model to the api
    api.register(MyModel, {'allowed_methods': ['GET', 'POST']})

    # The api view that is generated by api.generate_view
    # should return the expected json data when requested
    # via POST
    request = Mock(method='POST')
    request.POST = QueryDict('text=Shalom')
    response_from_POST = (api.generate_list_view("table_mymodel")
                          .as_view()(request))
    expected_response_json = {'id': 1, 'text': 'Shalom'}
    response_json = ast.literal_eval(response_from_POST.content)
    expect(response_json).to.equal(expected_response_json)
    MyModel.objects.create.assert_called_once_with(text='Shalom')

def test_generate_single_item_view_for_get():
    # Given a restroom API instance
    api = API()

    # And we have a model
    MyModel = Mock()
    fields = [Mock(attname='id'), Mock(attname='text')]
    db_table = "table_mymodel"
    _meta = Mock(fields=fields, db_table=db_table)
    MyModel._meta = _meta

    _id = 1

    returned_object = Mock()
    returned_object.id = _id
    returned_object.text = 'Shalom'
    returned_object.__class__ = MyModel
    MyModel.objects.get.return_value = returned_object

    MyModel.objects.values.return_value = [{'id': _id, 'text': 'Shalom'}]
    # And we register this model to the api
    api.register(MyModel)

    # The api view that is generated by api.generate_view
    # should return the expected json data when requested
    # via GET
    request = Mock(method='GET')
    response_from_GET = (api.generate_single_item_view("table_mymodel")
                         .as_view()(request, _id))
    expected_response_json = {'id': _id, 'text': 'Shalom'}
    response_json = ast.literal_eval(response_from_GET.content)
    expect(response_json).to.equal(expected_response_json)

def test_generate_single_item_view_for_delete():
    # Given a restroom API instance
    api = API()

    # And we have a model
    MyModel = Mock()
    fields = [Mock(attname='id'), Mock(attname='text')]
    db_table = "table_mymodel"
    _meta = Mock(fields=fields, db_table=db_table)
    MyModel._meta = _meta

    _id = 1

    returned_object = Mock()
    returned_object.id = _id
    returned_object.text = 'Shalom'
    returned_object.__class__ = MyModel
    MyModel.objects.get.return_value = returned_object

    MyModel.objects.values.return_value = [{'id': _id, 'text': 'Shalom'}]
    # And we register this model to the api
    api.register(MyModel, {'allowed_methods': ['GET', 'POST', 'DELETE']})

    request = Mock(method='DELETE')
    response_from_DELETE = (api.generate_single_item_view("table_mymodel")
                            .as_view()(request, _id))
    expected_response_json = {'status': 'deletion successful'}
    response_json = ast.literal_eval(response_from_DELETE.content)
    expect(response_json).to.equal(expected_response_json)
    returned_object.delete.assert_called_once()
<commit_msg>adding func tests for when to expect a forbidden response<commit_after>import ast

from restroom import API, RestroomError
from mock import Mock
from sure import expect
from django.conf.urls import url, patterns
from django.http import QueryDict, HttpResponseForbidden

def transform_to_attrs_dict(url_patterns):
    """
    Transforms url patterns to dictionaries of
    their attributes (regex, view and name)
    """
    return [{'regex': pattern._regex,
             'view': pattern._callback_str,
             'name': pattern.name}
            for pattern in url_patterns]

def test_validate_registration_invalid_fields_raises_error():
    api = API()

    model_class = Mock()
    fields = [
        Mock(attname='id'),
        Mock(attname='text'),
        Mock(attname='status'),
    ]
    model_class._meta = Mock(fields=fields, object_name='MyModel')
    options = {
        'fields': ['id', 'timestamp']
    }
    (api.register
     .when.called_with(model_class, options)
     .should
     .throw(RestroomError, "timestamp is not a valid field of MyModel"))

def test_validate_fields_with_valid_fields():
    api = API()

    (expect(api.validate_fields(['id', 'address'],
     ['id', 'address', 'city', 'ZIP']))
     .to.equal((True, None)))

def test_validate_fields_with_invalid_fields():
    api = API()

    (expect(api.validate_fields(
     ['id', 'name'],
     ['id', 'address', 'city', 'ZIP']))
     .to.equal((False, 'name')))

def test_validate_allowed_methods_with_valid_methods():
    api = API()

    (expect(api.validate_allowed_methods(
     ['GET', 'POST']))
     .to.equal((True, None)))

def test_validate_allowed_methods_with_invalid_methods():
    api = API()

    (expect(api.validate_allowed_methods(
     ['GET', 'RANDOMMETHOD']))
     .to.equal((False, 'RANDOMMETHOD')))

def test_validate_registration_invalid_http_methods_raises_error():
    api = API()

    model_class = Mock()
    fields = [
        Mock(attname='id'),
        Mock(attname='text'),
        Mock(attname='status'),
    ]
    model_class._meta = Mock(fields=fields, object_name='MyModel')
    options = {
        'fields': ['id', 'text'],
        'allowed_methods': ['GET', 'HELLO']
    }
    (api.register
     .when.called_with(model_class, options)
     .should
     .throw(RestroomError, "HELLO is not a valid allowable HTTP method"))

def test_get_url_data():
    # Given a restroom API instance
    api = API()

    api.generate_list_view = Mock()
    api.generate_single_item_view = Mock()
    api.generate_list_view.return_value.as_view.return_value = 'list view'
    (api.generate_single_item_view
     .return_value
     .as_view
     .return_value) = 'single item view'

    # When I get the url_data for the table `table_model`
    # with data {}
    url_data = api.get_url_data('table_model', {})

    # I should get back a dictionary containing
    # the URL Regex, the View string and the URL name
    expected_data = {
        'list_regex': r'^table_model/$',
        'list_view': 'list view',
        'list_name': 'table_model_list_api',
        'single_item_regex': r'^table_model/(?P<_id>[\d]+)/$',
        'single_item_view': 'single item view',
        'single_item_name': 'table_model_single_item_api',
    }
    expect(url_data).to.equal(expected_data)

def test_api_url_data_property():
    # Given a restroom API instance
    api = API()

    # And we have two models, MyModel and YourModel
    MyModel = Mock()
    MyModel._meta.fields = [Mock(attname='id'), Mock(attname='text')]
    MyModel._meta.db_table = "table_mymodel"

    YourModel = Mock()
    YourModel._meta.fields = [Mock(attname='id'), Mock(attname='name')]
    YourModel._meta.db_table = "table_yourmodel"

    # And we register these models to the api
    api.register(MyModel)
    api.register(YourModel)

    api.generate_list_view = Mock()
    api.generate_single_item_view = Mock()
    api.generate_list_view.return_value.as_view.return_value = 'list view'
    (api.generate_single_item_view
     .return_value
     .as_view
     .return_value) = 'single item view'

    # We should get back a list of dictionaries containing
    # the url data for each of these models
    # when we call url_data
    expected_data = [
        {
            'list_regex': r'^table_mymodel/$',
            'list_view': 'list view',
            'list_name': 'table_mymodel_list_api',
            'single_item_regex': r'^table_mymodel/(?P<_id>[\d]+)/$',
            'single_item_view': 'single item view',
            'single_item_name': 'table_mymodel_single_item_api',
        },
        {
            'list_regex': r'^table_yourmodel/$',
            'list_view': 'list view',
            'list_name': 'table_yourmodel_list_api',
            'single_item_regex': r'^table_yourmodel/(?P<_id>[\d]+)/$',
            'single_item_view': 'single item view',
            'single_item_name': 'table_yourmodel_single_item_api',
        },
    ]
    expect(api.url_data).to.equal(expected_data)

def test_api_construct_url_pattern():
    # Given a restroom API instance
    api = API()

    url_data = [{
        'list_regex': r'^table_mymodel/$',
        'list_view': 'list view',
        'list_name': 'table_mymodel_list_api',
        'single_item_regex': r'^table_mymodel/(?P<_id>[\d]+)/$',
        'single_item_view': 'single item view',
        'single_item_name': 'table_mymodel_single_item_api',
    }]

    url_patterns = api.construct_url_patterns(url_data)
    expected_patterns = patterns('',
                                 url(r'^table_mymodel/$',
                                     'list view',
                                     name='table_mymodel_list_api'),
                                 url(r'^table_mymodel/(?P<_id>[\d]+)/$',
                                     'single item view',
                                     name='table_mymodel_single_item_api'),
                                 )

    (expect(transform_to_attrs_dict(url_patterns))
     .to.equal(transform_to_attrs_dict(expected_patterns)))

def test_url_patterns():
    # Given a restroom API instance
    api = API()

    # And we have two models, MyModel and YourModel
    MyModel = Mock()
    MyModel._meta.fields = [Mock(attname='id'), Mock(attname='text')]
    MyModel._meta.db_table = "table_mymodel"

    YourModel = Mock()
    YourModel._meta.fields = [Mock(attname='id'), Mock(attname='name')]
    YourModel._meta.db_table = "table_yourmodel"

    # And we register these models to the api
    api.register(MyModel)
    api.register(YourModel)

    api.generate_list_view = Mock()
    api.generate_single_item_view = Mock()
    api.generate_list_view.return_value.as_view.return_value = 'list view'
    (api.generate_single_item_view
     .return_value
     .as_view
     .return_value) = 'single item view'

    expected_patterns = patterns('',
                                 url(r'^table_mymodel/$',
                                     'list view',
                                     name='table_mymodel_list_api'),
                                 url(r'^table_mymodel/(?P<_id>[\d]+)/$',
                                     'single item view',
                                     name='table_mymodel_single_item_api'),
                                 url(r'^table_yourmodel/$',
                                     'list view',
                                     name='table_yourmodel_list_api'),
                                 url(r'^table_yourmodel/(?P<_id>[\d]+)/$',
                                     'single item view',
                                     name='table_yourmodel_single_item_api'),
                                 )
    url_patterns = api.url_patterns

    (expect(transform_to_attrs_dict(url_patterns))
     .to.equal(transform_to_attrs_dict(expected_patterns)))

def test_generate_list_view_for_get():
    # Given a restroom API instance
    api = API()

    # And we have a model
    MyModel = Mock()
    MyModel._meta.fields = [Mock(attname='id'), Mock(attname='text')]
    MyModel._meta.db_table = "table_mymodel"

    MyModel.objects.values.return_value = [{'id': 1, 'text': 'Shalom'}]
    # And we register this model to the api
    api.register(MyModel)

    # The api view that is generated by api.generate_view
    # should return the expected json data when requested
    # via GET
    request = Mock(method='GET')
    response_from_GET = (api.generate_list_view("table_mymodel")
                         .as_view()(request))
    expected_response_json = [{'id': 1, 'text': 'Shalom'}]
    response_json = ast.literal_eval(response_from_GET.content)
    expect(response_json).to.equal(expected_response_json)

def test_generate_list_view_for_GET_when_not_allowed_gets_forbidden():
    # Given a restroom API instance
    api = API()

    # And we have a model
    MyModel = Mock()
    MyModel._meta.fields = [Mock(attname='id'), Mock(attname='text')]
    MyModel._meta.db_table = "table_mymodel"

    # And we register this model to the api
    api.register(MyModel, {'allowed_methods': ['POST']})

    # The api view that is generated by api.generate_view
    # should return the expected json data when requested
    # via GET
    request = Mock(method='GET')
    response_from_GET = (api.generate_list_view("table_mymodel")
                         .as_view()(request))

    forbidden_response = HttpResponseForbidden()
    expect(response_from_GET.status_code).to.equal(forbidden_response.status_code)
    expect(response_from_GET.content).to.equal(forbidden_response.content)
    assert not MyModel.objects.values.called

def test_generate_list_view_for_post():
    # Given a restroom API instance
    api = API()

    # And we have a model
    MyModel = Mock()
    MyModel._meta.fields = [Mock(attname='id'), Mock(attname='text')]
    MyModel._meta.db_table = "table_mymodel"

    # We mock the return of Django .create
    # to be an object with id 1 and text=Shalom
    # whose __class__._meta.db_table is `table_mymodel`
    returned_obj = Mock()
    returned_obj.id = 1
    returned_obj.text = 'Shalom'
    _meta = Mock(db_table='table_mymodel')
    returned_obj.__class__ = Mock(_meta=_meta)
    MyModel.objects.create.return_value = returned_obj

    # And we register this model to the api
    api.register(MyModel, {'allowed_methods': ['GET', 'POST']})

    # The api view that is generated by api.generate_view
    # should return the expected json data when requested
    # via POST
    request = Mock(method='POST')
    request.POST = QueryDict('text=Shalom')
    response_from_POST = (api.generate_list_view("table_mymodel")
                          .as_view()(request))
    expected_response_json = {'id': 1, 'text': 'Shalom'}
    response_json = ast.literal_eval(response_from_POST.content)
    expect(response_json).to.equal(expected_response_json)
    MyModel.objects.create.assert_called_once_with(text='Shalom')

def test_generate_list_view_for_POST_when_not_allowed_gets_forbidden():
    # Given a restroom API instance
    api = API()

    # And we have a model
    MyModel = Mock()
    MyModel._meta.fields = [Mock(attname='id'), Mock(attname='text')]
    MyModel._meta.db_table = "table_mymodel"

    # We mock the return of Django .create
    # to be an object with id 1 and text=Shalom
    # whose __class__._meta.db_table is `table_mymodel`
    returned_obj = Mock()
    returned_obj.id = 1
    returned_obj.text = 'Shalom'
    _meta = Mock(db_table='table_mymodel')
    returned_obj.__class__ = Mock(_meta=_meta)

    # And we register this model to the api
    api.register(MyModel, {'allowed_methods': ['GET']})

    # The api view that is generated by api.generate_view
    # should return the expected json data when requested
    # via POST
    request = Mock(method='POST')
    request.POST = QueryDict('text=Shalom')
    response_from_POST = (api.generate_list_view("table_mymodel")
                          .as_view()(request))

    forbidden_response = HttpResponseForbidden()
    expect(response_from_POST.status_code).to.equal(forbidden_response.status_code)
    expect(response_from_POST.content).to.equal(forbidden_response.content)
    assert not MyModel.objects.create.called

def test_generate_single_item_view_for_GET():
    # Given a restroom API instance
    api = API()

    # And we have a model
    MyModel = Mock()
    fields = [Mock(attname='id'), Mock(attname='text')]
    db_table = "table_mymodel"
    _meta = Mock(fields=fields, db_table=db_table)
    MyModel._meta = _meta

    _id = 1

    returned_object = Mock()
    returned_object.id = _id
    returned_object.text = 'Shalom'
    returned_object.__class__ = MyModel
    MyModel.objects.get.return_value = returned_object

    MyModel.objects.values.return_value = [{'id': _id, 'text': 'Shalom'}]
    # And we register this model to the api
    api.register(MyModel)

    # The api view that is generated by api.generate_view
    # should return the expected json data when requested
    # via GET
    request = Mock(method='GET')
    response_from_GET = (api.generate_single_item_view("table_mymodel")
                         .as_view()(request, _id))
    expected_response_json = {'id': _id, 'text': 'Shalom'}
    response_json = ast.literal_eval(response_from_GET.content)
    expect(response_json).to.equal(expected_response_json)

def test_generate_single_item_view_for_GET_when_not_allowed_gets_forbidden():
    # Given a restroom API instance
    api = API()

    # And we have a model
    MyModel = Mock()
    fields = [Mock(attname='id'), Mock(attname='text')]
    db_table = "table_mymodel"
    _meta = Mock(fields=fields, db_table=db_table)
    MyModel._meta = _meta

    _id = 1

    returned_object = Mock()
    returned_object.id = _id
    returned_object.text = 'Shalom'
    returned_object.__class__ = MyModel

    # And we register this model to the api
    api.register(MyModel, {'allowed_methods': ['POST', 'PUT', 'DELETE']})

    # The api view that is generated by api.generate_view
    # should return the expected json data when requested
    # via GET
    request = Mock(method='GET')
    response_from_GET = (api.generate_single_item_view("table_mymodel")
                         .as_view()(request, _id))

    forbidden_response = HttpResponseForbidden()
    expect(response_from_GET.status_code).to.equal(forbidden_response.status_code)
    expect(response_from_GET.content).to.equal(forbidden_response.content)
    assert not MyModel.objects.values.called

def test_generate_single_item_view_for_DELETE():
    # Given a restroom API instance
    api = API()

    # And we have a model
    MyModel = Mock()
    fields = [Mock(attname='id'), Mock(attname='text')]
    db_table = "table_mymodel"
    _meta = Mock(fields=fields, db_table=db_table)
    MyModel._meta = _meta

    _id = 1

    returned_object = Mock()
    returned_object.id = _id
    returned_object.text = 'Shalom'
    returned_object.__class__ = MyModel
    MyModel.objects.get.return_value = returned_object

    MyModel.objects.values.return_value = [{'id': _id, 'text': 'Shalom'}]
    # And we register this model to the api
    api.register(MyModel, {'allowed_methods': ['GET', 'POST', 'DELETE']})

    request = Mock(method='DELETE')
    response_from_DELETE = (api.generate_single_item_view("table_mymodel")
                            .as_view()(request, _id))
    expected_response_json = {'status': 'deletion successful'}
    response_json = ast.literal_eval(response_from_DELETE.content)
    expect(response_json).to.equal(expected_response_json)
    returned_object.delete.assert_called_once()

def test_generate_single_item_view_for_DELETE_when_not_allowed_gets_forbidden():
    # Given a restroom API instance
    api = API()

    # And we have a model
    MyModel = Mock()
    fields = [Mock(attname='id'), Mock(attname='text')]
    db_table = "table_mymodel"
    _meta = Mock(fields=fields, db_table=db_table)
    MyModel._meta = _meta

    _id = 1

    returned_object = Mock()
    returned_object.id = _id
    returned_object.text = 'Shalom'
    returned_object.__class__ = MyModel

    MyModel.objects.values.return_value = [{'id': _id, 'text': 'Shalom'}]
    # And we register this model to the api
    api.register(MyModel, {'allowed_methods': ['GET', 'POST']})

    request = Mock(method='DELETE')
    response_from_DELETE = (api.generate_single_item_view("table_mymodel")
                            .as_view()(request, _id))
    forbidden_response = HttpResponseForbidden()

    expect(response_from_DELETE.status_code).to.equal(forbidden_response.status_code)
    expect(response_from_DELETE.content).to.equal(forbidden_response.content)
    assert not MyModel.objects.get.called
    assert not returned_object.delete.called
<|endoftext|>