<commit_before>"""Caching loader for the 20 newsgroups text classification dataset

The description of the dataset is available on the official website at:

    http://people.csail.mit.edu/jrennie/20Newsgroups/

Quoting the introduction:

    The 20 Newsgroups data set is a collection of approximately 20,000
    newsgroup documents, partitioned (nearly) evenly across 20 different
    newsgroups. To the best of my knowledge, it was originally collected
    by Ken Lang, probably for his Newsweeder: Learning to filter netnews
    paper, though he does not explicitly mention this collection. The 20
    newsgroups collection has become a popular data set for experiments
    in text applications of machine learning techniques, such as text
    classification and text clustering.

This dataset loader will download the recommended "by date" variant of the
dataset and which features a point in time split between the train and
test sets. The compressed dataset size is around 14 Mb compressed. Once
uncompressed the train set is 52 MB and the test set is 34 MB.

The data is downloaded, extracted and cached in the '~/scikit_learn_data'
folder. However contrary to other datasets in the scikit, the data is
not vectorized into numpy arrays but the dataset list the filenames of
the posts and there categories as target signal.

The lack of vector feature extraction is intentional: there is no single
best way to turn text into vectors. Depending on the task various
preprocessing and text transformation are useful or not (n-grams,
lowercasing, stemming, stop-words filtering, TF-IDF weighting...).

"""
# Copyright (c) 2011 Olivier Grisel <olivier.grisel@ensta.org>
# License: Simplified BSD

import os
import urllib
import logging
import tarfile
import pickle
import warnings
import shutil

import numpy as np

from .base import get_data_home
from .base import load_files
from ..utils import check_random_state, deprecated
from ..utils.fixes import in1d

logger = logging.getLogger(__name__)

URL = ("http://people.csail.mit.edu/jrennie/"
            "20Newsgroups/20news-bydate.tar.gz")
ARCHIVE_NAME = "20news-bydate.tar.gz"
CACHE_NAME = "20news-bydate.pkz"
TRAIN_FOLDER = "20news-bydate-train"
TEST_FOLDER = "20news-bydate-test"

def download_20newsgroups(target_dir, cache_path):
    """ Download the 20Newsgroups data and convert is in a zipped pickle
        storage.
    """
    archive_path = os.path.join(target_dir, ARCHIVE_NAME)
    train_path = os.path.join(target_dir, TRAIN_FOLDER)
    test_path = os.path.join(target_dir, TEST_FOLDER)

    if not os.path.exists(target_dir):
        os.makedirs(target_dir)

    if not os.path.exists(archive_path):
        logger.warn("Downloading dataset from %s (14 MB)", URL)
        opener = urllib.urlopen(URL)
        open(archive_path, 'wb').write(opener.read())

    logger.info("Decompressing %s", archive_path)
    tarfile.open(archive_path, "r:gz").extractall(path=target_dir)
    os.remove(archive_path)

    # Store a zipped pickle
    cache = dict(
            train=load_files(train_path),
            test=load_files(test_path)
        )
    open(cache_path, 'wb').write(pickle.dumps(cache).encode('zip'))
    shutil.rmtree(target_dir)
    return cache

def fetch_20newsgroups(data_home=None, subset='train', categories=None,
                      shuffle=True, random_state=42, download_if_missing=True):
    """Load the filenames of the 20 newsgroups dataset

    Parameters
    ----------
    subset: 'train' or 'test', 'all', optional
        Select the dataset to load: 'train' for the training set, 'test'
        for the test set, 'all' for both, with shuffled ordering.

    data_home: optional, default: None
        Specify an download and cache folder for the datasets. If None,
        all scikit-learn data is stored in '~/scikit_learn_data' subfolders.

    categories: None or collection of string or unicode
        If None (default), load all the categories.
        If not None, list of category names to load (other categories
        ignored).

    shuffle: bool, optional
        Whether or not to shuffle the data: might be important for models that
        make the assumption that the samples are independent and identically
        distributed (i.i.d.), such as stochastic gradient descent.

    random_state: numpy random number generator or seed integer
        Used to shuffle the dataset.

    download_if_missing: optional, True by default
        If False, raise an IOError if the data is not locally available
        instead of trying to download the data from the source site.
    """

    data_home = get_data_home(data_home=data_home)
    cache_path = os.path.join(data_home, CACHE_NAME)
    twenty_home = os.path.join(data_home, "20news_home")
    cache = None
    if os.path.exists(cache_path):
        try:
            cache = pickle.loads(open(cache_path, 'rb').read().decode('zip'))
        except Exception, e:
            print 80*'_'
            print 'Cache loading failed'
            print 80*'_'
            print e

    if cache is None:
        if download_if_missing:
            cache = download_20newsgroups(target_dir=twenty_home,
                                          cache_path=cache_path)
        else:
            raise IOError('20Newsgroups dataset not found')

    if subset in ('train', 'test'):
        data = cache[subset]
    elif subset == 'all':
        data_lst = list()
        target = list()
        for subset in ('train', 'test'):
            data = cache[subset]
            data_lst.extend(data.data)
            target.extend(data.target)

        data.data = data_lst
        data.target = np.array(target)
        data.description = 'the 20 newsgroups by date dataset'
    else:
        raise ValueError(
            "subset can only be 'train', 'test' or 'all', got '%s'" % subset)

    if categories is not None:
        labels = [(data.target_names.index(cat), cat) for cat in categories]
        # Sort the categories to have the ordering of the labels
        labels.sort()
        labels, categories = zip(*labels)
        mask = in1d(data.target, labels)
        data.target = data.target[mask]
        # searchsorted to have continuous labels
        data.target = np.searchsorted(labels, data.target)
        data.target_names = list(categories)
        # Use an object array to shuffle: avoids memory copy
        data_lst = np.array(data.data, dtype=object)
        data_lst = data_lst[mask]
        data.data = data_lst.tolist()

    if shuffle:
        random_state = check_random_state(random_state)
        indices = np.arange(data.target.shape[0])
        random_state.shuffle(indices)
        data.target = data.target[indices]
        # Use an object array to shuffle: avoids memory copy
        data_lst = np.array(data.data, dtype=object)
        data_lst = data_lst[indices]
        data.data = data_lst.tolist()

    return data

@deprecated("Use fetch_20newsgroups instead with download_if_missing=False")
def load_20newsgroups(download_if_missing=False, **kwargs):
    """Alias for fetch_20newsgroups(download_if_missing=False).

    See fetch_20newsgroups.__doc__ for documentation and parameter list.
    """
    return fetch_20newsgroups(download_if_missing=download_if_missing, **kwargs)
<commit_msg>Unused import<commit_after>"""Caching loader for the 20 newsgroups text classification dataset

The description of the dataset is available on the official website at:

    http://people.csail.mit.edu/jrennie/20Newsgroups/

Quoting the introduction:

    The 20 Newsgroups data set is a collection of approximately 20,000
    newsgroup documents, partitioned (nearly) evenly across 20 different
    newsgroups. To the best of my knowledge, it was originally collected
    by Ken Lang, probably for his Newsweeder: Learning to filter netnews
    paper, though he does not explicitly mention this collection. The 20
    newsgroups collection has become a popular data set for experiments
    in text applications of machine learning techniques, such as text
    classification and text clustering.

This dataset loader will download the recommended "by date" variant of the
dataset and which features a point in time split between the train and
test sets. The compressed dataset size is around 14 Mb compressed. Once
uncompressed the train set is 52 MB and the test set is 34 MB.

The data is downloaded, extracted and cached in the '~/scikit_learn_data'
folder. However contrary to other datasets in the scikit, the data is
not vectorized into numpy arrays but the dataset list the filenames of
the posts and there categories as target signal.

The lack of vector feature extraction is intentional: there is no single
best way to turn text into vectors. Depending on the task various
preprocessing and text transformation are useful or not (n-grams,
lowercasing, stemming, stop-words filtering, TF-IDF weighting...).

"""
# Copyright (c) 2011 Olivier Grisel <olivier.grisel@ensta.org>
# License: Simplified BSD

import os
import urllib
import logging
import tarfile
import pickle
import shutil

import numpy as np

from .base import get_data_home
from .base import load_files
from ..utils import check_random_state, deprecated
from ..utils.fixes import in1d

logger = logging.getLogger(__name__)

URL = ("http://people.csail.mit.edu/jrennie/"
            "20Newsgroups/20news-bydate.tar.gz")
ARCHIVE_NAME = "20news-bydate.tar.gz"
CACHE_NAME = "20news-bydate.pkz"
TRAIN_FOLDER = "20news-bydate-train"
TEST_FOLDER = "20news-bydate-test"

def download_20newsgroups(target_dir, cache_path):
    """ Download the 20Newsgroups data and convert is in a zipped pickle
        storage.
    """
    archive_path = os.path.join(target_dir, ARCHIVE_NAME)
    train_path = os.path.join(target_dir, TRAIN_FOLDER)
    test_path = os.path.join(target_dir, TEST_FOLDER)

    if not os.path.exists(target_dir):
        os.makedirs(target_dir)

    if not os.path.exists(archive_path):
        logger.warn("Downloading dataset from %s (14 MB)", URL)
        opener = urllib.urlopen(URL)
        open(archive_path, 'wb').write(opener.read())

    logger.info("Decompressing %s", archive_path)
    tarfile.open(archive_path, "r:gz").extractall(path=target_dir)
    os.remove(archive_path)

    # Store a zipped pickle
    cache = dict(
            train=load_files(train_path),
            test=load_files(test_path)
        )
    open(cache_path, 'wb').write(pickle.dumps(cache).encode('zip'))
    shutil.rmtree(target_dir)
    return cache

def fetch_20newsgroups(data_home=None, subset='train', categories=None,
                      shuffle=True, random_state=42, download_if_missing=True):
    """Load the filenames of the 20 newsgroups dataset

    Parameters
    ----------
    subset: 'train' or 'test', 'all', optional
        Select the dataset to load: 'train' for the training set, 'test'
        for the test set, 'all' for both, with shuffled ordering.

    data_home: optional, default: None
        Specify an download and cache folder for the datasets. If None,
        all scikit-learn data is stored in '~/scikit_learn_data' subfolders.

    categories: None or collection of string or unicode
        If None (default), load all the categories.
        If not None, list of category names to load (other categories
        ignored).

    shuffle: bool, optional
        Whether or not to shuffle the data: might be important for models that
        make the assumption that the samples are independent and identically
        distributed (i.i.d.), such as stochastic gradient descent.

    random_state: numpy random number generator or seed integer
        Used to shuffle the dataset.

    download_if_missing: optional, True by default
        If False, raise an IOError if the data is not locally available
        instead of trying to download the data from the source site.
    """

    data_home = get_data_home(data_home=data_home)
    cache_path = os.path.join(data_home, CACHE_NAME)
    twenty_home = os.path.join(data_home, "20news_home")
    cache = None
    if os.path.exists(cache_path):
        try:
            cache = pickle.loads(open(cache_path, 'rb').read().decode('zip'))
        except Exception, e:
            print 80*'_'
            print 'Cache loading failed'
            print 80*'_'
            print e

    if cache is None:
        if download_if_missing:
            cache = download_20newsgroups(target_dir=twenty_home,
                                          cache_path=cache_path)
        else:
            raise IOError('20Newsgroups dataset not found')

    if subset in ('train', 'test'):
        data = cache[subset]
    elif subset == 'all':
        data_lst = list()
        target = list()
        for subset in ('train', 'test'):
            data = cache[subset]
            data_lst.extend(data.data)
            target.extend(data.target)

        data.data = data_lst
        data.target = np.array(target)
        data.description = 'the 20 newsgroups by date dataset'
    else:
        raise ValueError(
            "subset can only be 'train', 'test' or 'all', got '%s'" % subset)

    if categories is not None:
        labels = [(data.target_names.index(cat), cat) for cat in categories]
        # Sort the categories to have the ordering of the labels
        labels.sort()
        labels, categories = zip(*labels)
        mask = in1d(data.target, labels)
        data.target = data.target[mask]
        # searchsorted to have continuous labels
        data.target = np.searchsorted(labels, data.target)
        data.target_names = list(categories)
        # Use an object array to shuffle: avoids memory copy
        data_lst = np.array(data.data, dtype=object)
        data_lst = data_lst[mask]
        data.data = data_lst.tolist()

    if shuffle:
        random_state = check_random_state(random_state)
        indices = np.arange(data.target.shape[0])
        random_state.shuffle(indices)
        data.target = data.target[indices]
        # Use an object array to shuffle: avoids memory copy
        data_lst = np.array(data.data, dtype=object)
        data_lst = data_lst[indices]
        data.data = data_lst.tolist()

    return data

@deprecated("Use fetch_20newsgroups instead with download_if_missing=False")
def load_20newsgroups(download_if_missing=False, **kwargs):
    """Alias for fetch_20newsgroups(download_if_missing=False).

    See fetch_20newsgroups.__doc__ for documentation and parameter list.
    """
    return fetch_20newsgroups(download_if_missing=download_if_missing, **kwargs)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
====================================================================
Structure data atoms (:mod:`sknano.structure_io.atoms`)
====================================================================

.. currentmodule:: sknano.structure_io.atoms

Contents
========

.. autosummary::
   :toctree: generated/

   Atom
   Atoms
   LAMMPSAtom
   LAMMPSAtoms

"""
from __future__ import division, print_function, absolute_import

__docformat__ = 'restructuredtext'

from ._atom import *
from ._atoms import *
from ._lammps_atom import *
from ._lammps_atoms import *

__all__ = [s for s in dir() if not s.startswith('_')]
<commit_msg>adding note to warn user that the structure_io.atoms subpackage is not implemented in the structure generating code<commit_after># -*- coding: utf-8 -*-
"""
====================================================================
Structure data atoms (:mod:`sknano.structure_io.atoms`)
====================================================================

.. currentmodule:: sknano.structure_io.atoms

.. note::

   These classes have **NOT** yet been implemented in any of the
   :py:mod:`~sknano.structure_io` classes and are still in development.
   Currently, the :py:mod:`~sknano.structure_io` classes use the
   :py:class:`~sknano.chemistry.Atom` and :py:class:`~sknano.chemistry.Atoms`
   classes provided by the :py:mod:`~sknano.chemistry` module.

Contents
========

.. autosummary::
   :toctree: generated/

   Atom
   Atoms
   LAMMPSAtom
   LAMMPSAtoms

"""
from __future__ import division, print_function, absolute_import

__docformat__ = 'restructuredtext'

from ._atom import *
from ._atoms import *
from ._lammps_atom import *
from ._lammps_atoms import *

__all__ = [s for s in dir() if not s.startswith('_')]
<|endoftext|>


<commit_before>import textwrap
from ..api import Topos, load
from ..sgp4lib import EarthSatellite

def dedent(s):
    return textwrap.dedent(s.rstrip())

def eph():
    yield load('de421.bsp')

def test_jpl_segment(eph):
    e = eph['mercury barycenter']
    expected = dedent("""\
        Segment 'de421.bsp' 0 SOLAR SYSTEM BARYCENTER -> 1 MERCURY BARYCENTER
    """)
    assert str(e) == expected
    expected = dedent("""\
        <Segment 'de421.bsp' 0 SOLAR SYSTEM BARYCENTER -> 1 MERCURY BARYCENTER>
    """)
    assert repr(e) == expected

def test_satellite(eph):
    lines = [
        'ISS (ZARYA)             ',
        '1 25544U 98067A   13330.58127943  .00000814  00000-0  21834-4 0  1064',
        '2 25544  51.6484  23.7537 0001246  74.1647  18.7420 15.50540527859894',
    ]
    s = EarthSatellite(lines, None)
    expected = dedent("""\
        EarthSatellite 'ISS (ZARYA)' number=25544 epoch=2013-11-26T13:57:03Z
    """)
    assert str(s) == expected
    expected = dedent("""\
        <EarthSatellite 'ISS (ZARYA)' number=25544 epoch=2013-11-26T13:57:03Z>
    """)
    assert repr(s) == expected

def test_topos(eph):
    t = Topos(latitude_degrees=42.2, longitude_degrees=-88.1)
    expected = dedent("""\
        Topos 42deg 12' 00.0" N -88deg 06' 00.0" E
    """)
    assert str(t) == expected
    expected = dedent("""\
        <Topos 42deg 12' 00.0" N -88deg 06' 00.0" E>
    """)
    assert repr(t) == expected

def test_vector_sum(eph):
    e = eph['earth']
    expected = dedent("""\
        Sum of 2 vectors:
         + Segment 'de421.bsp' 0 SOLAR SYSTEM BARYCENTER -> 3 EARTH BARYCENTER
         + Segment 'de421.bsp' 3 EARTH BARYCENTER -> 399 EARTH
    """)
    assert str(e) == expected
    expected = dedent("""\
        <VectorSum of 2 vectors 0 SOLAR SYSTEM BARYCENTER -> 399 EARTH>
    """)
    assert repr(e) == expected
<commit_msg>Add test of str/repr for satellite without a name<commit_after>import textwrap
from ..api import Topos, load
from ..sgp4lib import EarthSatellite

def dedent(s):
    return textwrap.dedent(s.rstrip())

def eph():
    yield load('de421.bsp')

def test_jpl_segment(eph):
    e = eph['mercury barycenter']
    expected = dedent("""\
        Segment 'de421.bsp' 0 SOLAR SYSTEM BARYCENTER -> 1 MERCURY BARYCENTER
    """)
    assert str(e) == expected
    expected = dedent("""\
        <Segment 'de421.bsp' 0 SOLAR SYSTEM BARYCENTER -> 1 MERCURY BARYCENTER>
    """)
    assert repr(e) == expected

def test_satellite_with_name(eph):
    lines = [
        'ISS (ZARYA)             ',
        '1 25544U 98067A   13330.58127943  .00000814  00000-0  21834-4 0  1064',
        '2 25544  51.6484  23.7537 0001246  74.1647  18.7420 15.50540527859894',
    ]
    s = EarthSatellite(lines, None)
    expected = dedent("""\
        EarthSatellite 'ISS (ZARYA)' number=25544 epoch=2013-11-26T13:57:03Z
    """)
    assert str(s) == expected
    expected = dedent("""\
        <EarthSatellite 'ISS (ZARYA)' number=25544 epoch=2013-11-26T13:57:03Z>
    """)
    assert repr(s) == expected

def test_satellite_without_name(eph):
    lines = [
        '1 25544U 98067A   13330.58127943  .00000814  00000-0  21834-4 0  1064',
        '2 25544  51.6484  23.7537 0001246  74.1647  18.7420 15.50540527859894',
    ]
    s = EarthSatellite(lines, None)
    expected = dedent("""\
        EarthSatellite number=25544 epoch=2013-11-26T13:57:03Z
    """)
    assert str(s) == expected
    expected = dedent("""\
        <EarthSatellite number=25544 epoch=2013-11-26T13:57:03Z>
    """)
    assert repr(s) == expected

def test_topos(eph):
    t = Topos(latitude_degrees=42.2, longitude_degrees=-88.1)
    expected = dedent("""\
        Topos 42deg 12' 00.0" N -88deg 06' 00.0" E
    """)
    assert str(t) == expected
    expected = dedent("""\
        <Topos 42deg 12' 00.0" N -88deg 06' 00.0" E>
    """)
    assert repr(t) == expected

def test_vector_sum(eph):
    e = eph['earth']
    expected = dedent("""\
        Sum of 2 vectors:
         + Segment 'de421.bsp' 0 SOLAR SYSTEM BARYCENTER -> 3 EARTH BARYCENTER
         + Segment 'de421.bsp' 3 EARTH BARYCENTER -> 399 EARTH
    """)
    assert str(e) == expected
    expected = dedent("""\
        <VectorSum of 2 vectors 0 SOLAR SYSTEM BARYCENTER -> 399 EARTH>
    """)
    assert repr(e) == expected
<|endoftext|>


<commit_before># source: http://stackoverflow.com/questions/2758159/how-to-embed-a-python-interpreter-in-a-pyqt-widget

import sys, os, re
import traceback
from PyQt4 import QtCore
from PyQt4 import QtGui
from electrum import util

class Console(QtGui.QPlainTextEdit):
    def __init__(self, prompt='>> ', startup_message='', parent=None):
        QtGui.QPlainTextEdit.__init__(self, parent)

        self.prompt = prompt
        self.history = []
        self.namespace = {}
        self.construct = []

        self.setGeometry(50, 75, 600, 400)
        self.setWordWrapMode(QtGui.QTextOption.WrapAnywhere)
        self.setUndoRedoEnabled(False)
        self.document().setDefaultFont(QtGui.QFont("monospace", 10, QtGui.QFont.Normal))
        self.showMessage(startup_message)

        self.updateNamespace({'run':self.run_script})
        self.set_json(False)

    def set_json(self, b):
        self.is_json = b

    def run_script(self, filename):
        with open(filename) as f:
            script = f.read()
        result = eval(script, self.namespace, self.namespace)

    def updateNamespace(self, namespace):
        self.namespace.update(namespace)

    def showMessage(self, message):
        self.appendPlainText(message)
        self.newPrompt()

    def clear(self):
        self.setPlainText('')
        self.newPrompt()

    def newPrompt(self):
        if self.construct:
            prompt = '.' * len(self.prompt)
        else:
            prompt = self.prompt

        self.completions_pos = self.textCursor().position()
        self.completions_visible = False

        self.appendPlainText(prompt)
        self.moveCursor(QtGui.QTextCursor.End)

    def getCommand(self):
        doc = self.document()
        curr_line = unicode(doc.findBlockByLineNumber(doc.lineCount() - 1).text())
        curr_line = curr_line.rstrip()
        curr_line = curr_line[len(self.prompt):]
        return curr_line

    def setCommand(self, command):
        if self.getCommand() == command:
            return
        self.moveCursor(QtGui.QTextCursor.End)
        self.moveCursor(QtGui.QTextCursor.StartOfLine, QtGui.QTextCursor.KeepAnchor)
        for i in range(len(self.prompt)):
            self.moveCursor(QtGui.QTextCursor.Right, QtGui.QTextCursor.KeepAnchor)
        self.textCursor().removeSelectedText()
        self.textCursor().insertText(command)
        self.moveCursor(QtGui.QTextCursor.End)

    def show_completions(self, completions):
        if self.completions_visible:
            self.hide_completions()

        c = self.textCursor()
        c.setPosition(self.completions_pos)

        completions = map(lambda x: x.split('.')[-1], completions)
        t = '\n' + ' '.join(completions)
        if len(t) > 500:
            t = t[:500] + '...'
        c.insertText(t)
        self.completions_end = c.position()

        self.moveCursor(QtGui.QTextCursor.End)
        self.completions_visible = True

    def hide_completions(self):
        if not self.completions_visible:
            return
        c = self.textCursor()
        c.setPosition(self.completions_pos)
        l = self.completions_end - self.completions_pos
        for x in range(l): c.deleteChar()

        self.moveCursor(QtGui.QTextCursor.End)
        self.completions_visible = False

    def getConstruct(self, command):
        if self.construct:
            prev_command = self.construct[-1]
            self.construct.append(command)
            if not prev_command and not command:
                ret_val = '\n'.join(self.construct)
                self.construct = []
                return ret_val
            else:
                return ''
        else:
            if command and command[-1] == (':'):
                self.construct.append(command)
                return ''
            else:
                return command

    def getHistory(self):
        return self.history

    def setHisory(self, history):
        self.history = history

    def addToHistory(self, command):
        if command and (not self.history or self.history[-1] != command):
            self.history.append(command)
        self.history_index = len(self.history)

    def getPrevHistoryEntry(self):
        if self.history:
            self.history_index = max(0, self.history_index - 1)
            return self.history[self.history_index]
        return ''

    def getNextHistoryEntry(self):
        if self.history:
            hist_len = len(self.history)
            self.history_index = min(hist_len, self.history_index + 1)
            if self.history_index < hist_len:
                return self.history[self.history_index]
        return ''

    def getCursorPosition(self):
        return self.textCursor().columnNumber() - len(self.prompt)

    def setCursorPosition(self, position):
        self.moveCursor(QtGui.QTextCursor.StartOfLine)
        for i in range(len(self.prompt) + position):
            self.moveCursor(QtGui.QTextCursor.Right)

    def register_command(self, c, func):
        methods = { c: func}
        self.updateNamespace(methods)

    def runCommand(self):
        command = self.getCommand()
        self.addToHistory(command)

        command = self.getConstruct(command)

        if command:
            tmp_stdout = sys.stdout

            class stdoutProxy():
                def __init__(self, write_func):
                    self.write_func = write_func
                    self.skip = False

                def flush(self):
                    pass

                def write(self, text):
                    if not self.skip:
                        stripped_text = text.rstrip('\n')
                        self.write_func(stripped_text)
                        QtCore.QCoreApplication.processEvents()
                    self.skip = not self.skip

            if command == 'help()':
                self.appendPlainText("no help here!")
                self.newPrompt()
                return

            sys.stdout = stdoutProxy(self.appendPlainText)
            try:
                try:
                    result = eval(command, self.namespace, self.namespace)
                    if result != None:
                        if self.is_json:
                            util.print_json(result)
                        else:
                            self.appendPlainText(repr(result))
                except SyntaxError:
                    exec command in self.namespace
            except SystemExit:
                self.close()
            except:
                traceback_lines = traceback.format_exc().split('\n')
                # Remove traceback mentioning this file, and a linebreak
                for i in (3,2,1,-1):
                    traceback_lines.pop(i)
                self.appendPlainText('\n'.join(traceback_lines))
            sys.stdout = tmp_stdout
        self.newPrompt()
        self.set_json(False)

    def keyPressEvent(self, event):
        if event.key() == QtCore.Qt.Key_Tab:
            self.completions()
            return

        self.hide_completions()

        if event.key() in (QtCore.Qt.Key_Enter, QtCore.Qt.Key_Return):
            self.runCommand()
            return
        if event.key() == QtCore.Qt.Key_Home:
            self.setCursorPosition(0)
            return
        if event.key() == QtCore.Qt.Key_PageUp:
            return
        elif event.key() in (QtCore.Qt.Key_Left, QtCore.Qt.Key_Backspace):
            if self.getCursorPosition() == 0:
                return
        elif event.key() == QtCore.Qt.Key_Up:
            self.setCommand(self.getPrevHistoryEntry())
            return
        elif event.key() == QtCore.Qt.Key_Down:
            self.setCommand(self.getNextHistoryEntry())
            return
        elif event.key() == QtCore.Qt.Key_L and event.modifiers() == QtCore.Qt.ControlModifier:
            self.clear()

        super(Console, self).keyPressEvent(event)

    def completions(self):
        cmd = self.getCommand()
        lastword = re.split(' |\(|\)',cmd)[-1]
        beginning = cmd[0:-len(lastword)]

        path = lastword.split('.')
        ns = self.namespace.keys()

        if len(path) == 1:
            ns = ns
            prefix = ''
        else:
            obj = self.namespace.get(path[0])
            prefix = path[0] + '.'
            ns = dir(obj)

        completions = []
        for x in ns:
            if x[0] == '_':continue
            xx = prefix + x
            if xx.startswith(lastword):
                completions.append(xx)
        completions.sort()

        if not completions:
            self.hide_completions()
        elif len(completions) == 1:
            self.hide_completions()
            self.setCommand(beginning + completions[0])
        else:
            # find common prefix
            p = os.path.commonprefix(completions)
            if len(p)>len(lastword):
                self.hide_completions()
                self.setCommand(beginning + p)
            else:
                self.show_completions(completions)

welcome_message = '''
   ---------------------------------------------------------------
     Welcome to a primitive Python interpreter.
   ---------------------------------------------------------------
'''

if __name__ == '__main__':
    app = QtGui.QApplication(sys.argv)
    console = Console(startup_message=welcome_message)
    console.updateNamespace({'myVar1' : app, 'myVar2' : 1234})
    console.show();
    sys.exit(app.exec_())
<commit_msg>fix history in console<commit_after># source: http://stackoverflow.com/questions/2758159/how-to-embed-a-python-interpreter-in-a-pyqt-widget

import sys, os, re
import traceback
from PyQt4 import QtCore
from PyQt4 import QtGui
from electrum import util

class Console(QtGui.QPlainTextEdit):
    def __init__(self, prompt='>> ', startup_message='', parent=None):
        QtGui.QPlainTextEdit.__init__(self, parent)

        self.prompt = prompt
        self.history = []
        self.namespace = {}
        self.construct = []

        self.setGeometry(50, 75, 600, 400)
        self.setWordWrapMode(QtGui.QTextOption.WrapAnywhere)
        self.setUndoRedoEnabled(False)
        self.document().setDefaultFont(QtGui.QFont("monospace", 10, QtGui.QFont.Normal))
        self.showMessage(startup_message)

        self.updateNamespace({'run':self.run_script})
        self.set_json(False)

    def set_json(self, b):
        self.is_json = b

    def run_script(self, filename):
        with open(filename) as f:
            script = f.read()
        result = eval(script, self.namespace, self.namespace)

    def updateNamespace(self, namespace):
        self.namespace.update(namespace)

    def showMessage(self, message):
        self.appendPlainText(message)
        self.newPrompt()

    def clear(self):
        self.setPlainText('')
        self.newPrompt()

    def newPrompt(self):
        if self.construct:
            prompt = '.' * len(self.prompt)
        else:
            prompt = self.prompt

        self.completions_pos = self.textCursor().position()
        self.completions_visible = False

        self.appendPlainText(prompt)
        self.moveCursor(QtGui.QTextCursor.End)

    def getCommand(self):
        doc = self.document()
        curr_line = unicode(doc.findBlockByLineNumber(doc.lineCount() - 1).text())
        curr_line = curr_line.rstrip()
        curr_line = curr_line[len(self.prompt):]
        return curr_line

    def setCommand(self, command):
        if self.getCommand() == command:
            return

        doc = self.document()
        curr_line = unicode(doc.findBlockByLineNumber(doc.lineCount() - 1).text())
        self.moveCursor(QtGui.QTextCursor.End)
        for i in range(len(curr_line) - len(self.prompt)):
            self.moveCursor(QtGui.QTextCursor.Left, QtGui.QTextCursor.KeepAnchor)

        self.textCursor().removeSelectedText()
        self.textCursor().insertText(command)
        self.moveCursor(QtGui.QTextCursor.End)

    def show_completions(self, completions):
        if self.completions_visible:
            self.hide_completions()

        c = self.textCursor()
        c.setPosition(self.completions_pos)

        completions = map(lambda x: x.split('.')[-1], completions)
        t = '\n' + ' '.join(completions)
        if len(t) > 500:
            t = t[:500] + '...'
        c.insertText(t)
        self.completions_end = c.position()

        self.moveCursor(QtGui.QTextCursor.End)
        self.completions_visible = True

    def hide_completions(self):
        if not self.completions_visible:
            return
        c = self.textCursor()
        c.setPosition(self.completions_pos)
        l = self.completions_end - self.completions_pos
        for x in range(l): c.deleteChar()

        self.moveCursor(QtGui.QTextCursor.End)
        self.completions_visible = False

    def getConstruct(self, command):
        if self.construct:
            prev_command = self.construct[-1]
            self.construct.append(command)
            if not prev_command and not command:
                ret_val = '\n'.join(self.construct)
                self.construct = []
                return ret_val
            else:
                return ''
        else:
            if command and command[-1] == (':'):
                self.construct.append(command)
                return ''
            else:
                return command

    def getHistory(self):
        return self.history

    def setHisory(self, history):
        self.history = history

    def addToHistory(self, command):
        if command and (not self.history or self.history[-1] != command):
            self.history.append(command)
        self.history_index = len(self.history)

    def getPrevHistoryEntry(self):
        if self.history:
            self.history_index = max(0, self.history_index - 1)
            return self.history[self.history_index]
        return ''

    def getNextHistoryEntry(self):
        if self.history:
            hist_len = len(self.history)
            self.history_index = min(hist_len, self.history_index + 1)
            if self.history_index < hist_len:
                return self.history[self.history_index]
        return ''

    def getCursorPosition(self):
        c = self.textCursor()
        return c.position() - c.block().position() - len(self.prompt)

    def setCursorPosition(self, position):
        self.moveCursor(QtGui.QTextCursor.StartOfLine)
        for i in range(len(self.prompt) + position):
            self.moveCursor(QtGui.QTextCursor.Right)

    def register_command(self, c, func):
        methods = { c: func}
        self.updateNamespace(methods)

    def runCommand(self):
        command = self.getCommand()
        self.addToHistory(command)

        command = self.getConstruct(command)

        if command:
            tmp_stdout = sys.stdout

            class stdoutProxy():
                def __init__(self, write_func):
                    self.write_func = write_func
                    self.skip = False

                def flush(self):
                    pass

                def write(self, text):
                    if not self.skip:
                        stripped_text = text.rstrip('\n')
                        self.write_func(stripped_text)
                        QtCore.QCoreApplication.processEvents()
                    self.skip = not self.skip

            if command == 'help()':
                self.appendPlainText("no help here!")
                self.newPrompt()
                return

            sys.stdout = stdoutProxy(self.appendPlainText)
            try:
                try:
                    result = eval(command, self.namespace, self.namespace)
                    if result != None:
                        if self.is_json:
                            util.print_json(result)
                        else:
                            self.appendPlainText(repr(result))
                except SyntaxError:
                    exec command in self.namespace
            except SystemExit:
                self.close()
            except:
                traceback_lines = traceback.format_exc().split('\n')
                # Remove traceback mentioning this file, and a linebreak
                for i in (3,2,1,-1):
                    traceback_lines.pop(i)
                self.appendPlainText('\n'.join(traceback_lines))
            sys.stdout = tmp_stdout
        self.newPrompt()
        self.set_json(False)

    def keyPressEvent(self, event):
        if event.key() == QtCore.Qt.Key_Tab:
            self.completions()
            return

        self.hide_completions()

        if event.key() in (QtCore.Qt.Key_Enter, QtCore.Qt.Key_Return):
            self.runCommand()
            return
        if event.key() == QtCore.Qt.Key_Home:
            self.setCursorPosition(0)
            return
        if event.key() == QtCore.Qt.Key_PageUp:
            return
        elif event.key() in (QtCore.Qt.Key_Left, QtCore.Qt.Key_Backspace):
            if self.getCursorPosition() == 0:
                return
        elif event.key() == QtCore.Qt.Key_Up:
            self.setCommand(self.getPrevHistoryEntry())
            return
        elif event.key() == QtCore.Qt.Key_Down:
            self.setCommand(self.getNextHistoryEntry())
            return
        elif event.key() == QtCore.Qt.Key_L and event.modifiers() == QtCore.Qt.ControlModifier:
            self.clear()

        super(Console, self).keyPressEvent(event)

    def completions(self):
        cmd = self.getCommand()
        lastword = re.split(' |\(|\)',cmd)[-1]
        beginning = cmd[0:-len(lastword)]

        path = lastword.split('.')
        ns = self.namespace.keys()

        if len(path) == 1:
            ns = ns
            prefix = ''
        else:
            obj = self.namespace.get(path[0])
            prefix = path[0] + '.'
            ns = dir(obj)

        completions = []
        for x in ns:
            if x[0] == '_':continue
            xx = prefix + x
            if xx.startswith(lastword):
                completions.append(xx)
        completions.sort()

        if not completions:
            self.hide_completions()
        elif len(completions) == 1:
            self.hide_completions()
            self.setCommand(beginning + completions[0])
        else:
            # find common prefix
            p = os.path.commonprefix(completions)
            if len(p)>len(lastword):
                self.hide_completions()
                self.setCommand(beginning + p)
            else:
                self.show_completions(completions)

welcome_message = '''
   ---------------------------------------------------------------
     Welcome to a primitive Python interpreter.
   ---------------------------------------------------------------
'''

if __name__ == '__main__':
    app = QtGui.QApplication(sys.argv)
    console = Console(startup_message=welcome_message)
    console.updateNamespace({'myVar1' : app, 'myVar2' : 1234})
    console.show();
    sys.exit(app.exec_())
<|endoftext|>


<commit_before>from collections import OrderedDict

import base64
import hashlib
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
from mdstudio.db.database import IDatabase

from db.key_repository import KeyRepository
from mdstudio.api.endpoint import endpoint
from mdstudio.component.impl.core import CoreComponentSession
from mdstudio.db.connection_type import ConnectionType
from mdstudio.db.fields import Fields
from mdstudio.db.impl.mongo_client_wrapper import MongoClientWrapper
from mdstudio.db.index import Index
from mdstudio.deferred.chainable import chainable
from mdstudio.deferred.lock import Lock
from mdstudio.deferred.return_value import return_value

class DBComponent(CoreComponentSession):
    """
    Database management WAMP methods.
    """

    def pre_init(self):

        self.load_environment(OrderedDict([
            ('host', (['MD_MONGO_HOST'], None)),
            ('port', (['MD_MONGO_PORT'], None, int)),
            ('secret', (['MD_MONGO_SECRET'], None))
        ]))

        self.component_waiters.append(self.ComponentWaiter(self, 'schema'))

        self._client = MongoClientWrapper(self.component_config.settings['host'], self.component_config.settings['port'])

        super(DBComponent, self).pre_init()

    def on_init(self):

        assert self.component_config.settings['secret'], 'The database must have a secret set!\n' \
                                                         'Please modify your configuration or set "MD_MONGO_SECRET"'
        assert len(self.component_config.settings['secret']) >= 20, 'The database secret must be at least 20 characters long! ' \
                                                                    'Please make sure it is larger than it is now.'
        self._set_secret()

        self.database_lock = Lock()

    @property
    def secret(self):
        return self._secret

    @chainable
    def _on_join(self):
        yield self.call('mdstudio.auth.endpoint.ring0.set-status', {'status': True})

        yield super(DBComponent, self)._on_join()

    @endpoint('more', 'cursor/more-request/v1', 'cursor/more-response/v1', scope='write')
    def more(self, request, claims=None):
        database = self.get_database(claims)

        return database.more(request['cursorId'], claims=claims)

    @endpoint('rewind',
              'cursor/rewind-request',
              'cursor/rewind-response', scope='write')
    def rewind(self, request, claims=None):
        database = self.get_database(claims)

        return database.rewind(request['cursorId'], claims=claims)

    @endpoint('insert_one',
              'insert/insert-one-request',
              'insert/insert-one-response',
              scope='write')
    def insert_one(self, request, claims=None):
        database = self.get_database(claims)
        kwargs = {}

        self.set_fields(claims, kwargs, request)

        return database.insert_one(request['collection'], request['insert'], **kwargs)

    @endpoint('insert_many',
              'insert/insert-many-request',
              'insert/insert-many-response',
              scope='write')
    def insert_many(self, request, claims=None):
        database = self.get_database(claims)
        kwargs = {}

        self.set_fields(claims, kwargs, request)

        return database.insert_many(request['collection'], request['insert'], **kwargs)

    @endpoint('replace_one',
              'replace/replace-one-request',
              'replace/replace-one-response',
              scope='write')
    def replace_one(self, request, claims=None):
        database = self.get_database(claims)
        kwargs = {}
        if 'upsert' in request:
            kwargs['upsert'] = request['upsert']

        self.set_fields(claims, kwargs, request)

        return database.replace_one(request['collection'], request['filter'],
                                    request['replacement'], **kwargs)

    @endpoint('count',
              'count/count-request',
              'count/count-response', scope='read')
    def count(self, request, claims=None):
        database = self.get_database(claims)

        if 'cursorId' in request:
            kwargs = {
                'cursor_id': request['cursorId'],
                'with_limit_and_skip': request['withLimitAndSkip']
            }

        else:
            kwargs = {
                'collection': request['collection'],
                'filter': request.get('filter', {})
            }
            if 'skip' in request:
                kwargs['skip'] = request['skip']
            if 'limit' in request:
                kwargs['limit'] = request['limit']
            if 'fields' in request:
                kwargs['fields'] = Fields.from_dict(request['fields'])

        return database.count(**kwargs)

    @endpoint('update_one',
              'update/update-one-request',
              'update/update-one-response',
              scope='write')
    def update_one(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'upsert' in request:
            kwargs['upsert'] = request['upsert']

        self.set_fields(claims, kwargs, request)

        return database.update_one(request['collection'], request['filter'],
                                   request['update'], **kwargs)

    @endpoint('update_many',
              'update/update-many-request',
              'update/update-many-response',
              scope='write')
    def update_many(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'upsert' in request:
            kwargs['upsert'] = request['upsert']

        self.set_fields(claims, kwargs, request)

        return database.update_many(request['collection'], request['filter'],
                                    request['update'], **kwargs)

    @endpoint('find_one',
              'find/find-one-request',
              'find/find-one-response', scope='read')
    def find_one(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'projection' in request:
            kwargs['projection'] = request['projection']
        if 'skip' in request:
            kwargs['skip'] = request['skip']
        if 'sort' in request:
            kwargs['sort'] = request['sort']

        self.set_fields(claims, kwargs, request)

        return database.find_one(request['collection'], request['filter'], **kwargs)

    @endpoint('find_many',
              'find/find-many-request',
              'find/find-many-response', scope='read')
    def find_many(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'projection' in request:
            kwargs['projection'] = request['projection']
        if 'skip' in request:
            kwargs['skip'] = request['skip']
        if 'limit' in request:
            kwargs['limit'] = request['limit']
        if 'sort' in request:
            kwargs['sort'] = request['sort']

        self.set_fields(claims, kwargs, request)

        return database.find_many(request['collection'], request['filter'], **kwargs)

    @endpoint('find_one_and_update',
              'find/find-one-and-update-request',
              'find/find-one-and-update-response',
              scope='read')
    def find_one_and_update(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'upsert' in request:
            kwargs['upsert'] = request['upsert']
        if 'projection' in request:
            kwargs['projection'] = request['projection']
        if 'sort' in request:
            kwargs['sort'] = request['sort']
        if 'returnUpdated' in request:
            kwargs['return_updated'] = request['returnUpdated']

        self.set_fields(claims, kwargs, request)

        return database.find_one_and_update(request['collection'], request['filter'],
                                            request['update'], **kwargs)

    @endpoint('find_one_and_replace',
              'find/find-one-and-replace-request',
              'find/find-one-and-replace-response',
              scope='read')
    def find_one_and_replace(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'upsert' in request:
            kwargs['upsert'] = request['upsert']
        if 'projection' in request:
            kwargs['projection'] = request['projection']
        if 'sort' in request:
            kwargs['sort'] = request['sort']
        if 'returnUpdated' in request:
            kwargs['return_updated'] = request['returnUpdated']

        self.set_fields(claims, kwargs, request)

        return database.find_one_and_replace(request['collection'], request['filter'],
                                             request['replacement'], **kwargs)

    @endpoint('find_one_and_delete',
              'find/find-one-and-delete-request',
              'find/find-one-and-delete-response',
              scope='read')
    def find_one_and_delete(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'projection' in request:
            kwargs['projection'] = request['projection']
        if 'sort' in request:
            kwargs['sort'] = request['sort']

        self.set_fields(claims, kwargs, request)

        return database.find_one_and_delete(request['collection'], request['filter'], **kwargs)

    @endpoint('distinct',
              'distinct/distinct-request',
              'distinct/distinct-response',
              scope='read')
    def distinct(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'filter' in request:
            kwargs['filter'] = request['filter']

        self.set_fields(claims, kwargs, request)

        return database.distinct(request['collection'], request['field'], **kwargs)

    @endpoint('aggregate',
              'aggregate/aggregate-request',
              'aggregate/aggregate-response',
              scope='read')
    def aggregate(self, request, claims=None):
        database = self.get_database(claims)
        return database.aggregate(request['collection'], request['pipeline'])

    @endpoint('delete_one',
              'delete/delete-one',
              'delete/delete-response',
              scope='delete')
    def delete_one(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}

        self.set_fields(claims, kwargs, request)

        return database.delete_one(request['collection'], request['filter'], **kwargs)

    @endpoint('delete_many',
              'delete/delete-many',
              'delete/delete-response',
              scope='delete')
    def delete_many(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}

        self.set_fields(claims, kwargs, request)

        return database.delete_many(request['collection'], request['filter'], **kwargs)

    @endpoint('create_indexes',
              'index/create-request/v1',
              'index/create-response/v1',
              scope='index')
    def create_indexes(self, request, claims=None):
        database = self.get_database(claims)

        return database.create_indexes(request['collection'], [Index.from_dict(d) for d in request['indexes']])

    @endpoint('drop-indexes',
              'index/drop-request/v1',
              scope='index')
    def drop_indexes(self, request, claims=None):
        database = self.get_database(claims)

        return database.drop_indexes(request['collection'], [Index.from_dict(d) for d in request['indexes']])

    @endpoint('drop-all-indexes',
              'index/drop-all-request/v1',
              scope='index')
    def drop_all_indexes(self, request, claims=None):
        database = self.get_database(claims)

        return database.drop_all_indexes(request['collection'])

    @chainable
    def get_database(self, claims):
        # type: (dict) -> IDatabase
        connection_type = ConnectionType.from_string(claims['connectionType'])

        if connection_type == ConnectionType.User:
            database_name = 'users~{user}'.format(user=claims['username'])

            assert database_name.count('~') <= 1, 'Someone tried to spoof the key database!'
        elif connection_type == ConnectionType.Group:
            database_name = 'groups~{group}'.format(group=claims['group'])

            assert database_name.count('~') <= 1, 'Someone tried to spoof the key database!'
        elif connection_type == ConnectionType.GroupRole:
            database_name = 'grouproles~{group}~{group_role}'.format(group=claims['group'], group_role=claims['role'])

            assert database_name.count('~') <= 2, 'Someone tried to spoof the key database!'
        else:
            raise NotImplemented('This distinction does not exist')

        result = None
        if database_name:
            assert database_name.strip() != 'users~db', 'Someone tried to spoof the key database!'

            yield self.database_lock.acquire()
            result = self._client.get_database(database_name)
            yield self.database_lock.release()

        return_value(result)

    def set_fields(self, claims, kwargs, request):
        if 'fields' in request:
            repo = KeyRepository(self, self._client.get_database('users~db'))
            kwargs['fields'] = Fields.from_dict(request['fields'], repo)
            if kwargs['fields'].uses_encryption:
                kwargs['claims'] = claims

    def authorize_request(self, uri, claims):
        connection_type = ConnectionType.from_string(claims['connectionType'])

        # @todo: solve this using jsonschema
        # @todo: authorize cursor
        if connection_type == ConnectionType.User:
            return ('username' in claims) == True
        elif connection_type == ConnectionType.Group:
            return ('group' in claims) == True
        elif connection_type == ConnectionType.GroupRole:
            return all(key in claims for key in ['group', 'role'])

        return False

    def _set_secret(self):
        secret = str.encode(self.component_config.settings['secret'])
        kdf = PBKDF2HMAC(
            algorithm=hashes.SHA256(),
            length=32,
            salt=hashlib.sha512(secret).digest(),
            iterations=150000,
            backend=default_backend()
        )
        self._secret = base64.urlsafe_b64encode(kdf.derive(secret))
<commit_msg>Use correct defaults<commit_after>from collections import OrderedDict

import base64
import hashlib
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
from mdstudio.db.database import IDatabase

from db.key_repository import KeyRepository
from mdstudio.api.endpoint import endpoint
from mdstudio.component.impl.core import CoreComponentSession
from mdstudio.db.connection_type import ConnectionType
from mdstudio.db.fields import Fields
from mdstudio.db.impl.mongo_client_wrapper import MongoClientWrapper
from mdstudio.db.index import Index
from mdstudio.deferred.chainable import chainable
from mdstudio.deferred.lock import Lock
from mdstudio.deferred.return_value import return_value

class DBComponent(CoreComponentSession):
    """
    Database management WAMP methods.
    """

    def pre_init(self):

        self.load_environment(OrderedDict([
            ('host', (['MD_MONGO_HOST'], 'localhost')),
            ('port', (['MD_MONGO_PORT'], 217127, int)),
            ('secret', (['MD_MONGO_SECRET'], ''))
        ]))

        self.component_waiters.append(self.ComponentWaiter(self, 'schema'))

        self._client = MongoClientWrapper(self.component_config.settings['host'], self.component_config.settings['port'])

        super(DBComponent, self).pre_init()

    def on_init(self):

        assert self.component_config.settings['secret'], 'The database must have a secret set!\n' \
                                                         'Please modify your configuration or set "MD_MONGO_SECRET"'
        assert len(self.component_config.settings['secret']) >= 20, 'The database secret must be at least 20 characters long! ' \
                                                                    'Please make sure it is larger than it is now.'
        self._set_secret()

        self.database_lock = Lock()

    @property
    def secret(self):
        return self._secret

    @chainable
    def _on_join(self):
        yield self.call('mdstudio.auth.endpoint.ring0.set-status', {'status': True})

        yield super(DBComponent, self)._on_join()

    @endpoint('more', 'cursor/more-request/v1', 'cursor/more-response/v1', scope='write')
    def more(self, request, claims=None):
        database = self.get_database(claims)

        return database.more(request['cursorId'], claims=claims)

    @endpoint('rewind',
              'cursor/rewind-request',
              'cursor/rewind-response', scope='write')
    def rewind(self, request, claims=None):
        database = self.get_database(claims)

        return database.rewind(request['cursorId'], claims=claims)

    @endpoint('insert_one',
              'insert/insert-one-request',
              'insert/insert-one-response',
              scope='write')
    def insert_one(self, request, claims=None):
        database = self.get_database(claims)
        kwargs = {}

        self.set_fields(claims, kwargs, request)

        return database.insert_one(request['collection'], request['insert'], **kwargs)

    @endpoint('insert_many',
              'insert/insert-many-request',
              'insert/insert-many-response',
              scope='write')
    def insert_many(self, request, claims=None):
        database = self.get_database(claims)
        kwargs = {}

        self.set_fields(claims, kwargs, request)

        return database.insert_many(request['collection'], request['insert'], **kwargs)

    @endpoint('replace_one',
              'replace/replace-one-request',
              'replace/replace-one-response',
              scope='write')
    def replace_one(self, request, claims=None):
        database = self.get_database(claims)
        kwargs = {}
        if 'upsert' in request:
            kwargs['upsert'] = request['upsert']

        self.set_fields(claims, kwargs, request)

        return database.replace_one(request['collection'], request['filter'],
                                    request['replacement'], **kwargs)

    @endpoint('count',
              'count/count-request',
              'count/count-response', scope='read')
    def count(self, request, claims=None):
        database = self.get_database(claims)

        if 'cursorId' in request:
            kwargs = {
                'cursor_id': request['cursorId'],
                'with_limit_and_skip': request['withLimitAndSkip']
            }

        else:
            kwargs = {
                'collection': request['collection'],
                'filter': request.get('filter', {})
            }
            if 'skip' in request:
                kwargs['skip'] = request['skip']
            if 'limit' in request:
                kwargs['limit'] = request['limit']
            if 'fields' in request:
                kwargs['fields'] = Fields.from_dict(request['fields'])

        return database.count(**kwargs)

    @endpoint('update_one',
              'update/update-one-request',
              'update/update-one-response',
              scope='write')
    def update_one(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'upsert' in request:
            kwargs['upsert'] = request['upsert']

        self.set_fields(claims, kwargs, request)

        return database.update_one(request['collection'], request['filter'],
                                   request['update'], **kwargs)

    @endpoint('update_many',
              'update/update-many-request',
              'update/update-many-response',
              scope='write')
    def update_many(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'upsert' in request:
            kwargs['upsert'] = request['upsert']

        self.set_fields(claims, kwargs, request)

        return database.update_many(request['collection'], request['filter'],
                                    request['update'], **kwargs)

    @endpoint('find_one',
              'find/find-one-request',
              'find/find-one-response', scope='read')
    def find_one(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'projection' in request:
            kwargs['projection'] = request['projection']
        if 'skip' in request:
            kwargs['skip'] = request['skip']
        if 'sort' in request:
            kwargs['sort'] = request['sort']

        self.set_fields(claims, kwargs, request)

        return database.find_one(request['collection'], request['filter'], **kwargs)

    @endpoint('find_many',
              'find/find-many-request',
              'find/find-many-response', scope='read')
    def find_many(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'projection' in request:
            kwargs['projection'] = request['projection']
        if 'skip' in request:
            kwargs['skip'] = request['skip']
        if 'limit' in request:
            kwargs['limit'] = request['limit']
        if 'sort' in request:
            kwargs['sort'] = request['sort']

        self.set_fields(claims, kwargs, request)

        return database.find_many(request['collection'], request['filter'], **kwargs)

    @endpoint('find_one_and_update',
              'find/find-one-and-update-request',
              'find/find-one-and-update-response',
              scope='read')
    def find_one_and_update(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'upsert' in request:
            kwargs['upsert'] = request['upsert']
        if 'projection' in request:
            kwargs['projection'] = request['projection']
        if 'sort' in request:
            kwargs['sort'] = request['sort']
        if 'returnUpdated' in request:
            kwargs['return_updated'] = request['returnUpdated']

        self.set_fields(claims, kwargs, request)

        return database.find_one_and_update(request['collection'], request['filter'],
                                            request['update'], **kwargs)

    @endpoint('find_one_and_replace',
              'find/find-one-and-replace-request',
              'find/find-one-and-replace-response',
              scope='read')
    def find_one_and_replace(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'upsert' in request:
            kwargs['upsert'] = request['upsert']
        if 'projection' in request:
            kwargs['projection'] = request['projection']
        if 'sort' in request:
            kwargs['sort'] = request['sort']
        if 'returnUpdated' in request:
            kwargs['return_updated'] = request['returnUpdated']

        self.set_fields(claims, kwargs, request)

        return database.find_one_and_replace(request['collection'], request['filter'],
                                             request['replacement'], **kwargs)

    @endpoint('find_one_and_delete',
              'find/find-one-and-delete-request',
              'find/find-one-and-delete-response',
              scope='read')
    def find_one_and_delete(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'projection' in request:
            kwargs['projection'] = request['projection']
        if 'sort' in request:
            kwargs['sort'] = request['sort']

        self.set_fields(claims, kwargs, request)

        return database.find_one_and_delete(request['collection'], request['filter'], **kwargs)

    @endpoint('distinct',
              'distinct/distinct-request',
              'distinct/distinct-response',
              scope='read')
    def distinct(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}
        if 'filter' in request:
            kwargs['filter'] = request['filter']

        self.set_fields(claims, kwargs, request)

        return database.distinct(request['collection'], request['field'], **kwargs)

    @endpoint('aggregate',
              'aggregate/aggregate-request',
              'aggregate/aggregate-response',
              scope='read')
    def aggregate(self, request, claims=None):
        database = self.get_database(claims)
        return database.aggregate(request['collection'], request['pipeline'])

    @endpoint('delete_one',
              'delete/delete-one',
              'delete/delete-response',
              scope='delete')
    def delete_one(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}

        self.set_fields(claims, kwargs, request)

        return database.delete_one(request['collection'], request['filter'], **kwargs)

    @endpoint('delete_many',
              'delete/delete-many',
              'delete/delete-response',
              scope='delete')
    def delete_many(self, request, claims=None):
        database = self.get_database(claims)

        kwargs = {}

        self.set_fields(claims, kwargs, request)

        return database.delete_many(request['collection'], request['filter'], **kwargs)

    @endpoint('create_indexes',
              'index/create-request/v1',
              'index/create-response/v1',
              scope='index')
    def create_indexes(self, request, claims=None):
        database = self.get_database(claims)

        return database.create_indexes(request['collection'], [Index.from_dict(d) for d in request['indexes']])

    @endpoint('drop-indexes',
              'index/drop-request/v1',
              scope='index')
    def drop_indexes(self, request, claims=None):
        database = self.get_database(claims)

        return database.drop_indexes(request['collection'], [Index.from_dict(d) for d in request['indexes']])

    @endpoint('drop-all-indexes',
              'index/drop-all-request/v1',
              scope='index')
    def drop_all_indexes(self, request, claims=None):
        database = self.get_database(claims)

        return database.drop_all_indexes(request['collection'])

    @chainable
    def get_database(self, claims):
        # type: (dict) -> IDatabase
        connection_type = ConnectionType.from_string(claims['connectionType'])

        if connection_type == ConnectionType.User:
            database_name = 'users~{user}'.format(user=claims['username'])

            assert database_name.count('~') <= 1, 'Someone tried to spoof the key database!'
        elif connection_type == ConnectionType.Group:
            database_name = 'groups~{group}'.format(group=claims['group'])

            assert database_name.count('~') <= 1, 'Someone tried to spoof the key database!'
        elif connection_type == ConnectionType.GroupRole:
            database_name = 'grouproles~{group}~{group_role}'.format(group=claims['group'], group_role=claims['role'])

            assert database_name.count('~') <= 2, 'Someone tried to spoof the key database!'
        else:
            raise NotImplemented('This distinction does not exist')

        result = None
        if database_name:
            assert database_name.strip() != 'users~db', 'Someone tried to spoof the key database!'

            yield self.database_lock.acquire()
            result = self._client.get_database(database_name)
            yield self.database_lock.release()

        return_value(result)

    def set_fields(self, claims, kwargs, request):
        if 'fields' in request:
            repo = KeyRepository(self, self._client.get_database('users~db'))
            kwargs['fields'] = Fields.from_dict(request['fields'], repo)
            if kwargs['fields'].uses_encryption:
                kwargs['claims'] = claims

    def authorize_request(self, uri, claims):
        connection_type = ConnectionType.from_string(claims['connectionType'])

        # @todo: solve this using jsonschema
        # @todo: authorize cursor
        if connection_type == ConnectionType.User:
            return ('username' in claims) == True
        elif connection_type == ConnectionType.Group:
            return ('group' in claims) == True
        elif connection_type == ConnectionType.GroupRole:
            return all(key in claims for key in ['group', 'role'])

        return False

    def _set_secret(self):
        secret = str.encode(self.component_config.settings['secret'])
        kdf = PBKDF2HMAC(
            algorithm=hashes.SHA256(),
            length=32,
            salt=hashlib.sha512(secret).digest(),
            iterations=150000,
            backend=default_backend()
        )
        self._secret = base64.urlsafe_b64encode(kdf.derive(secret))
<|endoftext|>


<commit_before>from birdy.twitter import AppClient
import os, re

from totalimpact.providers import provider
from totalimpact.providers.provider import Provider, ProviderContentMalformedError

import logging
logger = logging.getLogger('ti.providers.twitter_account')

class Twitter_Account(Provider):

    example_id = ("url", "http://twitter.com/jasonpriem")

    url = "http://twitter.com"
    descr = "Social networking and microblogging service."
    member_items_url_template = "http://twitter.com/%s"
    provenance_url_template = "%s"
    biblio_url_template = "https://api.github.com/repos/%s?client_id=" + os.environ["GITHUB_CLIENT_ID"] + "&client_secret=" + os.environ["GITHUB_CLIENT_SECRET"]
    metrics_url_template = "https://api.github.com/repos/%s?client_id=" + os.environ["GITHUB_CLIENT_ID"] + "&client_secret=" + os.environ["GITHUB_CLIENT_SECRET"]

    static_meta_dict = {
        "followers": {
            "display_name": "followers",
            "provider": "Twitter",
            "provider_url": "http://twitter.com",
            "description": "The number of people following this Twitter account",
            "icon": "https://twitter.com/favicon.ico"
        },
        "lists": {
            "display_name": "lists",
            "provider": "Twitter",
            "provider_url": "http://twitter.com",
            "description": "The number of people who have included this Twitter account in a Twitter list",
            "icon": "https://twitter.com/favicon.ico"
            }
    }

    def __init__(self):
        super(Twitter_Account, self).__init__()
        self.client = AppClient(os.getenv("TWITTER_CONSUMER_KEY"),
                            os.getenv("TWITTER_CONSUMER_SECRET"),
                            os.getenv("TWITTER_ACCESS_TOKEN"))

    # overriding default because overriding member_items method
    @property
    def provides_members(self):
        return True

    @property
    def provides_biblio(self):
         return True

    @property
    def provides_metrics(self):
         return True

    def is_relevant_alias(self, alias):
        (namespace, nid) = alias
        try:
            nid = nid.lower()
        except AttributeError:
            return False
        if (namespace == "url"):
            if ("twitter.com" in nid) and ("/status/" not in nid):
                return True
        return False

    def screen_name(self, nid):
        #regex from http://stackoverflow.com/questions/4424179/how-to-validate-a-twitter-username-using-regex
        match = re.findall("twitter.com/([A-Za-z0-9_]{1,15}$)", nid)
        return match[0]

    # default method; providers can override
    def member_items(self,
            query_string,
            provider_url_template=None,
            cache_enabled=True):

        twitter_username = query_string.replace("@", "")
        url = self._get_templated_url(self.member_items_url_template, twitter_username, "members")
        members = [("url", url)]
        return(members)

    def get_account_data(self, aliases):
        nid = self.get_best_id(aliases)
        if not nid:
            return None

        screen_name = self.screen_name(nid)

        r = self.client.api.users.show.get(screen_name=screen_name)
        return r.data

    # default method; providers can override
    def biblio(self,
            aliases,
            provider_url_template=None,
            cache_enabled=True):

        data = self.get_account_data(aliases)
        if not data:
            return {}

        biblio_dict = {}
        biblio_dict["repository"] = "Twitter"
        biblio_dict["title"] = data["screen_name"]
        biblio_dict["authors"] = data["name"]
        biblio_dict["description"] = data["description"]
        biblio_dict["created_at"] = data["created_at"]

        return biblio_dict

    # default method; providers can override
    def metrics(self,
            aliases,
            provider_url_template=None,
            cache_enabled=True):

        data = self.get_account_data(aliases)
        if not data:
            return {}

        dict_of_keylists = {
            'twitter_account:followers' : ['followers_count'],
            'twitter_account:lists' : ['listed_count']
        }

        metrics_dict = {}
        for field in dict_of_keylists:
            metric_value = data[dict_of_keylists[field][0]]
            if metric_value:
                metrics_dict[field] = metric_value

        metrics_and_drilldown = {}
        for metric_name in metrics_dict:
            drilldown_url = self.provenance_url(metric_name, aliases)
            metrics_and_drilldown[metric_name] = (metrics_dict[metric_name], drilldown_url)

        return metrics_and_drilldown

<commit_msg>debug twitter access token<commit_after>from birdy.twitter import AppClient
import os, re

from totalimpact.providers import provider
from totalimpact.providers.provider import Provider, ProviderContentMalformedError

import logging
logger = logging.getLogger('ti.providers.twitter_account')

class Twitter_Account(Provider):

    example_id = ("url", "http://twitter.com/jasonpriem")

    url = "http://twitter.com"
    descr = "Social networking and microblogging service."
    member_items_url_template = "http://twitter.com/%s"
    provenance_url_template = "%s"
    biblio_url_template = "https://api.github.com/repos/%s?client_id=" + os.environ["GITHUB_CLIENT_ID"] + "&client_secret=" + os.environ["GITHUB_CLIENT_SECRET"]
    metrics_url_template = "https://api.github.com/repos/%s?client_id=" + os.environ["GITHUB_CLIENT_ID"] + "&client_secret=" + os.environ["GITHUB_CLIENT_SECRET"]

    static_meta_dict = {
        "followers": {
            "display_name": "followers",
            "provider": "Twitter",
            "provider_url": "http://twitter.com",
            "description": "The number of people following this Twitter account",
            "icon": "https://twitter.com/favicon.ico"
        },
        "lists": {
            "display_name": "lists",
            "provider": "Twitter",
            "provider_url": "http://twitter.com",
            "description": "The number of people who have included this Twitter account in a Twitter list",
            "icon": "https://twitter.com/favicon.ico"
            }
    }

    def __init__(self):
        super(Twitter_Account, self).__init__()
        temp_client = AppClient(os.getenv("TWITTER_CONSUMER_KEY"),
                            os.getenv("TWITTER_CONSUMER_SECRET"))
        access_token = temp_client.get_access_token()
        logger.debug(u"generated access token = {generated}, env = {env}".format(
            generated=access_token, env=os.getenv("TWITTER_ACCESS_TOKEN")))
        self.client = AppClient(os.getenv("TWITTER_CONSUMER_KEY"),
                            os.getenv("TWITTER_CONSUMER_SECRET"),
                            os.getenv("TWITTER_ACCESS_TOKEN"))

    # overriding default because overriding member_items method
    @property
    def provides_members(self):
        return True

    @property
    def provides_biblio(self):
         return True

    @property
    def provides_metrics(self):
         return True

    def is_relevant_alias(self, alias):
        (namespace, nid) = alias
        try:
            nid = nid.lower()
        except AttributeError:
            return False
        if (namespace == "url"):
            if ("twitter.com" in nid) and ("/status/" not in nid):
                return True
        return False

    def screen_name(self, nid):
        #regex from http://stackoverflow.com/questions/4424179/how-to-validate-a-twitter-username-using-regex
        match = re.findall("twitter.com/([A-Za-z0-9_]{1,15}$)", nid)
        return match[0]

    # default method; providers can override
    def member_items(self,
            query_string,
            provider_url_template=None,
            cache_enabled=True):

        twitter_username = query_string.replace("@", "")
        url = self._get_templated_url(self.member_items_url_template, twitter_username, "members")
        members = [("url", url)]
        return(members)

    def get_account_data(self, aliases):
        nid = self.get_best_id(aliases)
        if not nid:
            return None

        screen_name = self.screen_name(nid)

        r = self.client.api.users.show.get(screen_name=screen_name)
        return r.data

    # default method; providers can override
    def biblio(self,
            aliases,
            provider_url_template=None,
            cache_enabled=True):

        data = self.get_account_data(aliases)
        if not data:
            return {}

        biblio_dict = {}
        biblio_dict["repository"] = "Twitter"
        biblio_dict["title"] = data["screen_name"]
        biblio_dict["authors"] = data["name"]
        biblio_dict["description"] = data["description"]
        biblio_dict["created_at"] = data["created_at"]

        return biblio_dict

    # default method; providers can override
    def metrics(self,
            aliases,
            provider_url_template=None,
            cache_enabled=True):

        data = self.get_account_data(aliases)
        if not data:
            return {}

        dict_of_keylists = {
            'twitter_account:followers' : ['followers_count'],
            'twitter_account:lists' : ['listed_count']
        }

        metrics_dict = {}
        for field in dict_of_keylists:
            metric_value = data[dict_of_keylists[field][0]]
            if metric_value:
                metrics_dict[field] = metric_value

        metrics_and_drilldown = {}
        for metric_name in metrics_dict:
            drilldown_url = self.provenance_url(metric_name, aliases)
            metrics_and_drilldown[metric_name] = (metrics_dict[metric_name], drilldown_url)

        return metrics_and_drilldown

<|endoftext|>


<commit_before># Copyright 2014-2015 Isotoma Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import uuid

from touchdown.core.resource import Resource
from touchdown.core.plan import Plan, Present
from touchdown.core import argument, serializers

from ..account import Account
from ..common import SimpleDescribe, SimpleApply, SimpleDestroy, RefreshMetadata

from ..iam import ServerCertificate
from ..s3 import Bucket
from .. import route53

from .common import S3Origin, CloudFrontList, CloudFrontResourceList

class CustomOrigin(Resource):

    resource_name = "custom_origin"
    dot_ignore = True
    extra_serializers = {
        "CustomOriginConfig": serializers.Dict(
            HTTPPort=serializers.Argument("http_port"),
            HTTPSPort=serializers.Argument("https_port"),
            OriginProtocolPolicy=serializers.Argument("protocol"),
        )
    }

    name = argument.String(field='Id')
    domain_name = argument.String(field='DomainName')
    origin_path = argument.String(default='', field='OriginPath')
    http_port = argument.Integer(default=80)
    https_port = argument.Integer(default=443)
    protocol = argument.String(choices=['http-only', 'match-viewer'], default='match-viewer')

class DefaultCacheBehavior(Resource):

    resource_name = "default_cache_behaviour"
    dot_ignore = True

    extra_serializers = {
        # TrustedSigners are not supported yet, so include stub in serialized form
        "TrustedSigners": serializers.Const({
            "Enabled": False,
            "Quantity": 0,
        }),
        "AllowedMethods": CloudFrontList(
            inner=serializers.Context(serializers.Argument("allowed_methods"), serializers.List()),
            CachedMethods=serializers.Context(serializers.Argument("cached_methods"), CloudFrontList()),
        ),
        "ForwardedValues": serializers.Resource(
            group="forwarded-values",
            Cookies=serializers.Resource(
                group="cookies",
                Forward=serializers.Expression(lambda r, o: "all" if o.forward_cookies == ["*"] else "none" if len(o.forward_cookies) == 0 else "whitelist"),
            )
        )
    }

    target_origin = argument.String(field='TargetOriginId')

    forward_query_string = argument.Boolean(default=True, field="QueryString", group="forwarded-values")
    forward_headers = argument.List(field="Headers", serializer=CloudFrontList(serializers.List()), group="forwarded-values")
    forward_cookies = argument.List(field="WhitelistedNames", serializer=CloudFrontList(serializers.Expression(lambda r, o: [] if o == ['*'] else o)), group="cookies")

    allowed_methods = argument.List(default=lambda x: ["GET", "HEAD"],)
    cached_methods = argument.List(default=lambda x: ["GET", "HEAD"])

    min_ttl = argument.Integer(default=0, field="MinTTL")
    viewer_protocol_policy = argument.String(choices=['allow-all', 'https-only', 'redirect-to-https'], default='allow-all', field="ViewerProtocolPolicy")
    smooth_streaming = argument.Boolean(default=False, field='SmoothStreaming')

class CacheBehavior(DefaultCacheBehavior):

    resource_name = "cache_behaviour"
    path_pattern = argument.String(field='PathPattern')

class ErrorResponse(Resource):

    resource_name = "error_response"
    dot_ignore = True

    error_code = argument.Integer(field="ErrorCode")
    response_page_path = argument.String(field="ResponsePagePath")
    response_code = argument.Integer(field="ResponseCode")
    min_ttl = argument.Integer(field="ErrorCachingMinTTL")

class LoggingConfig(Resource):

    resource_name = "logging_config"
    dot_ignore = True

    enabled = argument.Boolean(field="Enabled", default=False)
    include_cookies = argument.Boolean(field="IncludeCookies", default=False)
    bucket = argument.Resource(Bucket, field="Bucket", serializer=serializers.Default(default=None), default="")
    prefix = argument.String(field="Prefix", default="")

class Distribution(Resource):

    resource_name = "distribution"

    extra_serializers = {
        "CallerReference": serializers.Expression(
            lambda runner, object: runner.get_plan(object).object.get('CallerReference', str(uuid.uuid4()))
        ),
        "Aliases": CloudFrontList(serializers.Chain(
            serializers.Context(serializers.Argument("name"), serializers.ListOfOne()),
            serializers.Context(serializers.Argument("aliases"), serializers.List()),
        )),
        # We don't support GeoRestrictions yet - so include a stubbed default
        # when serializing
        "Restrictions": serializers.Const({
            "GeoRestriction": {
                "RestrictionType": "none",
                "Quantity": 0,
            },
        }),
        "ViewerCertificate": serializers.Resource(
            group="viewer-certificate",
            CloudFrontDefaultCertificate=serializers.Expression(lambda r, o: False if o.ssl_certificate else True),
        )
    }

    name = argument.String()
    comment = argument.String(field='Comment', default=lambda instance: instance.name)
    aliases = argument.List()
    root_object = argument.String(default='/', field="DefaultRootObject")
    enabled = argument.Boolean(default=True, field="Enabled")
    origins = argument.ResourceList(
        (S3Origin, CustomOrigin),
        field="Origins",
        serializer=CloudFrontResourceList(),
    )
    default_cache_behavior = argument.Resource(
        DefaultCacheBehavior,
        field="DefaultCacheBehavior",
        serializer=serializers.Resource(),
    )
    behaviors = argument.ResourceList(
        CacheBehavior,
        field="CacheBehaviors",
        serializer=CloudFrontResourceList(),
    )
    error_responses = argument.ResourceList(
        ErrorResponse,
        field="CustomErrorResponses",
        serializer=CloudFrontResourceList(),
    )
    logging = argument.Resource(
        LoggingConfig,
        default=lambda instance: dict(enabled=False),
        field="Logging",
        serializer=serializers.Resource(),
    )
    price_class = argument.String(
        default="PriceClass_100",
        choices=['PriceClass_100', 'PriceClass_200', 'PriceClass_All'],
        field="PriceClass",
    )

    ssl_certificate = argument.Resource(
        ServerCertificate,
        field="IAMCertificateId",
        serializer=serializers.Property("ServerCertificateId"),
        group="viewer-certificate",
    )

    ssl_support_method = argument.String(
        default="sni-only",
        choices=["sni-only", "vip"],
        field="SSLSupportMethod",
        group="viewer-certificate",
    )

    ssl_minimum_protocol_version = argument.String(
        default="TLSv1",
        choices=["TLSv1", "SSLv3"],
        field="MinimumProtocolVersion",
        group="viewer-certificate",
    )

    account = argument.Resource(Account)

class Describe(SimpleDescribe, Plan):

    resource = Distribution
    service_name = 'cloudfront'
    describe_filters = {}
    describe_action = "list_distributions"
    describe_envelope = 'DistributionList.Items'
    key = 'Id'

    def get_describe_filters(self):
        return {"Id": self.object['Id']}

    def describe_object_matches(self, d):
        return self.resource.name in d['Aliases'].get('Items', [])

    def describe_object(self):
        distribution = super(Describe, self).describe_object()
        if distribution:
            result = self.client.get_distribution(Id=distribution['Id'])
            distribution = {"ETag": result["ETag"], "Id": distribution["Id"]}
            distribution.update(result['Distribution']['DistributionConfig'])
            return distribution

class Apply(SimpleApply, Describe):

    create_action = "create_distribution"
    #update_action = "update_distribution"
    create_response = "not-that-useful"
    waiter = "distribution_deployed"

    signature = (
        Present("name"),
        Present("origins"),
        Present("default_cache_behavior"),
    )

    def get_create_serializer(self):
        return serializers.Dict(
            DistributionConfig=serializers.Resource(),
        )

    def get_update_serializer(self):
        return serializers.Dict(
            Id=serializers.Identifier(),
            DistributionConfig=serializers.Resource(),
            IfMatch=serializers.Property("ETag"),
        )

class Destroy(SimpleDestroy, Describe):

    destroy_action = "delete_distribution"

    def get_destroy_serializer(self):
        return serializers.Dict(
            Id=self.resource_id,
            IfMatch=serializers.Property('ETag'),
        )

    def destroy_object(self):
        if not self.object:
            return

        if self.object.get('Enabled', False):
            yield self.generic_action(
                "Disable distribution",
                self.client.update_distribution,
                Id=self.object['Id'],
                IfMatch=self.object['ETag'],
                DistributionConfig=serializers.Resource(
                    Enabled=False,
                ),
            )

            yield self.get_waiter(
                ["Waiting for distribution to enter disabled state"],
                "distribution_deployed",
            )

            yield RefreshMetadata(self)

        for change in super(Destroy, self).destroy_object():
            yield change

class AliasTarget(route53.AliasTarget):

    """ Adapts a Distribution into a AliasTarget """

    input = Distribution

    def get_serializer(self, runner, **kwargs):
        return serializers.Context(
            serializers.Const(self.adapts),
            serializers.Dict(
                DNSName=serializers.Context(
                    serializers.Property("DomainName"),
                    serializers.Expression(lambda r, o: route53._normalize(o)),
                ),
                HostedZoneId="Z2FDTNDATAQYW2",
                EvaluateTargetHealth=False,
            )
        )
<commit_msg>Properly pickup the DomainName field<commit_after># Copyright 2014-2015 Isotoma Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import uuid

from touchdown.core.resource import Resource
from touchdown.core.plan import Plan, Present
from touchdown.core import argument, serializers

from ..account import Account
from ..common import SimpleDescribe, SimpleApply, SimpleDestroy, RefreshMetadata

from ..iam import ServerCertificate
from ..s3 import Bucket
from .. import route53

from .common import S3Origin, CloudFrontList, CloudFrontResourceList

class CustomOrigin(Resource):

    resource_name = "custom_origin"
    dot_ignore = True
    extra_serializers = {
        "CustomOriginConfig": serializers.Dict(
            HTTPPort=serializers.Argument("http_port"),
            HTTPSPort=serializers.Argument("https_port"),
            OriginProtocolPolicy=serializers.Argument("protocol"),
        )
    }

    name = argument.String(field='Id')
    domain_name = argument.String(field='DomainName')
    origin_path = argument.String(default='', field='OriginPath')
    http_port = argument.Integer(default=80)
    https_port = argument.Integer(default=443)
    protocol = argument.String(choices=['http-only', 'match-viewer'], default='match-viewer')

class DefaultCacheBehavior(Resource):

    resource_name = "default_cache_behaviour"
    dot_ignore = True

    extra_serializers = {
        # TrustedSigners are not supported yet, so include stub in serialized form
        "TrustedSigners": serializers.Const({
            "Enabled": False,
            "Quantity": 0,
        }),
        "AllowedMethods": CloudFrontList(
            inner=serializers.Context(serializers.Argument("allowed_methods"), serializers.List()),
            CachedMethods=serializers.Context(serializers.Argument("cached_methods"), CloudFrontList()),
        ),
        "ForwardedValues": serializers.Resource(
            group="forwarded-values",
            Cookies=serializers.Resource(
                group="cookies",
                Forward=serializers.Expression(lambda r, o: "all" if o.forward_cookies == ["*"] else "none" if len(o.forward_cookies) == 0 else "whitelist"),
            )
        )
    }

    target_origin = argument.String(field='TargetOriginId')

    forward_query_string = argument.Boolean(default=True, field="QueryString", group="forwarded-values")
    forward_headers = argument.List(field="Headers", serializer=CloudFrontList(serializers.List()), group="forwarded-values")
    forward_cookies = argument.List(field="WhitelistedNames", serializer=CloudFrontList(serializers.Expression(lambda r, o: [] if o == ['*'] else o)), group="cookies")

    allowed_methods = argument.List(default=lambda x: ["GET", "HEAD"],)
    cached_methods = argument.List(default=lambda x: ["GET", "HEAD"])

    min_ttl = argument.Integer(default=0, field="MinTTL")
    viewer_protocol_policy = argument.String(choices=['allow-all', 'https-only', 'redirect-to-https'], default='allow-all', field="ViewerProtocolPolicy")
    smooth_streaming = argument.Boolean(default=False, field='SmoothStreaming')

class CacheBehavior(DefaultCacheBehavior):

    resource_name = "cache_behaviour"
    path_pattern = argument.String(field='PathPattern')

class ErrorResponse(Resource):

    resource_name = "error_response"
    dot_ignore = True

    error_code = argument.Integer(field="ErrorCode")
    response_page_path = argument.String(field="ResponsePagePath")
    response_code = argument.Integer(field="ResponseCode")
    min_ttl = argument.Integer(field="ErrorCachingMinTTL")

class LoggingConfig(Resource):

    resource_name = "logging_config"
    dot_ignore = True

    enabled = argument.Boolean(field="Enabled", default=False)
    include_cookies = argument.Boolean(field="IncludeCookies", default=False)
    bucket = argument.Resource(Bucket, field="Bucket", serializer=serializers.Default(default=None), default="")
    prefix = argument.String(field="Prefix", default="")

class Distribution(Resource):

    resource_name = "distribution"

    extra_serializers = {
        "CallerReference": serializers.Expression(
            lambda runner, object: runner.get_plan(object).object.get('CallerReference', str(uuid.uuid4()))
        ),
        "Aliases": CloudFrontList(serializers.Chain(
            serializers.Context(serializers.Argument("name"), serializers.ListOfOne()),
            serializers.Context(serializers.Argument("aliases"), serializers.List()),
        )),
        # We don't support GeoRestrictions yet - so include a stubbed default
        # when serializing
        "Restrictions": serializers.Const({
            "GeoRestriction": {
                "RestrictionType": "none",
                "Quantity": 0,
            },
        }),
        "ViewerCertificate": serializers.Resource(
            group="viewer-certificate",
            CloudFrontDefaultCertificate=serializers.Expression(lambda r, o: False if o.ssl_certificate else True),
        )
    }

    name = argument.String()
    comment = argument.String(field='Comment', default=lambda instance: instance.name)
    aliases = argument.List()
    root_object = argument.String(default='/', field="DefaultRootObject")
    enabled = argument.Boolean(default=True, field="Enabled")
    origins = argument.ResourceList(
        (S3Origin, CustomOrigin),
        field="Origins",
        serializer=CloudFrontResourceList(),
    )
    default_cache_behavior = argument.Resource(
        DefaultCacheBehavior,
        field="DefaultCacheBehavior",
        serializer=serializers.Resource(),
    )
    behaviors = argument.ResourceList(
        CacheBehavior,
        field="CacheBehaviors",
        serializer=CloudFrontResourceList(),
    )
    error_responses = argument.ResourceList(
        ErrorResponse,
        field="CustomErrorResponses",
        serializer=CloudFrontResourceList(),
    )
    logging = argument.Resource(
        LoggingConfig,
        default=lambda instance: dict(enabled=False),
        field="Logging",
        serializer=serializers.Resource(),
    )
    price_class = argument.String(
        default="PriceClass_100",
        choices=['PriceClass_100', 'PriceClass_200', 'PriceClass_All'],
        field="PriceClass",
    )

    ssl_certificate = argument.Resource(
        ServerCertificate,
        field="IAMCertificateId",
        serializer=serializers.Property("ServerCertificateId"),
        group="viewer-certificate",
    )

    ssl_support_method = argument.String(
        default="sni-only",
        choices=["sni-only", "vip"],
        field="SSLSupportMethod",
        group="viewer-certificate",
    )

    ssl_minimum_protocol_version = argument.String(
        default="TLSv1",
        choices=["TLSv1", "SSLv3"],
        field="MinimumProtocolVersion",
        group="viewer-certificate",
    )

    account = argument.Resource(Account)

class Describe(SimpleDescribe, Plan):

    resource = Distribution
    service_name = 'cloudfront'
    describe_filters = {}
    describe_action = "list_distributions"
    describe_envelope = 'DistributionList.Items'
    key = 'Id'

    def get_describe_filters(self):
        return {"Id": self.object['Id']}

    def describe_object_matches(self, d):
        return self.resource.name in d['Aliases'].get('Items', [])

    def describe_object(self):
        distribution = super(Describe, self).describe_object()
        if distribution:
            result = self.client.get_distribution(Id=distribution['Id'])
            distribution = {
                "ETag": result["ETag"],
                "Id": distribution["Id"],
                "DomainName": result["Distribution"]["DomainName"],
            }
            distribution.update(result['Distribution']['DistributionConfig'])
            return distribution

class Apply(SimpleApply, Describe):

    create_action = "create_distribution"
    #update_action = "update_distribution"
    create_response = "not-that-useful"
    waiter = "distribution_deployed"

    signature = (
        Present("name"),
        Present("origins"),
        Present("default_cache_behavior"),
    )

    def get_create_serializer(self):
        return serializers.Dict(
            DistributionConfig=serializers.Resource(),
        )

    def get_update_serializer(self):
        return serializers.Dict(
            Id=serializers.Identifier(),
            DistributionConfig=serializers.Resource(),
            IfMatch=serializers.Property("ETag"),
        )

class Destroy(SimpleDestroy, Describe):

    destroy_action = "delete_distribution"

    def get_destroy_serializer(self):
        return serializers.Dict(
            Id=self.resource_id,
            IfMatch=serializers.Property('ETag'),
        )

    def destroy_object(self):
        if not self.object:
            return

        if self.object.get('Enabled', False):
            yield self.generic_action(
                "Disable distribution",
                self.client.update_distribution,
                Id=self.object['Id'],
                IfMatch=self.object['ETag'],
                DistributionConfig=serializers.Resource(
                    Enabled=False,
                ),
            )

            yield self.get_waiter(
                ["Waiting for distribution to enter disabled state"],
                "distribution_deployed",
            )

            yield RefreshMetadata(self)

        for change in super(Destroy, self).destroy_object():
            yield change

class AliasTarget(route53.AliasTarget):

    """ Adapts a Distribution into a AliasTarget """

    input = Distribution

    def get_serializer(self, runner, **kwargs):
        return serializers.Context(
            serializers.Const(self.adapts),
            serializers.Dict(
                DNSName=serializers.Context(
                    serializers.Property("DomainName"),
                    serializers.Expression(lambda r, o: route53._normalize(o)),
                ),
                HostedZoneId="Z2FDTNDATAQYW2",
                EvaluateTargetHealth=False,
            )
        )
<|endoftext|>