<commit_before>"""ML-ENSEMBLE

:author: Sebastian Flennerhag
:copyright: 2017
:licence: MIT

Classes for partitioning training data.
"""

import numpy as np
import warnings

class FullIndex(object):

    """Indexer that generates the full size of X.

    K-Fold iterator that generates fold index tuples.

    FullIndex creates a generator that returns a tuple of stop and start
    positions to be used for numpy array slicing [stop:start]. Note that
    slicing works well for the test set, but for the training set it is
    recommended to concatenate the index for training data that comes before
    the current test set with the index for the training data that comes after.
    This can easily be achieved with::

        for train_tup, test_tup in self.generate():
            train_slice = numpy.hstack([numpy.arange(t0, t1) for t0, t1 in
                                      train_tup])

            xtrain, xtest = X[train_slice], X[test_tup[0]:test_tup[1]]

    Warnings
    --------
    Slicing only works for the test set, which is convex, but typically not for
    the training set. To build get a training index, use
    ``hstack([np.arange(t0, t1) for t0, t1 in tri])``.

    Examples
    --------

    Creating arrays of folds and checking overlap

    >>> import numpy as np
    >>> from mlens.base.indexer import FullIndex
    >>> X = np.arange(10)
    >>> print("Data set: %r" % X)
    >>> print()
    >>>
    >>> idx = FullIndex(X, 4)
    >>>
    >>> for train, test in idx.generate(as_array=True):
    ...     print('TRAIN IDX: %32r | TEST IDX: %16r' % (train, test))
    >>>
    >>> print()
    >>>
    >>> for train, test in idx.generate(as_array=True):
            print('TRAIN SET: %32r | TEST SET: %16r' % (X[train], X[test]))
    >>>
    >>> for train_idx, test_idx in idx.generate(as_array=True):
    ...     assert not any([i in X[test_idx] for i in X[train_idx]])
    >>>
    >>>print()
    >>>
    >>> print("No overlap between train set and test set.")
    Data set: array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])

    TRAIN IDX:     array([3, 4, 5, 6, 7, 8, 9]) | TEST IDX: array([0, 1, 2])
    TRAIN IDX:     array([0, 1, 2, 6, 7, 8, 9]) | TEST IDX: array([3, 4, 5])
    TRAIN IDX:  array([0, 1, 2, 3, 4, 5, 8, 9]) | TEST IDX:    array([6, 7])
    TRAIN IDX:  array([0, 1, 2, 3, 4, 5, 6, 7]) | TEST IDX:    array([8, 9])

    TRAIN SET:     array([3, 4, 5, 6, 7, 8, 9]) | TEST SET: array([0, 1, 2])
    TRAIN SET:     array([0, 1, 2, 6, 7, 8, 9]) | TEST SET: array([3, 4, 5])
    TRAIN SET:  array([0, 1, 2, 3, 4, 5, 8, 9]) | TEST SET:    array([6, 7])
    TRAIN SET:  array([0, 1, 2, 3, 4, 5, 6, 7]) | TEST SET:    array([8, 9])

    No overlap between train set and test set.

    Passing only one fold with raise_on_exception set to False

    >>> import numpy as np
    >>> from mlens.base.indexer import FullIndex
    >>> X = np.arange(3)
    >>> print("Data set: %r" % X)
    >>> print()
    >>>
    >>> idx = FullIndex(X, 1, raise_on_exception=False)
    >>>
    >>> for train, test in idx.generate(as_array=True):
    ...     print('TRAIN IDX: %10r | TEST IDX: %10r' % (train, test))
    Data set: array([0, 1, 2])

    /../mlens/base/indexer.py:167: UserWarning: 'n_splits' is 1, will return
    full index as both training set and test set.
    warnings.warn("'n_splits' is 1, will return full index as "

    Data set: array([0, 1, 2])
    TRAIN IDX: array([0, 1, 2]) | TEST IDX: array([0, 1, 2])
    """

    def __init__(self, n_splits=2, X=None, raise_on_exception=True):
        self.n_splits = n_splits
        self.raise_on_exception = raise_on_exception

        if X is not None:
            self.fit(X)

    def fit(self, X):
        """Set indexer up for slicing an array of length X."""
        n = X.shape[0]
        check_index(n, self.n_splits, self.raise_on_exception)

        self.n_samples = n

    def _gen_indices(self):
        n_samples = self.n_samples
        n_splits = self.n_splits

        if n_splits == 1:
            # Return the full index as both training and test set
            yield ((0, n_samples),),  (0, n_samples)
        else:

            # Get the length of the test set. If n_samples mod n_splits is
            #  not 0, increment n_remainder folds by 1
            tei_len = (n_samples // n_splits) * np.ones(n_splits, dtype=np.int)
            tei_len[:n_samples % n_splits] += 1

            last = 0  # counter for where last test index stopped
            for size in tei_len:
                tei_start, tei_stop = last, last + size
                tri_start_below, tri_stop_below = 0, tei_start
                tri_start_above, tri_stop_above = tei_stop, n_samples

                # Build test set tuple
                tei = (tei_start, tei_stop)

                # Build train set tuple(s)
                # Check if the first tuple is (0, 0) - if so drop
                if tri_start_below == tri_stop_below:
                    tri = ((tri_start_above, tri_stop_above),)

                # Check if the first tuple is (n, n) - if so drop
                elif tri_start_below == tri_stop_below:
                    tri = ((tri_start_above, tri_stop_above),)

                else:
                    tri = ((tri_start_below, tri_stop_below),
                           (tri_start_above, tri_stop_above))

                yield (tri, tei)
                last = tei_stop

    def generate(self, X=None, as_array=False):
        """Generator."""
        # Check that the instance have some array information to work with
        if not hasattr(self, 'n_samples'):
            if X is None:
                raise AttributeError("No array provided to indexer. Either "
                                     "pass an array to the 'generate' method, "
                                     "or call the 'fit' method first or "
                                     "initiate the instance with an array X "
                                     "as argument.")
            else:
                # Need to call fit to continue
                self.fit(X)

        for tri, tei in self._gen_indices():

            # return np.arrays
            if as_array:
                tri = np.hstack([np.arange(t0, t1) for t0, t1 in tri])
                tei = np.arange(tei[0], tei[1])

            yield tri, tei

def check_index(n_samples, n_splits, raise_on_exception):
    """Check that folds can be constructed from passed arguments."""
    if not isinstance(n_splits, int):
        raise ValueError("'n_splits' must be an integer. "
                         "type(%s) was passed." % type(n_splits))

    if n_splits <= 1:
        if raise_on_exception:
            raise ValueError("Need at least 2 folds to partition data. "
                             "Got %i." % n_splits)
        else:
            if n_splits == 1:
                warnings.warn("'n_splits' is 1, will return full index as "
                              "both training set and test set.")
    if n_splits > n_samples:
        raise ValueError("Number of splits %i is greater than the number "
                         "of samples: %i." % (n_splits, n_samples))
<commit_msg>docstrings<commit_after>"""ML-ENSEMBLE

:author: Sebastian Flennerhag
:copyright: 2017
:licence: MIT

Classes for partitioning training data.
"""

from abc import abstractmethod
from numbers import Integral
import numpy as np
import warnings

class BaseIndex(object):
    """Base Index class."""

    @abstractmethod
    def fit(self, X):
        """Method for storing array data.

        Parameters
        ----------
        X : array-like
            array to collect dimension data from.

        Returns
        -------
        instance
        """

    @abstractmethod
    def _gen_indices(self):
        """Method for constructing the index generator.

        Returns
        -------
        iterable
        """

    def generate(self, X=None, as_array=False):
        """Generator."""
        # Check that the instance have some array information to work with
        if not hasattr(self, 'n_samples'):
            if X is None:
                raise AttributeError("No array provided to indexer. Either "
                                     "pass an array to the 'generate' method, "
                                     "or call the 'fit' method first or "
                                     "initiate the instance with an array X "
                                     "as argument.")
            else:
                # Need to call fit to continue
                self.fit(X)

        for tri, tei in self._gen_indices():

            if as_array:
                # return np.arrays
                if isinstance(tri[0], tuple):
                    # If a tuple of indices, build iteratively
                    tri = np.hstack([np.arange(t0, t1) for t0, t1 in tri])
                else:
                    tri = np.arange(tri[0], tri[1])

                tei = np.arange(tei[0], tei[1])

            yield tri, tei

class BlendIndex(BaseIndex):

    """Indexer that generates two non-overlapping subsets of X.

    Iterator that generates one training fold and one test fold that are
    non-overlapping and that may or may not partition all of X depending on the
    user's specification.

    BlendIndex creates a singleton generator (has on iteration) that
    yields two tuples of ``(start, stop)`` integers that can be used for
    numpy array slicing (i.e. ``X[stop:start]``). If a full array index
    is desired this can easily be achieved with::

        for train_tup, test_tup in self.generate():
            train_slice = numpy.hstack([numpy.arange(t0, t1) for t0, t1 in
                                      train_tup])

            test_slice = numpy.hstack([numpy.arange(t0, t1) for t0, t1 in
                                      test_tup])

    See Also
    --------
    :class:`FullIndex`

    Examples
    --------

    Selecting an absolute test size, with train size as the remainder

    >>> import numpy as np
    >>> from mlens.base.indexer import BlendIndex
    >>> X = np.arange(8)
    >>> idx = BlendIndex(3)
    >>> print('Test size: 3')
    >>> for tri, tei in idx.generate(X):
    ...     print('TEST (idx | array): (%i, %i) | %r ' % (tei[0], tei[1],
    ...                                                   X[tei[0]:tei[1]]))
    ...     print('TRAIN (idx | array): (%i, %i) | %r ' % (tri[0], tri[1],
    ...                                                    X[tri[0]:tri[1]]))
    Test size: 3
    TEST (idx | array): (5, 8) | array([5, 6, 7])
    TRAIN (idx | array): (0, 5) | array([0, 1, 2, 3, 4])

    Selecting a test and train size less than the total

    >>> import numpy as np
    >>> from mlens.base.indexer import BlendIndex
    >>> X = np.arange(8)
    >>> idx = BlendIndex(3, 4, X)
    >>> print('Test size: 3')
    >>> print('Train size: 4')
    >>> for tri, tei in idx.generate(X):
    ...     print('TEST (idx | array): (%i, %i) | %r ' % (tei[0], tei[1],
    ...                                                   X[tei[0]:tei[1]]))
    ...     print('TRAIN (idx | array): (%i, %i) | %r ' % (tri[0], tri[1],
    ...                                                    X[tri[0]:tri[1]]))
    Test size: 3
    Train size: 4
    TEST (idx | array): (4, 7) | array([4, 5, 6])
    TRAIN (idx | array): (0, 4) | array([0, 1, 2, 3])

    Selecting a percentage of observations as test and train set

    >>> import numpy as np
    >>> from mlens.base.indexer import BlendIndex
    >>> X = np.arange(8)
    >>> idx = BlendIndex(0.25, 0.45, X)
    >>> print('Test size: 25% * 8 = 2')
    >>> print('Train size: 45% * 8 < 4 -> 3')
    >>> for tri, tei in idx.generate(X):
    ...     print('TEST (idx | array): (%i, %i) | %r ' % (tei[0], tei[1],
    ...                                                   X[tei[0]:tei[1]]))
    ...     print('TRAIN (idx | array): (%i, %i) | %r ' % (tri[0], tri[1],
    ...                                                    X[tri[0]:tri[1]]))
    Test size: 25% * 8 = 2
    Train size: 50% * 8 < 4 ->
    TEST (idx | array): (3, 5) | array([[3, 4]])
    TRAIN (idx | array): (0, 3) | array([[0, 1, 2]])
    """

    def __init__(self, test_size, train_size=None,
                 X=None, raise_on_exception=True):

        self.test_size = test_size
        self.train_size = train_size
        self.raise_on_exception = raise_on_exception

        if X is not None:
            self.fit(X)

    def fit(self, X):
        """Set indexer up for slicing an array of length X."""
        n = X.shape[0]

        self.n_samples = n

        # Get number of test samples
        if isinstance(self.test_size, int):
            self.n_test = self.test_size
        else:
            self.n_test = int(np.floor(self.test_size * n))

        # Get number of train samples
        if self.train_size is None:
            # Partition X
            self.n_train = int(n - self.n_test)

        elif isinstance(self.train_size, int):
            self.n_train = self.train_size

        else:
            self.n_train = int(np.floor(self.train_size * n))

        _check_partial_index(self.n_samples, self.test_size, self.train_size,
                              self.n_test, self.n_train)

        return self

    def _gen_indices(self):
        yield (0, self.n_train), (self.n_train, self.n_train + self.n_test)

class FullIndex(BaseIndex):

    """Indexer that generates the full size of X.

    K-Fold iterator that generates fold index tuples.

    FullIndex creates a generator that returns a tuple of stop and start
    positions to be used for numpy array slicing [stop:start]. Note that
    slicing works well for the test set, but for the training set it is
    recommended to concatenate the index for training data that comes before
    the current test set with the index for the training data that comes after.
    This can easily be achieved with::

        for train_tup, test_tup in self.generate():
            train_slice = numpy.hstack([numpy.arange(t0, t1) for t0, t1 in
                                      train_tup])

            xtrain, xtest = X[train_slice], X[test_tup[0]:test_tup[1]]

    Warnings
    --------
    Slicing only works for the test set, which is convex, but typically not for
    the training set. To build get a training index, use
    ``hstack([np.arange(t0, t1) for t0, t1 in tri])``.

    See Also
    --------
    :class:`BlendIndex`

    Examples
    --------

    Creating arrays of folds and checking overlap

    >>> import numpy as np
    >>> from mlens.base.indexer import FullIndex
    >>> X = np.arange(10)
    >>> print("Data set: %r" % X)
    >>> print()
    >>>
    >>> idx = FullIndex(4, X)
    >>>
    >>> for train, test in idx.generate(as_array=True):
    ...     print('TRAIN IDX: %32r | TEST IDX: %16r' % (train, test))
    >>>
    >>> print()
    >>>
    >>> for train, test in idx.generate(as_array=True):
    ...     print('TRAIN SET: %32r | TEST SET: %16r' % (X[train], X[test]))
    >>>
    >>> for train_idx, test_idx in idx.generate(as_array=True):
    ...     assert not any([i in X[test_idx] for i in X[train_idx]])
    >>>
    >>> print()
    >>>
    >>> print("No overlap between train set and test set.")
    Data set: array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])

    TRAIN IDX:     array([3, 4, 5, 6, 7, 8, 9]) | TEST IDX: array([0, 1, 2])
    TRAIN IDX:     array([0, 1, 2, 6, 7, 8, 9]) | TEST IDX: array([3, 4, 5])
    TRAIN IDX:  array([0, 1, 2, 3, 4, 5, 8, 9]) | TEST IDX:    array([6, 7])
    TRAIN IDX:  array([0, 1, 2, 3, 4, 5, 6, 7]) | TEST IDX:    array([8, 9])

    TRAIN SET:     array([3, 4, 5, 6, 7, 8, 9]) | TEST SET: array([0, 1, 2])
    TRAIN SET:     array([0, 1, 2, 6, 7, 8, 9]) | TEST SET: array([3, 4, 5])
    TRAIN SET:  array([0, 1, 2, 3, 4, 5, 8, 9]) | TEST SET:    array([6, 7])
    TRAIN SET:  array([0, 1, 2, 3, 4, 5, 6, 7]) | TEST SET:    array([8, 9])

    No overlap between train set and test set.

    Passing only one fold with raise_on_exception set to False

    >>> import numpy as np
    >>> from mlens.base.indexer import FullIndex
    >>> X = np.arange(3)
    >>> print("Data set: %r" % X)
    >>> print()
    >>>
    >>> idx = FullIndex(1, X, raise_on_exception=False)
    >>>
    >>> for train, test in idx.generate(as_array=True):
    ...     print('TRAIN IDX: %10r | TEST IDX: %10r' % (train, test))
    Data set: array([0, 1, 2])

    /../mlens/base/indexer.py:167: UserWarning: 'n_splits' is 1, will return
    full index as both training set and test set.
    warnings.warn("'n_splits' is 1, will return full index as "

    Data set: array([0, 1, 2])
    TRAIN IDX: array([0, 1, 2]) | TEST IDX: array([0, 1, 2])
    """

    def __init__(self, n_splits=2, X=None, raise_on_exception=True):
        self.n_splits = n_splits
        self.raise_on_exception = raise_on_exception

        if X is not None:
            self.fit(X)

    def fit(self, X):
        """Set indexer up for slicing an array of length X."""
        n = X.shape[0]
        _check_full_index(n, self.n_splits, self.raise_on_exception)

        self.n_samples = n

    def _gen_indices(self):
        n_samples = self.n_samples
        n_splits = self.n_splits

        if n_splits == 1:
            # Return the full index as both training and test set
            yield ((0, n_samples),),  (0, n_samples)
        else:

            # Get the length of the test set. If n_samples mod n_splits is
            #  not 0, increment n_remainder folds by 1
            tei_len = (n_samples // n_splits) * np.ones(n_splits, dtype=np.int)
            tei_len[:n_samples % n_splits] += 1

            last = 0  # counter for where last test index stopped
            for size in tei_len:
                tei_start, tei_stop = last, last + size
                tri_start_below, tri_stop_below = 0, tei_start
                tri_start_above, tri_stop_above = tei_stop, n_samples

                # Build test set tuple
                tei = (tei_start, tei_stop)

                # Build train set tuple(s)
                # Check if the first tuple is (0, 0) - if so drop
                if tri_start_below == tri_stop_below:
                    tri = ((tri_start_above, tri_stop_above),)

                # Check if the first tuple is (n, n) - if so drop
                elif tri_start_below == tri_stop_below:
                    tri = ((tri_start_above, tri_stop_above),)

                else:
                    tri = ((tri_start_below, tri_stop_below),
                           (tri_start_above, tri_stop_above))

                yield (tri, tei)
                last = tei_stop

###############################################################################
def _check_full_index(n_samples, n_splits, raise_on_exception):
    """Check that folds can be constructed from passed arguments."""
    if not isinstance(n_splits, Integral):
        raise ValueError("'n_splits' must be an integer. "
                         "type(%s) was passed." % type(n_splits))

    if n_splits <= 1:
        if raise_on_exception:
            raise ValueError("Need at least 2 folds to partition data. "
                             "Got %i." % n_splits)
        else:
            if n_splits == 1:
                warnings.warn("'n_splits' is 1, will return full index as "
                              "both training set and test set.")
    if n_splits > n_samples:
        raise ValueError("Number of splits %i is greater than the number "
                         "of samples: %i." % (n_splits, n_samples))

def _check_partial_index(n_samples, test_size, train_size,
                         n_test, n_train):
    """Check that folds can be constructed from passed arguments."""

    if not n_test + n_train <= n_samples:
        raise ValueError("The selection of train (%r) and test (%r) samples "
                         "lead to a selection greater than the number of "
                         "observations (%i). Test size: %i, train size: "
                         "%i." % (test_size, train_size,
                                  n_samples, n_test, n_train))

    for n, i, j in zip(('test', 'train'),
                    (n_test, n_train),
                    (test_size, train_size)):
        if n == 0:
            raise ValueError("The %s set size is 0 with current selection ("
                             "%r): "
                             "cannot create %s subset. Assign a greater "
                             "proportion of samples to the %s set (total "
                             "samples size: %i)." % (i, j, i, i, n_samples))

    if n_samples == 0:
        raise ValueError("Sample size is 0: nothing to create subset from.")
<|endoftext|>


<commit_before>import unittest
from mock import Mock
from cellardoor import errors
from cellardoor.api import API, StandardOptionsMixin, InterfaceProxy, FilterProxy, LinkProxy
from cellardoor.api.methods import ALL
from cellardoor.storage import Storage
from cellardoor.model import Model, Entity

storage = Storage()
storage.create = Mock()
model = Model(storage=storage)
api = API(model)

class Foo(model.Entity):
	pass

class Foos(api.Interface):
	entity = Foo
	method_authorization = {
		ALL: None
	}

class TestStandardOptionsMixin(unittest.TestCase):

	def test_has_options(self):
		options = StandardOptionsMixin()
		self.assertTrue(hasattr(options, '_options'))
		self.assertEquals(options._options, {})

	def test_identity(self):
		options = StandardOptionsMixin()
		self.assertTrue(hasattr(options, 'identity'))
		self.assertEquals(options.identity(), None)
		result = options.identity('foo')
		self.assertEquals(result, options)
		self.assertEquals(options.identity(), 'foo')
		self.assertEquals(options._options, {'context':{'identity':'foo'}})

	def test_create_accessor(self):
		options = StandardOptionsMixin('foo', 'bar')
		self.assertTrue(hasattr(options, 'foo'))
		self.assertTrue(hasattr(options, 'bar'))
		result = options.foo(23)
		self.assertEquals(result, options)
		self.assertEquals(options.foo(), 23)

	def test_merge(self):
		options = StandardOptionsMixin('foo')
		options.foo(1)
		other_options = {'foo':2, 'bar':3}
		merged_options = options._merge_options(other_options)
		self.assertEquals(merged_options, {'foo':1, 'bar':3})

def get_fake_interface():
	interface = Mock()
	interface.hooks.listeners.keys = Mock(return_value=[])
	interface.hooks.listeners.items = Mock(return_value=[])
	interface.entity.get_link = lambda x: True
	return interface

class TestAPI(unittest.TestCase):

	def test_standard_options(self):
		api = API(None)
		self.assertTrue(hasattr(api, 'bypass_authorization'))

	def test_getattr(self):
		api = API(None)
		api.interfaces['foo'] = get_fake_interface()
		interface_proxy = api.foo
		self.assertIsInstance(interface_proxy, InterfaceProxy)
		self.assertEquals(interface_proxy._interface, api.interfaces['foo'])
		self.assertEquals(interface_proxy._api_options, api._options)

	def test_getitem(self):
		api = API(None)
		api.__getattr__ = Mock()
		api['foo']
		api.__getattr__.assert_called_once_with('foo')

class TestInterfaceProxy(unittest.TestCase):

	def test_options(self):
		interface = get_fake_interface()
		interface_proxy = InterfaceProxy(interface)
		self.assertTrue(hasattr(interface_proxy, 'fields'))
		self.assertTrue(hasattr(interface_proxy, 'embed'))
		self.assertTrue(hasattr(interface_proxy, 'bypass_authorization'))
		self.assertTrue(hasattr(interface_proxy, 'show_hidden'))

	def test_save(self):
		interface = get_fake_interface()
		interface.replace = Mock(return_value='replace')
		interface.create = Mock(return_value='create')
		interface_proxy = InterfaceProxy(interface)
		interface_proxy.show_hidden(True)

		create_result = interface_proxy.save({})
		self.assertEquals(create_result, 'create')
		interface.create.assert_called_once_with({}, show_hidden=True)

		replace_result = interface_proxy.save({'_id':'123'})
		self.assertEquals(replace_result, 'replace')
		interface.replace.assert_called_once_with('123', {'_id':'123'}, show_hidden=True)

	def test_update(self):
		interface = get_fake_interface()
		interface.update = Mock(return_value='update')
		interface_proxy = InterfaceProxy(interface)
		interface_proxy.show_hidden(True)

		result = interface_proxy.update('123', {'foo':'bar'})
		self.assertEquals(result, 'update')
		interface.update.assert_called_once_with('123', {'foo':'bar'}, show_hidden=True)

	def test_delete(self):
		interface = get_fake_interface()
		interface.delete = Mock()
		interface_proxy = InterfaceProxy(interface)
		interface_proxy.show_hidden(True)

		result = interface_proxy.delete('123')
		self.assertEquals(result, interface_proxy)
		interface.delete.assert_called_once_with('123', show_hidden=True)

	def test_get(self):
		interface = get_fake_interface()
		interface.get = Mock(return_value=1)
		interface_proxy = InterfaceProxy(interface)
		interface_proxy.show_hidden(True)

		result = interface_proxy.get('123')
		self.assertEquals(result, 1)
		interface.get.assert_called_once_with('123', show_hidden=True)

		interface.list = Mock(return_value=[2])
		result = interface_proxy.get({'foo':'bar'})
		self.assertEquals(result, 2)
		interface.list.assert_called_once_with(filter={'foo':'bar'}, limit=1, show_hidden=True)

		interface.list = Mock(return_value=[])
		with self.assertRaises(errors.NotFoundError):
			interface_proxy.get({'foo':'bar'})

	def test_find(self):
		interface = get_fake_interface()
		interface_proxy = InterfaceProxy(interface)
		result = interface_proxy.find({'foo':'bar'})
		self.assertIsInstance(result, FilterProxy)
		self.assertEquals(result._interface, interface)
		self.assertEquals(result._base_options, interface_proxy._options)
		self.assertEquals(result._options['filter'], {'foo':'bar'})

	def test_link(self):
		interface = get_fake_interface()
		interface_proxy = InterfaceProxy(interface)
		result = interface_proxy.related_things('foo')
		self.assertIsInstance(result, FilterProxy)
		self.assertIsInstance(result, LinkProxy)
		self.assertEquals(result._name, 'related_things')
		self.assertEquals(result._id, 'foo')

	def test_hooks(self):
		@api.foos.before_create
		def before_create(fields, context):
			print 'Stuff!!!'
			fields['foo'] = 'bar'

		fields = {}
		api.foos.save(fields)

		self.assertEquals(fields, {'foo':'bar'})

class TestFilterProxy(unittest.TestCase):

	def test_options(self):
		interface = get_fake_interface()
		filter_proxy = FilterProxy(interface, {}, {})
		self.assertTrue(hasattr(filter_proxy, 'fields'))
		self.assertTrue(hasattr(filter_proxy, 'embed'))
		self.assertTrue(hasattr(filter_proxy, 'sort'))
		self.assertTrue(hasattr(filter_proxy, 'offset'))
		self.assertTrue(hasattr(filter_proxy, 'limit'))
		self.assertTrue(hasattr(filter_proxy, 'bypass_authorization'))
		self.assertTrue(hasattr(filter_proxy, 'show_hidden'))
		self.assertTrue(hasattr(filter_proxy, 'filter'))

	def test_iter(self):
		interface = get_fake_interface()
		interface.list = Mock(return_value=[1,2,3])
		filter_proxy = FilterProxy(interface, {}, {'foo':'bar'})
		filter_proxy.show_hidden(True)
		result = list(iter(filter_proxy))
		self.assertEquals(result, [1,2,3])
		interface.list.assert_called_once_with(filter={'foo':'bar'}, show_hidden=True)

	def test_count(self):
		interface = get_fake_interface()
		interface.list = Mock(return_value=42)
		filter_proxy = FilterProxy(interface, {}, {'foo':'bar'})
		filter_proxy.show_hidden(True)
		result = filter_proxy.count()
		self.assertEquals(result, 42)
		interface.list.assert_called_once_with(filter={'foo':'bar'}, show_hidden=True, count=True)

	def test_len(self):
		filter_proxy = FilterProxy(None, {}, {})
		filter_proxy.count = Mock(return_value=33)
		result = len(filter_proxy)
		self.assertEquals(result, 33)
		filter_proxy.count.assert_called_once()

	def test_contains(self):
		interface = get_fake_interface()
		interface.list = Mock(return_value=[])
		filter_proxy = FilterProxy(interface, {}, {'foo':'bar'})
		filter_proxy.show_hidden(True)

		result = '123' in filter_proxy
		self.assertEquals(result, False)
		interface.list.assert_called_once_with(filter={'_id':'123', 'foo':'bar'}, limit=1, show_hidden=True)

		interface.list = Mock(return_value=[1])
		result = '123' in filter_proxy
		self.assertEquals(result, True)

		interface.list = Mock(return_value=[1])
		result = {'_id':'123'} in filter_proxy
		self.assertEquals(result, True)
		interface.list.assert_called_once_with(filter={'_id':'123', 'foo':'bar'}, limit=1, show_hidden=True)

class TestLinkProxy(unittest.TestCase):

	def test_list(self):
		interface = get_fake_interface()
		interface.link = Mock(return_value=[1,2,3])
		link_proxy = LinkProxy(interface, {}, 'related_things', 'an_id')
		result = link_proxy.list()
		self.assertEquals(result, [1,2,3])
		interface.link.assert_called_once_with('an_id', 'related_things', filter={})<commit_msg>Added test for Interface create() proxy function in API<commit_after>import unittest
from mock import Mock
from cellardoor import errors
from cellardoor.api import API, StandardOptionsMixin, InterfaceProxy, FilterProxy, LinkProxy
from cellardoor.api.methods import ALL
from cellardoor.storage import Storage
from cellardoor.model import Model, Entity
from cellardoor.errors import NotFoundError

storage = Storage()
storage.create = Mock()
model = Model(storage=storage)
api = API(model)

class Foo(model.Entity):
	pass

class Foos(api.Interface):
	entity = Foo
	method_authorization = {
		ALL: None
	}

class TestStandardOptionsMixin(unittest.TestCase):

	def test_has_options(self):
		options = StandardOptionsMixin()
		self.assertTrue(hasattr(options, '_options'))
		self.assertEquals(options._options, {})

	def test_identity(self):
		options = StandardOptionsMixin()
		self.assertTrue(hasattr(options, 'identity'))
		self.assertEquals(options.identity(), None)
		result = options.identity('foo')
		self.assertEquals(result, options)
		self.assertEquals(options.identity(), 'foo')
		self.assertEquals(options._options, {'context':{'identity':'foo'}})

	def test_create_accessor(self):
		options = StandardOptionsMixin('foo', 'bar')
		self.assertTrue(hasattr(options, 'foo'))
		self.assertTrue(hasattr(options, 'bar'))
		result = options.foo(23)
		self.assertEquals(result, options)
		self.assertEquals(options.foo(), 23)

	def test_merge(self):
		options = StandardOptionsMixin('foo')
		options.foo(1)
		other_options = {'foo':2, 'bar':3}
		merged_options = options._merge_options(other_options)
		self.assertEquals(merged_options, {'foo':1, 'bar':3})

def get_fake_interface():
	interface = Mock()
	interface.hooks.listeners.keys = Mock(return_value=[])
	interface.hooks.listeners.items = Mock(return_value=[])
	interface.entity.get_link = lambda x: True
	return interface

class TestAPI(unittest.TestCase):

	def test_standard_options(self):
		api = API(None)
		self.assertTrue(hasattr(api, 'bypass_authorization'))

	def test_getattr(self):
		api = API(None)
		api.interfaces['foo'] = get_fake_interface()
		interface_proxy = api.foo
		self.assertIsInstance(interface_proxy, InterfaceProxy)
		self.assertEquals(interface_proxy._interface, api.interfaces['foo'])
		self.assertEquals(interface_proxy._api_options, api._options)

	def test_getitem(self):
		api = API(None)
		api.__getattr__ = Mock()
		api['foo']
		api.__getattr__.assert_called_once_with('foo')

class TestInterfaceProxy(unittest.TestCase):

	def test_options(self):
		interface = get_fake_interface()
		interface_proxy = InterfaceProxy(interface)
		self.assertTrue(hasattr(interface_proxy, 'fields'))
		self.assertTrue(hasattr(interface_proxy, 'embed'))
		self.assertTrue(hasattr(interface_proxy, 'bypass_authorization'))
		self.assertTrue(hasattr(interface_proxy, 'show_hidden'))

	def test_create(self):
		interface = get_fake_interface()
		interface.create = Mock(return_value='create')
		interface_proxy = InterfaceProxy(interface)
		interface_proxy.show_hidden(True)

		result = interface_proxy.create({'foo':'bar'})
		self.assertEquals(result, 'create')
		interface.create.assert_called_once_with({'foo':'bar'}, show_hidden=True)

	def test_save(self):
		interface = get_fake_interface()
		interface.replace = Mock(return_value='replace')
		interface.create = Mock(return_value='create')
		interface_proxy = InterfaceProxy(interface)
		interface_proxy.show_hidden(True)

		create_result = interface_proxy.save({})
		self.assertEquals(create_result, 'create')
		interface.create.assert_called_once_with({}, show_hidden=True)

		replace_result = interface_proxy.save({'_id':'123'})
		self.assertEquals(replace_result, 'replace')
		interface.replace.assert_called_once_with('123', {'_id':'123'}, show_hidden=True)

	def test_update(self):
		interface = get_fake_interface()
		interface.update = Mock(return_value='update')
		interface_proxy = InterfaceProxy(interface)
		interface_proxy.show_hidden(True)

		result = interface_proxy.update('123', {'foo':'bar'})
		self.assertEquals(result, 'update')
		interface.update.assert_called_once_with('123', {'foo':'bar'}, show_hidden=True)

	def test_delete(self):
		interface = get_fake_interface()
		interface.delete = Mock()
		interface_proxy = InterfaceProxy(interface)
		interface_proxy.show_hidden(True)

		result = interface_proxy.delete('123')
		self.assertEquals(result, interface_proxy)
		interface.delete.assert_called_once_with('123', show_hidden=True)

	def test_get(self):
		interface = get_fake_interface()
		interface.get = Mock(return_value=1)
		interface_proxy = InterfaceProxy(interface)
		interface_proxy.show_hidden(True)

		result = interface_proxy.get('123')
		self.assertEquals(result, 1)
		interface.get.assert_called_once_with('123', show_hidden=True)

		interface.list = Mock(return_value=[2])
		result = interface_proxy.get({'foo':'bar'})
		self.assertEquals(result, 2)
		interface.list.assert_called_once_with(filter={'foo':'bar'}, limit=1, show_hidden=True)

		interface.list = Mock(return_value=[])
		with self.assertRaises(errors.NotFoundError):
			interface_proxy.get({'foo':'bar'})

	def test_find(self):
		interface = get_fake_interface()
		interface_proxy = InterfaceProxy(interface)
		result = interface_proxy.find({'foo':'bar'})
		self.assertIsInstance(result, FilterProxy)
		self.assertEquals(result._interface, interface)
		self.assertEquals(result._base_options, interface_proxy._options)
		self.assertEquals(result._options['filter'], {'foo':'bar'})

	def test_link(self):
		interface = get_fake_interface()
		interface_proxy = InterfaceProxy(interface)
		result = interface_proxy.related_things('foo')
		self.assertIsInstance(result, FilterProxy)
		self.assertIsInstance(result, LinkProxy)
		self.assertEquals(result._name, 'related_things')
		self.assertEquals(result._id, 'foo')

	def test_hooks(self):
		@api.foos.before_create
		def before_create(fields, context):
			print 'Stuff!!!'
			fields['foo'] = 'bar'

		fields = {}
		api.foos.save(fields)

		self.assertEquals(fields, {'foo':'bar'})

class TestFilterProxy(unittest.TestCase):

	def test_options(self):
		interface = get_fake_interface()
		filter_proxy = FilterProxy(interface, {}, {})
		self.assertTrue(hasattr(filter_proxy, 'fields'))
		self.assertTrue(hasattr(filter_proxy, 'embed'))
		self.assertTrue(hasattr(filter_proxy, 'sort'))
		self.assertTrue(hasattr(filter_proxy, 'offset'))
		self.assertTrue(hasattr(filter_proxy, 'limit'))
		self.assertTrue(hasattr(filter_proxy, 'bypass_authorization'))
		self.assertTrue(hasattr(filter_proxy, 'show_hidden'))
		self.assertTrue(hasattr(filter_proxy, 'filter'))

	def test_iter(self):
		interface = get_fake_interface()
		interface.list = Mock(return_value=[1,2,3])
		filter_proxy = FilterProxy(interface, {}, {'foo':'bar'})
		filter_proxy.show_hidden(True)
		result = list(iter(filter_proxy))
		self.assertEquals(result, [1,2,3])
		interface.list.assert_called_once_with(filter={'foo':'bar'}, show_hidden=True)

	def test_count(self):
		interface = get_fake_interface()
		interface.list = Mock(return_value=42)
		filter_proxy = FilterProxy(interface, {}, {'foo':'bar'})
		filter_proxy.show_hidden(True)
		result = filter_proxy.count()
		self.assertEquals(result, 42)
		interface.list.assert_called_once_with(filter={'foo':'bar'}, show_hidden=True, count=True)

	def test_len(self):
		filter_proxy = FilterProxy(None, {}, {})
		filter_proxy.count = Mock(return_value=33)
		result = len(filter_proxy)
		self.assertEquals(result, 33)
		filter_proxy.count.assert_called_once()

	def test_contains(self):
		interface = get_fake_interface()
		interface.list = Mock(return_value=[])
		filter_proxy = FilterProxy(interface, {}, {'foo':'bar'})
		filter_proxy.show_hidden(True)

		result = '123' in filter_proxy
		self.assertEquals(result, False)
		interface.list.assert_called_once_with(filter={'_id':'123', 'foo':'bar'}, limit=1, show_hidden=True)

		interface.list = Mock(return_value=[1])
		result = '123' in filter_proxy
		self.assertEquals(result, True)

		interface.list = Mock(return_value=[1])
		result = {'_id':'123'} in filter_proxy
		self.assertEquals(result, True)
		interface.list.assert_called_once_with(filter={'_id':'123', 'foo':'bar'}, limit=1, show_hidden=True)

class TestLinkProxy(unittest.TestCase):

	def test_list(self):
		interface = get_fake_interface()
		interface.link = Mock(return_value=[1,2,3])
		link_proxy = LinkProxy(interface, {}, 'related_things', 'an_id')
		result = link_proxy.list()
		self.assertEquals(result, [1,2,3])
		interface.link.assert_called_once_with('an_id', 'related_things', filter={})<|endoftext|>


<commit_before>import discord
import time
import twitter
import os
from lxml import html
import requests
import datetime
import poker
import copy
import threading

card_sort = {'A':1,'2':2,'3':3,'4':4,'5':5,'6':6,'7':7,'8':8,'9':9,'10':10,'J':11,'Q':12,'K':13}

participants = []
in_progress = 0

my_user_id = '<@110629657417633792>'
log_channel_id = '91029061601615872'
poker_channel_id = '111655731958161408'

last_joke_time = datetime.datetime.now()

def fetch_latest_tweet(the_client, the_message):
    api = twitter.Api(consumer_key=os.environ['twitter_consumer_key'],
                      consumer_secret=os.environ['twitter_consumer_secret'],
                      access_token_key=os.environ['twitter_access_token_key'],
                      access_token_secret=os.environ['twitter_access_token_secret'])
    tweet = api.GetUserTimeline(screen_name='ftwaegwynn',count=1)[0]
    the_client.send_message(the_message.channel,
"""The most recent FTWAegwynn tweet is: %s

Read it at http://twitter.com/ftwaegwynn/status/%s.""" % (tweet.text, tweet.id))

client = discord.Client()
client.login(os.environ['discord_user'],os.environ['discord_password'])

def fetch_latest_logs(the_client, the_message):
    page = requests.get('http://www.warcraftlogs.com/guilds/recent_reports/79198/')
    tree = html.fromstring(page.text)
    wol = tree.xpath('/html/body/div/table/tbody/tr[1]/td[1]/a')[0]

    the_client.send_message(log_channel_id,
"""Here are the WCL for tonight's %s
%s""" % (wol.text, wol.attrib['href']))

    the_client.send_message(the_message.author, 'Logs are posted in #logs-raid.')

def execute_poker(the_client, the_message):
    global participants, in_progress
    rolls = poker.poker(participants)

    announcement = ''

    for player in rolls:
        announcement += player + ':     ' + str.join('     ',sorted(rolls[player], key=lambda x: card_sort[x[0]])) + '\n'

#    the_client.send_message(the_message.channel, 'Join us in #poker to see the results.')
    the_client.send_message(the_message.channel, announcement)

    in_progress = 0
    participants = []

    return rolls

@client.event
def on_message(message):
    if message.content.startswith('!joke'):
        global last_joke_time
        if last_joke_time - datetime.datetime.now() > datetime.timedelta(minutes=1):
            client.send_message(message.channel, 'What side of a turkey has the most feathers?')
            time.sleep(0.5)
            client.send_message(message.channel, 'The outside!!! LMAO')
            last_joke_time = datetime.datetime.now()

    elif message.content.startswith('!hello'):
        client.send_message(message.channel, 'Hello to you, {}.'.format(message.author.mention()))

    elif message.content.startswith('!fail'):
        client.send_message(message.channel, 'You are fail, {}.'.format(message.author.mention()))

    elif message.content.startswith('!poker'):
        global in_progress, participants
        if in_progress == 0:
            client.send_message(message.channel, 'Poker started by %s. If you would like to join, type !poker within 15 seconds.' % (message.author.mention(),))
            in_progress = 1
            threading.Timer(15.0, execute_poker, args=[client, message]).start()
            participants.append(message.author.name)
        elif in_progress == 1 and message.author.name not in participants and len(participants) < 11:
            participants.append(message.author.name)

    elif message.content.startswith('!twitter'):
        try:
            fetch_latest_tweet(client, message)
        except:
            client.send_message(message.channel, 'Sorry, I am not able to get a tweet right now.')

    elif message.content.startswith('!logs') and message.author.roles[0].name == 'Officer':
        try:
            fetch_latest_logs(client, message)
        except:
            client.send.message(message.channel, 'Sorry, I am unable to get logs right now.')

    elif message.content.startswith(my_user_id) or message.content.startswith('@forthewynnbot'):
        if 'help' in message.content:
            client.send_message(message.author,
"""Hi there and welcome to forthewynnbot. It was constructed by Emann.
The following commands are available for use:

    !fail: tells you how awesome you are
    !hello: greets you
    !joke: tells a very funny joke
    !logs: posts the latest Warcraft Logs in #logs-raid if you are an Officer
    !poker: initiates a poker game (not implemented yet)
    !twitter: retrieves the latest FTWAegwynn tweet from Twitter

Commands you can direct at me:

    @forthewynnbot help: displays this message
    @forthewynnbot stop: turns me off for 5 minutes
""")
        elif 'stop' in message.content:
            client.send_message(message.channel, 'Shutting down for 5 minutes.')
            time.sleep(300.0)
        else:
            client.send_message(message.channel, 'Sorry, I do not understand that command.')

client.run()
<commit_msg>Added ping/pong<commit_after>import discord
import time
import twitter
import os
from lxml import html
import requests
import datetime
import poker
import copy
import threading

card_sort = {'A':1,'2':2,'3':3,'4':4,'5':5,'6':6,'7':7,'8':8,'9':9,'10':10,'J':11,'Q':12,'K':13}

participants = []
in_progress = 0

my_user_id = '<@110629657417633792>'
log_channel_id = '91029061601615872'
poker_channel_id = '111655731958161408'

last_joke_time = datetime.datetime.now()

def fetch_latest_tweet(the_client, the_message):
    api = twitter.Api(consumer_key=os.environ['twitter_consumer_key'],
                      consumer_secret=os.environ['twitter_consumer_secret'],
                      access_token_key=os.environ['twitter_access_token_key'],
                      access_token_secret=os.environ['twitter_access_token_secret'])
    tweet = api.GetUserTimeline(screen_name='ftwaegwynn',count=1)[0]
    the_client.send_message(the_message.channel,
"""The most recent FTWAegwynn tweet is: %s

Read it at http://twitter.com/ftwaegwynn/status/%s.""" % (tweet.text, tweet.id))

client = discord.Client()
client.login(os.environ['discord_user'],os.environ['discord_password'])

def fetch_latest_logs(the_client, the_message):
    page = requests.get('http://www.warcraftlogs.com/guilds/recent_reports/79198/')
    tree = html.fromstring(page.text)
    wol = tree.xpath('/html/body/div/table/tbody/tr[1]/td[1]/a')[0]

    the_client.send_message(log_channel_id,
"""Here are the WCL for tonight's %s
%s""" % (wol.text, wol.attrib['href']))

    the_client.send_message(the_message.author, 'Logs are posted in #logs-raid.')

def execute_poker(the_client, the_message):
    global participants, in_progress
    rolls = poker.poker(participants)

    announcement = ''

    for player in rolls:
        announcement += player + ':     ' + str.join('     ',sorted(rolls[player], key=lambda x: card_sort[x[0]])) + '\n'

#    the_client.send_message(the_message.channel, 'Join us in #poker to see the results.')
    the_client.send_message(the_message.channel, announcement)

    in_progress = 0
    participants = []

    return rolls

@client.event
def on_message(message):
    if message.content.startswith('!joke'):
        global last_joke_time
        if last_joke_time - datetime.datetime.now() > datetime.timedelta(minutes=1):
            client.send_message(message.channel, 'What side of a turkey has the most feathers?')
            time.sleep(0.5)
            client.send_message(message.channel, 'The outside!!! LMAO')
            last_joke_time = datetime.datetime.now()

    elif message.content.startswith('!hello'):
        client.send_message(message.channel, 'Hello to you, {}.'.format(message.author.mention()))

    elif message.content.startswith('!fail'):
        client.send_message(message.channel, 'You are fail, {}.'.format(message.author.mention()))

    elif message.content.startswith('!ping'):
        client.send_message(message.channel, 'Pong back atcha, %s.' % (message.author.mention(),))

    elif message.content.startswith('!poker'):
        global in_progress, participants
        if in_progress == 0:
            client.send_message(message.channel, 'Poker started by %s. If you would like to join, type !poker within 15 seconds.' % (message.author.mention(),))
            in_progress = 1
            threading.Timer(15.0, execute_poker, args=[client, message]).start()
            participants.append(message.author.name)
        elif in_progress == 1 and message.author.name not in participants and len(participants) < 11:
            participants.append(message.author.name)

    elif message.content.startswith('!twitter'):
        try:
            fetch_latest_tweet(client, message)
        except:
            client.send_message(message.channel, 'Sorry, I am not able to get a tweet right now.')

    elif message.content.startswith('!logs') and message.author.roles[0].name == 'Officer':
        try:
            fetch_latest_logs(client, message)
        except:
            client.send.message(message.channel, 'Sorry, I am unable to get logs right now.')

    elif message.content.startswith(my_user_id) or message.content.startswith('@forthewynnbot'):
        if 'help' in message.content:
            client.send_message(message.author,
"""Hi there and welcome to forthewynnbot. It was constructed by Emann.
The following commands are available for use:

    !fail: tells you how awesome you are
    !hello: greets you
    !joke: tells a very funny joke
    !logs: posts the latest Warcraft Logs in #logs-raid if you are an Officer
    !poker: initiates a poker game (not implemented yet)
    !twitter: retrieves the latest FTWAegwynn tweet from Twitter

Commands you can direct at me:

    @forthewynnbot help: displays this message
    @forthewynnbot stop: turns me off for 5 minutes
""")
        elif 'stop' in message.content:
            client.send_message(message.channel, 'Shutting down for 5 minutes.')
            time.sleep(300.0)
        else:
            client.send_message(message.channel, 'Sorry, I do not understand that command.')

client.run()
<|endoftext|>


<commit_before>import unittest
import json

import responses

from pyshk.api import Api
from pyshk import models
from pyshk import errors

class ApiTests(unittest.TestCase):

    """ Test various API functions """

    def test_api_creation(self):
        a = Api(consumer_key='test',
                consumer_secret='test',
                access_token_key='test',
                access_token_secret='supersecret')
        self.assertEqual(a.base_url, 'http://mlkshk.com')
        a = Api(base_url='http://example.com')
        self.assertEqual(a.base_url, 'http://example.com')

    def test_api_unauthorized(self):
        a = Api(consumer_key='test', consumer_secret='test')
        self.assertRaises(errors.ApiInstanceUnauthorized, a.get_user)

    # @responses.activate
    # def test_api_auth(self):
    #     auth_url = (
    #         'https://mlkshk.com/api/authorize?response_type='
    #         'code&client_id=test&redirect_uri=http://localhost:8000')

    #     with open('tests/test_data/api/authorize') as f:
    #         resp_data = f.read()
    #         responses.add(
    #             responses.POST,
    #             auth_url,
    #             body=resp_data,
    #             status_code=200)

    #         a = Api(
    #             consumer_key='test',
    #             consumer_key='test',
    #             redirect_uri='http://localhost:8000')
    #         req = a.get_auth()

class TestAPIResponses(unittest.TestCase):

    def setUp(self):
        self.api = Api(
            consumer_key='test',
            consumer_secret='test',
            access_token_key='test',
            access_token_secret='test')

    @responses.activate
    def test_get_user(self):
        with open('tests/test_data/api/user') as f:
            resp_data = f.read()

            responses.add(
                responses.GET,
                'http://mlkshk.com/api/user',
                body=resp_data,
                status=200)
            user = self.api.get_user()
        self.assertEqual(user.name, 'jcbl')

        # By user_id
        with open('tests/test_data/api/user_id/67136') as f:
            resp_data = f.read()

            responses.add(
                responses.GET,
                'http://mlkshk.com/api/user_id/67136',
                body=resp_data,
                status=200)
            user = self.api.get_user(user_id=67136)
        self.assertEqual(user.name, 'jcbl')

        # By user_name
        with open('tests/test_data/api/user_name/jcbl') as f:
            resp_data = f.read()

            responses.add(
                responses.GET,
                'http://mlkshk.com/api/user_name/jcbl',
                body=resp_data,
                status=200)
            user = self.api.get_user(user_name='jcbl')
        self.assertEqual(user.name, 'jcbl')

    @responses.activate
    def test_get_authd_user_shakes(self):
        with open('tests/test_data/api/shakes') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/shakes',
            body=resp_data,
            status=200)
        shakes = self.api.get_user_shakes()
        self.assertEqual(len(shakes), 1)

    @responses.activate
    def test_get_sharedfiles_from_shake(self):
        with open('tests/test_data/api/shakes.59884') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/shakes/59884',
            body=resp_data,
            status=200)
        files = self.api.get_shared_files_from_shake(shake_id=59884)
        self.assertEqual(len(files), 10)

        with open('tests/test_data/api/sharedfile/16509') as f:
            sharedfile = models.SharedFile.NewFromJSON(json.load(f))

        self.assertTrue(sharedfile == files[0])

    @responses.activate
    def test_upload(self):
        with open('tests/test_data/api/upload') as f:
            resp_data = f.read().encode('utf8')

            responses.add(
                responses.POST,
                'http://mlkshk.com/api/upload',
                body=resp_data,
                status=200
            )
            resp = self.api.post_shared_file(
                image_file='tests/test_data/like-tiny.gif')
        expected_resp = {"share_key": "164L4", "name": "like-tiny.gif"}
        self.assertEqual(resp['share_key'], expected_resp['share_key'])

    def test_headers(self):
        test_headers = self.api._make_headers(
            nonce='bcbd15da3e38847ebad5655231912ac8',
            timestamp=1444770686)
        std_headers = (
            'MAC token="test", timestamp="1444770686", '
            'nonce="bcbd15da3e38847ebad5655231912ac8", '
            'signature="vdjjGj07papwf0H28Nkc3SbWuUQ="')
        self.assertEqual(test_headers, std_headers)

    @responses.activate
    def test_get_shared_file(self):
        with open('tests/test_data/api/sharedfile/164DW') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/sharedfile/164DW',
            body=resp_data,
            status=200
        )
        sf = self.api.get_shared_file(sharekey='164DW')
        sf.user = None
        sf2 = models.SharedFile(
            comments=0,
            pivot_id="164DW",
            title="Shakeys",
            name="tumblr_ntg61qKrOe1rtr67io1_500.jpg",
            posted_at="2015-10-10T15:14:30Z",
            like=False,
            saved=False,
            permalink_page="http://mlkshk.com/p/164DW",
            nsfw=False,
            height=645,
            likes=1,
            description=None,
            sharekey="164DW",
            original_image_url="http://s.mlkshk.com/r/164DW",
            saves=0,
            width=500,
            views=0)
        self.assertTrue(sf == sf2)

    @responses.activate
    def test_like_shared_file(self):
        with open('tests/test_data/api/sharedfile/164DW') as f:
            resp_data = f.read()

        responses.add(
            responses.POST,
            'http://mlkshk.com/api/sharedfile/164DW/like',
            body=resp_data,
            status=200)
        sf = self.api.like_shared_file(sharekey='164DW')

        self.assertTrue(sf.liked)
        self.assertRaises(Exception, lambda: self.api.like_shared_file())

    @responses.activate
    def test_save_shared_file(self):
        with open('tests/test_data/api/sharedfile/165PZ/save') as f:
            resp_data = f.read()

        responses.add(
            responses.POST,
            'http://mlkshk.com/api/sharedfile/165PZ/save',
            body=resp_data,
            status=200)

        sf = self.api.save_shared_file(sharekey='165PZ')
        self.assertTrue(sf.saved)
        self.assertEqual(sf.user.id, 14430)

    @responses.activate
    def test_get_favorites(self):
        with open('tests/test_data/api/favorites') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/favorites',
            body=resp_data,
            status=200)
        favs = self.api.get_favorites()

        self.assertEqual(len(favs), 10)
        self.assertIsInstance(favs[0], models.SharedFile)

    @responses.activate
    def test_get_friends_shake(self):
        with open('tests/test_data/api/friends') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/friends',
            body=resp_data,
            status=200
        )
        friends_shake = self.api.get_friends_shake()
        self.assertIsInstance(friends_shake[0], models.SharedFile)
        self.assertIsInstance(friends_shake, list)

    @responses.activate
    def test_get_incoming_shake(self):
        with open('tests/test_data/api/incoming') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/incoming',
            body=resp_data,
            status=200
        )
        incoming_shake = self.api.get_incoming_shake()
        self.assertIsInstance(incoming_shake[0], models.SharedFile)
        self.assertIsInstance(incoming_shake, list)

    @responses.activate
    def test_get_magic_files(self):
        with open('tests/test_data/api/magicfiles') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/magicfiles',
            body=resp_data,
            status=200
        )
        magic_files = self.api.get_magic_shake()
        self.assertIsInstance(magic_files[0], models.SharedFile)
        self.assertIsInstance(magic_files, list)

    @responses.activate
    def test_get_magic_files_after(self):
        with open('tests/test_data/api/magicfiles.after') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/magicfiles/after/1643X',
            body=resp_data,
            status=200
        )
        magic_files = self.api.get_magic_shake(after='1643X')
        self.assertIsInstance(magic_files[0], models.SharedFile)
        self.assertEqual(magic_files[0].sharekey, 'CO04')
        self.assertIsInstance(magic_files, list)
        self.assertEqual(len(magic_files), 10)

    @responses.activate
    def test_get_magic_files_before(self):
        with open('tests/test_data/api/magicfiles.before') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/magicfiles/before/1643X',
            body=resp_data,
            status=200
        )
        magic_files = self.api.get_magic_shake(before='1643X')
        self.assertIsInstance(magic_files[0], models.SharedFile)
        self.assertEqual(magic_files[0].sharekey, 'CO0E')
        self.assertIsInstance(magic_files, list)
        self.assertEqual(len(magic_files), 10)

    @responses.activate
    def test_get_user_shakes(self):
        with open('tests/test_data/api/shakes') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/shakes',
            body=resp_data,
            status=200
        )
        shakes = self.api.get_user_shakes()
        self.assertIsInstance(shakes[0], models.Shake)
        self.assertIsInstance(shakes[0].owner, models.User)
        self.assertEqual(len(shakes), 1)

    @responses.activate
    def test_update_shared_file(self):
        with open('tests/test_data/api/sharedfile/1669X.POST') as f:
            resp_data = f.read()

        responses.add(
            responses.POST,
            'http://mlkshk.com/api/sharedfile/1669X',
            body=resp_data,
            status=200
        )

        sf = self.api.update_shared_file(sharekey='1669X', description='test')

        self.assertIsInstance(sf, models.SharedFile)
        self.assertEqual(sf.description, 'test')

        with open('tests/test_data/api/sharedfile/1669X.POST.title') as f:
            resp_data = f.read()

        responses.add(
            responses.POST,
            'http://mlkshk.com/api/sharedfile/1669X',
            body=resp_data,
            status=200
        )
        sf = self.api.update_shared_file(sharekey='1669X',
                                         title='journalism')
        self.assertIsInstance(sf, models.SharedFile)
        self.assertEqual(sf.title, 'journalism')

        self.assertRaises(
            Exception,
            lambda: self.api.update_shared_file(sharekey='1669X'))
        self.assertRaises(
            Exception,
            lambda: self.api.update_shared_file())

    @responses.activate
    def test_get_comments(self):
        with open('tests/test_data/api/sharedfile/162LG/comments') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/sharedfile/162LG/comments',
            body=resp_data,
            status=200
        )
        comments = self.api.get_comments(sharekey='162LG')
        self.assertEqual(len(comments), 4)
        self.assertIsInstance(comments[0], models.Comment)
        self.assertEqual(comments[0].body, '!!!!!!!')
<commit_msg>adds exception tests for a couple functions<commit_after>import unittest
import json

import responses

from pyshk.api import Api
from pyshk import models
from pyshk import errors

class ApiTests(unittest.TestCase):

    """ Test various API functions """

    def test_api_creation(self):
        a = Api(consumer_key='test',
                consumer_secret='test',
                access_token_key='test',
                access_token_secret='supersecret')
        self.assertEqual(a.base_url, 'http://mlkshk.com')
        a = Api(base_url='http://example.com')
        self.assertEqual(a.base_url, 'http://example.com')

    def test_api_unauthorized(self):
        a = Api(consumer_key='test', consumer_secret='test')
        self.assertRaises(errors.ApiInstanceUnauthorized, a.get_user)

    # @responses.activate
    # def test_api_auth(self):
    #     auth_url = (
    #         'https://mlkshk.com/api/authorize?response_type='
    #         'code&client_id=test&redirect_uri=http://localhost:8000')

    #     with open('tests/test_data/api/authorize') as f:
    #         resp_data = f.read()
    #         responses.add(
    #             responses.POST,
    #             auth_url,
    #             body=resp_data,
    #             status_code=200)

    #         a = Api(
    #             consumer_key='test',
    #             consumer_key='test',
    #             redirect_uri='http://localhost:8000')
    #         req = a.get_auth()

class TestAPIResponses(unittest.TestCase):

    def setUp(self):
        self.api = Api(
            consumer_key='test',
            consumer_secret='test',
            access_token_key='test',
            access_token_secret='test')

    @responses.activate
    def test_get_user(self):
        with open('tests/test_data/api/user') as f:
            resp_data = f.read()

            responses.add(
                responses.GET,
                'http://mlkshk.com/api/user',
                body=resp_data,
                status=200)
            user = self.api.get_user()
        self.assertEqual(user.name, 'jcbl')

        # By user_id
        with open('tests/test_data/api/user_id/67136') as f:
            resp_data = f.read()

            responses.add(
                responses.GET,
                'http://mlkshk.com/api/user_id/67136',
                body=resp_data,
                status=200)
            user = self.api.get_user(user_id=67136)
        self.assertEqual(user.name, 'jcbl')

        # By user_name
        with open('tests/test_data/api/user_name/jcbl') as f:
            resp_data = f.read()

            responses.add(
                responses.GET,
                'http://mlkshk.com/api/user_name/jcbl',
                body=resp_data,
                status=200)
            user = self.api.get_user(user_name='jcbl')
        self.assertEqual(user.name, 'jcbl')

    @responses.activate
    def test_get_authd_user_shakes(self):
        with open('tests/test_data/api/shakes') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/shakes',
            body=resp_data,
            status=200)
        shakes = self.api.get_user_shakes()
        self.assertEqual(len(shakes), 1)

    @responses.activate
    def test_get_sharedfiles_from_shake(self):
        with open('tests/test_data/api/shakes.59884') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/shakes/59884',
            body=resp_data,
            status=200)
        files = self.api.get_shared_files_from_shake(shake_id=59884)
        self.assertEqual(len(files), 10)

        with open('tests/test_data/api/sharedfile/16509') as f:
            sharedfile = models.SharedFile.NewFromJSON(json.load(f))

        self.assertTrue(sharedfile == files[0])

        self.assertRaises(
            Exception,
            lambda: self.api.get_shared_files_from_shake(
                before='0001', after='0002'))

    @responses.activate
    def test_post_shared_file(self):
        with open('tests/test_data/api/upload') as f:
            resp_data = f.read().encode('utf8')

            responses.add(
                responses.POST,
                'http://mlkshk.com/api/upload',
                body=resp_data,
                status=200
            )
            resp = self.api.post_shared_file(
                image_file='tests/test_data/like-tiny.gif')
        expected_resp = {"share_key": "164L4", "name": "like-tiny.gif"}
        self.assertEqual(resp['share_key'], expected_resp['share_key'])
        self.assertRaises(
            Exception,
            lambda: self.api.post_shared_file(
                image_file='1.jpg',
                source_link='http://example.com'))

    def test_headers(self):
        test_headers = self.api._make_headers(
            nonce='bcbd15da3e38847ebad5655231912ac8',
            timestamp=1444770686)
        std_headers = (
            'MAC token="test", timestamp="1444770686", '
            'nonce="bcbd15da3e38847ebad5655231912ac8", '
            'signature="vdjjGj07papwf0H28Nkc3SbWuUQ="')
        self.assertEqual(test_headers, std_headers)

    @responses.activate
    def test_get_shared_file(self):
        with open('tests/test_data/api/sharedfile/164DW') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/sharedfile/164DW',
            body=resp_data,
            status=200
        )
        sf = self.api.get_shared_file(sharekey='164DW')
        sf.user = None
        sf2 = models.SharedFile(
            comments=0,
            pivot_id="164DW",
            title="Shakeys",
            name="tumblr_ntg61qKrOe1rtr67io1_500.jpg",
            posted_at="2015-10-10T15:14:30Z",
            like=False,
            saved=False,
            permalink_page="http://mlkshk.com/p/164DW",
            nsfw=False,
            height=645,
            likes=1,
            description=None,
            sharekey="164DW",
            original_image_url="http://s.mlkshk.com/r/164DW",
            saves=0,
            width=500,
            views=0)
        self.assertTrue(sf == sf2)

    @responses.activate
    def test_like_shared_file(self):
        with open('tests/test_data/api/sharedfile/164DW') as f:
            resp_data = f.read()

        responses.add(
            responses.POST,
            'http://mlkshk.com/api/sharedfile/164DW/like',
            body=resp_data,
            status=200)
        sf = self.api.like_shared_file(sharekey='164DW')

        self.assertTrue(sf.liked)
        self.assertRaises(Exception, lambda: self.api.like_shared_file())

    @responses.activate
    def test_save_shared_file(self):
        with open('tests/test_data/api/sharedfile/165PZ/save') as f:
            resp_data = f.read()

        responses.add(
            responses.POST,
            'http://mlkshk.com/api/sharedfile/165PZ/save',
            body=resp_data,
            status=200)

        sf = self.api.save_shared_file(sharekey='165PZ')
        self.assertTrue(sf.saved)
        self.assertEqual(sf.user.id, 14430)

    @responses.activate
    def test_get_favorites(self):
        with open('tests/test_data/api/favorites') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/favorites',
            body=resp_data,
            status=200)
        favs = self.api.get_favorites()

        self.assertEqual(len(favs), 10)
        self.assertIsInstance(favs[0], models.SharedFile)

    @responses.activate
    def test_get_friends_shake(self):
        with open('tests/test_data/api/friends') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/friends',
            body=resp_data,
            status=200
        )
        friends_shake = self.api.get_friends_shake()
        self.assertIsInstance(friends_shake[0], models.SharedFile)
        self.assertIsInstance(friends_shake, list)
        self.assertRaises(
            Exception,
            lambda: self.api.get_friends_shake(
                before='0001', after='0002'))

    @responses.activate
    def test_get_incoming_shake(self):
        with open('tests/test_data/api/incoming') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/incoming',
            body=resp_data,
            status=200
        )
        incoming_shake = self.api.get_incoming_shake()
        self.assertIsInstance(incoming_shake[0], models.SharedFile)
        self.assertIsInstance(incoming_shake, list)

    @responses.activate
    def test_get_magic_files(self):
        with open('tests/test_data/api/magicfiles') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/magicfiles',
            body=resp_data,
            status=200
        )
        magic_files = self.api.get_magic_shake()
        self.assertIsInstance(magic_files[0], models.SharedFile)
        self.assertIsInstance(magic_files, list)

    @responses.activate
    def test_get_magic_files_after(self):
        with open('tests/test_data/api/magicfiles.after') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/magicfiles/after/1643X',
            body=resp_data,
            status=200
        )
        magic_files = self.api.get_magic_shake(after='1643X')
        self.assertIsInstance(magic_files[0], models.SharedFile)
        self.assertEqual(magic_files[0].sharekey, 'CO04')
        self.assertIsInstance(magic_files, list)
        self.assertEqual(len(magic_files), 10)

    @responses.activate
    def test_get_magic_files_before(self):
        with open('tests/test_data/api/magicfiles.before') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/magicfiles/before/1643X',
            body=resp_data,
            status=200
        )
        magic_files = self.api.get_magic_shake(before='1643X')
        self.assertIsInstance(magic_files[0], models.SharedFile)
        self.assertEqual(magic_files[0].sharekey, 'CO0E')
        self.assertIsInstance(magic_files, list)
        self.assertEqual(len(magic_files), 10)

    @responses.activate
    def test_get_user_shakes(self):
        with open('tests/test_data/api/shakes') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/shakes',
            body=resp_data,
            status=200
        )
        shakes = self.api.get_user_shakes()
        self.assertIsInstance(shakes[0], models.Shake)
        self.assertIsInstance(shakes[0].owner, models.User)
        self.assertEqual(len(shakes), 1)

    @responses.activate
    def test_update_shared_file(self):
        with open('tests/test_data/api/sharedfile/1669X.POST') as f:
            resp_data = f.read()

        responses.add(
            responses.POST,
            'http://mlkshk.com/api/sharedfile/1669X',
            body=resp_data,
            status=200
        )

        sf = self.api.update_shared_file(sharekey='1669X', description='test')

        self.assertIsInstance(sf, models.SharedFile)
        self.assertEqual(sf.description, 'test')

        with open('tests/test_data/api/sharedfile/1669X.POST.title') as f:
            resp_data = f.read()

        responses.add(
            responses.POST,
            'http://mlkshk.com/api/sharedfile/1669X',
            body=resp_data,
            status=200
        )
        sf = self.api.update_shared_file(sharekey='1669X',
                                         title='journalism')
        self.assertIsInstance(sf, models.SharedFile)
        self.assertEqual(sf.title, 'journalism')

        self.assertRaises(
            Exception,
            lambda: self.api.update_shared_file(sharekey='1669X'))
        self.assertRaises(
            Exception,
            lambda: self.api.update_shared_file())

    @responses.activate
    def test_get_comments(self):
        with open('tests/test_data/api/sharedfile/162LG/comments') as f:
            resp_data = f.read()

        responses.add(
            responses.GET,
            'http://mlkshk.com/api/sharedfile/162LG/comments',
            body=resp_data,
            status=200
        )
        comments = self.api.get_comments(sharekey='162LG')
        self.assertEqual(len(comments), 4)
        self.assertIsInstance(comments[0], models.Comment)
        self.assertEqual(comments[0].body, '!!!!!!!')
        self.assertRaises(Exception,
                          lambda: self.api.get_comments())
<|endoftext|>


<commit_before>#!/usr/bin/env python2

import sys
import requests
import argparse
import os.path
from datetime import datetime
import xml.etree.ElementTree as et
from xml.dom import minidom

if ( len( sys.argv ) < 2 ):
    print "\nMissing parameters. Run \"%s -h\" for help.\n" %(sys.argv[0]);
    exit();

parser = argparse.ArgumentParser( description='SOAP web service Fuzzer' );
parser.add_argument( 'url', help='Web service URL to fuzz' );
parser.add_argument( '--no-cert-validate', action='store_true', help="Disable certificates validation" );
parser.add_argument( '--auto', action='store_true', help="Enable automatic testing" );
header_group = parser.add_mutually_exclusive_group();
header_group.add_argument( '--header', nargs='*', help='Specify required request headers' );
header_group.add_argument( '--fheader', help='Specify a file containing the required request headers' );
parser.add_argument( '--ua', help='Specify User-Agent header' );
parser.add_argument( '--ct', help='Specify Content-Type header' );
data_group = parser.add_mutually_exclusive_group();
data_group.add_argument( '--data', help='Data to be sent inside the request body' );
data_group.add_argument( '--fdata', help='Specify a file containing the data to be sent inside the request body' );
args = parser.parse_args()

#args.url = args.url.lower()
#url = args.url;

def end( reason ):
    print reason;
    exit();

def check_url_syntax( url ):
    if ((url.find('http://',0,7) == -1) and (url.find('https://',0,8) == -1)):
        end('\nERROR: not starting with http or https.\nCheck your URL and try again.\n');

def verify_url( url ):
    print('\nChecking if URL is available...');
    if (args.no_cert_validate):
        req = requests.post(url, data='', verify=False);
    else:
        req = requests.post(url, data='');
    if (req.status_code == requests.codes.ok):
        return
    else:
        print "HTTP Response Status Code %s - %s" %( req.status_code, req.reason );
        end("Exiting.\n");

def make_request( url, usr_headers, content='None' ):
    if (args.no_cert_validate):
        http = requests.post( url, data=content, headers=usr_headers, verify=False );
    else:
        http = requests.post( url, data=content, headers=usr_headers );
    http.close();
    return http;

def get_save_filename( url, clock ):
    if url.startswith( 'http://' ):
        address = url.replace( 'http://', '', 1 );
        address = address.replace( '/', '_' );
        address = address + '_' + clock;    # www.example.com_webservice.php_20141110_1822.15.059238.xml
        return address;
    elif url.startswith( 'https://' ):
        address = url.replace( 'https://', '', 1 );
        address = address.replace( '/', '_' );
        address = address + '_' + clock;    # www.example.com_webservice.php_20141110_1822.15.059238.xml
        return address;
    else:
        end('Malformed URL');

def save_data( request_content, response_content, clock ):
    path = 'requests/';     # path to write requests's response
    if not os.path.exists(path):
        os.makedirs(path);
    with open( path + get_save_filename( args.url, clock ) + '_req.xml', 'w+' ) as fp_req:     # requests/www.example.com_webservice.php_20141110_1822.15.059238_req.xml
        fp_req.write( request_content.encode('utf-8') );    # fixing problems with unicode characters
    with open( path + get_save_filename( args.url, clock ) + '_resp.xml', 'w+' ) as fp_resp:    # requests/www.example.com_webservice.php_20141110_1822.15.059238_resp.xml
        fp_resp.write( response_content.encode( 'utf-8' ) );  # fixing problems with unicode characters

def add_default_headers():
    return dict({ 'Content-Type': 'text/xml; charset=ascii', 'User-Agent': 'FuzzML/1.0' });

def add_header( header_dict, field_value_dict ):
    header_dict.update( field_value_dict );
    return

def add_headers( args ):
    print ( 'Adding headers...' );
    hr = add_default_headers();
    if ( args.header is not None ):
        add_header( hr, list2dict( args.header ) );
    if ( args.fheader is not None ):
        add_header( hr, get_headers_from_file( args.fheader ) );
    if ( args.ua is not None ):
        add_header( hr, list2dict( [ 'User-Agent', args.ua ] ) );
    if ( args.ct is not None ):
        add_header( hr, list2dict( [ 'Content-Type', args.ct ] ) );
    return hr;

def get_headers_from_file( hdr_file ):
    if (os.path.exists( hdr_file )):
        fp_hdr = open( hdr_file, 'r');
        lines = dict();
        for line in fp_hdr:
            line = replace_tabs(line);
            lines.update( list2dict( line ) );
        fp_hdr.close();
        return lines;
    else:
        end( "File not found: %s\n" %( hdr_file ) );

def list2dict( llist ):
    converted_list = list();
    for i in range( 0, len( llist ) ):
        converted_list.extend( llist[i].split(' ') );
    return dict( converted_list[i:i+2] for i in range( 0, len( converted_list ), 2) )

def replace_tabs( string ):
    return string.split();

def set_req_body( cmdline_data, file_data ):
    if ( ( cmdline_data is not None ) and ( file_data is None ) ):      # if the user defined data through command line
        return cmdline_data;
    elif ( ( cmdline_data is None ) and (file_data is not None ) ):     # if the user defined data through a file
        if os.path.exists( file_data ):
            with open( file_data, 'r' ) as f:
                body = f.read( 20 * 1024 );    # read at most 20 KB from file
            return body;
    elif ( ( cmdline_data is not None ) and ( file_data is not None ) ):
        end( 'You cannot specify BOTH parameters: --data AND --fdata. Choose only one.\n' );

def parse_xml_req( xml_data ):
    if ( ( not isinstance( xml_data, str ) ) and ( not isinstance( xml_data, et.Element ) ) ):
        end( 'Unrecognized xml data.' );
    if ( isinstance( xml_data, str ) ):
        root = et.fromstring( xml_data );
    elif ( isinstance( xml_data, et.Element ) ):
        root = xml_data;
    return root;

def parse_xml_resp( xml_data ):
    if ( ( not isinstance( xml_data, str ) ) and ( not isinstance( xml_data, et.Element ) ) ):
        end( 'Unrecognized xml data.' );
    if ( isinstance( xml_data, str ) ):
        root = et.fromstring( xml_data );
    elif ( isinstance( xml_data, et.Element ) ):
        root = xml_data;
    for child in root.iter():
        if ( '}' in child.tag ):
            print child.tag.split('}')[1], child.attrib, '-' + child.text.strip();  # transforms '{scope}TAG' in 'TAG'

def fuzzml_element_duplication( root, url, hr ):
    new_tree = copy_tree ( root );
    tree_root = new_tree.getroot();
    nodes_to_duplicate = get_nodes_list( tree_root );
    if ( nodes_to_duplicate ):
        print 'Fuzzing elements (by duplication) and saving responses...'
        for node in nodes_to_duplicate:
            children = get_children( node );
            for child in children:
                child_tree = copy_tree( child );
                child_dup = child_tree.getroot();
                node.insert( get_children( node ).index( child ), child_dup );  # places the duplicated node side-by-side with the original node
                keep_information( et.tostring( tree_root ), url, hr );
                node.remove( child_dup );
    else:
        end( 'Tree has only one Element\n' );

def fuzzml_element_omission( root, url, hr ):
    new_tree = copy_tree ( root );
    tree_root = new_tree.getroot();
    nodes_to_remove = get_nodes_list( tree_root );
    if ( nodes_to_remove ):
        print 'Fuzzing elements (by omission) and saving responses...'
        for node in nodes_to_remove:
            children = get_children( node );
            for child in children:
                #child_tree = copy_tree( child );
                #child_dup = child_tree.getroot();
                index = get_children( node ).index( child );    # remember element position
                node.remove( child );
                keep_information( et.tostring( tree_root ), url, hr );
                node.insert( index, child );  # places the removed node back into place
    else:
        end( 'Tree has only one Element\n' );

def keep_information( fuzzed_xml, url, hr ):
    fuzzed_xml_request = minidom.parseString( fuzzed_xml );
    fuzzed_xml_response = make_request( url, hr, fuzzed_xml_request.toprettyxml() );
    #fuzzed_xml_response = minidom.parseString( response.text.encode( 'utf-8' ) );
    save_data( fuzzed_xml_request.toprettyxml(), fuzzed_xml_response.text, datetime.now().strftime( "%Y%m%d_%H%M.%S.%f" ) );

def get_nodes_list( node ):
    return list( node.iter() );

def copy_tree( tree_root ):
    return et.ElementTree( tree_root );

def get_children( node ):
    return list( node );

def main():
    check_url_syntax( args.url );
    if args.auto:
        verify_url( args.url );
    hr = add_headers( args );
    content = set_req_body( args.data, args.fdata );
    print 'Parsing...';
    #print content;
    xml_root = parse_xml_req( content );
    #fuzzml_element_duplication( xml_root, args.url, hr );
    fuzzml_element_omission( xml_root, args.url, hr );

if __name__ == '__main__':
    main();

<commit_msg>added fuzzing by tag malformation<commit_after>#!/usr/bin/env python2

import sys
import requests
import argparse
import os.path
from datetime import datetime
import xml.etree.ElementTree as et
from xml.dom import minidom
import random

if ( len( sys.argv ) < 2 ):
    print "\nMissing parameters. Run \"%s -h\" for help.\n" %(sys.argv[0]);
    exit();

parser = argparse.ArgumentParser( description='SOAP web service Fuzzer' );
parser.add_argument( 'url', help='Web service URL to fuzz' );
parser.add_argument( '--no-cert-validate', action='store_true', help="Disable certificates validation" );
parser.add_argument( '--auto', action='store_true', help="Enable automatic testing" );
header_group = parser.add_mutually_exclusive_group();
header_group.add_argument( '--header', nargs='*', help='Specify required request headers' );
header_group.add_argument( '--fheader', help='Specify a file containing the required request headers' );
parser.add_argument( '--ua', help='Specify User-Agent header' );
parser.add_argument( '--ct', help='Specify Content-Type header' );
data_group = parser.add_mutually_exclusive_group();
data_group.add_argument( '--data', help='Data to be sent inside the request body' );
data_group.add_argument( '--fdata', help='Specify a file containing the data to be sent inside the request body' );
args = parser.parse_args()

def end( reason ):
    print '[-] ' + reason;
    exit();

def check_url_syntax( url ):
    if ((url.find('http://',0,7) == -1) and (url.find('https://',0,8) == -1)):
        end('\nERROR: address not starting with http or https.\nCheck your URL and try again.\n');

def verify_url( url ):
    print('\n[+] Checking if URL is available...');
    if (args.no_cert_validate):
        req = requests.post(url, data='', verify=False);
    else:
        req = requests.post(url, data='');
    if (req.status_code == requests.codes.ok):
        return
    else:
        print "[+] HTTP Response Status Code %s - %s" %( req.status_code, req.reason );
        end("Exiting.\n");

def make_request( url, usr_headers, content='None' ):
    if (args.no_cert_validate):
        http = requests.post( url, data=content, headers=usr_headers, verify=False );
    else:
        http = requests.post( url, data=content, headers=usr_headers );
    http.close();
    return http;

def get_save_filename( url, clock ):
    if url.startswith( 'http://' ):
        address = url.replace( 'http://', '', 1 );
        address = address.replace( '/', '_' );
        address = address + '_' + clock;    # www.example.com_webservice.php_20141110_1822.15.059238.xml
        return address;
    elif url.startswith( 'https://' ):
        address = url.replace( 'https://', '', 1 );
        address = address.replace( '/', '_' );
        address = address + '_' + clock;    # www.example.com_webservice.php_20141110_1822.15.059238.xml
        return address;
    else:
        end('Malformed URL');

def save_data( request_content, response_content, clock ):
    path = 'requests/';     # path to write requests's response
    if not os.path.exists(path):
        os.makedirs(path);
    with open( path + get_save_filename( args.url, clock ) + '_req.xml', 'w+' ) as fp_req:     # requests/www.example.com_webservice.php_20141110_1822.15.059238_req.xml
        fp_req.write( request_content.encode('utf-8') );    # fixing problems with unicode characters
    with open( path + get_save_filename( args.url, clock ) + '_resp.xml', 'w+' ) as fp_resp:    # requests/www.example.com_webservice.php_20141110_1822.15.059238_resp.xml
        fp_resp.write( response_content.encode( 'utf-8' ) );  # fixing problems with unicode characters

def add_default_headers():
    return dict({ 'Content-Type': 'text/xml; charset=ascii', 'User-Agent': 'FuzzML/1.0' });

def add_header( header_dict, field_value_dict ):
    header_dict.update( field_value_dict );
    return

def add_headers( args ):
    print ( '[+] Adding headers...' );
    hr = add_default_headers();
    if ( args.header is not None ):
        add_header( hr, list2dict( args.header ) );
    if ( args.fheader is not None ):
        add_header( hr, get_headers_from_file( args.fheader ) );
    if ( args.ua is not None ):
        add_header( hr, list2dict( [ 'User-Agent', args.ua ] ) );
    if ( args.ct is not None ):
        add_header( hr, list2dict( [ 'Content-Type', args.ct ] ) );
    return hr;

def get_headers_from_file( hdr_file ):
    if (os.path.exists( hdr_file )):
        fp_hdr = open( hdr_file, 'r');
        lines = dict();
        for line in fp_hdr:
            line = replace_tabs(line);
            lines.update( list2dict( line ) );
        fp_hdr.close();
        return lines;
    else:
        end( "File not found: %s\n" %( hdr_file ) );

def list2dict( llist ):
    converted_list = list();
    for i in range( 0, len( llist ) ):
        converted_list.extend( llist[i].split(' ') );
    return dict( converted_list[i:i+2] for i in range( 0, len( converted_list ), 2) )

def replace_tabs( string ):
    return string.split();

def set_req_body( cmdline_data, file_data ):
    if ( ( cmdline_data is not None ) and ( file_data is None ) ):      # if the user defined data through command line
        return cmdline_data;
    elif ( ( cmdline_data is None ) and (file_data is not None ) ):     # if the user defined data through a file
        if os.path.exists( file_data ):
            with open( file_data, 'r' ) as f:
                body = f.read( 20 * 1024 );    # read at most 20 KB from file
            return body;
    elif ( ( cmdline_data is not None ) and ( file_data is not None ) ):
        end( 'You cannot specify BOTH parameters: --data AND --fdata. Choose only one.\n' );

def parse_xml_req( xml_data ):
    if ( ( not isinstance( xml_data, str ) ) and ( not isinstance( xml_data, et.Element ) ) ):
        end( 'Unrecognized xml data.' );
    if ( isinstance( xml_data, str ) ):
        root = et.fromstring( xml_data );
    elif ( isinstance( xml_data, et.Element ) ):
        root = xml_data;
    return root;

def fuzzml_element_duplication( root, url, hr ):
    new_tree = copy_tree ( root );
    tree_root = new_tree.getroot();
    nodes_to_duplicate = get_nodes_list( tree_root );
    if ( nodes_to_duplicate ):
        print( '[+] Fuzzing elements (by duplication) and saving responses...' );
        for node in nodes_to_duplicate:
            children = get_children( node );
            for child in children:
                child_tree = copy_tree( child );
                child_dup = child_tree.getroot();
                node.insert( get_children( node ).index( child ), child_dup );  # places the duplicated node side-by-side with the original node
                keep_information( et.tostring( tree_root ), url, hr );
                node.remove( child_dup );
    else:
        end( 'Tree has only one Element\n' );

def fuzzml_element_omission( root, url, hr ):
    new_tree = copy_tree ( root );
    tree_root = new_tree.getroot();
    nodes_to_remove = get_nodes_list( tree_root );
    if ( nodes_to_remove ):
        print( '[+] Fuzzing elements (by omission) and saving responses...' );
        for node in nodes_to_remove:
            children = get_children( node );
            for child in children:
                index = get_children( node ).index( child );    # remember element position
                node.remove( child );
                keep_information( et.tostring( tree_root ), url, hr );
                node.insert( index, child );  # places the removed node back into place
    else:
        end( 'Tree has only one Element\n' );

def fuzzml_element_tag_malforming( root, url, hr ):
    new_tree = copy_tree ( root );
    tree_root = new_tree.getroot();
    nodes_to_remove = get_nodes_list( tree_root );
    if ( nodes_to_remove ):
        print( '[+] Fuzzing elements (by tag malformation) and saving responses...' );
        xml_allowed_chars = sum( list( ( range(65,90),[95],range(97,122) ) ), [] );     # creating list of acceptable chars in an xml tag
        for node in nodes_to_remove:
            children = get_children( node );
            for child in children:
                if ( '}' in child.tag ):
                    namespace = child.tag.split('}')[0];
                    tag = child.tag.split('}')[1];
                else:
                    tag = str( child.tag );
                element_position = get_children( node ).index( child );    # remember element position
                char_index = random.choice(range(0, len( tag )));  # choosing a random char to strip from tag
                if (namespace):
                    child.tag = namespace + '}' + tag.replace( tag[ char_index ], '' );       # stripping char from tag
                else:
                    child.tag = tag.replace( tag[ char_index ], '' );       # stripping char from tag
                child.tag = child.tag + chr(random.choice( xml_allowed_chars ) ) ;     # adding a random ascii char suffix
                keep_information( et.tostring( tree_root ), url, hr );
                if (namespace):
                    child.tag = namespace + '}' + tag ;
                else:
                    child.tag = tag ;   # putting everything back to normal
    else:
        end( 'Tree has only one Element\n' );

def keep_information( fuzzed_xml, url, hr ):
    fuzzed_xml_request = minidom.parseString( fuzzed_xml );
    fuzzed_xml_response = make_request( url, hr, fuzzed_xml_request.toprettyxml() );
    fuzzed_xml_response = minidom.parseString( fuzzed_xml_response.text.encode( 'utf-8' ) );
    save_data( fuzzed_xml_request.toprettyxml(), fuzzed_xml_response.toxml(), datetime.now().strftime( "%Y%m%d_%H%M.%S.%f" ) );

def get_nodes_list( node ):
    return list( node.iter() );

def copy_tree( tree_root ):
    return et.ElementTree( tree_root );

def get_children( node ):
    return list( node );

def main():
    check_url_syntax( args.url );
    if args.auto:
        verify_url( args.url );
    hr = add_headers( args );
    content = set_req_body( args.data, args.fdata );
    print( '[+] Parsing...' );
    xml_root = parse_xml_req( content );
    fuzzml_element_duplication( xml_root, args.url, hr );
    fuzzml_element_omission( xml_root, args.url, hr );
    fuzzml_element_tag_malforming( xml_root, args.url, hr );

if __name__ == '__main__':
    main();

<|endoftext|>


<commit_before>"""
:Author: Jonathan Karr <karr@mssm.edu>
:Date: 2018-03-12
:Copyright: 2018, Karr Lab
:License: MIT
"""

import obj_model
import types
import unittest

class ApiTestCase(unittest.TestCase):
    def test(self):
        self.assertIsInstance(obj_model.Model, type)
        self.assertIsInstance(obj_model.abstract, types.ModuleType)
<commit_msg>removing trailing white space<commit_after>"""
:Author: Jonathan Karr <karr@mssm.edu>
:Date: 2018-03-12
:Copyright: 2018, Karr Lab
:License: MIT
"""

import obj_model
import types
import unittest

class ApiTestCase(unittest.TestCase):
    def test(self):
        self.assertIsInstance(obj_model.Model, type)
        self.assertIsInstance(obj_model.abstract, types.ModuleType)
<|endoftext|>