<commit_before>import numpy as np
import pandas as pd
import scanpy as sc
from sklearn.metrics import silhouette_score
from sc.preprocessing._combat import stand_data

def test_norm():
    # this test trivially checks wether mean normalisation worked

    # load in data
    adata = sc.datasets.blobs()
    key = 'blobs'
    data = pd.DataFrame(data=adata.X.T, index=adata.var_names,
                        columns=adata.obs_names)

    # construct a pandas series of the batch annotation
    batch = pd.Series(adata.obs[key])
    model = pd.DataFrame({'batch': batch})

    # standardize the data
    s_data, design, var_pooled,  stand_mean = stand_data(model, data)

    assert np.allclose(s_data.mean(axis = 1), np.zeros(s_data.shape[0]))

def test_shilhouette():
    # this test checks wether combat can align data from several gaussians
    # it checks this by computing the silhouette coefficient in a pca embedding

    # load in data
    adata = sc.datasets.blobs()

    # apply combat
    sc.pp.combat(adata, 'blobs')

    # compute pca
    sc.tl.pca(adata)
    X_pca = adata.obsm['X_pca']

    # compute silhouette coefficient in pca
    sh = silhouette_score(X_pca[:, :2], adata.obs['blobs'].values )

    assert sh < 0.1
<commit_msg>fixed another bug<commit_after>import numpy as np
import pandas as pd
import scanpy as sc
import scanpy
from sklearn.metrics import silhouette_score
from scanpy.preprocessing._combat import stand_data

def test_norm():
    # this test trivially checks wether mean normalisation worked

    # load in data
    adata = sc.datasets.blobs()
    key = 'blobs'
    data = pd.DataFrame(data=adata.X.T, index=adata.var_names,
                        columns=adata.obs_names)

    # construct a pandas series of the batch annotation
    batch = pd.Series(adata.obs[key])
    model = pd.DataFrame({'batch': batch})

    # standardize the data
    s_data, design, var_pooled,  stand_mean = stand_data(model, data)

    assert np.allclose(s_data.mean(axis = 1), np.zeros(s_data.shape[0]))

def test_shilhouette():
    # this test checks wether combat can align data from several gaussians
    # it checks this by computing the silhouette coefficient in a pca embedding

    # load in data
    adata = sc.datasets.blobs()

    # apply combat
    sc.pp.combat(adata, 'blobs')

    # compute pca
    sc.tl.pca(adata)
    X_pca = adata.obsm['X_pca']

    # compute silhouette coefficient in pca
    sh = silhouette_score(X_pca[:, :2], adata.obs['blobs'].values )

    assert sh < 0.1
<|endoftext|>


<commit_before># Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Memory leak detection utility."""

from tensorflow.python.framework.python_memory_checker import _PythonMemoryChecker
from tensorflow.python.profiler import trace
from tensorflow.python.util import tf_inspect

try:
  from tensorflow.python.platform.cpp_memory_checker import _CppMemoryChecker as CppMemoryChecker  # pylint:disable=g-import-not-at-top
except ImportError:
  CppMemoryChecker = None

def _get_test_name_best_effort():
  """If available, return the current test name. Otherwise, `None`."""
  for stack in tf_inspect.stack():
    function_name = stack[3]
    if function_name.startswith('test'):
      try:
        class_name = stack[0].f_locals['self'].__class__.__name__
        return class_name + '.' + function_name
      except:  # pylint:disable=bare-except
        pass

  return None

# TODO(kkb): Also create decorator versions for convenience.
class MemoryChecker(object):
  """Memory leak detection class.

  This is a utility class to detect Python and C++ memory leaks. It's intended
  for both testing and debugging. Basic usage:

  >>> # MemoryChecker() context manager tracks memory status inside its scope.
  >>> with MemoryChecker() as memory_checker:
  >>>   tensors = []
  >>>   for _ in range(10):
  >>>     # Simulating `tf.constant(1)` object leak every iteration.
  >>>     tensors.append(tf.constant(1))
  >>>
  >>>     # Take a memory snapshot for later analysis.
  >>>     memory_checker.record_snapshot()
  >>>
  >>> # `report()` generates a html graph file showing allocations over
  >>> # snapshots per every stack trace.
  >>> memory_checker.report()
  >>>
  >>> # This assertion will detect `tf.constant(1)` object leak.
  >>> memory_checker.assert_no_leak_if_all_possibly_except_one()

  `record_snapshot()` must be called once every iteration at the same location.
  This is because the detection algorithm relies on the assumption that if there
  is a leak, it's happening similarly on every snapshot.
  """

  @trace.trace_wrapper
  def __enter__(self):
    self._python_memory_checker = _PythonMemoryChecker()
    if CppMemoryChecker:
      self._cpp_memory_checker = CppMemoryChecker(_get_test_name_best_effort())
    return self

  @trace.trace_wrapper
  def __exit__(self, exc_type, exc_value, traceback):
    if CppMemoryChecker:
      self._cpp_memory_checker.stop()

  # We do not enable trace_wrapper on this function to avoid contaminating
  # the snapshot.
  def record_snapshot(self):
    """Take a memory snapshot for later analysis.

    `record_snapshot()` must be called once every iteration at the same
    location. This is because the detection algorithm relies on the assumption
    that if there is a leak, it's happening similarly on every snapshot.

    The recommended number of `record_snapshot()` call depends on the testing
    code complexity and the allcoation pattern.
    """
    self._python_memory_checker.record_snapshot()
    if CppMemoryChecker:
      self._cpp_memory_checker.record_snapshot()

  @trace.trace_wrapper
  def report(self):
    """Generates a html graph file showing allocations over snapshots.

    It create a temporary directory and put all the output files there.
    If this is running under Google internal testing infra, it will use the
    directory provided the infra instead.
    """
    self._python_memory_checker.report()
    if CppMemoryChecker:
      self._cpp_memory_checker.report()

  @trace.trace_wrapper
  def assert_no_leak_if_all_possibly_except_one(self):
    """Raises an exception if a leak is detected.

    This algorithm classifies a series of allocations as a leak if it's the same
    type(Python) orit happens at the same stack trace(C++) at every snapshot,
    but possibly except one snapshot.
    """

    self._python_memory_checker.assert_no_leak_if_all_possibly_except_one()
    if CppMemoryChecker:
      self._cpp_memory_checker.assert_no_leak_if_all_possibly_except_one()

  @trace.trace_wrapper
  def assert_no_new_python_objects(self, threshold=None):
    """Raises an exception if there are new Python objects created.

    It computes the number of new Python objects per type using the first and
    the last snapshots.

    Args:
      threshold: A dictionary of [Type name string], [count] pair. It won't
        raise an exception if the new Python objects are under this threshold.
    """
    self._python_memory_checker.assert_no_new_objects(threshold=threshold)
<commit_msg>Fix a typo<commit_after># Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Memory leak detection utility."""

from tensorflow.python.framework.python_memory_checker import _PythonMemoryChecker
from tensorflow.python.profiler import trace
from tensorflow.python.util import tf_inspect

try:
  from tensorflow.python.platform.cpp_memory_checker import _CppMemoryChecker as CppMemoryChecker  # pylint:disable=g-import-not-at-top
except ImportError:
  CppMemoryChecker = None

def _get_test_name_best_effort():
  """If available, return the current test name. Otherwise, `None`."""
  for stack in tf_inspect.stack():
    function_name = stack[3]
    if function_name.startswith('test'):
      try:
        class_name = stack[0].f_locals['self'].__class__.__name__
        return class_name + '.' + function_name
      except:  # pylint:disable=bare-except
        pass

  return None

# TODO(kkb): Also create decorator versions for convenience.
class MemoryChecker(object):
  """Memory leak detection class.

  This is a utility class to detect Python and C++ memory leaks. It's intended
  for both testing and debugging. Basic usage:

  >>> # MemoryChecker() context manager tracks memory status inside its scope.
  >>> with MemoryChecker() as memory_checker:
  >>>   tensors = []
  >>>   for _ in range(10):
  >>>     # Simulating `tf.constant(1)` object leak every iteration.
  >>>     tensors.append(tf.constant(1))
  >>>
  >>>     # Take a memory snapshot for later analysis.
  >>>     memory_checker.record_snapshot()
  >>>
  >>> # `report()` generates a html graph file showing allocations over
  >>> # snapshots per every stack trace.
  >>> memory_checker.report()
  >>>
  >>> # This assertion will detect `tf.constant(1)` object leak.
  >>> memory_checker.assert_no_leak_if_all_possibly_except_one()

  `record_snapshot()` must be called once every iteration at the same location.
  This is because the detection algorithm relies on the assumption that if there
  is a leak, it's happening similarly on every snapshot.
  """

  @trace.trace_wrapper
  def __enter__(self):
    self._python_memory_checker = _PythonMemoryChecker()
    if CppMemoryChecker:
      self._cpp_memory_checker = CppMemoryChecker(_get_test_name_best_effort())
    return self

  @trace.trace_wrapper
  def __exit__(self, exc_type, exc_value, traceback):
    if CppMemoryChecker:
      self._cpp_memory_checker.stop()

  # We do not enable trace_wrapper on this function to avoid contaminating
  # the snapshot.
  def record_snapshot(self):
    """Take a memory snapshot for later analysis.

    `record_snapshot()` must be called once every iteration at the same
    location. This is because the detection algorithm relies on the assumption
    that if there is a leak, it's happening similarly on every snapshot.

    The recommended number of `record_snapshot()` call depends on the testing
    code complexity and the allcoation pattern.
    """
    self._python_memory_checker.record_snapshot()
    if CppMemoryChecker:
      self._cpp_memory_checker.record_snapshot()

  @trace.trace_wrapper
  def report(self):
    """Generates a html graph file showing allocations over snapshots.

    It create a temporary directory and put all the output files there.
    If this is running under Google internal testing infra, it will use the
    directory provided the infra instead.
    """
    self._python_memory_checker.report()
    if CppMemoryChecker:
      self._cpp_memory_checker.report()

  @trace.trace_wrapper
  def assert_no_leak_if_all_possibly_except_one(self):
    """Raises an exception if a leak is detected.

    This algorithm classifies a series of allocations as a leak if it's the same
    type(Python) or it happens at the same stack trace(C++) at every snapshot,
    but possibly except one snapshot.
    """

    self._python_memory_checker.assert_no_leak_if_all_possibly_except_one()
    if CppMemoryChecker:
      self._cpp_memory_checker.assert_no_leak_if_all_possibly_except_one()

  @trace.trace_wrapper
  def assert_no_new_python_objects(self, threshold=None):
    """Raises an exception if there are new Python objects created.

    It computes the number of new Python objects per type using the first and
    the last snapshots.

    Args:
      threshold: A dictionary of [Type name string], [count] pair. It won't
        raise an exception if the new Python objects are under this threshold.
    """
    self._python_memory_checker.assert_no_new_objects(threshold=threshold)
<|endoftext|>


<commit_before>"""DataTables unit tests."""
import unittest
import faker
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from datatables import DataTables, ColumnDT

from .models import Base, User, Address

class DataTablesTest(unittest.TestCase):

    """Class defining a DataTables test plan."""

    def setUp(self):
        """Set up fake database session before all tests."""
        engine = create_engine('sqlite://', echo=False)  # echo=True for debug
        Base.metadata.create_all(engine)
        Session = sessionmaker(bind=engine)
        self.session = Session()

    def populate(self, nbUsers):
        """Create nbUsers in fake database."""
        users = []
        f = faker.Faker()

        for i in range(nbUsers):
            user, addr = self.create_user(f.name(), f.address())
            users.append(user)

        self.session.add_all(users)
        self.session.commit()

    def create_user(self, name, address):
        """Create a fake user."""
        addr = Address(description=address)

        user = User(name=name, address=addr)

        return user, addr

    def create_columns(self, columns):
        """Create a fake DataTables columns."""
        cols = []

        for col in columns:
            cols.append(ColumnDT(col))

        return cols

    def create_dt_params(self, search='', start=0, length=10, order=None):
        """Create DataTables input parameters."""
        params = {
            'draw': '1',
            'start': str(start),
            'length': str(length),
            'search[value]': str(search),
            'search[regex]': 'false'
        }

        for i, item in enumerate(('id', 'name', 'address', 'created_at')):
            cols = 'columns[%s]' % i
            params['%s%s' % (cols, '[data]')] = i
            params['%s%s' % (cols, '[name]')] = ''
            params['%s%s' % (cols, '[searchable]')] = 'true'
            params['%s%s' % (cols, '[orderable]')] = 'true'
            params['%s%s' % (cols, '[search][value]')] = ''
            params['%s%s' % (cols, '[search][regex]')] = 'false'

        for i, item in enumerate(order or [{'column': 0, 'dir': 'asc'}]):
            for key, value in item.items():
                params['order[%s][%s]' % (i, key)] = str(value)

        return params

    def test_list_users(self):
        """Test if it returns a simple users list."""
        self.populate(5)

        columns = self.create_columns(['id', 'name', 'address.description',
                                       'created_at'])

        req = self.create_dt_params()

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert len(res['data']) == 5
        assert res['recordsTotal'] == '5'
        assert res['recordsFiltered'] == '5'

    def test_list_users_with_hybrid_attribute(self):
        """Test if it returns a users list with a Hybrid column."""
        self.populate(5)

        user6, addr6 = self.create_user('Homer', 'Simpson')

        self.session.add(user6)
        self.session.commit()

        columns = self.create_columns(['id', 'dummy', 'name', 'created_at'])

        req = self.create_dt_params()

        rowTable = DataTables(
            req, User, self.session.query(User), columns)

        res = rowTable.output_result()

        assert len(res['data']) == 6
        assert res['data'][5]['1'] == 'H6-DUMMY'
        assert res['data'][5]['2'] == 'Homer'

    def test_list_page_x(self):
        """Test if it lists users that are not on page 1."""
        self.populate(10)

        user11, addr11 = self.create_user('The Number of', 'The Beast')
        user12, addr12 = self.create_user('Hallowed Be', 'Thy Name')

        self.session.add(user11)
        self.session.add(user12)
        self.session.commit()

        columns = self.create_columns(['id', 'name', 'address.description',
                                       'created_at'])

        req = self.create_dt_params(start=10, length=10)

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert len(res['data']) == 2
        assert res['recordsTotal'] == '12'
        assert res['recordsFiltered'] == '12'
        assert res['data'][0]['1'] == 'The Number of'
        assert res['data'][1]['1'] == 'Hallowed Be'

    def test_set_mdata(self):
        """Test if result's data have mData set."""
        self.populate(5)

        columns = []
        columns.append(ColumnDT('id', mData='ID'))
        columns.append(ColumnDT('name', mData='Username'))
        columns.append(ColumnDT('address.description', mData='Address'))
        columns.append(ColumnDT('created_at', mData='Created at'))

        req = self.create_dt_params()

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert len(res['data']) == 5
        assert 'ID' in res['data'][0]
        assert 'Username' in res['data'][0]
        assert 'Address' in res['data'][0]
        assert 'Created at' in res['data'][0]

    def test_global_search_filtering(self):
        """Test if result's are filtered from global search field."""
        self.populate(5)

        user6, addr6 = self.create_user('Run To', 'The Hills')
        user7, addr7 = self.create_user('Fear Of', 'The Dark')

        self.session.add(user6)
        self.session.add(user7)
        self.session.commit()

        columns = self.create_columns(['id', 'name', 'address.description',
                                       'created_at'])

        req = self.create_dt_params(search='Fear')

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert len(res['data']) == 1
        assert res['recordsTotal'] == '7'
        assert res['recordsFiltered'] == '1'
        assert res['data'][0]['1'] == 'Fear Of'
        assert res['data'][0]['2'] == 'The Dark'

    def test_column_not_searchable(self):
        """Test if a column is not searchable."""
        self.populate(5)

        user6, addr6 = self.create_user('Run To', 'The Hills')
        user7, addr7 = self.create_user('Fear Of', 'The Dark')

        self.session.add(user6)
        self.session.add(user7)
        self.session.commit()

        columns = []
        columns.append(ColumnDT('id', mData='ID'))
        columns.append(ColumnDT('name', mData='Username', searchable=False))
        columns.append(ColumnDT('address.description', mData='Address'))
        columns.append(ColumnDT('created_at', mData='Created at'))

        req = self.create_dt_params(search='Fear')

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert len(res['data']) == 0
        assert res['recordsTotal'] == '7'
        assert res['recordsFiltered'] == '0'

    def test_column_ordering(self):
        """Test if a column is orderable."""
        self.populate(5)

        user6, addr6 = self.create_user('000_aaa', 'Whatever')
        user7, addr7 = self.create_user('zzz_aaa', 'Whatif')

        self.session.add(user6)
        self.session.add(user7)
        self.session.commit()

        columns = self.create_columns(['id', 'name', 'address.description',
                                       'created_at'])

        # DESC
        req = self.create_dt_params(order=[{"column": 1, "dir": "desc"}])

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert res['data'][0]['1'] == 'zzz_aaa'

        # ASC
        req = self.create_dt_params(order=[{"column": 1, "dir": "asc"}])

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert res['data'][0]['1'] == '000_aaa'

    def test_column_ordering_relation(self):
        """Test if a foreign key column is orderable."""
        self.populate(5)

        user6, addr6 = self.create_user('000_Whatever', '000_aaa')
        user7, addr7 = self.create_user('zzz_Whatif', 'zzz_aaa')

        self.session.add(user6)
        self.session.add(user7)
        self.session.commit()

        columns = self.create_columns(['id', 'name', 'address.description',
                                       'created_at'])

        # DESC
        req = self.create_dt_params(order=[{"column": 2, "dir": "desc"}])

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert res['data'][0]['2'] == 'zzz_aaa'

        # ASC
        req = self.create_dt_params(order=[{"column": 2, "dir": "asc"}])

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert res['data'][0]['2'] == '000_aaa'
<commit_msg>Unit test for sort on first column<commit_after>"""DataTables unit tests."""
import unittest
import faker
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from datatables import DataTables, ColumnDT

from .models import Base, User, Address

class DataTablesTest(unittest.TestCase):

    """Class defining a DataTables test plan."""

    def setUp(self):
        """Set up fake database session before all tests."""
        engine = create_engine('sqlite://', echo=False)  # echo=True for debug
        Base.metadata.create_all(engine)
        Session = sessionmaker(bind=engine)
        self.session = Session()

    def populate(self, nbUsers):
        """Create nbUsers in fake database."""
        users = []
        f = faker.Faker()

        for i in range(nbUsers):
            user, addr = self.create_user(f.name(), f.address())
            users.append(user)

        self.session.add_all(users)
        self.session.commit()

    def create_user(self, name, address):
        """Create a fake user."""
        addr = Address(description=address)

        user = User(name=name, address=addr)

        return user, addr

    def create_columns(self, columns):
        """Create a fake DataTables columns."""
        cols = []

        for col in columns:
            cols.append(ColumnDT(col))

        return cols

    def create_dt_params(self, search='', start=0, length=10, order=None):
        """Create DataTables input parameters."""
        params = {
            'draw': '1',
            'start': str(start),
            'length': str(length),
            'search[value]': str(search),
            'search[regex]': 'false'
        }

        for i, item in enumerate(('id', 'name', 'address', 'created_at')):
            cols = 'columns[%s]' % i
            params['%s%s' % (cols, '[data]')] = i
            params['%s%s' % (cols, '[name]')] = ''
            params['%s%s' % (cols, '[searchable]')] = 'true'
            params['%s%s' % (cols, '[orderable]')] = 'true'
            params['%s%s' % (cols, '[search][value]')] = ''
            params['%s%s' % (cols, '[search][regex]')] = 'false'

        for i, item in enumerate(order or [{'column': 0, 'dir': 'asc'}]):
            for key, value in item.items():
                params['order[%s][%s]' % (i, key)] = str(value)

        return params

    def test_list_users(self):
        """Test if it returns a simple users list."""
        self.populate(5)

        columns = self.create_columns(['id', 'name', 'address.description',
                                       'created_at'])

        req = self.create_dt_params()

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert len(res['data']) == 5
        assert res['recordsTotal'] == '5'
        assert res['recordsFiltered'] == '5'

    def test_list_users_with_hybrid_attribute(self):
        """Test if it returns a users list with a Hybrid column."""
        self.populate(5)

        user6, addr6 = self.create_user('Homer', 'Simpson')

        self.session.add(user6)
        self.session.commit()

        columns = self.create_columns(['id', 'dummy', 'name', 'created_at'])

        req = self.create_dt_params()

        rowTable = DataTables(
            req, User, self.session.query(User), columns)

        res = rowTable.output_result()

        assert len(res['data']) == 6
        assert res['data'][5]['1'] == 'H6-DUMMY'
        assert res['data'][5]['2'] == 'Homer'

    def test_list_page_x(self):
        """Test if it lists users that are not on page 1."""
        self.populate(10)

        user11, addr11 = self.create_user('The Number of', 'The Beast')
        user12, addr12 = self.create_user('Hallowed Be', 'Thy Name')

        self.session.add(user11)
        self.session.add(user12)
        self.session.commit()

        columns = self.create_columns(['id', 'name', 'address.description',
                                       'created_at'])

        req = self.create_dt_params(start=10, length=10)

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert len(res['data']) == 2
        assert res['recordsTotal'] == '12'
        assert res['recordsFiltered'] == '12'
        assert res['data'][0]['1'] == 'The Number of'
        assert res['data'][1]['1'] == 'Hallowed Be'

    def test_set_mdata(self):
        """Test if result's data have mData set."""
        self.populate(5)

        columns = []
        columns.append(ColumnDT('id', mData='ID'))
        columns.append(ColumnDT('name', mData='Username'))
        columns.append(ColumnDT('address.description', mData='Address'))
        columns.append(ColumnDT('created_at', mData='Created at'))

        req = self.create_dt_params()

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert len(res['data']) == 5
        assert 'ID' in res['data'][0]
        assert 'Username' in res['data'][0]
        assert 'Address' in res['data'][0]
        assert 'Created at' in res['data'][0]

    def test_global_search_filtering(self):
        """Test if result's are filtered from global search field."""
        self.populate(5)

        user6, addr6 = self.create_user('Run To', 'The Hills')
        user7, addr7 = self.create_user('Fear Of', 'The Dark')

        self.session.add(user6)
        self.session.add(user7)
        self.session.commit()

        columns = self.create_columns(['id', 'name', 'address.description',
                                       'created_at'])

        req = self.create_dt_params(search='Fear')

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert len(res['data']) == 1
        assert res['recordsTotal'] == '7'
        assert res['recordsFiltered'] == '1'
        assert res['data'][0]['1'] == 'Fear Of'
        assert res['data'][0]['2'] == 'The Dark'

    def test_column_not_searchable(self):
        """Test if a column is not searchable."""
        self.populate(5)

        user6, addr6 = self.create_user('Run To', 'The Hills')
        user7, addr7 = self.create_user('Fear Of', 'The Dark')

        self.session.add(user6)
        self.session.add(user7)
        self.session.commit()

        columns = []
        columns.append(ColumnDT('id', mData='ID'))
        columns.append(ColumnDT('name', mData='Username', searchable=False))
        columns.append(ColumnDT('address.description', mData='Address'))
        columns.append(ColumnDT('created_at', mData='Created at'))

        req = self.create_dt_params(search='Fear')

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert len(res['data']) == 0
        assert res['recordsTotal'] == '7'
        assert res['recordsFiltered'] == '0'

    def test_column_ordering(self):
        """Test if a column is orderable."""
        self.populate(5)

        user6, addr6 = self.create_user('000_aaa', 'Whatever')
        user7, addr7 = self.create_user('zzz_aaa', 'Whatif')

        self.session.add(user6)
        self.session.add(user7)
        self.session.commit()

        columns = self.create_columns(['id', 'name', 'address.description',
                                       'created_at'])

        # DESC
        req = self.create_dt_params(order=[{"column": 1, "dir": "desc"}])

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert res['data'][0]['1'] == 'zzz_aaa'

        # ASC
        req = self.create_dt_params(order=[{"column": 1, "dir": "asc"}])

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert res['data'][0]['1'] == '000_aaa'

        # DESC first column
        req = self.create_dt_params(order=[{"column": 0, "dir": "desc"}])

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert res['data'][0]['1'] == 'zzz_aaa'

    def test_column_ordering_relation(self):
        """Test if a foreign key column is orderable."""
        self.populate(5)

        user6, addr6 = self.create_user('000_Whatever', '000_aaa')
        user7, addr7 = self.create_user('zzz_Whatif', 'zzz_aaa')

        self.session.add(user6)
        self.session.add(user7)
        self.session.commit()

        columns = self.create_columns(['id', 'name', 'address.description',
                                       'created_at'])

        # DESC
        req = self.create_dt_params(order=[{"column": 2, "dir": "desc"}])

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert res['data'][0]['2'] == 'zzz_aaa'

        # ASC
        req = self.create_dt_params(order=[{"column": 2, "dir": "asc"}])

        rowTable = DataTables(
            req, User, self.session.query(User).join(Address), columns)

        res = rowTable.output_result()

        assert res['data'][0]['2'] == '000_aaa'
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
duffel.py

Flask app for routing to Rail outputs on various hosts. Currently supports only
Amazon Cloud Drive. Requires https://github.com/yadayada/acd_cli is authorized
and set up as owner of shared directory.
"""
from flask import Flask, redirect, render_template, abort, request
from contextlib import closing
import subprocess
import json
app = Flask(__name__)

# Path to ACD CLI is hardcoded so app works on Webfaction
_ACDCLI = '/home/verve/anaconda3/bin/acdcli'

@app.before_request
def before_request():
    if request.method == 'HEAD':
        request.method = 'GET'
        return 'hello'

@app.route('/')
def duffout():
    return "Duffel is your friendly neighborhood data broker."

@app.route('/<resource>/<path:identifier>')
def forward(resource, identifier):
    """ Redirects request for file to direct URL.

        Requires global "paths" dictionary is active.

        resource: a given resource, like "recount2"
        identifier: relative path to file or directory

        Return value: Flask redirect response object
    """
    if resource == 'recount':
        try:
            # Redirect to temp URL obtained from ACD CLI
            return redirect(
                        json.loads(
                            subprocess.check_output(
                                [
                                    _ACDCLI,
                                    'metadata',
                                    '/'.join(['', resource, identifier])
                            ]
                        )
                    )['tempLink']
                )
        except Exception as e:
            # 404 out below
            pass
    abort(404)

if __name__ == '__main__':
    app.run()
<commit_msg>attempts constructing response manually<commit_after>#!/usr/bin/env python
"""
duffel.py

Flask app for routing to Rail outputs on various hosts. Currently supports only
Amazon Cloud Drive. Requires https://github.com/yadayada/acd_cli is authorized
and set up as owner of shared directory.
"""
from flask import Flask, redirect, render_template, abort, request, Response
from contextlib import closing
import subprocess
import json
import requests
app = Flask(__name__)

# Path to ACD CLI is hardcoded so app works on Webfaction
_ACDCLI = '/home/verve/anaconda3/bin/acdcli'

@app.route('/')
def duffout():
    return "Duffel is your friendly neighborhood data broker."

@app.route('/<resource>/<path:identifier>')
def forward(resource, identifier):
    """ Redirects request for file to direct URL.

        Requires global "paths" dictionary is active.

        resource: a given resource, like "recount2"
        identifier: relative path to file or directory

        Return value: Flask redirect response object
    """
    if resource == 'recount':
        try:
            templink = json.loads(
                            subprocess.check_output(
                                [
                                    _ACDCLI,
                                    'metadata',
                                    '/'.join(['', resource, identifier])
                            ]
                        )
                    )['tempLink']
        except Exception as e:
            # 404 out below
            pass
        else:
            if request.method == 'HEAD':
                # Workaround: use GET and simulate header
                aws_response = requests.get(
                        templink,
                        headers={'range' : 'bytes=0-0'}
                    )
                return Response(
                        headers=aws_response.headers,
                        status=aws_response.status_code,
                        content_type=aws_response.headers['content-type']
                    )
            # Redirect to temp URL obtained from ACD CLI
            return redirect(templink)
        except Exception as e:
            # 404 out below
            pass
    abort(404)

if __name__ == '__main__':
    app.run()
<|endoftext|>


<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Keras estimator API."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.util.tf_export import keras_export

# Keras has undeclared dependency on tensorflow/estimator:estimator_py.
# As long as you depend //third_party/py/tensorflow:tensorflow target
# everything will work as normal.

# LINT.IfChange
@keras_export(v1=['keras.estimator.model_to_estimator'])
def model_to_estimator(
    keras_model=None,
    keras_model_path=None,
    custom_objects=None,
    model_dir=None,
    config=None,
    checkpoint_format='saver'):
  """Constructs an `Estimator` instance from given keras model.

  For usage example, please see:
  [Creating estimators from Keras
  Models](https://tensorflow.org/guide/estimators#model_to_estimator).

  Args:
    keras_model: A compiled Keras model object. This argument is mutually
      exclusive with `keras_model_path`.
    keras_model_path: Path to a compiled Keras model saved on disk, in HDF5
      format, which can be generated with the `save()` method of a Keras model.
      This argument is mutually exclusive with `keras_model`.
    custom_objects: Dictionary for custom objects.
    model_dir: Directory to save `Estimator` model parameters, graph, summary
      files for TensorBoard, etc.
    config: `RunConfig` to config `Estimator`.
    checkpoint_format: Sets the format of the checkpoint saved by the estimator
      when training. May be `saver` or `checkpoint`, depending on whether to
      save checkpoints from `tf.train.Saver` or `tf.train.Checkpoint`. This
      argument currently defaults to `saver`. When 2.0 is released, the default
      will be `checkpoint`. Estimators use name-based `tf.train.Saver`
      checkpoints, while Keras models use object-based checkpoints from
      `tf.train.Checkpoint`. Currently, saving object-based checkpoints from
      `model_to_estimator` is only supported by Functional and Sequential
      models.

  Returns:
    An Estimator from given keras model.

  Raises:
    ValueError: if neither keras_model nor keras_model_path was given.
    ValueError: if both keras_model and keras_model_path was given.
    ValueError: if the keras_model_path is a GCS URI.
    ValueError: if keras_model has not been compiled.
    ValueError: if an invalid checkpoint_format was given.
  """
  try:
    from tensorflow_estimator.python.estimator import keras as keras_lib  # pylint: disable=g-import-not-at-top
  except ImportError:
    raise NotImplementedError(
        'tf.keras.estimator.model_to_estimator function not available in your '
        'installation.')
  return keras_lib.model_to_estimator(  # pylint:disable=unexpected-keyword-arg
      keras_model=keras_model,
      keras_model_path=keras_model_path,
      custom_objects=custom_objects,
      model_dir=model_dir,
      config=config,
      checkpoint_format=checkpoint_format)

@keras_export('keras.estimator.model_to_estimator', v1=[])
def model_to_estimator_v2(
    keras_model=None,
    keras_model_path=None,
    custom_objects=None,
    model_dir=None,
    config=None,
    checkpoint_format='checkpoint'):
  """Constructs an `Estimator` instance from given keras model.

  For usage example, please see:
  [Creating estimators from Keras
  Models](https://tensorflow.org/guide/estimators#model_to_estimator).

  Args:
    keras_model: A compiled Keras model object. This argument is mutually
      exclusive with `keras_model_path`.
    keras_model_path: Path to a compiled Keras model saved on disk, in HDF5
      format, which can be generated with the `save()` method of a Keras model.
      This argument is mutually exclusive with `keras_model`.
    custom_objects: Dictionary for custom objects.
    model_dir: Directory to save `Estimator` model parameters, graph, summary
      files for TensorBoard, etc.
    config: `RunConfig` to config `Estimator`.
    checkpoint_format: Sets the format of the checkpoint saved by the estimator
      when training. May be `saver` or `checkpoint`, depending on whether to
      save checkpoints from `tf.compat.v1.train.Saver` or `tf.train.Checkpoint`.
      The default is `checkpoint`. Estimators use name-based `tf.train.Saver`
      checkpoints, while Keras models use object-based checkpoints from
      `tf.train.Checkpoint`. Currently, saving object-based checkpoints from
      `model_to_estimator` is only supported by Functional and Sequential
      models.

  Returns:
    An Estimator from given keras model.

  Raises:
    ValueError: if neither keras_model nor keras_model_path was given.
    ValueError: if both keras_model and keras_model_path was given.
    ValueError: if the keras_model_path is a GCS URI.
    ValueError: if keras_model has not been compiled.
    ValueError: if an invalid checkpoint_format was given.
  """
  try:
    from tensorflow_estimator.python.estimator import keras as keras_lib  # pylint: disable=g-import-not-at-top
  except ImportError:
    raise NotImplementedError(
        'tf.keras.estimator.model_to_estimator function not available in your '
        'installation.')
  return keras_lib.model_to_estimator(  # pylint:disable=unexpected-keyword-arg
      keras_model=keras_model,
      keras_model_path=keras_model_path,
      custom_objects=custom_objects,
      model_dir=model_dir,
      config=config,
      checkpoint_format=checkpoint_format)
# LINT.ThenChange(//tensorflow_estimator/python/estimator/keras.py)

<commit_msg>Support `sample_weights` in `model_to_estimator`.<commit_after># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Keras estimator API."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.util.tf_export import keras_export

# Keras has undeclared dependency on tensorflow/estimator:estimator_py.
# As long as you depend //third_party/py/tensorflow:tensorflow target
# everything will work as normal.

# LINT.IfChange
@keras_export(v1=['keras.estimator.model_to_estimator'])
def model_to_estimator(
    keras_model=None,
    keras_model_path=None,
    custom_objects=None,
    model_dir=None,
    config=None,
    checkpoint_format='saver'):
  """Constructs an `Estimator` instance from given keras model.

  For usage example, please see:
  [Creating estimators from Keras
  Models](https://tensorflow.org/guide/estimators#model_to_estimator).

  __Sample Weights__
  Estimators returned by `model_to_estimator` are configured to handle sample
  weights (similar to `keras_model.fit(x, y, sample_weights)`). To pass sample
  weights when training or evaluating the Estimator, the first item returned by
  the input function should be a dictionary with keys `features` and
  `sample_weights`. Example below:

  ```
  keras_model = tf.keras.Model(...)
  keras_model.compile(...)

  estimator = tf.keras.estimator.model_to_estimator(keras_model)

  def input_fn():
    return dataset_ops.Dataset.from_tensors(
        ({'features': features, 'sample_weights': sample_weights},
         targets))

  estimator.train(input_fn, steps=1)
  ```

  Args:
    keras_model: A compiled Keras model object. This argument is mutually
      exclusive with `keras_model_path`.
    keras_model_path: Path to a compiled Keras model saved on disk, in HDF5
      format, which can be generated with the `save()` method of a Keras model.
      This argument is mutually exclusive with `keras_model`.
    custom_objects: Dictionary for custom objects.
    model_dir: Directory to save `Estimator` model parameters, graph, summary
      files for TensorBoard, etc.
    config: `RunConfig` to config `Estimator`.
    checkpoint_format: Sets the format of the checkpoint saved by the estimator
      when training. May be `saver` or `checkpoint`, depending on whether to
      save checkpoints from `tf.train.Saver` or `tf.train.Checkpoint`. This
      argument currently defaults to `saver`. When 2.0 is released, the default
      will be `checkpoint`. Estimators use name-based `tf.train.Saver`
      checkpoints, while Keras models use object-based checkpoints from
      `tf.train.Checkpoint`. Currently, saving object-based checkpoints from
      `model_to_estimator` is only supported by Functional and Sequential
      models.

  Returns:
    An Estimator from given keras model.

  Raises:
    ValueError: if neither keras_model nor keras_model_path was given.
    ValueError: if both keras_model and keras_model_path was given.
    ValueError: if the keras_model_path is a GCS URI.
    ValueError: if keras_model has not been compiled.
    ValueError: if an invalid checkpoint_format was given.
  """
  try:
    from tensorflow_estimator.python.estimator import keras as keras_lib  # pylint: disable=g-import-not-at-top
  except ImportError:
    raise NotImplementedError(
        'tf.keras.estimator.model_to_estimator function not available in your '
        'installation.')
  return keras_lib.model_to_estimator(  # pylint:disable=unexpected-keyword-arg
      keras_model=keras_model,
      keras_model_path=keras_model_path,
      custom_objects=custom_objects,
      model_dir=model_dir,
      config=config,
      checkpoint_format=checkpoint_format)

@keras_export('keras.estimator.model_to_estimator', v1=[])
def model_to_estimator_v2(
    keras_model=None,
    keras_model_path=None,
    custom_objects=None,
    model_dir=None,
    config=None,
    checkpoint_format='checkpoint'):
  """Constructs an `Estimator` instance from given keras model.

  For usage example, please see:
  [Creating estimators from Keras
  Models](https://tensorflow.org/guide/estimators#model_to_estimator).

  Args:
    keras_model: A compiled Keras model object. This argument is mutually
      exclusive with `keras_model_path`.
    keras_model_path: Path to a compiled Keras model saved on disk, in HDF5
      format, which can be generated with the `save()` method of a Keras model.
      This argument is mutually exclusive with `keras_model`.
    custom_objects: Dictionary for custom objects.
    model_dir: Directory to save `Estimator` model parameters, graph, summary
      files for TensorBoard, etc.
    config: `RunConfig` to config `Estimator`.
    checkpoint_format: Sets the format of the checkpoint saved by the estimator
      when training. May be `saver` or `checkpoint`, depending on whether to
      save checkpoints from `tf.compat.v1.train.Saver` or `tf.train.Checkpoint`.
      The default is `checkpoint`. Estimators use name-based `tf.train.Saver`
      checkpoints, while Keras models use object-based checkpoints from
      `tf.train.Checkpoint`. Currently, saving object-based checkpoints from
      `model_to_estimator` is only supported by Functional and Sequential
      models.

  Returns:
    An Estimator from given keras model.

  Raises:
    ValueError: if neither keras_model nor keras_model_path was given.
    ValueError: if both keras_model and keras_model_path was given.
    ValueError: if the keras_model_path is a GCS URI.
    ValueError: if keras_model has not been compiled.
    ValueError: if an invalid checkpoint_format was given.
  """
  try:
    from tensorflow_estimator.python.estimator import keras as keras_lib  # pylint: disable=g-import-not-at-top
  except ImportError:
    raise NotImplementedError(
        'tf.keras.estimator.model_to_estimator function not available in your '
        'installation.')
  return keras_lib.model_to_estimator(  # pylint:disable=unexpected-keyword-arg
      keras_model=keras_model,
      keras_model_path=keras_model_path,
      custom_objects=custom_objects,
      model_dir=model_dir,
      config=config,
      checkpoint_format=checkpoint_format)
# LINT.ThenChange(//tensorflow_estimator/python/estimator/keras.py)

<|endoftext|>


<commit_before>import unittest
from contextlib import contextmanager

import warnings
warnings.simplefilter('default')
warnings.filterwarnings('error', category=DeprecationWarning, module='compdb')

# Make sure the storages created for this test are unique.
import uuid
test_token = {'test_token': str(uuid.uuid4())}

def test_doc():
    import uuid
    ret = dict(test_token)
    ret.update({
        'name' : str(uuid.uuid4())})
    return ret

def test_data():
    import uuid
    return str(uuid.uuid4())

@contextmanager
def remove_files(storage, spec):
    yield
    docs = storage.find(spec)
    for doc in docs:
        storage.delete(doc['_id'])

class StorageTest(unittest.TestCase):

    def setUp(self):
        import tempfile
        from pymongo import MongoClient
        from compdb.db import Storage
        from compdb.core.config import load_config
        config = load_config()
        client = MongoClient(config['database_host'])
        db = client['testing']
        self._mc = db['compdb_storage_test']
        self._tmp_fs = tempfile.TemporaryDirectory()
        self._storage = Storage(self._mc, self._tmp_fs.name)

    def tearDown(self):
        self._tmp_fs.cleanup()
        self._mc.drop()

    def test_new_file(self):
        storage = self._storage
        doc = test_doc()
        with remove_files(storage, doc):
            data = test_data()
            with storage.new_file(** doc) as file:
                file.write(data.encode())

    def test_delete_file(self):
        import os.path
        storage = self._storage
        doc = test_doc()
        with remove_files(storage, doc):
            data = test_data()
            with storage.new_file(** doc) as file:
                file.write(data.encode())
        self.assertEqual(len(list(storage.find(doc))), 0)

    def test_find(self):
        import uuid
        storage = self._storage
        doc = test_doc()
        data = test_data()
        with remove_files(storage, doc):
            with storage.new_file(** doc) as file:
                file.write(data.encode())
            files = storage.find(dict(name = 'my_file'))
            for file in files:
                read_back = file.read().decode()
                self.assertEqual(data, read_back)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Remove db_storage unit test.<commit_after><|endoftext|>


<commit_before>import vobject

from django.http import HttpResponse

EVENT_ITEMS = (
    ('uid', 'uid'),
    ('dtstart', 'start'),
    ('dtend', 'end'),
    ('summary', 'summary'),
    ('location', 'location'),
    ('last_modified', 'last_modified'),
    ('created', 'created'),
)

class ICalendarFeed(object):

    def __call__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs

        cal = vobject.iCalendar()

        for item in self.items():

            event = cal.add('vevent')

            for vkey, key in EVENT_ITEMS:
                value = getattr(self, 'item_' + key)(item)
                if value:
                    event.add(vkey).value = value

        response = HttpResponse(cal.serialize())
        response['Content-Type'] = 'text/calendar'

        return response

    def items(self):
        return []

    def item_uid(self, item):
        pass

    def item_start(self, item):
        pass

    def item_end(self, item):
        pass

    def item_summary(self, item):
        return str(item)

    def item_location(self, item):
        pass

    def item_last_modified(self, item):
        pass

    def item_created(self, item):
        pass
<commit_msg>Added recurrence to icalendar feed.<commit_after>import vobject
import pytz
import settings

from django.http import HttpResponse

EVENT_ITEMS = (
    ('uid', 'uid'),
    ('dtstart', 'start'),
    ('dtend', 'end'),
    ('summary', 'summary'),
    ('location', 'location'),
    ('last_modified', 'last_modified'),
    ('created', 'created'),
)

FREQNAMES = ['YEARLY','MONTHLY','WEEKLY','DAILY','HOURLY','MINUTELY','SECONDLY']

def reqstr(self):
    parts = ['FREQ='+FREQNAMES[self._freq]]
    if self._interval != 1:
        parts.append('INTERVAL='+str(self._interval))
    if self._wkst:
        parts.append('WKST='+str(self._wkst))
    if self._count:
        parts.append('COUNT='+str(self._count))

    for name, value in [
            ('BYSETPOS', self._bysetpos),
            ('BYMONTH', self._bymonth),
            ('BYMONTHDAY', self._bymonthday),
            ('BYYEARDAY', self._byyearday),
            ('BYWEEKNO', self._byweekno),
            ('BYWEEKDAY', self._byweekday),
            ('BYHOUR', self._byhour),
            ('BYMINUTE', self._byminute),
            ('BYSECOND', self._bysecond),
            ]:
        if value:
            parts.append(name+'='+','.join(str(v) for v in value))

    return ';'.join(parts)

class ICalendarFeed(object):

    def __call__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs

        cal = vobject.iCalendar()
        tz = pytz.timezone(settings.TIME_ZONE)

        for item in self.items():

            event = cal.add('vevent')

            for vkey, key in EVENT_ITEMS:
                value = getattr(self, 'item_' + key)(item)
                if value:
                    event.add(vkey).value = value

            rule = item.get_rrule_object(tz)
            event.add('rrule').value = reqstr(rule)

        response = HttpResponse(cal.serialize())
        response['Content-Type'] = 'text/calendar'

        return response

    def items(self):
        return []

    def item_uid(self, item):
        pass

    def item_start(self, item):
        pass

    def item_end(self, item):
        pass

    def item_summary(self, item):
        return str(item)

    def item_location(self, item):
        pass

    def item_last_modified(self, item):
        pass

    def item_created(self, item):
        pass
<|endoftext|>


<commit_before>from dispatch import Signal
import unittest
import sys
import gc

if sys.platform.startswith('java'):
    def garbage_collect():
        """Run the garbage collector and wait a bit to let it do his work"""
        import time
        gc.collect()
        time.sleep(0.1)
else:
    def garbage_collect():
        gc.collect()

def receiver_1_arg(val, **kwargs):
    return val

class Callable(object):
    def __call__(self, val, **kwargs):
        return val

    def a(self, val, **kwargs):
        return val

a_signal = Signal(providing_args=["val"])

class DispatcherTests(unittest.TestCase):
    """Test suite for dispatcher (barely started)"""

    def _testIsClean(self, signal):
        """Assert that everything has been cleaned up automatically"""
        self.assertEqual(signal.receivers, [])

        # force cleanup just in case
        signal.receivers = []

    def testExact(self):
        a_signal.connect(receiver_1_arg, sender=self)
        expected = [(receiver_1_arg,"test")]
        result = a_signal.send(sender=self, val="test")
        self.assertEqual(result, expected)
        a_signal.disconnect(receiver_1_arg, sender=self)
        self._testIsClean(a_signal)

    def testIgnoredSender(self):
        a_signal.connect(receiver_1_arg)
        expected = [(receiver_1_arg,"test")]
        result = a_signal.send(sender=self, val="test")
        self.assertEqual(result, expected)
        a_signal.disconnect(receiver_1_arg)
        self._testIsClean(a_signal)

    def testGarbageCollected(self):
        a = Callable()
        a_signal.connect(a.a, sender=self)
        expected = []
        del a
        garbage_collect()
        result = a_signal.send(sender=self, val="test")
        self.assertEqual(result, expected)
        self._testIsClean(a_signal)

    def testMultipleRegistration(self):
        a = Callable()
        a_signal.connect(a)
        a_signal.connect(a)
        a_signal.connect(a)
        a_signal.connect(a)
        a_signal.connect(a)
        a_signal.connect(a)
        result = a_signal.send(sender=self, val="test")
        self.assertEqual(len(result), 1)
        self.assertEqual(len(a_signal.receivers), 1)
        del a
        del result
        garbage_collect()
        self._testIsClean(a_signal)

    def testUidRegistration(self):
        def uid_based_receiver_1(**kwargs):
            pass

        def uid_based_receiver_2(**kwargs):
            pass

        a_signal.connect(uid_based_receiver_1, dispatch_uid = "uid")
        a_signal.connect(uid_based_receiver_2, dispatch_uid = "uid")
        self.assertEqual(len(a_signal.receivers), 1)
        a_signal.disconnect(dispatch_uid = "uid")
        self._testIsClean(a_signal)

    def testRobust(self):
        """Test the sendRobust function"""
        def fails(val, **kwargs):
            raise ValueError('this')
        a_signal.connect(fails)
        result = a_signal.send_robust(sender=self, val="test")
        err = result[0][1]
        self.assert_(isinstance(err, ValueError))
        self.assertEqual(err.args, ('this',))
        a_signal.disconnect(fails)
        self._testIsClean(a_signal)

    def testDisconnection(self):
        receiver_1 = Callable()
        receiver_2 = Callable()
        receiver_3 = Callable()
        a_signal.connect(receiver_1)
        a_signal.connect(receiver_2)
        a_signal.connect(receiver_3)
        a_signal.disconnect(receiver_1)
        del receiver_2
        garbage_collect()
        a_signal.disconnect(receiver_3)
        self._testIsClean(a_signal)

def getSuite():
    return unittest.makeSuite(DispatcherTests,'test')

if __name__ == "__main__":
    unittest.main()
<commit_msg>test sender filtering by instance<commit_after>from dispatch import Signal
import unittest
import sys
import gc

if sys.platform.startswith('java'):
    def garbage_collect():
        """Run the garbage collector and wait a bit to let it do his work"""
        import time
        gc.collect()
        time.sleep(0.1)
else:
    def garbage_collect():
        gc.collect()

def receiver_1_arg(val, **kwargs):
    return val

class Callable(object):
    def __call__(self, val, **kwargs):
        return val

    def a(self, val, **kwargs):
        return val

class Sender(object):
    signal = Signal(providing_args=["val"])
    def send(self):
        self.signal.send(sender=self, val="test")

a_signal = Signal(providing_args=["val"])

class DispatcherTests(unittest.TestCase):
    """Test suite for dispatcher (barely started)"""

    def _testIsClean(self, signal):
        """Assert that everything has been cleaned up automatically"""
        self.assertEqual(signal.receivers, [])

        # force cleanup just in case
        signal.receivers = []

    def testExact(self):
        a_signal.connect(receiver_1_arg, sender=self)
        expected = [(receiver_1_arg,"test")]
        result = a_signal.send(sender=self, val="test")
        self.assertEqual(result, expected)
        a_signal.disconnect(receiver_1_arg, sender=self)
        self._testIsClean(a_signal)

    def testInstanceSender(self):
        s1 = Sender()
        s2 = Sender()
        Sender.signal.connect(receiver_1_arg, sender=s1)
        result = Sender.signal.send(sender=s2, val="test")
        expected = []
        self.assertEqual(result, expected)
        result = Sender.signal.send(sender=s1, val="test")
        expected = [(receiver_1_arg, "test")]
        self.assertEqual(result, expected)

    def testIgnoredSender(self):
        a_signal.connect(receiver_1_arg)
        expected = [(receiver_1_arg,"test")]
        result = a_signal.send(sender=self, val="test")
        self.assertEqual(result, expected)
        a_signal.disconnect(receiver_1_arg)
        self._testIsClean(a_signal)

    def testGarbageCollected(self):
        a = Callable()
        a_signal.connect(a.a, sender=self)
        expected = []
        del a
        garbage_collect()
        result = a_signal.send(sender=self, val="test")
        self.assertEqual(result, expected)
        self._testIsClean(a_signal)

    def testMultipleRegistration(self):
        a = Callable()
        a_signal.connect(a)
        a_signal.connect(a)
        a_signal.connect(a)
        a_signal.connect(a)
        a_signal.connect(a)
        a_signal.connect(a)
        result = a_signal.send(sender=self, val="test")
        self.assertEqual(len(result), 1)
        self.assertEqual(len(a_signal.receivers), 1)
        del a
        del result
        garbage_collect()
        self._testIsClean(a_signal)

    def testUidRegistration(self):
        def uid_based_receiver_1(**kwargs):
            pass

        def uid_based_receiver_2(**kwargs):
            pass

        a_signal.connect(uid_based_receiver_1, dispatch_uid = "uid")
        a_signal.connect(uid_based_receiver_2, dispatch_uid = "uid")
        self.assertEqual(len(a_signal.receivers), 1)
        a_signal.disconnect(dispatch_uid = "uid")
        self._testIsClean(a_signal)

    def testRobust(self):
        """Test the sendRobust function"""
        def fails(val, **kwargs):
            raise ValueError('this')
        a_signal.connect(fails)
        result = a_signal.send_robust(sender=self, val="test")
        err = result[0][1]
        self.assert_(isinstance(err, ValueError))
        self.assertEqual(err.args, ('this',))
        a_signal.disconnect(fails)
        self._testIsClean(a_signal)

    def testDisconnection(self):
        receiver_1 = Callable()
        receiver_2 = Callable()
        receiver_3 = Callable()
        a_signal.connect(receiver_1)
        a_signal.connect(receiver_2)
        a_signal.connect(receiver_3)
        a_signal.disconnect(receiver_1)
        del receiver_2
        garbage_collect()
        a_signal.disconnect(receiver_3)
        self._testIsClean(a_signal)

def getSuite():
    return unittest.makeSuite(DispatcherTests,'test')

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>if __name__ == '__main__':
    from run_tests import setup_environ
    setup_environ()

import json
import common
import base64
import hashlib
import unittest2
from mock import patch, MagicMock

from google.appengine.api import memcache, urlfetch

content = json.dumps({
    "tree": [{
        "type": "blob",
        "path": "README.md",
        "mode": "100644",
        "sha": "ac178f6489f2d3f601df6a9a5e641b62a0388eae",
        "size": 314
    }]
})
mock_authed_fetch = MagicMock()
mock_authed_fetch.return_value.raw = MagicMock(read=lambda: content)
mock_authed_fetch.return_value.content = content

class TestDTMMUtils(common.DMSTestCase):
    @patch('google.appengine.api.urlfetch.fetch')
    def test_authed_fetch(self, fetch):
        fetch.return_value.headers = {'x-ratelimit-remaining': 'lots'}
        fetch.return_value.content = (
            'Lorem ipsum dolor sit amet, consectetur adipisicing elit.')

        import dtmm_utils
        end_data = dtmm_utils.authed_fetch('http://mock.com')
        self.assertEqual(fetch.return_value.content, end_data.content)

        fetch.return_value.headers['x-ratelimit-remaining'] = None
        end_data = dtmm_utils.authed_fetch('http://mock.com')

    @patch('dtmm_utils.authed_fetch', mock_authed_fetch)
    def test_get_url_content_fetch_from_remote(self):

        import dtmm_utils
        end_data = dtmm_utils.get_url_content(None, 'http://mock.com')
        self.assertEqual(
            end_data,
            {
                u'tree': [{
                    u'sha': u'ac178f6489f2d3f601df6a9a5e641b62a0388eae',
                    u'mode': u'100644',
                    u'path': u'README.md',
                    u'type': u'blob',
                    u'size': 314
                }]
            }
        )

    def test_get_url_content_retrieve_from_memcache(self):
        import dtmm_utils

        url_digest = hashlib.md5('http://mock.com').hexdigest()

        memcache.set(url_digest, {'content': 'word'})
        end_data = dtmm_utils.get_url_content(None, 'http://mock.com')
        self.assertEqual(end_data, {'content': 'word'})

    @patch('dtmm_utils.authed_fetch')
    def test_get_url_content_download_error_handleing(self, authed_fetch):
        import dtmm_utils

        mock_handler = MagicMock()
        authed_fetch.side_effect = urlfetch.DownloadError

        dtmm_utils.get_url_content(mock_handler, 'http://mock.com')
        self.assertEqual(mock_handler.error.call_args[0][0], 408)

    @patch('dtmm_utils.authed_fetch', mock_authed_fetch)
    def test_get_tree(self):

        import dtmm_utils
        end_data = dtmm_utils.get_tree()
        self.assertEqual(
            end_data,
            [{
                u'sha': u'ac178f6489f2d3f601df6a9a5e641b62a0388eae',
                u'mode': u'100644',
                u'path': u'README.md',
                u'type': u'blob',
                u'size': 314
            }]
        )

    @patch('dtmm_utils.get_url_content')
    def test_get_live_module_data(self, get_url_content):
        get_url_content.return_value = {
            'content': base64.b64encode('''
                MODULE = {
                    Type = "Hardware",
                    Name = "HMD2043",
                    Version = "1.1",
                    SDescription = "Deprecated HMD2043 hardware device",
                    URL = "False URL"
                };''')
        }

        import dtmm_utils
        end_data = dtmm_utils.get_live_module_data(
            None, {"url": "http://mock.url/hardware_file"})
        self.assertEqual(
            end_data,
            {
                'URL': 'False URL',
                'SDescription': 'Deprecated HMD2043 hardware device',
                'Version': '1.1',
                'Type': 'Hardware',
                'Name': 'HMD2043'
            }
        )

    @patch('dtmm_utils.get_url_content')
    def test_get_live_hardware_data(self, get_url_content):
        get_url_content.return_value = {
            'content': base64.b64encode('''
                HARDWARE = {
                    ID = 0x74fa4cae,
                    Version = 0x07c2,
                    Manufacturer = 0x21544948 -- HAROLD_IT
                };''')
        }

        import dtmm_utils
        end_data = dtmm_utils.get_live_hardware_data(
            None, {"url": "http://mock.url/hardware_file"})
        self.assertEqual(
            end_data,
            {
                'Version': 1986,
                'ID': 1962560686,
                'Manufacturer': 559171912
            }
        )

def main():
    unittest2.main()

if __name__ == '__main__':
    main()
<commit_msg>Split out authed_fetch tests<commit_after>if __name__ == '__main__':
    from run_tests import setup_environ
    setup_environ()

import json
import common
import base64
import hashlib
import unittest2
from mock import patch, MagicMock

from google.appengine.api import memcache, urlfetch

content = json.dumps({
    "tree": [{
        "type": "blob",
        "path": "README.md",
        "mode": "100644",
        "sha": "ac178f6489f2d3f601df6a9a5e641b62a0388eae",
        "size": 314
    }]
})
mock_authed_fetch = MagicMock()
mock_authed_fetch.return_value.raw = MagicMock(read=lambda: content)
mock_authed_fetch.return_value.content = content

class TestDTMMUtils(common.DMSTestCase):
    @patch('google.appengine.api.urlfetch.fetch')
    def test_authed_fetch_with_remaining(self, fetch):
        fetch.return_value.headers = {'x-ratelimit-remaining': 'lots'}
        fetch.return_value.content = (
            'Lorem ipsum dolor sit amet, consectetur adipisicing elit.')

        import dtmm_utils
        end_data = dtmm_utils.authed_fetch('http://mock.com')
        self.assertEqual(fetch.return_value.content, end_data.content)

    @patch('google.appengine.api.urlfetch.fetch')
    def test_authed_fetch_without_remaining(self, fetch):
        fetch.return_value.headers['x-ratelimit-remaining'] = None
        fetch.return_value.content = (
            'Lorem ipsum dolor sit amet, consectetur adipisicing elit.')

        import dtmm_utils
        dtmm_utils.authed_fetch('http://mock.com')

        # a little unsure how to ensure correct behaviour here

    @patch('dtmm_utils.authed_fetch', mock_authed_fetch)
    def test_get_url_content_fetch_from_remote(self):

        import dtmm_utils
        end_data = dtmm_utils.get_url_content(None, 'http://mock.com')
        self.assertEqual(
            end_data,
            {
                u'tree': [{
                    u'sha': u'ac178f6489f2d3f601df6a9a5e641b62a0388eae',
                    u'mode': u'100644',
                    u'path': u'README.md',
                    u'type': u'blob',
                    u'size': 314
                }]
            }
        )

    def test_get_url_content_retrieve_from_memcache(self):
        import dtmm_utils

        url_digest = hashlib.md5('http://mock.com').hexdigest()

        memcache.set(url_digest, {'content': 'word'})
        end_data = dtmm_utils.get_url_content(None, 'http://mock.com')
        self.assertEqual(end_data, {'content': 'word'})

    @patch('dtmm_utils.authed_fetch')
    def test_get_url_content_download_error_handleing(self, authed_fetch):
        import dtmm_utils

        mock_handler = MagicMock()
        authed_fetch.side_effect = urlfetch.DownloadError

        dtmm_utils.get_url_content(mock_handler, 'http://mock.com')
        self.assertEqual(mock_handler.error.call_args[0][0], 408)

    @patch('dtmm_utils.authed_fetch', mock_authed_fetch)
    def test_get_tree(self):

        import dtmm_utils
        end_data = dtmm_utils.get_tree()
        self.assertEqual(
            end_data,
            [{
                u'sha': u'ac178f6489f2d3f601df6a9a5e641b62a0388eae',
                u'mode': u'100644',
                u'path': u'README.md',
                u'type': u'blob',
                u'size': 314
            }]
        )

    @patch('dtmm_utils.get_url_content')
    def test_get_live_module_data(self, get_url_content):
        get_url_content.return_value = {
            'content': base64.b64encode('''
                MODULE = {
                    Type = "Hardware",
                    Name = "HMD2043",
                    Version = "1.1",
                    SDescription = "Deprecated HMD2043 hardware device",
                    URL = "False URL"
                };''')
        }

        import dtmm_utils
        end_data = dtmm_utils.get_live_module_data(
            None, {"url": "http://mock.url/hardware_file"})
        self.assertEqual(
            end_data,
            {
                'URL': 'False URL',
                'SDescription': 'Deprecated HMD2043 hardware device',
                'Version': '1.1',
                'Type': 'Hardware',
                'Name': 'HMD2043'
            }
        )

    @patch('dtmm_utils.get_url_content')
    def test_get_live_hardware_data(self, get_url_content):
        get_url_content.return_value = {
            'content': base64.b64encode('''
                HARDWARE = {
                    ID = 0x74fa4cae,
                    Version = 0x07c2,
                    Manufacturer = 0x21544948 -- HAROLD_IT
                };''')
        }

        import dtmm_utils
        end_data = dtmm_utils.get_live_hardware_data(
            None, {"url": "http://mock.url/hardware_file"})
        self.assertEqual(
            end_data,
            {
                'Version': 1986,
                'ID': 1962560686,
                'Manufacturer': 559171912
            }
        )

def main():
    unittest2.main()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Solve the orthogonal Procrustes problem.

"""
from __future__ import division, print_function, absolute_import

import numpy as np
from scipy.linalg import svd

__all__ = ['orthogonal_procrustes']

def orthogonal_procrustes(A, B, check_finite=True):
    """
    Compute the matrix solution of the orthogonal Procrustes problem.

    Given matrices A and B of equal shape, find an orthogonal matrix R
    that most closely maps A to B [1]_.
    Note that unlike higher level Procrustes analyses of spatial data,
    this function only uses orthogonal transformations like rotations
    and reflections, and it does not use scaling or translation.

    Parameters
    ----------
    A : (M, N) array_like
        Matrix to be mapped.
    B : (M, N) array_like
        Target matrix.

    Returns
    -------
    R : (N, N) ndarray
        The matrix solution of the orthogonal Procrustes problem.
        Minimizes the Frobenius norm of dot(A, R) - B, subject to
        dot(R.T, R) == I.

    References
    ----------
    .. [1] Peter H. Schonemann, "A generalized solution of the orthogonal
           Procrustes problem", Psychometrica -- Vol. 31, No. 1, March, 1996.

    """
    if check_finite:
        A = np.asarray_chkfinite(A)
        B = np.asarray_chkfinite(B)
    else:
        A = np.asarray(A)
        B = np.asarray(B)
    if A.ndim != 2:
        raise ValueError('expected ndim to be 2, but observed %s' % A.ndim)
    if A.shape != B.shape:
        raise ValueError('the shapes of A and B differ (%s vs %s)' % (
            A.shape, B.shape))
    u, w, vt = svd(A.T.dot(B))
    return u.dot(vt)

<commit_msg>MAINT: replace asarray with more specific format request<commit_after>"""
Solve the orthogonal Procrustes problem.

"""
from __future__ import division, print_function, absolute_import

import numpy as np
from scipy.linalg import svd

__all__ = ['orthogonal_procrustes']

def orthogonal_procrustes(A, B, check_finite=True):
    """
    Compute the matrix solution of the orthogonal Procrustes problem.

    Given matrices A and B of equal shape, find an orthogonal matrix R
    that most closely maps A to B [1]_.
    Note that unlike higher level Procrustes analyses of spatial data,
    this function only uses orthogonal transformations like rotations
    and reflections, and it does not use scaling or translation.

    Parameters
    ----------
    A : (M, N) array_like
        Matrix to be mapped.
    B : (M, N) array_like
        Target matrix.

    Returns
    -------
    R : (N, N) ndarray
        The matrix solution of the orthogonal Procrustes problem.
        Minimizes the Frobenius norm of dot(A, R) - B, subject to
        dot(R.T, R) == I.

    References
    ----------
    .. [1] Peter H. Schonemann, "A generalized solution of the orthogonal
           Procrustes problem", Psychometrica -- Vol. 31, No. 1, March, 1996.

    """
    if check_finite:
        A = np.asarray_chkfinite(A)
        B = np.asarray_chkfinite(B)
    else:
        A = np.ascontiguousarray(A)
        B = np.asfortranarray(B)
    if A.ndim != 2:
        raise ValueError('expected ndim to be 2, but observed %s' % A.ndim)
    if A.shape != B.shape:
        raise ValueError('the shapes of A and B differ (%s vs %s)' % (
            A.shape, B.shape))
    u, w, vt = svd(A.T.dot(B))
    return u.dot(vt)

<|endoftext|>


<commit_before>"""Tests the experiment module of pyexperiment

Written by Peter Duerr
"""

from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import

import unittest
import argparse
import sys
import io

from pyexperiment import experiment
from pyexperiment.utils.stdout_redirector import stdout_redirector, \
    stdout_err_redirector

class TestExperiment(unittest.TestCase):
    """Test the experiment module
    """
    def setUp(self):
        """Setup the test fixure
        """
        pass

    def tearDown(self):
        """Tear down the test fixure
        """
        pass

    def test_main_runs_function(self):
        """Test running main calls function
        """
        run = [False]

        def custom_function():
            """User function
            """
            run[0] = True

        # Need to monkey patch arg parser here
        argparse._sys.argv = ["test", "custom_function"]

        buf = io.StringIO()
        with stdout_redirector(buf):
            experiment.main(commands=[custom_function])

        self.assertTrue(run[0])
        self.assertEqual(len(buf.getvalue()), 0)

    def test_main_does_not_run_function(self):
        """Test running main does not call unnecessary function but complains
        """
        run = [False]

        def custom_function():
            """User function
            """
            run[0] = True

        # Need to monkey patch arg parser here
        argparse._sys.argv = ["test", "help"]

        buf = io.StringIO()
        with stdout_redirector(buf):
            experiment.main(commands=[custom_function])

        self.assertFalse(run[0])
        self.assertNotEqual(len(buf.getvalue()), 0)

    def test_main_gives_help(self):
        """Test running help shows docstring
        """
        run = [False]

        def custom_function():
            """This should be printed!!
            """
            run[0] = True

        # Need to monkey patch arg parser here
        argparse._sys.argv = ["test", "help", "custom_function"]

        buf = io.StringIO()
        with stdout_redirector(buf):
            experiment.main(commands=[custom_function])

        self.assertFalse(run[0])
        self.assertIn("This should be printed!!", buf.getvalue())

    # Need to mock unittest to avoid printing
    # def test_main_runs_test(self):
    #     """Test running main calls tests
    #     """
    #     run = [False]

    #     class ExampleTest(unittest.TestCase):
    #         pass
    #     #     """Test case for the test
    #     #     """
    #     #     def custom_test(self):
    #     #         """Test something
    #     #         """
    #     #         run[0] = True

    #     # # Need to monkey patch arg parser and stdout here
    #     argparse._sys.argv = ["test", "test"]

    #     buf_out, buf_err = io.StringIO(), io.StringIO()
    #     with stdout_err_redirector(buf_out, buf_err):
    #         experiment.main(commands=[], tests=[ExampleTest])
<commit_msg>Remove unused imports<commit_after>"""Tests the experiment module of pyexperiment

Written by Peter Duerr
"""

from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import

import unittest
import argparse
import io

from pyexperiment import experiment
from pyexperiment.utils.stdout_redirector import stdout_redirector

class TestExperiment(unittest.TestCase):
    """Test the experiment module
    """
    def setUp(self):
        """Setup the test fixure
        """
        pass

    def tearDown(self):
        """Tear down the test fixure
        """
        pass

    def test_main_runs_function(self):
        """Test running main calls function
        """
        run = [False]

        def custom_function():
            """User function
            """
            run[0] = True

        # Need to monkey patch arg parser here
        argparse._sys.argv = ["test", "custom_function"]

        buf = io.StringIO()
        with stdout_redirector(buf):
            experiment.main(commands=[custom_function])

        self.assertTrue(run[0])
        self.assertEqual(len(buf.getvalue()), 0)

    def test_main_does_not_run_function(self):
        """Test running main does not call unnecessary function but complains
        """
        run = [False]

        def custom_function():
            """User function
            """
            run[0] = True

        # Need to monkey patch arg parser here
        argparse._sys.argv = ["test", "help"]

        buf = io.StringIO()
        with stdout_redirector(buf):
            experiment.main(commands=[custom_function])

        self.assertFalse(run[0])
        self.assertNotEqual(len(buf.getvalue()), 0)

    def test_main_gives_help(self):
        """Test running help shows docstring
        """
        run = [False]

        def custom_function():
            """This should be printed!!
            """
            run[0] = True

        # Need to monkey patch arg parser here
        argparse._sys.argv = ["test", "help", "custom_function"]

        buf = io.StringIO()
        with stdout_redirector(buf):
            experiment.main(commands=[custom_function])

        self.assertFalse(run[0])
        self.assertIn("This should be printed!!", buf.getvalue())

    # Need to mock unittest to avoid printing
    # def test_main_runs_test(self):
    #     """Test running main calls tests
    #     """
    #     run = [False]

    #     class ExampleTest(unittest.TestCase):
    #         pass
    #     #     """Test case for the test
    #     #     """
    #     #     def custom_test(self):
    #     #         """Test something
    #     #         """
    #     #         run[0] = True

    #     # # Need to monkey patch arg parser and stdout here
    #     argparse._sys.argv = ["test", "test"]

    #     buf_out, buf_err = io.StringIO(), io.StringIO()
    #     with stdout_err_redirector(buf_out, buf_err):
    #         experiment.main(commands=[], tests=[ExampleTest])
<|endoftext|>


<commit_before>from collections import OrderedDict
from itertools import groupby

import os
import urlparse

class Post:
    def __init__(self, api_post, run_time):
        self.api_post = api_post
        self.run_time = run_time

    def serialize(self):
        return OrderedDict([
            ('run_time', self.run_time),
            ('headline', self.headline),
            ('post_type', self.post_type),
            ('art_url', self.art_url),
            ('link_url', self.link_url),
        ])

    @property
    def id(self):
        return self.api_post['id']

    @property
    def headline(self):
        return self.api_post['name']

    @property
    def post_type(self):
        return self.api_post['type']

    @property
    def art_url(self):
        url = self.api_post['picture']
        params = urlparse.parse_qs(url[url.find('?'):])
        image_url = params['url'][0]
        return image_url[:image_url.find('?')]

    @property
    def link_url(self):
        url = self.api_post['link']
        return url[:url.find('?')]

class Insights:
    def __init__(self, post, api_insights):
        self.post = post
        self.api_insights = api_insights

    def serialize(self):
        return OrderedDict([
            ('shares', self.shares),
            ('likes', self.likes),
            ('comments', self.comments),
            ('people_reached', self.people_reached),
            ('photo_view_clicks', self.photo_view_clicks),
            ('link_clicks', self.link_clicks)
        ])

    def _insights(self):
        insights = {}

        for row in self.api_insights['data']:
            insights[row['name']] = row

        return insights

    def _post_stories_by_action_type(self, action_type):
        insight =  self._insights()['post_stories_by_action_type']
        value = insight['values'][0]['value']

        if not value:
            return 0

        if action_type not in value:
            return 0

        return value[action_type]

    def _post_consumptions_by_type(self, action_type):
        insight =  self._insights()['post_consumptions_by_type']
        value = insight['values'][0]['value']

        if not value:
            return 0

        if action_type not in value:
            return 0

        return value[action_type]

    @property
    def shares(self):
        return self._post_stories_by_action_type('share')

    @property
    def likes(self):
        return self._post_stories_by_action_type('like')

    @property
    def comments(self):
        return self._post_stories_by_action_type('comment')

    @property
    def people_reached(self):
        return self._insights()['post_impressions_unique']['values'][0]['value']

    @property
    def photo_view_clicks(self):
        return self._post_consumptions_by_type('photo view')

    @property
    def link_clicks(self):
        return self._post_consumptions_by_type('link clicks')

    @property
    def other_clicks(self):
        """
        Numbers for this metric don't seem to match what is displayed on the
        post insights popup. Leaving out of reporting.
        """
        return self._post_consumptions_by_type('other clicks')
<commit_msg>account for query string and non-querystring links in fb scraper<commit_after>from collections import OrderedDict
from itertools import groupby

import os
import urlparse

class Post:
    def __init__(self, api_post, run_time):
        self.api_post = api_post
        self.run_time = run_time

    def serialize(self):
        return OrderedDict([
            ('run_time', self.run_time),
            ('headline', self.headline),
            ('post_type', self.post_type),
            ('art_url', self.art_url),
            ('link_url', self.link_url),
        ])

    @property
    def id(self):
        return self.api_post['id']

    @property
    def headline(self):
        return self.api_post['name']

    @property
    def post_type(self):
        return self.api_post['type']

    @property
    def art_url(self):
        url = self.api_post['picture']
        params = urlparse.parse_qs(url[url.find('?'):])
        image_url = params['url'][0]
        if image_url.find('?') > -1:
            return image_url[:image_url.find('?')]
        else:
            return image_url

    @property
    def link_url(self):
        url = self.api_post['link']
        if url.find('?') > -1:
            return url[:url.find('?')]
        else:
            return url

class Insights:
    def __init__(self, post, api_insights):
        self.post = post
        self.api_insights = api_insights

    def serialize(self):
        return OrderedDict([
            ('shares', self.shares),
            ('likes', self.likes),
            ('comments', self.comments),
            ('people_reached', self.people_reached),
            ('photo_view_clicks', self.photo_view_clicks),
            ('link_clicks', self.link_clicks)
        ])

    def _insights(self):
        insights = {}

        for row in self.api_insights['data']:
            insights[row['name']] = row

        return insights

    def _post_stories_by_action_type(self, action_type):
        insight =  self._insights()['post_stories_by_action_type']
        value = insight['values'][0]['value']

        if not value:
            return 0

        if action_type not in value:
            return 0

        return value[action_type]

    def _post_consumptions_by_type(self, action_type):
        insight =  self._insights()['post_consumptions_by_type']
        value = insight['values'][0]['value']

        if not value:
            return 0

        if action_type not in value:
            return 0

        return value[action_type]

    @property
    def shares(self):
        return self._post_stories_by_action_type('share')

    @property
    def likes(self):
        return self._post_stories_by_action_type('like')

    @property
    def comments(self):
        return self._post_stories_by_action_type('comment')

    @property
    def people_reached(self):
        return self._insights()['post_impressions_unique']['values'][0]['value']

    @property
    def photo_view_clicks(self):
        return self._post_consumptions_by_type('photo view')

    @property
    def link_clicks(self):
        return self._post_consumptions_by_type('link clicks')

    @property
    def other_clicks(self):
        """
        Numbers for this metric don't seem to match what is displayed on the
        post insights popup. Leaving out of reporting.
        """
        return self._post_consumptions_by_type('other clicks')
<|endoftext|>