<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module contains the Organization Model.
"""

__authors__ = [
  '"Pawel Solyga" <pawel.solyga@gmail.com>',
]

from google.appengine.ext import db

from django.utils.translation import ugettext_lazy

import soc.models.group

class Organization(soc.models.group.Group):
  """Organization details.

  A Organization entity participates in the following relationships implemented
  as a db.ReferenceProperty elsewhere in another db.Model:

   reviewers)  a many:1 relationship associating Reviewers with
     a specific Organization. This relation is implemented as the
     'reviewers' back-reference Query of the Organization model 'org'
     reference.
  """

  #: Optional development mailing list.
  dev_mailing_list = db.StringProperty(required=False,
    verbose_name=ugettext_lazy('Development Mailing List'))
  dev_mailing_list.help_text = ugettext_lazy(
    'Mailing list email address, URL to sign-up page, etc.')

  member_template = db.ReferenceProperty(
    reference_class=soc.models.document.Document, required=False,
    collection_name='group_app_member_template',
    verbose_name=ugettext_lazy('Application template'))
  member_template.help_text = ugettext_lazy(
    'This template will be presented to potential members when they'
    ' apply to the group.')

<commit_msg>Changed the collection name and help text for Organization member template.<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module contains the Organization Model.
"""

__authors__ = [
  '"Pawel Solyga" <pawel.solyga@gmail.com>',
]

from google.appengine.ext import db

from django.utils.translation import ugettext_lazy

import soc.models.group

class Organization(soc.models.group.Group):
  """Organization details.

  A Organization entity participates in the following relationships implemented
  as a db.ReferenceProperty elsewhere in another db.Model:

   reviewers)  a many:1 relationship associating Reviewers with
     a specific Organization. This relation is implemented as the
     'reviewers' back-reference Query of the Organization model 'org'
     reference.
  """

  #: Optional development mailing list.
  dev_mailing_list = db.StringProperty(required=False,
    verbose_name=ugettext_lazy('Development Mailing List'))
  dev_mailing_list.help_text = ugettext_lazy(
    'Mailing list email address, URL to sign-up page, etc.')

  member_template = db.ReferenceProperty(
    reference_class=soc.models.document.Document, required=False,
    collection_name='org_app_member_template',
    verbose_name=ugettext_lazy('Application template'))
  member_template.help_text = ugettext_lazy(
    'This template will be presented to potential members when they'
    ' apply to the organization.')

<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst

import pytest
import numpy as np
from numpy.testing import assert_array_equal

from ..nduncertainty import (StdDevUncertainty, NDUncertainty,
                             IncompatibleUncertaintiesException,
                             UnknownUncertainty)
from ..nddata import NDData
from ... import units as u

# Regarding setter tests:
# No need to test setters since the uncertainty is considered immutable after
# creation except of the parent_nddata attribute and this accepts just
# everything.
# Additionally they should be covered by NDData, NDArithmeticMixin which rely
# on it

# Regarding propagate, _convert_uncert, _propagate_* tests:
# They should be covered by NDArithmeticMixin since there is generally no need
# to test them without this mixin.

# Regarding __getitem__ tests:
# Should be covered by NDSlicingMixin.

# Regarding StdDevUncertainty tests:
# This subclass only overrides the methods for propagation so the same
# they should be covered in NDArithmeticMixin.

# Not really fake but the minimum an uncertainty has to override not to be
# abstract.

class FakeUncertainty(NDUncertainty):

    @property
    def uncertainty_type(self):
        return 'fake'

    def _propagate_add(self, data, final_data):
        pass

    def _propagate_subtract(self, data, final_data):
        pass

    def _propagate_multiply(self, data, final_data):
        pass

    def _propagate_divide(self, data, final_data):
        pass

# Test the fake (added also StdDevUncertainty which should behave identical)

@pytest.mark.parametrize(('UncertClass'), [FakeUncertainty, StdDevUncertainty,
                                           UnknownUncertainty])
def test_init_fake_with_list(UncertClass):
    fake_uncert = UncertClass([1, 2, 3])
    assert_array_equal(fake_uncert.array, np.array([1, 2, 3]))
    # Copy makes no difference since casting a list to an np.ndarray always
    # makes a copy.
    # But let's give the uncertainty a unit too
    fake_uncert = UncertClass([1, 2, 3], unit=u.adu)
    assert_array_equal(fake_uncert.array, np.array([1, 2, 3]))
    assert fake_uncert.unit is u.adu

@pytest.mark.parametrize(('UncertClass'), [FakeUncertainty, StdDevUncertainty,
                                           UnknownUncertainty])
def test_init_fake_with_ndarray(UncertClass):
    uncert = np.arange(100).reshape(10, 10)
    fake_uncert = UncertClass(uncert)
    # Numpy Arrays are copied by default
    assert_array_equal(fake_uncert.array, uncert)
    assert fake_uncert.array is not uncert
    # Now try it without copy
    fake_uncert = UncertClass(uncert, copy=False)
    assert fake_uncert.array is uncert
    # let's provide a unit
    fake_uncert = UncertClass(uncert, unit=u.adu)
    assert_array_equal(fake_uncert.array, uncert)
    assert fake_uncert.array is not uncert
    assert fake_uncert.unit is u.adu

@pytest.mark.parametrize(('UncertClass'), [FakeUncertainty, StdDevUncertainty,
                                           UnknownUncertainty])
def test_init_fake_with_quantity(UncertClass):
    uncert = np.arange(10).reshape(2, 5) * u.adu
    fake_uncert = UncertClass(uncert)
    # Numpy Arrays are copied by default
    assert_array_equal(fake_uncert.array, uncert.value)
    assert fake_uncert.array is not uncert.value
    assert fake_uncert.unit is u.adu
    # Try without copy (should not work, quantity.value always returns a copy)
    fake_uncert = UncertClass(uncert, copy=False)
    assert fake_uncert.array is not uncert.value
    assert fake_uncert.unit is u.adu
    # Now try with an explicit unit parameter too
    fake_uncert = UncertClass(uncert, unit=u.m)
    assert_array_equal(fake_uncert.array, uncert.value)  # No conversion done
    assert fake_uncert.array is not uncert.value
    assert fake_uncert.unit is u.m  # It took the explicit one

@pytest.mark.parametrize(('UncertClass'), [FakeUncertainty, StdDevUncertainty,
                                           UnknownUncertainty])
def test_init_fake_with_fake(UncertClass):
    uncert = np.arange(5).reshape(5, 1)
    fake_uncert1 = UncertClass(uncert)
    fake_uncert2 = UncertClass(fake_uncert1)
    assert_array_equal(fake_uncert2.array, uncert)
    assert fake_uncert2.array is not uncert
    # Without making copies
    fake_uncert1 = UncertClass(uncert, copy=False)
    fake_uncert2 = UncertClass(fake_uncert1, copy=False)
    assert_array_equal(fake_uncert2.array, fake_uncert1.array)
    assert fake_uncert2.array is fake_uncert1.array
    # With a unit
    uncert = np.arange(5).reshape(5, 1) * u.adu
    fake_uncert1 = UncertClass(uncert)
    fake_uncert2 = UncertClass(fake_uncert1)
    assert_array_equal(fake_uncert2.array, uncert.value)
    assert fake_uncert2.array is not uncert.value
    assert fake_uncert2.unit is u.adu
    # With a unit and an explicit unit-parameter
    fake_uncert2 = UncertClass(fake_uncert1, unit=u.cm)
    assert_array_equal(fake_uncert2.array, uncert.value)
    assert fake_uncert2.array is not uncert.value
    assert fake_uncert2.unit is u.cm

@pytest.mark.parametrize(('UncertClass'), [FakeUncertainty, StdDevUncertainty,
                                           UnknownUncertainty])
def test_init_fake_with_somethingElse(UncertClass):
    # What about a dict?
    uncert = {'rdnoise': 2.9, 'gain': 0.6}
    fake_uncert = UncertClass(uncert)
    assert fake_uncert.array == uncert
    # We can pass a unit too but since we cannot do uncertainty propagation
    # the interpretation is up to the user
    fake_uncert = UncertClass(uncert, unit=u.s)
    assert fake_uncert.array == uncert
    assert fake_uncert.unit is u.s
    # So, now check what happens if copy is False
    fake_uncert = UncertClass(uncert, copy=False)
    assert fake_uncert.array == uncert
    assert id(fake_uncert) != id(uncert)
    # dicts cannot be referenced without copy
    # TODO : Find something that can be referenced without copy :-)

def test_init_fake_with_StdDevUncertainty():
    # Different instances of uncertainties are not directly convertible so this
    # should fail
    uncert = np.arange(5).reshape(5, 1)
    std_uncert = StdDevUncertainty(uncert)
    with pytest.raises(IncompatibleUncertaintiesException):
        FakeUncertainty(std_uncert)
    # Ok try it the other way around
    fake_uncert = FakeUncertainty(uncert)
    with pytest.raises(IncompatibleUncertaintiesException):
        StdDevUncertainty(fake_uncert)

def test_uncertainty_type():
    fake_uncert = FakeUncertainty([10, 2])
    assert fake_uncert.uncertainty_type == 'fake'
    std_uncert = StdDevUncertainty([10, 2])
    assert std_uncert.uncertainty_type == 'std'

def test_uncertainty_correlated():
    fake_uncert = FakeUncertainty([10, 2])
    assert not fake_uncert.supports_correlated
    std_uncert = StdDevUncertainty([10, 2])
    assert std_uncert.supports_correlated

def test_for_leak_with_uncertainty():
    # Regression test for memory leak because of cyclic references between
    # NDData and uncertainty
    from collections import defaultdict
    from gc import get_objects

    def test_leak(func, specific_objects=None):
        """Function based on gc.get_objects to determine if any object or
        a specific object leaks.

        It requires a function to be given and if any objects survive the
        function scope it's considered a leak (so don't return anything).
        """
        before = defaultdict(int)
        for i in get_objects():
            before[type(i)] += 1

        func()

        after = defaultdict(int)
        for i in get_objects():
            after[type(i)] += 1

        if specific_objects is None:
            assert all(after[k] - before[k] == 0 for k in after)
        else:
            assert after[specific_objects] - before[specific_objects] == 0

    def non_leaker_nddata():
        # Without uncertainty there is no reason to assume that there is a
        # memory leak but test it nevertheless.
        NDData(np.ones(100))

    def leaker_nddata():
        # With uncertainty there was a memory leak!
        NDData(np.ones(100), uncertainty=StdDevUncertainty(np.ones(100)))

    test_leak(non_leaker_nddata, NDData)
    test_leak(leaker_nddata, NDData)

    # Same for NDDataArray:

    from ..compat import NDDataArray

    def non_leaker_nddataarray():
        NDDataArray(np.ones(100))

    def leaker_nddataarray():
        NDDataArray(np.ones(100), uncertainty=StdDevUncertainty(np.ones(100)))

    test_leak(non_leaker_nddataarray, NDDataArray)
    test_leak(leaker_nddataarray, NDDataArray)

def test_for_stolen_uncertainty():
    # Sharing uncertainties should not overwrite the parent_nddata attribute
    ndd1 = NDData(1, uncertainty=1)
    ndd2 = NDData(2, uncertainty=ndd1.uncertainty)
    # uncertainty.parent_nddata.data should be the original data!
    assert ndd1.uncertainty.parent_nddata.data == ndd1.data

@pytest.mark.parametrize(('UncertClass'), [FakeUncertainty, StdDevUncertainty,
                                           UnknownUncertainty])
def test_quantity(UncertClass):
    fake_uncert = UncertClass([1, 2, 3], unit=u.adu)
    assert isinstance(fake_uncert.quantity, u.Quantity)
    assert fake_uncert.quantity.unit.is_equivalent(u.adu)

    fake_uncert_nounit = UncertClass([1, 2, 3])
    assert isinstance(fake_uncert_nounit.quantity, u.Quantity)
    assert fake_uncert_nounit.quantity.unit.is_equivalent(u.dimensionless_unscaled)
<commit_msg>make parameterization of uncertainties a variable<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst

import pytest
import numpy as np
from numpy.testing import assert_array_equal

from ..nduncertainty import (StdDevUncertainty, NDUncertainty,
                             IncompatibleUncertaintiesException,
                             UnknownUncertainty)
from ..nddata import NDData
from ... import units as u

# Regarding setter tests:
# No need to test setters since the uncertainty is considered immutable after
# creation except of the parent_nddata attribute and this accepts just
# everything.
# Additionally they should be covered by NDData, NDArithmeticMixin which rely
# on it

# Regarding propagate, _convert_uncert, _propagate_* tests:
# They should be covered by NDArithmeticMixin since there is generally no need
# to test them without this mixin.

# Regarding __getitem__ tests:
# Should be covered by NDSlicingMixin.

# Regarding StdDevUncertainty tests:
# This subclass only overrides the methods for propagation so the same
# they should be covered in NDArithmeticMixin.

# Not really fake but the minimum an uncertainty has to override not to be
# abstract.

class FakeUncertainty(NDUncertainty):

    @property
    def uncertainty_type(self):
        return 'fake'

    def _propagate_add(self, data, final_data):
        pass

    def _propagate_subtract(self, data, final_data):
        pass

    def _propagate_multiply(self, data, final_data):
        pass

    def _propagate_divide(self, data, final_data):
        pass

# Test the fake (added also StdDevUncertainty which should behave identical)

# the list of classes used for parametrization in tests below
paramet_uncert_classes = [FakeUncertainty, StdDevUncertainty, UnknownUncertainty]

@pytest.mark.parametrize(('UncertClass'), paramet_uncert_classes)
def test_init_fake_with_list(UncertClass):
    fake_uncert = UncertClass([1, 2, 3])
    assert_array_equal(fake_uncert.array, np.array([1, 2, 3]))
    # Copy makes no difference since casting a list to an np.ndarray always
    # makes a copy.
    # But let's give the uncertainty a unit too
    fake_uncert = UncertClass([1, 2, 3], unit=u.adu)
    assert_array_equal(fake_uncert.array, np.array([1, 2, 3]))
    assert fake_uncert.unit is u.adu

@pytest.mark.parametrize(('UncertClass'), paramet_uncert_classes)
def test_init_fake_with_ndarray(UncertClass):
    uncert = np.arange(100).reshape(10, 10)
    fake_uncert = UncertClass(uncert)
    # Numpy Arrays are copied by default
    assert_array_equal(fake_uncert.array, uncert)
    assert fake_uncert.array is not uncert
    # Now try it without copy
    fake_uncert = UncertClass(uncert, copy=False)
    assert fake_uncert.array is uncert
    # let's provide a unit
    fake_uncert = UncertClass(uncert, unit=u.adu)
    assert_array_equal(fake_uncert.array, uncert)
    assert fake_uncert.array is not uncert
    assert fake_uncert.unit is u.adu

@pytest.mark.parametrize(('UncertClass'), paramet_uncert_classes)
def test_init_fake_with_quantity(UncertClass):
    uncert = np.arange(10).reshape(2, 5) * u.adu
    fake_uncert = UncertClass(uncert)
    # Numpy Arrays are copied by default
    assert_array_equal(fake_uncert.array, uncert.value)
    assert fake_uncert.array is not uncert.value
    assert fake_uncert.unit is u.adu
    # Try without copy (should not work, quantity.value always returns a copy)
    fake_uncert = UncertClass(uncert, copy=False)
    assert fake_uncert.array is not uncert.value
    assert fake_uncert.unit is u.adu
    # Now try with an explicit unit parameter too
    fake_uncert = UncertClass(uncert, unit=u.m)
    assert_array_equal(fake_uncert.array, uncert.value)  # No conversion done
    assert fake_uncert.array is not uncert.value
    assert fake_uncert.unit is u.m  # It took the explicit one

@pytest.mark.parametrize(('UncertClass'), paramet_uncert_classes)
def test_init_fake_with_fake(UncertClass):
    uncert = np.arange(5).reshape(5, 1)
    fake_uncert1 = UncertClass(uncert)
    fake_uncert2 = UncertClass(fake_uncert1)
    assert_array_equal(fake_uncert2.array, uncert)
    assert fake_uncert2.array is not uncert
    # Without making copies
    fake_uncert1 = UncertClass(uncert, copy=False)
    fake_uncert2 = UncertClass(fake_uncert1, copy=False)
    assert_array_equal(fake_uncert2.array, fake_uncert1.array)
    assert fake_uncert2.array is fake_uncert1.array
    # With a unit
    uncert = np.arange(5).reshape(5, 1) * u.adu
    fake_uncert1 = UncertClass(uncert)
    fake_uncert2 = UncertClass(fake_uncert1)
    assert_array_equal(fake_uncert2.array, uncert.value)
    assert fake_uncert2.array is not uncert.value
    assert fake_uncert2.unit is u.adu
    # With a unit and an explicit unit-parameter
    fake_uncert2 = UncertClass(fake_uncert1, unit=u.cm)
    assert_array_equal(fake_uncert2.array, uncert.value)
    assert fake_uncert2.array is not uncert.value
    assert fake_uncert2.unit is u.cm

@pytest.mark.parametrize(('UncertClass'), paramet_uncert_classes)
def test_init_fake_with_somethingElse(UncertClass):
    # What about a dict?
    uncert = {'rdnoise': 2.9, 'gain': 0.6}
    fake_uncert = UncertClass(uncert)
    assert fake_uncert.array == uncert
    # We can pass a unit too but since we cannot do uncertainty propagation
    # the interpretation is up to the user
    fake_uncert = UncertClass(uncert, unit=u.s)
    assert fake_uncert.array == uncert
    assert fake_uncert.unit is u.s
    # So, now check what happens if copy is False
    fake_uncert = UncertClass(uncert, copy=False)
    assert fake_uncert.array == uncert
    assert id(fake_uncert) != id(uncert)
    # dicts cannot be referenced without copy
    # TODO : Find something that can be referenced without copy :-)

def test_init_fake_with_StdDevUncertainty():
    # Different instances of uncertainties are not directly convertible so this
    # should fail
    uncert = np.arange(5).reshape(5, 1)
    std_uncert = StdDevUncertainty(uncert)
    with pytest.raises(IncompatibleUncertaintiesException):
        FakeUncertainty(std_uncert)
    # Ok try it the other way around
    fake_uncert = FakeUncertainty(uncert)
    with pytest.raises(IncompatibleUncertaintiesException):
        StdDevUncertainty(fake_uncert)

def test_uncertainty_type():
    fake_uncert = FakeUncertainty([10, 2])
    assert fake_uncert.uncertainty_type == 'fake'
    std_uncert = StdDevUncertainty([10, 2])
    assert std_uncert.uncertainty_type == 'std'

def test_uncertainty_correlated():
    fake_uncert = FakeUncertainty([10, 2])
    assert not fake_uncert.supports_correlated
    std_uncert = StdDevUncertainty([10, 2])
    assert std_uncert.supports_correlated

def test_for_leak_with_uncertainty():
    # Regression test for memory leak because of cyclic references between
    # NDData and uncertainty
    from collections import defaultdict
    from gc import get_objects

    def test_leak(func, specific_objects=None):
        """Function based on gc.get_objects to determine if any object or
        a specific object leaks.

        It requires a function to be given and if any objects survive the
        function scope it's considered a leak (so don't return anything).
        """
        before = defaultdict(int)
        for i in get_objects():
            before[type(i)] += 1

        func()

        after = defaultdict(int)
        for i in get_objects():
            after[type(i)] += 1

        if specific_objects is None:
            assert all(after[k] - before[k] == 0 for k in after)
        else:
            assert after[specific_objects] - before[specific_objects] == 0

    def non_leaker_nddata():
        # Without uncertainty there is no reason to assume that there is a
        # memory leak but test it nevertheless.
        NDData(np.ones(100))

    def leaker_nddata():
        # With uncertainty there was a memory leak!
        NDData(np.ones(100), uncertainty=StdDevUncertainty(np.ones(100)))

    test_leak(non_leaker_nddata, NDData)
    test_leak(leaker_nddata, NDData)

    # Same for NDDataArray:

    from ..compat import NDDataArray

    def non_leaker_nddataarray():
        NDDataArray(np.ones(100))

    def leaker_nddataarray():
        NDDataArray(np.ones(100), uncertainty=StdDevUncertainty(np.ones(100)))

    test_leak(non_leaker_nddataarray, NDDataArray)
    test_leak(leaker_nddataarray, NDDataArray)

def test_for_stolen_uncertainty():
    # Sharing uncertainties should not overwrite the parent_nddata attribute
    ndd1 = NDData(1, uncertainty=1)
    ndd2 = NDData(2, uncertainty=ndd1.uncertainty)
    # uncertainty.parent_nddata.data should be the original data!
    assert ndd1.uncertainty.parent_nddata.data == ndd1.data

@pytest.mark.parametrize(('UncertClass'), paramet_uncert_classes)
def test_quantity(UncertClass):
    fake_uncert = UncertClass([1, 2, 3], unit=u.adu)
    assert isinstance(fake_uncert.quantity, u.Quantity)
    assert fake_uncert.quantity.unit.is_equivalent(u.adu)

    fake_uncert_nounit = UncertClass([1, 2, 3])
    assert isinstance(fake_uncert_nounit.quantity, u.Quantity)
    assert fake_uncert_nounit.quantity.unit.is_equivalent(u.dimensionless_unscaled)
<|endoftext|>


<commit_before># mpf-monitor
__version__ = '0.54.0-dev.0'
__short_version__ = '0.54'
__bcp_version__ = '1.1'
__config_version__ = '5'
__mpf_version_required__ = '0.54.0-dev.0'

version = "MPF Monitor v{} (config_version={}, BCP v{}, Requires MPF v{})".format(
    __version__, __config_version__, __bcp_version__, __mpf_version_required__)
<commit_msg>Increment dev version, expect push to PyPi<commit_after># mpf-monitor
__version__ = '0.54.0-dev.1'
__short_version__ = '0.54'
__bcp_version__ = '1.1'
__config_version__ = '5'
__mpf_version_required__ = '0.54.0-dev.0'

version = "MPF Monitor v{} (config_version={}, BCP v{}, Requires MPF v{})".format(
    __version__, __config_version__, __bcp_version__, __mpf_version_required__)
<|endoftext|>


<commit_before>import csv
import logging
import subprocess
import flask
from flask.ext.script import Manager
from flask.ext.rq import job
from path import path
from mptracker import models
from mptracker.common import temp_dir
from mptracker.scraper.common import get_cached_session
from mptracker.nlp import match_names, get_placenames

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

questions = flask.Blueprint('questions', __name__)

questions_manager = Manager()

@job
def ocr_question(question_id):
    question = models.Question.query.get(question_id)
    http_session = get_cached_session('question-pdf')

    pages = []
    with temp_dir() as tmp:
        pdf_data = http_session.get(question.pdf_url).content
        pdf_path = tmp / 'document.pdf'
        with pdf_path.open('wb') as f:
            f.write(pdf_data)
        subprocess.check_call(['pdfimages', pdf_path, tmp / 'img'])
        for image_path in sorted(tmp.listdir('img-*'))[:10]:
            subprocess.check_call(['tesseract',
                                   image_path, image_path,
                                   '-l', 'ron'],
                                  stderr=subprocess.DEVNULL)
            text = (image_path + '.txt').text()
            pages.append(text)

    question.text = '\n\n'.join(pages)

    models.db.session.add(question)
    models.db.session.commit()
    logger.info("done OCR for %s (%d pages)", question, len(pages))

@questions_manager.command
def ocr_all(number=None):
    count = 0
    for question in models.Question.query:
        if not question.pdf_url:
            logger.info("Skipping %s, no URL", question)
            continue
        if question.text:
            logger.info("Skipping %s, already done OCR", question)
            continue
        ocr_question.delay(question.id)
        count += 1
        if number and count >= int(number):
            break
    logger.info("enqueued %d jobs", count)

def match_question(question):
    local_names = get_placenames(question.person.county.geonames_code)

    mp_info = {'name': question.person.name}
    matches = match_names(question.text, local_names, mp_info=mp_info)
    top_matches = sorted(matches,
                         key=lambda m: m['distance'],
                         reverse=True)[:10]
    return {'top_matches': top_matches}

@job
@questions_manager.command
def analyze_question(question_id):
    question = models.Question.query.get(question_id)
    result = match_question(question)
    question.match_data = flask.json.dumps(result)
    question.match_score = len(result['top_matches'])
    models.db.session.commit()

@questions_manager.command
def analyze_all(number=None, force=False):
    n_jobs = n_skip = n_ok = 0
    for question in models.Question.query:
        if not force:
            if question.match_data is not None:
                n_ok += 1
                continue
        county = question.person.county
        if (question.text is None or
            county is None or
            county.geonames_code is None):
            n_skip += 1
            continue
        analyze_question.delay(question.id)
        n_jobs += 1
        if number and n_jobs >= int(number):
            break
    logger.info("enqueued %d jobs, skipped %d, ok %d", n_jobs, n_skip, n_ok)

@questions.route('/questions/')
def person_index():
    from sqlalchemy import func
    from sqlalchemy.orm import subqueryload
    question_count_for_person = dict(
        models.db.session
            .query(models.Question.person_id,
                   func.count(models.Question.person_id))
            .group_by(models.Question.person_id)
    )
    people_rows = (models.Person.query
                .filter(models.Person.cdep_id)
                .options(subqueryload(models.Person.county)))
    people = [{
                'id': p.id,
                'name': p.name,
                'county_name': p.county.name if p.county else '',
                'question_count': question_count_for_person.get(p.id, 0),
            } for p in people_rows]
    people.sort(key=lambda p: p['question_count'], reverse=True)
    return flask.render_template('questions/person_index.html', **{
        'people': people,
    })

@questions.route('/person/<person_id>/questions')
def person_questions(person_id):
    person = models.Person.query.get_or_404(person_id)
    questions = list(person.questions)
    questions = [{
            'id': q.id,
            'title': q.title,
            'date': q.date,
            'score': q.match_score,
        } for q in person.questions]
    questions.sort(key=lambda q: q['score'] or 0, reverse=True)
    return flask.render_template('questions/person.html', **{
        'person': person,
        'questions': questions,
    })

@questions.route('/questions/<question_id>')
def question_detail(question_id):
    question = models.Question.query.get_or_404(question_id)
    match_result = (flask.json.loads(question.match_data)
                    if question.match_data else None)

    return flask.render_template('questions/detail.html', **{
        'person': question.person,
        'question': question,
        'match_result': match_result,
    })
<commit_msg>match some extra phrases<commit_after>import csv
import logging
import subprocess
import flask
from flask.ext.script import Manager
from flask.ext.rq import job
from path import path
from mptracker import models
from mptracker.common import temp_dir
from mptracker.scraper.common import get_cached_session
from mptracker.nlp import match_names, get_placenames

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

questions = flask.Blueprint('questions', __name__)

questions_manager = Manager()

other_phrases = [
    'memoriu',
    'memoriul',
    'petiție',
    'petiția',
    'audiență',
    'audiențe',
    'cabinet parlamentar',
    'cabinetul parlamentar',
    'cabinetul meu parlamentar',
]

@job
def ocr_question(question_id):
    question = models.Question.query.get(question_id)
    http_session = get_cached_session('question-pdf')

    pages = []
    with temp_dir() as tmp:
        pdf_data = http_session.get(question.pdf_url).content
        pdf_path = tmp / 'document.pdf'
        with pdf_path.open('wb') as f:
            f.write(pdf_data)
        subprocess.check_call(['pdfimages', pdf_path, tmp / 'img'])
        for image_path in sorted(tmp.listdir('img-*'))[:10]:
            subprocess.check_call(['tesseract',
                                   image_path, image_path,
                                   '-l', 'ron'],
                                  stderr=subprocess.DEVNULL)
            text = (image_path + '.txt').text()
            pages.append(text)

    question.text = '\n\n'.join(pages)

    models.db.session.add(question)
    models.db.session.commit()
    logger.info("done OCR for %s (%d pages)", question, len(pages))

@questions_manager.command
def ocr_all(number=None):
    count = 0
    for question in models.Question.query:
        if not question.pdf_url:
            logger.info("Skipping %s, no URL", question)
            continue
        if question.text:
            logger.info("Skipping %s, already done OCR", question)
            continue
        ocr_question.delay(question.id)
        count += 1
        if number and count >= int(number):
            break
    logger.info("enqueued %d jobs", count)

def match_question(question):
    local_names = get_placenames(question.person.county.geonames_code)
    all_names = local_names + other_phrases

    mp_info = {'name': question.person.name}
    matches = match_names(question.text, all_names, mp_info=mp_info)
    top_matches = sorted(matches,
                         key=lambda m: m['distance'],
                         reverse=True)[:10]
    return {'top_matches': top_matches}

@job
@questions_manager.command
def analyze_question(question_id):
    question = models.Question.query.get(question_id)
    result = match_question(question)
    question.match_data = flask.json.dumps(result)
    question.match_score = len(result['top_matches'])
    models.db.session.commit()

@questions_manager.command
def analyze_all(number=None, force=False):
    n_jobs = n_skip = n_ok = 0
    for question in models.Question.query:
        if not force:
            if question.match_data is not None:
                n_ok += 1
                continue
        county = question.person.county
        if (question.text is None or
            county is None or
            county.geonames_code is None):
            n_skip += 1
            continue
        analyze_question.delay(question.id)
        n_jobs += 1
        if number and n_jobs >= int(number):
            break
    logger.info("enqueued %d jobs, skipped %d, ok %d", n_jobs, n_skip, n_ok)

@questions.route('/questions/')
def person_index():
    from sqlalchemy import func
    from sqlalchemy.orm import subqueryload
    question_count_for_person = dict(
        models.db.session
            .query(models.Question.person_id,
                   func.count(models.Question.person_id))
            .group_by(models.Question.person_id)
    )
    people_rows = (models.Person.query
                .filter(models.Person.cdep_id)
                .options(subqueryload(models.Person.county)))
    people = [{
                'id': p.id,
                'name': p.name,
                'county_name': p.county.name if p.county else '',
                'question_count': question_count_for_person.get(p.id, 0),
            } for p in people_rows]
    people.sort(key=lambda p: p['question_count'], reverse=True)
    return flask.render_template('questions/person_index.html', **{
        'people': people,
    })

@questions.route('/person/<person_id>/questions')
def person_questions(person_id):
    person = models.Person.query.get_or_404(person_id)
    questions = list(person.questions)
    questions = [{
            'id': q.id,
            'title': q.title,
            'date': q.date,
            'score': q.match_score,
        } for q in person.questions]
    questions.sort(key=lambda q: q['score'] or 0, reverse=True)
    return flask.render_template('questions/person.html', **{
        'person': person,
        'questions': questions,
    })

@questions.route('/questions/<question_id>')
def question_detail(question_id):
    question = models.Question.query.get_or_404(question_id)
    match_result = (flask.json.loads(question.match_data)
                    if question.match_data else None)

    return flask.render_template('questions/detail.html', **{
        'person': question.person,
        'question': question,
        'match_result': match_result,
    })
<|endoftext|>


<commit_before># -*- test-case-name: mqtt.client.test.test_factory -*-
# ----------------------------------------------------------------------
# Copyright (C) 2015 by Rafael Gonzalez
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# ----------------------------------------------------------------------

# ----------------
# Standard modules
# ----------------

from collections import deque

# ----------------
# Twisted  modules
# ----------------

from twisted.internet.protocol import ReconnectingClientFactory
from twisted.logger import Logger

# -----------
# Own modules
# -----------

log = Logger()

class MQTTFactory(ReconnectingClientFactory):

    SUBSCRIBER = 0x1
    PUBLISHER  = 0x2

    def __init__(self, profile):
        self.profile  = profile
        self.factor   = 2
        self.maxDelay = 2*3600
        # Packet Id generator
        self.id       = 0
        self.queuePublishTx  = deque() # PUBLISH messages Waiting for PUBREC/PUBACK
        self.queuePubRelease  = deque() # PUBREL  messages (qos=2) waiting for PUBCOMP (publisher)
        self.queuePublishRx   = deque() # PUBLISH messages (qos=2) waiting for PUBREL (subscriber side)

    def buildProtocol(self, addr):

        if   self.profile == self.SUBSCRIBER:
            from mqtt.client.subscriber import MQTTProtocol
        elif self.profile == self.PUBLISHER:
            from mqtt.client.publisher import MQTTProtocol
        elif self.profile == (self.SUBSCRIBER | self.PUBLISHER):
            from mqtt.client.pubsubs import MQTTProtocol
        else:
            raise ValueError("profile value not supported" , self.profile)

        return MQTTProtocol(self)

    def clientConnectionLost(self, connector, reason):
        log.warn('Lost connection. Reason {reason!r}:', reason=reason)
        ReconnectingClientFactory.clientConnectionLost(self, connector, reason)

    def clientConnectionFailed(self, connector, reason):
        log.warn('Conenction failed. Reason {reason!r}:', reason=reason)
        ReconnectingClientFactory.clientConnectionFailed(self, connector,
                                                         reason)

    # -------------
    # Helper methods
    # --------------

    def makeId(self):
        '''Produce ids for Protocol packets, outliving their sessions'''
        self.id = (self.id + 1) % 65536
        self.id = self.id or 1   # avoid id 0
        return self.id

__all__ = [MQTTFactory]
<commit_msg>added logging of version info in the factory constructor<commit_after># -*- test-case-name: mqtt.client.test.test_factory -*-
# ----------------------------------------------------------------------
# Copyright (C) 2015 by Rafael Gonzalez
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# ----------------------------------------------------------------------

# ----------------
# Standard modules
# ----------------

from collections import deque

# ----------------
# Twisted  modules
# ----------------

from twisted.internet.protocol import ReconnectingClientFactory
from twisted.logger import Logger

# -----------
# Own modules
# -----------

from .. import __version__

log = Logger()

class MQTTFactory(ReconnectingClientFactory):

    SUBSCRIBER = 0x1
    PUBLISHER  = 0x2

    def __init__(self, profile):
        self.profile  = profile
        self.factor   = 2
        self.maxDelay = 2*3600
        # Packet Id generator
        self.id       = 0
        self.queuePublishTx   = deque() # PUBLISH messages Waiting for PUBREC/PUBACK
        self.queuePubRelease  = deque() # PUBREL  messages (qos=2) waiting for PUBCOMP (publisher)
        self.queuePublishRx   = deque() # PUBLISH messages (qos=2) waiting for PUBREL (subscriber side)
        log.info("MQTT Client library version {version}", version=__version__)

    def buildProtocol(self, addr):

        if   self.profile == self.SUBSCRIBER:
            from mqtt.client.subscriber import MQTTProtocol
        elif self.profile == self.PUBLISHER:
            from mqtt.client.publisher import MQTTProtocol
        elif self.profile == (self.SUBSCRIBER | self.PUBLISHER):
            from mqtt.client.pubsubs import MQTTProtocol
        else:
            raise ValueError("profile value not supported" , self.profile)

        return MQTTProtocol(self)

    def clientConnectionLost(self, connector, reason):
        log.warn('Lost connection. Reason {reason!r}:', reason=reason)
        ReconnectingClientFactory.clientConnectionLost(self, connector, reason)

    def clientConnectionFailed(self, connector, reason):
        log.warn('Conenction failed. Reason {reason!r}:', reason=reason)
        ReconnectingClientFactory.clientConnectionFailed(self, connector,
                                                         reason)

    # -------------
    # Helper methods
    # --------------

    def makeId(self):
        '''Produce ids for Protocol packets, outliving their sessions'''
        self.id = (self.id + 1) % 65536
        self.id = self.id or 1   # avoid id 0
        return self.id

__all__ = [MQTTFactory]
<|endoftext|>


<commit_before>from docutils import nodes
from sphinx import addnodes
from sphinx.writers.html import SmartyPantsHTMLTranslator
from sphinx.builders.html import StandaloneHTMLBuilder
import os

def setup(app):
    app.add_builder(PxBuilder)

BaseHtmlXlator = SmartyPantsHTMLTranslator
class PxTranslator(BaseHtmlXlator):
    """Adjust the HTML translator into a .px translator.

    """

    def __init__(self, *args, **kwargs):
        BaseHtmlXlator.__init__(self, *args, **kwargs)
        #self.document.reporter.debug_flag = 1
        # To make the doc title be h0 (skipped), and the next h1.
        self.initial_header_level = 0

    def visit_section(self, node):
        self.section_level += 1

    def depart_section(self, node):
        self.section_level -= 1

    def visit_title(self, node):
        if self.section_level == 1:
            raise nodes.SkipNode
        else:
            # The id for the h2 tag is on the parent, move it
            # down here so we'll get the right HTML.
            if not node['ids'] and len(node.parent['ids']) > 1:
                node['ids'] = [node.parent['ids'][1]]
            BaseHtmlXlator.visit_title(self, node)

    def visit_field_list(self, node):
        self.history = []

    def depart_field_list(self, node):
        if self.history:
            self.body.append("<history>\n")
            for hist in self.history:
                when, what = hist.split(',', 1)
                self.body.append("<what when='%s'>%s</what>\n" % (when, self.encode(what.strip())))
            self.body.append("</history>\n")

        if "b" in self.builder.config.release:
            self.body.append("""
                <box>
                These docs are for a beta release, %s.
                For the latest released version, see coverage.py.
                </box>
                """ % self.builder.config.release)

    def visit_field(self, node):
        if node.children[0].astext() == 'history':
            self.history.append(node.children[1].astext())
        raise nodes.SkipChildren

    def depart_field(self, node):
        pass

    def visit_literal_block(self, node):
        if node.rawsource != node.astext():
            # most probably a parsed-literal block -- don't highlight
            return BaseHtmlXlator.visit_literal_block(self, node)
        lang = self.highlightlang
        if node.has_key('language'):
            # code-block directives
            lang = node['language']
        self.body.append('' % lang)
        self.body.append(self.encode(node.rawsource))
        self.body.append('\n')
        raise nodes.SkipNode

    def visit_desc_parameterlist(self, node):
        # I'm overriding this method so that the base class doesn't write out
        # <big>(</big>, but I also have to handle the logic from the base class,
        # so most of this is just copied from sphinx/writers/html.py...
        self.body.append('(')
        self.first_param = 1
        self.optional_param_level = 0
        # How many required parameters are left.
        self.required_params_left = sum([isinstance(c, addnodes.desc_parameter)
                                         for c in node.children])
        self.param_separator = node.child_text_separator
    def depart_desc_parameterlist(self, node):
        self.body.append(')')

class PxBuilder(StandaloneHTMLBuilder):
    name = 'px'

    def init(self):
        self.config.html_theme = 'px'
        self.config.html_translator_class = "px_xlator.PxTranslator"

        super(PxBuilder, self).init()

        self.out_suffix = '.px'
        self.link_suffix = '.html'

        if "b" in self.config.release:
            self.px_uri = "/code/coverage/beta/"
        else:
            self.px_uri = "/code/coverage/"

    def get_target_uri(self, docname, typ=None):
        return self.px_uri + docname + self.link_suffix
<commit_msg>Properly handle alpha releases.<commit_after>from docutils import nodes
from sphinx import addnodes
from sphinx.writers.html import SmartyPantsHTMLTranslator
from sphinx.builders.html import StandaloneHTMLBuilder
import os

def setup(app):
    app.add_builder(PxBuilder)

BaseHtmlXlator = SmartyPantsHTMLTranslator
class PxTranslator(BaseHtmlXlator):
    """Adjust the HTML translator into a .px translator.

    """

    def __init__(self, *args, **kwargs):
        BaseHtmlXlator.__init__(self, *args, **kwargs)
        #self.document.reporter.debug_flag = 1
        # To make the doc title be h0 (skipped), and the next h1.
        self.initial_header_level = 0

    def visit_section(self, node):
        self.section_level += 1

    def depart_section(self, node):
        self.section_level -= 1

    def visit_title(self, node):
        if self.section_level == 1:
            raise nodes.SkipNode
        else:
            # The id for the h2 tag is on the parent, move it
            # down here so we'll get the right HTML.
            if not node['ids'] and len(node.parent['ids']) > 1:
                node['ids'] = [node.parent['ids'][1]]
            BaseHtmlXlator.visit_title(self, node)

    def visit_field_list(self, node):
        self.history = []

    def depart_field_list(self, node):
        if self.history:
            self.body.append("<history>\n")
            for hist in self.history:
                when, what = hist.split(',', 1)
                self.body.append("<what when='%s'>%s</what>\n" % (when, self.encode(what.strip())))
            self.body.append("</history>\n")

        prerel = None
        if "b" in self.builder.config.release:
            prerel = "a beta"
        if "a" in self.builder.config.release:
            prerel = "an ALPHA"
        if prerel:
            self.body.append("""
                <box>
                These docs are for %s release, %s.
                For the latest released version, see coverage.py.
                </box>
                """ % (prerel, self.builder.config.release))

    def visit_field(self, node):
        if node.children[0].astext() == 'history':
            self.history.append(node.children[1].astext())
        raise nodes.SkipChildren

    def depart_field(self, node):
        pass

    def visit_literal_block(self, node):
        if node.rawsource != node.astext():
            # most probably a parsed-literal block -- don't highlight
            return BaseHtmlXlator.visit_literal_block(self, node)
        lang = self.highlightlang
        if node.has_key('language'):
            # code-block directives
            lang = node['language']
        self.body.append('' % lang)
        self.body.append(self.encode(node.rawsource))
        self.body.append('\n')
        raise nodes.SkipNode

    def visit_desc_parameterlist(self, node):
        # I'm overriding this method so that the base class doesn't write out
        # <big>(</big>, but I also have to handle the logic from the base class,
        # so most of this is just copied from sphinx/writers/html.py...
        self.body.append('(')
        self.first_param = 1
        self.optional_param_level = 0
        # How many required parameters are left.
        self.required_params_left = sum([isinstance(c, addnodes.desc_parameter)
                                         for c in node.children])
        self.param_separator = node.child_text_separator
    def depart_desc_parameterlist(self, node):
        self.body.append(')')

class PxBuilder(StandaloneHTMLBuilder):
    name = 'px'

    def init(self):
        self.config.html_theme = 'px'
        self.config.html_translator_class = "px_xlator.PxTranslator"

        super(PxBuilder, self).init()

        self.out_suffix = '.px'
        self.link_suffix = '.html'

        if max(self.config.release).isalpha():
            self.px_uri = "/code/coverage/beta/"
        else:
            self.px_uri = "/code/coverage/"

    def get_target_uri(self, docname, typ=None):
        return self.px_uri + docname + self.link_suffix
<|endoftext|>


<commit_before>import logging

from django.middleware.csrf import CsrfViewMiddleware, REASON_NO_CSRF_COOKIE, REASON_BAD_TOKEN
from django.conf import settings
from django.contrib import messages
from django.utils.translation import ugettext as _

from corehq.util.soft_assert import soft_assert

logger = logging.getLogger('')

class HQCsrfViewMiddleWare(CsrfViewMiddleware):

    def _reject(self, request, reason):
        messages.error(request, _('If cookies are enabled and you still see this error, please report an issue'))
        request_url = request.path
        referring_url = request.META.get('HTTP_REFERER', 'Unknown URL')
        warning = "Request at {url} doesn't contain a csrf token. "\
                  "Referring url is {referer}. Letting the request pass through for now. "\
                  "Check if we are sending csrf_token in the corresponding POST form, if not fix it. "\
                  "Read more here https://github.com/dimagi/commcare-hq/pull/9227".format(
                      url=request_url,
                      referer=referring_url,
                  )
        logger.error(warning)  # send it couchlog for log-analysis
        _assert = soft_assert('{}@{}'.format('sreddy+logs', 'dimagi.com'), exponential_backoff=False)
        _assert(False, warning)

        if '/login/' in request_url:
            # exempt login URLs from csrf, but still get failure logs
            self._accept(request)

        if settings.CSRF_SOFT_MODE and reason in [REASON_NO_CSRF_COOKIE, REASON_BAD_TOKEN]:
            return self._accept(request)
        else:
            return super(HQCsrfViewMiddleWare, self)._reject(request, reason)
<commit_msg>remove CSRF user warning, soft-assert, login check<commit_after>import logging

from django.middleware.csrf import CsrfViewMiddleware, REASON_NO_CSRF_COOKIE, REASON_BAD_TOKEN
from django.conf import settings

logger = logging.getLogger('')

class HQCsrfViewMiddleWare(CsrfViewMiddleware):

    def _reject(self, request, reason):
        request_url = request.path
        referring_url = request.META.get('HTTP_REFERER', 'Unknown URL')
        warning = "Request at {url} doesn't contain a csrf token. "\
                  "Referring url is {referer}. Letting the request pass through for now. "\
                  "Check if we are sending csrf_token in the corresponding POST form, if not fix it. "\
                  "Read more here https://github.com/dimagi/commcare-hq/pull/9227".format(
                      url=request_url,
                      referer=referring_url,
                  )
        logger.error(warning)

        if settings.CSRF_SOFT_MODE and reason in [REASON_NO_CSRF_COOKIE, REASON_BAD_TOKEN]:
            return self._accept(request)
        else:
            return super(HQCsrfViewMiddleWare, self)._reject(request, reason)
<|endoftext|>


<commit_before>from google.appengine.ext import db
import string

class Tag(db.Model):
    "Google AppEngine model for store of tags."

    tag = db.StringProperty(required=True)
    "The actual string value of the tag."

    added = db.DateTimeProperty(auto_now_add=True)
    "The date and time that the tag was first added to the datastore."

    tagged = db.ListProperty(db.Key)
    "A List of db.Key values for the datastore objects that have been tagged with this tag value."

    tagged_count = db.IntegerProperty(default=0)
    "The number of entities in tagged."

    @staticmethod
    def __key_name(tag_name):
        return "tag_%s" % tag_name

    def remove_tagged(self, key):
        def remove_tagged_txn():
            if key in self.tagged:
                self.tagged.remove(key)
                self.tagged_count -= 1
                self.put()
        db.run_in_transaction(remove_tagged_txn)
        Tag.expire_cached_tags()

    def add_tagged(self, key):
        def add_tagged_txn():
            if key not in self.tagged:
                self.tagged.append(key)
                self.tagged_count += 1
                self.put()
        db.run_in_transaction(add_tagged_txn)
        Tag.expire_cached_tags()

    def clear_tagged(self):
        def clear_tagged_txn():
            self.tagged = []
            self.tagged_count = 0
            self.put()
        db.run_in_transaction(clear_tagged_txn)
        Tag.expire_cached_tags()

    @classmethod
    def get_by_name(cls, tag_name):
        return Tag.get_by_key_name(Tag.__key_name(tag_name))

    @classmethod
    def get_tags_for_key(cls, key):
        "Set the tags for the datastore object represented by key."
        tags = db.Query(Tag).filter('tagged =', key).fetch(1000)
        return tags

    @classmethod
    def get_or_create(cls, tag_name):
        "Get the Tag object that has the tag value given by tag_value."
        tag_key_name = Tag.__key_name(tag_name)
        existing_tag = Tag.get_by_key_name(tag_key_name)
        if existing_tag is None:
            # The tag does not yet exist, so create it.
            def create_tag_txn():
                new_tag = Tag(key_name=tag_key_name, tag = tag_name)
                new_tag.put()
                return new_tag
            existing_tag = db.run_in_transaction(create_tag_txn)
        return existing_tag

    @classmethod
    def get_tags_by_frequency(cls, limit=1000):
        """Return a list of Tags sorted by the number of objects to which they have been applied,
        most frequently-used first.  If limit is given, return only that many tags; otherwise,
        return all."""
        tag_list = db.Query(Tag).filter('tagged_count >', 0).order("-tagged_count").fetch(limit)

        return tag_list

    @classmethod
    def get_tags_by_name(cls, limit=1000, ascending=True):
        """Return a list of Tags sorted alphabetically by the name of the tag.
        If a limit is given, return only that many tags; otherwise, return all.
        If ascending is True, sort from a-z; otherwise, sort from z-a."""

        from google.appengine.api import memcache

        cache_name = 'tags_by_name'
        if ascending:
            cache_name += '_asc'
        else:
            cache_name += '_desc'

        tags = memcache.get(cache_name)
        if tags is None or len(tags) < limit:
            order_by = "tag"
            if not ascending:
                order_by = "-tag"

            tags = db.Query(Tag).order(order_by).fetch(limit)
            memcache.add(cache_name, tags, 3600)
        else:
            if len(tags) > limit:
                # Return only as many as requested.
                tags = tags[:limit]

        return tags

    @classmethod
    def popular_tags(cls, limit=5):
        from google.appengine.api import memcache

        tags = memcache.get('popular_tags')
        if tags is None:
            tags = Tag.get_tags_by_frequency(limit)
            memcache.add('popular_tags', tags, 3600)

        return tags

    @classmethod
    def expire_cached_tags(cls):
        from google.appengine.api import memcache

        memcache.delete('popular_tags')
        memcache.delete('tags_by_name_asc')
        memcache.delete('tags_by_name_desc')

class Taggable:
    """A mixin class that is used for making Google AppEnigne Model classes taggable.
        Usage:
            class Post(db.Model, taggable.Taggable):
                body = db.TextProperty(required = True)
                title = db.StringProperty()
                added = db.DateTimeProperty(auto_now_add=True)
                edited = db.DateTimeProperty()

                def __init__(self, parent=None, key_name=None, app=None, **entity_values):
                    db.Model.__init__(self, parent, key_name, app, **entity_values)
                    taggable.Taggable.__init__(self)
    """

    def __init__(self):
        self.__tags = None
        self.tag_separator = ","
        """The string that is used to separate individual tags in a string
        representation of a list of tags.  Used by tags_string() to join the tags
        into a string representation and tags setter to split a string into
        individual tags."""

    def __get_tags(self):
        "Get a List of Tag objects for all Tags that apply to this object."
        if self.__tags is None or len(self.__tags) == 0:
            self.__tags = Tag.get_tags_for_key(self.key())
        return self.__tags

    def __set_tags(self, tags):
        import types
        if type(tags) is types.UnicodeType:
            # Convert unicode to a plain string
            tags = str(tags)
        if type(tags) is types.StringType:
            # Tags is a string, split it on tag_seperator into a list
            tags = string.split(tags, self.tag_separator)
        if type(tags) is types.ListType:
            self.__get_tags()
            # Firstly, we will check to see if any tags have been removed.
            # Iterate over a copy of __tags, as we may need to modify __tags
            for each_tag in self.__tags[:]:
                if each_tag not in tags:
                    # A tag that was previously assigned to this entity is
                    # missing in the list that is being assigned, so we
                    # disassocaite this entity and the tag.
                    each_tag.remove_tagged(self.key())
                    self.__tags.remove(each_tag)
            # Secondly, we will check to see if any tags have been added.
            for each_tag in tags:
                each_tag = string.strip(each_tag)
                if len(each_tag) > 0 and each_tag not in self.__tags:
                    # A tag that was not previously assigned to this entity
                    # is present in the list that is being assigned, so we
                    # associate this entity with the tag.
                    tag = Tag.get_or_create(each_tag)
                    tag.add_tagged(self.key())
                    self.__tags.append(tag)
        else:
            raise Exception, "tags must be either a unicode, a string or a list"

    tags = property(__get_tags, __set_tags, None, None)

    def tags_string(self):
        "Create a formatted string version of this entity's tags"
        to_str = ""
        for each_tag in self.tags:
            to_str += each_tag.tag
            if each_tag != self.tags[-1]:
                to_str += self.tag_separator
        return to_str

<commit_msg>Extend taggable-mixin to support different Tag models.<commit_after>from google.appengine.ext import db
import string

class Tag(db.Model):
    "Google AppEngine model for store of tags."

    tag = db.StringProperty(required=True)
    "The actual string value of the tag."

    added = db.DateTimeProperty(auto_now_add=True)
    "The date and time that the tag was first added to the datastore."

    tagged = db.ListProperty(db.Key)
    "A List of db.Key values for the datastore objects that have been tagged with this tag value."

    tagged_count = db.IntegerProperty(default=0)
    "The number of entities in tagged."

    @classmethod
    def __key_name(cls, tag_name):
        return cls.__name__ + '_' + tag_name

    def remove_tagged(self, key):
        def remove_tagged_txn():
            if key in self.tagged:
                self.tagged.remove(key)
                self.tagged_count -= 1
                self.put()
        db.run_in_transaction(remove_tagged_txn)
        self.__class__.expire_cached_tags()

    def add_tagged(self, key):
        def add_tagged_txn():
            if key not in self.tagged:
                self.tagged.append(key)
                self.tagged_count += 1
                self.put()
        db.run_in_transaction(add_tagged_txn)
        self.__class__.expire_cached_tags()

    def clear_tagged(self):
        def clear_tagged_txn():
            self.tagged = []
            self.tagged_count = 0
            self.put()
        db.run_in_transaction(clear_tagged_txn)
        self.__class__.expire_cached_tags()

    @classmethod
    def get_by_name(cls, tag_name):
        return cls.get_by_key_name(cls.__key_name(tag_name))

    @classmethod
    def get_tags_for_key(cls, key):
        "Set the tags for the datastore object represented by key."
        tags = db.Query(cls).filter('tagged =', key).fetch(1000)
        return tags

    @classmethod
    def get_or_create(cls, tag_name):
        "Get the Tag object that has the tag value given by tag_value."
        tag_key_name = cls.__key_name(tag_name)
        existing_tag = cls.get_by_key_name(tag_key_name)
        if existing_tag is None:
            # The tag does not yet exist, so create it.
            def create_tag_txn():
                new_tag = cls(key_name=tag_key_name, tag=tag_name)
                new_tag.put()
                return new_tag
            existing_tag = db.run_in_transaction(create_tag_txn)
        return existing_tag

    @classmethod
    def get_tags_by_frequency(cls, limit=1000):
        """Return a list of Tags sorted by the number of objects to which they have been applied,
        most frequently-used first.  If limit is given, return only that many tags; otherwise,
        return all."""
        tag_list = db.Query(cls).filter('tagged_count >', 0).order("-tagged_count").fetch(limit)

        return tag_list

    @classmethod
    def get_tags_by_name(cls, limit=1000, ascending=True):
        """Return a list of Tags sorted alphabetically by the name of the tag.
        If a limit is given, return only that many tags; otherwise, return all.
        If ascending is True, sort from a-z; otherwise, sort from z-a."""

        from google.appengine.api import memcache

        cache_name = cls.__name__ + '_tags_by_name'
        if ascending:
            cache_name += '_asc'
        else:
            cache_name += '_desc'

        tags = memcache.get(cache_name)
        if tags is None or len(tags) < limit:
            order_by = "tag"
            if not ascending:
                order_by = "-tag"

            tags = db.Query(cls).order(order_by).fetch(limit)
            memcache.add(cache_name, tags, 3600)
        else:
            if len(tags) > limit:
                # Return only as many as requested.
                tags = tags[:limit]

        return tags

    @classmethod
    def popular_tags(cls, limit=5):
        from google.appengine.api import memcache

        tags = memcache.get(cls.__name__ + '_popular_tags')
        if tags is None:
            tags = cls.get_tags_by_frequency(limit)
            memcache.add(cls.__name__ + '_popular_tags', tags, 3600)

        return tags

    @classmethod
    def expire_cached_tags(cls):
        from google.appengine.api import memcache

        memcache.delete(cls.__name__ + '_popular_tags')
        memcache.delete(cls.__name__ + '_tags_by_name_asc')
        memcache.delete(cls.__name__ + '_tags_by_name_desc')

class Taggable:
    """A mixin class that is used for making Google AppEngine Model classes taggable.
        Usage:
            class Post(db.Model, taggable.Taggable):
                body = db.TextProperty(required = True)
                title = db.StringProperty()
                added = db.DateTimeProperty(auto_now_add=True)
                edited = db.DateTimeProperty()

                def __init__(self, parent=None, key_name=None, app=None, **entity_values):
                    db.Model.__init__(self, parent, key_name, app, **entity_values)
                    taggable.Taggable.__init__(self)
    """

    def __init__(self, tag_model = Tag):
        self.__tags = None
        self.__tag_model = tag_model
        self.tag_separator = ","
        """The string that is used to separate individual tags in a string
        representation of a list of tags.  Used by tags_string() to join the tags
        into a string representation and tags setter to split a string into
        individual tags."""

    def __get_tags(self):
        "Get a List of Tag objects for all Tags that apply to this object."
        if self.__tags is None or len(self.__tags) == 0:
            self.__tags = self.__tag_model.get_tags_for_key(self.key())
        return self.__tags

    def __set_tags(self, tags):
        import types
        if type(tags) is types.UnicodeType:
            # Convert unicode to a plain string
            tags = str(tags)
        if type(tags) is types.StringType:
            # Tags is a string, split it on tag_seperator into a list
            tags = string.split(tags, self.tag_separator)
        if type(tags) is types.ListType:
            self.__get_tags()
            # Firstly, we will check to see if any tags have been removed.
            # Iterate over a copy of __tags, as we may need to modify __tags
            for each_tag in self.__tags[:]:
                if each_tag not in tags:
                    # A tag that was previously assigned to this entity is
                    # missing in the list that is being assigned, so we
                    # disassocaite this entity and the tag.
                    each_tag.remove_tagged(self.key())
                    self.__tags.remove(each_tag)
            # Secondly, we will check to see if any tags have been added.
            for each_tag in tags:
                each_tag = string.strip(each_tag)
                if len(each_tag) > 0 and each_tag not in self.__tags:
                    # A tag that was not previously assigned to this entity
                    # is present in the list that is being assigned, so we
                    # associate this entity with the tag.
                    tag = self.__tag_model.get_or_create(each_tag)
                    tag.add_tagged(self.key())
                    self.__tags.append(tag)
        else:
            raise Exception, "tags must be either a unicode, a string or a list"

    tags = property(__get_tags, __set_tags, None, None)

    def tags_string(self):
        "Create a formatted string version of this entity's tags"
        to_str = ""
        for each_tag in self.tags:
            to_str += each_tag.tag
            if each_tag != self.tags[-1]:
                to_str += self.tag_separator
        return to_str

<|endoftext|>


<commit_before>from django.test import TestCase

from corehq.apps.sso.tests import generator

class BaseIdPTest(TestCase):

    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.account = generator.get_billing_account_for_idp()

    def setUp(self):
        super().setUp()
        self.idp = generator.create_idp('vaultwax', self.account)

    def set_idp_active(self, active):
        self.idp.is_active = active
        self.idp.save()

    def tearDown(self):
        self.idp.delete()
        super().tearDown()

    @classmethod
    def tearDownClass(cls):
        cls.account.delete()
        super().tearDownClass()
<commit_msg>Remove duplicate method<commit_after>from django.test import TestCase

from corehq.apps.sso.tests import generator

class BaseIdPTest(TestCase):

    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.account = generator.get_billing_account_for_idp()

    def setUp(self):
        super().setUp()
        self.idp = generator.create_idp('vaultwax', self.account)

    def tearDown(self):
        self.idp.delete()
        super().tearDown()

    @classmethod
    def tearDownClass(cls):
        cls.account.delete()
        super().tearDownClass()
<|endoftext|>


<commit_before>from collections import defaultdict
from dimagi.ext.couchdbkit import (Document, DictProperty,
    StringProperty, ListProperty)
import commcare_translations
from dimagi.utils.couch import CouchDocLockableMixIn

class TranslationMixin(Document):
    translations = DictProperty()

    def init(self, lang):
        self.translations[lang] = Translation.get_translations(lang, one=True)

    def set_translation(self, lang, key, value):
        if lang not in self.translations:
            self.translations[lang] = {}
        if value is not None:
            self.translations[lang][key] = value
        else:
            del self.translations[lang][key]

    def set_translations(self, lang, translations):
        self.translations[lang] = translations

class TranslationDoc(TranslationMixin):
    @classmethod
    def create_from_txt(cls, lang, txt=None):
        """
        from corehq.apps.translations.models import *
        TranslationDoc.create_from_txt("pt")

        """
        if txt:
            dct = commcare_translations.loads(txt)
        else:
            dct = commcare_translations.load_translations(lang)
        t = cls(translations={lang: dct})
        t.save()
        return t

class StandaloneTranslationDoc(TranslationDoc, CouchDocLockableMixIn):
    """
    There is either 0 or 1 StandaloneTranslationDoc doc for each (domain, area).
    """
    domain = StringProperty()
    # For example, "sms"
    area = StringProperty()
    langs = ListProperty()

    @property
    def default_lang(self):
        if len(self.langs) > 0:
            return self.langs[0]
        else:
            return None

    @classmethod
    def get_obj(cls, domain, area, *args, **kwargs):
        return StandaloneTranslationDoc.view(
            "translations/standalone",
            key=[domain, area],
            include_docs=True
        ).one()

    @classmethod
    def create_obj(cls, domain, area, *args, **kwargs):
        obj = StandaloneTranslationDoc(
            domain=domain,
            area=area,
        )
        obj.save()
        return obj

class Translation(object):
    @classmethod
    def get_translations(cls, lang, key=None, one=False):
        from corehq.apps.app_manager.models import Application
        if key:
            translations = []
            r = Application.get_db().view('app_translations_by_popularity/view',
                startkey=[lang, key],
                endkey=[lang, key, {}],
                group=True
            ).all()
            r.sort(key=lambda x: -x['value'])
            for row in r:
                _, _, translation = row['key']
                translations.append(translation)
            if one:
                return translations[0] if translations else None
            return translations
        else:
            translations = defaultdict(list)
            r = Application.get_db().view('app_translations_by_popularity/view',
                startkey=[lang],
                endkey=[lang, {}],
                group=True
            ).all()
            r.sort(key=lambda x: (x['key'][1], -x['value']))
            for row in r:
                _, key, translation = row['key']
                translations[key].append(translation)
            if one:
                return dict([(key, val[0]) for key, val in translations.items()])
            else:
                return translations
<commit_msg>Remove unused method<commit_after>from collections import defaultdict
from dimagi.ext.couchdbkit import (Document, DictProperty,
    StringProperty, ListProperty)
import commcare_translations
from dimagi.utils.couch import CouchDocLockableMixIn

class TranslationMixin(Document):
    translations = DictProperty()

    def init(self, lang):
        self.translations[lang] = Translation.get_translations(lang, one=True)

    def set_translation(self, lang, key, value):
        if lang not in self.translations:
            self.translations[lang] = {}
        if value is not None:
            self.translations[lang][key] = value
        else:
            del self.translations[lang][key]

    def set_translations(self, lang, translations):
        self.translations[lang] = translations

class TranslationDoc(TranslationMixin):
    pass

class StandaloneTranslationDoc(TranslationDoc, CouchDocLockableMixIn):
    """
    There is either 0 or 1 StandaloneTranslationDoc doc for each (domain, area).
    """
    domain = StringProperty()
    # For example, "sms"
    area = StringProperty()
    langs = ListProperty()

    @property
    def default_lang(self):
        if len(self.langs) > 0:
            return self.langs[0]
        else:
            return None

    @classmethod
    def get_obj(cls, domain, area, *args, **kwargs):
        return StandaloneTranslationDoc.view(
            "translations/standalone",
            key=[domain, area],
            include_docs=True
        ).one()

    @classmethod
    def create_obj(cls, domain, area, *args, **kwargs):
        obj = StandaloneTranslationDoc(
            domain=domain,
            area=area,
        )
        obj.save()
        return obj

class Translation(object):
    @classmethod
    def get_translations(cls, lang, key=None, one=False):
        from corehq.apps.app_manager.models import Application
        if key:
            translations = []
            r = Application.get_db().view('app_translations_by_popularity/view',
                startkey=[lang, key],
                endkey=[lang, key, {}],
                group=True
            ).all()
            r.sort(key=lambda x: -x['value'])
            for row in r:
                _, _, translation = row['key']
                translations.append(translation)
            if one:
                return translations[0] if translations else None
            return translations
        else:
            translations = defaultdict(list)
            r = Application.get_db().view('app_translations_by_popularity/view',
                startkey=[lang],
                endkey=[lang, {}],
                group=True
            ).all()
            r.sort(key=lambda x: (x['key'][1], -x['value']))
            for row in r:
                _, key, translation = row['key']
                translations[key].append(translation)
            if one:
                return dict([(key, val[0]) for key, val in translations.items()])
            else:
                return translations
<|endoftext|>


<commit_before># coding=utf-8
import json
from os.path import join

import corehq.blobs.migrate as mod
from corehq.blobs import get_blob_db
from corehq.blobs.mixin import BlobMixin
from corehq.blobs.s3db import maybe_not_found
from corehq.blobs.tests.util import (TemporaryFilesystemBlobDB,
    TemporaryMigratingBlobDB, TemporaryS3BlobDB)
from corehq.util.test_utils import trap_extra_setup

from django.conf import settings
from django.test import TestCase
from testil import replattr, tempdir

from couchexport.models import SavedBasicExport, ExportConfiguration

NOT_SET = object()

class BaseMigrationTest(TestCase):

    def setUp(self):
        mod.BlobMigrationState.objects.filter(slug=self.slug).delete()
        self._old_flags = {}
        for model in mod.MIGRATIONS[self.slug].doc_type_map.values():
            self._old_flags[model] = model.migrating_blobs_from_couch
            model.migrating_blobs_from_couch = True

    def tearDown(self):
        mod.BlobMigrationState.objects.filter(slug=self.slug).delete()
        for model, flag in self._old_flags.items():
            if flag is NOT_SET:
                del model.migrating_blobs_from_couch
            else:
                model.migrating_blobs_from_couch = flag

    # abstract property, must be overridden in base class
    slug = None

    @property
    def doc_types(self):
        return set(mod.MIGRATIONS[self.slug].doc_type_map)

    def do_migration(self, docs, num_attachments=1):
        if not docs or not num_attachments:
            raise Exception("bad test: must have at least one document and "
                            "one attachment")

        for doc in docs:
            # verify: attachment is in couch and migration not complete
            self.assertEqual(len(doc._attachments), num_attachments)
            self.assertEqual(len(doc.external_blobs), 0)

        with tempdir() as tmp:
            filename = join(tmp, "file.txt")

            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
            self.assertGreaterEqual(migrated, len(docs))

            # verify: migration state recorded
            mod.BlobMigrationState.objects.get(slug=self.slug)

            # verify: migrated data was written to the file
            with open(filename) as fh:
                lines = list(fh)
            lines_by_id = {d["_id"]: d for d in (json.loads(x) for x in lines)}
            for doc in docs:
                self.assertEqual(lines_by_id[doc._id]["_rev"], doc._rev)
            self.assertEqual(len(lines), migrated, lines)

        for doc in docs:
            # verify: attachments were moved to blob db
            exp = type(doc).get(doc._id)
            self.assertNotEqual(exp._rev, doc._rev)
            self.assertEqual(len(exp.blobs), num_attachments, repr(exp.blobs))
            self.assertFalse(exp._attachments, exp._attachments)
            self.assertEqual(len(exp.external_blobs), num_attachments)

    def do_failed_migration(self, docs, modify_docs):
        if len(docs) < len(self.doc_types):
            raise Exception("bad test: must have at least one document per doc type")
        modified = []
        print_status = mod.print_status

        # setup concurrent modification
        def modify_doc_and_print_status(num, total, elapsed):
            if not modified:
                # do concurrent modification
                modify_docs()
                modified.append(True)
            print_status(num, total, elapsed)

        # verify: attachments are in couch, not blob db
        for doc in docs:
            self.assertGreaterEqual(len(doc._attachments), 1)
            self.assertEqual(len(doc.external_blobs), 0)

        # hook print_status() call to simulate concurrent modification
        with replattr(mod, "print_status", modify_doc_and_print_status):
            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug].migrate()
            self.assertGreaterEqual(skipped, len(docs))

        self.assertTrue(modified)

        # verify: migration state not set when docs are skipped
        with self.assertRaises(mod.BlobMigrationState.DoesNotExist):
            mod.BlobMigrationState.objects.get(slug=self.slug)

        tested = set()
        for doc, (num_attachments, num_blobs) in docs.items():
            tested.add(doc.doc_type)
            exp = type(doc).get(doc._id)
            if not num_attachments:
                raise Exception("bad test: modify function should leave "
                                "unmigrated attachments")
            # verify: attachments were not migrated
            print(exp)
            self.assertEqual(len(exp._attachments), num_attachments)
            self.assertEqual(len(exp.external_blobs), num_blobs)

        self.assertEqual(self.doc_types, tested) # were all the model types tested?

class TestSavedExportsMigrations(BaseMigrationTest):

    slug = "saved_exports"

    def test_migrate_saved_exports(self):
        saved = SavedBasicExport(configuration=_mk_config())
        saved.save()
        payload = b'binary data not valid utf-8 \xe4\x94'
        name = saved.get_attachment_name()
        super(BlobMixin, saved).put_attachment(payload, name)
        saved.save()

        self.do_migration([saved])

        exp = SavedBasicExport.get(saved._id)
        self.assertEqual(exp.get_payload(), payload)

    def test_migrate_with_concurrent_modification(self):
        saved = SavedBasicExport(configuration=_mk_config())
        saved.save()
        name = saved.get_attachment_name()
        new_payload = 'something new'
        old_payload = 'something old'
        super(BlobMixin, saved).put_attachment(old_payload, name)
        super(BlobMixin, saved).put_attachment(old_payload, "other")
        saved.save()
        self.assertEqual(len(saved._attachments), 2)

        def modify():
            doc = SavedBasicExport.get(saved._id)
            doc.set_payload(new_payload)
            doc.save()

        self.do_failed_migration({saved: (1, 1)}, modify)

        exp = SavedBasicExport.get(saved._id)
        self.assertEqual(exp.get_payload(), new_payload)
        self.assertEqual(exp.fetch_attachment("other"), old_payload)

class TestMigrateBackend(TestCase):

    slug = "migrate_backend"
    test_size = 5

    def setUp(self):
        with trap_extra_setup(AttributeError, msg="S3_BLOB_DB_SETTINGS not configured"):
            config = settings.S3_BLOB_DB_SETTINGS

        fsdb = TemporaryFilesystemBlobDB()
        assert get_blob_db() is fsdb, (get_blob_db(), fsdb)
        self.migrate_docs = docs = []
        for i in range(self.test_size):
            doc = SavedBasicExport(configuration=_mk_config("config-%s" % i))
            doc.save()
            doc.set_payload(("content %s" % i).encode('utf-8'))
            docs.append(doc)

        s3db = TemporaryS3BlobDB(config)
        self.db = TemporaryMigratingBlobDB(s3db, fsdb)
        assert get_blob_db() is self.db, (get_blob_db(), self.db)
        mod.BlobMigrationState.objects.filter(slug=self.slug).delete()

    def tearDown(self):
        self.db.close()
        mod.BlobMigrationState.objects.filter(slug=self.slug).delete()

    def test_migrate_backend(self):
        # verify: attachment is in couch and migration not complete
        with maybe_not_found():
            s3_blobs = sum(1 for b in self.db.new_db._s3_bucket().objects.all())
            self.assertEqual(s3_blobs, 0)

        with tempdir() as tmp:
            filename = join(tmp, "file.txt")

            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
            self.assertGreaterEqual(migrated, self.test_size)

            # verify: migration state recorded
            mod.BlobMigrationState.objects.get(slug=self.slug)

            # verify: migrated data was written to the file
            with open(filename) as fh:
                lines = list(fh)
            ids = {d._id for d in self.migrate_docs}
            migrated = {d["_id"] for d in (json.loads(x) for x in lines)}
            self.assertEqual(len(ids.intersection(migrated)), self.test_size)

        # verify: attachment was copied to new blob db
        for doc in self.migrate_docs:
            exp = SavedBasicExport.get(doc._id)
            self.assertEqual(exp._rev, doc._rev)  # rev should not change
            self.assertTrue(doc.blobs)
            bucket = doc._blobdb_bucket()
            for meta in doc.blobs.values():
                content = self.db.new_db.get(meta.id, bucket)
                self.assertEqual(len(content.read()), meta.content_length)

def _mk_config(name='some export name', index='dummy_index'):
    return ExportConfiguration(index=index, name=name, format='xlsx')
<commit_msg>Improve blob migration test precondition checks<commit_after># coding=utf-8
import json
from os.path import join

import corehq.blobs.migrate as mod
from corehq.blobs import get_blob_db
from corehq.blobs.mixin import BlobMixin
from corehq.blobs.s3db import maybe_not_found
from corehq.blobs.tests.util import (TemporaryFilesystemBlobDB,
    TemporaryMigratingBlobDB, TemporaryS3BlobDB)
from corehq.util.test_utils import trap_extra_setup

from django.conf import settings
from django.test import TestCase
from testil import replattr, tempdir

from couchexport.models import SavedBasicExport, ExportConfiguration

NOT_SET = object()

class BaseMigrationTest(TestCase):

    def setUp(self):
        mod.BlobMigrationState.objects.filter(slug=self.slug).delete()
        self._old_flags = {}
        for model in mod.MIGRATIONS[self.slug].doc_type_map.values():
            self._old_flags[model] = model.migrating_blobs_from_couch
            model.migrating_blobs_from_couch = True

    def tearDown(self):
        mod.BlobMigrationState.objects.filter(slug=self.slug).delete()
        for model, flag in self._old_flags.items():
            if flag is NOT_SET:
                del model.migrating_blobs_from_couch
            else:
                model.migrating_blobs_from_couch = flag

    # abstract property, must be overridden in base class
    slug = None

    @property
    def doc_types(self):
        return set(mod.MIGRATIONS[self.slug].doc_type_map)

    def do_migration(self, docs, num_attachments=1):
        test_types = {d.doc_type for d in docs}
        if test_types != self.doc_types:
            raise Exception("bad test: must have at least one document per doc "
                            "type (got: {})".format(test_types))
        if not num_attachments:
            raise Exception("bad test: must have at least one attachment")

        for doc in docs:
            # verify: attachment is in couch and migration not complete
            self.assertEqual(len(doc._attachments), num_attachments)
            self.assertEqual(len(doc.external_blobs), 0)

        with tempdir() as tmp:
            filename = join(tmp, "file.txt")

            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
            self.assertGreaterEqual(migrated, len(docs))

            # verify: migration state recorded
            mod.BlobMigrationState.objects.get(slug=self.slug)

            # verify: migrated data was written to the file
            with open(filename) as fh:
                lines = list(fh)
            lines_by_id = {d["_id"]: d for d in (json.loads(x) for x in lines)}
            for doc in docs:
                self.assertEqual(lines_by_id[doc._id]["_rev"], doc._rev)
            self.assertEqual(len(lines), migrated, lines)

        for doc in docs:
            # verify: attachments were moved to blob db
            exp = type(doc).get(doc._id)
            self.assertEqual(exp.doc_type, doc.doc_type)
            self.assertNotEqual(exp._rev, doc._rev)
            self.assertEqual(len(exp.blobs), num_attachments, repr(exp.blobs))
            self.assertFalse(exp._attachments, exp._attachments)
            self.assertEqual(len(exp.external_blobs), num_attachments)

    def do_failed_migration(self, docs, modify_docs):
        test_types = {d.doc_type for d in docs}
        if test_types != self.doc_types:
            raise Exception("bad test: must have at least one document per doc "
                            "type (got: {})".format(test_types))
        modified = []
        print_status = mod.print_status

        # setup concurrent modification
        def modify_doc_and_print_status(num, total, elapsed):
            if not modified:
                # do concurrent modification
                modify_docs()
                modified.append(True)
            print_status(num, total, elapsed)

        # verify: attachments are in couch, not blob db
        for doc in docs:
            self.assertGreaterEqual(len(doc._attachments), 1)
            self.assertEqual(len(doc.external_blobs), 0)

        # hook print_status() call to simulate concurrent modification
        with replattr(mod, "print_status", modify_doc_and_print_status):
            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug].migrate()
            self.assertGreaterEqual(skipped, len(docs))

        self.assertTrue(modified)

        # verify: migration state not set when docs are skipped
        with self.assertRaises(mod.BlobMigrationState.DoesNotExist):
            mod.BlobMigrationState.objects.get(slug=self.slug)

        for doc, (num_attachments, num_blobs) in docs.items():
            exp = type(doc).get(doc._id)
            if not num_attachments:
                raise Exception("bad test: modify function should leave "
                                "unmigrated attachments")
            # verify: attachments were not migrated
            print(exp)
            self.assertEqual(len(exp._attachments), num_attachments)
            self.assertEqual(len(exp.external_blobs), num_blobs)

class TestSavedExportsMigrations(BaseMigrationTest):

    slug = "saved_exports"

    def test_migrate_saved_exports(self):
        saved = SavedBasicExport(configuration=_mk_config())
        saved.save()
        payload = b'binary data not valid utf-8 \xe4\x94'
        name = saved.get_attachment_name()
        super(BlobMixin, saved).put_attachment(payload, name)
        saved.save()

        self.do_migration([saved])

        exp = SavedBasicExport.get(saved._id)
        self.assertEqual(exp.get_payload(), payload)

    def test_migrate_with_concurrent_modification(self):
        saved = SavedBasicExport(configuration=_mk_config())
        saved.save()
        name = saved.get_attachment_name()
        new_payload = 'something new'
        old_payload = 'something old'
        super(BlobMixin, saved).put_attachment(old_payload, name)
        super(BlobMixin, saved).put_attachment(old_payload, "other")
        saved.save()
        self.assertEqual(len(saved._attachments), 2)

        def modify():
            doc = SavedBasicExport.get(saved._id)
            doc.set_payload(new_payload)
            doc.save()

        self.do_failed_migration({saved: (1, 1)}, modify)

        exp = SavedBasicExport.get(saved._id)
        self.assertEqual(exp.get_payload(), new_payload)
        self.assertEqual(exp.fetch_attachment("other"), old_payload)

class TestMigrateBackend(TestCase):

    slug = "migrate_backend"
    test_size = 5

    def setUp(self):
        with trap_extra_setup(AttributeError, msg="S3_BLOB_DB_SETTINGS not configured"):
            config = settings.S3_BLOB_DB_SETTINGS

        fsdb = TemporaryFilesystemBlobDB()
        assert get_blob_db() is fsdb, (get_blob_db(), fsdb)
        self.migrate_docs = docs = []
        for i in range(self.test_size):
            doc = SavedBasicExport(configuration=_mk_config("config-%s" % i))
            doc.save()
            doc.set_payload(("content %s" % i).encode('utf-8'))
            docs.append(doc)

        s3db = TemporaryS3BlobDB(config)
        self.db = TemporaryMigratingBlobDB(s3db, fsdb)
        assert get_blob_db() is self.db, (get_blob_db(), self.db)
        mod.BlobMigrationState.objects.filter(slug=self.slug).delete()

    def tearDown(self):
        self.db.close()
        mod.BlobMigrationState.objects.filter(slug=self.slug).delete()

    def test_migrate_backend(self):
        # verify: attachment is in couch and migration not complete
        with maybe_not_found():
            s3_blobs = sum(1 for b in self.db.new_db._s3_bucket().objects.all())
            self.assertEqual(s3_blobs, 0)

        with tempdir() as tmp:
            filename = join(tmp, "file.txt")

            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
            self.assertGreaterEqual(migrated, self.test_size)

            # verify: migration state recorded
            mod.BlobMigrationState.objects.get(slug=self.slug)

            # verify: migrated data was written to the file
            with open(filename) as fh:
                lines = list(fh)
            ids = {d._id for d in self.migrate_docs}
            migrated = {d["_id"] for d in (json.loads(x) for x in lines)}
            self.assertEqual(len(ids.intersection(migrated)), self.test_size)

        # verify: attachment was copied to new blob db
        for doc in self.migrate_docs:
            exp = SavedBasicExport.get(doc._id)
            self.assertEqual(exp._rev, doc._rev)  # rev should not change
            self.assertTrue(doc.blobs)
            bucket = doc._blobdb_bucket()
            for meta in doc.blobs.values():
                content = self.db.new_db.get(meta.id, bucket)
                self.assertEqual(len(content.read()), meta.content_length)

def _mk_config(name='some export name', index='dummy_index'):
    return ExportConfiguration(index=index, name=name, format='xlsx')
<|endoftext|>


<commit_before>from datetime import date, datetime, timedelta

from django.test import TestCase
from icalendar import Calendar
from rest_framework.exceptions import ParseError

from .renderers import ICalRenderer
from .utils import parse_date

class ICalSerializerTest(TestCase):

    def test_data(self):
        renderer = ICalRenderer()
        events = [{'full_url': 'http://oxtalks.com/test/1',
                   'title': 'Talk 1',
                   'description': 'Description'},
                  {'full_url': 'http://oxtalks.com/test/2',
                   'title': 'Talk 2',
                   'description': ''}]
        data = renderer.render(events)
        cal = Calendar.from_ical(data)
        self.assertEquals(len(cal.subcomponents), 2)
        self.assertEquals(cal.subcomponents[0]['SUMMARY'], 'Talk 1')
        self.assertEquals(cal.subcomponents[0]['DESCRIPTION'], 'Description')
        self.assertEquals(cal.subcomponents[0]['URL'], 'http://oxtalks.com/test/1')
        self.assertEquals(cal.subcomponents[1]['SUMMARY'], 'Talk 2')
        self.assertEquals(cal.subcomponents[1]['DESCRIPTION'], '')
        self.assertEquals(cal.subcomponents[1]['URL'], 'http://oxtalks.com/test/2')

    def test_event_to_ics(self):
        renderer = ICalRenderer()
        events = {'full_url': 'http://oxtalks.com/test/1',
                  'title': 'Talk 1',
                  'description': 'Description'}
        data = renderer.render(events)
        cal = Calendar.from_ical(data)
        self.assertEquals(len(cal.subcomponents), 1)
        self.assertEquals(cal.subcomponents[0]['SUMMARY'], 'Talk 1')
        self.assertEquals(cal.subcomponents[0]['DESCRIPTION'], 'Description')
        self.assertEquals(cal.subcomponents[0]['URL'], 'http://oxtalks.com/test/1')

class UtilsParseDate(TestCase):

    def test_today(self):
        result = parse_date("today")
        self.assertEquals(result, date.today())

    def test_tomorrow(self):
        result = parse_date("tomorrow")
        self.assertEquals(result, date.today() + timedelta(days=1))

    def test_invalid_date(self):
        self.assertRaises(ParseError, parse_date, "01/01/aa")

    def test_custom_date(self):
        result = parse_date("13/02/15")
        self.assertEquals(result, datetime(2015, 2, 13, 0, 0))
<commit_msg>Fix tests broken in 0f43f8f<commit_after>from datetime import date, datetime, timedelta

from django.test import TestCase
from icalendar import Calendar
from rest_framework.exceptions import ParseError

from .renderers import ICalRenderer
from .utils import parse_date

class ICalSerializerTest(TestCase):

    def test_data(self):
        renderer = ICalRenderer()
        events = [{'full_url': 'http://oxtalks.com/test/1',
                   'title_display': 'Talk 1',
                   'description': 'Description'},
                  {'full_url': 'http://oxtalks.com/test/2',
                   'title_display': 'Talk 2',
                   'description': ''}]
        data = renderer.render(events)
        cal = Calendar.from_ical(data)
        self.assertEquals(len(cal.subcomponents), 2)
        self.assertEquals(cal.subcomponents[0]['SUMMARY'], 'Talk 1')
        self.assertEquals(cal.subcomponents[0]['DESCRIPTION'], 'Description')
        self.assertEquals(cal.subcomponents[0]['URL'], 'http://oxtalks.com/test/1')
        self.assertEquals(cal.subcomponents[1]['SUMMARY'], 'Talk 2')
        self.assertEquals(cal.subcomponents[1]['DESCRIPTION'], '')
        self.assertEquals(cal.subcomponents[1]['URL'], 'http://oxtalks.com/test/2')

    def test_event_to_ics(self):
        renderer = ICalRenderer()
        events = {'full_url': 'http://oxtalks.com/test/1',
                  'title_display': 'Talk 1',
                  'description': 'Description'}
        data = renderer.render(events)
        cal = Calendar.from_ical(data)
        self.assertEquals(len(cal.subcomponents), 1)
        self.assertEquals(cal.subcomponents[0]['SUMMARY'], 'Talk 1')
        self.assertEquals(cal.subcomponents[0]['DESCRIPTION'], 'Description')
        self.assertEquals(cal.subcomponents[0]['URL'], 'http://oxtalks.com/test/1')

class UtilsParseDate(TestCase):

    def test_today(self):
        result = parse_date("today")
        self.assertEquals(result, date.today())

    def test_tomorrow(self):
        result = parse_date("tomorrow")
        self.assertEquals(result, date.today() + timedelta(days=1))

    def test_invalid_date(self):
        self.assertRaises(ParseError, parse_date, "01/01/aa")

    def test_custom_date(self):
        result = parse_date("13/02/15")
        self.assertEquals(result, datetime(2015, 2, 13, 0, 0))
<|endoftext|>