<commit_before># -*- coding: utf-8 -*-

from io import BytesIO

import json
import os.path
import shutil
import tempfile

from crawlcrunch.compat import GzipFile
from crawlcrunch.compat import StringIO
from crawlcrunch.tests import EXAMPLES_PATH
from crawlcrunch.tests import DummyRoot
from crawlcrunch.tests import prepare_url_open
from crawlcrunch.tests import unittest

import mock

class LocalFilesRootTests(unittest.TestCase):

    def _make_one(self, path):
        from crawlcrunch.model.local_files import LocalFilesRoot
        return LocalFilesRoot(path)

    def test_companies_list_creation(self):
        root = self._make_one('foo')
        companies = root.get('companies')
        from crawlcrunch.model.local_files import CompanyList
        self.assertIsInstance(companies, CompanyList)

class CompanyTests(unittest.TestCase):

    def _make_one(self, local_data, name):
        from crawlcrunch.model.local_files import Company
        return Company(local_data, name)

    def test_url_generation(self):
        company = self._make_one('foo', 'facebook')
        expected = 'http://api.crunchbase.com/v/1/company/facebook.js'
        self.assertEqual(company.query_url(), expected)

    @mock.patch('crawlcrunch.model.local_files.url_open')
    def test_update(self, url_open):
        foo_url = 'http://api.crunchbase.com/v/1/company/foo.js'
        prepare_url_open(url_open,
                         {foo_url: {'foo': 'bar', }})
        from crawlcrunch.model.local_files import ZippedJsonFile
        from crawlcrunch.model.local_files import Company
        with tempfile.NamedTemporaryFile() as fp:
            local_data = ZippedJsonFile(fp.name)
            company = Company(local_data, 'foo')
            company.update()
        self.assertEqual(local_data.data, {'foo': 'bar'})
        self.assertEqual(company.data, {'foo': 'bar'})
        url_open.assert_called_once_with(foo_url)

    @mock.patch('crawlcrunch.model.local_files.url_open')
    def test_control_chars_in_response(self, urlopen):
        buf = BytesIO(b'["\x12fo\x14", "ba\x0b"]')
        buf.seek(0)
        urlopen.return_value = buf
        from crawlcrunch.model.local_files import ZippedJsonFile
        from crawlcrunch.model.local_files import Company
        with tempfile.NamedTemporaryFile() as fp:
            local_data = ZippedJsonFile(fp.name)
            company = Company(local_data, 'foo')
            company.update()
        self.assertEqual(local_data.data, ['fo', 'ba'])

class CompanyListTests(unittest.TestCase):

    def _make_one(self, path):
        from crawlcrunch.model.local_files import CompanyList
        return CompanyList(DummyRoot(), path)

    def test_list_creation_when_all_companies_files_present(self):
        cl = self._make_one(EXAMPLES_PATH['company_files_empty'])
        self.assertEqual(list(cl.list_not_local()), [])

    def test_local_list_when_company_files_empty(self):
        cl = self._make_one(EXAMPLES_PATH['company_files_empty'])
        result = list(cl.list_local())
        result.sort()
        expected = ['de-revolutione', 'group-laurier',
                    'hiconversion', 'pivotshare', 'vaporstream']
        self.assertEqual(result, expected)

    def test_get(self):
        cl = self._make_one(EXAMPLES_PATH['company_files_empty'])
        result = cl.get('de-revolutione')
        from crawlcrunch.model.local_files import Company
        self.assertIsInstance(result, Company)

class IntegrationTests(unittest.TestCase):

    def setUp(self):
        self.tmpd = tempfile.mkdtemp()

    def tearDown(self):
        shutil.rmtree(self.tmpd)

    def _make_one(self, path):
        from crawlcrunch.model.local_files import LocalFilesRoot
        return LocalFilesRoot(path)

    @mock.patch('crawlcrunch.model.local_files.url_open')
    def test_list_is_fetched_when_not_present(self, url_open):
        url = 'http://api.crunchbase.com/v/1/companies.js'
        prepare_url_open(url_open,
                         {url: [{'permalink': 'foo'}]})
        from crawlcrunch.model.local_files import CompanyList
        root = self._make_one(self.tmpd)
        companies = root.get('companies')
        companies.update()
        url_open.assert_called_once_with(url)
<commit_msg>gardening<commit_after># -*- coding: utf-8 -*-

from io import BytesIO

import json
import os.path
import shutil
import tempfile

from crawlcrunch.compat import GzipFile
from crawlcrunch.compat import StringIO
from crawlcrunch.tests import EXAMPLES_PATH
from crawlcrunch.tests import DummyRoot
from crawlcrunch.tests import prepare_url_open
from crawlcrunch.tests import unittest

import mock

class LocalFilesRootTests(unittest.TestCase):

    def _make_one(self, path):
        from crawlcrunch.model.local_files import LocalFilesRoot
        return LocalFilesRoot(path)

    def test_companies_list_creation(self):
        root = self._make_one('foo')
        companies = root.get('companies')
        from crawlcrunch.model.local_files import CompanyList
        self.assertIsInstance(companies, CompanyList)

class CompanyTests(unittest.TestCase):

    def _make_one(self, local_data, name):
        from crawlcrunch.model.local_files import Company
        return Company(local_data, name)

    def test_url_generation(self):
        company = self._make_one('foo', 'facebook')
        expected = 'http://api.crunchbase.com/v/1/company/facebook.js'
        self.assertEqual(company.query_url(), expected)

    @mock.patch('crawlcrunch.model.local_files.url_open')
    def test_update(self, url_open):
        foo_url = 'http://api.crunchbase.com/v/1/company/foo.js'
        prepare_url_open(url_open,
                         {foo_url: {'foo': 'bar', }})
        from crawlcrunch.model.local_files import ZippedJsonFile
        with tempfile.NamedTemporaryFile() as fp:
            local_data = ZippedJsonFile(fp.name)
            company = self._make_one(local_data, 'foo')
            company.update()
        self.assertEqual(local_data.data, {'foo': 'bar'})
        self.assertEqual(company.data, {'foo': 'bar'})
        url_open.assert_called_once_with(foo_url)

    @mock.patch('crawlcrunch.model.local_files.url_open')
    def test_control_chars_in_response(self, urlopen):
        buf = BytesIO(b'["\x12fo\x14", "ba\x0b"]')
        buf.seek(0)
        urlopen.return_value = buf
        from crawlcrunch.model.local_files import ZippedJsonFile
        with tempfile.NamedTemporaryFile() as fp:
            local_data = ZippedJsonFile(fp.name)
            company = self._make_one(local_data, 'foo')
            company.update()
        self.assertEqual(local_data.data, ['fo', 'ba'])

class CompanyListTests(unittest.TestCase):

    def _make_one(self, path):
        from crawlcrunch.model.local_files import CompanyList
        return CompanyList(DummyRoot(), path)

    def test_list_creation_when_all_companies_files_present(self):
        cl = self._make_one(EXAMPLES_PATH['company_files_empty'])
        self.assertEqual(list(cl.list_not_local()), [])

    def test_local_list_when_company_files_empty(self):
        cl = self._make_one(EXAMPLES_PATH['company_files_empty'])
        result = list(cl.list_local())
        result.sort()
        expected = ['de-revolutione', 'group-laurier',
                    'hiconversion', 'pivotshare', 'vaporstream']
        self.assertEqual(result, expected)

    def test_get(self):
        cl = self._make_one(EXAMPLES_PATH['company_files_empty'])
        result = cl.get('de-revolutione')
        from crawlcrunch.model.local_files import Company
        self.assertIsInstance(result, Company)

class IntegrationTests(unittest.TestCase):

    def setUp(self):
        self.tmpd = tempfile.mkdtemp()

    def tearDown(self):
        shutil.rmtree(self.tmpd)

    def _make_one(self, path):
        from crawlcrunch.model.local_files import LocalFilesRoot
        return LocalFilesRoot(path)

    @mock.patch('crawlcrunch.model.local_files.url_open')
    def test_list_is_fetched_when_not_present(self, url_open):
        url = 'http://api.crunchbase.com/v/1/companies.js'
        prepare_url_open(url_open,
                         {url: [{'permalink': 'foo'}]})
        from crawlcrunch.model.local_files import CompanyList
        root = self._make_one(self.tmpd)
        companies = root.get('companies')
        companies.update()
        url_open.assert_called_once_with(url)
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
Routes for end to end testing, can only be called when in test mode.

'''
__author__ = 'M@Campbell'

from flask import jsonify, current_app, request
from ooiservices.app.main import api
import urllib
import subprocess
import json

@api.route('/list_routes', methods=['GET'])
def list_routes():
    """
    Create a list of (url, endpoint) tuples from app.url_map
    Used for performance testing
    :return:
    """
    routes = []
    for rule in current_app.url_map.iter_rules():
        if 'GET' in rule.methods:
            url = urllib.unquote("{}".format(rule))
            routes.append((url, rule.endpoint))

    if not routes:
        return '{}', 204

    return jsonify({'routes': routes})

@api.route('/cache_keys', methods=['GET'])
def get_cache_list():
    # setup the container for the list of current cache items
    flask_cache = []

    # issue the command to get the list from redis
    pipe_output = subprocess.Popen(['redis-cli', 'scan', '0'],
                                   stdout=subprocess.PIPE)
    output, err = pipe_output.communicate()

    # the output is a string, so lets load it into the array based
    # in it's delimiter
    flask_cache = output.split('\n')

    # we don't want to provide system level redis keys, so iterate
    # over the list and only provide those that are prefixed
    # by 'flask_cache'
    temp_list = []
    for cache_key in flask_cache:
        if 'flask_cache' in cache_key:
            temp_list.append({'key': cache_key})

    # assign the list to the main bin
    flask_cache = temp_list

    # lets get the TTL of each of the keys so we can see how long ago
    # they were created.
    for cache_key in flask_cache:
        pipe_output = subprocess.Popen(['redis-cli', 'TTL', cache_key['key']],
                                       stdout=subprocess.PIPE)
        output, err = pipe_output.communicate()
        cache_key['TTL'] = output.split('\n')[0]

    # clear out this for garbage collection of flask_cache ref
    temp_list = None

    return jsonify({'cache_list': flask_cache}), 200

@api.route('/delete_cache', methods=['POST'])
def delete_cache():
    '''
    Example Post:
    {delete: [<cache_item_1>, <cache_item_2>, ...]}

    '''

    redis_response = 1

    try:
        # grab the json and load the payload
        redis_list = json.loads(request.data)

        # for each item in the list, delete the cache
        for cache_key in redis_list['delete']:
            pipe_output = subprocess.Popen(['redis-cli', 'del', cache_key],
                                           stdout=subprocess.PIPE)
            output, err = pipe_output.communicate()

            # figure out if the response is a great success
            # by multiplying the redis_response by the output
            # which will be a 1 for success and a 0 otherwise
            redis_response = redis_response*output

        return jsonify({'response': int(redis_response)}), 200

    except Exception as e:
        return jsonify({'error': 'Exception in cache deletion. %s' % e}), 500
<commit_msg>consolidated the route for easier access from the UI<commit_after>#!/usr/bin/env python
'''
Routes for end to end testing, can only be called when in test mode.

'''
__author__ = 'M@Campbell'

from flask import jsonify, current_app, request
from ooiservices.app.main import api
import urllib
import subprocess
import json

@api.route('/list_routes', methods=['GET'])
def list_routes():
    """
    Create a list of (url, endpoint) tuples from app.url_map
    Used for performance testing
    :return:
    """
    routes = []
    for rule in current_app.url_map.iter_rules():
        if 'GET' in rule.methods:
            url = urllib.unquote("{}".format(rule))
            routes.append((url, rule.endpoint))

    if not routes:
        return '{}', 204

    return jsonify({'routes': routes})

@api.route('/cache_keys', methods=['GET', 'POST'])
def cache_list():
    '''
    @method GET: returns this list of flask cache keys and their
                 time to live (TTL)
    @method POST: Submitting a post request with a list of keys
                  will delete each key in the list
                  Example: {delete: [<cache_item_1>, <cache_item_2>, ...]}
    '''
    if request.method == 'GET':
        # setup the container for the list of current cache items
        flask_cache = []

        # issue the command to get the list from redis
        pipe_output = subprocess.Popen(['redis-cli', 'scan', '0'],
                                       stdout=subprocess.PIPE)
        output, err = pipe_output.communicate()

        # the output is a string, so lets load it into the array based
        # in it's delimiter
        flask_cache = output.split('\n')

        # we don't want to provide system level redis keys, so iterate
        # over the list and only provide those that are prefixed
        # by 'flask_cache'
        temp_list = []
        for cache_key in flask_cache:
            if 'flask_cache' in cache_key:
                temp_list.append({'key': cache_key})

        # assign the list to the main bin
        flask_cache = temp_list

        # lets get the TTL of each of the keys so we can see how long ago
        # they were created.
        for cache_key in flask_cache:
            pipe_output = subprocess.Popen(['redis-cli', 'TTL',
                                            cache_key['key']],
                                           stdout=subprocess.PIPE)
            output, err = pipe_output.communicate()
            cache_key['TTL'] = output.split('\n')[0]

        # clear out this for garbage collection of flask_cache ref
        temp_list = None

        return jsonify({'cache_list': flask_cache}), 200

    elif request.method == 'POST':
        redis_response = 1

        try:
            # grab the json and load the payload
            redis_list = json.loads(request.data)

            # for each item in the list, delete the cache
            for cache_key in redis_list['delete']:
                pipe_output = subprocess.Popen(['redis-cli', 'del', cache_key],
                                               stdout=subprocess.PIPE)
                output, err = pipe_output.communicate()

                # figure out if the response is a great success
                # by multiplying the redis_response by the output
                # which will be a 1 for success and a 0 otherwise
                redis_response = redis_response*output

            return jsonify({'response': int(redis_response)}), 200

        except Exception as e:
            return jsonify({'error': 'Exception in cache delete. %s' % e}), 500
<|endoftext|>


<commit_before>from cached_property import cached_property
import concurrent.futures
from functools import partial
from itertools import chain
import logging
import multiprocessing
import os

from mapchete.config import MULTIPROCESSING_DEFAULT_START_METHOD
from mapchete.log import set_log_level

logger = logging.getLogger(__name__)

class Executor:
    """
    Executor factory for dask and concurrent.futures executor

    Will move into the mapchete core package.
    """

    def __new__(self, *args, concurrency=None, **kwargs):
        if concurrency == "dask":
            try:
                return DaskExecutor(*args, **kwargs)
            except ImportError as e:  # pragma: no cover
                raise ImportError(
                    f"this feature requires the mapchete[dask] extra: {e}"
                )
        elif concurrency is None or kwargs.get("max_workers") == 1:
            return SequentialExecutor(*args, **kwargs)
        elif concurrency in ["processes", "threads"]:
            return ConcurrentFuturesExecutor(*args, concurrency=concurrency, **kwargs)
        else:  # pragma: no cover
            raise ValueError(
                f"concurrency must be one of None, 'processes', 'threads' or 'dask', not {concurrency}"
            )

class _ExecutorBase:
    """Define base methods and properties of executors."""

    futures = None
    _as_completed = None
    _executor = None
    _executor_cls = None
    _executor_args = ()
    _executor_kwargs = {}

    def as_completed(self, func, iterable, fargs=None, fkwargs=None):
        """Yield finished tasks."""
        fargs = fargs or ()
        fkwargs = fkwargs or {}
        logger.debug("submitting tasks to executor")
        futures = [
            self._executor.submit(func, *chain([item], fargs), **fkwargs)
            for item in iterable
        ]
        self.futures.extend(futures)
        logger.debug(f"added {len(futures)} tasks")
        for future in self._as_completed(futures):
            yield future

    def cancel(self):
        logger.debug(f"cancel {len(self.futures)} futures...")
        for future in self.futures:
            future.cancel()
        logger.debug(f"{len(self.futures)} futures cancelled")

    def close(self):  # pragma: no cover
        self.__exit__(None, None, None)

    def _as_completed(self, *args, **kwargs):  # pragma: no cover
        raise NotImplementedError()

    @cached_property
    def _executor(self):
        return self._executor_cls(*self._executor_args, **self._executor_kwargs)

    def __enter__(self):
        """Enter context manager."""
        return self

    def __exit__(self, *args):
        """Exit context manager."""
        logger.debug(f"closing executor {self._executor}...")
        try:
            self._executor.close()
        except Exception:
            self._executor.__exit__(*args)
        logger.debug(f"closed executor {self._executor}")

    def __repr__(self):  # pragma: no cover
        return f"<Executor ({self._executor_cls})>"

class DaskExecutor(_ExecutorBase):
    """Execute tasks using dask cluster."""

    def __init__(
        self,
        *args,
        address=None,
        dask_scheduler=None,
        dask_client=None,
        max_workers=None,
        **kwargs,
    ):
        from dask.distributed import Client, LocalCluster

        self.futures = []
        self._executor_client = dask_client
        if self._executor_client:  # pragma: no cover
            logger.debug(f"using existing dask client: {dask_client}")
        else:
            local_cluster_kwargs = dict(
                n_workers=max_workers or os.cpu_count(), threads_per_worker=1
            )
            self._executor_cls = Client
            self._executor_kwargs = dict(
                address=dask_scheduler or LocalCluster(**local_cluster_kwargs),
            )
            logger.debug(
                f"starting dask.distributed.Client with kwargs {self._executor_kwargs}"
            )

    def cancel(self):
        logger.debug(f"cancel {len(self.futures)} futures...")
        for future in self.futures:
            future.cancel()
        logger.debug(f"{len(self.futures)} futures cancelled")

    def _as_completed(self, futures):
        from dask.distributed import as_completed

        if futures:
            for future in as_completed(futures):
                yield future

    @cached_property
    def _executor(self):
        return self._executor_client or self._executor_cls(
            *self._executor_args, **self._executor_kwargs
        )

    def __exit__(self, *args):
        """Exit context manager."""
        if self._executor_client:  # pragma: no cover
            logger.debug("client not closing as it was passed on as kwarg")
        else:
            logger.debug(f"closing executor {self._executor}...")
            try:
                self._executor.close()
            except Exception:  # pragma: no cover
                self._executor.__exit__(*args)
            logger.debug(f"closed executor {self._executor}")

class ConcurrentFuturesExecutor(_ExecutorBase):
    """Execute tasks using concurrent.futures."""

    def __init__(
        self,
        *args,
        max_workers=None,
        concurrency="processes",
        **kwargs,
    ):
        """Set attributes."""

        self.max_workers = max_workers or os.cpu_count()
        self.futures = []
        self._executor_kwargs = dict(
            max_workers=self.max_workers,
            initializer=set_log_level,
            initargs=(logger.getEffectiveLevel(),),
        )
        if concurrency == "processes":
            self._executor_cls = concurrent.futures.ProcessPoolExecutor
            start_method = (
                kwargs.get("multiprocessing_start_method")
                or MULTIPROCESSING_DEFAULT_START_METHOD
            )
            self._executor_kwargs.update(
                mp_context=multiprocessing.get_context(method=start_method)
            )
        elif concurrency == "threads":
            self._executor_cls = concurrent.futures.ThreadPoolExecutor
        else:  # pragma: no cover
            raise ValueError("concurrency must either be 'processes' or 'threads'")
        logger.debug(
            f"init ConcurrentFuturesExecutor using {concurrency} with {self.max_workers} workers"
        )

    def _as_completed(self, futures):
        """Yield finished tasks."""
        for future in concurrent.futures.as_completed(futures):
            yield future

class SequentialExecutor(_ExecutorBase):
    """Execute tasks sequentially in single process."""

    def __init__(self, *args, **kwargs):
        """Set attributes."""
        logger.debug("init SequentialExecutor")
        self.futures = []
        self._cancel = False

    def as_completed(self, func, iterable, fargs=None, fkwargs=None):
        """Yield finished tasks."""
        fargs = fargs or []
        fkwargs = fkwargs or {}
        for i in iterable:
            if self._cancel:
                return
            yield FakeFuture(func, fargs=[i, *fargs], fkwargs=fkwargs)

    def cancel(self):
        self._cancel = True

    def __exit__(self, *args):
        """Exit context manager."""
        logger.debug("SequentialExecutor closed")

    def __repr__(self):  # pragma: no cover
        """Return string representation."""
        return "SequentialExecutor"

class FakeFuture:
    """Wrapper class to mimick future interface."""

    def __init__(self, func, fargs=None, fkwargs=None):
        """Set attributes."""
        fargs = fargs or []
        fkwargs = fkwargs or {}
        try:
            self._result, self._exception = func(*fargs, **fkwargs), None
        except Exception as e:  # pragma: no cover
            self._result, self._exception = None, e

    def result(self):
        """Return task result."""
        if self._exception:
            logger.exception(self._exception)
            raise self._exception
        else:
            return self._result

    def exception(self):
        """Raise task exception if any."""
        return self._exception

    def cancelled(self):
        return False

    def __repr__(self):  # pragma: no cover
        """Return string representation."""
        return f"FakeFuture(result={self._result}, exception={self._exception})"
<commit_msg>we don't need to test this<commit_after>from cached_property import cached_property
import concurrent.futures
from functools import partial
from itertools import chain
import logging
import multiprocessing
import os

from mapchete.config import MULTIPROCESSING_DEFAULT_START_METHOD
from mapchete.log import set_log_level

logger = logging.getLogger(__name__)

class Executor:
    """
    Executor factory for dask and concurrent.futures executor

    Will move into the mapchete core package.
    """

    def __new__(self, *args, concurrency=None, **kwargs):
        if concurrency == "dask":
            try:
                return DaskExecutor(*args, **kwargs)
            except ImportError as e:  # pragma: no cover
                raise ImportError(
                    f"this feature requires the mapchete[dask] extra: {e}"
                )
        elif concurrency is None or kwargs.get("max_workers") == 1:
            return SequentialExecutor(*args, **kwargs)
        elif concurrency in ["processes", "threads"]:
            return ConcurrentFuturesExecutor(*args, concurrency=concurrency, **kwargs)
        else:  # pragma: no cover
            raise ValueError(
                f"concurrency must be one of None, 'processes', 'threads' or 'dask', not {concurrency}"
            )

class _ExecutorBase:
    """Define base methods and properties of executors."""

    futures = None
    _as_completed = None
    _executor = None
    _executor_cls = None
    _executor_args = ()
    _executor_kwargs = {}

    def as_completed(self, func, iterable, fargs=None, fkwargs=None):
        """Yield finished tasks."""
        fargs = fargs or ()
        fkwargs = fkwargs or {}
        logger.debug("submitting tasks to executor")
        futures = [
            self._executor.submit(func, *chain([item], fargs), **fkwargs)
            for item in iterable
        ]
        self.futures.extend(futures)
        logger.debug(f"added {len(futures)} tasks")
        for future in self._as_completed(futures):
            yield future

    def cancel(self):
        logger.debug(f"cancel {len(self.futures)} futures...")
        for future in self.futures:
            future.cancel()
        logger.debug(f"{len(self.futures)} futures cancelled")

    def close(self):  # pragma: no cover
        self.__exit__(None, None, None)

    def _as_completed(self, *args, **kwargs):  # pragma: no cover
        raise NotImplementedError()

    @cached_property
    def _executor(self):
        return self._executor_cls(*self._executor_args, **self._executor_kwargs)

    def __enter__(self):
        """Enter context manager."""
        return self

    def __exit__(self, *args):
        """Exit context manager."""
        logger.debug(f"closing executor {self._executor}...")
        try:
            self._executor.close()
        except Exception:
            self._executor.__exit__(*args)
        logger.debug(f"closed executor {self._executor}")

    def __repr__(self):  # pragma: no cover
        return f"<Executor ({self._executor_cls})>"

class DaskExecutor(_ExecutorBase):
    """Execute tasks using dask cluster."""

    def __init__(
        self,
        *args,
        address=None,
        dask_scheduler=None,
        dask_client=None,
        max_workers=None,
        **kwargs,
    ):
        from dask.distributed import Client, LocalCluster

        self.futures = []
        self._executor_client = dask_client
        if self._executor_client:  # pragma: no cover
            logger.debug(f"using existing dask client: {dask_client}")
        else:
            local_cluster_kwargs = dict(
                n_workers=max_workers or os.cpu_count(), threads_per_worker=1
            )
            self._executor_cls = Client
            self._executor_kwargs = dict(
                address=dask_scheduler or LocalCluster(**local_cluster_kwargs),
            )
            logger.debug(
                f"starting dask.distributed.Client with kwargs {self._executor_kwargs}"
            )

    def cancel(self):
        logger.debug(f"cancel {len(self.futures)} futures...")
        for future in self.futures:
            future.cancel()
        logger.debug(f"{len(self.futures)} futures cancelled")

    def _as_completed(self, futures):
        from dask.distributed import as_completed

        if futures:
            for future in as_completed(futures):
                yield future

    @cached_property
    def _executor(self):
        return self._executor_client or self._executor_cls(
            *self._executor_args, **self._executor_kwargs
        )

    def __exit__(self, *args):
        """Exit context manager."""
        if self._executor_client:  # pragma: no cover
            logger.debug("client not closing as it was passed on as kwarg")
        else:
            logger.debug(f"closing executor {self._executor}...")
            try:
                self._executor.close()
            except Exception:  # pragma: no cover
                self._executor.__exit__(*args)
            logger.debug(f"closed executor {self._executor}")

class ConcurrentFuturesExecutor(_ExecutorBase):
    """Execute tasks using concurrent.futures."""

    def __init__(
        self,
        *args,
        max_workers=None,
        concurrency="processes",
        **kwargs,
    ):
        """Set attributes."""

        self.max_workers = max_workers or os.cpu_count()
        self.futures = []
        self._executor_kwargs = dict(
            max_workers=self.max_workers,
            initializer=set_log_level,
            initargs=(logger.getEffectiveLevel(),),
        )
        if concurrency == "processes":
            self._executor_cls = concurrent.futures.ProcessPoolExecutor
            start_method = (
                kwargs.get("multiprocessing_start_method")
                or MULTIPROCESSING_DEFAULT_START_METHOD
            )
            self._executor_kwargs.update(
                mp_context=multiprocessing.get_context(method=start_method)
            )
        elif concurrency == "threads":
            self._executor_cls = concurrent.futures.ThreadPoolExecutor
        else:  # pragma: no cover
            raise ValueError("concurrency must either be 'processes' or 'threads'")
        logger.debug(
            f"init ConcurrentFuturesExecutor using {concurrency} with {self.max_workers} workers"
        )

    def _as_completed(self, futures):
        """Yield finished tasks."""
        for future in concurrent.futures.as_completed(futures):
            yield future

class SequentialExecutor(_ExecutorBase):
    """Execute tasks sequentially in single process."""

    def __init__(self, *args, **kwargs):
        """Set attributes."""
        logger.debug("init SequentialExecutor")
        self.futures = []
        self._cancel = False

    def as_completed(self, func, iterable, fargs=None, fkwargs=None):
        """Yield finished tasks."""
        fargs = fargs or []
        fkwargs = fkwargs or {}
        for i in iterable:
            if self._cancel:
                return
            yield FakeFuture(func, fargs=[i, *fargs], fkwargs=fkwargs)

    def cancel(self):
        self._cancel = True

    def __exit__(self, *args):
        """Exit context manager."""
        logger.debug("SequentialExecutor closed")

    def __repr__(self):  # pragma: no cover
        """Return string representation."""
        return "SequentialExecutor"

class FakeFuture:
    """Wrapper class to mimick future interface."""

    def __init__(self, func, fargs=None, fkwargs=None):
        """Set attributes."""
        fargs = fargs or []
        fkwargs = fkwargs or {}
        try:
            self._result, self._exception = func(*fargs, **fkwargs), None
        except Exception as e:  # pragma: no cover
            self._result, self._exception = None, e

    def result(self):
        """Return task result."""
        if self._exception:
            logger.exception(self._exception)
            raise self._exception
        else:
            return self._result

    def exception(self):
        """Raise task exception if any."""
        return self._exception

    def cancelled(self):  # pragma: no cover
        """Sequential futures cannot be cancelled."""
        return False

    def __repr__(self):  # pragma: no cover
        """Return string representation."""
        return f"FakeFuture(result={self._result}, exception={self._exception})"
<|endoftext|>


<commit_before># This file is part of the MapProxy project.
# Copyright (C) 2010, 2011 Omniscale <http://omniscale.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function, division

import os
import sys
import stat
import math
import time
from datetime import datetime

try:
    import cPickle as pickle
except ImportError:
    import pickle

from mapproxy.layer import map_extent_from_grid

import logging
log = logging.getLogger(__name__)

class bidict(dict):
    """
    Simplest bi-directional dictionary.
    """
    def __init__(self, iterator):
        for key, val in iterator:
            dict.__setitem__(self, key, val)
            dict.__setitem__(self, val, key)

class ETA(object):
    def __init__(self):
        self.avgs = []
        self.last_tick_start = time.time()
        self.progress = 0.0
        self.ticks = 10000
        self.tick_duration_sums = 0.0
        self.tick_duration_divisor = 0.0
        self.tick_count = 0

    def update(self, progress):
        self.progress = progress
        missing_ticks = (self.progress * self.ticks) - self.tick_count
        if missing_ticks:
            tick_duration = (time.time() - self.last_tick_start) / missing_ticks

            while missing_ticks > 0:

                # reduce the influence of older messurements
                self.tick_duration_sums *= 0.999
                self.tick_duration_divisor *= 0.999

                self.tick_count += 1

                self.tick_duration_sums += tick_duration
                self.tick_duration_divisor += 1

                missing_ticks -= 1

            self.last_tick_start = time.time()

    def eta_string(self):
        timestamp = self.eta()
        if timestamp is None:
            return 'N/A'
        try:
            return time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(timestamp))
        except ValueError:
            # raised when time is out of range (e.g. year >2038)
            return 'N/A'

    def eta(self):
        if not self.tick_count: return
        return (self.last_tick_start +
                ((self.tick_duration_sums/self.tick_duration_divisor)
                 * (self.ticks - self.tick_count)))

    def __str__(self):
        return self.eta_string()

class ProgressStore(object):
    """
    Reads and stores seed progresses to a file.
    """
    def __init__(self, filename=None, continue_seed=True):
        self.filename = filename
        if continue_seed:
            self.status = self.load()
        else:
            self.status = {}

    def load(self):
        if not os.path.exists(self.filename):
            pass
        elif os.stat(self.filename).st_mode & stat.S_IWOTH:
            log.error('progress file (%s) is world writable, ignoring file',
                self.filename)
        else:
            with open(self.filename, 'rb') as f:
                try:
                    return pickle.load(f)
                except (pickle.UnpicklingError, AttributeError,
                    EOFError, ImportError, IndexError):
                    log.error('unable to read progress file (%s), ignoring file',
                        self.filename)

        return {}

    def write(self):
        try:
            with open(self.filename + '.tmp', 'wb') as f:
                f.write(pickle.dumps(self.status))
                f.flush()
                os.fsync(f.fileno())
            os.rename(self.filename + '.tmp', self.filename)
        except (IOError, OSError) as ex:
            log.error('unable to write seed progress: %s', ex)

    def remove(self):
        self.status = {}
        if os.path.exists(self.filename):
            os.remove(self.filename)

    def get(self, task_identifier):
        return self.status.get(task_identifier, None)

    def add(self, task_identifier, progress_identifier):
        self.status[task_identifier] = progress_identifier

class ProgressLog(object):
    def __init__(self, out=None, silent=False, verbose=True, progress_store=None):
        if not out:
            out = sys.stdout
        self.out = out
        self.lastlog = time.time()
        self.verbose = verbose
        self.silent = silent
        self.current_task_id = None
        self.progress_store = progress_store

    def log_message(self, msg):
        self.out.write('[%s] %s\n' % (
            timestamp(), msg,
        ))
        self.out.flush()

    def log_step(self, progress):
        if not self.verbose:
            return
        if (self.lastlog + .1) < time.time():
            # log progress at most every 100ms
            self.out.write('[%s] %6.2f%%\t%-20s ETA: %s\r' % (
                timestamp(), progress.progress*100, progress.progress_str,
                progress.eta
            ))
            self.out.flush()
            self.lastlog = time.time()

    def log_progress(self, progress, level, bbox, tiles):
        if self.progress_store and self.current_task_id:
            self.progress_store.add(self.current_task_id,
                progress.current_progress_identifier())
            self.progress_store.write()

        if self.silent:
            return
        self.out.write('[%s] %2s %6.2f%% %s (%d tiles) ETA: %s\n' % (
            timestamp(), level, progress.progress*100,
            format_bbox(bbox), tiles, progress.eta))
        self.out.flush()

def limit_sub_bbox(bbox, sub_bbox):
    """
    >>> limit_sub_bbox((0, 1, 10, 11), (-1, -1, 9, 8))
    (0, 1, 9, 8)
    >>> limit_sub_bbox((0, 0, 10, 10), (5, 2, 18, 18))
    (5, 2, 10, 10)
    """
    minx = max(bbox[0], sub_bbox[0])
    miny = max(bbox[1], sub_bbox[1])
    maxx = min(bbox[2], sub_bbox[2])
    maxy = min(bbox[3], sub_bbox[3])
    return minx, miny, maxx, maxy

def timestamp():
    return datetime.now().strftime('%H:%M:%S')

def format_bbox(bbox):
    return ('%.5f, %.5f, %.5f, %.5f') % tuple(bbox)

def status_symbol(i, total):
    """
    >>> status_symbol(0, 1)
    '0'
    >>> [status_symbol(i, 4) for i in range(5)]
    ['.', 'o', 'O', '0', 'X']
    >>> [status_symbol(i, 10) for i in range(11)]
    ['.', '.', 'o', 'o', 'o', 'O', 'O', '0', '0', '0', 'X']
    """
    symbols = list(' .oO0')
    i += 1
    if 0 < i > total:
        return 'X'
    else:
        return symbols[int(math.ceil(i/(total/4)))]

class BackoffError(Exception):
    pass

def exp_backoff(func, args=(), kw={}, max_repeat=10, start_backoff_sec=2,
        exceptions=(Exception,), ignore_exceptions=tuple(), max_backoff=60):
    n = 0
    while True:
        try:
            result = func(*args, **kw)
        except ignore_exceptions:
            time.sleep(0.01)
        except exceptions as ex:
            if n >= max_repeat:
                print >>sys.stderr, "An error occured. Giving up"
                raise BackoffError
            wait_for = start_backoff_sec * 2**n
            if wait_for > max_backoff:
                wait_for = max_backoff
            print("An error occured. Retry in %d seconds: %r. Retries left: %d" %
                (wait_for, ex, (max_repeat - n)), file=sys.stderr)
            time.sleep(wait_for)
            n += 1
        else:
            return result

def format_seed_task(task):
    info = []
    info.append('  %s:' % (task.md['name'], ))
    if task.coverage is False:
        info.append("    Empty coverage given for this task")
        info.append("    Skipped")
        return '\n'.join(info)

    info.append("    Seeding cache '%s' with grid '%s' in %s" % (
                 task.md['cache_name'], task.md['grid_name'], task.grid.srs.srs_code))
    if task.coverage:
        info.append('    Limited to: %s (EPSG:4326)' % (format_bbox(task.coverage.extent.llbbox), ))
    else:
        info.append('   Complete grid: %s (EPSG:4326)' % (format_bbox(map_extent_from_grid(task.grid).llbbox), ))
    info.append('    Levels: %s' % (task.levels, ))

    if task.refresh_timestamp:
        info.append('    Overwriting: tiles older than %s' %
                    datetime.fromtimestamp(task.refresh_timestamp))
    elif task.refresh_timestamp == 0:
        info.append('    Overwriting: all tiles')
    else:
        info.append('    Overwriting: no tiles')

    return '\n'.join(info)

def format_cleanup_task(task):
    info = []
    info.append('  %s:' % (task.md['name'], ))
    if task.coverage is False:
        info.append("    Empty coverage given for this task")
        info.append("    Skipped")
        return '\n'.join(info)

    info.append("    Cleaning up cache '%s' with grid '%s' in %s" % (
                 task.md['cache_name'], task.md['grid_name'], task.grid.srs.srs_code))
    if task.coverage:
        info.append('    Limited to: %s (EPSG:4326)' % (format_bbox(task.coverage.extent.llbbox), ))
    else:
        info.append('    Complete grid: %s (EPSG:4326)' % (format_bbox(map_extent_from_grid(task.grid).llbbox), ))
    info.append('    Levels: %s' % (task.levels, ))

    if task.remove_timestamp:
        info.append('    Remove: tiles older than %s' %
                    datetime.fromtimestamp(task.remove_timestamp))
    else:
        info.append('    Remove: all tiles')

    return '\n'.join(info)
<commit_msg>mention coverage in maproxy-seed --summary<commit_after># This file is part of the MapProxy project.
# Copyright (C) 2010, 2011 Omniscale <http://omniscale.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function, division

import os
import sys
import stat
import math
import time
from datetime import datetime

try:
    import cPickle as pickle
except ImportError:
    import pickle

from mapproxy.layer import map_extent_from_grid

import logging
log = logging.getLogger(__name__)

class bidict(dict):
    """
    Simplest bi-directional dictionary.
    """
    def __init__(self, iterator):
        for key, val in iterator:
            dict.__setitem__(self, key, val)
            dict.__setitem__(self, val, key)

class ETA(object):
    def __init__(self):
        self.avgs = []
        self.last_tick_start = time.time()
        self.progress = 0.0
        self.ticks = 10000
        self.tick_duration_sums = 0.0
        self.tick_duration_divisor = 0.0
        self.tick_count = 0

    def update(self, progress):
        self.progress = progress
        missing_ticks = (self.progress * self.ticks) - self.tick_count
        if missing_ticks:
            tick_duration = (time.time() - self.last_tick_start) / missing_ticks

            while missing_ticks > 0:

                # reduce the influence of older messurements
                self.tick_duration_sums *= 0.999
                self.tick_duration_divisor *= 0.999

                self.tick_count += 1

                self.tick_duration_sums += tick_duration
                self.tick_duration_divisor += 1

                missing_ticks -= 1

            self.last_tick_start = time.time()

    def eta_string(self):
        timestamp = self.eta()
        if timestamp is None:
            return 'N/A'
        try:
            return time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(timestamp))
        except ValueError:
            # raised when time is out of range (e.g. year >2038)
            return 'N/A'

    def eta(self):
        if not self.tick_count: return
        return (self.last_tick_start +
                ((self.tick_duration_sums/self.tick_duration_divisor)
                 * (self.ticks - self.tick_count)))

    def __str__(self):
        return self.eta_string()

class ProgressStore(object):
    """
    Reads and stores seed progresses to a file.
    """
    def __init__(self, filename=None, continue_seed=True):
        self.filename = filename
        if continue_seed:
            self.status = self.load()
        else:
            self.status = {}

    def load(self):
        if not os.path.exists(self.filename):
            pass
        elif os.stat(self.filename).st_mode & stat.S_IWOTH:
            log.error('progress file (%s) is world writable, ignoring file',
                self.filename)
        else:
            with open(self.filename, 'rb') as f:
                try:
                    return pickle.load(f)
                except (pickle.UnpicklingError, AttributeError,
                    EOFError, ImportError, IndexError):
                    log.error('unable to read progress file (%s), ignoring file',
                        self.filename)

        return {}

    def write(self):
        try:
            with open(self.filename + '.tmp', 'wb') as f:
                f.write(pickle.dumps(self.status))
                f.flush()
                os.fsync(f.fileno())
            os.rename(self.filename + '.tmp', self.filename)
        except (IOError, OSError) as ex:
            log.error('unable to write seed progress: %s', ex)

    def remove(self):
        self.status = {}
        if os.path.exists(self.filename):
            os.remove(self.filename)

    def get(self, task_identifier):
        return self.status.get(task_identifier, None)

    def add(self, task_identifier, progress_identifier):
        self.status[task_identifier] = progress_identifier

class ProgressLog(object):
    def __init__(self, out=None, silent=False, verbose=True, progress_store=None):
        if not out:
            out = sys.stdout
        self.out = out
        self.lastlog = time.time()
        self.verbose = verbose
        self.silent = silent
        self.current_task_id = None
        self.progress_store = progress_store

    def log_message(self, msg):
        self.out.write('[%s] %s\n' % (
            timestamp(), msg,
        ))
        self.out.flush()

    def log_step(self, progress):
        if not self.verbose:
            return
        if (self.lastlog + .1) < time.time():
            # log progress at most every 100ms
            self.out.write('[%s] %6.2f%%\t%-20s ETA: %s\r' % (
                timestamp(), progress.progress*100, progress.progress_str,
                progress.eta
            ))
            self.out.flush()
            self.lastlog = time.time()

    def log_progress(self, progress, level, bbox, tiles):
        if self.progress_store and self.current_task_id:
            self.progress_store.add(self.current_task_id,
                progress.current_progress_identifier())
            self.progress_store.write()

        if self.silent:
            return
        self.out.write('[%s] %2s %6.2f%% %s (%d tiles) ETA: %s\n' % (
            timestamp(), level, progress.progress*100,
            format_bbox(bbox), tiles, progress.eta))
        self.out.flush()

def limit_sub_bbox(bbox, sub_bbox):
    """
    >>> limit_sub_bbox((0, 1, 10, 11), (-1, -1, 9, 8))
    (0, 1, 9, 8)
    >>> limit_sub_bbox((0, 0, 10, 10), (5, 2, 18, 18))
    (5, 2, 10, 10)
    """
    minx = max(bbox[0], sub_bbox[0])
    miny = max(bbox[1], sub_bbox[1])
    maxx = min(bbox[2], sub_bbox[2])
    maxy = min(bbox[3], sub_bbox[3])
    return minx, miny, maxx, maxy

def timestamp():
    return datetime.now().strftime('%H:%M:%S')

def format_bbox(bbox):
    return ('%.5f, %.5f, %.5f, %.5f') % tuple(bbox)

def status_symbol(i, total):
    """
    >>> status_symbol(0, 1)
    '0'
    >>> [status_symbol(i, 4) for i in range(5)]
    ['.', 'o', 'O', '0', 'X']
    >>> [status_symbol(i, 10) for i in range(11)]
    ['.', '.', 'o', 'o', 'o', 'O', 'O', '0', '0', '0', 'X']
    """
    symbols = list(' .oO0')
    i += 1
    if 0 < i > total:
        return 'X'
    else:
        return symbols[int(math.ceil(i/(total/4)))]

class BackoffError(Exception):
    pass

def exp_backoff(func, args=(), kw={}, max_repeat=10, start_backoff_sec=2,
        exceptions=(Exception,), ignore_exceptions=tuple(), max_backoff=60):
    n = 0
    while True:
        try:
            result = func(*args, **kw)
        except ignore_exceptions:
            time.sleep(0.01)
        except exceptions as ex:
            if n >= max_repeat:
                print >>sys.stderr, "An error occured. Giving up"
                raise BackoffError
            wait_for = start_backoff_sec * 2**n
            if wait_for > max_backoff:
                wait_for = max_backoff
            print("An error occured. Retry in %d seconds: %r. Retries left: %d" %
                (wait_for, ex, (max_repeat - n)), file=sys.stderr)
            time.sleep(wait_for)
            n += 1
        else:
            return result

def format_seed_task(task):
    info = []
    info.append('  %s:' % (task.md['name'], ))
    if task.coverage is False:
        info.append("    Empty coverage given for this task")
        info.append("    Skipped")
        return '\n'.join(info)

    info.append("    Seeding cache '%s' with grid '%s' in %s" % (
                 task.md['cache_name'], task.md['grid_name'], task.grid.srs.srs_code))
    if task.coverage:
        info.append('    Limited to coverage in: %s (EPSG:4326)' % (format_bbox(task.coverage.extent.llbbox), ))
    else:
        info.append('   Complete grid: %s (EPSG:4326)' % (format_bbox(map_extent_from_grid(task.grid).llbbox), ))
    info.append('    Levels: %s' % (task.levels, ))

    if task.refresh_timestamp:
        info.append('    Overwriting: tiles older than %s' %
                    datetime.fromtimestamp(task.refresh_timestamp))
    elif task.refresh_timestamp == 0:
        info.append('    Overwriting: all tiles')
    else:
        info.append('    Overwriting: no tiles')

    return '\n'.join(info)

def format_cleanup_task(task):
    info = []
    info.append('  %s:' % (task.md['name'], ))
    if task.coverage is False:
        info.append("    Empty coverage given for this task")
        info.append("    Skipped")
        return '\n'.join(info)

    info.append("    Cleaning up cache '%s' with grid '%s' in %s" % (
                 task.md['cache_name'], task.md['grid_name'], task.grid.srs.srs_code))
    if task.coverage:
        info.append('    Limited to coverage in: %s (EPSG:4326)' % (format_bbox(task.coverage.extent.llbbox), ))
    else:
        info.append('    Complete grid: %s (EPSG:4326)' % (format_bbox(map_extent_from_grid(task.grid).llbbox), ))
    info.append('    Levels: %s' % (task.levels, ))

    if task.remove_timestamp:
        info.append('    Remove: tiles older than %s' %
                    datetime.fromtimestamp(task.remove_timestamp))
    else:
        info.append('    Remove: all tiles')

    return '\n'.join(info)
<|endoftext|>


<commit_before>from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from flask.ext.sqlalchemy import SQLAlchemy
from geoalchemy2.types import Geometry
from random import random
from datetime import datetime
from maproulette import app

# set up the ORM engine and database object
engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'], convert_unicode=True)
db_session = scoped_session(sessionmaker(autocommit=False,
    autoflush=False,
    bind=engine))
Base = declarative_base()
Base.query = db_session.query_property()
db = SQLAlchemy(app)

challenge_types = {
    'default': []}

class User(db.Model):
    __tablename__ = 'users'

    id = db.Column(db.Integer, unique=True, primary_key=True)
    oauth_token = db.Column(db.String)
    oauth_secret = db.Column(db.String)
    display_name = db.Column(db.String)
    home_location = db.Column(Geometry('POINT'))
    languages = db.Column(db.String)
    changeset_count = db.Column(db.Integer)
    last_changeset_id = db.Column(db.Integer)
    last_changeset_date = db.Column(db.DateTime)
    last_changeset_bbox = db.Column(Geometry('POLYGON'))
    osm_account_created = db.Column(db.DateTime)
    difficulty = db.Column(db.SmallInteger)

    def __unicode__(self):
        return self.display_name

# challenge
#  -tasks
#    -actions

# a challenge is like 'fix all highway tags' and does not belong to anything
# else - it has no foreign keys
class Challenge(db.Model):
    __tablename__ = 'challenges'

    id = db.Column(db.Integer, unique=True, primary_key=True)
    slug = db.Column(db.String(72), primary_key=True)
    title = db.Column(db.String(128))
    description = db.Column(db.String)
    blurb = db.Column(db.String)
    polygon = db.Column(Geometry('POLYGON'))
    help = db.Column(db.String)
    instruction = db.Column(db.String)
    run = db.Column(db.String(72))
    active = db.Column(db.Boolean)
    difficulty = db.Column(db.SmallInteger)
    type = db.Column(db.String, default = 'default')
    done_dialog = db.Column(db.String, default =
'{"Done": "I am Done!","Skipped": "Did not do it."}')
# no idea if this is an appropriate default

    __table_args__ = (
        db.Index('idx_geom', polygon, postgresql_using='gist'),
        db.Index('idx_run', run)
        )

    def __init__(self, slug):
        self.slug = slug

    def __unicode__(self):
        return self.slug

    def _get_task_available(self, task):
        """The function for a task to determine
        if it's available or not."""
        # Most tasks will use this method
        action = task.current_action
        if action.status == 'available':
            return True
        else:
            return False

    def _set_task_status(self, task):
        """This is the function that runs after a task action is set,
        to set its secondary availability."""
        current = task.current
        if current.status == 'skipped':
            task.state = 'available'
        elif current.status == 'fixed':
            task.state = 'done'
        elif (current.status == 'alreadyfixed' or
            current.status == 'falsepositive'):
            l = [i for i in task.actions if i.status == "falsepositive" \
                     or i.status == "alreadyfixed"]
            if len(l) >= 2:
                task.status = 'done'
            else:
                task.status = 'available'
        else:
            # This is a catchall that a task should never get to
            task.status = 'available'

    @property
    def meta(self):
        """Return a dictionary of metadata for the challenge"""
        return {'slug': self.slug,
                'description': self.description,
                'help': self.help,
                'blurb': self.blurb,
                'instruction': self.instruction,
                'doneDlg': self.done_dialog,
                'editors': self.editors
                }

# a task is like 'fix this highway here' and belongs to a challenge
# and has actions associated with it
class Task(db.Model):
    __tablename__ = 'tasks'

    id = db.Column(db.Integer, unique=True, primary_key=True)
    identifier = db.Column(db.String(72))
    challenge_id = db.Column(db.Integer, db.ForeignKey('challenges.id'))
    location = db.Column(Geometry('POINT'))
    run  = db.Column(db.String(72))
    random = db.Column(db.Float, default=random())
    manifest = db.Column(db.String)
    actions = db.relationship("Action", backref = db.backref("task"))
    instructions = db.Column(db.String())
    challenge = db.relationship("Challenge",
                             backref=db.backref('tasks', order_by=id))
    __table_args__ = (
        db.Index('idx_location', location, postgresql_using='gist'),
        db.Index('idx_id', id),
        db.Index('idx_challenge', challenge_id),
        db.Index('idx_random', random)
        )

    def __init__(self, challenge_id):
        self.challenge_id = challenge_id

    def __repr__(self):
        return '<Task %d>' % (self.id)

    def current_state(self):
        """Displays the current state of a task"""
        return self.current_action.state

# actions are associated with tasks and belong to users
class Action(db.Model):
    __tablename__ = 'actions'

    id = db.Column(db.Integer, unique=True, primary_key=True)
    timestamp = db.Column(db.DateTime, default = datetime.now)
    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
    task_id = db.Column(db.Integer, db.ForeignKey('tasks.id'))
    status = db.Column(db.String(32))

    def __init__(self, task_id, status, user_id = None):
        self.task_id = task_id
        self.status = status
        if user_id:
            self.user_id = user_id
<commit_msg>Cleaned up models file<commit_after>"""This file contains the SQLAlechemy ORM models"""

from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from flask.ext.sqlalchemy import SQLAlchemy
from geoalchemy2.types import Geometry
from random import random
from datetime import datetime
from maproulette import app

# set up the ORM engine and database object
engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'],
                       convert_unicode=True)
db_session = scoped_session(sessionmaker(autocommit=False,
                                         autoflush=False,
                                         bind=engine))
Base = declarative_base()
Base.query = db_session.query_property()
db = SQLAlchemy(app)

challenge_types = {
    'default': []}

class User(db.Model):
    __tablename__ = 'users'

    id = db.Column(db.Integer, unique=True, primary_key=True)
    oauth_token = db.Column(db.String)
    oauth_secret = db.Column(db.String)
    display_name = db.Column(db.String)
    home_location = db.Column(Geometry('POINT'))
    languages = db.Column(db.String)
    changeset_count = db.Column(db.Integer)
    last_changeset_id = db.Column(db.Integer)
    last_changeset_date = db.Column(db.DateTime)
    last_changeset_bbox = db.Column(Geometry('POLYGON'))
    osm_account_created = db.Column(db.DateTime)
    difficulty = db.Column(db.SmallInteger)

    def __unicode__(self):
        return self.display_name

class Challenge(db.Model):
    __tablename__ = 'challenges'

    id = db.Column(db.Integer, unique=True, primary_key=True)
    slug = db.Column(db.String(72), primary_key=True)
    title = db.Column(db.String(128))
    description = db.Column(db.String)
    blurb = db.Column(db.String)
    polygon = db.Column(Geometry('POLYGON'))
    help = db.Column(db.String)
    instruction = db.Column(db.String)
    run = db.Column(db.String(72))
    active = db.Column(db.Boolean)
    difficulty = db.Column(db.SmallInteger)
    type = db.Column(db.String, default='default')
    done_dialog = db.Column(db.String)

    __table_args__ = (db.Index('idx_geom', polygon, postgresql_using='gist'),
                      db.Index('idx_run', run))

    def __init__(self, slug):
        self.slug = slug

    def __unicode__(self):
        return self.slug

    def _get_task_available(self, task):
        """The function for a task to determine if it's available or not."""
        action = task.current_action
        if action.status == 'available':
            return True
        else:
            return False

    def _set_task_status(self, task):
        """This is the function that runs after a task action is set,
        to set its secondary availability."""
        current = task.current
        if current.status == 'skipped':
            task.state = 'available'
        elif current.status == 'fixed':
            task.state = 'done'
        elif (current.status == 'alreadyfixed' or
              current.status == 'falsepositive'):
            l = [i for i in task.actions if i.status == "falsepositive"
                 or i.status == "alreadyfixed"]
            if len(l) >= 2:
                task.status = 'done'
            else:
                task.status = 'available'
        else:
            # This is a catchall that a task should never get to
            task.status = 'available'

    @property
    def meta(self):
        """Return a dictionary of metadata for the challenge"""
        return {'slug': self.slug,
                'description': self.description,
                'help': self.help,
                'blurb': self.blurb,
                'instruction': self.instruction,
                'doneDlg': self.done_dialog,
                'editors': self.editors
                }

class Task(db.Model):
    __tablename__ = 'tasks'

    id = db.Column(db.Integer, unique=True, primary_key=True)
    identifier = db.Column(db.String(72))
    challenge_id = db.Column(db.Integer, db.ForeignKey('challenges.id'))
    location = db.Column(Geometry('POINT'))
    run = db.Column(db.String(72))
    random = db.Column(db.Float, default=random())
    manifest = db.Column(db.String)
    actions = db.relationship("Action", backref=db.backref("task"))
    instructions = db.Column(db.String())
    challenge = db.relationship("Challenge",
                                backref=db.backref('tasks', order_by=id))
    __table_args__ = (
        db.Index('idx_location', location, postgresql_using='gist'),
        db.Index('idx_id', id),
        db.Index('idx_challenge', challenge_id),
        db.Index('idx_random', random))

    def __init__(self, challenge_id):
        self.challenge_id = challenge_id

    def __repr__(self):
        return '<Task %d>' % (self.id)

    def current_state(self):
        """Displays the current state of a task"""
        return self.current_action.state

class Action(db.Model):
    __tablename__ = 'actions'

    id = db.Column(db.Integer, unique=True, primary_key=True)
    timestamp = db.Column(db.DateTime, default=datetime.now)
    user_id = db.Column(db.Integer, db.ForeignKey('users.id'))
    task_id = db.Column(db.Integer, db.ForeignKey('tasks.id'))
    status = db.Column(db.String(32))

    def __init__(self, task_id, status, user_id=None):
        self.task_id = task_id
        self.status = status
        if user_id:
            self.user_id = user_id
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2019 Akretion France (http://www.akretion.com/)
# @author: Alexis de Lattre <alexis.delattre@akretion.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo import models, fields, _
from odoo.exceptions import UserError
from dateutil.relativedelta import relativedelta
from odoo.tools import float_compare, float_round

class AccountCutoff(models.Model):
    _inherit = 'account.cutoff'

    def get_lines(self):
        res = super(AccountCutoff, self).get_lines()
        if self.type not in ['accrued_expense', 'accrued_revenue']:
            return res
        aml_obj = self.env['account.move.line']
        line_obj = self.env['account.cutoff.line']
        sub_obj = self.env['account.cutoff.accrual.subscription']
        type2subtype = {
            'accrued_expense': 'expense',
            'accrued_revenue': 'revenue',
            }
        sub_type = type2subtype[self.type]
        sign = sub_type == 'revenue' and -1 or 1
        subs = sub_obj.search([
            ('company_id', '=', self.company_id.id),
            ('subscription_type', '=', sub_type),
            ('start_date', '<=', self.cutoff_date),
            ])
        cutoff_date = fields.Date.from_string(self.cutoff_date)
        if subs:
            # check that the cutoff is the last day of a month
            # otherwise, we have pb with when we compute intervals
            # with cutoff date is a day which is not present for all months
            last_day_same_month = cutoff_date + relativedelta(day=31)
            if last_day_same_month.day != cutoff_date.day:
                raise UserError(_(
                    "The cutoffs with subscription only work when the cutoff "
                    "date (%s) is the last day of a month.")
                    % self.cutoff_date)
            if not self.source_journal_ids:
                raise UserError(_('Missing source journals.'))
        periodicity2months = {
            'month': 1,
            'quarter': 3,
            'semester': 6,
            'year': 12,
            }
        company_currency = self.company_currency_id
        prec = company_currency.rounding
        work = {}
        # Generate time intervals and computing existing expenses/revenue
        for sub in subs:
            sub_start_date_dt = fields.Date.from_string(sub.start_date)
            months = periodicity2months[sub.periodicity]
            work[sub] = {'intervals': []}
            end_date = cutoff_date
            for i in range(12 / months):
                start_date = end_date + relativedelta(
                    day=1, months=-(months-1))
                if start_date < sub_start_date_dt:
                    break
                # compute amount
                amount = 0
                # 1. No start/end dates
                no_start_end_res = aml_obj.read_group([
                    ('company_id', '=', sub.company_id.id),
                    ('journal_id', 'in', self.source_journal_ids.ids),
                    ('account_id', '=', sub.account_id.id),
                    ('analytic_account_id', '=',
                        sub.analytic_account_id.id or False),
                    ('date', '<=', fields.Date.to_string(end_date)),
                    ('date', '>=', fields.Date.to_string(start_date)),
                    ('start_date', '=', False),
                    ('end_date', '=', False),
                    ], ['balance'], [])
                amount_no_start_end = no_start_end_res and\
                    no_start_end_res[0]['balance'] or 0
                amount += amount_no_start_end * sign
                # 2. Start/end dates, INSIDE interval
                domain_base = [
                    ('company_id', '=', sub.company_id.id),
                    ('journal_id', 'in', self.source_journal_ids.ids),
                    ('start_date', '!=', False),
                    ('end_date', '!=', False),
                    ('account_id', '=', sub.account_id.id),
                    ('analytic_account_id', '=',
                        sub.analytic_account_id.id or False),
                    ]
                inside_res = aml_obj.read_group(
                    domain_base + [
                        ('start_date', '>=', start_date),
                        ('end_date', '<=', end_date),
                    ], ['balance'], [])
                amount_inside = inside_res and inside_res[0]['balance'] or 0
                amount += amount_inside * sign
                # 3. Start/end dates, OVER interval
                mlines = aml_obj.search(
                    domain_base + [
                        ('start_date', '<', start_date),
                        ('end_date', '>', end_date),
                    ])
                for mline in mlines:
                    total_days = (
                        fields.Date.from_string(mline.end_date) -
                        fields.Date.from_string(mline.start_date)
                        ).days + 1
                    days_in_interval = (end_date - start_date).days + 1
                    amount_in_interval = mline.balance *\
                        days_in_interval / float(total_days)
                    amount += amount_in_interval * sign
                # 4. Start/end dates, start_date before, end_date inside
                mlines = aml_obj.search(
                    domain_base + [
                        ('start_date', '<', start_date),
                        ('end_date', '>=', start_date),
                        ('end_date', '<=', end_date),
                        ])
                for mline in mlines:
                    total_days = (
                        fields.Date.from_string(mline.end_date) -
                        fields.Date.from_string(mline.start_date)
                        ).days + 1
                    days_in_interval = (
                        fields.Date.from_string(mline.end_date)
                        - start_date).days + 1
                    amount_in_interval = mline.balance *\
                        days_in_interval / float(total_days)
                    amount += amount_in_interval * sign
                # 5. Start/end dates, start_date inside, end_date after
                mlines = aml_obj.search(
                    domain_base + [
                        ('start_date', '>=', start_date),
                        ('start_date', '<=', end_date),
                        ('end_date', '>', end_date),
                        ])
                for mline in mlines:
                    total_days = (
                        fields.Date.from_string(mline.end_date) -
                        fields.Date.from_string(mline.start_date)
                        ).days + 1
                    days_in_interval = (
                        end_date -
                        fields.Date.from_string(mline.start_date)).days + 1
                    amount_in_interval = mline.balance *\
                        days_in_interval / float(total_days)
                    amount += amount_in_interval * sign

                work[sub]['intervals'].append({
                    'start': start_date,
                    'end': end_date,
                    'amount': float_round(amount, precision_rounding=prec),
                    })
                # prepare next interval
                end_date = start_date + relativedelta(days=-1)
        # Create mapping dict
        mapping = self.env['account.cutoff.mapping']._get_mapping_dict(
            self.company_id.id, self.type)
        sub_type_label = sub_type == 'expense' and _('Expense') or _('Revenue')
        lsign = sub_type == 'expense' and -1 or 1
        # Compute provisions for each subscription
        # Write the details of the computation in the notes
        for sub in work.keys():
            notes = []
            cutoff_amount = 0
            if sub.account_id.id in mapping:
                cutoff_account_id = mapping[sub.account_id.id]
            else:
                cutoff_account_id = sub.account_id.id
            for interval in work[sub]['intervals']:
                if float_compare(
                        interval['amount'], sub.min_amount,
                        precision_rounding=prec) < 0:
                    period_cutoff_amount = float_round(
                        sub.provision_amount - interval['amount'],
                        precision_rounding=prec)
                    notes.append(_(
                        "Period from %s to %s: %s %s under the minimum "
                        "amount %s. Default provisionning amount is %s, "
                        "therefore provisionning %s.") % (
                            fields.Date.to_string(interval['start']),
                            fields.Date.to_string(interval['end']),
                            sub_type_label,
                            interval['amount'],
                            float_round(
                                sub.min_amount, precision_rounding=prec),
                            float_round(
                                sub.provision_amount, precision_rounding=prec),
                            period_cutoff_amount))
                    cutoff_amount += period_cutoff_amount * lsign
                else:
                    notes.append(_(
                        "Period from %s to %s: %s %s over the minimum "
                        "amount %s => no provisionning.") % (
                            fields.Date.to_string(interval['start']),
                            fields.Date.to_string(interval['end']),
                            sub_type_label,
                            interval['amount'],
                            float_round(
                                sub.min_amount, precision_rounding=prec)))
            vals = {
                'parent_id': self.id,
                'partner_id': sub.partner_id.id,
                'account_id': sub.account_id.id,
                'analytic_account_id': sub.analytic_account_id.id or False,
                'name': sub.name,
                'currency_id': sub.company_currency_id.id,
                'tax_ids': [(6, 0, sub.tax_ids.ids)],
                'amount': 0,
                'cutoff_amount': cutoff_amount,
                'cutoff_account_id': cutoff_account_id,
                'notes': '\n'.join(notes),
                }
            if sub.tax_ids:
                tax_compute_all_res = sub.tax_ids.compute_all(
                    cutoff_amount, partner=sub.partner_id)
                vals['tax_line_ids'] = self._prepare_tax_lines(
                    tax_compute_all_res, company_currency)
            line_obj.create(vals)
        return res
<commit_msg>[FIX] Add partner_id in domain when searching account moves<commit_after># -*- coding: utf-8 -*-
# Copyright 2019 Akretion France (http://www.akretion.com/)
# @author: Alexis de Lattre <alexis.delattre@akretion.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo import models, fields, _
from odoo.exceptions import UserError
from dateutil.relativedelta import relativedelta
from odoo.tools import float_compare, float_round

class AccountCutoff(models.Model):
    _inherit = 'account.cutoff'

    def get_lines(self):
        res = super(AccountCutoff, self).get_lines()
        if self.type not in ['accrued_expense', 'accrued_revenue']:
            return res
        aml_obj = self.env['account.move.line']
        line_obj = self.env['account.cutoff.line']
        sub_obj = self.env['account.cutoff.accrual.subscription']
        type2subtype = {
            'accrued_expense': 'expense',
            'accrued_revenue': 'revenue',
            }
        sub_type = type2subtype[self.type]
        sign = sub_type == 'revenue' and -1 or 1
        subs = sub_obj.search([
            ('company_id', '=', self.company_id.id),
            ('subscription_type', '=', sub_type),
            ('start_date', '<=', self.cutoff_date),
            ])
        cutoff_date = fields.Date.from_string(self.cutoff_date)
        if subs:
            # check that the cutoff is the last day of a month
            # otherwise, we have pb with when we compute intervals
            # with cutoff date is a day which is not present for all months
            last_day_same_month = cutoff_date + relativedelta(day=31)
            if last_day_same_month.day != cutoff_date.day:
                raise UserError(_(
                    "The cutoffs with subscription only work when the cutoff "
                    "date (%s) is the last day of a month.")
                    % self.cutoff_date)
            if not self.source_journal_ids:
                raise UserError(_('Missing source journals.'))
        periodicity2months = {
            'month': 1,
            'quarter': 3,
            'semester': 6,
            'year': 12,
            }
        company_currency = self.company_currency_id
        prec = company_currency.rounding
        work = {}
        # Generate time intervals and computing existing expenses/revenue
        for sub in subs:
            sub_start_date_dt = fields.Date.from_string(sub.start_date)
            months = periodicity2months[sub.periodicity]
            work[sub] = {'intervals': []}
            end_date = cutoff_date
            for i in range(12 / months):
                start_date = end_date + relativedelta(
                    day=1, months=-(months-1))
                if start_date < sub_start_date_dt:
                    break
                # compute amount
                domain_base = [
                    ('company_id', '=', sub.company_id.id),
                    ('journal_id', 'in', self.source_journal_ids.ids),
                    ('account_id', '=', sub.account_id.id),
                    ('analytic_account_id', '=',
                        sub.analytic_account_id.id or False),
                    ('partner_id', '=', sub.partner_id.id),
                    ]
                domain_base_w_start_end = domain_base + [
                    ('start_date', '!=', False),
                    ('end_date', '!=', False),
                    ]
                amount = 0
                # 1. No start/end dates
                no_start_end_res = aml_obj.read_group(
                    domain_base + [
                        ('date', '<=', fields.Date.to_string(end_date)),
                        ('date', '>=', fields.Date.to_string(start_date)),
                        ('start_date', '=', False),
                        ('end_date', '=', False),
                    ], ['balance'], [])
                amount_no_start_end = no_start_end_res and\
                    no_start_end_res[0]['balance'] or 0
                amount += amount_no_start_end * sign
                # 2. Start/end dates, INSIDE interval
                inside_res = aml_obj.read_group(
                    domain_base_w_start_end + [
                        ('start_date', '>=', start_date),
                        ('end_date', '<=', end_date),
                    ], ['balance'], [])
                amount_inside = inside_res and inside_res[0]['balance'] or 0
                amount += amount_inside * sign
                # 3. Start/end dates, OVER interval
                mlines = aml_obj.search(
                    domain_base_w_start_end + [
                        ('start_date', '<', start_date),
                        ('end_date', '>', end_date),
                    ])
                for mline in mlines:
                    total_days = (
                        fields.Date.from_string(mline.end_date) -
                        fields.Date.from_string(mline.start_date)
                        ).days + 1
                    days_in_interval = (end_date - start_date).days + 1
                    amount_in_interval = mline.balance *\
                        days_in_interval / float(total_days)
                    amount += amount_in_interval * sign
                # 4. Start/end dates, start_date before, end_date inside
                mlines = aml_obj.search(
                    domain_base_w_start_end + [
                        ('start_date', '<', start_date),
                        ('end_date', '>=', start_date),
                        ('end_date', '<=', end_date),
                        ])
                for mline in mlines:
                    total_days = (
                        fields.Date.from_string(mline.end_date) -
                        fields.Date.from_string(mline.start_date)
                        ).days + 1
                    days_in_interval = (
                        fields.Date.from_string(mline.end_date)
                        - start_date).days + 1
                    amount_in_interval = mline.balance *\
                        days_in_interval / float(total_days)
                    amount += amount_in_interval * sign
                # 5. Start/end dates, start_date inside, end_date after
                mlines = aml_obj.search(
                    domain_base_w_start_end + [
                        ('start_date', '>=', start_date),
                        ('start_date', '<=', end_date),
                        ('end_date', '>', end_date),
                        ])
                for mline in mlines:
                    total_days = (
                        fields.Date.from_string(mline.end_date) -
                        fields.Date.from_string(mline.start_date)
                        ).days + 1
                    days_in_interval = (
                        end_date -
                        fields.Date.from_string(mline.start_date)).days + 1
                    amount_in_interval = mline.balance *\
                        days_in_interval / float(total_days)
                    amount += amount_in_interval * sign

                work[sub]['intervals'].append({
                    'start': start_date,
                    'end': end_date,
                    'amount': float_round(amount, precision_rounding=prec),
                    })
                # prepare next interval
                end_date = start_date + relativedelta(days=-1)
        # Create mapping dict
        mapping = self.env['account.cutoff.mapping']._get_mapping_dict(
            self.company_id.id, self.type)
        sub_type_label = sub_type == 'expense' and _('Expense') or _('Revenue')
        lsign = sub_type == 'expense' and -1 or 1
        # Compute provisions for each subscription
        # Write the details of the computation in the notes
        for sub in work.keys():
            notes = []
            cutoff_amount = 0
            if sub.account_id.id in mapping:
                cutoff_account_id = mapping[sub.account_id.id]
            else:
                cutoff_account_id = sub.account_id.id
            for interval in work[sub]['intervals']:
                if float_compare(
                        interval['amount'], sub.min_amount,
                        precision_rounding=prec) < 0:
                    period_cutoff_amount = float_round(
                        sub.provision_amount - interval['amount'],
                        precision_rounding=prec)
                    notes.append(_(
                        "Period from %s to %s: %s %s under the minimum "
                        "amount %s. Default provisionning amount is %s, "
                        "therefore provisionning %s.") % (
                            fields.Date.to_string(interval['start']),
                            fields.Date.to_string(interval['end']),
                            sub_type_label,
                            interval['amount'],
                            float_round(
                                sub.min_amount, precision_rounding=prec),
                            float_round(
                                sub.provision_amount, precision_rounding=prec),
                            period_cutoff_amount))
                    cutoff_amount += period_cutoff_amount * lsign
                else:
                    notes.append(_(
                        "Period from %s to %s: %s %s over the minimum "
                        "amount %s => no provisionning.") % (
                            fields.Date.to_string(interval['start']),
                            fields.Date.to_string(interval['end']),
                            sub_type_label,
                            interval['amount'],
                            float_round(
                                sub.min_amount, precision_rounding=prec)))
            vals = {
                'parent_id': self.id,
                'partner_id': sub.partner_id.id,
                'account_id': sub.account_id.id,
                'analytic_account_id': sub.analytic_account_id.id or False,
                'name': sub.name,
                'currency_id': sub.company_currency_id.id,
                'tax_ids': [(6, 0, sub.tax_ids.ids)],
                'amount': 0,
                'cutoff_amount': cutoff_amount,
                'cutoff_account_id': cutoff_account_id,
                'notes': '\n'.join(notes),
                }
            if sub.tax_ids:
                tax_compute_all_res = sub.tax_ids.compute_all(
                    cutoff_amount, partner=sub.partner_id)
                vals['tax_line_ids'] = self._prepare_tax_lines(
                    tax_compute_all_res, company_currency)
            line_obj.create(vals)
        return res
<|endoftext|>


<commit_before>import logging
import json

from smtplib import SMTPException

from django.shortcuts import render_to_response
from django.http import HttpResponse, HttpResponseRedirect
from django.template import RequestContext
from django.contrib import messages
from django.contrib.auth import login, logout
from django.utils import translation
from django.contrib.auth.forms import PasswordChangeForm
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext as _
from django.core.exceptions import PermissionDenied
from django.db import IntegrityError

from django_fixmystreet.backoffice.forms import FmsUserForm, GroupForm
from django_fixmystreet.fixmystreet.models import OrganisationEntity, UserOrganisationMembership, FMSUser

logger = logging.getLogger(__name__)

def list_groups(request):
    current_user = request.fmsuser

    groups = OrganisationEntity.objects.filter(type__in=['D', 'S'])

    #~ if current_user.organisation:
        #~ groups = groups.filter(dependency=current_user.organisation)
    #~ elif not request.user.is_superuser:
        #~ raise PermissionDenied()

    return render_to_response("pro/auth/groups_list.html", {
        'groups': groups,
        'can_create' : current_user.leader
    }, context_instance=RequestContext(request))

def create_group(request,):
    can_edit = request.fmsuser.leader

    if request.method == "POST" and can_edit:
        group_form = GroupForm(request.POST)

        if group_form.is_valid():
            group = group_form.save(commit=False)
            group.dependency = request.fmsuser.organisation
            group.save()

            messages.add_message(request, messages.SUCCESS, _("Group has been created successfully"))
            return HttpResponseRedirect(reverse('list_groups'))

    else:
        group_form = GroupForm()

    return render_to_response("pro/auth/group_create.html",
            {
                "group_form": group_form,
                "can_edit":  can_edit,
            },
            context_instance=RequestContext(request))

def edit_group(request, group_id):
    can_edit = request.fmsuser.leader

    instance = OrganisationEntity.objects.get(id=group_id)
    if request.method == "POST" and can_edit:
        group_form = GroupForm(request.POST, instance=instance)

        if group_form.is_valid():
            group = group_form.save()

            messages.add_message(request, messages.SUCCESS, _("Group has been updated successfully"))
            return HttpResponseRedirect(reverse('list_groups'))

    else:
        group_form = GroupForm(edit=True, instance=instance)

    return render_to_response("pro/auth/group_create.html",
            {
                "group_id"   : instance.id,
                "group_form" : group_form,
                "memberships"  : UserOrganisationMembership.objects.filter(organisation=instance),
                "can_edit" : can_edit
            },
            context_instance=RequestContext(request))

def delete_group(request, group_id):
    can_edit = request.fmsuser.leader

    instance = OrganisationEntity.objects.get(id=group_id)
    if request.method == "GET" and can_edit:
            # Delete group and memberships (in pre_delete)
            instance.delete()

            messages.add_message(request, messages.SUCCESS, _("Group has been deleted successfully"))

    return HttpResponseRedirect(reverse('list_groups'))

def add_membership(request, group_id, user_id):
    can_edit = request.fmsuser.leader

    if can_edit:
        response_data = {}
        organisation  = OrganisationEntity.objects.get(id=group_id)
        user          = FMSUser.objects.get(id=user_id)

        user_organisation_membership = UserOrganisationMembership(user=user, organisation=organisation)

        try:
            user_organisation_membership.save()
            response_data['status'] = 'OK'
        except IntegrityError:
            response_data['status'] = 'IntegrityError'

        response_data['membership_id'] = user_organisation_membership.id
    else:
        response_data['status'] = 'Permission Denied'

    return HttpResponse(json.dumps(response_data), content_type="application/json")

def remove_membership(request, membership_id):
    can_edit = request.fmsuser.leader

    if not can_edit:
        return HttpResponse('Permission Denied');

    isManager = request.fmsuser.manager

    user_organisation_membership = UserOrganisationMembership.objects.get(id=membership_id)
    user_organisation_membership.delete()

    return HttpResponse('OK');
<commit_msg>added tests to test group administration<commit_after>import logging
import json

from smtplib import SMTPException

from django.shortcuts import render_to_response
from django.http import HttpResponse, HttpResponseRedirect
from django.template import RequestContext
from django.contrib import messages
from django.contrib.auth import login, logout
from django.utils import translation
from django.contrib.auth.forms import PasswordChangeForm
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext as _
from django.core.exceptions import PermissionDenied
from django.db import IntegrityError

from django_fixmystreet.backoffice.forms import FmsUserForm, GroupForm
from django_fixmystreet.fixmystreet.models import OrganisationEntity, UserOrganisationMembership, FMSUser

logger = logging.getLogger(__name__)

def list_groups(request):
    current_user = request.fmsuser

    groups = OrganisationEntity.objects.filter(type__in=['D', 'S'])

    #~ if current_user.organisation:
        #~ groups = groups.filter(dependency=current_user.organisation)
    #~ elif not request.user.is_superuser:
        #~ raise PermissionDenied()

    return render_to_response("pro/auth/groups_list.html", {
        'groups': groups,
        'can_create' : current_user.leader
    }, context_instance=RequestContext(request))

def create_group(request,):
    can_edit = request.fmsuser.leader

    if request.method == "POST" and can_edit:
        group_form = GroupForm(request.POST)

        if group_form.is_valid():
            group = group_form.save(commit=False)
            group.dependency = request.fmsuser.organisation
            group.save()

            messages.add_message(request, messages.SUCCESS, _("Group has been created successfully"))
            return HttpResponseRedirect(reverse('list_groups'))

    else:
        group_form = GroupForm()

    return render_to_response("pro/auth/group_create.html",
            {
                "group_form": group_form,
                "can_edit":  can_edit,
            },
            context_instance=RequestContext(request))

def edit_group(request, group_id):
    can_edit = request.fmsuser.leader

    instance = OrganisationEntity.objects.get(id=group_id)
    if request.method == "POST" and can_edit:
        group_form = GroupForm(request.POST, instance=instance)

        if group_form.is_valid():
            group = group_form.save()

            messages.add_message(request, messages.SUCCESS, _("Group has been updated successfully"))
            return HttpResponseRedirect(reverse('list_groups'))

    else:
        group_form = GroupForm(edit=True, instance=instance)

    return render_to_response("pro/auth/group_create.html",
            {
                "group_id"   : instance.id,
                "group_form" : group_form,
                "memberships"  : UserOrganisationMembership.objects.filter(organisation=instance),
                "can_edit" : can_edit
            },
            context_instance=RequestContext(request))

def delete_group(request, group_id):
    can_edit = request.fmsuser.leader

    instance = OrganisationEntity.objects.get(id=group_id)
    if request.method == "GET" and can_edit:
            # Delete group and memberships (in pre_delete)
            instance.delete()

            messages.add_message(request, messages.SUCCESS, _("Group has been deleted successfully"))

    return HttpResponseRedirect(reverse('list_groups'))

def add_membership(request, group_id, user_id):
    can_edit = request.fmsuser.leader
    response_data = {}
    if can_edit:
        organisation  = OrganisationEntity.objects.get(id=group_id)
        user          = FMSUser.objects.get(id=user_id)

        user_organisation_membership = UserOrganisationMembership(user=user, organisation=organisation)

        try:
            user_organisation_membership.save()
            response_data['status'] = 'OK'
        except IntegrityError:
            response_data['status'] = 'IntegrityError'

        response_data['membership_id'] = user_organisation_membership.id
    else:
        response_data['status'] = 'Permission Denied'

    return HttpResponse(json.dumps(response_data), content_type="application/json")

def remove_membership(request, membership_id):
    can_edit = request.fmsuser.leader

    if not can_edit:
        return HttpResponse('Permission Denied');

    isManager = request.fmsuser.manager

    user_organisation_membership = UserOrganisationMembership.objects.get(id=membership_id)
    user_organisation_membership.delete()

    return HttpResponse('OK');
<|endoftext|>