<commit_before>import asyncio
from urllib.parse import urlencode

import aiohttp
import requests
import slacker

from .compat import PY_350, create_task

__version__ = '0.0.2'

Error = slacker.Error

Response = slacker.Response

class BaseAPI(slacker.BaseAPI):

    def __init__(
        self,
        token=None,
        timeout=slacker.DEFAULT_TIMEOUT,
        *, loop=None
    ):
        if loop is None:
            loop = asyncio.get_event_loop()

        self.loop = loop

        self.session = aiohttp.ClientSession(
            connector=aiohttp.TCPConnector(
                use_dns_cache=False,
                loop=self.loop,
            ),
        )

        self.methods = {
            requests.get: 'GET',
            requests.post: 'POST',
        }

        self.futs = set()

        super().__init__(token=token, timeout=timeout)

    @asyncio.coroutine
    def __request(self, method, api, **kwargs):
        method = self.methods[method]

        kwargs.setdefault('params', {})
        kwargs.setdefault('timeout', None)

        if self.token:
            kwargs['params']['token'] = self.token

        kwargs['params'] = urlencode(kwargs['params'], doseq=True)

        if method == 'POST':
            files = kwargs.pop('files', None)

            if files is not None:
                data = kwargs.pop('data', {})

                _data = aiohttp.FormData()

                for k, v in files.items():
                    _data.add_field(k, open(v.name, 'rb'))

                for k, v in data.items():
                    if v is not None:
                        _data.add_field(k, str(v))

                kwargs['data'] = _data

        _url = slacker.API_BASE_URL.format(api=api)

        _request = self.session.request(method, _url, **kwargs)

        _response = None

        try:
            with aiohttp.Timeout(self.timeout, loop=self.loop):
                _response = yield from _request

            _response.raise_for_status()

            text = yield from _response.text()
        finally:
            if _response is not None:
                yield from _response.release()

        response = Response(text)

        if not response.successful:
            raise Error(response.error)

        return response

    def _request(self, method, api, **kwargs):
        coro = self.__request(method, api, **kwargs)
        fut = create_task(loop=self.loop)(coro)

        self.futs.add(fut)
        fut.add_done_callback(self.futs.remove)

        return fut

    @asyncio.coroutine
    def close(self):
        if self.futs:
            yield from asyncio.gather(*self.futs)

        yield from self.session.close()

class IM(BaseAPI, slacker.IM):
    pass

class API(BaseAPI, slacker.API):
    pass

class DND(BaseAPI, slacker.DND):
    pass

class RTM(BaseAPI, slacker.RTM):
    pass

class Auth(BaseAPI, slacker.Auth):
    pass

class Bots(BaseAPI, slacker.Bots):
    pass

class Chat(BaseAPI, slacker.Chat):
    pass

class TeamProfile(BaseAPI, slacker.TeamProfile):
    pass

class Team(BaseAPI, slacker.Team):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._profile = TeamProfile(*args, **kwargs)

    def close(self):
        coros = [
            super().close(),
            self._profile.close(),
        ]
        return asyncio.gather(*coros, loop=self.loop)

class Pins(BaseAPI, slacker.Pins):
    pass

class MPIM(BaseAPI, slacker.MPIM):
    pass

class UsersProfile(BaseAPI, slacker.UsersProfile):
    pass

class Users(BaseAPI, slacker.Users):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._profile = UsersProfile(*args, **kwargs)

    def close(self):
        coros = [
            super().close(),
            self._profile.close(),
        ]
        return asyncio.gather(*coros, loop=self.loop)

class FilesComments(BaseAPI, slacker.FilesComments):
    pass

class Files(BaseAPI, slacker.Files):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._comments = FilesComments(*args, **kwargs)

    def close(self):
        coros = [
            super().close(),
            self._comments.close(),
        ]
        return asyncio.gather(*coros, loop=self.loop)

class Stars(BaseAPI, slacker.Stars):
    pass

class Emoji(BaseAPI, slacker.Emoji):
    pass

class Search(BaseAPI, slacker.Search):
    pass

class Groups(BaseAPI, slacker.Groups):
    pass

class Channels(BaseAPI, slacker.Channels):
    pass

class Presence(BaseAPI, slacker.Presence):
    pass

class Reminders(BaseAPI, slacker.Reminders):
    pass

class Reactions(BaseAPI, slacker.Reactions):
    pass

class IDPGroups(BaseAPI, slacker.IDPGroups):
    pass

class UserGroupsUsers(BaseAPI, slacker.UserGroupsUsers):
    pass

class UserGroups(BaseAPI, slacker.UserGroups):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._users = UserGroupsUsers(*args, **kwargs)

    def close(self):
        coros = [
            super().close(),
            self._users.close(),
        ]
        return asyncio.gather(*coros, loop=self.loop)

class IncomingWebhook(BaseAPI, slacker.IncomingWebhook):

    def __init__(
        self,
        url=None,
        timeout=slacker.DEFAULT_TIMEOUT,
        *, loop=None
    ):
        self.url = url

        super().__init__(token=None, timeout=timeout, loop=loop)

    @asyncio.coroutine
    def post(self, data):
        if not self.url:
            raise slacker.Error('URL for incoming webhook is undefined')

        _request = self.session.request(
            'POST',
            self.url,
            data=data,
            timeout=None,
        )

        _response = None

        try:
            with aiohttp.Timeout(self.timeout, loop=self.loop):
                _response = yield from _request

            _response.raise_for_status()

            _json = yield from _response.json()
        finally:
            if _response is not None:
                yield from _response.release()

        return _json

class Slacker(slacker.Slacker):

    def __init__(
        self,
        token,
        incoming_webhook_url=None,
        timeout=slacker.DEFAULT_TIMEOUT,
        *, loop=None
    ):
        if loop is None:
            loop = asyncio.get_event_loop()

        self.loop = loop

        self.im = IM(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.api = API(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.dnd = DND(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.rtm = RTM(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.auth = Auth(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.bots = Bots(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.chat = Chat(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.team = Team(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.pins = Pins(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.mpim = MPIM(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.users = Users(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.files = Files(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.stars = Stars(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.emoji = Emoji(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.search = Search(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.groups = Groups(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.channels = Channels(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.presence = Presence(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.reminders = Reminders(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.reactions = Reactions(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.idpgroups = IDPGroups(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.usergroups = UserGroups(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.incomingwebhook = IncomingWebhook(
            url=incoming_webhook_url,
            timeout=timeout,
            loop=self.loop,
        )

    def close(self):
        coros = [
            self.im.close(),
            self.api.close(),
            self.dnd.close(),
            self.rtm.close(),
            self.auth.close(),
            self.bots.close(),
            self.chat.close(),
            self.team.close(),
            self.pins.close(),
            self.mpim.close(),
            self.users.close(),
            self.files.close(),
            self.stars.close(),
            self.emoji.close(),
            self.search.close(),
            self.groups.close(),
            self.channels.close(),
            self.presence.close(),
            self.reminders.close(),
            self.reactions.close(),
            self.idpgroups.close(),
            self.usergroups.close(),
            self.incomingwebhook.close(),
        ]

        return asyncio.gather(*coros, loop=self.loop)

    if PY_350:
        @asyncio.coroutine
        def __aenter__(self):  # noqa
            return self

        @asyncio.coroutine
        def __aexit__(self, *exc_info):  # noqa
            yield from self.close()
<commit_msg>Close return future instead of async calls<commit_after>import asyncio
from urllib.parse import urlencode

import aiohttp
import requests
import slacker

from .compat import PY_350, create_task

__version__ = '0.0.2'

Error = slacker.Error

Response = slacker.Response

class BaseAPI(slacker.BaseAPI):

    def __init__(
        self,
        token=None,
        timeout=slacker.DEFAULT_TIMEOUT,
        *, loop=None
    ):
        if loop is None:
            loop = asyncio.get_event_loop()

        self.loop = loop

        self.session = aiohttp.ClientSession(
            connector=aiohttp.TCPConnector(
                use_dns_cache=False,
                loop=self.loop,
            ),
        )

        self.methods = {
            requests.get: 'GET',
            requests.post: 'POST',
        }

        self.futs = set()

        super().__init__(token=token, timeout=timeout)

    @asyncio.coroutine
    def __request(self, method, api, **kwargs):
        method = self.methods[method]

        kwargs.setdefault('params', {})
        kwargs.setdefault('timeout', None)

        if self.token:
            kwargs['params']['token'] = self.token

        kwargs['params'] = urlencode(kwargs['params'], doseq=True)

        if method == 'POST':
            files = kwargs.pop('files', None)

            if files is not None:
                data = kwargs.pop('data', {})

                _data = aiohttp.FormData()

                for k, v in files.items():
                    _data.add_field(k, open(v.name, 'rb'))

                for k, v in data.items():
                    if v is not None:
                        _data.add_field(k, str(v))

                kwargs['data'] = _data

        _url = slacker.API_BASE_URL.format(api=api)

        _request = self.session.request(method, _url, **kwargs)

        _response = None

        try:
            with aiohttp.Timeout(self.timeout, loop=self.loop):
                _response = yield from _request

            _response.raise_for_status()

            text = yield from _response.text()
        finally:
            if _response is not None:
                yield from _response.release()

        response = Response(text)

        if not response.successful:
            raise Error(response.error)

        return response

    def _request(self, method, api, **kwargs):
        coro = self.__request(method, api, **kwargs)
        fut = create_task(loop=self.loop)(coro)

        self.futs.add(fut)
        fut.add_done_callback(self.futs.remove)

        return fut

    def close(self):
        if self.futs:
            return asyncio.gather(*self.futs)

        return self.session.close()

class IM(BaseAPI, slacker.IM):
    pass

class API(BaseAPI, slacker.API):
    pass

class DND(BaseAPI, slacker.DND):
    pass

class RTM(BaseAPI, slacker.RTM):
    pass

class Auth(BaseAPI, slacker.Auth):
    pass

class Bots(BaseAPI, slacker.Bots):
    pass

class Chat(BaseAPI, slacker.Chat):
    pass

class TeamProfile(BaseAPI, slacker.TeamProfile):
    pass

class Team(BaseAPI, slacker.Team):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._profile = TeamProfile(*args, **kwargs)

    def close(self):
        coros = [
            super().close(),
            self._profile.close(),
        ]
        return asyncio.gather(*coros, loop=self.loop)

class Pins(BaseAPI, slacker.Pins):
    pass

class MPIM(BaseAPI, slacker.MPIM):
    pass

class UsersProfile(BaseAPI, slacker.UsersProfile):
    pass

class Users(BaseAPI, slacker.Users):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._profile = UsersProfile(*args, **kwargs)

    def close(self):
        coros = [
            super().close(),
            self._profile.close(),
        ]
        return asyncio.gather(*coros, loop=self.loop)

class FilesComments(BaseAPI, slacker.FilesComments):
    pass

class Files(BaseAPI, slacker.Files):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._comments = FilesComments(*args, **kwargs)

    def close(self):
        coros = [
            super().close(),
            self._comments.close(),
        ]
        return asyncio.gather(*coros, loop=self.loop)

class Stars(BaseAPI, slacker.Stars):
    pass

class Emoji(BaseAPI, slacker.Emoji):
    pass

class Search(BaseAPI, slacker.Search):
    pass

class Groups(BaseAPI, slacker.Groups):
    pass

class Channels(BaseAPI, slacker.Channels):
    pass

class Presence(BaseAPI, slacker.Presence):
    pass

class Reminders(BaseAPI, slacker.Reminders):
    pass

class Reactions(BaseAPI, slacker.Reactions):
    pass

class IDPGroups(BaseAPI, slacker.IDPGroups):
    pass

class UserGroupsUsers(BaseAPI, slacker.UserGroupsUsers):
    pass

class UserGroups(BaseAPI, slacker.UserGroups):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._users = UserGroupsUsers(*args, **kwargs)

    def close(self):
        coros = [
            super().close(),
            self._users.close(),
        ]
        return asyncio.gather(*coros, loop=self.loop)

class IncomingWebhook(BaseAPI, slacker.IncomingWebhook):

    def __init__(
        self,
        url=None,
        timeout=slacker.DEFAULT_TIMEOUT,
        *, loop=None
    ):
        self.url = url

        super().__init__(token=None, timeout=timeout, loop=loop)

    @asyncio.coroutine
    def post(self, data):
        if not self.url:
            raise slacker.Error('URL for incoming webhook is undefined')

        _request = self.session.request(
            'POST',
            self.url,
            data=data,
            timeout=None,
        )

        _response = None

        try:
            with aiohttp.Timeout(self.timeout, loop=self.loop):
                _response = yield from _request

            _response.raise_for_status()

            _json = yield from _response.json()
        finally:
            if _response is not None:
                yield from _response.release()

        return _json

class Slacker(slacker.Slacker):

    def __init__(
        self,
        token,
        incoming_webhook_url=None,
        timeout=slacker.DEFAULT_TIMEOUT,
        *, loop=None
    ):
        if loop is None:
            loop = asyncio.get_event_loop()

        self.loop = loop

        self.im = IM(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.api = API(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.dnd = DND(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.rtm = RTM(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.auth = Auth(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.bots = Bots(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.chat = Chat(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.team = Team(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.pins = Pins(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.mpim = MPIM(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.users = Users(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.files = Files(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.stars = Stars(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.emoji = Emoji(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.search = Search(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.groups = Groups(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.channels = Channels(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.presence = Presence(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.reminders = Reminders(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.reactions = Reactions(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.idpgroups = IDPGroups(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.usergroups = UserGroups(
            token=token,
            timeout=timeout,
            loop=self.loop,
        )
        self.incomingwebhook = IncomingWebhook(
            url=incoming_webhook_url,
            timeout=timeout,
            loop=self.loop,
        )

    def close(self):
        coros = [
            self.im.close(),
            self.api.close(),
            self.dnd.close(),
            self.rtm.close(),
            self.auth.close(),
            self.bots.close(),
            self.chat.close(),
            self.team.close(),
            self.pins.close(),
            self.mpim.close(),
            self.users.close(),
            self.files.close(),
            self.stars.close(),
            self.emoji.close(),
            self.search.close(),
            self.groups.close(),
            self.channels.close(),
            self.presence.close(),
            self.reminders.close(),
            self.reactions.close(),
            self.idpgroups.close(),
            self.usergroups.close(),
            self.incomingwebhook.close(),
        ]

        return asyncio.gather(*coros, loop=self.loop)

    if PY_350:
        @asyncio.coroutine
        def __aenter__(self):  # noqa
            return self

        def __aexit__(self, *exc_info):  # noqa
            return self.close()
<|endoftext|>


<commit_before># -*- coding: utf-8  -*-
import pywikibot
import pywikibot.data.wikidataquery as pwq
import json

def item_from_id(repo, identifier):
    """ Get item from its ID.

    Args:
        repo: data repository of the site.
        identifier (int): ID of an item (eg. 120 for Q120)
    """
    return pywikibot.ItemPage(repo, "Q%s" % identifier)

def fetch_property_values(item, prop):
    """ Fetch a property from an item. """
    target = None
    if prop in item.claims:
        claim = item.claims[prop][0]
        target = claim.getTarget()
    return target

class Airport:
    """ Object to represent an airport and its data. """

    def __init__(self, iata=None, icao=None, en=None, geo=None, country=None):
        """ Constructor of airport.

        Args:
            iata (str): IATA code of the Airport (P238)
            icao (str): ICAO code of the Airport (P239)
            en (str): Name of the english wikipedia article about the airport
            geo (str): lat, long of the airport.
        """
        self.values = {
            'iata': iata,
            'icao': icao,
            'en': en,
            'geo': geo,
            'country': country
            }

    def __repr__(self):
        """ string to for print. """
        vals = [self.values['iata'], self.values['icao'], self.values['en'], self.values['geo'], self.values['country']]
        display_vals = ['' if v is None else v for v in vals]
        return u';'.join(display_vals).encode('utf-8')

def main():
    """ Main script of airport dabatase creation. """
    from argparse import ArgumentParser

    parser = ArgumentParser(description="Commons Cat Metrics")
    parser.add_argument("-f", "--f",
        type=str,
        dest="output",
        metavar="FILE",
        required=True,
        help="output file")
    args = parser.parse_args()

    with open(args.output, 'w', 0) as output:

        site = pywikibot.Site("fr", "wikipedia")
        repo = site.data_repository()
        page = pywikibot.Page(site, u"Aéroport de Paris-Orly")
        item = pywikibot.ItemPage.fromPage(page)
        dictionary = item.get()

        # retrieve airport list
        q= pwq.HasClaim(31,items=[1248784])
        dat = pwq.WikidataQuery(cacheMaxAge=600).query(q)
        items = dat['items']
        print "Found %d items for query: %s\n" % (len(items), q)
        for i in items:
            item = item_from_id(repo, i)
            item.get()
            name = ''
            if 'enwiki' in item.sitelinks:
                name = item.sitelinks['enwiki']
            coordinate = fetch_property_values(item, 'P625')
            latitude = ''
            longitude= ''
            if coordinate is not None:
                latitude = coordinate.lat
                longitude = coordinate.lon
            airport = Airport(iata=fetch_property_values(item, 'P238'),
                    icao= fetch_property_values(item, 'P239'),
                    geo= "%s, %s" % (latitude, longitude),
                    country=fetch_property_values(item, 'P17'),
                    en=name)
            output.write(repr(airport)+'\n')

if __name__ == '__main__':
    main()
<commit_msg>Improved style, removed unused import, etc.<commit_after># -*- coding: utf-8  -*-

import pywikibot
import pywikibot.data.wikidataquery as pwq

def item_from_id(repo, identifier):
    """ Get item from its ID.

    Args:
        repo: data repository of the site.
        identifier (int): ID of an item (eg. 120 for Q120)
    """
    return pywikibot.ItemPage(repo, "Q%s" % identifier)

def fetch_property_values(item, prop):
    """ Fetch a property from an item. """
    target = None
    if prop in item.claims:
        claim = item.claims[prop][0]
        target = claim.getTarget()
    return target

class Airport(object):

    """ Object to represent an airport and its data. """

    def __init__(self, iata=None, icao=None, en=None, geo=None, country=None):
        """ Constructor of airport.

        Args:
            iata (str): IATA code of the Airport (P238)
            icao (str): ICAO code of the Airport (P239)
            en (str): Name of the english wikipedia article about the airport
            geo (str): lat, long of the airport.
        """
        self.values = {
            'iata': iata,
            'icao': icao,
            'en': en,
            'geo': geo,
            'country': country
        }

    def __repr__(self):
        """ string to for print. """
        vals = [self.values['iata'],
                self.values['icao'],
                self.values['en'],
                self.values['geo'],
                self.values['country']]
        display_vals = ['' if v is None else v for v in vals]
        return u';'.join(display_vals).encode('utf-8')

def main():
    """ Main script of airport dabatase creation. """
    from argparse import ArgumentParser

    parser = ArgumentParser(description="Airport DB")
    parser.add_argument("-f", "--f",
                        type=str,
                        dest="output",
                        metavar="FILE",
                        required=True,
                        help="output file")
    args = parser.parse_args()

    with open(args.output, 'w', 0) as output:

        site = pywikibot.Site("fr", "wikipedia")
        repo = site.data_repository()

        # retrieve airport list
        query = pwq.HasClaim(31, items=[1248784])
        dat = pwq.WikidataQuery(cacheMaxAge=600).query(query)
        items = dat['items']
        print "Found %d items for query: %s\n" % (len(items), query)
        for i in items:
            item = item_from_id(repo, i)
            item.get()
            name = ''
            if 'enwiki' in item.sitelinks:
                name = item.sitelinks['enwiki']
            coordinate = fetch_property_values(item, 'P625')
            latitude = ''
            longitude = ''
            if coordinate is not None:
                latitude = coordinate.lat
                longitude = coordinate.lon
            airport = Airport(iata=fetch_property_values(item, 'P238'),
                              icao=fetch_property_values(item, 'P239'),
                              geo="%s, %s" % (latitude, longitude),
                              country=fetch_property_values(item, 'P17'),
                              en=name)
            output.write(repr(airport) + '\n')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
    Akvo RSR is covered by the GNU Affero General Public License.
    See more details in the license.txt file located at the root folder of the
    Akvo RSR module. For additional details on the GNU license please
    see < http://www.gnu.org/licenses/agpl.html >.
"""

from django.conf import settings
from django.contrib.sites.models import Site
from django.core.urlresolvers import LocaleRegexURLResolver, is_valid_path, get_resolver
from django.http import HttpResponseRedirect
from django.middleware.locale import LocaleMiddleware
from django.shortcuts import redirect
from django.utils import translation
from django.utils.cache import patch_vary_headers

from akvo.rsr.models import PartnerSite

def make_tls_property(default=None):
    "Creates a class-wide instance property with a thread-specific value."
    class TLSProperty(object):
        def __init__(self):
            from threading import local
            self.local = local()

        def __get__(self, instance, cls):
            if not instance:
                return self
            return self.value

        def __set__(self, instance, value):
            self.value = value

        def _get_value(self):
            return getattr(self.local, 'value', default)

        def _set_value(self, value):
            self.local.value = value
        value = property(_get_value, _set_value)

    return TLSProperty()

DEFAULT_SITE_ID = getattr(settings, 'SITE_ID', None)
settings.__class__.SITE_ID = make_tls_property(DEFAULT_SITE_ID)

DEFAULT_PARTNER_SITE = getattr(settings, 'PARTNER_SITE', None)
settings.__class__.PARTNER_SITE = make_tls_property(DEFAULT_PARTNER_SITE)

PARTNER_SITES_DEVELOPMENT_DOMAIN = getattr(settings,
                                           'PARTNER_SITES_DEVELOPMENT_DOMAIN',
                                           'akvoapp.dev')
PARTNER_SITES_DOMAINS = getattr(settings,
                                'PARTNER_SITES_DOMAINS',
                                (
                                    'akvoapp.org',
                                    'akvotest.org',
                                    'akvotest2.org',
                                    'akvotest3.org',
                                    PARTNER_SITES_DEVELOPMENT_DOMAIN
                                ))
PARTNER_SITES_MARKETING_SITE = getattr(settings,
                                       'PARTNER_SITES_MARKETING_SITE',
                                       'http://www.akvoapp.org/')

def normalize_domain(domain):
    domain_parts = domain.split(".")
    if len(domain_parts) > 3:
        domain = "%s.%s.%s" % tuple(domain_parts[-3:])
    return domain

def is_rsr_instance(domain):
    """Predicate to determine if an incoming request domain should be handled
    as a regular instance of Akvo RSR."""
    dev_domains = ('localhost', '127.0.0.1', 'akvo.dev', '77.53.15.119')
    return domain == 'akvo.org' or domain.endswith('.akvo.org') or domain in dev_domains

def is_partner_site_instance(domain):
    """Predicate to determine if an incoming request domain should be
    handled as a partner site instance."""
    domain_parts = domain.split('.')
    if len(domain_parts) >= 3:
        domain_name = "%s.%s" % tuple(domain_parts[-2:])
        if domain_name in PARTNER_SITES_DOMAINS:
            return True
    return False

def get_or_create_site(domain):
    """Helper function to get or create a `django.contrib.sites.models.Site` object.
    Also takes care of removing any duplicates."""
    if domain == "akvo.org":
        domain = "www.akvo.org"
    sites = Site.objects.filter(domain=domain)
    if sites.count() >= 1:
        site, duplicates = sites[0], sites[1:]
        if duplicates:
            for duplicate in duplicates:
                duplicate.delete()
    else:
        site = Site(domain=domain, name=domain)
        site.save()
    return site

class PartnerSitesRouterMiddleware(object):

    def process_request(self, request, partner_site=None):
        domain = normalize_domain(request.get_host().split(':')[0])
        if is_rsr_instance(domain):  # Vanilla Akvo RSR instance
            request.urlconf = 'akvo.urls.rsr'
        elif is_partner_site_instance(domain):  # Partner site instance
            hostname = domain.split('.')[-3]
            try:
                partner_site = PartnerSite.objects.get(hostname=hostname)
            except:
                pass
            if partner_site is None or not partner_site.enabled:
                return redirect(PARTNER_SITES_MARKETING_SITE)
        else:  # Partner site instance on partner-nominated domain
            try:
                partner_site = PartnerSite.objects.get(cname=domain)
            except:
                return redirect(PARTNER_SITES_MARKETING_SITE)
        if partner_site is not None and partner_site.enabled:
            request.urlconf = 'akvo.urls.partner_sites'
            request.partner_site = settings.PARTNER_SITE = partner_site
            request.organisation_id = partner_site.organisation.id
            request.default_language = partner_site.default_language
        site = get_or_create_site(domain)
        settings.SITE_ID = site.id
        return

class PartnerSitesLocaleMiddleware(LocaleMiddleware):
    """Partner sites aware version of Django's LocaleMiddleware. Since we
    swap out the root urlconf for a partner sites specific one, and the
    original Django LocaleMiddleware didn't like that."""

    def process_request(self, request):
        check_path = self.is_language_prefix_patterns_used(request)
        language = translation.get_language_from_request(
            request, check_path=check_path)
        translation.activate(language)
        request.LANGUAGE_CODE = translation.get_language()

    def process_response(self, request, response):
        # First set the default language, this will be used if there is none
        # in the path
        default_language = getattr(request, 'default_language', '')
        if default_language:
            translation.activate(default_language)

        language = translation.get_language()
        if (response.status_code == 404 and
                not translation.get_language_from_path(request.path_info)
                    and self.is_language_prefix_patterns_used(request)):
            urlconf = getattr(request, 'urlconf', None)
            language_path = '/%s%s' % (language, request.path_info)
            if settings.APPEND_SLASH and not language_path.endswith('/'):
                language_path = language_path + '/'

            if is_valid_path(language_path, urlconf):
                language_url = "%s://%s/%s%s" % (
                    request.is_secure() and 'https' or 'http',
                    request.get_host(), language, request.get_full_path())
                return HttpResponseRedirect(language_url)
        translation.deactivate()

        patch_vary_headers(response, ('Accept-Language',))
        if 'Content-Language' not in response:
            response['Content-Language'] = language
        return response

    def is_language_prefix_patterns_used(self, request):
        """
        Returns `True` if the `LocaleRegexURLResolver` is used
        at root level of the urlpatterns, else it returns `False`.
        """
        for url_pattern in get_resolver(request.urlconf).url_patterns:
            if isinstance(url_pattern, LocaleRegexURLResolver):
                return True
        return False
<commit_msg>Refacotored akvo.rsr.middleware.normalize_domain. It is now akvo.rsr.middleware.get_domain. Updated akvo.rsr.middleware.PartnerSitesRouterMiddleware to use it.<commit_after># -*- coding: utf-8 -*-

"""
    Akvo RSR is covered by the GNU Affero General Public License.
    See more details in the license.txt file located at the root folder of the
    Akvo RSR module. For additional details on the GNU license please
    see < http://www.gnu.org/licenses/agpl.html >.
"""

from django.conf import settings
from django.contrib.sites.models import Site
from django.core.urlresolvers import LocaleRegexURLResolver, is_valid_path, get_resolver
from django.http import HttpResponseRedirect
from django.middleware.locale import LocaleMiddleware
from django.shortcuts import redirect
from django.utils import translation
from django.utils.cache import patch_vary_headers

from akvo.rsr.models import PartnerSite

def make_tls_property(default=None):
    "Creates a class-wide instance property with a thread-specific value."
    class TLSProperty(object):
        def __init__(self):
            from threading import local
            self.local = local()

        def __get__(self, instance, cls):
            if not instance:
                return self
            return self.value

        def __set__(self, instance, value):
            self.value = value

        def _get_value(self):
            return getattr(self.local, 'value', default)

        def _set_value(self, value):
            self.local.value = value
        value = property(_get_value, _set_value)

    return TLSProperty()

DEFAULT_SITE_ID = getattr(settings, 'SITE_ID', None)
settings.__class__.SITE_ID = make_tls_property(DEFAULT_SITE_ID)

DEFAULT_PARTNER_SITE = getattr(settings, 'PARTNER_SITE', None)
settings.__class__.PARTNER_SITE = make_tls_property(DEFAULT_PARTNER_SITE)

PARTNER_SITES_DEVELOPMENT_DOMAIN = getattr(settings,
                                           'PARTNER_SITES_DEVELOPMENT_DOMAIN',
                                           'akvoapp.dev')
PARTNER_SITES_DOMAINS = getattr(settings,
                                'PARTNER_SITES_DOMAINS',
                                ('akvoapp.org',
                                 'akvotest.org',
                                 'akvotest2.org',
                                 'akvotest3.org',
                                 PARTNER_SITES_DEVELOPMENT_DOMAIN))
PARTNER_SITES_MARKETING_SITE = getattr(settings,
                                       'PARTNER_SITES_MARKETING_SITE',
                                       'http://www.akvoapp.org/')

def get_domain(request):
    domain = request.get_host().split(":")[0]
    domain_parts = domain.split(".")
    if len(domain_parts) > 3:
        domain = "%s.%s.%s" % tuple(domain_parts[-3:])
    return domain

def is_rsr_instance(domain):
    """Predicate to determine if an incoming request domain should be handled
    as a regular instance of Akvo RSR."""
    dev_domains = ('localhost', '127.0.0.1', 'akvo.dev', '77.53.15.119')
    return domain == 'akvo.org' or domain.endswith('.akvo.org') or domain in dev_domains

def is_partner_site_instance(domain):
    """Predicate to determine if an incoming request domain should be
    handled as a partner site instance."""
    domain_parts = domain.split(".")
    if len(domain_parts) >= 3:
        domain_name = "%s.%s" % tuple(domain_parts[-2:])
        if domain_name in PARTNER_SITES_DOMAINS:
            return True
    return False

def get_or_create_site(domain):
    """Helper function to get or create a `django.contrib.sites.models.Site` object.
    Also takes care of removing any duplicates."""
    if domain == "akvo.org":
        domain = "www.akvo.org"
    sites = Site.objects.filter(domain=domain)
    if sites.count() >= 1:
        site, duplicates = sites[0], sites[1:]
        if duplicates:
            for duplicate in duplicates:
                duplicate.delete()
    else:
        site = Site(domain=domain, name=domain)
        site.save()
    return site

class PartnerSitesRouterMiddleware(object):

    def process_request(self, request, partner_site=None):
        domain = get_domain(request)
        if is_rsr_instance(domain):  # Vanilla Akvo RSR instance
            request.urlconf = 'akvo.urls.rsr'
        elif is_partner_site_instance(domain):  # Partner site instance
            hostname = domain.split('.')[-3]
            try:
                partner_site = PartnerSite.objects.get(hostname=hostname)
            except:
                pass
            if partner_site is None or not partner_site.enabled:
                return redirect(PARTNER_SITES_MARKETING_SITE)
        else:  # Partner site instance on partner-nominated domain
            try:
                partner_site = PartnerSite.objects.get(cname=domain)
            except:
                return redirect(PARTNER_SITES_MARKETING_SITE)
        if partner_site is not None and partner_site.enabled:
            request.urlconf = 'akvo.urls.partner_sites'
            request.partner_site = settings.PARTNER_SITE = partner_site
            request.organisation_id = partner_site.organisation.id
            request.default_language = partner_site.default_language
        site = get_or_create_site(domain)
        settings.SITE_ID = site.id
        return

class PartnerSitesLocaleMiddleware(LocaleMiddleware):
    """Partner sites aware version of Django's LocaleMiddleware. Since we
    swap out the root urlconf for a partner sites specific one, and the
    original Django LocaleMiddleware didn't like that."""

    def process_request(self, request):
        check_path = self.is_language_prefix_patterns_used(request)
        language = translation.get_language_from_request(
            request, check_path=check_path)
        translation.activate(language)
        request.LANGUAGE_CODE = translation.get_language()

    def process_response(self, request, response):
        # First set the default language, this will be used if there is none
        # in the path
        default_language = getattr(request, 'default_language', '')
        if default_language:
            translation.activate(default_language)

        language = translation.get_language()
        if (response.status_code == 404 and
                not translation.get_language_from_path(request.path_info)
                    and self.is_language_prefix_patterns_used(request)):
            urlconf = getattr(request, 'urlconf', None)
            language_path = '/%s%s' % (language, request.path_info)
            if settings.APPEND_SLASH and not language_path.endswith('/'):
                language_path = language_path + '/'

            if is_valid_path(language_path, urlconf):
                language_url = "%s://%s/%s%s" % (
                    request.is_secure() and 'https' or 'http',
                    request.get_host(), language, request.get_full_path())
                return HttpResponseRedirect(language_url)
        translation.deactivate()

        patch_vary_headers(response, ('Accept-Language',))
        if 'Content-Language' not in response:
            response['Content-Language'] = language
        return response

    def is_language_prefix_patterns_used(self, request):
        """
        Returns `True` if the `LocaleRegexURLResolver` is used
        at root level of the urlpatterns, else it returns `False`.
        """
        for url_pattern in get_resolver(request.urlconf).url_patterns:
            if isinstance(url_pattern, LocaleRegexURLResolver):
                return True
        return False
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Akvo RSR is covered by the GNU Affero General Public License.

See more details in the license.txt file located at the root folder of the
Akvo RSR module. For additional details on the GNU license please see
< http://www.gnu.org/licenses/agpl.html >.
"""

import logging

from django.conf import settings
from django.core.exceptions import DisallowedHost
from django.db.models import Q
from django.shortcuts import redirect

from akvo.rsr.context_processors import extra_context
from akvo.rsr.models import PartnerSite

def _is_rsr_host(hostname):
    """Predicate function that checks if request is made to the RSR_DOMAIN."""
    rsr_hosts = ['127.0.0.1', 'localhost', settings.RSR_DOMAIN]
    return hostname in rsr_hosts

def _is_naked_app_host(hostname):
    """Predicate function that checks if request is made to the RSR_DOMAIN."""
    if hostname == settings.AKVOAPP_DOMAIN:
        return True
    return False

def _partner_site(netloc):
    """From a netloc return PartnerSite or raise a DoesNotExist."""
    return PartnerSite.objects.get(
        Q(hostname=PartnerSite.yank_hostname(netloc)) | Q(cname=netloc)
    )

def _build_api_link(request, resource, object_id):
    """
    Build a new link that will redirect from the '/v1/api/project/?depth=X' resource to
    '/v1/api/project_extra(_deep)/' resource.
    """
    protocol = 'https' if request.is_secure() else 'http'
    object_id_part = '/' if not object_id else '/{0}/'.format(object_id)
    return '{0}://{1}/api/v1/{2}{3}?{4}'.format(
        protocol, request.META['HTTP_HOST'], resource, object_id_part, request.GET.urlencode()
    )

class HostDispatchMiddleware(object):
    """RSR page dispatch middleware."""

    def process_request(self, request):
        """Route on request."""
        request.rsr_page = None
        DEFAULT_REDIRECT_URL = "{}://{}".format(request.scheme, settings.RSR_DOMAIN)

        try:
            # Make sure host is valid - otherwise redirect to RSR_DOMAIN.
            # Do nothing if called on "normal" RSR host.
            host = request.get_host()
            if _is_rsr_host(host):
                return None
        except DisallowedHost:
            request.META['HTTP_HOST'] = settings.RSR_DOMAIN
            return redirect(DEFAULT_REDIRECT_URL)

        # Check if called on naked app domain - if so redirect
        if _is_naked_app_host(host):
            return redirect(DEFAULT_REDIRECT_URL)

        # Check if site exists
        try:
            site = _partner_site(host)
        except PartnerSite.DoesNotExist:
            return redirect(DEFAULT_REDIRECT_URL)

        # Check if site is enabled
        if not site.enabled:
            return redirect(DEFAULT_REDIRECT_URL)

        # Check if the request if for a partner's CNAME
        if site.redirect_cname and site.is_cname_request(host):
            return redirect("{}://{}.{}".format(request.scheme, site.hostname,
                                                settings.AKVOAPP_DOMAIN))
        # Set site to request object
        request.rsr_page = site
        return None

class ExceptionLoggingMiddleware(object):

    """Used to log exceptions on production systems."""

    def process_exception(self, request, exception):
        """."""
        logging.exception('Exception handling request for ' + request.path)

class RSRVersionHeaderMiddleware(object):

    """Add a response header with RSR version info."""

    def process_response(self, request, response):
        """Add the X-RSR-Version header."""
        context = extra_context(request)

        if response is not None:
            response['X-RSR-Version'] = "tag={}, commit={}, branch={}".format(
                context['deploy_tag'],
                context['deploy_commit_id'],
                context['deploy_branch'])
        return response

class APIRedirectMiddleware(object):
    """
    In special cases, the old API links should be redirected:

    - /api/v1/project/ with depth = 1 should be redirected to /api/v1/project_extra/.
    - /api/v1/project/ with depth > 1 should be redirected to /api/v1/project_extra_deep/.
    """
    @staticmethod
    def process_response(request, response):
        project_extra_fields = ['api', 'v1', 'project', ]
        path_list = request.path.split('/')

        if all(field in path_list for field in project_extra_fields):
            try:
                object_id = path_list[4] if len(path_list) > 4 and int(path_list[4]) else None
            except ValueError:
                object_id = None

            depth = request.GET.get('depth')
            if depth == '1':
                return redirect(_build_api_link(request, 'project_extra', object_id))
            if depth > '1':
                return redirect(_build_api_link(request, 'project_extra_deep', object_id))
        return response
<commit_msg>[#3337] Temporarily returning 200 to k8s health check<commit_after># -*- coding: utf-8 -*-
"""
Akvo RSR is covered by the GNU Affero General Public License.

See more details in the license.txt file located at the root folder of the
Akvo RSR module. For additional details on the GNU license please see
< http://www.gnu.org/licenses/agpl.html >.
"""

import logging

from django.conf import settings
from django.core.exceptions import DisallowedHost
from django.db.models import Q
from django.http import HttpResponse
from django.shortcuts import redirect

from akvo.rsr.context_processors import extra_context
from akvo.rsr.models import PartnerSite

def _is_rsr_host(hostname):
    """Predicate function that checks if request is made to the RSR_DOMAIN."""
    rsr_hosts = ['127.0.0.1', 'localhost', settings.RSR_DOMAIN]
    return hostname in rsr_hosts

def _is_naked_app_host(hostname):
    """Predicate function that checks if request is made to the RSR_DOMAIN."""
    if hostname == settings.AKVOAPP_DOMAIN:
        return True
    return False

def _partner_site(netloc):
    """From a netloc return PartnerSite or raise a DoesNotExist."""
    return PartnerSite.objects.get(
        Q(hostname=PartnerSite.yank_hostname(netloc)) | Q(cname=netloc)
    )

def _build_api_link(request, resource, object_id):
    """
    Build a new link that will redirect from the '/v1/api/project/?depth=X' resource to
    '/v1/api/project_extra(_deep)/' resource.
    """
    protocol = 'https' if request.is_secure() else 'http'
    object_id_part = '/' if not object_id else '/{0}/'.format(object_id)
    return '{0}://{1}/api/v1/{2}{3}?{4}'.format(
        protocol, request.META['HTTP_HOST'], resource, object_id_part, request.GET.urlencode()
    )

class HostDispatchMiddleware(object):
    """RSR page dispatch middleware."""

    def process_request(self, request):
        """Route on request."""
        request.rsr_page = None
        DEFAULT_REDIRECT_URL = "{}://{}".format(request.scheme, settings.RSR_DOMAIN)

        if request.META['HTTP_USER_AGENT'] == 'GoogleHC/1.0':
            return HttpResponse("OK")

        try:
            # Make sure host is valid - otherwise redirect to RSR_DOMAIN.
            # Do nothing if called on "normal" RSR host.
            host = request.get_host()
            if _is_rsr_host(host):
                return None
        except DisallowedHost:
            request.META['HTTP_HOST'] = settings.RSR_DOMAIN
            return redirect(DEFAULT_REDIRECT_URL)

        # Check if called on naked app domain - if so redirect
        if _is_naked_app_host(host):
            return redirect(DEFAULT_REDIRECT_URL)

        # Check if site exists
        try:
            site = _partner_site(host)
        except PartnerSite.DoesNotExist:
            return redirect(DEFAULT_REDIRECT_URL)

        # Check if site is enabled
        if not site.enabled:
            return redirect(DEFAULT_REDIRECT_URL)

        # Check if the request if for a partner's CNAME
        if site.redirect_cname and site.is_cname_request(host):
            return redirect("{}://{}.{}".format(request.scheme, site.hostname,
                                                settings.AKVOAPP_DOMAIN))
        # Set site to request object
        request.rsr_page = site
        return None

class ExceptionLoggingMiddleware(object):

    """Used to log exceptions on production systems."""

    def process_exception(self, request, exception):
        """."""
        logging.exception('Exception handling request for ' + request.path)

class RSRVersionHeaderMiddleware(object):

    """Add a response header with RSR version info."""

    def process_response(self, request, response):
        """Add the X-RSR-Version header."""
        context = extra_context(request)

        if response is not None:
            response['X-RSR-Version'] = "tag={}, commit={}, branch={}".format(
                context['deploy_tag'],
                context['deploy_commit_id'],
                context['deploy_branch'])
        return response

class APIRedirectMiddleware(object):
    """
    In special cases, the old API links should be redirected:

    - /api/v1/project/ with depth = 1 should be redirected to /api/v1/project_extra/.
    - /api/v1/project/ with depth > 1 should be redirected to /api/v1/project_extra_deep/.
    """
    @staticmethod
    def process_response(request, response):
        project_extra_fields = ['api', 'v1', 'project', ]
        path_list = request.path.split('/')

        if all(field in path_list for field in project_extra_fields):
            try:
                object_id = path_list[4] if len(path_list) > 4 and int(path_list[4]) else None
            except ValueError:
                object_id = None

            depth = request.GET.get('depth')
            if depth == '1':
                return redirect(_build_api_link(request, 'project_extra', object_id))
            if depth > '1':
                return redirect(_build_api_link(request, 'project_extra_deep', object_id))
        return response
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
    Akvo RSR is covered by the GNU Affero General Public License.
    See more details in the license.txt file located at the root folder of the
    Akvo RSR module. For additional details on the GNU license please
    see < http://www.gnu.org/licenses/agpl.html >.
"""

from django.conf import settings
from django.contrib.sites.models import Site

from akvo.rsr.utils import make_tls_property()

SITE_ID = settings.__dict__['_wrapped'].__class__.SITE_ID = make_tls_property()

class PartnerSitesRouterMiddleware(object):
    def process_request(self, request, partner_site=None):
        domain = request.get_host().split(':')[0]
        parts = domain.split('.')
        num_parts = len(parts)
        if domain.endswith('.dev'):  # local development domain
            hostname = parts[-2]
            if not hostname == 'akvo':
                try:
                    partner_site = Site.objects.get(development_domain=domain)
                except:
                    pass
        elif domain.endswith('.akvo.org'):  # akvo development/production domain
            if num_parts >= 3:
                hostname = parts[-3]
                if not hostname == 'www':
                    if ((num_parts == 3 and not hostname in settings.RESERVED_HOSTNAMES) or
                        (num_parts >=4 and hostname in settings.RESERVED_HOSTNAMES)):
                        try:
                            partner_site = Site.objects.get(domain=domain)
                        except:
                            pass
        else:  # probably a partner-nominated domain
            try:
                partner_site = Site.objects.get(partner_domain=domain)
            except:
                pass
        if partner_site is not None and partner_site.enabled:
            SITE_ID.value = partner_site.id
            request.urlconf = 'akvo.urls_partner_sites'
        return
<commit_msg>Added organisation_id to request in PartnerSitesRouterMiddleware.<commit_after># -*- coding: utf-8 -*-

"""
    Akvo RSR is covered by the GNU Affero General Public License.
    See more details in the license.txt file located at the root folder of the
    Akvo RSR module. For additional details on the GNU license please
    see < http://www.gnu.org/licenses/agpl.html >.
"""

from django.conf import settings
from django.contrib.sites.models import Site

from akvo.rsr.utils import make_tls_property()

SITE_ID = settings.__dict__['_wrapped'].__class__.SITE_ID = make_tls_property()

class PartnerSitesRouterMiddleware(object):
    def process_request(self, request, partner_site=None):
        domain = request.get_host().split(':')[0]
        parts = domain.split('.')
        num_parts = len(parts)
        if domain.endswith('.dev'):  # local development domain
            hostname = parts[-2]
            if not hostname == 'akvo':
                try:
                    partner_site = Site.objects.get(development_domain=domain)
                except:
                    pass
        elif domain.endswith('.akvo.org'):  # akvo development/production domain
            if num_parts >= 3:
                hostname = parts[-3]
                if not hostname == 'www':
                    if ((num_parts == 3 and not hostname in settings.RESERVED_HOSTNAMES) or
                        (num_parts >=4 and hostname in settings.RESERVED_HOSTNAMES)):
                        try:
                            partner_site = Site.objects.get(domain=domain)
                        except:
                            pass
        else:  # probably a partner-nominated domain
            try:
                partner_site = Site.objects.get(partner_domain=domain)
            except:
                pass
        if partner_site is not None and partner_site.enabled:
            SITE_ID.value = partner_site.id
            request.organisation_id = partner_site.organisation.id
            request.urlconf = 'akvo.urls_partner_sites'
        return
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import six

from lxml.html.clean import Cleaner as LxmlCleaner

from django.core.exceptions import ImproperlyConfigured
from django.db import models
try:
    from django.utils.encoding import force_unicode
except ImportError:
    from django.utils.encoding import force_text as force_unicode
from django.utils.importlib import import_module
from django.utils.html import strip_tags as _strip_tags

from haystack import DEFAULT_ALIAS
from haystack.indexes import SearchIndex

from cms.utils.i18n import get_language_code

from .conf import settings

def alias_from_language(language):
    """
    Returns alias if alias is a valid language.
    """
    language = get_language_code(language)

    if language == settings.ALDRYN_SEARCH_DEFAULT_LANGUAGE:
        return DEFAULT_ALIAS
    return language

def clean_join(separator, iterable):
    """
    Filters out iterable to only join non empty items.
    """
    return separator.join(filter(None, iterable))

def get_callable(string_or_callable):
    """
    If given a callable then it returns it, otherwise it resolves the path
    and returns an object.
    """
    if callable(string_or_callable):
        return string_or_callable
    else:
        module_name, object_name = string_or_callable.rsplit('.', 1)
        module = import_module(module_name)
        return getattr(module, object_name)

def _get_language_from_alias_func():
    path_or_callable = settings.ALDRYN_SEARCH_LANGUAGE_FROM_ALIAS

    if path_or_callable:
        try:
            func = get_callable(path_or_callable)
        except AttributeError as error:
            raise ImproperlyConfigured('ALDRYN_SEARCH_LANGUAGE_FROM_ALIAS: %s' % (str(error)))
        if not callable(func):
            raise ImproperlyConfigured('ALDRYN_SEARCH_LANGUAGE_FROM_ALIAS: %s is not callable' % func)
    else:
        func = None
    return func

def get_index_base():
    index_string = settings.ALDRYN_SEARCH_INDEX_BASE_CLASS
    try:
        BaseClass = get_callable(index_string)
    except AttributeError as error:
        raise ImproperlyConfigured('ALDRYN_SEARCH_INDEX_BASE_CLASS: %s' % (str(error)))

    if not issubclass(BaseClass, SearchIndex):
        raise ImproperlyConfigured('ALDRYN_SEARCH_INDEX_BASE_CLASS: %s is not a subclass of haystack.indexes.SearchIndex' % index_string)

    required_fields = ['text', 'language']

    if not all(field in BaseClass.fields for field in required_fields):
        raise ImproperlyConfigured('ALDRYN_SEARCH_INDEX_BASE_CLASS: %s must contain at least these fields: %s' % (index_string, required_fields))
    return BaseClass

def language_from_alias(alias):
    """
    Returns alias if alias is a valid language.
    """
    languages = [language[0] for language in settings.LANGUAGES]

    return alias if alias in languages else None

def get_field_value(obj, name):
    """
    Given a model instance and a field name (or attribute),
    returns the value of the field or an empty string.
    """
    fields = name.split('__')

    name = fields[0]

    try:
        obj._meta.get_field(name)
    except (AttributeError, models.FieldDoesNotExist):
        # we catch attribute error because obj will not always be a model
        # specially when going through multiple relationships.
        value = getattr(obj, name, None) or ''
    else:
        value = getattr(obj, name)

    if len(fields) > 1:
        remaining = '__'.join(fields[1:])
        return get_field_value(value, remaining)
    return value

def get_model_path(model_or_string):
    if not isinstance(model_or_string, six.string_types):
        # it's a model class
        app_label = model_or_string._meta.app_label
        model_name = model_or_string._meta.object_name
        model_or_string = '{0}.{1}'.format(app_label, model_name)
    return model_or_string.lower()

def strip_tags(value):
    """
    Returns the given HTML with all tags stripped.
    We use lxml to strip all js tags and then hand the result to django's
    strip tags. If value isn't valid, just return value since there is
    no tags to strip.
    """
    # borrowed from aldryn-search

    if isinstance(value, six.string_types):
        value = value.strip()

        if not value:
            return

        try:
            partial_strip = LxmlCleaner().clean_html(value)
        except ParserError:
            # error could occur because of invalid html document
            # we don't want to return empty handed.
            partial_strip = value
        value = _strip_tags(partial_strip)
        return value.strip()  # clean cases we have
\n\n

    return value
<commit_msg>Fix strip_tags<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import six

from lxml.html.clean import Cleaner as LxmlCleaner
from lxml.etree import ParseError

from django.core.exceptions import ImproperlyConfigured
from django.db import models
try:
    from django.utils.encoding import force_unicode
except ImportError:
    from django.utils.encoding import force_text as force_unicode
from django.utils.importlib import import_module
from django.utils.html import strip_tags as _strip_tags

from haystack import DEFAULT_ALIAS
from haystack.indexes import SearchIndex

from cms.utils.i18n import get_language_code

from .conf import settings

def alias_from_language(language):
    """
    Returns alias if alias is a valid language.
    """
    language = get_language_code(language)

    if language == settings.ALDRYN_SEARCH_DEFAULT_LANGUAGE:
        return DEFAULT_ALIAS
    return language

def clean_join(separator, iterable):
    """
    Filters out iterable to only join non empty items.
    """
    return separator.join(filter(None, iterable))

def get_callable(string_or_callable):
    """
    If given a callable then it returns it, otherwise it resolves the path
    and returns an object.
    """
    if callable(string_or_callable):
        return string_or_callable
    else:
        module_name, object_name = string_or_callable.rsplit('.', 1)
        module = import_module(module_name)
        return getattr(module, object_name)

def _get_language_from_alias_func():
    path_or_callable = settings.ALDRYN_SEARCH_LANGUAGE_FROM_ALIAS

    if path_or_callable:
        try:
            func = get_callable(path_or_callable)
        except AttributeError as error:
            raise ImproperlyConfigured('ALDRYN_SEARCH_LANGUAGE_FROM_ALIAS: %s' % (str(error)))
        if not callable(func):
            raise ImproperlyConfigured('ALDRYN_SEARCH_LANGUAGE_FROM_ALIAS: %s is not callable' % func)
    else:
        func = None
    return func

def get_index_base():
    index_string = settings.ALDRYN_SEARCH_INDEX_BASE_CLASS
    try:
        BaseClass = get_callable(index_string)
    except AttributeError as error:
        raise ImproperlyConfigured('ALDRYN_SEARCH_INDEX_BASE_CLASS: %s' % (str(error)))

    if not issubclass(BaseClass, SearchIndex):
        raise ImproperlyConfigured('ALDRYN_SEARCH_INDEX_BASE_CLASS: %s is not a subclass of haystack.indexes.SearchIndex' % index_string)

    required_fields = ['text', 'language']

    if not all(field in BaseClass.fields for field in required_fields):
        raise ImproperlyConfigured('ALDRYN_SEARCH_INDEX_BASE_CLASS: %s must contain at least these fields: %s' % (index_string, required_fields))
    return BaseClass

def language_from_alias(alias):
    """
    Returns alias if alias is a valid language.
    """
    languages = [language[0] for language in settings.LANGUAGES]

    return alias if alias in languages else None

def get_field_value(obj, name):
    """
    Given a model instance and a field name (or attribute),
    returns the value of the field or an empty string.
    """
    fields = name.split('__')

    name = fields[0]

    try:
        obj._meta.get_field(name)
    except (AttributeError, models.FieldDoesNotExist):
        # we catch attribute error because obj will not always be a model
        # specially when going through multiple relationships.
        value = getattr(obj, name, None) or ''
    else:
        value = getattr(obj, name)

    if len(fields) > 1:
        remaining = '__'.join(fields[1:])
        return get_field_value(value, remaining)
    return value

def get_model_path(model_or_string):
    if not isinstance(model_or_string, six.string_types):
        # it's a model class
        app_label = model_or_string._meta.app_label
        model_name = model_or_string._meta.object_name
        model_or_string = '{0}.{1}'.format(app_label, model_name)
    return model_or_string.lower()

def strip_tags(value):
    """
    Returns the given HTML with all tags stripped.
    We use lxml to strip all js tags and then hand the result to django's
    strip tags. If value isn't valid, just return value since there is
    no tags to strip.
    """
    if isinstance(value, six.string_types):
        value = value.strip()

        try:
            partial_strip = LxmlCleaner().clean_html(value)
        except ParseError:
            # Error could occur because of invalid html document,
            # including '' values. We don't want to return empty handed.
            partial_strip = value
        value = _strip_tags(partial_strip)
        return value.strip()  # clean cases we have
\n\n

    return value
<|endoftext|>


<commit_before># Copyright (C) 2015 Bitquant Research Laboratories (Asia) Limited
# Released under the Simplified BSD License

import logging
import zmq.green as zmq
import msgpack
import time
import traceback

def logger(s : str):
    logger = logging.getLogger(s)
    logger.setLevel(logging.INFO)
    ch = logging.StreamHandler()
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    formatter.converter = time.gmtime
    ch.setFormatter(formatter)
    logger.addHandler(ch)
    return logger

loglevels = {
    "CRITICAL" : logging.CRITICAL,
    "ERROR" : logging.ERROR,
    "WARNING" : logging.WARNING,
    "INFO" : logging.INFO,
    "DEBUG" : logging.DEBUG,
    "NOTSET" : logging.NOTSET
    }

ports = {
    "data" : {
    "dispatcher" : "tcp://127.0.0.1:5557",
    "broker_plivo" : "tcp://127.0.0.1:5558",
    "strategy_alert" : "tcp://127.0.0.1:5559",
    "ticker_yahoo" : "tcp://127.0.0.1:5560",
    "ticker_bitfutures" : "tcp://127.0.0.1:5561",
    "broker_bitmex" : "tcp://127.0.0.1:5562",
    "ticker_bravenewcoin" : "tcp://127.0.0.1:5563",
    "strategy_xbt_close" : "tcp://127.0.0.1:5564",
    "broker_desk_alert" : "tcp://127.0.0.1:5565",
    "broker_twilio" : "tcp://127.0.0.1:5566",
    "broker_web" : "tcp://127.0.0.1:5567",
    },
    "control" : {
    "dispatcher" : "tcp://127.0.0.1:5577",
    "broker_plivo" : "tcp://127.0.0.1:5578",
    "strategy_alert" : "tcp://127.0.0.1:5579",
    "ticker_yahoo" : "tcp://127.0.0.1:5580",
    "ticker_bitfutures" : "tcp://127.0.0.1:5581",
    "broker_bitmex" : "tcp://127.0.0.1:5582",
    "ticker_bravenewcoin" : "tcp://127.0.0.1:5583",
    "strategy_xbt_close" : "tcp://127.0.0.1:5584",
    "broker_desk_alert" : "tcp://127.0.0.1:5585",
    "broker_twilio" : "tcp://127.0.0.1:5586",
    "broker_web" : "tcp://127.0.0.1:5587",
    }
    }

def send(name, data):
    context = zmq.Context()
    for i in data:
        socket = context.socket(zmq.PUSH)
        socket.connect(ports[name][i['dest']])
        socket.send(msgpack.packb(i))

class AlgoObject(object):
    def __init__(self, name : str, socket_type):
        self._logger = logger(name)
        self._context = zmq.Context()
        self._poller = zmq.Poller()
        self._data_socket = self.socket(socket_type)

        self._control_socket = self.socket(zmq.PULL)
        self._control_socket.bind(ports['control'][name])
        self._poller.register(self._data_socket,
                              zmq.POLLIN)
        self._poller.register(self._control_socket,
                              zmq.POLLIN)
        self.info("starting %s" % name)
        self.timeout = None
    def socket(self, socket_type):
        return self._context.socket(socket_type)
    def recv_data(self):
        return msgpack.unpackb(self._data_socket.recv(),
                               encoding='utf-8')
    def recv_control(self):
        return msgpack.unpackb(self._control_socket.recv(),
                               encoding='utf-8')
    def debug(self, s):
        self._logger.debug(s)
    def info(self, s):
        self._logger.info(s)
    def error(self, s):
        self._logger.error(s)
    def warning(self, s):
        self._logger.warning(s)
    def set_logger_level(self, level):
        self._logger.setLevel(loglevels[level])
    def process_data(self, data):
        raise NotImplementedError
    def process_control(self, data):
        self.debug("received control message")
        if data.get('cmd', None) == 'loglevel':
            if data.get('level', None) in loglevels:
                self.set_logger_level(loglevels[data['level']])
                self.info(("Setting loglevel to %s", data['level']))
                return True
    def run_once(self):
        pass
    def run(self):
        while True:
            try:
                socks = dict(self._poller.poll(self.timeout))
            except KeyboardInterrupt:
                break
            try:
                if self._control_socket in socks:
                    control = self.recv_control()
                    self.process_control(control)
                if self._data_socket in socks:
                    data = self.recv_data()
                    self.process_data(data)
                self.run_once()
            except:
                self.error("error processing control message")
                self.error(traceback.format_exc())

class Strategy(AlgoObject):
    def __init__(self, name, tickers):
        AlgoObject.__init__(self, name, zmq.SUB)
        for i in tickers:
            self._data_socket.connect(ports['data'][i])
        self._data_socket.setsockopt(zmq.SUBSCRIBE, b'')
        self._action_socket = self.socket(zmq.PUSH)
        self._action_socket.connect(ports['data']['dispatcher'])
    def send_action(self, message):
        self._action_socket.send(msgpack.packb(message))
    def send_control(self, to, message):
        socket = self._context.socket(zmq.PUSH)
        socket.connect(ports['control'][to])
        socket.send(msgpack.packb(message))

class Broker(AlgoObject):
    def __init__(self, name):
        AlgoObject.__init__(self, name, zmq.PULL)
        self._data_socket.bind(ports['data'][name])
    def process_control(self, data):
        return AlgoObject.process_control(self, data)

class Ticker(AlgoObject):
    def __init__(self, name):
        AlgoObject.__init__(self, name, zmq.PUB)
        self._data_socket.bind(ports['data'][name])
        self.timeout = 30000
        self.quotes = {}
    def run_once(self):
        self.debug("running loop function")
        self.get_quotes()
        self.send_quotes()
    def process_control(self, data):
        return AlgoObject.process_control(self, data)
    def send_quotes(self):
        self.debug("Sending quotes")
        self.send_data(self.quotes)
    def send_data(self, message):
        self._data_socket.send(msgpack.packb(message))
    def test(self):
        self.get_quotes()
        socket = self._context.socket(zmq.PUSH)
        socket.bind(ports['data']['dispatcher'])
        message = { 'cmd' : 'log',
                    'item' : self.quotes }
        self._logger.debug("Sending data")
        socket.send(msgpack.packb(message))
<commit_msg>fix typo<commit_after># Copyright (C) 2015 Bitquant Research Laboratories (Asia) Limited
# Released under the Simplified BSD License

import logging
import zmq.green as zmq
import msgpack
import time
import traceback

def logger(s : str):
    logger = logging.getLogger(s)
    logger.setLevel(logging.INFO)
    ch = logging.StreamHandler()
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    formatter.converter = time.gmtime
    ch.setFormatter(formatter)
    logger.addHandler(ch)
    return logger

loglevels = {
    "CRITICAL" : logging.CRITICAL,
    "ERROR" : logging.ERROR,
    "WARNING" : logging.WARNING,
    "INFO" : logging.INFO,
    "DEBUG" : logging.DEBUG,
    "NOTSET" : logging.NOTSET
    }

ports = {
    "data" : {
    "dispatcher" : "tcp://127.0.0.1:5557",
    "broker_plivo" : "tcp://127.0.0.1:5558",
    "strategy_alert" : "tcp://127.0.0.1:5559",
    "ticker_yahoo" : "tcp://127.0.0.1:5560",
    "ticker_bitfutures" : "tcp://127.0.0.1:5561",
    "broker_bitmex" : "tcp://127.0.0.1:5562",
    "ticker_bravenewcoin" : "tcp://127.0.0.1:5563",
    "strategy_xbt_close" : "tcp://127.0.0.1:5564",
    "broker_desk_alert" : "tcp://127.0.0.1:5565",
    "broker_twilio" : "tcp://127.0.0.1:5566",
    "broker_web" : "tcp://127.0.0.1:5567",
    },
    "control" : {
    "dispatcher" : "tcp://127.0.0.1:5577",
    "broker_plivo" : "tcp://127.0.0.1:5578",
    "strategy_alert" : "tcp://127.0.0.1:5579",
    "ticker_yahoo" : "tcp://127.0.0.1:5580",
    "ticker_bitfutures" : "tcp://127.0.0.1:5581",
    "broker_bitmex" : "tcp://127.0.0.1:5582",
    "ticker_bravenewcoin" : "tcp://127.0.0.1:5583",
    "strategy_xbt_close" : "tcp://127.0.0.1:5584",
    "broker_desk_alert" : "tcp://127.0.0.1:5585",
    "broker_twilio" : "tcp://127.0.0.1:5586",
    "broker_web" : "tcp://127.0.0.1:5587",
    }
    }

def send(name, data):
    context = zmq.Context()
    for i in data:
        socket = context.socket(zmq.PUSH)
        socket.connect(ports[name][i['dest']])
        socket.send(msgpack.packb(i))

class AlgoObject(object):
    def __init__(self, name : str, socket_type):
        self._logger = logger(name)
        self._context = zmq.Context()
        self._poller = zmq.Poller()
        self._data_socket = self.socket(socket_type)

        self._control_socket = self.socket(zmq.PULL)
        self._control_socket.bind(ports['control'][name])
        self._poller.register(self._data_socket,
                              zmq.POLLIN)
        self._poller.register(self._control_socket,
                              zmq.POLLIN)
        self.info("starting %s" % name)
        self.timeout = None
    def socket(self, socket_type):
        return self._context.socket(socket_type)
    def recv_data(self):
        return msgpack.unpackb(self._data_socket.recv(),
                               encoding='utf-8')
    def recv_control(self):
        return msgpack.unpackb(self._control_socket.recv(),
                               encoding='utf-8')
    def debug(self, s):
        self._logger.debug(s)
    def info(self, s):
        self._logger.info(s)
    def error(self, s):
        self._logger.error(s)
    def warning(self, s):
        self._logger.warning(s)
    def set_logger_level(self, level):
        self._logger.setLevel(level)
    def process_data(self, data):
        raise NotImplementedError
    def process_control(self, data):
        self.debug("received control message")
        if data.get('cmd', None) == 'loglevel':
            if data.get('level', None) in loglevels:
                self.set_logger_level(loglevels[data['level']])
                self.info(("Setting loglevel to %s", data['level']))
                return True
    def run_once(self):
        pass
    def run(self):
        while True:
            try:
                socks = dict(self._poller.poll(self.timeout))
            except KeyboardInterrupt:
                break
            try:
                if self._control_socket in socks:
                    control = self.recv_control()
                    self.process_control(control)
                if self._data_socket in socks:
                    data = self.recv_data()
                    self.process_data(data)
                self.run_once()
            except:
                self.error("error processing control message")
                self.error(traceback.format_exc())

class Strategy(AlgoObject):
    def __init__(self, name, tickers):
        AlgoObject.__init__(self, name, zmq.SUB)
        for i in tickers:
            self._data_socket.connect(ports['data'][i])
        self._data_socket.setsockopt(zmq.SUBSCRIBE, b'')
        self._action_socket = self.socket(zmq.PUSH)
        self._action_socket.connect(ports['data']['dispatcher'])
    def send_action(self, message):
        self._action_socket.send(msgpack.packb(message))
    def send_control(self, to, message):
        socket = self._context.socket(zmq.PUSH)
        socket.connect(ports['control'][to])
        socket.send(msgpack.packb(message))

class Broker(AlgoObject):
    def __init__(self, name):
        AlgoObject.__init__(self, name, zmq.PULL)
        self._data_socket.bind(ports['data'][name])
    def process_control(self, data):
        return AlgoObject.process_control(self, data)

class Ticker(AlgoObject):
    def __init__(self, name):
        AlgoObject.__init__(self, name, zmq.PUB)
        self._data_socket.bind(ports['data'][name])
        self.timeout = 30000
        self.quotes = {}
    def run_once(self):
        self.debug("running loop function")
        self.get_quotes()
        self.send_quotes()
    def process_control(self, data):
        return AlgoObject.process_control(self, data)
    def send_quotes(self):
        self.debug("Sending quotes")
        self.send_data(self.quotes)
    def send_data(self, message):
        self._data_socket.send(msgpack.packb(message))
    def test(self):
        self.get_quotes()
        socket = self._context.socket(zmq.PUSH)
        socket.bind(ports['data']['dispatcher'])
        message = { 'cmd' : 'log',
                    'item' : self.quotes }
        self._logger.debug("Sending data")
        socket.send(msgpack.packb(message))
<|endoftext|>


<commit_before># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode:nil -*-
# vi: set ft=python sts=4 sw=4 et:

import numpy as np

def make_pm(mask, meth = 'all'):
    """
    Make probabilistic map
    ------------------------------
    Parameters:
        mask: mask
        meth: 'all' or 'part'.
              all, all subjects are taken into account
              part, part subjects are taken into account
    Return:
        pm = probabilistic map
    """
    if mask.ndim != 4:
        raise Exception('Masks should be a 4D nifti file contains subjects')
    labels = np.unique(mask)[1:]
    pm = np.empty((mask.shape[0], mask.shape[1], mask.shape[2], labels.shape[0]))
    if meth == 'all':
        for i in range(labels.shape[0]):
            pm[..., i] = np.mean(mask == labels[i], axis = 3)
    elif meth == 'part':
        for i in range(labels.shape[0]):
            mask_i = mask == labels[i]
            subj = np.any(mask_i, axis = (0,1,2))
            pm[..., i] = np.mean(mask_i[..., subj], axis = 3)
    else:
        raise Exception('method not supported')
    return pm

def make_mpm(pm, threshold):
    """
    Make maximum probabilistic map (mpm)
    ---------------------------------------
    Parameters:
        threshold: threholds to mask probabilistic maps
    Return:
        mpm: maximum probabilisic map
    """
    pm_temp = np.empty((pm.shape[0], pm.shape[1], pm.shape[2], pm.shape[3]+1))
    pm_temp[..., range(1, pm.shape[3]+1)] = pm
    pm_temp[pm_temp < threshold] = 0
    mpm = np.argmax(pm_temp, axis=3)
    return mpm

def sphere_roi(data, voxloc, radius, value):
    """
    Generate a sphere roi which centered in (x,y,z)
    Parameters:
        data: your output data
        voxloc: (x,y,z), center vox of spheres
        radius: radius (unit: vox), note that it's a list
        value: label value
    """
    for n_x in range(int(voxloc[0]-radius[0]), int(voxloc[0]+radius[0]+1)):
        for n_y in range(int(voxloc[1]-radius[1]), int(voxloc[1]+radius[1]+1)):
            for n_z in range(int(voxloc[2]-radius[2]), int(voxloc[2]+radius[2]+1)):
                n_coord = np.array((n_x, n_y, n_z))
                coord = np.array((voxloc[0], voxloc[1], voxloc[2]))
                minus = coord - n_coord
                if (np.square(minus) / np.square(np.array(radius)).astype(np.float)).sum()<=1:
                    try:
                        data[n_x, n_y, n_z] = value
                    except IndexError:
                        pass
    return data

def region_growing(image, coordinate, voxnumber):
    """
    Region growing
    Parameters:
        image: nifti data
        coordinate: raw coordinate
        voxnumber: max growing number
    Output:
        rg_image: growth region image
    """
    nt = voxnumber
    tmp_image = np.zeros_like(image)
    rg_image = np.zeros_like(image)
    image_shape = image.shape

    x = coordinate[0]
    y = coordinate[1]
    z = coordinate[2]

    # ensuring the coordinate is in the image
    inside = (x >= 0) and (x < image_shape[0]) and (y >= 0) and \
             (y <= image_shape[1]) and (z >= 0) and (z < image_shape[2])
    if inside is not True:
        print "The coordinate is out of the image range"
        return False

    # initialize region_mean and region_size
    region_mean = image[x,y,z]
    region_size = 0

    # initialize neighbour_list with 10000 rows 4 columns
    neighbour_free = 10000
    neighbour_pos = -1
    neighbour_list = np.zeros((neighbour_free, 4))

    # 26 direct neighbour points
    neighbours = [[1,0,0],\
                  [-1,0,0],\
                  [0,1,0],\
                  [0,-1,0],\
                  [0,0,-1],\
                  [0,0,1],\
                  [1,1,0],\
                  [1,1,1],\
                  [1,1,-1],\
                  [0,1,1],\
                  [-1,1,1],\
                  [1,0,1],\
                  [1,-1,1],\
                  [-1,-1,0],\
                  [-1,-1,-1],\
                  [-1,-1,1],\
                  [0,-1,-1],\
                  [1,-1,-1],\
                  [-1,0,-1],\
                  [-1,1,-1],\
                  [0,1,-1],\
                  [0,-1,1],\
                  [1,0,-1],\
                  [1,-1,0],\
                  [-1,0,1],\
                  [-1,1,0]]
    while region_size < nt:
        # (xn, yn, zn) stored direct neighbour of seed point
        for i in range(6):
            xn = x + neighbours[i][0]
            yn = y + neighbours[i][1]
            zn = z + neighbours[i][2]

            inside = (xn >= 0) and (xn < image_shape[0]) and (yn >=0) and \
                 (yn < image_shape[1]) and (zn >= 0) and (zn < image_shape[2])
            # ensure the original flag 0 is not changed
            if inside and tmp_image[xn, yn, zn] == 0:
                neighbour_pos = neighbour_pos + 1
                neighbour_list[neighbour_pos] = [xn, yn, zn, image[xn,yn,zn]]
                tmp_image[xn,yn,zn] = 1

        # ensure there's enough space to store neighbour_list
        if (neighbour_pos + 100 > neighbour_free):
            neighbour_free += 10000
            new_list = np.zeros((10000,4))
            neighbour_list = np.vstack((neighbour_list, new_list))

        # the distance between every neighbour point value to new region mean value
        distance = np.abs(neighbour_list[:neighbour_pos+1, 3] - np.tile(region_mean, neighbour_pos+1))

        # chose min distance point
        index = distance.argmin()

        # mark the new region point with 2 and update new image
        tmp_image[x, y, z] = 2
        rg_image[x, y, z] = image[x, y, z]
        region_size+=1

        # (x,y,z) the new seed point
        x = neighbour_list[index][0]
        y = neighbour_list[index][1]
        z = neighbour_list[index][2]

        # update region mean value
        region_mean = (region_mean*region_size + neighbour_list[index, 3])/(region_size + 1)

        # remove the seed point from neighbour_list
        neighbour_list[index] = neighbour_list[neighbour_pos]
        neighbour_pos -= 1
    return rg_image

<commit_msg>write a function to search for coordinate of peak values<commit_after># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode:nil -*-
# vi: set ft=python sts=4 sw=4 et:

import numpy as np

def make_pm(mask, meth = 'all'):
    """
    Make probabilistic map
    ------------------------------
    Parameters:
        mask: mask
        meth: 'all' or 'part'.
              all, all subjects are taken into account
              part, part subjects are taken into account
    Return:
        pm = probabilistic map
    """
    if mask.ndim != 4:
        raise Exception('Masks should be a 4D nifti file contains subjects')
    labels = np.unique(mask)[1:]
    pm = np.empty((mask.shape[0], mask.shape[1], mask.shape[2], labels.shape[0]))
    if meth == 'all':
        for i in range(labels.shape[0]):
            pm[..., i] = np.mean(mask == labels[i], axis = 3)
    elif meth == 'part':
        for i in range(labels.shape[0]):
            mask_i = mask == labels[i]
            subj = np.any(mask_i, axis = (0,1,2))
            pm[..., i] = np.mean(mask_i[..., subj], axis = 3)
    else:
        raise Exception('method not supported')
    return pm

def make_mpm(pm, threshold):
    """
    Make maximum probabilistic map (mpm)
    ---------------------------------------
    Parameters:
        threshold: threholds to mask probabilistic maps
    Return:
        mpm: maximum probabilisic map
    """
    pm_temp = np.empty((pm.shape[0], pm.shape[1], pm.shape[2], pm.shape[3]+1))
    pm_temp[..., range(1, pm.shape[3]+1)] = pm
    pm_temp[pm_temp < threshold] = 0
    mpm = np.argmax(pm_temp, axis=3)
    return mpm

def sphere_roi(voxloc, radius, value, datashape = [91,109,91]):
    """
    Generate a sphere roi which centered in (x,y,z)
    Parameters:
        data: your output data
        voxloc: (x,y,z), center vox of spheres
        radius: radius (unit: vox), note that it's a list
        value: label value
    output:
        data: sphere roi image data
        loc: sphere roi coordinates
    """
    loc = []
    data = np.zeros(datashape)
    for n_x in range(int(voxloc[0]-radius[0]), int(voxloc[0]+radius[0]+1)):
        for n_y in range(int(voxloc[1]-radius[1]), int(voxloc[1]+radius[1]+1)):
            for n_z in range(int(voxloc[2]-radius[2]), int(voxloc[2]+radius[2]+1)):
                n_coord = np.array((n_x, n_y, n_z))
                coord = np.array((voxloc[0], voxloc[1], voxloc[2]))
                minus = coord - n_coord
                if (np.square(minus) / np.square(np.array(radius)).astype(np.float)).sum()<=1:
                    try:
                        data[n_x, n_y, n_z] = value
                        loc.append([n_x,n_y,n_z])
                    except IndexError:
                        pass
    loc = np.array(loc)
    return data, loc

def region_growing(image, coordinate, voxnumber):
    """
    Region growing
    Parameters:
        image: nifti data
        coordinate: raw coordinate
        voxnumber: max growing number
    Output:
        rg_image: growth region image
        loc: region growth location
    """
    loc = []
    nt = voxnumber
    tmp_image = np.zeros_like(image)
    rg_image = np.zeros_like(image)
    image_shape = image.shape

    x = coordinate[0]
    y = coordinate[1]
    z = coordinate[2]

    # ensuring the coordinate is in the image
    # inside = (x >= 0) and (x < image_shape[0]) and (y >= 0) and \
    #          (y <= image_shape[1]) and (z >= 0) and (z < image_shape[2])
    # if inside is not True:
    #     print "The coordinate is out of the image range"
    #     return False

    # initialize region_mean and region_size
    region_mean = image[x,y,z]
    region_size = 0

    # initialize neighbour_list with 10000 rows 4 columns
    neighbour_free = 10000
    neighbour_pos = -1
    neighbour_list = np.zeros((neighbour_free, 4))

    # 26 direct neighbour points
    neighbours = [[1,0,0],\
                  [-1,0,0],\
                  [0,1,0],\
                  [0,-1,0],\
                  [0,0,-1],\
                  [0,0,1],\
                  [1,1,0],\
                  [1,1,1],\
                  [1,1,-1],\
                  [0,1,1],\
                  [-1,1,1],\
                  [1,0,1],\
                  [1,-1,1],\
                  [-1,-1,0],\
                  [-1,-1,-1],\
                  [-1,-1,1],\
                  [0,-1,-1],\
                  [1,-1,-1],\
                  [-1,0,-1],\
                  [-1,1,-1],\
                  [0,1,-1],\
                  [0,-1,1],\
                  [1,0,-1],\
                  [1,-1,0],\
                  [-1,0,1],\
                  [-1,1,0]]
    while region_size < nt:
        # (xn, yn, zn) stored direct neighbour of seed point
        for i in range(6):
            xn = x + neighbours[i][0]
            yn = y + neighbours[i][1]
            zn = z + neighbours[i][2]

            inside = (xn >= 0) and (xn < image_shape[0]) and (yn >=0) and \
                 (yn < image_shape[1]) and (zn >= 0) and (zn < image_shape[2])
            # ensure the original flag 0 is not changed
            if inside and tmp_image[xn, yn, zn] == 0:
                neighbour_pos = neighbour_pos + 1
                neighbour_list[neighbour_pos] = [xn, yn, zn, image[xn,yn,zn]]
                tmp_image[xn,yn,zn] = 1

        # ensure there's enough space to store neighbour_list
        if (neighbour_pos + 100 > neighbour_free):
            neighbour_free += 10000
            new_list = np.zeros((10000,4))
            neighbour_list = np.vstack((neighbour_list, new_list))

        # the distance between every neighbour point value to new region mean value
        distance = np.abs(neighbour_list[:neighbour_pos+1, 3] - np.tile(region_mean, neighbour_pos+1))

        # chose min distance point
        index = distance.argmin()

        # mark the new region point with 2 and update new image
        tmp_image[x, y, z] = 2
        rg_image[x, y, z] = image[x, y, z]
        loc.append([x,y,z])
        region_size+=1

        # (x,y,z) the new seed point
        x = neighbour_list[index][0]
        y = neighbour_list[index][1]
        z = neighbour_list[index][2]

        # update region mean value
        region_mean = (region_mean*region_size + neighbour_list[index, 3])/(region_size + 1)

        # remove the seed point from neighbour_list
        neighbour_list[index] = neighbour_list[neighbour_pos]
        neighbour_pos -= 1

    loc = np.array(loc)
    return rg_image, loc

def peakn_location(data, ncluster = 5, rgsize = 10, reverse = False):
    """
    Using region growth to extract highest/lowest clusters
    --------------------------------------
    Parameters:
        data: raw data
        ncluster: cluster numbers by using information of data values
        rgsize: region growth size (voxel), constraint neighbouring voxels
        reverse: if True, get locations start from the largest values
                 if False, start from the lowest values
    Return:
        nth_loc: list of locations
    """
    if reverse is True:
        filterdata = np.argmin
    else:
        filterdata = np.argmax
    median_data = np.median(data)
    nth_loc = []
    for i in range(ncluster):
        temploc = np.unravel_index(filterdata(data), data.shape)
        nth_loc.append(temploc)
        tempdata, loc_rg = region_growing(data, temploc, rgsize)
        for j in loc_rg:
            data[j[0], j[1], j[2]] = median_data
    return nth_loc, tempdata

<|endoftext|>