<commit_before># -*- coding: utf-8 -*-
from mapping import TRANSLIT_DICT
import sys

def __encode_utf8(_string):
    if sys.version_info < (3, 0):
        return _string.encode('utf-8')
    else:
        return _string

def __decode_utf8(_string):
    if sys.version_info < (3, 0):
        return _string.decode('utf-8')
    else:
        return _string

def to_latin(string_to_transliterate, lang_code='sr'):
    ''' Transliterate serbian cyrillic string of characters to latin string of characters.
    :param string_to_transliterate: The cyrillic string to transliterate into latin characters.
    :param lang_code: Indicates the cyrillic language code we are translating from. Defaults to Serbian (sr).
    :return: A string of latin characters transliterated from the given cyrillic string.
    '''

    # First check if we support the cyrillic alphabet we want to transliterate to latin.
    if lang_code.lower() not in TRANSLIT_DICT:
        # If we don't support it, then just return the original string.
        return string_to_transliterate

    # If we do support it, check if the implementation is not missing before proceeding.
    elif not TRANSLIT_DICT[lang_code.lower()]['tolatin']:
        return string_to_transliterate

    # Everything checks out, proceed with transliteration.
    else:

        # Get the character per character transliteration dictionary
        transliteration_dict = TRANSLIT_DICT[lang_code.lower()]['tolatin']

        # Initialize the output latin string variable
        latinized_str = ''

        # Transliterate by traversing the input string character by character.
        string_to_transliterate = __decode_utf8(string_to_transliterate)

        for c in string_to_transliterate:

            # If character is in dictionary, it means it's a cyrillic so let's transliterate that character.
            if c in transliteration_dict:
                # Transliterate current character.
                latinized_str += transliteration_dict[c]

            # If character is not in character transliteration dictionary,
            # it is most likely a number or a special character so just keep it.
            else:
                latinized_str += c

        # Return the transliterated string.
        return __encode_utf8(latinized_str)

def to_cyrillic(string_to_transliterate, lang_code='sr'):
    ''' Transliterate serbian latin string of characters to cyrillic string of characters.
    :param string_to_transliterate: The latin string to transliterate into cyrillic characters.
    :param lang_code: Indicates the cyrillic language code we are translating to. Defaults to Serbian (sr).
    :return: A string of cyrillic characters transliterated from the given latin string.
    '''

    # First check if we support the cyrillic alphabet we want to transliterate to latin.
    if lang_code.lower() not in TRANSLIT_DICT:
        # If we don't support it, then just return the original string.
        return string_to_transliterate

    # If we do support it, check if the implementation is not missing before proceeding.
    elif not TRANSLIT_DICT[lang_code.lower()]['tocyrillic']:
        return string_to_transliterate

    else:
        # Get the character per character transliteration dictionary
        transliteration_dict = TRANSLIT_DICT[lang_code.lower()]['tocyrillic']

        # Initialize the output cyrillic string variable
        cyrillic_str = ''

        string_to_transliterate = __decode_utf8(string_to_transliterate)

        # Transliterate by traversing the inputted string character by character.
        length_of_string_to_transliterate = len(string_to_transliterate)
        index = 0

        while index < length_of_string_to_transliterate:
            # Grab a character from the string at the current index
            c = string_to_transliterate[index]

            # Watch out for Lj and lj. Don't want to interpret Lj/lj as L/l and j.
            # Watch out for Nj and nj. Don't want to interpret Nj/nj as N/n and j.
            # Watch out for Dž and and dž. Don't want to interpret Dž/dž as D/d and j.
            c_plus_1 = u''
            if index != length_of_string_to_transliterate - 1:
                c_plus_1 = string_to_transliterate[index + 1]

            if ((c == u'L' or c == u'l') and c_plus_1 == u'j') or \
               ((c == u'N' or c == u'n') and c_plus_1 == u'j') or \
               ((c == u'D' or c == u'd') and c_plus_1 == u'ž') or \
               (lang_code == 'mk' and (c == u'D' or c == u'd') and c_plus_1 == u'z') or \
               (lang_code == 'ru' and (
                    (c in u'Cc' and c_plus_1 in u'Hh')   or  # c, ch
                    (c in u'Ee' and c_plus_1 in u'Hh')   or  # eh
                    (c == u'i'  and c_plus_1 == u'y' and
                     string_to_transliterate[index + 2:index + 3] not in u'aou') or  # iy[^AaOoUu]
                    (c in u'Jj' and c_plus_1 in u'UuAaEe') or  # j, ju, ja, je
                    (c in u'Ss' and c_plus_1 in u'HhZz') or  # s, sh, sz
                    (c in u'Yy' and c_plus_1 in u'AaOoUu') or  # y, ya, yo, yu
                    (c in u'Zz' and c_plus_1 in u'Hh')       # z, zh
               )):

                index += 1
                c += c_plus_1

            # If character is in dictionary, it means it's a cyrillic so let's transliterate that character.
            if c in transliteration_dict:
                # ay, ey, iy, oy, uy
                if lang_code == 'ru' and c in u'Yy' and \
                        cyrillic_str and cyrillic_str[-1].lower() in u"аеиоуэя":
                    cyrillic_str += u"й" if c == u'y' else u"Й"
                else:
                    # Transliterate current character.
                    cyrillic_str += transliteration_dict[c]

            # If character is not in character transliteration dictionary,
            # it is most likely a number or a special character so just keep it.
            else:
                cyrillic_str += c

            index += 1

        return __encode_utf8(cyrillic_str)

def supported():
    ''' Returns list of supported languages
    :return:
    '''
    return TRANSLIT_DICT.keys()
<commit_msg>used absolute path instead of relative for python 3 compatibility<commit_after># -*- coding: utf-8 -*-
from .mapping import TRANSLIT_DICT
import sys

def __encode_utf8(_string):
    if sys.version_info < (3, 0):
        return _string.encode('utf-8')
    else:
        return _string

def __decode_utf8(_string):
    if sys.version_info < (3, 0):
        return _string.decode('utf-8')
    else:
        return _string

def to_latin(string_to_transliterate, lang_code='sr'):
    ''' Transliterate serbian cyrillic string of characters to latin string of characters.
    :param string_to_transliterate: The cyrillic string to transliterate into latin characters.
    :param lang_code: Indicates the cyrillic language code we are translating from. Defaults to Serbian (sr).
    :return: A string of latin characters transliterated from the given cyrillic string.
    '''

    # First check if we support the cyrillic alphabet we want to transliterate to latin.
    if lang_code.lower() not in TRANSLIT_DICT:
        # If we don't support it, then just return the original string.
        return string_to_transliterate

    # If we do support it, check if the implementation is not missing before proceeding.
    elif not TRANSLIT_DICT[lang_code.lower()]['tolatin']:
        return string_to_transliterate

    # Everything checks out, proceed with transliteration.
    else:

        # Get the character per character transliteration dictionary
        transliteration_dict = TRANSLIT_DICT[lang_code.lower()]['tolatin']

        # Initialize the output latin string variable
        latinized_str = ''

        # Transliterate by traversing the input string character by character.
        string_to_transliterate = __decode_utf8(string_to_transliterate)

        for c in string_to_transliterate:

            # If character is in dictionary, it means it's a cyrillic so let's transliterate that character.
            if c in transliteration_dict:
                # Transliterate current character.
                latinized_str += transliteration_dict[c]

            # If character is not in character transliteration dictionary,
            # it is most likely a number or a special character so just keep it.
            else:
                latinized_str += c

        # Return the transliterated string.
        return __encode_utf8(latinized_str)

def to_cyrillic(string_to_transliterate, lang_code='sr'):
    ''' Transliterate serbian latin string of characters to cyrillic string of characters.
    :param string_to_transliterate: The latin string to transliterate into cyrillic characters.
    :param lang_code: Indicates the cyrillic language code we are translating to. Defaults to Serbian (sr).
    :return: A string of cyrillic characters transliterated from the given latin string.
    '''

    # First check if we support the cyrillic alphabet we want to transliterate to latin.
    if lang_code.lower() not in TRANSLIT_DICT:
        # If we don't support it, then just return the original string.
        return string_to_transliterate

    # If we do support it, check if the implementation is not missing before proceeding.
    elif not TRANSLIT_DICT[lang_code.lower()]['tocyrillic']:
        return string_to_transliterate

    else:
        # Get the character per character transliteration dictionary
        transliteration_dict = TRANSLIT_DICT[lang_code.lower()]['tocyrillic']

        # Initialize the output cyrillic string variable
        cyrillic_str = ''

        string_to_transliterate = __decode_utf8(string_to_transliterate)

        # Transliterate by traversing the inputted string character by character.
        length_of_string_to_transliterate = len(string_to_transliterate)
        index = 0

        while index < length_of_string_to_transliterate:
            # Grab a character from the string at the current index
            c = string_to_transliterate[index]

            # Watch out for Lj and lj. Don't want to interpret Lj/lj as L/l and j.
            # Watch out for Nj and nj. Don't want to interpret Nj/nj as N/n and j.
            # Watch out for Dž and and dž. Don't want to interpret Dž/dž as D/d and j.
            c_plus_1 = u''
            if index != length_of_string_to_transliterate - 1:
                c_plus_1 = string_to_transliterate[index + 1]

            if ((c == u'L' or c == u'l') and c_plus_1 == u'j') or \
               ((c == u'N' or c == u'n') and c_plus_1 == u'j') or \
               ((c == u'D' or c == u'd') and c_plus_1 == u'ž') or \
               (lang_code == 'mk' and (c == u'D' or c == u'd') and c_plus_1 == u'z') or \
               (lang_code == 'ru' and (
                    (c in u'Cc' and c_plus_1 in u'Hh')   or  # c, ch
                    (c in u'Ee' and c_plus_1 in u'Hh')   or  # eh
                    (c == u'i'  and c_plus_1 == u'y' and
                     string_to_transliterate[index + 2:index + 3] not in u'aou') or  # iy[^AaOoUu]
                    (c in u'Jj' and c_plus_1 in u'UuAaEe') or  # j, ju, ja, je
                    (c in u'Ss' and c_plus_1 in u'HhZz') or  # s, sh, sz
                    (c in u'Yy' and c_plus_1 in u'AaOoUu') or  # y, ya, yo, yu
                    (c in u'Zz' and c_plus_1 in u'Hh')       # z, zh
               )):

                index += 1
                c += c_plus_1

            # If character is in dictionary, it means it's a cyrillic so let's transliterate that character.
            if c in transliteration_dict:
                # ay, ey, iy, oy, uy
                if lang_code == 'ru' and c in u'Yy' and \
                        cyrillic_str and cyrillic_str[-1].lower() in u"аеиоуэя":
                    cyrillic_str += u"й" if c == u'y' else u"Й"
                else:
                    # Transliterate current character.
                    cyrillic_str += transliteration_dict[c]

            # If character is not in character transliteration dictionary,
            # it is most likely a number or a special character so just keep it.
            else:
                cyrillic_str += c

            index += 1

        return __encode_utf8(cyrillic_str)

def supported():
    ''' Returns list of supported languages
    :return:
    '''
    return TRANSLIT_DICT.keys()
<|endoftext|>


<commit_before>from django.http import Http404
from django.views.generic import View

from .utils import check_file, send_file

class FileView(View):

    def get(request, file_path=None):
        if not file_path:
            raise Http404

        # append 'index.html' when the file_path is a directory
        if file_path.endswith('/'):
            file_path += 'index.html'

        if check_file(request.user, file_path):
            return send_file(request, file_path)

        # if nothing worked, return 404
        raise Http404
<commit_msg>Fix FileView<commit_after>from django.http import Http404
from django.views.generic import View

from .utils import check_file, send_file

class FileView(View):

    def get(self, request, file_path):
        if not file_path:
            raise Http404

        # append 'index.html' when the file_path is a directory
        if file_path.endswith('/'):
            file_path += 'index.html'

        if check_file(request.user, file_path):
            return send_file(request, file_path)

        # if nothing worked, return 404
        raise Http404
<|endoftext|>


<commit_before>from {{cookiecutter.app_name}}.app import init_celery

app = init_celery()
app.conf.imports = app.conf.imports + ("{{cookiecutter.app_name}}.tasks",)
<commit_msg>fix celery's task import<commit_after>from {{cookiecutter.app_name}}.app import init_celery

app = init_celery()
app.conf.imports = app.conf.imports + ("{{cookiecutter.app_name}}.tasks.example",)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Akvo RSR is covered by the GNU Affero General Public License.
# See more details in the license.txt file located at the root folder of the Akvo RSR module.
# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.

from django.conf import settings
from django.db import models
from django.db.models import Sum
from django.db.models.query import QuerySet as DjangoQuerySet
from django.utils.translation import ugettext_lazy as _

from sorl.thumbnail.fields import ImageField

from akvo.utils import codelist_choices, rsr_image_path

from ..mixins import TimestampsMixin
from ..fields import ValidXMLCharField, ValidXMLTextField
from akvo.codelists.store.codelists_v201 import ORGANISATION_TYPE as IATI_LIST_ORGANISATION_TYPE

from .country import Country
from .partner_site import PartnerSite
from .partnership import Partnership
from .publishing_status import PublishingStatus

ORG_TYPE_NGO = 'N'
ORG_TYPE_GOV = 'G'
ORG_TYPE_COM = 'C'
ORG_TYPE_KNO = 'K'
ORG_TYPES = (
    (ORG_TYPE_NGO, _(u'NGO')),
    (ORG_TYPE_GOV, _(u'Governmental')),
    (ORG_TYPE_COM, _(u'Commercial')),
    (ORG_TYPE_KNO, _(u'Knowledge institution')),
)

def image_path(instance, file_name):
    return rsr_image_path(instance, file_name, 'db/org/%(instance_pk)s/%(file_name)s')

class OrgManager(models.Manager):
    def get_queryset(self):
        return self.model.QuerySet(self.model).extra(
            select={
                'lower_name': 'lower(rsr_organisation.name)'
            }
        ).order_by('lower_name')

    def __getattr__(self, attr, *args):
        try:
            return getattr(self.__class__, attr, *args)
        except AttributeError:
            return getattr(self.get_queryset(), attr, *args)

class Organisation(TimestampsMixin, models.Model):
    """
    There are four types of organisations in RSR, called Field,
    Support, Funding and Sponsor partner respectively.
    """
    NEW_TO_OLD_TYPES = [
        ORG_TYPE_GOV, ORG_TYPE_GOV, ORG_TYPE_NGO, ORG_TYPE_NGO, ORG_TYPE_NGO,
        ORG_TYPE_NGO, ORG_TYPE_NGO, ORG_TYPE_NGO, ORG_TYPE_COM, ORG_TYPE_KNO
    ]

    @classmethod
    def org_type_from_iati_type(cls, iati_type):
        """ utility that maps the IATI organisation types to the old Akvo organisation types
        """
        types = dict(zip([int(type) for type, name in IATI_LIST_ORGANISATION_TYPE[1:]],
            cls.NEW_TO_OLD_TYPES
        ))
        return types[iati_type]

    name = ValidXMLCharField(
        _(u'name'), max_length=25, db_index=True, unique=True,
        help_text=_(u'Short name which will appear in organisation and partner listings '
                    u'(25 characters).')
    )
    long_name = ValidXMLCharField(
        _(u'long name'), blank=True, max_length=75, unique=True,
        help_text=_(u'Full name of organisation (75 characters).'),
    )
    language = ValidXMLCharField(
        _(u'language'), max_length=2, choices=settings.LANGUAGES, default='en',
        help_text=_(u'The main language of the organisation'),
    )
    organisation_type = ValidXMLCharField(
        _(u'organisation type'), max_length=1, db_index=True, choices=ORG_TYPES
    )
    new_organisation_type = models.IntegerField(
        _(u'IATI organisation type'), db_index=True,
        choices=[(int(c[0]), c[1]) for c in codelist_choices(IATI_LIST_ORGANISATION_TYPE)],
        default=22, help_text=_(u'Check that this field is set to an organisation type that '
                                u'matches your organisation.'),
    )
    iati_org_id = ValidXMLCharField(
        _(u'IATI organisation ID'), max_length=75, blank=True, null=True, db_index=True, unique=True
    )
    internal_org_ids = models.ManyToManyField(
        'self', through='InternalOrganisationID', symmetrical=False,
        related_name='recording_organisation'
    )
    logo = ImageField(_(u'logo'), blank=True, upload_to=image_path,
                      help_text=_(u'Logos should be approximately 360x270 pixels '
                                  u'(approx. 100-200kB in size) on a white background.')
    )
    url = models.URLField(
        blank=True,
        help_text=_(u'Enter the full address of your web site, beginning with http://.'),
    )
    facebook = models.URLField(
        blank=True,
        help_text=_(u'Enter the full address of your Facebook page, beginning with http://.'),
    )
    twitter = models.URLField(
        blank=True,
        help_text=_(u'Enter the full address of your Twitter feed, beginning with http://.'),
    )
    linkedin = models.URLField(
        blank=True,
        help_text=_(u'Enter the full address of your LinkedIn page, beginning with http://.'),
    )
    phone = ValidXMLCharField(
        _(u'phone'), blank=True, max_length=20, help_text=_(u'(20 characters).')
    )
    mobile = ValidXMLCharField(
        _(u'mobile'), blank=True, max_length=20, help_text=_(u'(20 characters).')
    )
    fax = ValidXMLCharField(
        _(u'fax'), blank=True, max_length=20, help_text=_(u'(20 characters).')
    )
    contact_person = ValidXMLCharField(
        _(u'contact person'), blank=True, max_length=30,
        help_text=_(u'Name of external contact person for your organisation (30 characters).'),
    )
    contact_email = ValidXMLCharField(
        _(u'contact email'), blank=True, max_length=50,
        help_text=_(u'Email to which inquiries about your organisation should be sent '
                    u'(50 characters).'),
    )
    description = ValidXMLTextField(
        _(u'description'), blank=True, help_text=_(u'Describe your organisation.')
    )
    notes = ValidXMLTextField(verbose_name=_(u"Notes and comments"), blank=True, default='')
    primary_location = models.ForeignKey(
        'OrganisationLocation', null=True, on_delete=models.SET_NULL
    )
    content_owner = models.ForeignKey(
        'self', null=True, blank=True, on_delete=models.SET_NULL,
        help_text=_(u'Organisation that maintains content for this organisation through the API.')
    )
    allow_edit = models.BooleanField(
        _(u'Partner editors of this organisation are allowed to manually edit projects where '
          u'this organisation is support partner'),
        help_text=_(u'When manual edits are disallowed, partner admins and editors of other '
                    u'organisations are also not allowed to edit these projects.'), default=True
    )
    public_iati_file = models.BooleanField(
        _(u'Show latest exported IATI file on organisation page.'), default=True
    )
    can_become_reporting = models.BooleanField(
        _(u'Reportable'),
        help_text=_(u'Organisation is allowed to become a reporting organisation. '
                    u'Can be set by superusers.'),
        default=False)

    objects = OrgManager()

    @models.permalink
    def get_absolute_url(self):
        return ('organisation-main', (), {'organisation_id': self.pk})

    class QuerySet(DjangoQuerySet):
        def has_location(self):
            return self.filter(primary_location__isnull=False)

        def partners(self, role):
            "return the organisations in the queryset that are partners of type role"
            return self.filter(partnerships__iati_organisation_role__exact=role).distinct()

        def allpartners(self):
            return self.distinct()

        def fieldpartners(self):
            return self.partners(Partnership.IATI_IMPLEMENTING_PARTNER)

        def fundingpartners(self):
            return self.partners(Partnership.IATI_FUNDING_PARTNER)

        def sponsorpartners(self):
            return self.partners(Partnership.AKVO_SPONSOR_PARTNER)

        def supportpartners(self):
            return self.partners(Partnership.IATI_ACCOUNTABLE_PARTNER)

        def extendingpartners(self):
            return self.partners(Partnership.IATI_EXTENDING_PARTNER)

        def supportpartners_with_projects(self):
            """return the organisations in the queryset that are support partners with published
            projects, not counting archived projects"""
            from .project import Project
            return self.filter(
                partnerships__iati_organisation_role=Partnership.IATI_ACCOUNTABLE_PARTNER,
                partnerships__project__publishingstatus__status=PublishingStatus.STATUS_PUBLISHED,
                partnerships__project__status__in=[
                    Project.STATUS_ACTIVE,
                    Project.STATUS_COMPLETE,
                    Project.STATUS_NEEDS_FUNDING,
                    Project.STATUS_CANCELLED,
                ]
            ).distinct()

        def ngos(self):
            return self.filter(organisation_type__exact=ORG_TYPE_NGO)

        def governmental(self):
            return self.filter(organisation_type__exact=ORG_TYPE_GOV)

        def commercial(self):
            return self.filter(organisation_type__exact=ORG_TYPE_COM)

        def knowledge(self):
            return self.filter(organisation_type__exact=ORG_TYPE_KNO)

        def all_projects(self):
            "returns a queryset with all projects that has self as any kind of partner"
            from .project import Project
            return (Project.objects.filter(partnerships__organisation__in=self) |
                    Project.objects.filter(sync_owner__in=self)).distinct()

        def users(self):
            "returns a queryset of all users belonging to the organisation(s)"
            from .user import User
            return User.objects.filter(employers__organisation__in=self).distinct()

        def employments(self):
            "returns a queryset of all employments belonging to the organisation(s)"
            from .employment import Employment
            return Employment.objects.filter(organisation__in=self).distinct()

    def __unicode__(self):
        return self.name

    def iati_org_type(self):
        return dict(IATI_LIST_ORGANISATION_TYPE)[str(self.new_organisation_type)] if \
            self.new_organisation_type else ""

    def partnersites(self):
        "returns the partnersites belonging to the organisation in a PartnerSite queryset"
        return PartnerSite.objects.filter(organisation=self)

    def website(self):
        return '%s' % (self.url, self.url,)
    website.allow_tags = True

    def all_users(self):
        "returns a queryset of all users belonging to the organisation"
        from .user import User
        return User.objects.filter(employers__organisation=self).distinct()

    def published_projects(self):
        "returns a queryset with published projects that has self as any kind of partner"
        return self.projects.published().distinct()

    def all_projects(self):
        """returns a queryset with all projects that has self as any kind of partner or reporting
        organisation."""
        return (self.projects.all() | self.reporting_projects.all()).distinct()

    def active_projects(self):
        return self.published_projects().status_not_cancelled().status_not_archived()

    def partners(self):
        """returns a queryset of all organisations that self has at least one project
        in common with, excluding self"""
        return self.published_projects().all_partners().exclude(id__exact=self.id)

    def support_partners(self):
        """returns a queryset of support partners that self has at least one project
        in common with, excluding self"""
        return self.published_projects().support_partners().exclude(id__exact=self.id)

    def has_partner_types(self, project):
        """Return a list of partner types of this organisation to the project"""
        partner_types = []
        for ps in Partnership.objects.filter(project=project, organisation=self):
            if ps.iati_organisation_role:
                partner_types.append(ps.iati_organisation_role_label())
        return partner_types

    def countries_where_active(self):
        """Returns a Country queryset of countries where this organisation has
        published projects."""
        return Country.objects.filter(
            projectlocation__project__partnerships__organisation=self,
            projectlocation__project__publishingstatus__status=PublishingStatus.STATUS_PUBLISHED
        ).distinct()

    def iati_file(self):
        """
        Looks up the latest public IATI file of this organisation.

        :return: String of IATI file or None
        """
        for export in self.iati_exports.all().order_by('-last_modified_at'):
            if export.is_public and export.status == 3 and export.iati_file:
                return export.iati_file
        return None

    # New API

    def euros_pledged(self):
        "How much € the organisation has pledged to projects it is a partner to"
        return self.active_projects().euros().filter(
            partnerships__organisation__exact=self,
            partnerships__iati_organisation_role__exact=Partnership.IATI_FUNDING_PARTNER
        ).aggregate(
            euros_pledged=Sum('partnerships__funding_amount')
        )['euros_pledged'] or 0

    def dollars_pledged(self):
        "How much $ the organisation has pledged to projects"
        return self.active_projects().dollars().filter(
            partnerships__organisation__exact=self,
            partnerships__iati_organisation_role__exact=Partnership.IATI_FUNDING_PARTNER
        ).aggregate(
            dollars_pledged=Sum('partnerships__funding_amount')
        )['dollars_pledged'] or 0

    def euro_projects_count(self):
        "How many projects with budget in € the organisation is a partner to"
        return self.published_projects().euros().distinct().count()

    def dollar_projects_count(self):
        "How many projects with budget in $ the organisation is a partner to"
        return self.published_projects().dollars().distinct().count()

    def _aggregate_funds_needed(self, projects):
        return sum(projects.values_list('funds_needed', flat=True))

    def euro_funds_needed(self):
        """How much is still needed to fully fund all projects with € budget that the
        organisation is a partner to.

        The ORM aggregate() doesn't work here since we may have multiple partnership relations
        to the same project."""
        return self._aggregate_funds_needed(self.published_projects().euros().distinct())

    def dollar_funds_needed(self):
        """How much is still needed to fully fund all projects with $ budget that the
        organisation is a partner to.

        The ORM aggregate() doesn't work here since we may have multiple partnership relations
        to the same project."""
        return self._aggregate_funds_needed(self.published_projects().dollars().distinct())

    class Meta:
        app_label = 'rsr'
        verbose_name = _(u'organisation')
        verbose_name_plural = _(u'organisations')
        permissions = (
            ('user_management', u'Can manage users'),
        )
<commit_msg>[#795] Add function for recursive content_owner of orgs<commit_after># -*- coding: utf-8 -*-

# Akvo RSR is covered by the GNU Affero General Public License.
# See more details in the license.txt file located at the root folder of the Akvo RSR module.
# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.

from django.conf import settings
from django.db import models
from django.db.models import Sum
from django.db.models.query import QuerySet as DjangoQuerySet
from django.utils.translation import ugettext_lazy as _

from sorl.thumbnail.fields import ImageField

from akvo.utils import codelist_choices, rsr_image_path

from ..mixins import TimestampsMixin
from ..fields import ValidXMLCharField, ValidXMLTextField
from akvo.codelists.store.codelists_v201 import ORGANISATION_TYPE as IATI_LIST_ORGANISATION_TYPE

from .country import Country
from .partner_site import PartnerSite
from .partnership import Partnership
from .publishing_status import PublishingStatus

ORG_TYPE_NGO = 'N'
ORG_TYPE_GOV = 'G'
ORG_TYPE_COM = 'C'
ORG_TYPE_KNO = 'K'
ORG_TYPES = (
    (ORG_TYPE_NGO, _(u'NGO')),
    (ORG_TYPE_GOV, _(u'Governmental')),
    (ORG_TYPE_COM, _(u'Commercial')),
    (ORG_TYPE_KNO, _(u'Knowledge institution')),
)

def image_path(instance, file_name):
    return rsr_image_path(instance, file_name, 'db/org/%(instance_pk)s/%(file_name)s')

class OrgManager(models.Manager):
    def get_queryset(self):
        return self.model.QuerySet(self.model).extra(
            select={
                'lower_name': 'lower(rsr_organisation.name)'
            }
        ).order_by('lower_name')

    def __getattr__(self, attr, *args):
        try:
            return getattr(self.__class__, attr, *args)
        except AttributeError:
            return getattr(self.get_queryset(), attr, *args)

class Organisation(TimestampsMixin, models.Model):
    """
    There are four types of organisations in RSR, called Field,
    Support, Funding and Sponsor partner respectively.
    """
    NEW_TO_OLD_TYPES = [
        ORG_TYPE_GOV, ORG_TYPE_GOV, ORG_TYPE_NGO, ORG_TYPE_NGO, ORG_TYPE_NGO,
        ORG_TYPE_NGO, ORG_TYPE_NGO, ORG_TYPE_NGO, ORG_TYPE_COM, ORG_TYPE_KNO
    ]

    @classmethod
    def org_type_from_iati_type(cls, iati_type):
        """ utility that maps the IATI organisation types to the old Akvo organisation types
        """
        types = dict(zip([int(type) for type, name in IATI_LIST_ORGANISATION_TYPE[1:]],
            cls.NEW_TO_OLD_TYPES
        ))
        return types[iati_type]

    name = ValidXMLCharField(
        _(u'name'), max_length=25, db_index=True, unique=True,
        help_text=_(u'Short name which will appear in organisation and partner listings '
                    u'(25 characters).')
    )
    long_name = ValidXMLCharField(
        _(u'long name'), blank=True, max_length=75, unique=True,
        help_text=_(u'Full name of organisation (75 characters).'),
    )
    language = ValidXMLCharField(
        _(u'language'), max_length=2, choices=settings.LANGUAGES, default='en',
        help_text=_(u'The main language of the organisation'),
    )
    organisation_type = ValidXMLCharField(
        _(u'organisation type'), max_length=1, db_index=True, choices=ORG_TYPES
    )
    new_organisation_type = models.IntegerField(
        _(u'IATI organisation type'), db_index=True,
        choices=[(int(c[0]), c[1]) for c in codelist_choices(IATI_LIST_ORGANISATION_TYPE)],
        default=22, help_text=_(u'Check that this field is set to an organisation type that '
                                u'matches your organisation.'),
    )
    iati_org_id = ValidXMLCharField(
        _(u'IATI organisation ID'), max_length=75, blank=True, null=True, db_index=True, unique=True
    )
    internal_org_ids = models.ManyToManyField(
        'self', through='InternalOrganisationID', symmetrical=False,
        related_name='recording_organisation'
    )
    logo = ImageField(_(u'logo'), blank=True, upload_to=image_path,
                      help_text=_(u'Logos should be approximately 360x270 pixels '
                                  u'(approx. 100-200kB in size) on a white background.')
    )
    url = models.URLField(
        blank=True,
        help_text=_(u'Enter the full address of your web site, beginning with http://.'),
    )
    facebook = models.URLField(
        blank=True,
        help_text=_(u'Enter the full address of your Facebook page, beginning with http://.'),
    )
    twitter = models.URLField(
        blank=True,
        help_text=_(u'Enter the full address of your Twitter feed, beginning with http://.'),
    )
    linkedin = models.URLField(
        blank=True,
        help_text=_(u'Enter the full address of your LinkedIn page, beginning with http://.'),
    )
    phone = ValidXMLCharField(
        _(u'phone'), blank=True, max_length=20, help_text=_(u'(20 characters).')
    )
    mobile = ValidXMLCharField(
        _(u'mobile'), blank=True, max_length=20, help_text=_(u'(20 characters).')
    )
    fax = ValidXMLCharField(
        _(u'fax'), blank=True, max_length=20, help_text=_(u'(20 characters).')
    )
    contact_person = ValidXMLCharField(
        _(u'contact person'), blank=True, max_length=30,
        help_text=_(u'Name of external contact person for your organisation (30 characters).'),
    )
    contact_email = ValidXMLCharField(
        _(u'contact email'), blank=True, max_length=50,
        help_text=_(u'Email to which inquiries about your organisation should be sent '
                    u'(50 characters).'),
    )
    description = ValidXMLTextField(
        _(u'description'), blank=True, help_text=_(u'Describe your organisation.')
    )
    notes = ValidXMLTextField(verbose_name=_(u"Notes and comments"), blank=True, default='')
    primary_location = models.ForeignKey(
        'OrganisationLocation', null=True, on_delete=models.SET_NULL
    )
    content_owner = models.ForeignKey(
        'self', null=True, blank=True, on_delete=models.SET_NULL,
        help_text=_(u'Organisation that maintains content for this organisation through the API.')
    )
    allow_edit = models.BooleanField(
        _(u'Partner editors of this organisation are allowed to manually edit projects where '
          u'this organisation is support partner'),
        help_text=_(u'When manual edits are disallowed, partner admins and editors of other '
                    u'organisations are also not allowed to edit these projects.'), default=True
    )
    public_iati_file = models.BooleanField(
        _(u'Show latest exported IATI file on organisation page.'), default=True
    )
    can_become_reporting = models.BooleanField(
        _(u'Reportable'),
        help_text=_(u'Organisation is allowed to become a reporting organisation. '
                    u'Can be set by superusers.'),
        default=False)

    objects = OrgManager()

    @models.permalink
    def get_absolute_url(self):
        return ('organisation-main', (), {'organisation_id': self.pk})

    class QuerySet(DjangoQuerySet):
        def has_location(self):
            return self.filter(primary_location__isnull=False)

        def partners(self, role):
            "return the organisations in the queryset that are partners of type role"
            return self.filter(partnerships__iati_organisation_role__exact=role).distinct()

        def allpartners(self):
            return self.distinct()

        def fieldpartners(self):
            return self.partners(Partnership.IATI_IMPLEMENTING_PARTNER)

        def fundingpartners(self):
            return self.partners(Partnership.IATI_FUNDING_PARTNER)

        def sponsorpartners(self):
            return self.partners(Partnership.AKVO_SPONSOR_PARTNER)

        def supportpartners(self):
            return self.partners(Partnership.IATI_ACCOUNTABLE_PARTNER)

        def extendingpartners(self):
            return self.partners(Partnership.IATI_EXTENDING_PARTNER)

        def supportpartners_with_projects(self):
            """return the organisations in the queryset that are support partners with published
            projects, not counting archived projects"""
            from .project import Project
            return self.filter(
                partnerships__iati_organisation_role=Partnership.IATI_ACCOUNTABLE_PARTNER,
                partnerships__project__publishingstatus__status=PublishingStatus.STATUS_PUBLISHED,
                partnerships__project__status__in=[
                    Project.STATUS_ACTIVE,
                    Project.STATUS_COMPLETE,
                    Project.STATUS_NEEDS_FUNDING,
                    Project.STATUS_CANCELLED,
                ]
            ).distinct()

        def ngos(self):
            return self.filter(organisation_type__exact=ORG_TYPE_NGO)

        def governmental(self):
            return self.filter(organisation_type__exact=ORG_TYPE_GOV)

        def commercial(self):
            return self.filter(organisation_type__exact=ORG_TYPE_COM)

        def knowledge(self):
            return self.filter(organisation_type__exact=ORG_TYPE_KNO)

        def all_projects(self):
            "returns a queryset with all projects that has self as any kind of partner"
            from .project import Project
            return (Project.objects.filter(partnerships__organisation__in=self) |
                    Project.objects.filter(sync_owner__in=self)).distinct()

        def users(self):
            "returns a queryset of all users belonging to the organisation(s)"
            from .user import User
            return User.objects.filter(employers__organisation__in=self).distinct()

        def employments(self):
            "returns a queryset of all employments belonging to the organisation(s)"
            from .employment import Employment
            return Employment.objects.filter(organisation__in=self).distinct()

    def __unicode__(self):
        return self.name

    def iati_org_type(self):
        return dict(IATI_LIST_ORGANISATION_TYPE)[str(self.new_organisation_type)] if \
            self.new_organisation_type else ""

    def partnersites(self):
        "returns the partnersites belonging to the organisation in a PartnerSite queryset"
        return PartnerSite.objects.filter(organisation=self)

    def website(self):
        return '%s' % (self.url, self.url,)
    website.allow_tags = True

    def all_users(self):
        "returns a queryset of all users belonging to the organisation"
        from .user import User
        return User.objects.filter(employers__organisation=self).distinct()

    def published_projects(self):
        "returns a queryset with published projects that has self as any kind of partner"
        return self.projects.published().distinct()

    def all_projects(self):
        """returns a queryset with all projects that has self as any kind of partner or reporting
        organisation."""
        return (self.projects.all() | self.reporting_projects.all()).distinct()

    def active_projects(self):
        return self.published_projects().status_not_cancelled().status_not_archived()

    def partners(self):
        """returns a queryset of all organisations that self has at least one project
        in common with, excluding self"""
        return self.published_projects().all_partners().exclude(id__exact=self.id)

    def support_partners(self):
        """returns a queryset of support partners that self has at least one project
        in common with, excluding self"""
        return self.published_projects().support_partners().exclude(id__exact=self.id)

    def has_partner_types(self, project):
        """Return a list of partner types of this organisation to the project"""
        partner_types = []
        for ps in Partnership.objects.filter(project=project, organisation=self):
            if ps.iati_organisation_role:
                partner_types.append(ps.iati_organisation_role_label())
        return partner_types

    def content_owned_organisations(self):
        """
        Return a list of Organisation objects of which this organisation is the content owner.
        Includes self, is recursive.
        """
        org_set = set()
        org_set.add(self)
        self_content_owned = list(Organisation.objects.filter(content_owner=self))

        while self_content_owned:
            org = self_content_owned.pop()
            org_set.add(org)
            for add_org in org.content_owned_organisations():
                org_set.add(add_org)

        return list(org_set)

    def countries_where_active(self):
        """Returns a Country queryset of countries where this organisation has
        published projects."""
        return Country.objects.filter(
            projectlocation__project__partnerships__organisation=self,
            projectlocation__project__publishingstatus__status=PublishingStatus.STATUS_PUBLISHED
        ).distinct()

    def iati_file(self):
        """
        Looks up the latest public IATI file of this organisation.

        :return: String of IATI file or None
        """
        for export in self.iati_exports.all().order_by('-last_modified_at'):
            if export.is_public and export.status == 3 and export.iati_file:
                return export.iati_file
        return None

    # New API

    def euros_pledged(self):
        "How much € the organisation has pledged to projects it is a partner to"
        return self.active_projects().euros().filter(
            partnerships__organisation__exact=self,
            partnerships__iati_organisation_role__exact=Partnership.IATI_FUNDING_PARTNER
        ).aggregate(
            euros_pledged=Sum('partnerships__funding_amount')
        )['euros_pledged'] or 0

    def dollars_pledged(self):
        "How much $ the organisation has pledged to projects"
        return self.active_projects().dollars().filter(
            partnerships__organisation__exact=self,
            partnerships__iati_organisation_role__exact=Partnership.IATI_FUNDING_PARTNER
        ).aggregate(
            dollars_pledged=Sum('partnerships__funding_amount')
        )['dollars_pledged'] or 0

    def euro_projects_count(self):
        "How many projects with budget in € the organisation is a partner to"
        return self.published_projects().euros().distinct().count()

    def dollar_projects_count(self):
        "How many projects with budget in $ the organisation is a partner to"
        return self.published_projects().dollars().distinct().count()

    def _aggregate_funds_needed(self, projects):
        return sum(projects.values_list('funds_needed', flat=True))

    def euro_funds_needed(self):
        """How much is still needed to fully fund all projects with € budget that the
        organisation is a partner to.

        The ORM aggregate() doesn't work here since we may have multiple partnership relations
        to the same project."""
        return self._aggregate_funds_needed(self.published_projects().euros().distinct())

    def dollar_funds_needed(self):
        """How much is still needed to fully fund all projects with $ budget that the
        organisation is a partner to.

        The ORM aggregate() doesn't work here since we may have multiple partnership relations
        to the same project."""
        return self._aggregate_funds_needed(self.published_projects().dollars().distinct())

    class Meta:
        app_label = 'rsr'
        verbose_name = _(u'organisation')
        verbose_name_plural = _(u'organisations')
        permissions = (
            ('user_management', u'Can manage users'),
        )
<|endoftext|>


<commit_before># !/usr/bin/python
# -*- coding: cp1252 -*-
#
##################################################################################
#
#    Copyright 2016 Félix Brezo and Yaiza Rubio (i3visio, contacto@i3visio.com)
#
#    This program is part of OSRFramework. You can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##################################################################################

import argparse
import json
import re
import sys
import urllib2

import osrframework.utils.browser as browser
from osrframework.utils.platforms import Platform

class Castroller(Platform):
    """
        A <Platform> object for Castroller.
    """
    def __init__(self):
        """
            Constructor...
        """
        self.platformName = "Castroller"
        self.tags = ["audio", "video"]

        ########################
        # Defining valid modes #
        ########################
        self.isValidMode = {}
        self.isValidMode["phonefy"] = False
        self.isValidMode["usufy"] = True
        self.isValidMode["searchfy"] = False

        ######################################
        # Search URL for the different modes #
        ######################################
        # Strings with the URL for each and every mode
        self.url = {}
        #self.url["phonefy"] = "http://anyurl.com//phone/" + "<phonefy>"
        self.url["usufy"] = "http://castroller.com/" + "<usufy>"
        #self.url["searchfy"] = "http://anyurl.com/search/" + "<searchfy>"

        ######################################
        # Whether the user needs credentials #
        ######################################
        self.needsCredentials = {}
        #self.needsCredentials["phonefy"] = False
        self.needsCredentials["usufy"] = False
        #self.needsCredentials["searchfy"] = False

        #################
        # Valid queries #
        #################
        # Strings that will imply that the query number is not appearing
        self.validQuery = {}
        # The regular expression '.+' will match any query.
        #self.validQuery["phonefy"] = ".*"
        self.validQuery["usufy"] = ".+"
        #self.validQuery["searchfy"] = ".*"

        ###################
        # Not_found clues #
        ###################
        # Strings that will imply that the query number is not appearing
        self.notFoundText = {}
        #self.notFoundText["phonefy"] = []
        self.notFoundText["usufy"] = ["<title>404 - File or directory not found.</title>"]
        #self.notFoundText["searchfy"] = []

        #########################
        # Fields to be searched #
        #########################
        self.fieldsRegExp = {}

        # Definition of regular expressions to be searched in phonefy mode
        #self.fieldsRegExp["phonefy"] = {}
        # Example of fields:
        #self.fieldsRegExp["phonefy"]["i3visio.location"] = ""

        # Definition of regular expressions to be searched in usufy mode
        self.fieldsRegExp["usufy"] = {}
        # Example of fields:
        #self.fieldsRegExp["usufy"]["i3visio.location"] = ""

        # Definition of regular expressions to be searched in searchfy mode
        #self.fieldsRegExp["searchfy"] = {}
        # Example of fields:
        #self.fieldsRegExp["searchfy"]["i3visio.location"] = ""

        ################
        # Fields found #
        ################
        # This attribute will be feeded when running the program.
        self.foundFields = {}

<commit_msg>Remove Castroller wrapper<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Copyright (C) 2015-2015: Alignak team, see AUTHORS.txt file for contributors
#
# This file is part of Alignak.
#
# Alignak is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Alignak is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Alignak.  If not, see <http://www.gnu.org/licenses/>.
#
#
# This file incorporates work covered by the following copyright and
# permission notice:
#
#  Copyright (C) 2009-2014:
#     Romain Forlot, rforlot@yahoo.com
#     Hartmut Goebel, h.goebel@goebel-consult.de
#     Jean Gabes, naparuba@gmail.com
#     Nicolas Dupeux, nicolas@dupeux.net
#     Sebastien Coavoux, s.coavoux@free.fr

#  This file is part of Shinken.
#
#  Shinken is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  Shinken is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

'''
 This is a utility class for factorizing matching functions for
 discovery runners and rules.
'''

import re

from item import Item

class MatchingItem(Item):

    # Try to see if the key,value is matching one or
    # our rule. If value got ',' we must look for each value
    # If one match, we quit
    # We can find in matches or not_matches
    def is_matching(self, key, value, look_in='matches'):
        if look_in == 'matches':
            d = self.matches
        else:
            d = self.not_matches
        # If we do not even have the key, we bailout
        if not key.strip() in d:
            return False

        # Get my matching pattern
        m = d[key]
        if ',' in m:
            matchings = [mt.strip() for mt in m.split(',')]
        else:
            matchings = [m]

        # Split the value by , too
        values = value.split(',')
        for m in matchings:
            for v in values:
                print "Try to match", m, v
                # Maybe m is a list, if so should check one values
                if isinstance(m, list):
                    for _m in m:
                        if re.search(_m, v):
                            return True
                else:
                    if re.search(m, v):
                        return True
        return False

    # Look if we match all discovery data or not
    # a disco data look as a list of (key, values)
    def is_matching_disco_datas(self, datas):
        # If we got not data, no way we can match
        if len(datas) == 0:
            return False

        # First we look if it's possible to match
        # we must match All self.matches things
        for m in self.matches:
            # print "Compare to", m
            match_one = False
            for (k, v) in datas.iteritems():
                # We found at least one of our match key
                if m == k:
                    if self.is_matching(k, v):
                        # print "Got matching with", m, k, v
                        match_one = True
                        continue
            if not match_one:
                # It match none
                # print "Match none, False"
                return False
        # print "It's possible to be OK"

        # And now look if ANY of not_matches is reach. If so
        # it's False
        for m in self.not_matches:
            # print "Compare to NOT", m
            match_one = False
            for (k, v) in datas.iteritems():
                # print "K,V", k,v
                # We found at least one of our match key
                if m == k:
                    # print "Go loop"
                    if self.is_matching(k, v, look_in='not_matches'):
                        # print "Got matching with", m, k, v
                        match_one = True
                        continue
            if match_one:
                # print "I match one, I quit"
                return False

        # Ok we match ALL rules in self.matches
        # and NONE of self.not_matches, we can go :)
        return True
<commit_msg>Enh: Pylint - Docstring in matchingitem.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Copyright (C) 2015-2015: Alignak team, see AUTHORS.txt file for contributors
#
# This file is part of Alignak.
#
# Alignak is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Alignak is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Alignak.  If not, see <http://www.gnu.org/licenses/>.
#
#
# This file incorporates work covered by the following copyright and
# permission notice:
#
#  Copyright (C) 2009-2014:
#     Romain Forlot, rforlot@yahoo.com
#     Hartmut Goebel, h.goebel@goebel-consult.de
#     Jean Gabes, naparuba@gmail.com
#     Nicolas Dupeux, nicolas@dupeux.net
#     Sebastien Coavoux, s.coavoux@free.fr

#  This file is part of Shinken.
#
#  Shinken is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  Shinken is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

'''
 This is a utility class for factorizing matching functions for
 discovery runners and rules.
'''

import re

from item import Item

class MatchingItem(Item):
    """
    MatchingItem class provide function for regular expression matching
    """

    def is_matching(self, key, value, look_in='matches'):
        """Try to see if the key,value is matching one or
        our rule.

        :param key: key to find in dict
        :param value: value to find at dict[key]
        :param look_in: the attribute dict to look in (matches or not_matches)
        :return: True if  dict[key] match in value, False otherwise
        """
        if look_in == 'matches':
            d = self.matches
        else:
            d = self.not_matches
        # If we do not even have the key, we bailout
        if not key.strip() in d:
            return False

        # Get my matching pattern
        m = d[key]
        if ',' in m:
            matchings = [mt.strip() for mt in m.split(',')]
        else:
            matchings = [m]

        # Split the value by , too
        values = value.split(',')
        for m in matchings:
            for v in values:
                print "Try to match", m, v
                # Maybe m is a list, if so should check one values
                if isinstance(m, list):
                    for _m in m:
                        if re.search(_m, v):
                            return True
                else:
                    if re.search(m, v):
                        return True
        return False

    def is_matching_disco_datas(self, datas):
        """Check if data match one of our matching attribute (matches or not_matches)

        :param datas: data to parse
        :return: True if we match one pattern in matches and no pattern in not_matches,
                 False otherwise
        """
        # If we got not data, no way we can match
        if len(datas) == 0:
            return False

        # First we look if it's possible to match
        # we must match All self.matches things
        for m in self.matches:
            # print "Compare to", m
            match_one = False
            for (k, v) in datas.iteritems():
                # We found at least one of our match key
                if m == k:
                    if self.is_matching(k, v):
                        # print "Got matching with", m, k, v
                        match_one = True
                        continue
            if not match_one:
                # It match none
                # print "Match none, False"
                return False
        # print "It's possible to be OK"

        # And now look if ANY of not_matches is reach. If so
        # it's False
        for m in self.not_matches:
            # print "Compare to NOT", m
            match_one = False
            for (k, v) in datas.iteritems():
                # print "K,V", k,v
                # We found at least one of our match key
                if m == k:
                    # print "Go loop"
                    if self.is_matching(k, v, look_in='not_matches'):
                        # print "Got matching with", m, k, v
                        match_one = True
                        continue
            if match_one:
                # print "I match one, I quit"
                return False

        # Ok we match ALL rules in self.matches
        # and NONE of self.not_matches, we can go :)
        return True
<|endoftext|>


<commit_before>from typing import Callable, Generic, List, TypeVar, Union, cast

import entrypoints

PluginType = TypeVar('PluginType')

class PluginRegistry(Generic[PluginType]):
    """A registry for plugins.

    This is a plugin registry that allows plugins to be loaded/registered
    in two ways:

    1. Through an explicit call to ``.register(name, value)``.
    2. By looking for other Python packages that are installed and provide
       a setuptools entry point group.

    When you create an instance of this claass, provide the name of the
    entry point group to use::

        reg = PluginRegister('my_entrypoint_group')

    """

    def __init__(self, entry_point_group: str = '', plugin_type=object) -> None:
        """Create a PluginRegistry for a named entry point group.

        Parameters
        ==========
        entry_point_group: str
            The name of the entry point group.
        plugin_type: object
            A type that will optionally be used for runtime type checking of
            loaded plugins using isinstance.
        """
        self.entry_point_group = entry_point_group
        self.plugin_type = plugin_type
        self._active = None     # type: None
        self._plugins = {}      # type: dict
        self._options = {}      # type: dict

    def register(self, name: str, value: Union[PluginType,None]) -> PluginType:
        """Register a plugin by name and value.

        This method is used for explicit registration of a plugin and shouldn't be
        used to manage entry point managed plugins, which are auto-loaded.

        Parameters
        ==========
        name: str
            The name of the plugin.
        value: PluginType or None
            The actual plugin object to register or None to unregister that plugin.

        Returns
        =======
        plugin: PluginType
            The plugin that was registered or unregistered.
        """
        if value is None and name in self._plugins:
            return self._plugins.pop(name)
        else:
            assert isinstance(value, self.plugin_type)
            self._plugins[name] = value
            return value

    def names(self) -> List[str]:
        """List the names of the registered and entry points plugins."""
        exts = list(self._plugins.keys())
        more_exts = [ep.name for ep in entrypoints.get_group_all(self.entry_point_group)]
        exts.extend(more_exts)
        return exts

    def enable(self, name: str) -> None:
        """Enable a plugin by name."""
        if name not in self._plugins:
            ep = entrypoints.get_single(self.entry_point_group, name)
            value = cast(PluginType, ep.load())
            assert isinstance(value, self.plugin_type)
            self.register(name, value)
        self._active = self._plugins[name]

    def get(self) -> PluginType:
        """Return the currently active plugin."""
        return self._active

    def set_options(self, **kwargs):
        self._options = kwargs

    def get_options(self):
        return self._options
<commit_msg>Remove unused methods of the plugin reg.<commit_after>from typing import Callable, Generic, List, TypeVar, Union, cast

import entrypoints

PluginType = TypeVar('PluginType')

class PluginRegistry(Generic[PluginType]):
    """A registry for plugins.

    This is a plugin registry that allows plugins to be loaded/registered
    in two ways:

    1. Through an explicit call to ``.register(name, value)``.
    2. By looking for other Python packages that are installed and provide
       a setuptools entry point group.

    When you create an instance of this claass, provide the name of the
    entry point group to use::

        reg = PluginRegister('my_entrypoint_group')

    """

    def __init__(self, entry_point_group: str = '', plugin_type=object) -> None:
        """Create a PluginRegistry for a named entry point group.

        Parameters
        ==========
        entry_point_group: str
            The name of the entry point group.
        plugin_type: object
            A type that will optionally be used for runtime type checking of
            loaded plugins using isinstance.
        """
        self.entry_point_group = entry_point_group
        self.plugin_type = plugin_type
        self._active = None     # type: None
        self._plugins = {}      # type: dict
        self._options = {}      # type: dict

    def register(self, name: str, value: Union[PluginType,None]) -> PluginType:
        """Register a plugin by name and value.

        This method is used for explicit registration of a plugin and shouldn't be
        used to manage entry point managed plugins, which are auto-loaded.

        Parameters
        ==========
        name: str
            The name of the plugin.
        value: PluginType or None
            The actual plugin object to register or None to unregister that plugin.

        Returns
        =======
        plugin: PluginType
            The plugin that was registered or unregistered.
        """
        if value is None and name in self._plugins:
            return self._plugins.pop(name)
        else:
            assert isinstance(value, self.plugin_type)
            self._plugins[name] = value
            return value

    def names(self) -> List[str]:
        """List the names of the registered and entry points plugins."""
        exts = list(self._plugins.keys())
        more_exts = [ep.name for ep in entrypoints.get_group_all(self.entry_point_group)]
        exts.extend(more_exts)
        return exts

    def enable(self, name: str) -> None:
        """Enable a plugin by name."""
        if name not in self._plugins:
            ep = entrypoints.get_single(self.entry_point_group, name)
            value = cast(PluginType, ep.load())
            assert isinstance(value, self.plugin_type)
            self.register(name, value)
        self._active = self._plugins[name]

    def get(self) -> PluginType:
        """Return the currently active plugin."""
        return self._active

<|endoftext|>


<commit_before>import numpy as np
import random
#
# Compute EM algorithm
#  uses helper functions
#

class BernoulliMixture:
    """
    Bernoulli Mixture model with an EM solver.
    """
    def __init__(self,num_mix,data_mat,init_type='unif_rand',
                 opt_type='expected'):
        self.num_mix = num_mix
        self.num_data = data_mat.shape[0]
        self.data_shape = data_mat.shape[1:]
        # flatten data to just be binary vectors
        self.data_length = np.prod(data_mat.shape[1:])
        self.data_mat = data_mat.reshape(self.num_data, self.data_length)

        # initializing weights
        self.weights = 1./num_mix * np.ones(num_mix)
        self.opt_type=opt_type
        self.init_affinities_templates(init_type)

    def run_EM(self,tol,save_template=False):
        """ EM algorithm
        First we compute the expected value of the label vector for each point
        """
        loglikelihood = -np.inf
        # First E step plus likelihood computation
        new_loglikelihood = self.compute_loglikelihoods()
        while new_loglikelihood - loglikelihood > tol:
            loglikelihood = new_loglikelihood
            # M-step
            self.M_step()
            # E-step
            new_loglikelihood = self.compute_loglikelihoods()
        self.set_templates()

    def M_step(self):
        self.weights = np.mean(self.affinities,axis=0)
        for mix_id in xrange(self.num_mix):
            self.work_templates[mix_id] = np.sum(self.data_mat * np.tile(self.affinities[:,mix_id],
                                                                    (self.data_length,1)).transpose(),
                                            axis=0)
            self.work_templates[mix_id] /= (self.weights[mix_id] * self.num_data)
        self.threshold_templates()
        self.log_templates = np.log(self.work_templates)
        self.log_invtemplates = np.log(1-self.work_templates)

    def threshold_templates(self):
        self.work_templates = np.maximum(np.minimum(self.work_templates,.95),.05)

    def init_affinities_templates(self,init_type):
        if init_type == 'unif_rand':
            random.seed()
            idx = range(self.num_data)
            random.shuffle(idx)
            self.affinities = np.zeros((self.num_data,
                                        self.num_mix))
            self.work_templates = np.zeros((self.num_mix,
                                       self.data_length))
            for mix_id in xrange(self.num_mix):
                self.affinities[self.num_mix*np.arange(self.num_data/self.num_mix)+mix_id,mix_id] = 1.
                self.work_templates[mix_id] = np.mean(self.data_mat[self.affinities[:,mix_id]==1],axis=0)
                self.threshold_templates()
        elif init_type == 'specific':
            random.seed()
            idx = range(self.num_data)
            random.shuffle(idx)
            self.affinities = np.zeros((self.num_data,
                                        self.num_mix))
            self.work_templates = np.zeros((self.num_mix,
                                       self.data_length))
            for mix_id in xrange(self.num_mix):
                self.affinities[self.num_mix*np.arange(self.num_data/self.num_mix)[1]+mix_id,mix_id] = 1.
                self.work_templates[mix_id] = np.mean(self.data_mat[self.affinities[:,mix_id]==1],axis=0)
                self.threshold_templates()

        self.log_templates = np.log(self.work_templates)
        self.log_invtemplates = np.log(1-self.work_templates)

    def get_templates(self):
        return self.templates

    def init_templates(self):
        self.work_templates = np.zeros((self.num_mix,
                                   self.data_length))
        self.templates = np.zeros((self.num_mix,
                                   self.data_length))

    def set_templates(self):
        self.templates = self.work_templates.reshape((self.num_mix,)+self.data_shape)

    def get_num_mix(self):
        return self.num_mix

    def get_weights(self):
        return self.weights

    def set_weights(self,new_weights):
        np.testing.assert_approx_equal(np.sum(new_weights),1.)
        assert(new_weights.shape==(self.num_mix,))
        self.weights = new_weights

    def compute_loglikelihoods(self):
        template_logscores = self.get_template_loglikelihoods(self.data_mat)
        loglikelihoods = template_logscores + np.tile(np.log(self.weights),(self.num_data,1))
        max_vals = np.amax(loglikelihoods,axis=1)
        # adjust the marginals by a value to avoid numerical
        # problems
        logmarginals_adj = np.sum(np.exp(loglikelihoods - np.tile(max_vals,(self.num_mix,1)).transpose()),axis=1)
        loglikelihood = np.sum(np.log(logmarginals_adj)) + np.sum(max_vals)
        self.affinities = np.exp(loglikelihoods-np.tile(logmarginals_adj+max_vals,
                                           (self.num_mix,1)).transpose())
        self.affinities/=np.tile(np.sum(self.affinities,axis=1),(self.num_mix,1)).transpose()
        return loglikelihood

    def get_template_loglikelihood(self,datum):
        return np.sum(self.log_templates[:,datum > .5],\
                           axis=1) +\
                           np.sum(self.log_invtemplates[:,datum < .5],\
                                       axis=1)

    def get_template_loglikelihoods(self,data_mat):
        """ Assumed to be called whenever
        """
        return np.array(map(lambda x:\
                                self.get_template_loglikelihood(x),\
                                data_mat))

    def set_template_vec_likelihoods(self):
        pass

<commit_msg>A simplification and speed improvement of BernoulliMixture.<commit_after>import numpy as np
import random
#
# Compute EM algorithm
#  uses helper functions
#

class BernoulliMixture:
    """
    Bernoulli Mixture model with an EM solver.
    """
    def __init__(self,num_mix,data_mat,init_type='unif_rand',
                 opt_type='expected'):
        self.num_mix = num_mix
        self.num_data = data_mat.shape[0]
        self.data_shape = data_mat.shape[1:]
        # flatten data to just be binary vectors
        self.data_length = np.prod(data_mat.shape[1:])
        self.data_mat = data_mat.reshape(self.num_data, self.data_length)

        # initializing weights
        self.weights = 1./num_mix * np.ones(num_mix)
        self.opt_type=opt_type
        self.init_affinities_templates(init_type)

    # TODO: save_template never used!
    def run_EM(self,tol,save_template=False):
        """ EM algorithm
        First we compute the expected value of the label vector for each point
        """
        loglikelihood = -np.inf
        # First E step plus likelihood computation
        new_loglikelihood = self.compute_loglikelihoods()
        while new_loglikelihood - loglikelihood > tol:
            loglikelihood = new_loglikelihood
            # M-step
            self.M_step()
            # E-step
            new_loglikelihood = self.compute_loglikelihoods()
        self.set_templates()

    def M_step(self):
        self.weights = np.mean(self.affinities,axis=0)
        for mix_id in xrange(self.num_mix):
            self.work_templates[mix_id] = np.sum(self.data_mat * np.tile(self.affinities[:,mix_id],
                                                                    (self.data_length,1)).transpose(),
                                            axis=0)
            self.work_templates[mix_id] /= (self.weights[mix_id] * self.num_data)
        self.threshold_templates()
        self.log_templates = np.log(self.work_templates)
        self.log_invtemplates = np.log(1-self.work_templates)

    def threshold_templates(self):
        self.work_templates = np.clip(self.work_templates, 0.05, 0.95)

    def init_affinities_templates(self,init_type):
        if init_type == 'unif_rand':
            random.seed()
            idx = range(self.num_data)
            random.shuffle(idx)
            self.affinities = np.zeros((self.num_data,
                                        self.num_mix))
            self.work_templates = np.zeros((self.num_mix,
                                       self.data_length))
            for mix_id in xrange(self.num_mix):
                self.affinities[self.num_mix*np.arange(self.num_data/self.num_mix)+mix_id,mix_id] = 1.
                self.work_templates[mix_id] = np.mean(self.data_mat[self.affinities[:,mix_id]==1],axis=0)
                self.threshold_templates()
        elif init_type == 'specific':
            random.seed()
            idx = range(self.num_data)
            random.shuffle(idx)
            self.affinities = np.zeros((self.num_data,
                                        self.num_mix))
            self.work_templates = np.zeros((self.num_mix,
                                       self.data_length))
            for mix_id in xrange(self.num_mix):
                self.affinities[self.num_mix*np.arange(self.num_data/self.num_mix)[1]+mix_id,mix_id] = 1.
                self.work_templates[mix_id] = np.mean(self.data_mat[self.affinities[:,mix_id]==1],axis=0)
                self.threshold_templates()

        self.log_templates = np.log(self.work_templates)
        self.log_invtemplates = np.log(1-self.work_templates)

    def get_templates(self):
        return self.templates

    def init_templates(self):
        self.work_templates = np.zeros((self.num_mix,
                                   self.data_length))
        self.templates = np.zeros((self.num_mix,
                                   self.data_length))

    def set_templates(self):
        self.templates = self.work_templates.reshape((self.num_mix,)+self.data_shape)

    def get_num_mix(self):
        return self.num_mix

    def get_weights(self):
        return self.weights

    def set_weights(self,new_weights):
        np.testing.assert_approx_equal(np.sum(new_weights),1.)
        assert(new_weights.shape==(self.num_mix,))
        self.weights = new_weights

    def compute_loglikelihoods(self):
        template_logscores = self.get_template_loglikelihoods(self.data_mat)
        loglikelihoods = template_logscores + np.tile(np.log(self.weights),(self.num_data,1))
        max_vals = np.amax(loglikelihoods,axis=1)
        # adjust the marginals by a value to avoid numerical
        # problems
        logmarginals_adj = np.sum(np.exp(loglikelihoods - np.tile(max_vals,(self.num_mix,1)).transpose()),axis=1)
        loglikelihood = np.sum(np.log(logmarginals_adj)) + np.sum(max_vals)
        self.affinities = np.exp(loglikelihoods-np.tile(logmarginals_adj+max_vals,
                                           (self.num_mix,1)).transpose())
        self.affinities/=np.tile(np.sum(self.affinities,axis=1),(self.num_mix,1)).transpose()
        return loglikelihood

    def get_template_loglikelihoods(self,data_mat):
        """ Assumed to be called whenever
        """
        return np.dot(data_mat, self.log_templates.T) + \
               np.dot(1-data_mat, self.log_invtemplates.T)

    def set_template_vec_likelihoods(self):
        pass

    def save(self, filename, save_affinities=False):
        """
        Save mixture components to a numpy npz file.
        """
        entries = dict(templates=self.templates, weights=self.weights)
        if save_affinities:
            entries['affinities'] = self.affinities
        np.savez(filename, **entries)
<|endoftext|>


<commit_before>from amqpstorm.management.base import ManagementHandler

API_CHANNEL = 'channels/%s'
API_CHANNELS = 'channels'

class Channel(ManagementHandler):
    def get(self, channel):
        """Get Connection details.

        :param channel: Channel name

        :raises ApiError: Raises if the remote server encountered an error.
                          We also raise an exception if the channel cannot
                          be found.

        :rtype: dict
        """
        return self.http_client.get(API_CHANNEL % channel)

    def list(self, name=None, page_size=None, use_regex=False):
        """List all Channels.

        :param name: Filter by name
        :param use_regex: Enables regular expression for the param name
        :param page_size: Number of elements per page

        :raises ApiError: Raises if the remote server encountered an error.
        :raises ApiConnectionError: Raises if there was a connectivity issue.

        :rtype: list
        """
        return self.http_client.list(
            API_CHANNELS,
            name=name, use_regex=use_regex, page_size=page_size,
        )
<commit_msg>Fixed minor typo<commit_after>from amqpstorm.management.base import ManagementHandler

API_CHANNEL = 'channels/%s'
API_CHANNELS = 'channels'

class Channel(ManagementHandler):
    def get(self, channel):
        """Get Connection details.

        :param channel: Channel name

        :raises ApiError: Raises if the remote server encountered an error.
                          We also raise an exception if the channel cannot
                          be found.
        :raises ApiConnectionError: Raises if there was a connectivity issue.

        :rtype: dict
        """
        return self.http_client.get(API_CHANNEL % channel)

    def list(self, name=None, page_size=None, use_regex=False):
        """List all Channels.

        :param name: Filter by name
        :param use_regex: Enables regular expression for the param name
        :param page_size: Number of elements per page

        :raises ApiError: Raises if the remote server encountered an error.
        :raises ApiConnectionError: Raises if there was a connectivity issue.

        :rtype: list
        """
        return self.http_client.list(
            API_CHANNELS,
            name=name, use_regex=use_regex, page_size=page_size,
        )
<|endoftext|>


<commit_before>from threading import Thread
from irc import client as irc
from queue import Empty
from time import sleep
import logging
from relaygram import events

class IRCHandler:
    def __init__(self, channel_map, config, my_queue, out_queues):
        self.log = logging.getLogger("relaygram.irc")
        self.channel_map = channel_map
        self.config = config
        self.my_queue = my_queue
        self.out_queues = out_queues

        self.initalized_servers = []

        # Build up IRC connections
        self.irc = irc.Reactor()

        # Message Events
        self.irc.add_global_handler("pubmsg", handler=self.irc_pubmsg)
        self.irc.add_global_handler("topic", handler=self.irc_topic)
        self.irc.add_global_handler("action", handler=self.irc_action)
        self.irc.add_global_handler("join", handler=self.irc_join)
        self.irc.add_global_handler("part", handler=self.irc_part)
        self.irc.add_global_handler("quit", handler=self.irc_quit)
        self.irc.add_global_handler("kick", handler=self.irc_kick)

        # System Events
        self.irc.add_global_handler("namreply", handler=self.irc_namreply)
        self.irc.add_global_handler("disconnect", handler=self.irc_disconnect)
        self.irc.add_global_handler("nicknameinuse", handler=self.irc_nicknameinuse)
        self.irc.add_global_handler("umode", handler=self.irc_umode)

        self.irc_servers = {}
        self.irc_channels = {}
        for server_name, server_params in self.config['irc']['servers'].items():
            self.initialize_server(server_params)

        self.thread = Thread(target=self.main_loop)

    def initialize_server(self, server_params):
        irc_server = self.irc.server()
        irc_server.connect(server_params['hostname'], server_params['port'], server_params['nickname'])
        self.irc_channels[irc_server.server] = {}
        for channel in server_params['channels']:
            self.irc_channels[irc_server.server][channel] = set()

        self.irc_servers[server_params['hostname']] = irc_server

    def irc_umode(self, connection, event):
        #  Set when server connection is finished, some servers don't like early join messages.
        if connection not in self.initalized_servers:
            for channel in self.irc_channels[connection.server].keys():
                connection.join(channel)
        self.initalized_servers.append(connection)

    def run(self):
        self.thread.start()
        return self

    def main_loop(self):
        while True:
            self.irc.process_once()
            self.process_queue_once()
            sleep(.1)

    def process_queue_once(self):
        try:
            self.process_event(self.my_queue.get_nowait())
        except Empty:
            pass

    def process_event(self, event):
        src_id = event.src[1]
        [server, channel] = self.channel_map.get_dest("tg", src_id).split(":")
        irc_config = self.config['irc']

        if event.type is events.Message:
            msg = irc_config['message_pattern'].format(nick=event.user, msg=event.msg)
        elif event.type is events.Join:
            msg = irc_config['join_pattern'].format(nick=event.user, msg=event.msg)
        elif event.type is events.Part:
            msg = irc_config['part_pattern'].format(nick=event.user, msg=event.msg)
        elif event.type is events.Kick:
            msg = irc_config['kick_pattern'].format(nick=event.user, msg=event.msg)
        elif event.type is events.Topic:
            msg = irc_config['topic_pattern'].format(nick=event.user, msg=event.msg)
        elif event.type is events.Action:
            msg = irc_config['topic_pattern'].format(nick=event.user, msg=event.msg)
        else:
            msg = None

        if msg:
            self.log.info("Sending to irc: {msg}".format(msg=msg))
            for line in msg.splitlines():
                self.irc_servers[server].privmsg(channel, line)
                sleep(0.75)

    def irc_pubmsg(self, connection, event):
        item = events.Message(src=(connection.server, event.target), user=event.source.nick, msg=event.arguments[0])
        [queue.put_nowait(item) for queue in self.out_queues]

    def irc_topic(self, connection, event):
        item = events.Topic(src=(connection.server, event.target), user=event.source.nick, msg=event.arguments[0])
        [queue.put_nowait(item) for queue in self.out_queues]

    def irc_action(self, connection, event):
        item = events.Action(src=(connection.server, event.target), user=event.source.nick, msg=event.arguments[0])
        [queue.put_nowait(item) for queue in self.out_queues]

    def irc_join(self, connection, event):
        self.irc_channels[connection.server][event.target].add(event.source.nick)

        item = events.Join(src=(connection.server, event.target), user=event.source.nick)
        [queue.put_nowait(item) for queue in self.out_queues]

    def irc_part(self, connection, event):
        item = events.Part(src=(connection.server, event.target), user=event.source.nick)
        [queue.put_nowait(item) for queue in self.out_queues]

    def irc_quit(self, connection, event):
        for channel, nick_list in self.irc_channels[connection.server].items():
            if event.source.nick in nick_list:
                nick_list.discard(event.source.nick)
                item = events.Part(src=(connection.server, channel), user=event.source.nick)
                [queue.put_nowait(item) for queue in self.out_queues]

    def irc_kick(self, connection, event):
        item = events.Kick(src=(connection.server, event.target), user=event.source.nick, msg=event.arguments)
        [queue.put_nowait(item) for queue in self.out_queues]

    def irc_nicknameinuse(self, connection, event):
        self.log.warning("Nickname in use, using {}".format(connection.get_nickname() + "_"))
        connection.nick(connection.get_nickname() + "_")

        for channel in self.irc_channels[connection.server].keys():
            connection.join(channel)

    def irc_namreply(self, connection, event):
        ch_type, channel, nick_list = event.arguments

        for nick in nick_list.split():
            nick_modes = []

            if nick[0] in connection.features.prefix:
                nick_modes.append(connection.features.prefix[nick[0]])
                nick = nick[1:]

            self.irc_channels[connection.server][channel].add(nick)

    def irc_disconnect(self, connection, event):
        # TODO Reconnect
        self.log.error("Disconnected from {}".format(connection.server))
<commit_msg>Limit the length of IRC messages on one line<commit_after>from threading import Thread
from irc import client as irc
from queue import Empty
from time import sleep
import logging
from relaygram import events

class IRCHandler:
    def __init__(self, channel_map, config, my_queue, out_queues):
        self.log = logging.getLogger("relaygram.irc")
        self.channel_map = channel_map
        self.config = config
        self.my_queue = my_queue
        self.out_queues = out_queues

        self.initalized_servers = []

        # Build up IRC connections
        self.irc = irc.Reactor()

        # Message Events
        self.irc.add_global_handler("pubmsg", handler=self.irc_pubmsg)
        self.irc.add_global_handler("topic", handler=self.irc_topic)
        self.irc.add_global_handler("action", handler=self.irc_action)
        self.irc.add_global_handler("join", handler=self.irc_join)
        self.irc.add_global_handler("part", handler=self.irc_part)
        self.irc.add_global_handler("quit", handler=self.irc_quit)
        self.irc.add_global_handler("kick", handler=self.irc_kick)

        # System Events
        self.irc.add_global_handler("namreply", handler=self.irc_namreply)
        self.irc.add_global_handler("disconnect", handler=self.irc_disconnect)
        self.irc.add_global_handler("nicknameinuse", handler=self.irc_nicknameinuse)
        self.irc.add_global_handler("umode", handler=self.irc_umode)

        self.irc_servers = {}
        self.irc_channels = {}
        for server_name, server_params in self.config['irc']['servers'].items():
            self.initialize_server(server_params)

        self.thread = Thread(target=self.main_loop)

    def initialize_server(self, server_params):
        irc_server = self.irc.server()
        irc_server.connect(server_params['hostname'], server_params['port'], server_params['nickname'])
        self.irc_channels[irc_server.server] = {}
        for channel in server_params['channels']:
            self.irc_channels[irc_server.server][channel] = set()

        self.irc_servers[server_params['hostname']] = irc_server

    def irc_umode(self, connection, event):
        #  Set when server connection is finished, some servers don't like early join messages.
        if connection not in self.initalized_servers:
            for channel in self.irc_channels[connection.server].keys():
                connection.join(channel)
        self.initalized_servers.append(connection)

    def run(self):
        self.thread.start()
        return self

    def main_loop(self):
        while True:
            self.irc.process_once()
            self.process_queue_once()
            sleep(.1)

    def process_queue_once(self):
        try:
            self.process_event(self.my_queue.get_nowait())
        except Empty:
            pass

    def process_event(self, event):
        src_id = event.src[1]
        [server, channel] = self.channel_map.get_dest("tg", src_id).split(":")
        irc_config = self.config['irc']

        if event.type is events.Message:
            msg = irc_config['message_pattern'].format(nick=event.user, msg=event.msg)
        elif event.type is events.Join:
            msg = irc_config['join_pattern'].format(nick=event.user, msg=event.msg)
        elif event.type is events.Part:
            msg = irc_config['part_pattern'].format(nick=event.user, msg=event.msg)
        elif event.type is events.Kick:
            msg = irc_config['kick_pattern'].format(nick=event.user, msg=event.msg)
        elif event.type is events.Topic:
            msg = irc_config['topic_pattern'].format(nick=event.user, msg=event.msg)
        elif event.type is events.Action:
            msg = irc_config['topic_pattern'].format(nick=event.user, msg=event.msg)
        else:
            msg = None

        if msg:
            self.log.info("Sending to irc: {msg}".format(msg=msg))
            for line in msg.splitlines():
                for msg in [line[i:i+400] for i in range(0, len(line), 400)]:
                    self.irc_servers[server].privmsg(channel, msg)
                    sleep(0.75)

    def irc_pubmsg(self, connection, event):
        item = events.Message(src=(connection.server, event.target), user=event.source.nick, msg=event.arguments[0])
        [queue.put_nowait(item) for queue in self.out_queues]

    def irc_topic(self, connection, event):
        item = events.Topic(src=(connection.server, event.target), user=event.source.nick, msg=event.arguments[0])
        [queue.put_nowait(item) for queue in self.out_queues]

    def irc_action(self, connection, event):
        item = events.Action(src=(connection.server, event.target), user=event.source.nick, msg=event.arguments[0])
        [queue.put_nowait(item) for queue in self.out_queues]

    def irc_join(self, connection, event):
        self.irc_channels[connection.server][event.target].add(event.source.nick)

        item = events.Join(src=(connection.server, event.target), user=event.source.nick)
        [queue.put_nowait(item) for queue in self.out_queues]

    def irc_part(self, connection, event):
        item = events.Part(src=(connection.server, event.target), user=event.source.nick)
        [queue.put_nowait(item) for queue in self.out_queues]

    def irc_quit(self, connection, event):
        for channel, nick_list in self.irc_channels[connection.server].items():
            if event.source.nick in nick_list:
                nick_list.discard(event.source.nick)
                item = events.Part(src=(connection.server, channel), user=event.source.nick)
                [queue.put_nowait(item) for queue in self.out_queues]

    def irc_kick(self, connection, event):
        item = events.Kick(src=(connection.server, event.target), user=event.source.nick, msg=event.arguments)
        [queue.put_nowait(item) for queue in self.out_queues]

    def irc_nicknameinuse(self, connection, event):
        self.log.warning("Nickname in use, using {}".format(connection.get_nickname() + "_"))
        connection.nick(connection.get_nickname() + "_")

        for channel in self.irc_channels[connection.server].keys():
            connection.join(channel)

    def irc_namreply(self, connection, event):
        ch_type, channel, nick_list = event.arguments

        for nick in nick_list.split():
            nick_modes = []

            if nick[0] in connection.features.prefix:
                nick_modes.append(connection.features.prefix[nick[0]])
                nick = nick[1:]

            self.irc_channels[connection.server][channel].add(nick)

    def irc_disconnect(self, connection, event):
        # TODO Reconnect
        self.log.error("Disconnected from {}".format(connection.server))
<|endoftext|>