<commit_before>from inbox.basicauth import NotSupportedError

__all__ = ['provider_info', 'providers']

def provider_info(provider_name):
    if provider_name not in providers:
        raise NotSupportedError("Provider: {} not supported.".format(
            provider_name))

    return providers[provider_name]

providers = {
    "gmail": {
        "imap": "imap.gmail.com",
        "smtp": "smtp.gmail.com:587",
        "auth": "oauth2",
        "mx_servers": ["aspmx.l.google.com",
                       "aspmx2.googlemail.com",
                       "aspmx3.googlemail.com",
                       "aspmx4.googlemail.com",
                       "aspmx5.googlemail.com",
                       "alt1.aspmx.l.google.com",
                       "alt2.aspmx.l.google.com",
                       "alt3.aspmx.l.google.com",
                       "alt4.aspmx.l.google.com",
                       "aspmx1.aspmx.l.google.com",
                       "aspmx2.aspmx.l.google.com",
                       "aspmx3.aspmx.l.google.com",
                       "aspmx4.aspmx.l.google.com",
                       "gmail-smtp-in.l.google.com",
                       "alt1.gmail-smtp-in.l.google.com",
                       "alt2.gmail-smtp-in.l.google.com",
                       "alt3.gmail-smtp-in.l.google.com",
                       "alt4.gmail-smtp-in.l.google.com"],
    },
    "yahoo": {
        "type": "generic",
        "imap": "imap.mail.yahoo.com",
        "smtp": "smtp.mail.yahoo.com:587",
        "auth": "password",
        "folder_map": {"Draft": "drafts", "Bulk Mail": "spam", "Inbox":
                       "inbox", "Sent": "sent", "Trash": "trash"},
        "domains": ["yahoo.com.ar", "yahoo.com.au", "yahoo.at", "yahoo.be",
                    "yahoo.fr", "yahoo.be", "yahoo.nl", "yahoo.com.br",
                    "yahoo.ca", "yahoo.en", "yahoo.ca", "yahoo.fr",
                    "yahoo.com.cn", "yahoo.cn", "yahoo.com.co", "yahoo.cz",
                    "yahoo.dk", "yahoo.fi", "yahoo.fr", "yahoo.de", "yahoo.gr",
                    "yahoo.com.hk", "yahoo.hu", "yahoo.co.in", "yahoo.in",
                    "yahoo.ie", "yahoo.co.il", "yahoo.it", "yahoo.co.jp",
                    "yahoo.com.my", "yahoo.com.mx", "yahoo.ae", "yahoo.nl",
                    "yahoo.co.nz", "yahoo.no", "yahoo.com.ph", "yahoo.pl",
                    "yahoo.pt", "yahoo.ro", "yahoo.ru", "yahoo.com.sg",
                    "yahoo.co.za", "yahoo.es", "yahoo.se", "yahoo.ch",
                    "yahoo.fr", "yahoo.ch", "yahoo.de", "yahoo.com.tw",
                    "yahoo.co.th", "yahoo.com.tr", "yahoo.co.uk", "yahoo.com",
                    "yahoo.com.vn", "ymail.com", "rocketmail.com"],
        "mx_servers": ["mx-biz.mail.am0.yahoodns.net",
                       "mx1.biz.mail.yahoo.com.", "mx5.biz.mail.yahoo.com.",
                       "mxvm2.mail.yahoo.com.", "mx-van.mail.am0.yahoodns.net"],
    },
    "outlook": {
        "type": "generic",
        "imap": "imap-mail.outlook.com",
        "smtp": "smtp.live.com:587",
        "auth": "oauth2",
        "folder_map": {"Junk": "spam", "Inbox": "inbox", "Sent": "sent",
                       "Trash": "trash", "Drafts": "drafts"},
        "domains": ["hotmail.com", "outlook.com", "outlook.com.ar",
                    "outlook.com.au", "outlook.at", "outlook.be",
                    "outlook.com.br", "outlook.cl", "outlook.cz", "outlook.dk",
                    "outlook.fr", "outlook.de", "outlook.com.gr",
                    "outlook.co.il", "outlook.in", "outlook.co.id",
                    "outlook.ie", "outlook.it", "outlook.hu", "outlook.jp",
                    "outlook.kr", "outlook.lv", "outlook.my", "outlook.co.nz",
                    "outlook.com.pe", "outlook.ph", "outlook.pt", "outlook.sa",
                    "outlook.sg", "outlook.sk", "outlook.es", "outlook.co.th",
                    "outlook.com.tr", "outlook.com.vn"],
    },
    "aol": {
        "type": "generic",
        "imap": "imap.aol.com",
        "smtp": "smtp.aol.com:587",
        "auth": "password",
        "folder_map": {"Spam": "spam", "INBOX": "inbox", "Sent": "sent",
                       "Trash": "trash", "Drafts": "drafts"},
        "domains": ["aol.com"],
    },
    "eas": {
        "auth": "password",
        "domains": [
            "onmicrosoft.com",
            "exchange.mit.edu"
            ],
    },
    "fastmail": {
        "type": "generic",
        "condstore": True,
        "imap": "mail.messagingengine.com",
        "smtp": "mail.messagingengine.com:587",
        "auth": "password",
        "folder_map": {"INBOX": "inbox", "INBOX.Archive": "archive",
                       "INBOX.Drafts": "drafts", "INBOX.Junk Mail": "spam",
                       "INBOX.Sent Items": "sent", "INBOX.Trash": "trash"},
        "domains": ["fastmail.fm"],
        "mx_servers": ["1-smtp.messagingengine.com",
                       "in2-smtp.messagingengine.com"],
        "ns_servers": ["ns1.messagingengine.com",
                       "ns2.messagingengine.com"],
    },
    "icloud": {
        "type": "generic",
        "imap": "imap.mail.me.com",
        "smtp": "smtp.mail.me.com:587",
        "auth": "password",
        "folder_map": {"INBOX": "inbox", "Archive": "archive",
                       "Drafts": "drafts", "Junk": "spam",
                       "Sent Messages": "sent",
                       "Deleted Messages": "trash"},
        "domains": ["icloud.com"],
    },
    "gmx": {
        "type": "generic",
        "imap": "imap.gmx.com",
        "smtp": "smtp.gmx.com:587",
        "auth": "password",
        "folder_map": {"INBOX": "inbox", "Drafts": "drafts", "Spam": "spam",
                       "Sent": "sent", "Trash": "trash"},
        "domains": ["gmx.us", "gmx.com"],
    },
    "gandi": {
        "type": "generic",
        "condstore": True,
        "imap": "mail.gandi.net",
        "smtp": "mail.gandi.net:587",
        "auth": "password",
        "folder_map": {"INBOX": "inbox"},
        "domains": ["debuggers.co"],
        "mx_servers": ["spool.mail.gandi.net", "fb.mail.gandi.net"],
    },
    "zimbra": {
        "type": "generic",
        "imap": "mail.you-got-mail.com",
        "smtp": "smtp.you-got-mail.com:587",
        "auth": "password",
        "folder_map": {"INBOX": "inbox", "Drafts": "drafts", "Junk": "spam",
                       "Sent": "sent", "Trash": "trash"},
        "domains": ["mrmail.com"],
    },
    "mail.ru": {
        "type": "generic",
        "imap": "imap.mail.ru",
        "smtp": "smtp.mail.ru:587",
        "auth": "password",
        "folder_map": {"INBOX": "inbox",
                       "\u0427\u0435\u0440\u043d\u043e\u0432\u0438\u043a\u0438": "drafts",
                       "\u0421\u043f\u0430\u043c": "spam",
                       "\u041e\u0442\u043f\u0440\u0430\u0432\u043b\u0435\u043d\u043d\u044b\u0435": "sent",
                       "\u041a\u043e\u0440\u0437\u0438\u043d\u0430": "trash"},
        "domains": ["mail.ru"],
    }
}
<commit_msg>Fix gandi folder mapping.<commit_after>from inbox.basicauth import NotSupportedError

__all__ = ['provider_info', 'providers']

def provider_info(provider_name):
    if provider_name not in providers:
        raise NotSupportedError("Provider: {} not supported.".format(
            provider_name))

    return providers[provider_name]

providers = {
    "gmail": {
        "imap": "imap.gmail.com",
        "smtp": "smtp.gmail.com:587",
        "auth": "oauth2",
        "mx_servers": ["aspmx.l.google.com",
                       "aspmx2.googlemail.com",
                       "aspmx3.googlemail.com",
                       "aspmx4.googlemail.com",
                       "aspmx5.googlemail.com",
                       "alt1.aspmx.l.google.com",
                       "alt2.aspmx.l.google.com",
                       "alt3.aspmx.l.google.com",
                       "alt4.aspmx.l.google.com",
                       "aspmx1.aspmx.l.google.com",
                       "aspmx2.aspmx.l.google.com",
                       "aspmx3.aspmx.l.google.com",
                       "aspmx4.aspmx.l.google.com",
                       "gmail-smtp-in.l.google.com",
                       "alt1.gmail-smtp-in.l.google.com",
                       "alt2.gmail-smtp-in.l.google.com",
                       "alt3.gmail-smtp-in.l.google.com",
                       "alt4.gmail-smtp-in.l.google.com"],
    },
    "yahoo": {
        "type": "generic",
        "imap": "imap.mail.yahoo.com",
        "smtp": "smtp.mail.yahoo.com:587",
        "auth": "password",
        "folder_map": {"Draft": "drafts", "Bulk Mail": "spam", "Inbox":
                       "inbox", "Sent": "sent", "Trash": "trash"},
        "domains": ["yahoo.com.ar", "yahoo.com.au", "yahoo.at", "yahoo.be",
                    "yahoo.fr", "yahoo.be", "yahoo.nl", "yahoo.com.br",
                    "yahoo.ca", "yahoo.en", "yahoo.ca", "yahoo.fr",
                    "yahoo.com.cn", "yahoo.cn", "yahoo.com.co", "yahoo.cz",
                    "yahoo.dk", "yahoo.fi", "yahoo.fr", "yahoo.de", "yahoo.gr",
                    "yahoo.com.hk", "yahoo.hu", "yahoo.co.in", "yahoo.in",
                    "yahoo.ie", "yahoo.co.il", "yahoo.it", "yahoo.co.jp",
                    "yahoo.com.my", "yahoo.com.mx", "yahoo.ae", "yahoo.nl",
                    "yahoo.co.nz", "yahoo.no", "yahoo.com.ph", "yahoo.pl",
                    "yahoo.pt", "yahoo.ro", "yahoo.ru", "yahoo.com.sg",
                    "yahoo.co.za", "yahoo.es", "yahoo.se", "yahoo.ch",
                    "yahoo.fr", "yahoo.ch", "yahoo.de", "yahoo.com.tw",
                    "yahoo.co.th", "yahoo.com.tr", "yahoo.co.uk", "yahoo.com",
                    "yahoo.com.vn", "ymail.com", "rocketmail.com"],
        "mx_servers": ["mx-biz.mail.am0.yahoodns.net",
                       "mx1.biz.mail.yahoo.com.", "mx5.biz.mail.yahoo.com.",
                       "mxvm2.mail.yahoo.com.", "mx-van.mail.am0.yahoodns.net"],
    },
    "outlook": {
        "type": "generic",
        "imap": "imap-mail.outlook.com",
        "smtp": "smtp.live.com:587",
        "auth": "oauth2",
        "folder_map": {"Junk": "spam", "Inbox": "inbox", "Sent": "sent",
                       "Trash": "trash", "Drafts": "drafts"},
        "domains": ["hotmail.com", "outlook.com", "outlook.com.ar",
                    "outlook.com.au", "outlook.at", "outlook.be",
                    "outlook.com.br", "outlook.cl", "outlook.cz", "outlook.dk",
                    "outlook.fr", "outlook.de", "outlook.com.gr",
                    "outlook.co.il", "outlook.in", "outlook.co.id",
                    "outlook.ie", "outlook.it", "outlook.hu", "outlook.jp",
                    "outlook.kr", "outlook.lv", "outlook.my", "outlook.co.nz",
                    "outlook.com.pe", "outlook.ph", "outlook.pt", "outlook.sa",
                    "outlook.sg", "outlook.sk", "outlook.es", "outlook.co.th",
                    "outlook.com.tr", "outlook.com.vn"],
    },
    "aol": {
        "type": "generic",
        "imap": "imap.aol.com",
        "smtp": "smtp.aol.com:587",
        "auth": "password",
        "folder_map": {"Spam": "spam", "INBOX": "inbox", "Sent": "sent",
                       "Trash": "trash", "Drafts": "drafts"},
        "domains": ["aol.com"],
    },
    "eas": {
        "auth": "password",
        "domains": [
            "onmicrosoft.com",
            "exchange.mit.edu"
            ],
    },
    "fastmail": {
        "type": "generic",
        "condstore": True,
        "imap": "mail.messagingengine.com",
        "smtp": "mail.messagingengine.com:587",
        "auth": "password",
        "folder_map": {"INBOX": "inbox", "INBOX.Archive": "archive",
                       "INBOX.Drafts": "drafts", "INBOX.Junk Mail": "spam",
                       "INBOX.Sent Items": "sent", "INBOX.Trash": "trash"},
        "domains": ["fastmail.fm"],
        "mx_servers": ["1-smtp.messagingengine.com",
                       "in2-smtp.messagingengine.com"],
        "ns_servers": ["ns1.messagingengine.com",
                       "ns2.messagingengine.com"],
    },
    "icloud": {
        "type": "generic",
        "imap": "imap.mail.me.com",
        "smtp": "smtp.mail.me.com:587",
        "auth": "password",
        "folder_map": {"INBOX": "inbox", "Archive": "archive",
                       "Drafts": "drafts", "Junk": "spam",
                       "Sent Messages": "sent",
                       "Deleted Messages": "trash"},
        "domains": ["icloud.com"],
    },
    "gmx": {
        "type": "generic",
        "imap": "imap.gmx.com",
        "smtp": "smtp.gmx.com:587",
        "auth": "password",
        "folder_map": {"INBOX": "inbox", "Drafts": "drafts", "Spam": "spam",
                       "Sent": "sent", "Trash": "trash"},
        "domains": ["gmx.us", "gmx.com"],
    },
    "gandi": {
        "type": "generic",
        "condstore": True,
        "imap": "mail.gandi.net",
        "smtp": "mail.gandi.net:587",
        "auth": "password",
        "folder_map": {"INBOX": "inbox", "Drafts": "drafts", "Junk": "spam",
                       "Sent": "sent", "Trash": "trash"},
        "domains": ["debuggers.co"],
        "mx_servers": ["spool.mail.gandi.net", "fb.mail.gandi.net"],
    },
    "zimbra": {
        "type": "generic",
        "imap": "mail.you-got-mail.com",
        "smtp": "smtp.you-got-mail.com:587",
        "auth": "password",
        "folder_map": {"INBOX": "inbox", "Drafts": "drafts", "Junk": "spam",
                       "Sent": "sent", "Trash": "trash"},
        "domains": ["mrmail.com"],
    },
    "mail.ru": {
        "type": "generic",
        "imap": "imap.mail.ru",
        "smtp": "smtp.mail.ru:587",
        "auth": "password",
        "folder_map": {"INBOX": "inbox",
                       "\u0427\u0435\u0440\u043d\u043e\u0432\u0438\u043a\u0438": "drafts",
                       "\u0421\u043f\u0430\u043c": "spam",
                       "\u041e\u0442\u043f\u0440\u0430\u0432\u043b\u0435\u043d\u043d\u044b\u0435": "sent",
                       "\u041a\u043e\u0440\u0437\u0438\u043d\u0430": "trash"},
        "domains": ["mail.ru"],
    }
}
<|endoftext|>


<commit_before># ogf4py3
# Copyright (C) 2017  Oscar Triano @cat_dotoscat

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from itertools import product, chain
from collections import deque
from . import toyblock3
from .component import Platform

@toyblock3.system("body")
def physics(system, entity, dt, gravity):
    entity.body.update(dt, gravity)

@toyblock3.system("body", "sprite")
def sprite(system, entity):
    body = entity.body
    entity.sprite.set_position(body.x, body.y)

@toyblock3.system("body", "collision")
def collision(system, entity):
    body = entity.body
    entity.collision.update(body.x, body.y)

class CheckCollision(toyblock3.System):
    """System for collisions. This is a naive implementation.

    With a :class:Collision component set the attribute *set* and then
    fills its *collides_with* the types, or groups, that you want collide with.

    Example:
        PLAYER = 1
        BALL = 1 << 1
        ENEMIES = 1 << 2

        Player.set(Collision, {"collides_with": BALL | ENEMIES, "type": PLAYER})
        Ball.set(Collision, {"type": BALL})

        my_collision = CheckCollision()
        my_collision.table[(PLAYER, BALL)] = player_ball_callback
    """
    def __init__(self):
        super().__init__("collision")
        self._table = {}
        self.callable = self._call

    @property
    def table(self):
        """Return a table where you assign a pair with its callable."""
        return self._table

    def _call(self, system, entity):
        entities = system.entities
        entity_collision = entity.collision
        table = self._table
        for sysentity in entities:
            if sysentity == entity: continue
            sysentity_collision = sysentity.collision
            if not entity_collision.collides_with & sysentity_collision.type_ == sysentity_collision.type_:
                continue
            if not entity_collision.intersects(sysentity_collision):
                continue
            table[(entity_collision.type_, sysentity_collision.type_)](entity, sysentity)

class PlatformCollision(toyblock3.System):
    def __init__(self):
        super().__init__("body", "platform")
        self.callable = self._call
        self.platforms = deque()
        self.walkers = deque()

    def _add_entity(self, entity):
        super()._add_entity(entity)
        if entity.platform.type_ == Platform.FOOT:
            self.walkers.append(entity)
        elif entity.platform.type_ == Platform.PLATFORM:
            self.platforms.append(entity)

    def _remove_entity(self, entity):
        super()._remove_entity(entity)
        if entity.platform.type_ == Platform.FOOT:
            self.walkers.remove(entity)
        elif entity.platform.type_ == Platform.PLATFORM:
            self.platforms.remove(entity)

    def _call(self, system, entity, callback=None):
        for platform in chain(self.platforms, self.walkers):
            platform.platform.update(platform.body.x, platform.body.y)

        for walker, platform in product(self.walkers, self.platforms):
            if walker.body.vel_y > 0.:
                continue
            walker.body.gravity = True
            touched = walker.platform.touch_floor
            walker.platform.platform = None
            if not walker.platform.intersects(platform.platform):
                continue
            walker.body.y += platform.platform.top - walker.platform.y
            walker.body.vel_y = 0.
            walker.body.gravity = False
            walker.platform.platform = platform
            if callable(callback) and not touched and walker.platform.touch_floor:
                callback(entity)
                break

@toyblock3.system("body", "floor_collision", "collision")
def platform_collision(system, entity, platforms, callback=None):
    """This system requires the next additional parameters

    *callback* accepts as parameter the entity.

    Parameters:
        platforms (iterable of platforms)
        callback (callable or None): callback when an entity touch the floor. If None, then is not triggered.
    """
    body = entity.body
    floor_collision = entity.floor_collision
    points = floor_collision.get_points(body.x, body.y)
    body.gravity = True
    touched = floor_collision.touch_floor
    floor_collision.platform = None
    for platform in platforms:
        platform_collision = platform.collision
        if (body.vel_y > 0.0 or
        (points[0] not in platform_collision and
        points[1] not in platform_collision)):
            continue
        body.y = platform_collision.top + -floor_collision.get_offset_y()
        body.vel_y = 0.0
        body.gravity = False
        floor_collision.platform = platform
        break
    if callable(callback) and not touched and floor_collision.touch_floor:
        callback(entity)

class AliveZone(toyblock3.System):
    """This is a basic system where the entities are freed if they are out of bounds.

    The parameters are relative to left and bottom.

    Parameters:
        x1 (float):
        y1 (float):
        x2 (float):
        y2 (float):

    Returns:
        An :class:AliveZone.

    Example:

        .. code-block:: python

            safezone = AliveZone(0., 0., SCREEN_WIDTH, SCREEN_HEIGHT)
            entity_systems = (safezone, physics, ...)
            #  ...
    """
    def __init__(self, x1, y1, x2, y2):
        super().__init__("body")
        self.callable = self._call
        self.x1 = x1
        self.y1 = y1
        self.x2 = x2
        self.y2 = y2

    def _call(self, system, entity):
        body = entity.body
        if not (self.x1 <= body.x <= self.x2 and self.y1 <= body.y <= self.y2):
            entity.free()

@toyblock3.system("timer")
def lifespan(system, entity, dt):
    timer = entity.timer
    timer.time += dt
    if timer.done:
        timer.time = 0.
        entity.free()
<commit_msg>Pasar walker en vez de entity<commit_after># ogf4py3
# Copyright (C) 2017  Oscar Triano @cat_dotoscat

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from itertools import product, chain
from collections import deque
from . import toyblock3
from .component import Platform

@toyblock3.system("body")
def physics(system, entity, dt, gravity):
    entity.body.update(dt, gravity)

@toyblock3.system("body", "sprite")
def sprite(system, entity):
    body = entity.body
    entity.sprite.set_position(body.x, body.y)

@toyblock3.system("body", "collision")
def collision(system, entity):
    body = entity.body
    entity.collision.update(body.x, body.y)

class CheckCollision(toyblock3.System):
    """System for collisions. This is a naive implementation.

    With a :class:Collision component set the attribute *set* and then
    fills its *collides_with* the types, or groups, that you want collide with.

    Example:
        PLAYER = 1
        BALL = 1 << 1
        ENEMIES = 1 << 2

        Player.set(Collision, {"collides_with": BALL | ENEMIES, "type": PLAYER})
        Ball.set(Collision, {"type": BALL})

        my_collision = CheckCollision()
        my_collision.table[(PLAYER, BALL)] = player_ball_callback
    """
    def __init__(self):
        super().__init__("collision")
        self._table = {}
        self.callable = self._call

    @property
    def table(self):
        """Return a table where you assign a pair with its callable."""
        return self._table

    def _call(self, system, entity):
        entities = system.entities
        entity_collision = entity.collision
        table = self._table
        for sysentity in entities:
            if sysentity == entity: continue
            sysentity_collision = sysentity.collision
            if not entity_collision.collides_with & sysentity_collision.type_ == sysentity_collision.type_:
                continue
            if not entity_collision.intersects(sysentity_collision):
                continue
            table[(entity_collision.type_, sysentity_collision.type_)](entity, sysentity)

class PlatformCollision(toyblock3.System):
    def __init__(self):
        super().__init__("body", "platform")
        self.callable = self._call
        self.platforms = deque()
        self.walkers = deque()

    def _add_entity(self, entity):
        super()._add_entity(entity)
        if entity.platform.type_ == Platform.FOOT:
            self.walkers.append(entity)
        elif entity.platform.type_ == Platform.PLATFORM:
            self.platforms.append(entity)

    def _remove_entity(self, entity):
        super()._remove_entity(entity)
        if entity.platform.type_ == Platform.FOOT:
            self.walkers.remove(entity)
        elif entity.platform.type_ == Platform.PLATFORM:
            self.platforms.remove(entity)

    def _call(self, system, entity, callback=None):
        for platform in chain(self.platforms, self.walkers):
            platform.platform.update(platform.body.x, platform.body.y)

        for walker, platform in product(self.walkers, self.platforms):
            if walker.body.vel_y > 0.:
                continue
            walker.body.gravity = True
            touched = walker.platform.touch_floor
            print(touched)
            walker.platform.platform = None
            if not walker.platform.intersects(platform.platform):
                continue
            walker.body.y += platform.platform.top - walker.platform.y
            walker.body.vel_y = 0.
            walker.body.gravity = False
            walker.platform.platform = platform
            if callable(callback) and not touched and walker.platform.touch_floor:
                callback(walker)
                break

@toyblock3.system("body", "floor_collision", "collision")
def platform_collision(system, entity, platforms, callback=None):
    """This system requires the next additional parameters

    *callback* accepts as parameter the entity.

    Parameters:
        platforms (iterable of platforms)
        callback (callable or None): callback when an entity touch the floor. If None, then is not triggered.
    """
    body = entity.body
    floor_collision = entity.floor_collision
    points = floor_collision.get_points(body.x, body.y)
    body.gravity = True
    touched = floor_collision.touch_floor
    floor_collision.platform = None
    for platform in platforms:
        platform_collision = platform.collision
        if (body.vel_y > 0.0 or
        (points[0] not in platform_collision and
        points[1] not in platform_collision)):
            continue
        body.y = platform_collision.top + -floor_collision.get_offset_y()
        body.vel_y = 0.0
        body.gravity = False
        floor_collision.platform = platform
        break
    if callable(callback) and not touched and floor_collision.touch_floor:
        callback(entity)

class AliveZone(toyblock3.System):
    """This is a basic system where the entities are freed if they are out of bounds.

    The parameters are relative to left and bottom.

    Parameters:
        x1 (float):
        y1 (float):
        x2 (float):
        y2 (float):

    Returns:
        An :class:AliveZone.

    Example:

        .. code-block:: python

            safezone = AliveZone(0., 0., SCREEN_WIDTH, SCREEN_HEIGHT)
            entity_systems = (safezone, physics, ...)
            #  ...
    """
    def __init__(self, x1, y1, x2, y2):
        super().__init__("body")
        self.callable = self._call
        self.x1 = x1
        self.y1 = y1
        self.x2 = x2
        self.y2 = y2

    def _call(self, system, entity):
        body = entity.body
        if not (self.x1 <= body.x <= self.x2 and self.y1 <= body.y <= self.y2):
            entity.free()

@toyblock3.system("timer")
def lifespan(system, entity, dt):
    timer = entity.timer
    timer.time += dt
    if timer.done:
        timer.time = 0.
        entity.free()
<|endoftext|>


<commit_before>import numpy as np
import cv2
from video import create_capture

if __name__ == '__main__':
    import sys, getopt

    args, video_src = getopt.getopt(sys.argv[1:], '', ['cascade=', 'nested-cascade='])
    try: video_src = video_src[0]
    except: video_src = '0'
    cascade_fn = args.get('--cascade', "../../data/haarcascades/haarcascade_frontalface_alt.xml")
    nested_fn  = args.get('--nested-cascade', "../../data/haarcascades/haarcascade_eye.xml.xml")

    cam = create_capture(video_src)

    while True:
        ret, img = cam.read()
        cv2.imshow('facedetect', img)

        if cv2.waitKey(5) == 27:
            break

<commit_msg>facedetect.py sample works<commit_after>import numpy as np
import cv2, cv
from video import create_capture

help_message = '''
USAGE: facedetect.py [--cascade <cascade_fn>] [--nested-cascade <cascade_fn>] [<video_source>]
'''

def detect(img, cascade):
    rects = cascade.detectMultiScale(img, scaleFactor=1.3, minNeighbors=4, minSize=(30, 30))
    if len(rects) == 0:
        return []
    rects[:,2:] += rects[:,:2]
    return rects

def draw_rects(img, rects, color):
    for x1, y1, x2, y2 in rects:
        cv2.rectangle(img, (x1, y1), (x2, y2), color)

if __name__ == '__main__':
    import sys, getopt
    print help_message

    args, video_src = getopt.getopt(sys.argv[1:], '', ['cascade=', 'nested-cascade='])
    try: video_src = video_src[0]
    except: video_src = 'synth:bg=../cpp/lena.jpg:noise=0.05'
    args = dict(args)
    cascade_fn = args.get('--cascade', "../../data/haarcascades/haarcascade_frontalface_alt.xml")
    nested_fn  = args.get('--nested-cascade', "../../data/haarcascades/haarcascade_eye.xml")

    cascade = cv2.CascadeClassifier(cascade_fn)
    nested = cv2.CascadeClassifier(nested_fn)

    cam = create_capture(video_src)

    while True:
        ret, img = cam.read()
        gray = cv2.cvtColor(img, cv.CV_BGR2GRAY)
        gray = cv2.equalizeHist(gray)
        rects = detect(gray, cascade)
        vis = cv2.cvtColor(gray, cv.CV_GRAY2BGR)
        draw_rects(vis, rects, (0, 255, 0))
        for x1, y1, x2, y2 in rects:
            roi = gray[y1:y2, x1:x2]
            vis_roi = vis[y1:y2, x1:x2]
            subrects = detect(roi.copy(), nested)
            draw_rects(vis_roi, subrects, (255, 0, 0))

        cv2.imshow('facedetect', vis)

        if cv2.waitKey(5) == 27:
            break

<|endoftext|>


<commit_before>import unittest
import random, sys, time, os
sys.path.extend(['.','..','py'])

import h2o, h2o_cmd, h2o_hosts, h2o_browse as h2b, h2o_import as h2i, h2o_glm

def define_params():
    paramDict = {
        'family': ['binomial'],
        'lambda': [1.0E-4],
        'alpha': [0.5],
        'max_iter': [2000],
        'weight': [1.0],
        'thresholds': [0.5],
        # 'link': [familyDefault],
        'num_cross_validation_folds': [1],
        'beta_epsilon': [1.0E-4],
        }
    return paramDict

class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        global SEED
        SEED = random.randint(0, sys.maxint)

        # SEED =
        random.seed(SEED)
        print "\nUsing random seed:", SEED
        localhost = h2o.decide_if_localhost()
        if (localhost):
            h2o.build_cloud(1,java_heap_GB=28)
        else:
            h2o_hosts.build_cloud_with_hosts()

    @classmethod
    def tearDownClass(cls):
        ### time.sleep(3600)
        h2o.tear_down_cloud()

    def test_many_cols_with_syn(self):
        ### h2b.browseTheCloud()

        csvFilename = "logreg_trisum_int_cat_10000x10.csv"
        csvPathname = "smalldata/logreg/" + csvFilename
        key2 = csvFilename + ".hex"

        parseKey = h2o_cmd.parseFile(None, h2o.find_file(csvPathname), key2=key2, timeoutSecs=10)
        print csvFilename, 'parse time:', parseKey['response']['time']
        print "Parse result['destination_key']:", parseKey['destination_key']

        # We should be able to see the parse result?
        inspect = h2o_cmd.runInspect(None, parseKey['destination_key'])
        print "\n" + csvFilename

        paramDict = define_params()
        paramDict2 = {}
        for k in paramDict:
            # sometimes we have a list to pick from in the value. now it's just list of 1.
            paramDict2[k] = paramDict[k][0]

        y = 10
        # FIX! what should we have for case? 1 should be okay because we have 1's in output col
        kwargs = {'y': y, 'max_iter': 50}
        kwargs.update(paramDict2)

        start = time.time()
        glm = h2o_cmd.runGLMOnly(parseKey=parseKey, timeoutSecs=20, **kwargs)
        print "glm end on ", csvPathname, 'took', time.time() - start, 'seconds'
        h2o_glm.simpleCheckGLM(self, glm, 8, **kwargs)

        if not h2o.browse_disable:
            h2b.browseJsonHistoryAsUrlLastMatch("Inspect")
            time.sleep(5)

if __name__ == '__main__':
    h2o.unit_main()
<commit_msg>fit in 12 GB dram (heap 10GB)<commit_after>import unittest
import random, sys, time, os
sys.path.extend(['.','..','py'])

import h2o, h2o_cmd, h2o_hosts, h2o_browse as h2b, h2o_import as h2i, h2o_glm

def define_params():
    paramDict = {
        'family': ['binomial'],
        'lambda': [1.0E-4],
        'alpha': [0.5],
        'max_iter': [2000],
        'weight': [1.0],
        'thresholds': [0.5],
        # 'link': [familyDefault],
        'num_cross_validation_folds': [1],
        'beta_epsilon': [1.0E-4],
        }
    return paramDict

class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        global SEED
        SEED = random.randint(0, sys.maxint)

        # SEED =
        random.seed(SEED)
        print "\nUsing random seed:", SEED
        localhost = h2o.decide_if_localhost()
        if (localhost):
            h2o.build_cloud(1,java_heap_GB=10)
        else:
            h2o_hosts.build_cloud_with_hosts()

    @classmethod
    def tearDownClass(cls):
        ### time.sleep(3600)
        h2o.tear_down_cloud()

    def test_many_cols_with_syn(self):
        ### h2b.browseTheCloud()

        csvFilename = "logreg_trisum_int_cat_10000x10.csv"
        csvPathname = "smalldata/logreg/" + csvFilename
        key2 = csvFilename + ".hex"

        parseKey = h2o_cmd.parseFile(None, h2o.find_file(csvPathname), key2=key2, timeoutSecs=10)
        print csvFilename, 'parse time:', parseKey['response']['time']
        print "Parse result['destination_key']:", parseKey['destination_key']

        # We should be able to see the parse result?
        inspect = h2o_cmd.runInspect(None, parseKey['destination_key'])
        print "\n" + csvFilename

        paramDict = define_params()
        paramDict2 = {}
        for k in paramDict:
            # sometimes we have a list to pick from in the value. now it's just list of 1.
            paramDict2[k] = paramDict[k][0]

        y = 10
        # FIX! what should we have for case? 1 should be okay because we have 1's in output col
        kwargs = {'y': y, 'max_iter': 50}
        kwargs.update(paramDict2)

        start = time.time()
        glm = h2o_cmd.runGLMOnly(parseKey=parseKey, timeoutSecs=20, **kwargs)
        print "glm end on ", csvPathname, 'took', time.time() - start, 'seconds'
        h2o_glm.simpleCheckGLM(self, glm, 8, **kwargs)

        if not h2o.browse_disable:
            h2b.browseJsonHistoryAsUrlLastMatch("Inspect")
            time.sleep(5)

if __name__ == '__main__':
    h2o.unit_main()
<|endoftext|>


<commit_before>#!/usr/bin/python
'''
Code Written by Jeff Peterson and Kerry Wang (2016)
Takes a DXF file and converts it into segments (which can be arcs or lines) and
vertexes that can then be converted into a CrysMAS or Cats2D mesh.

Requirements:
- Python 2.7x
- dxfgrabber 0.7.5
- matplotlib (if plotting)
- DXFtoSegments.py
- HelperFunctions.py

Requirements for Testing:
- DXFTestSuite.py
- CreateStandards.py
- DXFTests directory of DXF test files

Usage:
This script takes input from the command line and the usage for which can be
found by running this script with the '-h' flag afterwards specificaly,
'./DXFtoSegments.py -h'.
'''

import DXFTestSuite
from DXFtoSegments import DXFGeometry
import CreateStandards
import unittest
import argparse
from matplotlib import pyplot as plt
import os
import sys
import tarfile

def test_suite(verbose=False, dxf_test=True):
    '''Runs the test suite'''
    if verbose:
        verbosity = 2
    else:
        verbosity = 1
    suites = []
    suites.append(unittest.TestLoader().loadTestsFromTestCase(DXFTestSuite.TestVertex))
    suites.append(unittest.TestLoader().loadTestsFromTestCase(DXFTestSuite.TestVertexList))
    if dxf_test:
        suites.append(unittest.TestLoader().loadTestsFromTestCase(DXFTestSuite.TestDXFGeometry))
        suites.append(unittest.TestLoader().loadTestsFromTestCase(DXFTestSuite.DXFTestCases))
        suites.append(unittest.TestLoader().loadTestsFromTestCase(DXFTestSuite.DXFtoCats2DTests))
        suites.append(unittest.TestLoader().loadTestsFromTestCase(DXFTestSuite.DXFtoCrysMASTests))
        dxf1 = DXFGeometry('./DXFTests/DXFTest1.dxf', testing=True)
        dxf1.display()
        dxf2 = DXFGeometry('./DXFTests/DXFTest2.dxf', testing=True)
        dxf2.display()
        dxf3 = DXFGeometry('./DXFTests/DXFTest3.dxf', testing=True)
        dxf3.display()
        dxf4 = DXFGeometry('./DXFTests/DXFTest4.dxf', testing=True)
        dxf4.display()
    print
    alltests = unittest.TestSuite(suites)
    unittest.TextTestRunner(verbosity=verbosity).run(alltests)
    # Show the plotted DXF file if available
    plt.show()

def make_dist():
    '''Creates a tar file for the distribution of this code'''
    # Find out where this code is being run from
    root_path = os.path.dirname(os.getcwd())
    base_path = os.path.basename(os.getcwd())
    os.chdir('..')
    # Set file types to ignore
    ignore_types = ['.pyc', '.pcs', '.dwl', '.dwl2']
    tar_name = base_path+'.tar.gz'
    with tarfile.open(tar_name, 'w:gz') as tar:
        for root, dirs, files in os.walk(base_path):
            # Ignore hidden files and .pyc files
            files = [f for f in files if (not f[0] == '.') and (not [ext for ext in ignore_types if ext in f[-4:]])]
            # Ignore hidden directories
            dirs[:] = [d for d in dirs if not d[0] == '.']
            # Add these files to the tar file
            for f in files:
                print 'adding', os.path.join(root, f)
                tar.add(os.path.join(root, f))
        print 'packaging code as {} at {}'.format(tar_name, root_path)
        tar.close()
    os.chdir('./DXFtoMesh')

# NOTE: Argument parsing requires Python 2.7x or higher
# Parses command-line input arguemtns
help_string = '''Reads a DXF file and then converts it to a suitable form for
                 use in creating computational meshes for CrysMAS and Cats2D'''
parser = argparse.ArgumentParser(description=help_string)

# Specify the DXF file (required)
help_string = '''Specify the DXF file to convert, specify \'test\' to run the
                 test suite, or \'dist\' to create the distribution of this
                 program'''
parser.add_argument('dxf_file', action='store', type=str,
                    metavar='dxf_file or \'test\'', help=help_string)

# Create verbose mode
parser.add_argument('-v', '--verbose', action='store_true')

# Specify whether information should be output to CrysMAS and optionally specify
# a new filename
help_string = '''Turn the DXF file into a CrysMAS .pcs mesh file. A file name
                 for the .pcs file can optionally be specified'''
parser.add_argument('--crysmas', nargs='?', metavar='file',
                    const=True, help=help_string)

# Specify the units of the DXF file
help_string = 'Specify the units for the DXF file'
parser.add_argument('--units', action='store', default='mm', help=help_string)

# Skip DXF tests if option is passed
help_string = '''Skips the DXF tests if testing mode is activated'''
parser.add_argument('--nodxf', action='store_true', help=help_string)

# Create new standard DXF files for tests
help_string = '''Creates new standards for tests to compare to when in test
                mode. Use with caution'''
parser.add_argument('--newstandard', action='store_true', help=help_string)

args = parser.parse_args()

# Specify testing mode from the command line
if args.dxf_file == 'test':
    # Create new standards if specified
    if args.newstandard:
        print 'Creating new standards for tests'
        CreateStandards.save_standards(num_tests=4)
    else:
        test_suite(verbose=args.verbose, dxf_test=not(args.nodxf))
    # Exit immediately after testing is complete
    sys.exit()
# Create distribution from the command line
elif args.dxf_file == 'dist':
    make_dist()
# Otherwise create a DXF geometry object
else:
    dxf = DXFGeometry(args.dxf_file, verbose=args.verbose)

# Create a CrysMAS file
if args.crysmas:
    if args.crysmas != True:
        crys_file = args.crysmas
    else:
        crys_file = None
    dxf.output_to_crysmas(dxf_units=args.units, f_name=crys_file)

    <commit_msg>Small change to requirements<commit_after>#!/usr/bin/python
'''
Code Written by Jeff Peterson and Kerry Wang (2016)
Takes a DXF file and converts it into segments (which can be arcs or lines) and
vertexes that can then be converted into a CrysMAS or Cats2D mesh.

Requirements:
- Python 2.7x
- dxfgrabber 0.7.5
- SciPy Stack
- DXFtoSegments.py
- HelperFunctions.py

Requirements for Testing:
- DXFTestSuite.py
- CreateStandards.py
- DXFTests directory of DXF test files

Usage:
This script takes input from the command line and the usage for which can be
found by running this script with the '-h' flag afterwards specificaly,
'./DXFtoSegments.py -h'.
'''

import DXFTestSuite
from DXFtoSegments import DXFGeometry
import CreateStandards
import unittest
import argparse
from matplotlib import pyplot as plt
import os
import sys
import tarfile

def test_suite(verbose=False, dxf_test=True):
    '''Runs the test suite'''
    if verbose:
        verbosity = 2
    else:
        verbosity = 1
    suites = []
    suites.append(unittest.TestLoader().loadTestsFromTestCase(DXFTestSuite.TestVertex))
    suites.append(unittest.TestLoader().loadTestsFromTestCase(DXFTestSuite.TestVertexList))
    if dxf_test:
        suites.append(unittest.TestLoader().loadTestsFromTestCase(DXFTestSuite.TestDXFGeometry))
        suites.append(unittest.TestLoader().loadTestsFromTestCase(DXFTestSuite.DXFTestCases))
        suites.append(unittest.TestLoader().loadTestsFromTestCase(DXFTestSuite.DXFtoCats2DTests))
        suites.append(unittest.TestLoader().loadTestsFromTestCase(DXFTestSuite.DXFtoCrysMASTests))
        dxf1 = DXFGeometry('./DXFTests/DXFTest1.dxf', testing=True)
        dxf1.display()
        dxf2 = DXFGeometry('./DXFTests/DXFTest2.dxf', testing=True)
        dxf2.display()
        dxf3 = DXFGeometry('./DXFTests/DXFTest3.dxf', testing=True)
        dxf3.display()
        dxf4 = DXFGeometry('./DXFTests/DXFTest4.dxf', testing=True)
        dxf4.display()
    print
    alltests = unittest.TestSuite(suites)
    unittest.TextTestRunner(verbosity=verbosity).run(alltests)
    # Show the plotted DXF file if available
    plt.show()

def make_dist():
    '''Creates a tar file for the distribution of this code'''
    # Find out where this code is being run from
    root_path = os.path.dirname(os.getcwd())
    base_path = os.path.basename(os.getcwd())
    os.chdir('..')
    # Set file types to ignore
    ignore_types = ['.pyc', '.pcs', '.dwl', '.dwl2']
    tar_name = base_path+'.tar.gz'
    with tarfile.open(tar_name, 'w:gz') as tar:
        for root, dirs, files in os.walk(base_path):
            # Ignore hidden files and .pyc files
            files = [f for f in files if (not f[0] == '.') and (not [ext for ext in ignore_types if ext in f[-4:]])]
            # Ignore hidden directories
            dirs[:] = [d for d in dirs if not d[0] == '.']
            # Add these files to the tar file
            for f in files:
                print 'adding', os.path.join(root, f)
                tar.add(os.path.join(root, f))
        print 'packaging code as {} at {}'.format(tar_name, root_path)
        tar.close()
    os.chdir('./DXFtoMesh')

# NOTE: Argument parsing requires Python 2.7x or higher
# Parses command-line input arguemtns
help_string = '''Reads a DXF file and then converts it to a suitable form for
                 use in creating computational meshes for CrysMAS and Cats2D'''
parser = argparse.ArgumentParser(description=help_string)

# Specify the DXF file (required)
help_string = '''Specify the DXF file to convert, specify \'test\' to run the
                 test suite, or \'dist\' to create the distribution of this
                 program'''
parser.add_argument('dxf_file', action='store', type=str,
                    metavar='dxf_file or \'test\'', help=help_string)

# Create verbose mode
parser.add_argument('-v', '--verbose', action='store_true')

# Specify whether information should be output to CrysMAS and optionally specify
# a new filename
help_string = '''Turn the DXF file into a CrysMAS .pcs mesh file. A file name
                 for the .pcs file can optionally be specified'''
parser.add_argument('--crysmas', nargs='?', metavar='file',
                    const=True, help=help_string)

# Specify the units of the DXF file
help_string = 'Specify the units for the DXF file'
parser.add_argument('--units', action='store', default='mm', help=help_string)

# Skip DXF tests if option is passed
help_string = '''Skips the DXF tests if testing mode is activated'''
parser.add_argument('--nodxf', action='store_true', help=help_string)

# Create new standard DXF files for tests
help_string = '''Creates new standards for tests to compare to when in test
                mode. Use with caution'''
parser.add_argument('--newstandard', action='store_true', help=help_string)

args = parser.parse_args()

# Specify testing mode from the command line
if args.dxf_file == 'test':
    # Create new standards if specified
    if args.newstandard:
        print 'Creating new standards for tests'
        CreateStandards.save_standards(num_tests=4)
    else:
        test_suite(verbose=args.verbose, dxf_test=not(args.nodxf))
    # Exit immediately after testing is complete
    sys.exit()
# Create distribution from the command line
elif args.dxf_file == 'dist':
    make_dist()
# Otherwise create a DXF geometry object
else:
    dxf = DXFGeometry(args.dxf_file, verbose=args.verbose)

# Create a CrysMAS file
if args.crysmas:
    if args.crysmas != True:
        crys_file = args.crysmas
    else:
        crys_file = None
    dxf.output_to_crysmas(dxf_units=args.units, f_name=crys_file)

    <|endoftext|>


<commit_before>from django.contrib import admin
from django.shortcuts import get_object_or_404
from multiupload.admin import MultiUploadAdmin
from indicator.models import Indicator, IndicatorData, IndicatorSource, IncomeLevel, LendingType, IndicatorTopic
from django.conf.urls import patterns
from indicator.admin_tools import IndicatorAdminTools
from django.http import HttpResponse
from indicator.upload_indicators_helper import find_country, find_city, get_countries, get_cities
from indicator.wbi_parser import WBI_Parser

class IndicatorAdmin(admin.ModelAdmin):

    def get_urls(self):
        urls = super(IndicatorAdmin, self).get_urls()

        my_urls = patterns('',
            (r'^update-indicator/$', self.admin_site.admin_view(self.update_indicators)),
            (r'^update-indicator-data/$', self.admin_site.admin_view(self.update_indicator_data)),
            (r'^update-indicator-city-data/$', self.admin_site.admin_view(self.update_indicator_city_data)),
            (r'^update-wbi-indicator/$', self.admin_site.admin_view(self.update_WBI_indicators))
        )
        return my_urls + urls

    def update_indicator_data(self, request):
        admTools = IndicatorAdminTools()
        admTools.update_indicator_data()
        return HttpResponse('Success')

    def update_indicator_city_data(self, request):
        admTools = IndicatorAdminTools()
        admTools.update_indicator_city_data()
        return HttpResponse('Success')

    def update_indicators(self, request):
        admTools = IndicatorAdminTools()
        admTools.update_indicators()
        return HttpResponse('Success')

    def update_WBI_indicators(self, request):
        wbi_parser = WBI_Parser()
        wbi_parser.import_wbi_indicators()
        return HttpResponse('Success')

class IndicatorDataAdmin(admin.ModelAdmin):
    list_display = ['indicator', 'city','country', 'region', 'year', 'value']
    search_fields = ['year', 'indicator__friendly_label', 'value']
    list_filter = ['indicator', 'city', 'country', 'year']

class MyModelAdmin(MultiUploadAdmin):
    list_display = ['indicator','selection_type', 'city','country', 'region', 'year', 'value']
    search_fields = ['year', 'indicator__friendly_label', 'value']
    list_filter = ['indicator','selection_type', 'city', 'country', 'year']
    # default value of all parameters:
    change_form_template = 'multiupload/change_form.html'
    change_list_template = 'multiupload/change_list.html'
    multiupload_template = 'multiupload/upload.html'
    # if true, enable multiupload on list screen
    # generaly used when the model is the uploaded element
    multiupload_list = True
    # if true enable multiupload on edit screen
    # generaly used when the model is a container for uploaded files
    # eg: gallery
    # can upload files direct inside a gallery.
    multiupload_form = True
    # max allowed filesize for uploads in bytes
    multiupload_maxfilesize = 3 * 2 ** 20 # 3 Mb
    # min allowed filesize for uploads in bytes
    multiupload_minfilesize = 0
    # tuple with mimetype accepted
    multiupload_acceptedformats = ( "text/csv",)

    def process_uploaded_file(self, uploaded, object,not_sure, **kwargs):
        '''
        This method will be called for every file uploaded.
        Parameters:
            :uploaded: instance of uploaded file
            :object: instance of object if in form_multiupload else None
            :kwargs: request.POST received with file
        Return:
            It MUST return at least a dict with:
            {
                'url': 'url to download the file',
                'thumbnail_url': 'some url for an image_thumbnail or icon',
                'id': 'id of instance created in this method',
                'name': 'the name of created file',
            }
        '''
        # example:
        title = kwargs.get('title', [''])[0] or uploaded.name

        #check here how to save the file, i think we have to create a model, check https://github.com/gkuhn1/django-adminfiles/blob/master/adminfiles/models.py
        #f = self.model(upload=uploaded, title=title)
        #f.save()
        import csv
        try:
            dialect = csv.Sniffer().sniff(uploaded.read(4048))
        except csv.Error:
            dialect = csv.excel

        file = csv.DictReader(uploaded, dialect=dialect)
        keys = []
        for key in file.next().iterkeys():
            keys.append(key)

        line_counter = 0
        indicator_from_db = None
        city_found = []
        city_not_found = []
        country_found = []
        country_not_found = []

        cities = get_cities()
        countries = get_countries()
        for line in file:
            #getting data from the csv file
            city_csv = line.get('city')
            deprivation_type_csv = line.get('deprivation_type')
            description_csv = line.get('description')
            selection_type_csv = line.get('selection_type')
            country_csv = line.get('country')
            region_csv = line.get('region_csv')
            friendly_label_csv = line.get('friendly_name')
            value_csv = line.get('value')
            #todo replace , with . for floating numbers, check if this is always the case
            value_csv = value_csv.replace(',', '.')
            year_range_csv = line.get('year_range')
            indicator_id_csv = line.get('indicator_id')
            year_csv = line.get('year')
            type_data_csv = line.get('type_data')

            if line_counter == 0:
                #try to find the indicator that is uploaded or create a new one
                #todo I think an indicator is unique together with a selection_type or derivation_type
                indicator_from_db = Indicator.objects.get_or_create(id=indicator_id_csv)[0]

                #update the indicator fields
                indicator_from_db.description = description_csv
                indicator_from_db.friendly_label = friendly_label_csv
                indicator_from_db.type_data = type_data_csv
                indicator_from_db.selection_type = selection_type_csv
                indicator_from_db.deprivation_type = deprivation_type_csv
                indicator_from_db.save()

            #getting city from our database
            try:
                city_from_db = find_city(city_name=city_csv, cities=cities)
            except:
                city_from_db = None

                #continue

            #getting country from our database
            try:
                country_from_db = find_country(country_name=country_csv, countries=countries)
            except:
                country_from_db = None

            if country_from_db:
                country_found.append(country_csv)
            else:
                country_not_found.append(country_csv)

            if city_from_db:
                city_found.append(city_csv)
            else:
                city_not_found.append(city_csv)

            #if not (city_csv and city_from_db):
            #    file.next()
            #
            #if (not country_csv and country_from_db) or not city_from_db:
            #    file.next()
                #continue

            #try to update or create IndicatorData
            try:
                if city_from_db:
                    #if the indicator data a selection type contains than we need to store that correctly
                    if selection_type_csv:
                        indicator_data_from_db = IndicatorData.objects.get_or_create(year=year_csv, indicator=indicator_from_db, selection_type=selection_type_csv, city=city_from_db)[0]
                    else:
                        indicator_data_from_db = IndicatorData.objects.get_or_create(year=year_csv, indicator=indicator_from_db, city=city_from_db)[0]
                    if country_from_db:
                        indicator_data_from_db.country = country_from_db
                    indicator_data_from_db.city = city_from_db
                    #todo get region from db
                    indicator_data_from_db.value = float(value_csv)
                    indicator_data_from_db.save()
            except:
                pass

            line_counter += 1

        return {
            'url': 'f.image_thumb()',
            'thumbnail_url': 'f.image_thumb()',
            'id': 'f.id',
            'name': title
        }

    def delete_file(self, pk, request):
        '''
        Function to delete a file.
        '''
        # This is the default implementation.
        obj = get_object_or_404(self.queryset(request), pk=pk)
        obj.delete()

admin.site.register(Indicator, IndicatorAdmin)
admin.site.register(IndicatorData, MyModelAdmin)
admin.site.register(IndicatorSource)
admin.site.register(IncomeLevel)
admin.site.register(LendingType)
admin.site.register(IndicatorTopic)

<commit_msg>-fix uploader storing country related indicator data<commit_after>from django.contrib import admin
from django.shortcuts import get_object_or_404
from multiupload.admin import MultiUploadAdmin
from indicator.models import Indicator, IndicatorData, IndicatorSource, IncomeLevel, LendingType, IndicatorTopic
from django.conf.urls import patterns
from indicator.admin_tools import IndicatorAdminTools
from django.http import HttpResponse
from indicator.upload_indicators_helper import find_country, find_city, get_countries, get_cities
from indicator.wbi_parser import WBI_Parser

class IndicatorAdmin(admin.ModelAdmin):

    def get_urls(self):
        urls = super(IndicatorAdmin, self).get_urls()

        my_urls = patterns('',
            (r'^update-indicator/$', self.admin_site.admin_view(self.update_indicators)),
            (r'^update-indicator-data/$', self.admin_site.admin_view(self.update_indicator_data)),
            (r'^update-indicator-city-data/$', self.admin_site.admin_view(self.update_indicator_city_data)),
            (r'^update-wbi-indicator/$', self.admin_site.admin_view(self.update_WBI_indicators))
        )
        return my_urls + urls

    def update_indicator_data(self, request):
        admTools = IndicatorAdminTools()
        admTools.update_indicator_data()
        return HttpResponse('Success')

    def update_indicator_city_data(self, request):
        admTools = IndicatorAdminTools()
        admTools.update_indicator_city_data()
        return HttpResponse('Success')

    def update_indicators(self, request):
        admTools = IndicatorAdminTools()
        admTools.update_indicators()
        return HttpResponse('Success')

    def update_WBI_indicators(self, request):
        wbi_parser = WBI_Parser()
        wbi_parser.import_wbi_indicators()
        return HttpResponse('Success')

class IndicatorDataAdmin(admin.ModelAdmin):
    list_display = ['indicator', 'city','country', 'region', 'year', 'value']
    search_fields = ['year', 'indicator__friendly_label', 'value']
    list_filter = ['indicator', 'city', 'country', 'year']

class MyModelAdmin(MultiUploadAdmin):
    list_display = ['indicator','selection_type', 'city','country', 'region', 'year', 'value']
    search_fields = ['year', 'indicator__friendly_label', 'value']
    list_filter = ['indicator','selection_type', 'city', 'country', 'year']
    # default value of all parameters:
    change_form_template = 'multiupload/change_form.html'
    change_list_template = 'multiupload/change_list.html'
    multiupload_template = 'multiupload/upload.html'
    # if true, enable multiupload on list screen
    # generaly used when the model is the uploaded element
    multiupload_list = True
    # if true enable multiupload on edit screen
    # generaly used when the model is a container for uploaded files
    # eg: gallery
    # can upload files direct inside a gallery.
    multiupload_form = True
    # max allowed filesize for uploads in bytes
    multiupload_maxfilesize = 3 * 2 ** 20 # 3 Mb
    # min allowed filesize for uploads in bytes
    multiupload_minfilesize = 0
    # tuple with mimetype accepted
    multiupload_acceptedformats = ( "text/csv",)

    def process_uploaded_file(self, uploaded, object,not_sure, **kwargs):
        '''
        This method will be called for every file uploaded.
        Parameters:
            :uploaded: instance of uploaded file
            :object: instance of object if in form_multiupload else None
            :kwargs: request.POST received with file
        Return:
            It MUST return at least a dict with:
            {
                'url': 'url to download the file',
                'thumbnail_url': 'some url for an image_thumbnail or icon',
                'id': 'id of instance created in this method',
                'name': 'the name of created file',
            }
        '''
        # example:
        title = kwargs.get('title', [''])[0] or uploaded.name

        #check here how to save the file, i think we have to create a model, check https://github.com/gkuhn1/django-adminfiles/blob/master/adminfiles/models.py
        #f = self.model(upload=uploaded, title=title)
        #f.save()
        import csv
        try:
            dialect = csv.Sniffer().sniff(uploaded.read(4048))
        except csv.Error:
            dialect = csv.excel

        file = csv.DictReader(uploaded, dialect=dialect)
        keys = []
        for key in file.next().iterkeys():
            keys.append(key)

        line_counter = 0
        indicator_from_db = None
        city_found = []
        city_not_found = []
        country_found = []
        country_not_found = []

        cities = get_cities()
        countries = get_countries()
        for line in file:
            #getting data from the csv file
            city_csv = line.get('city')
            deprivation_type_csv = line.get('deprivation_type')
            description_csv = line.get('description')
            selection_type_csv = line.get('selection_type')
            country_csv = line.get('country')
            region_csv = line.get('region_csv')
            friendly_label_csv = line.get('friendly_name')
            value_csv = line.get('value')
            #todo replace , with . for floating numbers, check if this is always the case
            value_csv = value_csv.replace(',', '.')
            year_range_csv = line.get('year_range')
            indicator_id_csv = line.get('indicator_id')
            year_csv = line.get('year')
            type_data_csv = line.get('type_data')

            if line_counter == 0:
                #try to find the indicator that is uploaded or create a new one
                #todo I think an indicator is unique together with a selection_type or derivation_type
                indicator_from_db = Indicator.objects.get_or_create(id=indicator_id_csv)[0]

                #update the indicator fields
                indicator_from_db.description = description_csv
                indicator_from_db.friendly_label = friendly_label_csv
                indicator_from_db.type_data = type_data_csv
                indicator_from_db.selection_type = selection_type_csv
                indicator_from_db.deprivation_type = deprivation_type_csv
                indicator_from_db.save()

            #getting city from our database
            try:
                if city_csv:
                    city_from_db = find_city(city_name=city_csv, cities=cities)
                else:
                    city_from_db = None
            except:
                city_from_db = None

                #continue

            #getting country from our database
            try:
                if country_csv:
                    country_from_db = find_country(country_name=country_csv, countries=countries)
                else:
                    country_from_db = None
            except:
                country_from_db = None

            if country_from_db:
                country_found.append(country_csv)
            else:
                country_not_found.append(country_csv)

            if city_from_db:
                city_found.append(city_csv)
            else:
                city_not_found.append(city_csv)

            #this block is for storing data related to cities
            try:
                if city_from_db:
                    #if the indicator data a selection type contains than we need to store that correctly
                    if selection_type_csv:
                        indicator_data_from_db = IndicatorData.objects.get_or_create(year=year_csv, indicator=indicator_from_db, selection_type=selection_type_csv, city=city_from_db)[0]
                    else:
                        indicator_data_from_db = IndicatorData.objects.get_or_create(year=year_csv, indicator=indicator_from_db, city=city_from_db)[0]
                    if country_from_db:
                        indicator_data_from_db.country = country_from_db
                    indicator_data_from_db.city = city_from_db
                    #todo get region from db
                    indicator_data_from_db.value = float(value_csv)
                    indicator_data_from_db.save()
            except:
                pass

            #this block is for storing country related indicator data
            try:
                if country_from_db and not city_from_db:
                    if selection_type_csv:
                        indicator_data_from_db = IndicatorData.objects.get_or_create(year=year_csv, indicator=indicator_from_db, selection_type=selection_type_csv, country=country_from_db)[0]
                    else:
                        indicator_data_from_db = IndicatorData.objects.get_or_create(year=year_csv, indicator=indicator_from_db, country=country_from_db)[0]
                    #todo get region from db
                    indicator_data_from_db.value = float(value_csv)
                    indicator_data_from_db.save()
            except:
                pass

            line_counter += 1

        return {
            'url': 'f.image_thumb()',
            'thumbnail_url': 'f.image_thumb()',
            'id': 'f.id',
            'name': title
        }

    def delete_file(self, pk, request):
        '''
        Function to delete a file.
        '''
        # This is the default implementation.
        obj = get_object_or_404(self.queryset(request), pk=pk)
        obj.delete()

admin.site.register(Indicator, IndicatorAdmin)
admin.site.register(IndicatorData, MyModelAdmin)
admin.site.register(IndicatorSource)
admin.site.register(IncomeLevel)
admin.site.register(LendingType)
admin.site.register(IndicatorTopic)

<|endoftext|>


<commit_before>from flask import Markup
from newspaper import Article
from xml.etree import ElementTree

import markdown2
import html2text

def clean_content(url_to_clean):

    article = Article(url_to_clean)
    article.download()
    article.parse()

    html_string = ElementTree.tostring(article.clean_top_node)
    markdown = html2text.HTML2Text().handle(html_string)
    article_html = Markup(markdown2.markdown(markdown))

    return {
        'html': article_html,
        'authors': str(', '.join(article.authors)),
        'title': article.title,
        'plaintext': markdown.replace('\n', ' '),
        'markdown': markdown
    }
<commit_msg>Fix error importing newspaper<commit_after>from __future__ import absolute_import

from flask import Markup
from newspaper import Article
from xml.etree import ElementTree

import markdown2
import html2text

def clean_content(url_to_clean):

    article = Article(url_to_clean)
    article.download()
    article.parse()

    html_string = ElementTree.tostring(article.clean_top_node)
    markdown = html2text.HTML2Text().handle(html_string)
    article_html = Markup(markdown2.markdown(markdown))

    return {
        'html': article_html,
        'authors': str(', '.join(article.authors)),
        'title': article.title,
        'plaintext': markdown.replace('\n', ' '),
        'markdown': markdown
    }
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

'''
File: MongoSplitter.py
Author: NYU ITP Team
Description: Will calculate splits for a given collection/database
and store/return them in MongoSplit objects
'''
from pymongo import Connection, uri_parser
import logging

def calculate_splits(config):
    """reads config to find out what type of split to perform"""
    #pass
    uri = "mongodb://localhost/test.in" #config.getInputURI()
    uri_info = uri_parser.parse_uri(uri)

    host = uri_info['nodelist'][0][0]
    port = uri_info['nodelist'][0][1]
    database_name = uri_info['database']
    collection_name = uri_info['collection']

    connection = Connection(uri)
    db = connection[databaseName]
    stats = db.command("collstats", collection_name)

    isSharded = False if "sharded" not in stats else stats["sharded"]
    useShards = False #config.canReadSplitsFromShards()
    useChunks = False #config.isShardChunkedSplittingEnabled()
    laveOk = True #config.canReadSplitsFromSecondary()

    logging.info(" Calculate Splits Code ... Use Shards? " + useShards + ", Use Chunks? " + useChunks + "; Collection Sharded? " + isSharded);

    if config.createInputSplits():
        logging.info( "Creation of Input Splits is enabled." )
        if isSharded and (useShards or useChunks):
            if useShards and useChunks:
                logging.warn( "Combining 'use chunks' and 'read from shards directly' can have unexpected & erratic behavior in a live system due to chunk migrations. " );

                logging.info( "Sharding mode calculation entering." );
                return calculate_sharded_splits( config, useShards, useChunks, slaveOk, uri, mongo );

            else: # perfectly ok for sharded setups to run with a normally calculated split. May even be more efficient for some cases
                logging.info( "Using Unsharded Split mode (Calculating multiple splits though)" );
                return calculate_unsharded_splits( config, slaveOk, uri, coll );

    else:
        logging.info( "Creation of Input Splits is disabled; Non-Split mode calculation entering." );
        return calculate_single_split( config );

def calculate_unsharded_splits(config, etc):
    """@todo: Docstring for calculate_unsharded_splits

    :returns: @todo
    """
    pass

def _split(config, etc):
    """@todo: Docstring for _split

    :returns: an actual MongoSplit object
    """
    pass

def calculate_single_split(config):
    pass

def calculate_sharded_splits(config, etc):
    """Worry about this after unsharded splits are doen

    :returns: @todo
    """
    pass

def fetch_splits_from_shards(config):
    """@todo: Docstring for fetch_splits_from_shards

    :returns: @todo
    """
    pass

def fetch_splits_via_chunks(config):
    """@todo: Docstring for fetch_splits_via_chunks

    :returns: @todo
    """
    pass

def get_new_URI(original_URI, etc):
    """@todo: Docstring for get_new_URI

    :original_URI: @todo
    :etc: @todo
    :returns: a new Mongo_URI
    """

    pass

<commit_msg>split command for unsharded splits<commit_after>#!/usr/bin/env python
# encoding: utf-8

'''
File: MongoSplitter.py
Author: NYU ITP Team
Description: Will calculate splits for a given collection/database
and store/return them in MongoSplit objects
'''
from pymongo import Connection, uri_parser
import logging
import bson

def calculate_splits(config):
    """reads config to find out what type of split to perform"""
    #pass
    uri = "mongodb://localhost/test.in" #config.getInputURI()
    uri_info = uri_parser.parse_uri(uri)

    host = uri_info['nodelist'][0][0]
    port = uri_info['nodelist'][0][1]
    database_name = uri_info['database']
    collection_name = uri_info['collection']

    connection = Connection(uri)
    db = connection[database_name]
    stats = db.command("collstats", collection_name)

    isSharded = False if "sharded" not in stats else stats["sharded"]
    useShards = False #config.canReadSplitsFromShards()
    useChunks = False #config.isShardChunkedSplittingEnabled()
    slaveOk = True #config.canReadSplitsFromSecondary()

    logging.info(" Calculate Splits Code ... Use Shards? " + useShards + ", Use Chunks? " + useChunks + "; Collection Sharded? " + isSharded);

    if config.createInputSplits():
        logging.info( "Creation of Input Splits is enabled." )
        if isSharded and (useShards or useChunks):
            if useShards and useChunks:
                logging.warn( "Combining 'use chunks' and 'read from shards directly' can have unexpected & erratic behavior in a live system due to chunk migrations. " );

                logging.info( "Sharding mode calculation entering." );
                return calculate_sharded_splits( config, useShards, useChunks, slaveOk, uri, mongo );

            else: # perfectly ok for sharded setups to run with a normally calculated split. May even be more efficient for some cases
                logging.info( "Using Unsharded Split mode (Calculating multiple splits though)" );
                return calculate_unsharded_splits( config, slaveOk, uri, coll );

    else:
        logging.info( "Creation of Input Splits is disabled; Non-Split mode calculation entering." );
        return calculate_single_split( config );

def calculate_unsharded_splits(config, etc):
    """@todo: Docstring for calculate_unsharded_splits

    :returns: @todo

    """
    #create command
    #command to split should look like this VV
    #SON([('splitVector', u'test.test_data'), ('maxChunkSize', 2), ('force', False), ('keyPattern', {'x': 1})])
    split_key  = config.get('splitKey')
    split_size = config.get('splitSize')
    full_name  = config.get('full_name')
    logging.info("Calculating unsharded splits on collection %s with Split Key %s" % (full_name, split_key))
    logging.info("Max split size :: %sMB" % split_size)

    cmd = bson.bson.SON()
    cmd["splitVector"]  = full_name
    cmd["maxChunkSize"] = split_size
    cmd["keyPattern"]   = split_key
    cmd["force"]        = False

    # TODO: pass these fields VV as parameters? (02/26/12, 11:30, AFlock)
    connection = Connection(uri)
    db = connection[database_name]
    split_results = db.command(cmd)

    if split_results:
        logging.info()

    pass

def _split(config, etc):
    """@todo: Docstring for _split

    :returns: an actual MongoSplit object
    """
    pass

def calculate_single_split(config):
    pass

def calculate_sharded_splits(config, etc):
    """Worry about this after unsharded splits are doen

    :returns: @todo
    """
    pass

def fetch_splits_from_shards(config):
    """@todo: Docstring for fetch_splits_from_shards

    :returns: @todo
    """
    pass

def fetch_splits_via_chunks(config):
    """@todo: Docstring for fetch_splits_via_chunks

    :returns: @todo
    """
    pass

def get_new_URI(original_URI, etc):
    """@todo: Docstring for get_new_URI

    :original_URI: @todo
    :etc: @todo
    :returns: a new Mongo_URI
    """

    pass

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Main Controller"""
import json
import datetime
import unidecode
import collections
import webhelpers.feedgenerator as feedgenerator
from sqlalchemy.orm.exc import NoResultFound
from tg import expose, flash, require, lurl, config
from tg import predicates
from tg import request, redirect, tmpl_context
from tg.decorators import paginate
from tg.exceptions import HTTPFound
from tg.i18n import ugettext as _, lazy_ugettext as l_
from tgext.admin.controller import AdminController
from tgext.admin.tgadminconfig import BootstrapTGAdminConfig as TGAdminConfig

from pyjobsweb import model
from pyjobsweb.controllers.error import ErrorController
from pyjobsweb.controllers.secure import SecureController
from pyjobsweb.lib.base import BaseController
from pyjobsweb.lib.helpers import slugify, get_job_url
from pyjobsweb.lib.stats import StatsQuestioner
from pyjobsweb.model import DBSession, Log
from pyjobsweb.model.data import JobOfferSQLAlchemy, SOURCES
from pyjobsweb.forms.ResearchForm import ResearchForm

__all__ = ['RootController']
existing_fields = (
    'title',
    'publication_datetime',
    'company',
    'company_url',
    'address',
    'description',
    'tags',
)

class RootController(BaseController):
    """
    The root controller for the pyjobsweb application.

    All the other controllers and WSGI applications should be mounted on this
    controller. For example::

        panel = ControlPanelController()
        another_app = AnotherWSGIApplication()

    Keep in mind that WSGI applications shouldn't be mounted directly: They
    must be wrapped around with :class:`tg.controllers.WSGIAppController`.

    """
    secc = SecureController()
    admin = AdminController(model, DBSession, config_type=TGAdminConfig)

    error = ErrorController()

    def _before(self, *args, **kw):
        tmpl_context.project_name = "Algoo"

    items_per_page = 20

    @expose('pyjobsweb.templates.jobs')
    @paginate('jobs', items_per_page=items_per_page)
    def index(self, query=None, radius=None, center=None, unit=None):
        if not query and not center and not radius:
            job_offers = JobOfferSQLAlchemy.get_all_job_offers()
        else:
            import pyjobsweb.lib.search_query as sq

            search_query = model.ElasticsearchQuery(model.JobOfferElasticsearch, 0, self.items_per_page * 50)

            search_on = ['description', 'title', 'company']

            for q in query.split(','):
                if q:
                    search_query.add_elem(sq.KeywordFilter(search_on, [q]))

            try:
                geoloc_query = json.loads(center)
                lat, lon = (geoloc_query['lat'], geoloc_query['lon'])
                center_point = sq.GeolocationFilter.Center(lat, lon)
                unit = sq.GeolocationFilter.UnitsEnum(unit)
                radius = float(radius)
                search_query.add_elem(
                    sq.GeolocationFilter(center_point, radius, unit)
                )
                search_query.add_elem(
                    sq.BooleanFilter('geolocation_error', False)
                )
            except ValueError:
                # One of the following case has occurred:
                #     - Center, radius or unit weren't set properly
                #     - Radius couldn't be converted to float, therefore we
                #       ignore the geolocation filters
                pass

            ms = sq.Sort()
            ms.append(sq.DescSortStatement('publication_datetime'))
            search_query.add_elem(ms)

            job_offers = search_query.execute_query()

        search_form = ResearchForm(action='/', method='POST').req()

        return dict(
            sources=SOURCES,
            jobs=job_offers,
            job_offer_search_form=search_form
        )

    @expose('json')
    def geocomplete(self, **kwargs):
        if 'address' not in kwargs:
            return []

        import pyjobsweb.lib.search_query as sq

        query_tokens = kwargs['address'].split(' ')

        postal_code = None
        address = None

        for token in query_tokens:
            try:
                if len(token) == 5:
                    int_code = int(token)
                    postal_code = postal_code if postal_code else str(int_code)
                else:
                    raise ValueError
            except ValueError:
                address = u'%s %s' % (address, token) if address else token

        query = model.ElasticsearchQuery(model.Geocomplete, 0, 5)

        if address:
            query.add_elem(sq.KeywordFilter(['name'], [address]))

        if postal_code:
            query.add_elem(sq.KeywordFilter(['postal_code'], [postal_code]))

        raw_res = query.execute_query()

        res = list()
        duplicates = list()
        for s in raw_res:
            geoloc = dict(lat=s.geolocation.lat, lon=s.geolocation.lon)
            submit = json.dumps(geoloc)
            display = u'%s, France' % s.name

            if display not in duplicates:
                res.append(dict(to_submit=submit, to_display=display))
                duplicates.append(display)

        return dict(results=res)

    @expose()
    def rss(self, limit=50, source=None):
        """
        RSS feed of jobs
        :param source: source name
        :param limit: number of displayed jobs
        :return: RSS feed content
        """
        site_url = config.get('site.domain_base_url')
        feed = feedgenerator.Rss201rev2Feed(
            title=u"pyjobs : le job qu'il vous faut en python",
            link=site_url,
            description=u"Agrégation de jobs python",
            language=u"fr",
            feed_url=u"http://www.pyjobs.fr/rss?limit=%s" % limit
        )

        jobs = DBSession.query(JobOfferSQLAlchemy) \
            .order_by(JobOfferSQLAlchemy.publication_datetime.desc()) \
            .limit(limit)

        if source is not None:
            jobs = jobs.filter(JobOfferSQLAlchemy.source == source)

        for job in jobs:
            job_slug = slugify(job.title)
            feed.add_item(
                    title=job.title,
                    link=get_job_url(job.id, job_title=job.title, absolute=True),
                    description=job.description,
                    pubdate=job.publication_datetime,
                    unique_id="%s/job/%d/%s" % (site_url, job.id, job_slug)
            )

        return feed.writeString('utf-8')

    @expose('pyjobsweb.templates.job')
    def job(self, job_id, job_title=None, previous=None):
        """
        Job detail page
        :param job_id: Job identifier
        :param job_title: Job title (optional) for pretty url
        :return: dict
        """
        try:
            job = DBSession.query(JobOfferSQLAlchemy).filter_by(id=job_id).one()
        except NoResultFound:
            pass  # TODO: TubroGears 404 ?
        return dict(
                job=job,
                sources=SOURCES
        )

    @expose('pyjobsweb.templates.sources')
    def sources(self):
        sources_last_crawl = {}
        sorted_sources = collections.OrderedDict(sorted(SOURCES.items(), key=lambda x: x[1].label))
        for source_name in sorted_sources:
            try:
                sources_last_crawl[source_name] = DBSession.query(Log.datetime) \
                    .filter(Log.source == source_name) \
                    .order_by(Log.datetime.desc()) \
                    .limit(1)\
                    .one()[0]
            except NoResultFound:
                sources_last_crawl[source_name] = None

        return dict(
                sources=sorted_sources,
                existing_fields=existing_fields,
                sources_last_crawl=sources_last_crawl
        )

    @expose('pyjobsweb.templates.stats')
    def stats(self, since_months=4):
        stats = StatsQuestioner
        stats_questioner = stats(DBSession)
        date_from, date_to = stats.get_month_period(int(since_months))

        by_months = stats_questioner.by_complete_period(
            period=stats.PERIOD_MONTH,
            date_from=date_from,
            date_to=date_to,
        ).all()
        months = stats.extract(by_months, stats.FIELD_DATE)
        stats_month = stats.extract_stats(query_result=by_months, sources=SOURCES.keys())
        flat_month = stats.flat_query_by_y(
            query_result=by_months,
            sources=SOURCES.keys(),
            date_value_callback=lambda date: date.strftime('%Y-%m')
        )

        by_weeks = stats_questioner.by_complete_period(
            stats.PERIOD_WEEK,
            date_from=date_from,
            date_to=date_to,
        ).all()
        weeks = stats.extract(by_weeks, stats.FIELD_DATE)
        stats_week = stats.extract_stats(query_result=by_weeks, sources=SOURCES.keys())
        flat_week = stats.flat_query_by_y(
            query_result=by_weeks,
            sources=SOURCES.keys(),
            date_value_callback=lambda date: date.strftime('%Y-%m-%d')
        )

        return dict(
            sources=SOURCES,
            stats_month=stats_month,
            flat_month=flat_month,
            stats_week=stats_week,
            flat_week=flat_week,
            months=months,
            weeks=weeks,
            flat_x_field=stats.FIELDS[stats.FLAT_X_FIELD],
            flat_y_fields=SOURCES.keys(),
            sources_labels=[SOURCES[source].label for source in SOURCES]
        )

    @expose('pyjobsweb.templates.logs')
    def logs(self, source=None, last_days=1):

        logs_query = DBSession.query(Log)\
            .order_by(Log.datetime.desc())\
            .filter(Log.datetime >= datetime.datetime.now() + datetime.timedelta(days=-int(last_days)))\
            .filter(Log.message.in_(('CRAWL_LIST_START',
                                    'CRAWL_LIST_FINISHED',
                                    'ERROR_UNEXPECTED_END',
                                    'ERROR_CRAWNLING')))

        if source is not None:
            logs_query = logs_query.filter(Log.source == source)

        return dict(
            sources=SOURCES,
            logs=logs_query.all(),
            last_days=last_days
        )

    @expose('pyjobsweb.templates.about')
    def about(self):
        return dict()

    @expose('pyjobsweb.templates.index')
    @require(predicates.has_permission('manage', msg=l_('Only for managers')))
    def manage_permission_only(self, **kw):
        """Illustrate how a page for managers only works."""
        return dict(page='managers stuff')

    @expose('pyjobsweb.templates.index')
    @require(predicates.is_user('editor', msg=l_('Only for the editor')))
    def editor_user_only(self, **kw):
        """Illustrate how a page exclusive for the editor works."""
        return dict(page='editor stuff')

    @expose('pyjobsweb.templates.login')
    def login(self, came_from=lurl('/'), failure=None, login=''):
        """Start the user login."""
        if failure is not None:
            if failure == 'user-not-found':
                flash(_('User not found'), 'error')
            elif failure == 'invalid-password':
                flash(_('Invalid Password'), 'error')

        login_counter = request.environ.get('repoze.who.logins', 0)
        if failure is None and login_counter > 0:
            flash(_('Wrong credentials'), 'warning')

        return dict(page='login', login_counter=str(login_counter),
                    came_from=came_from, login=login)

    @expose()
    def post_login(self, came_from=lurl('/')):
        """
        Redirect the user to the initially requested page on successful
        authentication or redirect her back to the login page if login failed.

        """
        if not request.identity:
            login_counter = request.environ.get('repoze.who.logins', 0) + 1
            redirect('/login',
                     params=dict(came_from=came_from, __logins=login_counter))
        userid = request.identity['repoze.who.userid']
        flash(_('Welcome back, %s!') % userid)

        # Do not use tg.redirect with tg.url as it will add the mountpoint
        # of the application twice.
        return HTTPFound(location=came_from)

    @expose()
    def post_logout(self, came_from=lurl('/')):
        """
        Redirect the user to the initially requested page on logout and say
        goodbye as well.

        """
        flash(_('We hope to see you soon!'))
        return HTTPFound(location=came_from)
<commit_msg>Import cleanup<commit_after># -*- coding: utf-8 -*-
"""Main Controller"""
import json
import datetime
import collections
import webhelpers.feedgenerator as feedgenerator
from sqlalchemy.orm.exc import NoResultFound
from tg import expose, flash, require, lurl, config
from tg import predicates
from tg import request, redirect, tmpl_context
from tg.decorators import paginate
from tg.exceptions import HTTPFound
from tg.i18n import ugettext as _, lazy_ugettext as l_
from tgext.admin.controller import AdminController
from tgext.admin.tgadminconfig import BootstrapTGAdminConfig as TGAdminConfig

from pyjobsweb import model
from pyjobsweb.controllers.error import ErrorController
from pyjobsweb.controllers.secure import SecureController
from pyjobsweb.lib.base import BaseController
from pyjobsweb.lib.helpers import slugify, get_job_url
from pyjobsweb.lib.stats import StatsQuestioner
from pyjobsweb.model import DBSession, Log
from pyjobsweb.model.data import JobOfferSQLAlchemy, SOURCES
from pyjobsweb.forms.ResearchForm import ResearchForm

__all__ = ['RootController']
existing_fields = (
    'title',
    'publication_datetime',
    'company',
    'company_url',
    'address',
    'description',
    'tags',
)

class RootController(BaseController):
    """
    The root controller for the pyjobsweb application.

    All the other controllers and WSGI applications should be mounted on this
    controller. For example::

        panel = ControlPanelController()
        another_app = AnotherWSGIApplication()

    Keep in mind that WSGI applications shouldn't be mounted directly: They
    must be wrapped around with :class:`tg.controllers.WSGIAppController`.

    """
    secc = SecureController()
    admin = AdminController(model, DBSession, config_type=TGAdminConfig)

    error = ErrorController()

    def _before(self, *args, **kw):
        tmpl_context.project_name = "Algoo"

    items_per_page = 20

    @expose('pyjobsweb.templates.jobs')
    @paginate('jobs', items_per_page=items_per_page)
    def index(self, query=None, radius=None, center=None, unit=None):
        if not query and not center and not radius:
            job_offers = JobOfferSQLAlchemy.get_all_job_offers()
        else:
            import pyjobsweb.lib.search_query as sq

            search_query = model.ElasticsearchQuery(model.JobOfferElasticsearch, 0, self.items_per_page * 50)

            search_on = ['description', 'title', 'company']

            for q in query.split(','):
                if q:
                    search_query.add_elem(sq.KeywordFilter(search_on, [q]))

            try:
                geoloc_query = json.loads(center)
                lat, lon = (geoloc_query['lat'], geoloc_query['lon'])
                center_point = sq.GeolocationFilter.Center(lat, lon)
                unit = sq.GeolocationFilter.UnitsEnum(unit)
                radius = float(radius)
                search_query.add_elem(
                    sq.GeolocationFilter(center_point, radius, unit)
                )
                search_query.add_elem(
                    sq.BooleanFilter('geolocation_error', False)
                )
            except ValueError:
                # One of the following case has occurred:
                #     - Center, radius or unit weren't set properly
                #     - Radius couldn't be converted to float, therefore we
                #       ignore the geolocation filters
                pass

            ms = sq.Sort()
            ms.append(sq.DescSortStatement('publication_datetime'))
            search_query.add_elem(ms)

            job_offers = search_query.execute_query()

        search_form = ResearchForm(action='/', method='POST').req()

        return dict(
            sources=SOURCES,
            jobs=job_offers,
            job_offer_search_form=search_form
        )

    @expose('json')
    def geocomplete(self, **kwargs):
        if 'address' not in kwargs:
            return []

        import pyjobsweb.lib.search_query as sq

        query_tokens = kwargs['address'].split(' ')

        postal_code = None
        address = None

        for token in query_tokens:
            try:
                if len(token) == 5:
                    int_code = int(token)
                    postal_code = postal_code if postal_code else str(int_code)
                else:
                    raise ValueError
            except ValueError:
                address = u'%s %s' % (address, token) if address else token

        query = model.ElasticsearchQuery(model.Geocomplete, 0, 5)

        if address:
            query.add_elem(sq.KeywordFilter(['name'], [address]))

        if postal_code:
            query.add_elem(sq.KeywordFilter(['postal_code'], [postal_code]))

        raw_res = query.execute_query()

        res = list()
        duplicates = list()
        for s in raw_res:
            geoloc = dict(lat=s.geolocation.lat, lon=s.geolocation.lon)
            submit = json.dumps(geoloc)
            display = u'%s, France' % s.name

            if display not in duplicates:
                res.append(dict(to_submit=submit, to_display=display))
                duplicates.append(display)

        return dict(results=res)

    @expose()
    def rss(self, limit=50, source=None):
        """
        RSS feed of jobs
        :param source: source name
        :param limit: number of displayed jobs
        :return: RSS feed content
        """
        site_url = config.get('site.domain_base_url')
        feed = feedgenerator.Rss201rev2Feed(
            title=u"pyjobs : le job qu'il vous faut en python",
            link=site_url,
            description=u"Agrégation de jobs python",
            language=u"fr",
            feed_url=u"http://www.pyjobs.fr/rss?limit=%s" % limit
        )

        jobs = DBSession.query(JobOfferSQLAlchemy) \
            .order_by(JobOfferSQLAlchemy.publication_datetime.desc()) \
            .limit(limit)

        if source is not None:
            jobs = jobs.filter(JobOfferSQLAlchemy.source == source)

        for job in jobs:
            job_slug = slugify(job.title)
            feed.add_item(
                    title=job.title,
                    link=get_job_url(job.id, job_title=job.title, absolute=True),
                    description=job.description,
                    pubdate=job.publication_datetime,
                    unique_id="%s/job/%d/%s" % (site_url, job.id, job_slug)
            )

        return feed.writeString('utf-8')

    @expose('pyjobsweb.templates.job')
    def job(self, job_id, job_title=None, previous=None):
        """
        Job detail page
        :param job_id: Job identifier
        :param job_title: Job title (optional) for pretty url
        :return: dict
        """
        try:
            job = DBSession.query(JobOfferSQLAlchemy).filter_by(id=job_id).one()
        except NoResultFound:
            pass  # TODO: TubroGears 404 ?
        return dict(
                job=job,
                sources=SOURCES
        )

    @expose('pyjobsweb.templates.sources')
    def sources(self):
        sources_last_crawl = {}
        sorted_sources = collections.OrderedDict(sorted(SOURCES.items(), key=lambda x: x[1].label))
        for source_name in sorted_sources:
            try:
                sources_last_crawl[source_name] = DBSession.query(Log.datetime) \
                    .filter(Log.source == source_name) \
                    .order_by(Log.datetime.desc()) \
                    .limit(1)\
                    .one()[0]
            except NoResultFound:
                sources_last_crawl[source_name] = None

        return dict(
                sources=sorted_sources,
                existing_fields=existing_fields,
                sources_last_crawl=sources_last_crawl
        )

    @expose('pyjobsweb.templates.stats')
    def stats(self, since_months=4):
        stats = StatsQuestioner
        stats_questioner = stats(DBSession)
        date_from, date_to = stats.get_month_period(int(since_months))

        by_months = stats_questioner.by_complete_period(
            period=stats.PERIOD_MONTH,
            date_from=date_from,
            date_to=date_to,
        ).all()
        months = stats.extract(by_months, stats.FIELD_DATE)
        stats_month = stats.extract_stats(query_result=by_months, sources=SOURCES.keys())
        flat_month = stats.flat_query_by_y(
            query_result=by_months,
            sources=SOURCES.keys(),
            date_value_callback=lambda date: date.strftime('%Y-%m')
        )

        by_weeks = stats_questioner.by_complete_period(
            stats.PERIOD_WEEK,
            date_from=date_from,
            date_to=date_to,
        ).all()
        weeks = stats.extract(by_weeks, stats.FIELD_DATE)
        stats_week = stats.extract_stats(query_result=by_weeks, sources=SOURCES.keys())
        flat_week = stats.flat_query_by_y(
            query_result=by_weeks,
            sources=SOURCES.keys(),
            date_value_callback=lambda date: date.strftime('%Y-%m-%d')
        )

        return dict(
            sources=SOURCES,
            stats_month=stats_month,
            flat_month=flat_month,
            stats_week=stats_week,
            flat_week=flat_week,
            months=months,
            weeks=weeks,
            flat_x_field=stats.FIELDS[stats.FLAT_X_FIELD],
            flat_y_fields=SOURCES.keys(),
            sources_labels=[SOURCES[source].label for source in SOURCES]
        )

    @expose('pyjobsweb.templates.logs')
    def logs(self, source=None, last_days=1):

        logs_query = DBSession.query(Log)\
            .order_by(Log.datetime.desc())\
            .filter(Log.datetime >= datetime.datetime.now() + datetime.timedelta(days=-int(last_days)))\
            .filter(Log.message.in_(('CRAWL_LIST_START',
                                    'CRAWL_LIST_FINISHED',
                                    'ERROR_UNEXPECTED_END',
                                    'ERROR_CRAWNLING')))

        if source is not None:
            logs_query = logs_query.filter(Log.source == source)

        return dict(
            sources=SOURCES,
            logs=logs_query.all(),
            last_days=last_days
        )

    @expose('pyjobsweb.templates.about')
    def about(self):
        return dict()

    @expose('pyjobsweb.templates.index')
    @require(predicates.has_permission('manage', msg=l_('Only for managers')))
    def manage_permission_only(self, **kw):
        """Illustrate how a page for managers only works."""
        return dict(page='managers stuff')

    @expose('pyjobsweb.templates.index')
    @require(predicates.is_user('editor', msg=l_('Only for the editor')))
    def editor_user_only(self, **kw):
        """Illustrate how a page exclusive for the editor works."""
        return dict(page='editor stuff')

    @expose('pyjobsweb.templates.login')
    def login(self, came_from=lurl('/'), failure=None, login=''):
        """Start the user login."""
        if failure is not None:
            if failure == 'user-not-found':
                flash(_('User not found'), 'error')
            elif failure == 'invalid-password':
                flash(_('Invalid Password'), 'error')

        login_counter = request.environ.get('repoze.who.logins', 0)
        if failure is None and login_counter > 0:
            flash(_('Wrong credentials'), 'warning')

        return dict(page='login', login_counter=str(login_counter),
                    came_from=came_from, login=login)

    @expose()
    def post_login(self, came_from=lurl('/')):
        """
        Redirect the user to the initially requested page on successful
        authentication or redirect her back to the login page if login failed.

        """
        if not request.identity:
            login_counter = request.environ.get('repoze.who.logins', 0) + 1
            redirect('/login',
                     params=dict(came_from=came_from, __logins=login_counter))
        userid = request.identity['repoze.who.userid']
        flash(_('Welcome back, %s!') % userid)

        # Do not use tg.redirect with tg.url as it will add the mountpoint
        # of the application twice.
        return HTTPFound(location=came_from)

    @expose()
    def post_logout(self, came_from=lurl('/')):
        """
        Redirect the user to the initially requested page on logout and say
        goodbye as well.

        """
        flash(_('We hope to see you soon!'))
        return HTTPFound(location=came_from)
<|endoftext|>