<commit_before>'''
MIT License

Copyright (c) 2017 grokkers

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''
import json
from os import path

_path = path.join(
    path.dirname(path.realpath(__file__)),
    'chests.json'
    )

with open(_path.replace('\\','/')) as f:
    CHESTS = json.load(f)

class Base:
    '''
    Base class for models. Only thing that
    needs to be added is the from_data function
    '''

    def __init__(self, client, data):
        self.client = client
        self.raw_data = data
        self.name = data.get('name')
        self.tag = data.get('tag')
        self.from_data(data)
        endpoint = type(self).__name__.lower()
        self.url = '{0.client.BASE}/{1}/{0.tag}'.format(self, endpoint)

    async def from_data(self):
        return NotImplemented

    async def update(self):
        '''Update an object with current info.'''
        data = await self.client.request(self.url)

        self.raw_data = data
        self.from_data(data)

        return self

class ClanChest:
    '''Represents a clan chest'''
    def __init__(self, data):
        self.crowns = data.get('clanChestCrowns')
        self.percent = data.get('clanChestCrownsPercent')
        self.required = data.get('clanChestCrownsRequired')

class Arena:
    '''represents an arena'''
    def __init__(self, data):
        self.raw_data = data
        self.name = data.get('name')
        self.number = data.get('arenaID')
        self.trophies = data.get('trophyLimit')

    @property
    def image_url(self):
        return "http://api.cr-api.com" + self.raw_data.get('imageURL')

    def __str__(self):
        return self.name

class Shop:
    '''Represents shop offers'''
    def __init__(self, data):
        self.legendary = data.get('legendary')
        self.epic = data.get('epic')
        self.arena = data.get('arena')

class Cycle:
    '''Represents your chest cycle'''
    def __init__(self, data):
        self.position = data.get('position')
        self.super_magical = data.get('superMagicalPos')
        self.legendary = data.get('legendaryPos')
        self.epic = data.get('epicPos')

class CardInfo:
    '''Represents a Clash Royale card'''
    def __init__(self, data):
        self.raw_data = data
        self.name = data.get('name')
        self.rarity = data.get('rarity')
        self.id = data.get('card_id')
        self.elixir = data.get('elixir')
        self.type = data.get('type')
        self.arena = data.get('arena')
        self.description = data.get('description')
        self.decklink = data.get('decklink')

    def __repr__(self):
        return '<Card id={0.id}>'.format(self)

class PlayerCard(CardInfo):

    def __init__(self, data):
        super().__init__(data)
        self.level = data.get('level')
        self.count = data.get('count')
        self.to_upgrade = data.get('requiredForUpgrade')

class Member:
    '''Represents a member of a clan'''
    def __init__(self, client, clan, data):
        self.clan = clan
        self.client = client
        self.name = data.get('name')
        self.arena = Arena(data.get('arena'))
        self.role = data.get('role')
        self.role_name = data.get('roleName')
        self.level = data.get('expLevel')
        self.trophies = data.get('trophies')
        self.donations = data.get('donations')
        self.rank = data.get('currentRank')
        self.crowns = data.get('clanChestCrowns')
        self.tag = data.get('tag')

    def __str__(self):
        return '{0.name} (#{0.tag})'.format(self)

    def __repr__(self):
        return '<Member tag={0.tag}>'.format(self)

    def get_profile(self):
        return self.client.get_profile(self.tag)

class Alliance:
    def __init__(self, data):
        self.roles = data.get('roles')
        self.types = data.get('types')

class Country:
    def __init__(self, data):
        self.name = data.get('name')
        if data.get('isCountry') == 'true':
            self.is_country = True
        elif data.get('isCountry') == 'false':
            self.is_country = False
        else:
            self.is_country = data.get('isCountry')

    def __str__(self):
        return self.name

class Rarity:
    def __init__(self, data):
        self.name = data.get('name')
        self.balance_multiplier = data.get('balance_multiplier')
        self.chance_weight = data.get('chance_weight')
        self.clone_relative_level = data.get('clone_relative_level')
        self.donate_capacity = data.get('donate_capacity')
        self.donate_reward = data.get('donate_reward')
        self.donate_xp = data.get('donate_xp')
        self.gold_conversion_value = data.get('gold_conversion_value')
        self.max_level = data.get('level_count')
        self.mirror_relative_level = data.get('mirror_relative_level')
        self.power_level_multiplier = [c for c in data.get('power_level_multiplier')]
        self.refund_gems = data.get('refund_gems')
        self.relative_level = data.get('relative_level')
        self.sort_capacity = data.get('sort_capacity')
        self.upgrade_cost = [c for c in data.get('upgrade_cost')]
        self.upgrade_exp = [c for c in data.get('upgrade_exp')]
        self.upgrade_material_count = [c for c in data.get('upgrade_material_count')]

    def __str__(self):
        return self.name

class ClanInfo:
    def __init__(self, client, data):
        self.raw_data = data
        self.client = client
        self.name = data.get('name')
        self.tag = data.get('tag')
        self.trophies = data.get('trophies')
        self.region = data.get('region').get('name')
        self.member_count = data.get('memberCount')
        self.rank = data.get('rank')
        self.previous_rank = data.get('previousRank')

    @property
    def badge_url(self):
        url = self.raw_data.get('badge').get('url')
        return "http://api.cr-api.com" + url

    def get_clan(self):
        return self.client.get_clan(self.tag)

    def __repr__(self):
        return '<ClanInfo tag={0.tag}>'.format(self)

    def __str__(self):
        return '{0.name} (#{0.tag})'.format(self)

class Clan(Base):
    '''Represents a clan'''

    def from_data(self, data):
        self.name = data.get('name')
        self.score = data.get('score')
        self.required_trophies = data.get('requiredScore')
        self.donations = data.get('donations')
        self.rank = data.get('currentRank')
        self.description = data.get('description')
        self.type = data.get('type')
        self.type_name = data.get('typeName')
        self.region = data.get('region').get('name')
        self.clan_chest = ClanChest(data.get('clanChest'))
        self.members = [Member(self.client, self, m) for m in data.get('members')]

    @property
    def badge_url(self):
        url = self.raw_data.get('badge').get('url')
        return "http://api.cr-api.com" + url

    def __repr__(self):
        return '<Clan tag={0.tag}>'.format(self)

    def __str__(self):
        return '{0.name} (#{0.tag})'.format(self)

class Profile(Base):
    '''Represents a player profile.
    Includes a clan maybe? (requires a seperate request tho)
    '''
    def from_data(self, data):
        self.name = data.get('name')
        exp = data.get('experience')
        stats = data.get('stats')
        games = data.get('games')
        clan = data.get('clan')
        self.level = exp.get('level')
        self.experience = (
            exp.get('xp'),
            exp.get('xpRequiredForLevelUp')
            )
        self.xp = self.experience
        self.name_changed = data.get('nameChanged')
        self.global_rank = data.get('globalRank')
        self.current_trophies = data.get('trophies')
        self.highest_trophies = stats.get('maxTrophies')
        self.legend_trophies = data.get('legendaryTrophies')
        self.tournament_cards_won = stats.get('tournamentCardsWon')
        self.challenge_cards_won = stats.get('challengeCardsWon')
        self.favourite_card = stats.get('favoriteCard').title()
        self.total_donations = stats.get('totalDonations')
        self.max_wins = stats.get('challengeMaxWins')
        self.games_played = games.get('total')
        self.wins = games.get('wins')
        self.losses = games.get('losses')
        self.draws = games.get('draws')
        self.arena = Arena(data.get('arena'))
        self.shop_offers = Shop(data.get('shopOffers'))
        self.chest_cycle = Cycle(data.get('chestCycle'))
        self.deck = [PlayerCard(c) for c in data.get('currentDeck')]
        if clan is None:
            self.clan_tag = None
            self.clan_name = None
            self.clan_role = None
        else:
            self.clan_tag = clan.get('tag')
            self.clan_name = clan.get('name')
            self.clan_role = clan.get('role')

    @property
    def clan_badge_url(self):
        '''Returns clan badge url'''
        if self.clan_tag is None:
            return None
        url = self.raw_data.get('clan').get('badge').get('url')
        if not url:
            return None
        else:
            return "http://api.cr-api.com" + url

    def get_chest(self, index=0):
        '''Get your current chest +- the index'''
        index += self.chest_cycle.position % len(CHESTS)
        return CHESTS[index-1]

    def get_clan(self):
        if self.clan_tag is None:
            raise ValueError('Profile has no Clan')
        return self.client.get_clan(self.clan_tag)

    def __repr__(self):
        return '<Profile tag={0.tag}>'.format(self)

    def __str__(self):
        return '{0.name} (#{0.tag})'.format(self)

class Constants(Base):
    '''Represents the constants from cr-api'''
    def from_data(self, data):
        self.clan = Alliance(data.get('alliance'))
        self.arena = [Arena(c) for c in data.get('arenas')]
        self.badges = data.get('badges')
        self.chest_cycle = [c for c in data.get('chestCycle').get('order')]
        self.country_codes = [Country(c) for c in data.get('countryCodes')]
        self.rarities = [Rarity(c) for c in data.get('rarities')]
        self.card = [CardInfo(c) for c in data.get('cards')]

    def __repr__(self):
        return '<Clash Royale Constants Object>'

<commit_msg>some optimisation<commit_after>'''
MIT License

Copyright (c) 2017 grokkers

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''
import json
from os import path

_path = path.join(
    path.dirname(path.realpath(__file__)),
    'chests.json'
    )

with open(_path.replace('\\','/')) as f:
    CHESTS = json.load(f)

class Base:
    '''
    Base class for models. Only thing that
    needs to be added is the from_data function
    '''

    def __init__(self, client, data):
        self.client = client
        self.raw_data = data
        self.name = data.get('name')
        self.tag = data.get('tag')
        self.from_data(data)
        endpoint = type(self).__name__.lower()
        self.url = '{0.client.BASE}/{1}/{0.tag}'.format(self, endpoint)

    async def from_data(self):
        return NotImplemented

    async def update(self):
        '''Update an object with current info.'''
        data = await self.client.request(self.url)

        self.raw_data = data
        self.from_data(data)

        return self

class ClanChest:
    '''Represents a clan chest'''
    def __init__(self, data):
        self.crowns = data.get('clanChestCrowns')
        self.percent = data.get('clanChestCrownsPercent')
        self.required = data.get('clanChestCrownsRequired')

class Arena:
    '''represents an arena'''
    def __init__(self, data):
        self.raw_data = data
        self.name = data.get('name')
        self.number = data.get('arenaID')
        self.trophies = data.get('trophyLimit')

    @property
    def image_url(self):
        return "http://api.cr-api.com" + self.raw_data.get('imageURL')

    def __str__(self):
        return self.name

class Shop:
    '''Represents shop offers'''
    def __init__(self, data):
        self.legendary = data.get('legendary')
        self.epic = data.get('epic')
        self.arena = data.get('arena')

class Cycle:
    '''Represents your chest cycle'''
    def __init__(self, data):
        self.position = data.get('position')
        self.super_magical = data.get('superMagicalPos')
        self.legendary = data.get('legendaryPos')
        self.epic = data.get('epicPos')

class CardInfo:
    '''Represents a Clash Royale card'''
    def __init__(self, data):
        self.raw_data = data
        self.name = data.get('name')
        self.rarity = data.get('rarity')
        self.id = data.get('card_id')
        self.elixir = data.get('elixir')
        self.type = data.get('type')
        self.arena = data.get('arena')
        self.description = data.get('description')
        self.decklink = data.get('decklink')

    def __repr__(self):
        return '<Card id={0.id}>'.format(self)

class PlayerCard(CardInfo):

    def __init__(self, data):
        super().__init__(data)
        self.level = data.get('level')
        self.count = data.get('count')
        self.to_upgrade = data.get('requiredForUpgrade')

class Member:
    '''Represents a member of a clan'''
    def __init__(self, client, clan, data):
        self.clan = clan
        self.client = client
        self.name = data.get('name')
        self.arena = Arena(data.get('arena'))
        self.role = data.get('role')
        self.role_name = data.get('roleName')
        self.level = data.get('expLevel')
        self.trophies = data.get('trophies')
        self.donations = data.get('donations')
        self.rank = data.get('currentRank')
        self.crowns = data.get('clanChestCrowns')
        self.tag = data.get('tag')

    def __str__(self):
        return '{0.name} (#{0.tag})'.format(self)

    def __repr__(self):
        return '<Member tag={0.tag}>'.format(self)

    def get_profile(self):
        return self.client.get_profile(self.tag)

class Alliance:
    def __init__(self, data):
        self.roles = data.get('roles')
        self.types = data.get('types')

class Country:
    def __init__(self, data):
        self.name = data.get('name')
        if data.get('isCountry') == 'true':
            self.is_country = True
        elif data.get('isCountry') == 'false':
            self.is_country = False
        else:
            self.is_country = data.get('isCountry')

    def __str__(self):
        return self.name

class Rarity:
    def __init__(self, data):
        self.name = data.get('name')
        self.balance_multiplier = data.get('balance_multiplier')
        self.chance_weight = data.get('chance_weight')
        self.clone_relative_level = data.get('clone_relative_level')
        self.donate_capacity = data.get('donate_capacity')
        self.donate_reward = data.get('donate_reward')
        self.donate_xp = data.get('donate_xp')
        self.gold_conversion_value = data.get('gold_conversion_value')
        self.max_level = data.get('level_count')
        self.mirror_relative_level = data.get('mirror_relative_level')
        self.power_level_multiplier = [c for c in data.get('power_level_multiplier')]
        self.refund_gems = data.get('refund_gems')
        self.relative_level = data.get('relative_level')
        self.sort_capacity = data.get('sort_capacity')
        self.upgrade_cost = [c for c in data.get('upgrade_cost')]
        self.upgrade_exp = [c for c in data.get('upgrade_exp')]
        self.upgrade_material_count = [c for c in data.get('upgrade_material_count')]

    def __str__(self):
        return self.name

class ClanInfo:
    def __init__(self, client, data):
        self.raw_data = data
        self.client = client
        self.name = data.get('name')
        self.tag = data.get('tag')
        self.trophies = data.get('trophies')
        self.region = data.get('region').get('name')
        self.member_count = data.get('memberCount')
        self.rank = data.get('rank')
        self.previous_rank = data.get('previousRank')

    @property
    def badge_url(self):
        url = self.raw_data.get('badge').get('url')
        return "http://api.cr-api.com" + url

    def get_clan(self):
        return self.client.get_clan(self.tag)

    def __repr__(self):
        return '<ClanInfo tag={0.tag}>'.format(self)

    def __str__(self):
        return '{0.name} (#{0.tag})'.format(self)

class Clan(Base):
    '''Represents a clan'''

    def from_data(self, data):
        self.name = data.get('name')
        self.score = data.get('score')
        self.required_trophies = data.get('requiredScore')
        self.donations = data.get('donations')
        self.rank = data.get('currentRank')
        self.description = data.get('description')
        self.type = data.get('type')
        self.type_name = data.get('typeName')
        self.region = data.get('region').get('name')
        self.clan_chest = ClanChest(data.get('clanChest'))
        self.members = [Member(self.client, self, m) for m in data.get('members')]

    @property
    def badge_url(self):
        url = self.raw_data.get('badge').get('url')
        return "http://api.cr-api.com" + url

    def __repr__(self):
        return '<Clan tag={0.tag}>'.format(self)

    def __str__(self):
        return '{0.name} (#{0.tag})'.format(self)

class Profile(Base):
    '''Represents a player profile.
    Includes a clan maybe? (requires a seperate request tho)
    '''
    def from_data(self, data):
        self.name = data.get('name')
        exp = data.get('experience')
        stats = data.get('stats')
        games = data.get('games')
        clan = data.get('clan')
        self.level = exp.get('level')
        self.experience = (
            exp.get('xp'),
            exp.get('xpRequiredForLevelUp')
            )
        self.xp = self.experience
        self.name_changed = data.get('nameChanged')
        self.global_rank = data.get('globalRank')
        self.current_trophies = data.get('trophies')
        self.highest_trophies = stats.get('maxTrophies')
        self.legend_trophies = data.get('legendaryTrophies')
        self.tournament_cards_won = stats.get('tournamentCardsWon')
        self.challenge_cards_won = stats.get('challengeCardsWon')
        self.favourite_card = stats.get('favoriteCard').title()
        self.total_donations = stats.get('totalDonations')
        self.max_wins = stats.get('challengeMaxWins')
        self.games_played = games.get('total')
        self.wins = games.get('wins')
        self.losses = games.get('losses')
        self.draws = games.get('draws')
        self.arena = Arena(data.get('arena'))
        self.shop_offers = Shop(data.get('shopOffers'))
        self.chest_cycle = Cycle(data.get('chestCycle'))
        self.deck = [PlayerCard(c) for c in data.get('currentDeck')]
        self.clan_tag = None
        self.clan_name = None
        self.clan_role = None
        if clan is not None:
            self.clan_tag = clan.get('tag')
            self.clan_name = clan.get('name')
            self.clan_role = clan.get('role')

    @property
    def clan_badge_url(self):
        '''Returns clan badge url'''
        if self.clan_tag is None:
            return None
        url = self.raw_data.get('clan').get('badge').get('url')
        if not url:
            return None
        return "http://api.cr-api.com" + url

    def get_chest(self, index=0):
        '''Get your current chest +- the index'''
        index += self.chest_cycle.position % len(CHESTS)
        return CHESTS[index-1]

    def get_clan(self):
        if self.clan_tag is None:
            raise ValueError('Profile has no Clan')
        return self.client.get_clan(self.clan_tag)

    def __repr__(self):
        return '<Profile tag={0.tag}>'.format(self)

    def __str__(self):
        return '{0.name} (#{0.tag})'.format(self)

class Constants(Base):
    '''Represents the constants from cr-api'''
    def from_data(self, data):
        self.clan = Alliance(data.get('alliance'))
        self.arena = [Arena(c) for c in data.get('arenas')]
        self.badges = data.get('badges')
        self.chest_cycle = [c for c in data.get('chestCycle').get('order')]
        self.country_codes = [Country(c) for c in data.get('countryCodes')]
        self.rarities = [Rarity(c) for c in data.get('rarities')]
        self.card = [CardInfo(c) for c in data.get('cards')]

    def __repr__(self):
        return '<Clash Royale Constants Object>'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Author: Damien Crier
# Copyright 2016 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
from openerp import models, fields, api

class LedgerReportWizard(models.TransientModel):

    _name = "ledger.report.wizard"
    _description = "Ledger Report Wizard"

    company_id = fields.Many2one(comodel_name='res.company')
    # date_range = ??
    date_from = fields.Date()
    date_to = fields.Date()
    target_move = fields.Selection([('posted', 'All Posted Entries'),
                                    ('all', 'All Entries')],
                                   string='Target Moves',
                                   required=True,
                                   default='posted')
    account_ids = fields.Many2many(
        comodel_name='account.account',
        string='Filter accounts',
    )
    amount_currency = fields.Boolean(string='With currency',
                                     default=False)
    centralize = fields.Boolean(string='Activate centralization',
                                default=False)
    result_selection = fields.Selection(
        [('customer', 'Receivable Accounts'),
         ('supplier', 'Payable Accounts'),
         ('customer_supplier', 'Receivable and Payable Accounts')
         ],
        string="Partner's",
        default='customer')
    partner_ids = fields.Many2many(
        comodel_name='res.partner',
        string='Filter partners',
    )

    @api.multi
    def button_print(self):
        return True
<commit_msg>bring back code we're not inheriting anymore<commit_after># -*- coding: utf-8 -*-
# Author: Damien Crier
# Copyright 2016 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
from openerp import models, fields, api

class LedgerReportWizard(models.TransientModel):

    _name = "ledger.report.wizard"
    _description = "Ledger Report Wizard"

    company_id = fields.Many2one(comodel_name='res.company')
    # date_range = ??
    date_from = fields.Date()
    date_to = fields.Date()
    target_move = fields.Selection([('posted', 'All Posted Entries'),
                                    ('all', 'All Entries')],
                                   string='Target Moves',
                                   required=True,
                                   default='posted')
    account_ids = fields.Many2many(
        comodel_name='account.account',
        string='Filter accounts',
    )
    amount_currency = fields.Boolean(string='With currency',
                                     default=False)
    centralize = fields.Boolean(string='Activate centralization',
                                default=False)
    result_selection = fields.Selection(
        [('customer', 'Receivable Accounts'),
         ('supplier', 'Payable Accounts'),
         ('customer_supplier', 'Receivable and Payable Accounts')
         ],
        string="Partner's",
        default='customer')
    partner_ids = fields.Many2many(
        comodel_name='res.partner',
        string='Filter partners',
    )

    @api.multi
    def pre_print_report(self, data):
        data = {'form': {}}

        # will be used to attach the report on the main account
        vals = self.read(['amount_currency',
                          'account_ids',
                          'journal_ids',
                          'centralize',
                          'target_move',
                          'date_from',
                          'date_to',
                          'fiscalyear'])[0]
        data['form'].update(vals)
        return data

    @api.multi
    def _print_report(self, data):
        # we update form with display account value
        data = self.pre_print_report(data)
        Report = self.env['report'].with_context(landscape=True)
        return Report.get_action(
            self, 'account.report_generalledger_qweb',
            data=data)

    def _build_contexts(self, data):
        result = {}
        result['journal_ids'] = 'journal_ids' in data['form'] and data['form']['journal_ids'] or False
        result['state'] = 'target_move' in data['form'] and data['form']['target_move'] or ''
        result['date_from'] = data['form']['date_from'] or False
        result['date_to'] = data['form']['date_to'] or False
        result['strict_range'] = True if result['date_from'] else False
        return result

    @api.multi
    def check_report(self):
        self.ensure_one()
        data = {}
        data['ids'] = self.env.context.get('active_ids', [])
        data['model'] = self.env.context.get('active_model', 'ir.ui.menu')
        data['form'] = self.read(['date_from', 'date_to', 'journal_ids', 'target_move'])[0]
        used_context = self._build_contexts(data)
        data['form']['used_context'] = dict(used_context, lang=self.env.context.get('lang', 'en_US'))
        return self._print_report(data)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""Utility functions related to splitting work into tasks."""

import sys
import cPickle
import traceback
import psutil

from celery.task.sets import TaskSet
from celery.app import current_app
from celery.task import task

from openquake.engine import logs, no_distribute
from openquake.engine.db import models
from openquake.engine.utils import config
from openquake.engine.writer import CacheInserter
from openquake.engine.performance import EnginePerformanceMonitor, \
    LightMonitor

ONE_MB = 1024 * 1024

def check_mem_usage(used_mem_percent=90):
    """
    Display a warning if we are running out of memory

    :param int used_mem_percent: the memory limit as a percentage
    """
    mem_percent = psutil.virtmem_usage().percent
    if mem_percent > used_mem_percent:
        logs.LOG.warn('Using over %d%% of the memory!', mem_percent)

class Pickled(object):
    """
    An utility to manually pickling/unpickling objects.
    The reason is that celery does not use the HIGHEST_PROTOCOL,
    so relying on celery is slower. Moreover Pickled instances
    have a nice string representation and length giving the size
    of the pickled bytestring.

    :param obj: the object to pickle
    """
    def __init__(self, obj):
        self.clsname = obj.__class__.__name__
        self.pik = cPickle.dumps(obj, cPickle.HIGHEST_PROTOCOL)

    def __repr__(self):
        """String representation of the pickled object"""
        return '<Pickled %s %dK>' % (self.clsname, len(self) / 1024)

    def __len__(self):
        """Length of the pickled bytestring"""
        return len(self.pik)

    def unpickle(self):
        """Unpickle the underlying object"""
        return cPickle.loads(self.pik)

def pickle_sequence(objects):
    """
    Convert an iterable of objects into a list of pickled objects.
    If the iterable contains copies, the pickling will be done only once.
    If the iterable contains objects already pickled, they will not be
    pickled again.

    :param objects: a sequence of objects to pickle
    """
    cache = {}
    out = []
    for obj in objects:
        obj_id = id(obj)
        if obj_id not in cache:
            if isinstance(obj, Pickled):  # already pickled
                cache[obj_id] = obj
            else:  # pickle the object
                cache[obj_id] = Pickled(obj)
        out.append(cache[obj_id])
    return out

def safely_call(func, args):
    """
    Call the given function with the given arguments safely, i.e.
    by trapping the exceptions. Return a pair (result, exc_type)
    where exc_type is None if no exceptions occur, otherwise it
    is the exception class and the result is a string containing
    error message and traceback.

    :param func: the function to call
    :param args: the arguments
    """
    try:
        return func(*args), None
    except:
        etype, exc, tb = sys.exc_info()
        tb_str = ''.join(traceback.format_tb(tb))
        return '\n%s%s: %s' % (tb_str, etype.__name__, exc), etype

def map_reduce(task, task_args, agg, acc):
    """
    Given a task and an iterable of positional arguments, apply the
    task function to the arguments in parallel and return an aggregate
    result depending on the initial value of the accumulator
    and on the aggregation function. To save memory, the order is
    not preserved and there is no list with the intermediated results:
    the accumulator is incremented as soon as a task result comes.

    NB: if the environment variable OQ_NO_DISTRIBUTE is set the
    tasks are run sequentially in the current process and then
    map_reduce(task, task_args, agg, acc) is the same as
    reduce(agg, itertools.starmap(task, task_args), acc).
    Users of map_reduce should be aware of the fact that when
    thousands of tasks are spawned and large arguments are passed
    or large results are returned they may incur in memory issue:
    this is way the calculators limit the queue with the
    `concurrent_task` concept.

    :param task: a `celery` task callable.
    :param task_args: an iterable over positional arguments
    :param agg: the aggregation function, (acc, val) -> new acc
    :param acc: the initial value of the accumulator
    :returns: the final value of the accumulator
    """
    if no_distribute():
        for the_args in task_args:
            result, exctype = safely_call(task.task_func, the_args)
            if exctype:
                raise exctype(result)
            acc = agg(acc, result)
    else:
        backend = current_app().backend
        unpik = 0
        job_id = task_args[0][0]
        taskname = task.__name__
        mon = LightMonitor('unpickling %s' % taskname, job_id, task)
        to_send = 0
        pickled_args = []
        for args in task_args:
            piks = pickle_sequence(args)
            pickled_args.append(piks)
            to_send += sum(len(p) for p in piks)
        logs.LOG.info('Sending %dM', to_send / ONE_MB)
        taskset = TaskSet(tasks=map(task.subtask, pickled_args))
        for task_id, result_dict in taskset.apply_async().iter_native():
            check_mem_usage()  # log a warning if too much memory is used
            result_pik = result_dict['result']
            with mon:
                result, exctype = result_pik.unpickle()
            if exctype:
                raise exctype(result)
            unpik += len(result_pik)
            acc = agg(acc, result)
            del backend._cache[task_id]  # work around a celery bug
        logs.LOG.info('Unpickled %dM of received data in %s seconds',
                      unpik / ONE_MB, mon.duration)
    return acc

# used to implement BaseCalculator.parallelize, which takes in account
# the `concurrent_task` concept to avoid filling the Celery queue
def parallelize(task, task_args, side_effect=lambda val: None):
    """
    Given a celery task and an iterable of positional arguments, apply the
    callable to the arguments in parallel. It is possible to pass a
    function side_effect(val) which takes the return value of the
    callable and does something with it (such as saving or printing
    it). Notice that the order is not preserved. parallelize returns None.

    NB: if the environment variable OQ_NO_DISTRIBUTE is set the
    tasks are run sequentially in the current process.

    :param task: a celery task
    :param task_args: an iterable over positional arguments
    :param side_effect: a function val -> None
    """
    map_reduce(task, task_args, lambda acc, val: side_effect(val), None)

def oqtask(task_func):
    """
    Task function decorator which sets up logging and catches (and logs) any
    errors which occur inside the task. Also checks to make sure the job is
    actually still running. If it is not running, the task doesn't get
    executed, so we don't do useless computation.

    :param task_func: the function to decorate
    """
    def wrapped(*args):
        """
        Initialize logs, make sure the job is still running, and run the task
        code surrounded by a try-except. If any error occurs, log it as a
        critical failure.
        """
        # job_id is always assumed to be the first argument
        job_id = args[0]
        job = models.OqJob.objects.get(id=job_id)
        if job.is_running is False:
            # the job was killed, it is useless to run the task
            return

        # it is important to save the task id soon, so that
        # the revoke functionality can work
        EnginePerformanceMonitor.store_task_id(job_id, tsk)

        with EnginePerformanceMonitor(
                'total ' + task_func.__name__, job_id, tsk, flush=True):
            # tasks write on the celery log file
            logs.set_level(job.log_level)
            check_mem_usage()  # log a warning if too much memory is used
            try:
                # run the task
                return task_func(*args)
            finally:
                # save on the db
                CacheInserter.flushall()
                # the task finished, we can remove from the performance
                # table the associated row 'storing task id'
                models.Performance.objects.filter(
                    oq_job=job,
                    operation='storing task id',
                    task_id=tsk.request.id).delete()
    celery_queue = config.get('amqp', 'celery_queue')
    f = lambda *args: Pickled(
        safely_call(wrapped, [a.unpickle() for a in args]))
    f.__name__ = task_func.__name__
    tsk = task(f, queue=celery_queue)
    tsk.task_func = task_func
    return tsk
<commit_msg>Changed the limit to 80% of the physical memory<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""Utility functions related to splitting work into tasks."""

import sys
import cPickle
import traceback
import psutil

from celery.task.sets import TaskSet
from celery.app import current_app
from celery.task import task

from openquake.engine import logs, no_distribute
from openquake.engine.db import models
from openquake.engine.utils import config
from openquake.engine.writer import CacheInserter
from openquake.engine.performance import EnginePerformanceMonitor, \
    LightMonitor

ONE_MB = 1024 * 1024

def check_mem_usage(used_mem_percent=80):
    """
    Display a warning if we are running out of memory

    :param int used_mem_percent: the memory limit as a percentage
    """
    mem_percent = psutil.phymem_usage().percent
    if mem_percent > used_mem_percent:
        logs.LOG.warn('Using over %d%% of the memory!', mem_percent)

class Pickled(object):
    """
    An utility to manually pickling/unpickling objects.
    The reason is that celery does not use the HIGHEST_PROTOCOL,
    so relying on celery is slower. Moreover Pickled instances
    have a nice string representation and length giving the size
    of the pickled bytestring.

    :param obj: the object to pickle
    """
    def __init__(self, obj):
        self.clsname = obj.__class__.__name__
        self.pik = cPickle.dumps(obj, cPickle.HIGHEST_PROTOCOL)

    def __repr__(self):
        """String representation of the pickled object"""
        return '<Pickled %s %dK>' % (self.clsname, len(self) / 1024)

    def __len__(self):
        """Length of the pickled bytestring"""
        return len(self.pik)

    def unpickle(self):
        """Unpickle the underlying object"""
        return cPickle.loads(self.pik)

def pickle_sequence(objects):
    """
    Convert an iterable of objects into a list of pickled objects.
    If the iterable contains copies, the pickling will be done only once.
    If the iterable contains objects already pickled, they will not be
    pickled again.

    :param objects: a sequence of objects to pickle
    """
    cache = {}
    out = []
    for obj in objects:
        obj_id = id(obj)
        if obj_id not in cache:
            if isinstance(obj, Pickled):  # already pickled
                cache[obj_id] = obj
            else:  # pickle the object
                cache[obj_id] = Pickled(obj)
        out.append(cache[obj_id])
    return out

def safely_call(func, args):
    """
    Call the given function with the given arguments safely, i.e.
    by trapping the exceptions. Return a pair (result, exc_type)
    where exc_type is None if no exceptions occur, otherwise it
    is the exception class and the result is a string containing
    error message and traceback.

    :param func: the function to call
    :param args: the arguments
    """
    try:
        return func(*args), None
    except:
        etype, exc, tb = sys.exc_info()
        tb_str = ''.join(traceback.format_tb(tb))
        return '\n%s%s: %s' % (tb_str, etype.__name__, exc), etype

def map_reduce(task, task_args, agg, acc):
    """
    Given a task and an iterable of positional arguments, apply the
    task function to the arguments in parallel and return an aggregate
    result depending on the initial value of the accumulator
    and on the aggregation function. To save memory, the order is
    not preserved and there is no list with the intermediated results:
    the accumulator is incremented as soon as a task result comes.

    NB: if the environment variable OQ_NO_DISTRIBUTE is set the
    tasks are run sequentially in the current process and then
    map_reduce(task, task_args, agg, acc) is the same as
    reduce(agg, itertools.starmap(task, task_args), acc).
    Users of map_reduce should be aware of the fact that when
    thousands of tasks are spawned and large arguments are passed
    or large results are returned they may incur in memory issue:
    this is way the calculators limit the queue with the
    `concurrent_task` concept.

    :param task: a `celery` task callable.
    :param task_args: an iterable over positional arguments
    :param agg: the aggregation function, (acc, val) -> new acc
    :param acc: the initial value of the accumulator
    :returns: the final value of the accumulator
    """
    if no_distribute():
        for the_args in task_args:
            result, exctype = safely_call(task.task_func, the_args)
            if exctype:
                raise exctype(result)
            acc = agg(acc, result)
    else:
        backend = current_app().backend
        unpik = 0
        job_id = task_args[0][0]
        taskname = task.__name__
        mon = LightMonitor('unpickling %s' % taskname, job_id, task)
        to_send = 0
        pickled_args = []
        for args in task_args:
            piks = pickle_sequence(args)
            pickled_args.append(piks)
            to_send += sum(len(p) for p in piks)
        logs.LOG.info('Sending %dM', to_send / ONE_MB)
        taskset = TaskSet(tasks=map(task.subtask, pickled_args))
        for task_id, result_dict in taskset.apply_async().iter_native():
            check_mem_usage()  # log a warning if too much memory is used
            result_pik = result_dict['result']
            with mon:
                result, exctype = result_pik.unpickle()
            if exctype:
                raise exctype(result)
            unpik += len(result_pik)
            acc = agg(acc, result)
            del backend._cache[task_id]  # work around a celery bug
        logs.LOG.info('Unpickled %dM of received data in %s seconds',
                      unpik / ONE_MB, mon.duration)
    return acc

# used to implement BaseCalculator.parallelize, which takes in account
# the `concurrent_task` concept to avoid filling the Celery queue
def parallelize(task, task_args, side_effect=lambda val: None):
    """
    Given a celery task and an iterable of positional arguments, apply the
    callable to the arguments in parallel. It is possible to pass a
    function side_effect(val) which takes the return value of the
    callable and does something with it (such as saving or printing
    it). Notice that the order is not preserved. parallelize returns None.

    NB: if the environment variable OQ_NO_DISTRIBUTE is set the
    tasks are run sequentially in the current process.

    :param task: a celery task
    :param task_args: an iterable over positional arguments
    :param side_effect: a function val -> None
    """
    map_reduce(task, task_args, lambda acc, val: side_effect(val), None)

def oqtask(task_func):
    """
    Task function decorator which sets up logging and catches (and logs) any
    errors which occur inside the task. Also checks to make sure the job is
    actually still running. If it is not running, the task doesn't get
    executed, so we don't do useless computation.

    :param task_func: the function to decorate
    """
    def wrapped(*args):
        """
        Initialize logs, make sure the job is still running, and run the task
        code surrounded by a try-except. If any error occurs, log it as a
        critical failure.
        """
        # job_id is always assumed to be the first argument
        job_id = args[0]
        job = models.OqJob.objects.get(id=job_id)
        if job.is_running is False:
            # the job was killed, it is useless to run the task
            return

        # it is important to save the task id soon, so that
        # the revoke functionality can work
        EnginePerformanceMonitor.store_task_id(job_id, tsk)

        with EnginePerformanceMonitor(
                'total ' + task_func.__name__, job_id, tsk, flush=True):
            # tasks write on the celery log file
            logs.set_level(job.log_level)
            check_mem_usage()  # log a warning if too much memory is used
            try:
                # run the task
                return task_func(*args)
            finally:
                # save on the db
                CacheInserter.flushall()
                # the task finished, we can remove from the performance
                # table the associated row 'storing task id'
                models.Performance.objects.filter(
                    oq_job=job,
                    operation='storing task id',
                    task_id=tsk.request.id).delete()
    celery_queue = config.get('amqp', 'celery_queue')
    f = lambda *args: Pickled(
        safely_call(wrapped, [a.unpickle() for a in args]))
    f.__name__ = task_func.__name__
    tsk = task(f, queue=celery_queue)
    tsk.task_func = task_func
    return tsk
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-

import re
from itertools import zip_longest

import requests
import lxml.html

from utils.config import get_config_section

crawler_config = get_config_section('crawler')

encoding = crawler_config['encoding']  # big5 superset

form_url = crawler_config['form_url']
form_action_url = crawler_config['form_action_url']
syllabus_url = crawler_config['syllabus_url']
attachment_url = crawler_config['attachment_url']
dept_url = crawler_config['dept_url']

def xpath0(element, xpath):
    result = element.xpath(xpath)
    assert len(result) == 1, result
    return result[0]

def extract_text(element, joiner=''):
    return joiner.join(element.itertext()).strip()

def extract_int(element, allow_empty=False):
    text_list = element.xpath('.//text()')
    assert len(text_list) == 1, text_list
    if allow_empty and not text_list[0].strip():
        return None
    return int(text_list[0])

def extract_multirow_text(element):
    return re.sub(
        r'\n{4,}',
        '\n' * 3,
        '\n'.join(
            s.strip() for s in element.itertext()
        )
    )

class EmptyResponse(Exception):
    pass

def with_retry(request_function):
    def function(url, max_retries=32, **kwargs):
        '''
        get a valid response in <max_retries> retries
        change encoding before return
        raises EmptyResponse if not valid
        '''
        for r in range(max_retries):
            response = request_function(url, **kwargs)
            if response.content:
                response.encoding = encoding
                return response
        raise EmptyResponse(url)
    function.__name__ = request_function.__name__
    return function

def get_slfr(text):
    sl, s, fr = text.partition(u'大一新生保留')
    if not sl:
        return None, 0
    sl = int(sl)
    if fr:
        assert fr[-1] == u'人', fr
        fr = int(fr[:-1])
    else:
        fr = 0
    return sl, fr

get = with_retry(requests.get)
post = with_retry(requests.post)

def curriculum_to_trs(html):
    document = lxml.html.fromstring(html)
    course_trs = document.xpath("//tr[contains(@class, 'class3')]")
    assert len(course_trs) % 2 == 0, len(course_trs)
    return course_trs[::2]

def course_from_tr(main_tr):
    '''
    main_tr -> dict: course data

    data info: ([x] means implemented but disabled)
    no                  text            course number
    name_zh             text            Chinese course name
    name_en             text / None     English course name
    ge_hint             text / None     GE line text
    credit [x]          int             credit
    time [x]            text            time
    rc [x]              text            room & capacity
    teacher             text            teacher name, may contain English
    size_limit          int / None      quota size limit
    fr                  int             quota reserved for freshmen
    note                text            note
    enrollment          int             quota enrollment
    object              text            object
    has_prerequisite    bool            has prerequisite
    '''
    tds = main_tr.xpath('td')
    part = {
        'no': extract_text(tds[0]),
        # 'credit': extract_int(tds[2]),
        # 'time': extract_text(tds[3]),
        # 'room/capacity': extract_text(tds[4]),
        'teacher': extract_text(tds[5]),
        'note': extract_text(tds[7]),
        'enrollment': extract_int(tds[8]),
        'object': extract_text(tds[9]),
    }
    part['size_limit'], part['fr'] = get_slfr(extract_text(tds[6]))
    for key, text in zip_longest(
        ('name_zh', 'name_en', 'ge_hint'),
        tds[1].itertext(),
    ):
        if text is not None:
            text = text.strip()
        part[key] = text
    prerequisite = extract_text(tds[10])
    if prerequisite == u'擋修' or u'先修科目' in prerequisite:
        part['has_prerequisite'] = True
    else:
        assert not prerequisite, 'unknown prerequisite %r' % prerequisite
        part['has_prerequisite'] = False
    return part

def course_from_syllabus(html):
    '''
    syllabus html -> dict: course data

    data info:
    no              text            course number
    name_zh         text            Chinese course name
    name_en         text            English course name
    credit          int             credit
    time            text            time
    room            text            room
    syllabus        text            syllabus, no html
    has_attachment  bool            has attachment

    note: to get the attachment, simply visit:
    https://www.ccxp.nthu.edu.tw/ccxp/INQUIRE/JH/output/6_6.1_6.1.12/<no>.pdf
    where <no> is the course number
    '''
    document = lxml.html.fromstring(html)

    def xpath_text(xpath, joiner=''):
        return extract_text(xpath0(document, xpath), joiner=joiner)

    def xpath_int(xpath):
        return extract_int(xpath0(document, xpath))

    def patch_teacher(text):
        '''
        Save only Chinese names for teachers.

        See issue #86.
        '''
        return re.sub(r'\([^)]*\)', '', text)

    return {
        'no': xpath_text('/html/body/div/table[1]/tr[2]/td[2]'),
        'name_zh': xpath_text('/html/body/div/table[1]/tr[3]/td[2]'),
        'name_en': xpath_text('/html/body/div/table[1]/tr[4]/td[2]'),
        'credit': xpath_text('/html/body/div/table[1]/tr[2]/td[4]'),
        'teacher': patch_teacher(xpath_text(
            '/html/body/div/table[1]/tr[5]/td[2]', joiner=', ')),
        'time': xpath_text('/html/body/div/table[1]/tr[6]/td[2]'),
        'room': xpath_text('/html/body/div/table[1]/tr[6]/td[4]', joiner=', '),
        'syllabus': extract_multirow_text(
            xpath0(document, '/html/body/div/table[4]/tr[2]/td')),
        'has_attachment': bool(
            document.xpath('/html/body/div/table[4]/tr[2]/td/div/font[1]/a')),
    }

def get_syllabus(c_key, acixstore):
    return get(syllabus_url, params={'c_key': c_key, 'ACIXSTORE': acixstore})

def get_cou_codes():
    html = get(form_url).text
    document = lxml.html.fromstring(html)
    return document.xpath('//select[@name="cou_code"]/option/@value')
<commit_msg>Fix issue #109<commit_after># -*- encoding: utf-8 -*-

import re
from itertools import zip_longest

import requests
import lxml.html

from utils.config import get_config_section

crawler_config = get_config_section('crawler')

encoding = crawler_config['encoding']  # big5 superset

form_url = crawler_config['form_url']
form_action_url = crawler_config['form_action_url']
syllabus_url = crawler_config['syllabus_url']
attachment_url = crawler_config['attachment_url']
dept_url = crawler_config['dept_url']

def xpath0(element, xpath):
    result = element.xpath(xpath)
    assert len(result) == 1, result
    return result[0]

def extract_text(element, joiner=''):
    return joiner.join(element.itertext()).strip()

def extract_int(element, allow_empty=False):
    text_list = element.xpath('.//text()')
    assert len(text_list) == 1, text_list
    if allow_empty and not text_list[0].strip():
        return None
    return int(text_list[0])

def extract_multirow_text(element):
    return re.sub(
        r'\n{4,}',
        '\n' * 3,
        '\n'.join(
            s.strip() for s in element.itertext()
        )
    )

class EmptyResponse(Exception):
    pass

def with_retry(request_function):
    def function(url, max_retries=32, **kwargs):
        '''
        get a valid response in <max_retries> retries
        change encoding before return
        raises EmptyResponse if not valid
        '''
        for r in range(max_retries):
            response = request_function(url, **kwargs)
            if response.content:
                response.encoding = encoding
                return response
        raise EmptyResponse(url)
    function.__name__ = request_function.__name__
    return function

def get_slfr(text):
    sl, s, fr = text.partition(u'新生保留')
    if not sl:
        return None, 0
    sl = int(sl)
    if fr:
        assert fr[-1] == u'人', fr
        fr = int(fr[:-1])
    else:
        fr = 0
    return sl, fr

get = with_retry(requests.get)
post = with_retry(requests.post)

def curriculum_to_trs(html):
    document = lxml.html.fromstring(html)
    course_trs = document.xpath("//tr[contains(@class, 'class3')]")
    assert len(course_trs) % 2 == 0, len(course_trs)
    return course_trs[::2]

def course_from_tr(main_tr):
    '''
    main_tr -> dict: course data

    data info: ([x] means implemented but disabled)
    no                  text            course number
    name_zh             text            Chinese course name
    name_en             text / None     English course name
    ge_hint             text / None     GE line text
    credit [x]          int             credit
    time [x]            text            time
    rc [x]              text            room & capacity
    teacher             text            teacher name, may contain English
    size_limit          int / None      quota size limit
    fr                  int             quota reserved for freshmen
    note                text            note
    enrollment          int             quota enrollment
    object              text            object
    has_prerequisite    bool            has prerequisite
    '''
    tds = main_tr.xpath('td')
    part = {
        'no': extract_text(tds[0]),
        # 'credit': extract_int(tds[2]),
        # 'time': extract_text(tds[3]),
        # 'room/capacity': extract_text(tds[4]),
        'teacher': extract_text(tds[5]),
        'note': extract_text(tds[7]),
        'enrollment': extract_int(tds[8]),
        'object': extract_text(tds[9]),
    }
    part['size_limit'], part['fr'] = get_slfr(extract_text(tds[6]))
    for key, text in zip_longest(
        ('name_zh', 'name_en', 'ge_hint'),
        tds[1].itertext(),
    ):
        if text is not None:
            text = text.strip()
        part[key] = text
    prerequisite = extract_text(tds[10])
    if prerequisite == u'擋修' or u'先修科目' in prerequisite:
        part['has_prerequisite'] = True
    else:
        assert not prerequisite, 'unknown prerequisite %r' % prerequisite
        part['has_prerequisite'] = False
    return part

def course_from_syllabus(html):
    '''
    syllabus html -> dict: course data

    data info:
    no              text            course number
    name_zh         text            Chinese course name
    name_en         text            English course name
    credit          int             credit
    time            text            time
    room            text            room
    syllabus        text            syllabus, no html
    has_attachment  bool            has attachment

    note: to get the attachment, simply visit:
    https://www.ccxp.nthu.edu.tw/ccxp/INQUIRE/JH/output/6_6.1_6.1.12/<no>.pdf
    where <no> is the course number
    '''
    document = lxml.html.fromstring(html)

    def xpath_text(xpath, joiner=''):
        return extract_text(xpath0(document, xpath), joiner=joiner)

    def xpath_int(xpath):
        return extract_int(xpath0(document, xpath))

    def patch_teacher(text):
        '''
        Save only Chinese names for teachers.

        See issue #86.
        '''
        return re.sub(r'\([^)]*\)', '', text)

    return {
        'no': xpath_text('/html/body/div/table[1]/tr[2]/td[2]'),
        'name_zh': xpath_text('/html/body/div/table[1]/tr[3]/td[2]'),
        'name_en': xpath_text('/html/body/div/table[1]/tr[4]/td[2]'),
        'credit': xpath_text('/html/body/div/table[1]/tr[2]/td[4]'),
        'teacher': patch_teacher(xpath_text(
            '/html/body/div/table[1]/tr[5]/td[2]', joiner=', ')),
        'time': xpath_text('/html/body/div/table[1]/tr[6]/td[2]'),
        'room': xpath_text('/html/body/div/table[1]/tr[6]/td[4]', joiner=', '),
        'syllabus': extract_multirow_text(
            xpath0(document, '/html/body/div/table[4]/tr[2]/td')),
        'has_attachment': bool(
            document.xpath('/html/body/div/table[4]/tr[2]/td/div/font[1]/a')),
    }

def get_syllabus(c_key, acixstore):
    return get(syllabus_url, params={'c_key': c_key, 'ACIXSTORE': acixstore})

def get_cou_codes():
    html = get(form_url).text
    document = lxml.html.fromstring(html)
    return document.xpath('//select[@name="cou_code"]/option/@value')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2012-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

"""
Module :mod:`~openquake.hazardlib.calc.gmf` exports
:func:`ground_motion_fields`.
"""

import numpy
import scipy.stats

from openquake.baselib.python3compat import zip
from openquake.baselib.general import get_array
from openquake.hazardlib.const import StdDev
from openquake.hazardlib.calc import filters
from openquake.hazardlib.gsim.base import ContextMaker
from openquake.hazardlib.imt import from_string

U8 = numpy.uint8
U16 = numpy.uint16
U32 = numpy.uint32
F32 = numpy.float32

gmv_dt = numpy.dtype([('sid', U16), ('eid', U32), ('rlzi', U16),
                      ('imti', U8), ('gmv', F32)])

class CorrelationButNoInterIntraStdDevs(Exception):
    def __init__(self, corr, gsim):
        self.corr = corr
        self.gsim = gsim

    def __str__(self):
        return '''\
You cannot use the correlation model %s with the GSIM %s, \
that defines only the total standard deviation. If you want to use a \
correlation model you have to select a GMPE that provides the inter and \
intra event standard deviations.''' % (
            self.corr.__class__.__name__, self.gsim.__class__.__name__)

class GmfComputer(object):
    """
    Given an earthquake rupture, the ground motion field computer computes
    ground shaking over a set of sites, by randomly sampling a ground
    shaking intensity model.

    :param :class:`openquake.hazardlib.source.rupture.Rupture` rupture:
        Rupture to calculate ground motion fields radiated from.

    :param :class:`openquake.hazardlib.site.SiteCollection` sites:
        Sites of interest to calculate GMFs.

    :param imts:
        a sorted list of Intensity Measure Type strings

    :param truncation_level:
        Float, number of standard deviations for truncation of the intensity
        distribution, or ``None``.

    :param correlation_model:
        Instance of correlation model object. See
        :mod:`openquake.hazardlib.correlation`. Can be ``None``, in which
        case non-correlated ground motion fields are calculated.
        Correlation model is not used if ``truncation_level`` is zero.
    """
    def __init__(self, rupture, sites, imts, gsims,
                 truncation_level=None, correlation_model=None):
        assert sites, sites
        self.rupture = rupture
        self.sites = sites
        self.imts = [from_string(imt) for imt in imts]
        self.gsims = gsims
        self.truncation_level = truncation_level
        self.correlation_model = correlation_model
        self.ctx = ContextMaker(gsims).make_contexts(sites, rupture)

    def _compute(self, seed, gsim, realizations):
        # the method doing the real stuff; use compute instead
        if seed is not None:
            numpy.random.seed(seed)
        result = numpy.zeros(
            (len(self.imts), len(self.sites), realizations), F32)
        sctx, rctx, dctx = self.ctx

        if self.truncation_level == 0:
            assert self.correlation_model is None
            for imti, imt in enumerate(self.imts):
                mean, _stddevs = gsim.get_mean_and_stddevs(
                    sctx, rctx, dctx, imt, stddev_types=[])
                mean = gsim.to_imt_unit_values(mean)
                mean.shape += (1, )
                mean = mean.repeat(realizations, axis=1)
                result[imti] = mean
            return result
        elif self.truncation_level is None:
            distribution = scipy.stats.norm()
        else:
            assert self.truncation_level > 0
            distribution = scipy.stats.truncnorm(
                - self.truncation_level, self.truncation_level)

        for imti, imt in enumerate(self.imts):
            if gsim.DEFINED_FOR_STANDARD_DEVIATION_TYPES == \
               set([StdDev.TOTAL]):
                # If the GSIM provides only total standard deviation, we need
                # to compute mean and total standard deviation at the sites
                # of interest.
                # In this case, we also assume no correlation model is used.
                if self.correlation_model:
                    raise CorrelationButNoInterIntraStdDevs(
                        self.correlation_model, gsim)

                mean, [stddev_total] = gsim.get_mean_and_stddevs(
                    sctx, rctx, dctx, imt, [StdDev.TOTAL])
                stddev_total = stddev_total.reshape(stddev_total.shape + (1, ))
                mean = mean.reshape(mean.shape + (1, ))

                total_residual = stddev_total * distribution.rvs(
                    size=(len(self.sites), realizations))
                gmf = gsim.to_imt_unit_values(mean + total_residual)
            else:
                mean, [stddev_inter, stddev_intra] = gsim.get_mean_and_stddevs(
                    sctx, rctx, dctx, imt,
                    [StdDev.INTER_EVENT, StdDev.INTRA_EVENT])
                stddev_intra = stddev_intra.reshape(stddev_intra.shape + (1, ))
                stddev_inter = stddev_inter.reshape(stddev_inter.shape + (1, ))
                mean = mean.reshape(mean.shape + (1, ))

                intra_residual = stddev_intra * distribution.rvs(
                    size=(len(self.sites), realizations))

                if self.correlation_model is not None:
                    ir = self.correlation_model.apply_correlation(
                        self.sites, imt, intra_residual)
                    # this fixes a mysterious bug: ir[row] is actually
                    # a matrix of shape (E, 1) and not a vector of size E
                    intra_residual = numpy.zeros(ir.shape)
                    for i, val in numpy.ndenumerate(ir):
                        intra_residual[i] = val

                inter_residual = stddev_inter * distribution.rvs(
                    size=realizations)

                gmf = gsim.to_imt_unit_values(
                    mean + intra_residual + inter_residual)

            result[imti] = gmf

        return result

    def compute(self, seed, events, rlzs_by_gsim=None, min_iml=None):
        """
        Compute a ground motion array for the given sites.

        :param seed:
            seed for the numpy random number generator
        :param events:
            composite array of seismic events (eid, ses, occ)
        :param rlzs_by_gsim:
            a dictionary {gsim instance: realizations}
        :param min_iml:
            an array minimum intensity per intensity measure type
        :returns:
            a numpy array of dtype gmv_dt
        """
        gmfa = []
        for eid, imti, rlz, gmf_sids in self.calcgmfs(
                seed, events, rlzs_by_gsim, min_iml):
            for gmv, sid in zip(*gmf_sids):
                gmfa.append((sid, eid, rlz.ordinal, imti, gmv))
        return numpy.array(gmfa, gmv_dt)

    def calcgmfs(self, seed, events, rlzs_by_gsim, min_iml=None):
        """
        Compute the ground motion fields for the given gsims, sites,
        multiplicity and seed.

        :param seed:
            seed for the numpy random number generator
        :param events:
            composite array of seismic events (eid, ses, occ)
        :param rlzs_by_gsim:
            a dictionary {gsim instance: realizations}
        :yields:
            tuples (eid, imti, rlz, gmf_sids)
        """
        sids = self.sites.sids
        imt_range = range(len(self.imts))
        for i, gsim in enumerate(self.gsims):
            for j, rlz in enumerate(rlzs_by_gsim[gsim]):
                if isinstance(events, int):  # for scenario
                    eids = range(events)
                else:
                    eids = get_array(events, sample=rlz.sampleid)['eid']
                arr = self._compute(seed + j, gsim, len(eids)).transpose(
                    0, 2, 1)  # array of shape (I, E, S)
                for imti in imt_range:
                    for eid, gmf in zip(eids, arr[imti]):
                        if min_iml is not None:  # is an array
                            ok = gmf >= min_iml[imti]
                            gmf_sids = (gmf[ok], sids[ok])
                        else:
                            gmf_sids = (gmf, sids)
                        if len(gmf):
                            yield eid, imti, rlz, gmf_sids

# this is not used in the engine; it is still useful for usage in IPython
# when demonstrating hazardlib capabilities
def ground_motion_fields(rupture, sites, imts, gsim, truncation_level,
                         realizations, correlation_model=None,
                         rupture_site_filter=filters.rupture_site_noop_filter,
                         seed=None):
    """
    Given an earthquake rupture, the ground motion field calculator computes
    ground shaking over a set of sites, by randomly sampling a ground shaking
    intensity model. A ground motion field represents a possible 'realization'
    of the ground shaking due to an earthquake rupture. If a non-trivial
    filtering function is passed, the final result is expanded and filled
    with zeros in the places corresponding to the filtered out sites.

    .. note::

     This calculator is using random numbers. In order to reproduce the
     same results numpy random numbers generator needs to be seeded, see
     http://docs.scipy.org/doc/numpy/reference/generated/numpy.random.seed.html

    :param openquake.hazardlib.source.rupture.Rupture rupture:
        Rupture to calculate ground motion fields radiated from.
    :param openquake.hazardlib.site.SiteCollection sites:
        Sites of interest to calculate GMFs.
    :param imts:
        List of intensity measure type objects (see
        :mod:`openquake.hazardlib.imt`).
    :param gsim:
        Ground-shaking intensity model, instance of subclass of either
        :class:`~openquake.hazardlib.gsim.base.GMPE` or
        :class:`~openquake.hazardlib.gsim.base.IPE`.
    :param truncation_level:
        Float, number of standard deviations for truncation of the intensity
        distribution, or ``None``.
    :param realizations:
        Integer number of GMF realizations to compute.
    :param correlation_model:
        Instance of correlation model object. See
        :mod:`openquake.hazardlib.correlation`. Can be ``None``, in which case
        non-correlated ground motion fields are calculated. Correlation model
        is not used if ``truncation_level`` is zero.
    :param rupture_site_filter:
        Optional rupture-site filter function. See
        :mod:`openquake.hazardlib.calc.filters`.
    :param int seed:
        The seed used in the numpy random number generator
    :returns:
        Dictionary mapping intensity measure type objects (same
        as in parameter ``imts``) to 2d numpy arrays of floats,
        representing different realizations of ground shaking intensity
        for all sites in the collection. First dimension represents
        sites and second one is for realizations.
    """
    ruptures_sites = list(rupture_site_filter([(rupture, sites)]))
    if not ruptures_sites:
        return dict((imt, numpy.zeros((len(sites), realizations)))
                    for imt in imts)
    [(rupture, sites)] = ruptures_sites
    gc = GmfComputer(rupture, sites, [str(imt) for imt in imts], [gsim],
                     truncation_level, correlation_model)
    res = gc._compute(seed, gsim, realizations)
    result = {}
    for imti, imt in enumerate(gc.imts):
        # makes sure the lenght of the arrays in output is the same as sites
        if rupture_site_filter is not filters.rupture_site_noop_filter:
            result[imt] = sites.expand(res[imti], placeholder=0)
        else:
            result[imt] = res[imti]
    return result
<commit_msg>Removed GmfComputer.compute<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2012-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

"""
Module :mod:`~openquake.hazardlib.calc.gmf` exports
:func:`ground_motion_fields`.
"""

import numpy
import scipy.stats

from openquake.baselib.python3compat import zip
from openquake.baselib.general import get_array
from openquake.hazardlib.const import StdDev
from openquake.hazardlib.calc import filters
from openquake.hazardlib.gsim.base import ContextMaker
from openquake.hazardlib.imt import from_string

class CorrelationButNoInterIntraStdDevs(Exception):
    def __init__(self, corr, gsim):
        self.corr = corr
        self.gsim = gsim

    def __str__(self):
        return '''\
You cannot use the correlation model %s with the GSIM %s, \
that defines only the total standard deviation. If you want to use a \
correlation model you have to select a GMPE that provides the inter and \
intra event standard deviations.''' % (
            self.corr.__class__.__name__, self.gsim.__class__.__name__)

class GmfComputer(object):
    """
    Given an earthquake rupture, the ground motion field computer computes
    ground shaking over a set of sites, by randomly sampling a ground
    shaking intensity model.

    :param :class:`openquake.hazardlib.source.rupture.Rupture` rupture:
        Rupture to calculate ground motion fields radiated from.

    :param :class:`openquake.hazardlib.site.SiteCollection` sites:
        Sites of interest to calculate GMFs.

    :param imts:
        a sorted list of Intensity Measure Type strings

    :param truncation_level:
        Float, number of standard deviations for truncation of the intensity
        distribution, or ``None``.

    :param correlation_model:
        Instance of correlation model object. See
        :mod:`openquake.hazardlib.correlation`. Can be ``None``, in which
        case non-correlated ground motion fields are calculated.
        Correlation model is not used if ``truncation_level`` is zero.
    """
    def __init__(self, rupture, sites, imts, gsims,
                 truncation_level=None, correlation_model=None, samples=0):
        assert sites, sites
        self.rupture = rupture
        self.sites = sites
        self.imts = [from_string(imt) for imt in imts]
        self.gsims = gsims
        self.truncation_level = truncation_level
        self.correlation_model = correlation_model
        self.samples = samples
        self.ctx = ContextMaker(gsims).make_contexts(sites, rupture)

    def _compute(self, seed, gsim, realizations):
        # the method doing the real stuff; use compute instead
        if seed is not None:
            numpy.random.seed(seed)
        result = numpy.zeros(
            (len(self.imts), len(self.sites), realizations), numpy.float32)
        sctx, rctx, dctx = self.ctx

        if self.truncation_level == 0:
            assert self.correlation_model is None
            for imti, imt in enumerate(self.imts):
                mean, _stddevs = gsim.get_mean_and_stddevs(
                    sctx, rctx, dctx, imt, stddev_types=[])
                mean = gsim.to_imt_unit_values(mean)
                mean.shape += (1, )
                mean = mean.repeat(realizations, axis=1)
                result[imti] = mean
            return result
        elif self.truncation_level is None:
            distribution = scipy.stats.norm()
        else:
            assert self.truncation_level > 0
            distribution = scipy.stats.truncnorm(
                - self.truncation_level, self.truncation_level)

        for imti, imt in enumerate(self.imts):
            if gsim.DEFINED_FOR_STANDARD_DEVIATION_TYPES == \
               set([StdDev.TOTAL]):
                # If the GSIM provides only total standard deviation, we need
                # to compute mean and total standard deviation at the sites
                # of interest.
                # In this case, we also assume no correlation model is used.
                if self.correlation_model:
                    raise CorrelationButNoInterIntraStdDevs(
                        self.correlation_model, gsim)

                mean, [stddev_total] = gsim.get_mean_and_stddevs(
                    sctx, rctx, dctx, imt, [StdDev.TOTAL])
                stddev_total = stddev_total.reshape(stddev_total.shape + (1, ))
                mean = mean.reshape(mean.shape + (1, ))

                total_residual = stddev_total * distribution.rvs(
                    size=(len(self.sites), realizations))
                gmf = gsim.to_imt_unit_values(mean + total_residual)
            else:
                mean, [stddev_inter, stddev_intra] = gsim.get_mean_and_stddevs(
                    sctx, rctx, dctx, imt,
                    [StdDev.INTER_EVENT, StdDev.INTRA_EVENT])
                stddev_intra = stddev_intra.reshape(stddev_intra.shape + (1, ))
                stddev_inter = stddev_inter.reshape(stddev_inter.shape + (1, ))
                mean = mean.reshape(mean.shape + (1, ))

                intra_residual = stddev_intra * distribution.rvs(
                    size=(len(self.sites), realizations))

                if self.correlation_model is not None:
                    ir = self.correlation_model.apply_correlation(
                        self.sites, imt, intra_residual)
                    # this fixes a mysterious bug: ir[row] is actually
                    # a matrix of shape (E, 1) and not a vector of size E
                    intra_residual = numpy.zeros(ir.shape)
                    for i, val in numpy.ndenumerate(ir):
                        intra_residual[i] = val

                inter_residual = stddev_inter * distribution.rvs(
                    size=realizations)

                gmf = gsim.to_imt_unit_values(
                    mean + intra_residual + inter_residual)

            result[imti] = gmf

        return result

    def calcgmfs(self, seed, events, rlzs_by_gsim, min_iml=None):
        """
        Compute the ground motion fields for the given gsims, sites,
        multiplicity and seed.

        :param seed:
            seed for the numpy random number generator
        :param events:
            composite array of seismic events (eid, ses, occ)
        :param rlzs_by_gsim:
            a dictionary {gsim instance: realizations}
        :yields:
            tuples (eid, imti, rlz, gmf_sids)
        """
        sids = self.sites.sids
        imt_range = range(len(self.imts))
        for i, gsim in enumerate(self.gsims):
            for j, rlz in enumerate(rlzs_by_gsim[gsim]):
                if isinstance(events, int):  # for scenario
                    eids = range(events)
                elif self.samples:
                    eids = get_array(events, sample=rlz.sampleid)['eid']
                else:
                    eids = events['eid']
                arr = self._compute(seed + j, gsim, len(eids)).transpose(
                    0, 2, 1)  # array of shape (I, E, S)
                for imti in imt_range:
                    for eid, gmf in zip(eids, arr[imti]):
                        if min_iml is not None:  # is an array
                            ok = gmf >= min_iml[imti]
                            gmf_sids = (gmf[ok], sids[ok])
                        else:
                            gmf_sids = (gmf, sids)
                        if len(gmf):
                            yield eid, imti, rlz, gmf_sids

# this is not used in the engine; it is still useful for usage in IPython
# when demonstrating hazardlib capabilities
def ground_motion_fields(rupture, sites, imts, gsim, truncation_level,
                         realizations, correlation_model=None,
                         rupture_site_filter=filters.rupture_site_noop_filter,
                         seed=None):
    """
    Given an earthquake rupture, the ground motion field calculator computes
    ground shaking over a set of sites, by randomly sampling a ground shaking
    intensity model. A ground motion field represents a possible 'realization'
    of the ground shaking due to an earthquake rupture. If a non-trivial
    filtering function is passed, the final result is expanded and filled
    with zeros in the places corresponding to the filtered out sites.

    .. note::

     This calculator is using random numbers. In order to reproduce the
     same results numpy random numbers generator needs to be seeded, see
     http://docs.scipy.org/doc/numpy/reference/generated/numpy.random.seed.html

    :param openquake.hazardlib.source.rupture.Rupture rupture:
        Rupture to calculate ground motion fields radiated from.
    :param openquake.hazardlib.site.SiteCollection sites:
        Sites of interest to calculate GMFs.
    :param imts:
        List of intensity measure type objects (see
        :mod:`openquake.hazardlib.imt`).
    :param gsim:
        Ground-shaking intensity model, instance of subclass of either
        :class:`~openquake.hazardlib.gsim.base.GMPE` or
        :class:`~openquake.hazardlib.gsim.base.IPE`.
    :param truncation_level:
        Float, number of standard deviations for truncation of the intensity
        distribution, or ``None``.
    :param realizations:
        Integer number of GMF realizations to compute.
    :param correlation_model:
        Instance of correlation model object. See
        :mod:`openquake.hazardlib.correlation`. Can be ``None``, in which case
        non-correlated ground motion fields are calculated. Correlation model
        is not used if ``truncation_level`` is zero.
    :param rupture_site_filter:
        Optional rupture-site filter function. See
        :mod:`openquake.hazardlib.calc.filters`.
    :param int seed:
        The seed used in the numpy random number generator
    :returns:
        Dictionary mapping intensity measure type objects (same
        as in parameter ``imts``) to 2d numpy arrays of floats,
        representing different realizations of ground shaking intensity
        for all sites in the collection. First dimension represents
        sites and second one is for realizations.
    """
    ruptures_sites = list(rupture_site_filter([(rupture, sites)]))
    if not ruptures_sites:
        return dict((imt, numpy.zeros((len(sites), realizations)))
                    for imt in imts)
    [(rupture, sites)] = ruptures_sites
    gc = GmfComputer(rupture, sites, [str(imt) for imt in imts], [gsim],
                     truncation_level, correlation_model)
    res = gc._compute(seed, gsim, realizations)
    result = {}
    for imti, imt in enumerate(gc.imts):
        # makes sure the lenght of the arrays in output is the same as sites
        if rupture_site_filter is not filters.rupture_site_noop_filter:
            result[imt] = sites.expand(res[imti], placeholder=0)
        else:
            result[imt] = res[imti]
    return result
<|endoftext|>