<commit_before>
from django.test import TestCase
from django.contrib.auth import get_user_model

from model_mommy import mommy

from rolepermissions.exceptions import RoleDoesNotExist
from rolepermissions.roles import RolesManager, AbstractUserRole
from rolepermissions.shortcuts import (
    get_user_role, grant_permission,
    revoke_permission, retrieve_role,
    available_perm_status
)
from rolepermissions.verifications import has_permission

class ShoRole1(AbstractUserRole):
    available_permissions = {
        'permission1': True,
        'permission2': True,
    }

class ShoRole2(AbstractUserRole):
    available_permissions = {
        'permission3': True,
        'permission4': False,
    }

class ShoRole3(AbstractUserRole):
    role_name = 'sho_new_name'
    available_permissions = {
        'permission5': False,
        'permission6': False,
    }

class AvailablePermStatusTests(TestCase):

    def setUp(self):
        self.user = mommy.make(get_user_model())
        self.user_role = ShoRole2.assign_role_to_user(self.user)

    def test_permission_hash(self):
        perm_hash = available_perm_status(self.user)

        self.assertTrue(perm_hash['permission3'])
        self.assertFalse(perm_hash['permission4'])

class GetUserRoleTests(TestCase):

    def setUp(self):
        self.user = mommy.make(get_user_model())

    def test_get_user_role(self):
        user = self.user

        user_role = ShoRole1.assign_role_to_user(user)

        self.assertEquals(get_user_role(user), ShoRole1)

    def test_get_user_role_after_role_change(self):
        user = self.user

        user_role = ShoRole1.assign_role_to_user(user)
        user_role = ShoRole3.assign_role_to_user(user)

        self.assertEquals(get_user_role(user), ShoRole3)

    def test_user_without_role(self):
        user = self.user

        self.assertIsNone(get_user_role(user))

    def tearDown(self):
        RolesManager._roles = {}

class GrantPermissionTests(TestCase):

    def setUp(self):
        self.user = mommy.make(get_user_model())
        self.user_role = ShoRole2.assign_role_to_user(self.user)

    def test_grant_permission(self):
        user = self.user

        self.assertTrue(grant_permission(user, 'permission4'))

        self.assertTrue(has_permission(user, 'permission4'))

    def test_grat_granted_permission(self):
        user = self.user

        self.assertTrue(grant_permission(user, 'permission3'))

        self.assertTrue(has_permission(user, 'permission3'))

    def test_not_allowed_permission(self):
        user = self.user

        self.assertFalse(grant_permission(user, 'permission1'))

class RevokePermissionTests(TestCase):

    def setUp(self):
        self.user = mommy.make(get_user_model())
        self.user_role = ShoRole2.assign_role_to_user(self.user)

    def test_revoke_permission(self):
        user = self.user

        self.assertTrue(revoke_permission(user, 'permission3'))

        self.assertFalse(has_permission(user, 'permission3'))

    def test_revoke_revoked_permission(self):
        user = self.user

        self.assertTrue(revoke_permission(user, 'permission4'))

        self.assertFalse(has_permission(user, 'permission4'))

    def test_not_allowed_permission(self):
        user = self.user

        self.assertFalse(revoke_permission(user, 'permission1'))

class RetrieveRole(TestCase):

    def setUp(self):
        pass

    def test_retrive_role1(self):
        self.assertEquals(retrieve_role('sho_role1'), ShoRole1)

    def test_retrive_role2(self):
        self.assertEquals(retrieve_role('sho_role2'), ShoRole2)

    def test_retrive_role3(self):
        self.assertEquals(retrieve_role('sho_new_name'), ShoRole3)

    def test_retrieve_unknowun_role(self):
        role = retrieve_role('unknowun_role')
        self.assertIsNone(role)
<commit_msg>testing available_perm_status after perm change<commit_after>
from django.test import TestCase
from django.contrib.auth import get_user_model

from model_mommy import mommy

from rolepermissions.exceptions import RoleDoesNotExist
from rolepermissions.roles import RolesManager, AbstractUserRole
from rolepermissions.shortcuts import (
    get_user_role, grant_permission,
    revoke_permission, retrieve_role,
    available_perm_status
)
from rolepermissions.verifications import has_permission

class ShoRole1(AbstractUserRole):
    available_permissions = {
        'permission1': True,
        'permission2': True,
    }

class ShoRole2(AbstractUserRole):
    available_permissions = {
        'permission3': True,
        'permission4': False,
    }

class ShoRole3(AbstractUserRole):
    role_name = 'sho_new_name'
    available_permissions = {
        'permission5': False,
        'permission6': False,
    }

class AvailablePermStatusTests(TestCase):

    def setUp(self):
        self.user = mommy.make(get_user_model())
        self.user_role = ShoRole2.assign_role_to_user(self.user)

    def test_permission_hash(self):
        perm_hash = available_perm_status(self.user)

        self.assertTrue(perm_hash['permission3'])
        self.assertFalse(perm_hash['permission4'])

    def test_permission_hash_after_modification(self):
        revoke_permission(self.user, 'permission3')

        perm_hash = available_perm_status(self.user)

        self.assertFalse(perm_hash['permission3'])
        self.assertFalse(perm_hash['permission4'])

class GetUserRoleTests(TestCase):

    def setUp(self):
        self.user = mommy.make(get_user_model())

    def test_get_user_role(self):
        user = self.user

        user_role = ShoRole1.assign_role_to_user(user)

        self.assertEquals(get_user_role(user), ShoRole1)

    def test_get_user_role_after_role_change(self):
        user = self.user

        user_role = ShoRole1.assign_role_to_user(user)
        user_role = ShoRole3.assign_role_to_user(user)

        self.assertEquals(get_user_role(user), ShoRole3)

    def test_user_without_role(self):
        user = self.user

        self.assertIsNone(get_user_role(user))

    def tearDown(self):
        RolesManager._roles = {}

class GrantPermissionTests(TestCase):

    def setUp(self):
        self.user = mommy.make(get_user_model())
        self.user_role = ShoRole2.assign_role_to_user(self.user)

    def test_grant_permission(self):
        user = self.user

        self.assertTrue(grant_permission(user, 'permission4'))

        self.assertTrue(has_permission(user, 'permission4'))

    def test_grat_granted_permission(self):
        user = self.user

        self.assertTrue(grant_permission(user, 'permission3'))

        self.assertTrue(has_permission(user, 'permission3'))

    def test_not_allowed_permission(self):
        user = self.user

        self.assertFalse(grant_permission(user, 'permission1'))

class RevokePermissionTests(TestCase):

    def setUp(self):
        self.user = mommy.make(get_user_model())
        self.user_role = ShoRole2.assign_role_to_user(self.user)

    def test_revoke_permission(self):
        user = self.user

        self.assertTrue(revoke_permission(user, 'permission3'))

        self.assertFalse(has_permission(user, 'permission3'))

    def test_revoke_revoked_permission(self):
        user = self.user

        self.assertTrue(revoke_permission(user, 'permission4'))

        self.assertFalse(has_permission(user, 'permission4'))

    def test_not_allowed_permission(self):
        user = self.user

        self.assertFalse(revoke_permission(user, 'permission1'))

class RetrieveRole(TestCase):

    def setUp(self):
        pass

    def test_retrive_role1(self):
        self.assertEquals(retrieve_role('sho_role1'), ShoRole1)

    def test_retrive_role2(self):
        self.assertEquals(retrieve_role('sho_role2'), ShoRole2)

    def test_retrive_role3(self):
        self.assertEquals(retrieve_role('sho_new_name'), ShoRole3)

    def test_retrieve_unknowun_role(self):
        role = retrieve_role('unknowun_role')
        self.assertIsNone(role)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2017 Ricequant, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import six

from .base_account import BaseAccount
from ...environment import Environment
from ...events import EVENT
from ...const import ACCOUNT_TYPE, POSITION_EFFECT
from ...utils.i18n import gettext as _
from ...utils.logger import user_system_log

def margin_of(order_book_id, quantity, price):
    env = Environment.get_instance()
    margin_info = env.data_proxy.get_margin_info(order_book_id)
    margin_multiplier = env.config.base.margin_multiplier
    margin_rate = margin_info['long_margin_ratio'] * margin_multiplier
    contract_multiplier = env.get_instrument(order_book_id).contract_multiplier
    return quantity * contract_multiplier * price * margin_rate

class FutureAccount(BaseAccount):

    __abandon_properties__ = [
        "daily_holding_pnl",
        "daily_realized_pnl"
    ]

    def register_event(self):
        event_bus = Environment.get_instance().event_bus
        event_bus.prepend_listener(EVENT.SETTLEMENT, self._settlement)
        event_bus.prepend_listener(EVENT.ORDER_PENDING_NEW, self._on_order_pending_new)
        event_bus.prepend_listener(EVENT.ORDER_CREATION_REJECT, self._on_order_creation_reject)
        event_bus.prepend_listener(EVENT.ORDER_CANCELLATION_PASS, self._on_order_unsolicited_update)
        event_bus.prepend_listener(EVENT.ORDER_UNSOLICITED_UPDATE, self._on_order_unsolicited_update)
        event_bus.prepend_listener(EVENT.TRADE, self._on_trade)

    def fast_forward(self, orders, trades=list()):
        # 计算 Positions
        for trade in trades:
            if trade.exec_id in self._backward_trade_set:
                continue
            self._apply_trade(trade)
        # 计算 Frozen Cash
        self._frozen_cash = sum(self._frozen_cash_of_order(order) for order in orders if order.is_active())

    def get_state(self):
        return {
            'positions': {
                order_book_id: position.get_state()
                for order_book_id, position in six.iteritems(self._positions)
                },
            'frozen_cash': self._frozen_cash,
            'total_cash': self._total_cash,
            'backward_trade_set': list(self._backward_trade_set),
            'transaction_cost': self._transaction_cost,
        }

    def set_state(self, state):
        self._frozen_cash = state['frozen_cash']
        self._total_cash = state['total_cash']
        self._backward_trade_set = set(state['backward_trade_set'])
        self._transaction_cost = state['transaction_cost']
        self._positions.clear()
        for order_book_id, v in six.iteritems(state['positions']):
            position = self._positions.get_or_create(order_book_id)
            position.set_state(v)

    @property
    def type(self):
        return ACCOUNT_TYPE.FUTURE

    @staticmethod
    def _frozen_cash_of_order(order):
        if order.position_effect == POSITION_EFFECT.OPEN:
            return margin_of(order.order_book_id, order.unfilled_quantity, order.frozen_price)
        else:
            return 0

    @staticmethod
    def _frozen_cash_of_trade(trade):
        if trade.position_effect == POSITION_EFFECT.OPEN:
            return margin_of(trade.order_book_id, trade.last_quantity, trade.frozen_price)
        else:
            return 0

    @property
    def total_value(self):
        return self._total_cash + self.margin + self.holding_pnl

    # -- Margin 相关
    @property
    def margin(self):
        """
        [float] 总保证金
        """
        return sum(position.margin for position in six.itervalues(self._positions))

    @property
    def buy_margin(self):
        """
        [float] 买方向保证金
        """
        return sum(position.buy_margin for position in six.itervalues(self._positions))

    @property
    def sell_margin(self):
        """
        [float] 卖方向保证金
        """
        return sum(position.sell_margin for position in six.itervalues(self._positions))

    # -- PNL 相关
    @property
    def daily_pnl(self):
        """
        [float] 当日盈亏
        """
        return self.realized_pnl + self.holding_pnl - self.transaction_cost

    @property
    def holding_pnl(self):
        """
        [float] 浮动盈亏
        """
        return sum(position.holding_pnl for position in six.itervalues(self._positions))

    @property
    def realized_pnl(self):
        """
        [float] 平仓盈亏
        """
        return sum(position.realized_pnl for position in six.itervalues(self._positions))

    def _settlement(self, event):
        old_margin = self.margin
        old_holding_pnl = self.holding_pnl
        for position in list(self._positions.values()):
            order_book_id = position.order_book_id
            if position.is_de_listed() and position.buy_quantity + position.sell_quantity != 0:
                self._total_cash += position.market_value * position.margin_rate
                user_system_log.warn(
                    _(u"{order_book_id} is expired, close all positions by system").format(order_book_id=order_book_id))
                del self._positions[order_book_id]
            elif position.buy_quantity == 0 and position.sell_quantity == 0:
                del self._positions[order_book_id]
            else:
                position.apply_settlement()
        self._total_cash = self._total_cash + (old_margin - self.margin) + old_holding_pnl
        self._transaction_cost = 0

        # 如果 total_value <= 0 则认为已爆仓，清空仓位，资金归0
        if self.total_value <= 0:
            self._positions.clear()
            self._total_cash = 0

        self._backward_trade_set.clear()

    def _on_order_pending_new(self, event):
        if self != event.account:
            return
        self._frozen_cash += self._frozen_cash_of_order(event.order)

    def _on_order_creation_reject(self, event):
        if self != event.account:
            return
        self._frozen_cash -= self._frozen_cash_of_order(event.order)

    def _on_order_unsolicited_update(self, event):
        if self != event.account:
            return
        self._frozen_cash -= self._frozen_cash_of_order(event.order)

    def _on_trade(self, event):
        if self != event.account:
            return
        self._apply_trade(event.trade)

    def _apply_trade(self, trade):
        if trade.exec_id in self._backward_trade_set:
            return
        order_book_id = trade.order_book_id
        position = self._positions.get_or_create(order_book_id)
        delta_cash = position.apply_trade(trade)

        self._transaction_cost += trade.transaction_cost
        self._total_cash -= trade.transaction_cost
        self._total_cash += delta_cash
        self._frozen_cash -= self._frozen_cash_of_trade(trade)
        self._backward_trade_set.add(trade.exec_id)

    # ------------------------------------ Abandon Property ------------------------------------

    @property
    def daily_holding_pnl(self):
        """
        [已弃用] 请使用 holding_pnl
        """
        user_system_log.warn(_(u"[abandon] {} is no longer used.").format('future_account.daily_holding_pnl'))
        return self.holding_pnl

    @property
    def daily_realized_pnl(self):
        """
        [已弃用] 请使用 realized_pnl
        """
        user_system_log.warn(_(u"[abandon] {} is no longer used.").format('future_account.daily_realized_pnl'))
        return self.realized_pnl
<commit_msg>解决期货结算时总资金重复计算的问题<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2017 Ricequant, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import six

from .base_account import BaseAccount
from ...environment import Environment
from ...events import EVENT
from ...const import ACCOUNT_TYPE, POSITION_EFFECT
from ...utils.i18n import gettext as _
from ...utils.logger import user_system_log

def margin_of(order_book_id, quantity, price):
    env = Environment.get_instance()
    margin_info = env.data_proxy.get_margin_info(order_book_id)
    margin_multiplier = env.config.base.margin_multiplier
    margin_rate = margin_info['long_margin_ratio'] * margin_multiplier
    contract_multiplier = env.get_instrument(order_book_id).contract_multiplier
    return quantity * contract_multiplier * price * margin_rate

class FutureAccount(BaseAccount):

    __abandon_properties__ = [
        "daily_holding_pnl",
        "daily_realized_pnl"
    ]

    def register_event(self):
        event_bus = Environment.get_instance().event_bus
        event_bus.prepend_listener(EVENT.SETTLEMENT, self._settlement)
        event_bus.prepend_listener(EVENT.ORDER_PENDING_NEW, self._on_order_pending_new)
        event_bus.prepend_listener(EVENT.ORDER_CREATION_REJECT, self._on_order_creation_reject)
        event_bus.prepend_listener(EVENT.ORDER_CANCELLATION_PASS, self._on_order_unsolicited_update)
        event_bus.prepend_listener(EVENT.ORDER_UNSOLICITED_UPDATE, self._on_order_unsolicited_update)
        event_bus.prepend_listener(EVENT.TRADE, self._on_trade)

    def fast_forward(self, orders, trades=list()):
        # 计算 Positions
        for trade in trades:
            if trade.exec_id in self._backward_trade_set:
                continue
            self._apply_trade(trade)
        # 计算 Frozen Cash
        self._frozen_cash = sum(self._frozen_cash_of_order(order) for order in orders if order.is_active())

    def get_state(self):
        return {
            'positions': {
                order_book_id: position.get_state()
                for order_book_id, position in six.iteritems(self._positions)
                },
            'frozen_cash': self._frozen_cash,
            'total_cash': self._total_cash,
            'backward_trade_set': list(self._backward_trade_set),
            'transaction_cost': self._transaction_cost,
        }

    def set_state(self, state):
        self._frozen_cash = state['frozen_cash']
        self._total_cash = state['total_cash']
        self._backward_trade_set = set(state['backward_trade_set'])
        self._transaction_cost = state['transaction_cost']
        self._positions.clear()
        for order_book_id, v in six.iteritems(state['positions']):
            position = self._positions.get_or_create(order_book_id)
            position.set_state(v)

    @property
    def type(self):
        return ACCOUNT_TYPE.FUTURE

    @staticmethod
    def _frozen_cash_of_order(order):
        if order.position_effect == POSITION_EFFECT.OPEN:
            return margin_of(order.order_book_id, order.unfilled_quantity, order.frozen_price)
        else:
            return 0

    @staticmethod
    def _frozen_cash_of_trade(trade):
        if trade.position_effect == POSITION_EFFECT.OPEN:
            return margin_of(trade.order_book_id, trade.last_quantity, trade.frozen_price)
        else:
            return 0

    @property
    def total_value(self):
        return self._total_cash + self.margin + self.holding_pnl

    # -- Margin 相关
    @property
    def margin(self):
        """
        [float] 总保证金
        """
        return sum(position.margin for position in six.itervalues(self._positions))

    @property
    def buy_margin(self):
        """
        [float] 买方向保证金
        """
        return sum(position.buy_margin for position in six.itervalues(self._positions))

    @property
    def sell_margin(self):
        """
        [float] 卖方向保证金
        """
        return sum(position.sell_margin for position in six.itervalues(self._positions))

    # -- PNL 相关
    @property
    def daily_pnl(self):
        """
        [float] 当日盈亏
        """
        return self.realized_pnl + self.holding_pnl - self.transaction_cost

    @property
    def holding_pnl(self):
        """
        [float] 浮动盈亏
        """
        return sum(position.holding_pnl for position in six.itervalues(self._positions))

    @property
    def realized_pnl(self):
        """
        [float] 平仓盈亏
        """
        return sum(position.realized_pnl for position in six.itervalues(self._positions))

    def _settlement(self, event):
        old_margin = self.margin
        old_holding_pnl = self.holding_pnl
        for position in list(self._positions.values()):
            order_book_id = position.order_book_id
            if position.is_de_listed() and position.buy_quantity + position.sell_quantity != 0:
                user_system_log.warn(
                    _(u"{order_book_id} is expired, close all positions by system").format(order_book_id=order_book_id))
                del self._positions[order_book_id]
            elif position.buy_quantity == 0 and position.sell_quantity == 0:
                del self._positions[order_book_id]
            else:
                position.apply_settlement()
        self._total_cash = self._total_cash + (old_margin - self.margin) + old_holding_pnl
        self._transaction_cost = 0

        # 如果 total_value <= 0 则认为已爆仓，清空仓位，资金归0
        if self.total_value <= 0:
            self._positions.clear()
            self._total_cash = 0

        self._backward_trade_set.clear()

    def _on_order_pending_new(self, event):
        if self != event.account:
            return
        self._frozen_cash += self._frozen_cash_of_order(event.order)

    def _on_order_creation_reject(self, event):
        if self != event.account:
            return
        self._frozen_cash -= self._frozen_cash_of_order(event.order)

    def _on_order_unsolicited_update(self, event):
        if self != event.account:
            return
        self._frozen_cash -= self._frozen_cash_of_order(event.order)

    def _on_trade(self, event):
        if self != event.account:
            return
        self._apply_trade(event.trade)

    def _apply_trade(self, trade):
        if trade.exec_id in self._backward_trade_set:
            return
        order_book_id = trade.order_book_id
        position = self._positions.get_or_create(order_book_id)
        delta_cash = position.apply_trade(trade)

        self._transaction_cost += trade.transaction_cost
        self._total_cash -= trade.transaction_cost
        self._total_cash += delta_cash
        self._frozen_cash -= self._frozen_cash_of_trade(trade)
        self._backward_trade_set.add(trade.exec_id)

    # ------------------------------------ Abandon Property ------------------------------------

    @property
    def daily_holding_pnl(self):
        """
        [已弃用] 请使用 holding_pnl
        """
        user_system_log.warn(_(u"[abandon] {} is no longer used.").format('future_account.daily_holding_pnl'))
        return self.holding_pnl

    @property
    def daily_realized_pnl(self):
        """
        [已弃用] 请使用 realized_pnl
        """
        user_system_log.warn(_(u"[abandon] {} is no longer used.").format('future_account.daily_realized_pnl'))
        return self.realized_pnl
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright 2019 Language Technology, Universitaet Hamburg (author: Benjamin Milde)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import common_utils
import normalize_sentences
import spacy
import shutil

def process(text_kaldi_file):
    nlp = spacy.load('de')
    texts = []
    normalize_cache = {}
    i=0

    print('Making a backup of the original file:', text_kaldi_file, '=>', text_kaldi_file + '.orig')
    shutil.copyfile(text_kaldi_file, text_kaldi_file + '.orig')

    print('Opening and processing', text_kaldi_file)
    with open(text_kaldi_file) as infile:
        for line in infile:
            i +=1

            if i%10000 == 0:
                print('At line:', i)

            if line[-1] == '\n':
                line = line[:-1]
            split = line.split()
            if len(split) > 1:
                myid = split[0]
                text = ' '.join(split[1:])

                if text not in normalize_cache:
                    try:
                        normalized_text = normalize_sentences.normalize(nlp, text)
                        normalize_cache[text] = normalized_text
                    except:
                        print('Warning, error normalizing:', text)
                        continue
                else:
                    normalized_text = normalize_cache[text]

                texts.append(myid + ' ' + normalized_text)
            else:
                print('Warning,', myid, 'has no text!')

    print('Rewrite', text_kaldi_file)
    with open(text_kaldi_file, 'w') as outfile:
        outfile.write('\n'.join(texts))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Renormalize text file in Kaldi data dir format.')
    parser.add_argument('-t', '--text-kaldi-file', dest='text_kaldi_file', help='path to the Kaldi text file',  type=str)

    args = parser.parse_args()

    process(args.text_kaldi_file)
<commit_msg>dont overwrite old .orig<commit_after># -*- coding: utf-8 -*-

# Copyright 2019 Language Technology, Universitaet Hamburg (author: Benjamin Milde)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import common_utils
import normalize_sentences
import spacy
import shutil
import time

def process(text_kaldi_file):
    nlp = spacy.load('de')
    texts = []
    normalize_cache = {}
    i=0

    nonce = str(int(time.time()))
    print('Making a backup of the original file:', text_kaldi_file, '=>', text_kaldi_file + '.orig' + nonce)
    shutil.copyfile(text_kaldi_file, text_kaldi_file + '.orig' + nonce)

    print('Opening and processing', text_kaldi_file)
    with open(text_kaldi_file) as infile:
        for line in infile:
            i +=1

            if i%10000 == 0:
                print('At line:', i)

            if line[-1] == '\n':
                line = line[:-1]
            split = line.split()
            if len(split) > 1:
                myid = split[0]
                text = ' '.join(split[1:])

                if text not in normalize_cache:
                    try:
                        normalized_text = normalize_sentences.normalize(nlp, text)
                        normalize_cache[text] = normalized_text
                    except:
                        print('Warning, error normalizing:', text)
                        continue
                else:
                    normalized_text = normalize_cache[text]

                texts.append(myid + ' ' + normalized_text)
            else:
                print('Warning,', myid, 'has no text!')

    print('Rewrite', text_kaldi_file)
    with open(text_kaldi_file, 'w') as outfile:
        outfile.write('\n'.join(texts))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Renormalize text file in Kaldi data dir format.')
    parser.add_argument('-t', '--text-kaldi-file', dest='text_kaldi_file', help='path to the Kaldi text file',  type=str)

    args = parser.parse_args()

    process(args.text_kaldi_file)
<|endoftext|>


<commit_before># vim:ts=4:sts=4:sw=4:expandtab
"""
Security and authorization procedures.
"""

from datetime import datetime, timedelta
from types import NoneType
import string
import random
import urlparse
import urllib

from openid.consumer import consumer
from openid.extensions import sreg as oidsreg
from openid.extensions import pape as oidpape
from openid.extensions import ax as oidax

from satori.core.sec.tools import RightCheck, RoleSet, Token
from satori.core.sec.store import Store

from satori.objects import DispatchOn, Argument, ReturnValue
from satori.core.models import Object, User, Login, OpenIdentity, Global, Role
from satori.ars.wrapper import Struct, StaticWrapper, TypedMap

def openid_salt():
    chars = string.letters + string.digits
    salt = ''
    for i in range(16):
        salt += random.choice(chars)
    return salt

OpenIdRedirect = Struct('OpenIdRedirect', (
    ('token', Token, False),
    ('redirect', str, True),
    ('html', str, True)
))

security = StaticWrapper('Security')

@security.method
@ReturnValue(type=Role)
def anonymous():
    globe = Global.get_instance()
    return globe.anonymous

@security.method
@ReturnValue(type=Role)
def authenticated():
    globe = Global.get_instance()
    return globe.authenticated

@security.method
@Argument('token', type=Token)
@ReturnValue(type=User)
def whoami(token):
    if not token.valid:
        raise Exception("Provided token is invalid.")
    return token.user

@security.method
@Argument('token', type=Token)
@Argument('object', type=Object)
@Argument('right', type=str)
@ReturnValue(type=bool)
def right_have(token, object, right):
    checker = RightCheck()
    roleset = RoleSet(token=token)
    return checker(roleset, object, right)

@security.method
@Argument('token', type=Token)
@Argument('right', type=str)
@ReturnValue(type=bool)
def global_right_have(token, right):
    checker = RightCheck()
    roleset = RoleSet(token=token)
    object = Global.get_instance()
    return checker(roleset, object, right)

@security.method
@Argument('login', type=str)
@Argument('namespace', type=str, default='')
@ReturnValue(type=bool)
def login_free(login, namespace):
    return len(Login.objects.filter(namespace=namespace, login=login)) == 0

@security.method
@Argument('login', type=str)
@Argument('password', type=str)
@Argument('fullname', type=str)
@ReturnValue(type=User)
def register(login, password, fullname):
    user = User(login=login, fullname=fullname)
    user.save()
    auth = Login(login=login, user=user)
    auth.set_password(password)
    auth.save()
    return user

@security.method
@Argument('login', type=str)
@Argument('password', type=str)
@Argument('namespace', type=str, default='')
@ReturnValue(type=Token)
def login(login, password, namespace):
    login = Login.objects.get(namespace=namespace, login=login)
    if login.check_password(password):
        auth = 'login'
        if namespace != '':
            auth = auth + '.' + namespace
    	return Token(user=login.user, auth=auth, validity=timedelta(hours=6))

@security.method
@Argument('login', type=str)
@Argument('old', type=str)
@Argument('new', type=str)
@Argument('namespace', type=str, default='')
@ReturnValue(type=Token)
def passwd(login, old, new, namespace):
    login = Login.objects.get(namespace=namespace, login=login)
    if login.change_password(old, new):
        auth = 'login'
        if namespace != '':
            auth = auth + '.' + namespace
    	return Token(user=login.user, auth=auth, validity=timedelta(hours=6))

def openid_realm(url):
    callback = urlparse.urlparse(url)
    return urlparse.urlunparse((callback.scheme, callback.netloc, '', '', '', ''))

def openid_modify_callback(url, salt):
    callback = urlparse.urlparse(url)
    qs = urlparse.parse_qs(callback.query)
    qs['satori.openid.salt'] = (salt,)
    query = []
    for key, vlist in qs.items():
        for value in vlist:
        	query.append((key,value))
    query = urllib.urlencode(query)
    return urlparse.urlunparse((callback.scheme, callback.netloc, callback.path, callback.params, query, callback.fragment))

def openid_check_callback(url, salt):
    callback = urlparse.urlparse(url)
    qs = urlparse.parse_qs(callback.query)
    if 'satori.openid.salt' not in qs:
        return False
    if len(qs['satori.openid.salt']) != 1:
        return False
    return qs['satori.openid.salt'][0] == salt

def openid_add_ax(request):
    axr = oidax.FetchRequest()
    axr.add(oidax.AttrInfo('http://axschema.org/contact/country/home', 1, True, 'country'))
    axr.add(oidax.AttrInfo('http://axschema.org/contact/email', 1, True, 'email'))
    axr.add(oidax.AttrInfo('http://axschema.org/namePerson/first', 1, True, 'firstname'))
    axr.add(oidax.AttrInfo('http://axschema.org/namePerson/last', 1, True, 'lastname'))
    axr.add(oidax.AttrInfo('http://axschema.org/pref/language', 1, True, 'language'))
    request.addExtension(axr)

def openid_get_ax(response, identity, update =False):
    try:
        axr = oidax.FetchResponse.fromSuccessResponse(response)
        identity.country = axr.getSingle('http://axschema.org/contact/country/home', identity.country)
        identity.email = axr.getSingle('http://axschema.org/contact/email', identity.email)
        identity.language = axr.getSingle('http://axschema.org/pref/language', identity.language)
        firstname = axr.getSingle('http://axschema.org/namePerson/first', None)
        lastname = axr.getSingle('http://axschema.org/namePerson/last', None)
        if firstname != None and lastname != None:
            identity.name = firstname + ' ' + lastname
        identity.save()
        if update:
            user = identity.user
            user.fullname = identity.name
            user.save()
    except:
        pass

def openid_generic_start(openid, return_to, user_id ='', valid =1, ax =False):
    salt = openid_salt()
    session = { 'satori.openid.salt' : salt }
    store = Store()
    request = consumer.Consumer(session, store).begin(openid)
    if ax:
        openid_add_ax(request)
    redirect = ''
    html = ''
    realm = openid_realm(return_to)
    callback = openid_modify_callback(return_to, salt)
    if request.shouldSendRedirect():
        redirect = request.redirectURL(realm, callback)
    else:
        form = request.formMarkup(realm, callback, False, {'id': 'satori_openid_form'})
        html = '' + form + ''
    token = Token(user_id=user_id, auth='openid', data=session, validity=timedelta(hours=valid))
    return {
        'token' : token,
        'redirect' : redirect,
        'html' : html
    }

def openid_generic_finish(token, args, return_to, user =None):
    if not token.valid:
        raise Exception("Provided token is invalid.")
    if token.auth != 'openid':
        return token
    session = token.data
    store = Store()
    response = consumer.Consumer(session, store).complete(args, return_to)
    if response.status != consumer.SUCCESS:
        raise Exception("OpenID failed.")
    callback = response.getReturnTo()
    if not openid_check_callback(callback, session['satori.openid.salt']):
        raise Exception("OpenID failed.")
    if user:
        identity = OpenIdentity(identity=response.identity_url, user=user)
        identity.save()
    else:
        identity = OpenIdentity.objects.get(identity=response.identity_url)
    openid_get_ax(response, identity, update=True)
    token = Token(user=identity.user, auth='openid', validity=timedelta(hours=6))
    return token

@security.method
@Argument('openid', type=str)
@Argument('return_to', type=str)
@ReturnValue(type=OpenIdRedirect)
def openid_login_start(openid, return_to):
    return openid_generic_start(openid=openid, return_to=return_to, user_id='', ax=True)

@security.method
@Argument('token', type=Token)
@Argument('args', type=TypedMap(str, str))
@Argument('return_to', type=str)
@ReturnValue(type=Token)
def openid_login_finish(token, args, return_to):
    return openid_generic_finish(token, args, return_to)

@security.method
@Argument('login', type=str)
@Argument('openid', type=str)
@Argument('return_to', type=str)
@ReturnValue(type=OpenIdRedirect)
def openid_register_start(login, openid, return_to):
    return openid_generic_start(openid=openid, return_to=return_to, user_id='', ax=True)
    user = User(login=login, fullname='')
    user.save()
    session = res['token'].data
    session['satori.openid.user'] = user.id
    res['token'].data = session
    return res

@security.method
@Argument('token', type=Token)
@Argument('args', type=TypedMap(str, str))
@Argument('return_to', type=str)
@ReturnValue(type=Token)
def openid_register_finish(token, args, return_to):
    session = token.data
    user = User.objects.get(id=session['satori.openid.user'])
    res = openid_generic_finish(token, args, return_to, user)
    return res

@security.method
@Argument('login', type=str)
@Argument('password', type=str)
@Argument('namespace', type=str, default='')
@Argument('openid', type=str)
@Argument('return_to', type=str)
@ReturnValue(type=OpenIdRedirect)
def openid_add_start(login, openid, return_to):
    login = Login.objects.get(namespace=namespace, login=login)
    if login.check_password(password):
        res = openid_generic_start(openid=openid, return_to=return_to, user_id=str(token.user.id), ax=True)
        session = res['token'].data
        session['satori.openid.user'] = token.user.id
        res['token'].data = session
        return res
    raise Exception("Authorization failed.")

@security.method
@Argument('token', type=Token)
@Argument('args', type=TypedMap(str, str))
@Argument('return_to', type=str)
@ReturnValue(type=Token)
def openid_add_finish(token, args, return_to):
    session = token.data
    user = User.objects.get(id=session['satori.openid.user'])
    res = openid_generic_finish(token, args, return_to, user)
    return res

security._fill_module(__name__)

<commit_msg>Simplified cani.<commit_after># vim:ts=4:sts=4:sw=4:expandtab
"""
Security and authorization procedures.
"""

from datetime import datetime, timedelta
from types import NoneType
import string
import random
import urlparse
import urllib

from openid.consumer import consumer
from openid.extensions import sreg as oidsreg
from openid.extensions import pape as oidpape
from openid.extensions import ax as oidax

from satori.core.sec.tools import RightCheck, RoleSet, Token
from satori.core.sec.store import Store

from satori.objects import DispatchOn, Argument, ReturnValue
from satori.core.models import Object, User, Login, OpenIdentity, Global, Role
from satori.ars.wrapper import Struct, StaticWrapper, TypedMap

def openid_salt():
    chars = string.letters + string.digits
    salt = ''
    for i in range(16):
        salt += random.choice(chars)
    return salt

OpenIdRedirect = Struct('OpenIdRedirect', (
    ('token', Token, False),
    ('redirect', str, True),
    ('html', str, True)
))

security = StaticWrapper('Security')

@security.method
@ReturnValue(type=Role)
def anonymous():
    globe = Global.get_instance()
    return globe.anonymous

@security.method
@ReturnValue(type=Role)
def authenticated():
    globe = Global.get_instance()
    return globe.authenticated

@security.method
@Argument('token', type=Token)
@ReturnValue(type=User)
def whoami(token):
    if not token.valid:
        raise Exception("Provided token is invalid.")
    return token.user

@security.method
@Argument('token', type=Token)
@Argument('object', type=Object)
@Argument('right', type=str)
@ReturnValue(type=bool)
def right_have(token, object, right):
    return object.demand_right(token, right)

@security.method
@Argument('token', type=Token)
@Argument('right', type=str)
@ReturnValue(type=bool)
def global_right_have(token, right):
    return Global.get_instance().demand_right(token, right)

@security.method
@Argument('login', type=str)
@Argument('namespace', type=str, default='')
@ReturnValue(type=bool)
def login_free(login, namespace):
    return len(Login.objects.filter(namespace=namespace, login=login)) == 0

@security.method
@Argument('login', type=str)
@Argument('password', type=str)
@Argument('fullname', type=str)
@ReturnValue(type=User)
def register(login, password, fullname):
    user = User(login=login, fullname=fullname)
    user.save()
    auth = Login(login=login, user=user)
    auth.set_password(password)
    auth.save()
    return user

@security.method
@Argument('login', type=str)
@Argument('password', type=str)
@Argument('namespace', type=str, default='')
@ReturnValue(type=Token)
def login(login, password, namespace):
    login = Login.objects.get(namespace=namespace, login=login)
    if login.check_password(password):
        auth = 'login'
        if namespace != '':
            auth = auth + '.' + namespace
    	return Token(user=login.user, auth=auth, validity=timedelta(hours=6))

@security.method
@Argument('login', type=str)
@Argument('old', type=str)
@Argument('new', type=str)
@Argument('namespace', type=str, default='')
@ReturnValue(type=Token)
def passwd(login, old, new, namespace):
    login = Login.objects.get(namespace=namespace, login=login)
    if login.change_password(old, new):
        auth = 'login'
        if namespace != '':
            auth = auth + '.' + namespace
    	return Token(user=login.user, auth=auth, validity=timedelta(hours=6))

def openid_realm(url):
    callback = urlparse.urlparse(url)
    return urlparse.urlunparse((callback.scheme, callback.netloc, '', '', '', ''))

def openid_modify_callback(url, salt):
    callback = urlparse.urlparse(url)
    qs = urlparse.parse_qs(callback.query)
    qs['satori.openid.salt'] = (salt,)
    query = []
    for key, vlist in qs.items():
        for value in vlist:
        	query.append((key,value))
    query = urllib.urlencode(query)
    return urlparse.urlunparse((callback.scheme, callback.netloc, callback.path, callback.params, query, callback.fragment))

def openid_check_callback(url, salt):
    callback = urlparse.urlparse(url)
    qs = urlparse.parse_qs(callback.query)
    if 'satori.openid.salt' not in qs:
        return False
    if len(qs['satori.openid.salt']) != 1:
        return False
    return qs['satori.openid.salt'][0] == salt

def openid_add_ax(request):
    axr = oidax.FetchRequest()
    axr.add(oidax.AttrInfo('http://axschema.org/contact/country/home', 1, True, 'country'))
    axr.add(oidax.AttrInfo('http://axschema.org/contact/email', 1, True, 'email'))
    axr.add(oidax.AttrInfo('http://axschema.org/namePerson/first', 1, True, 'firstname'))
    axr.add(oidax.AttrInfo('http://axschema.org/namePerson/last', 1, True, 'lastname'))
    axr.add(oidax.AttrInfo('http://axschema.org/pref/language', 1, True, 'language'))
    request.addExtension(axr)

def openid_get_ax(response, identity, update =False):
    try:
        axr = oidax.FetchResponse.fromSuccessResponse(response)
        identity.country = axr.getSingle('http://axschema.org/contact/country/home', identity.country)
        identity.email = axr.getSingle('http://axschema.org/contact/email', identity.email)
        identity.language = axr.getSingle('http://axschema.org/pref/language', identity.language)
        firstname = axr.getSingle('http://axschema.org/namePerson/first', None)
        lastname = axr.getSingle('http://axschema.org/namePerson/last', None)
        if firstname != None and lastname != None:
            identity.name = firstname + ' ' + lastname
        identity.save()
        if update:
            user = identity.user
            user.fullname = identity.name
            user.save()
    except:
        pass

def openid_generic_start(openid, return_to, user_id ='', valid =1, ax =False):
    salt = openid_salt()
    session = { 'satori.openid.salt' : salt }
    store = Store()
    request = consumer.Consumer(session, store).begin(openid)
    if ax:
        openid_add_ax(request)
    redirect = ''
    html = ''
    realm = openid_realm(return_to)
    callback = openid_modify_callback(return_to, salt)
    if request.shouldSendRedirect():
        redirect = request.redirectURL(realm, callback)
    else:
        form = request.formMarkup(realm, callback, False, {'id': 'satori_openid_form'})
        html = '' + form + ''
    token = Token(user_id=user_id, auth='openid', data=session, validity=timedelta(hours=valid))
    return {
        'token' : token,
        'redirect' : redirect,
        'html' : html
    }

def openid_generic_finish(token, args, return_to, user =None):
    if not token.valid:
        raise Exception("Provided token is invalid.")
    if token.auth != 'openid':
        return token
    session = token.data
    store = Store()
    response = consumer.Consumer(session, store).complete(args, return_to)
    if response.status != consumer.SUCCESS:
        raise Exception("OpenID failed.")
    callback = response.getReturnTo()
    if not openid_check_callback(callback, session['satori.openid.salt']):
        raise Exception("OpenID failed.")
    if user:
        identity = OpenIdentity(identity=response.identity_url, user=user)
        identity.save()
    else:
        identity = OpenIdentity.objects.get(identity=response.identity_url)
    openid_get_ax(response, identity, update=True)
    token = Token(user=identity.user, auth='openid', validity=timedelta(hours=6))
    return token

@security.method
@Argument('openid', type=str)
@Argument('return_to', type=str)
@ReturnValue(type=OpenIdRedirect)
def openid_login_start(openid, return_to):
    return openid_generic_start(openid=openid, return_to=return_to, user_id='', ax=True)

@security.method
@Argument('token', type=Token)
@Argument('args', type=TypedMap(str, str))
@Argument('return_to', type=str)
@ReturnValue(type=Token)
def openid_login_finish(token, args, return_to):
    return openid_generic_finish(token, args, return_to)

@security.method
@Argument('login', type=str)
@Argument('openid', type=str)
@Argument('return_to', type=str)
@ReturnValue(type=OpenIdRedirect)
def openid_register_start(login, openid, return_to):
    return openid_generic_start(openid=openid, return_to=return_to, user_id='', ax=True)
    user = User(login=login, fullname='')
    user.save()
    session = res['token'].data
    session['satori.openid.user'] = user.id
    res['token'].data = session
    return res

@security.method
@Argument('token', type=Token)
@Argument('args', type=TypedMap(str, str))
@Argument('return_to', type=str)
@ReturnValue(type=Token)
def openid_register_finish(token, args, return_to):
    session = token.data
    user = User.objects.get(id=session['satori.openid.user'])
    res = openid_generic_finish(token, args, return_to, user)
    return res

@security.method
@Argument('login', type=str)
@Argument('password', type=str)
@Argument('namespace', type=str, default='')
@Argument('openid', type=str)
@Argument('return_to', type=str)
@ReturnValue(type=OpenIdRedirect)
def openid_add_start(login, openid, return_to):
    login = Login.objects.get(namespace=namespace, login=login)
    if login.check_password(password):
        res = openid_generic_start(openid=openid, return_to=return_to, user_id=str(token.user.id), ax=True)
        session = res['token'].data
        session['satori.openid.user'] = token.user.id
        res['token'].data = session
        return res
    raise Exception("Authorization failed.")

@security.method
@Argument('token', type=Token)
@Argument('args', type=TypedMap(str, str))
@Argument('return_to', type=str)
@ReturnValue(type=Token)
def openid_add_finish(token, args, return_to):
    session = token.data
    user = User.objects.get(id=session['satori.openid.user'])
    res = openid_generic_finish(token, args, return_to, user)
    return res

security._fill_module(__name__)

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import logging
from django.db import connections
from django.db.utils import IntegrityError, DataError, ProgrammingError

logger = logging.getLogger(__name__)

class ParserDataError(Exception):
    pass

class ParserRollbackError(Exception):
    pass

class ParserConnectionError(Exception):
    pass

class Parser:

    PUBLICATION_ATTRIBUTES = [
        'type',
        'title',
        'date',
        'booktitle'
        #'journal',
        'volume',
        'number',
        'pages_from',
        'pages_to',
        'series',
        'publisher',
        'isbn',
        'doi',
        'abstract',
        'copyright'
    ]

    def __init__(self, database):
        self.conn = connections[database]
        self.cursor = self.conn.cursor()
        self.count_publications = 0
        #self.count_citations = 0

    def commit(self):
        '''
        :raise ParserRollbackError When a problems occurred, that required to do a rollback
        '''
        try:
            logger.info('commit')
            self.conn.commit()
        except(IntegrityError) as e:
            logger.error(e, exc_info=True)
            self.conn.rollback()
            raise ParserRollbackError

    def __normalize_string(self, value):
        return value.strip().lower()

    def parse_author(self, name):
        '''
        If the author with the given name already exists, the ID of that author is returned.
        Otherwise a new author gets created.

        :param name: Name of the author
        :return: ID of the author
        :raise ParserDataError: When the name is too long
        '''
        name = self.__normalize_string(name)
        self.cursor.execute("SELECT id FROM core_author WHERE name LIKE %s LIMIT 1", [name])
        result = self.cursor.fetchone()

        if result:
            return result[0]
        else:
            if name and len(name) <= 100:
                self.cursor.execute("INSERT INTO core_author (name) VALUES (%s) RETURNING id", [name])
                return self.cursor.fetchone()[0]
            else:
                raise ParserDataError('Author name is too long')

    def parse_journal(self, name):
        '''
        If the journal with the given name already exists, the ID of that journal is returned.
        Otherwise a new journel gets created.

        :param name: Name of the journal
        :return: ID of the journal
        :raise ParserDataError: When the name is too long
        '''
        name = self.__normalize_string(name)
        self.cursor.execute("SELECT id FROM core_journal WHERE name LIKE %s LIMIT 1", [name])
        result = self.cursor.fetchone()

        if result:
            return result[0]
        else:
            if name and len(name) <= 250:
                self.cursor.execute("INSERT INTO core_journal (name) VALUES (%s) RETURNING id", [name])
                return self.cursor.fetchone()[0]
            else:
                raise ParserDataError('Journal name is too long')

    def parse_conference(self, short_name):
        short_name = self.__normalize_string(short_name)
        self.cursor.execute("SELECT id FROM core_conference WHERE short_name LIKE %s LIMIT 1", [short_name])
        result = self.cursor.fetchone()

        if result:
            return result[0]
        else:
            if short_name and len(short_name) <= 20:
                self.cursor.execute("INSERT INTO core_conference (short_name) VALUES (%s) RETURNING id", [short_name])
                return self.cursor.fetchone()[0]
            else:
                raise ParserDataError('Conference short name is too long: %s' % short_name)

    def parse_publication(self, type=None, title=None, year=None, date=None, booktitle=None, journal_id=None, volume=None, number=None, pages_from=None, pages_to=None, series=None, publisher=None, isbn=None, doi=None, abstract=None, copyright=None, conference_id=None, source=None):
        if title and len(title) <= 250:
            title = self.__normalize_string(title)
            if date and len(date) > 50:
                date = None
            if booktitle and len(booktitle) > 200:
                booktitle = None
            if volume and len(volume) > 20:
                volume = None
            if number and len(number) > 20:
                number = None
            if pages_from and len(pages_from) > 5:
                pages_from = None
            if pages_to and len(pages_to) > 5:
                pages_to = None
            if series and len(series) > 200:
                series = None
            if publisher and len(publisher) > 150:
                publisher = None
            if isbn and len(isbn) > 50:
                isbn = None
            if doi and len(doi) > 50:
                doi = None
            self.cursor.execute("INSERT INTO core_publication (type, title, year, date, booktitle, journal_id, volume, number, pages_from, pages_to, series, publisher, isbn, doi, abstract, copyright, conference_id, source) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id", [type, title, year, date, booktitle, journal_id, volume, number, pages_from, pages_to, series, publisher, isbn, doi, abstract, copyright, conference_id, source])
            return self.cursor.fetchone()[0]
        else:
            raise ParserDataError('Title does not exists or is too long')

    def parse_reference(self, publication_id, reference_id, context=None, source_id=None):
        if publication_id and reference_id:
            self.cursor.execute("INSERT INTO core_publicationreference (publication_id, reference_id, context, source_id) VALUES (%s, %s, %s, %s)", [publication_id, reference_id, context, source_id])
        else:
            raise ParserDataError('publication_id or reference_id does not exists')

    def parse(self, publication, conference_short_name=None, journal_name=None, authors=None, keywords=None, urls=None, reference=None):
        '''

        :param publication: Publication dictionary
        :param conference_short_name: Conference short name
        :param journal_name: Journal name
        :param authors: Authors name array
        :param keywords: Keywords array
        :param urls: Url array
        :param reference: Reference dictionary
        :return: Publication ID, if entry was successfully parsed; False otherwise
        :raise ParserRollbackError: When a problems occurred, that required to do a rollback
        '''

        self.count_publications += 1
        try:

            conference_id = None
            journal_id = None

            # conference
            if conference_short_name:
                try:
                    conference_id = self.parse_conference(conference_short_name)
                except(ParserDataError) as e:
                    logger.warn(str(e))
                del conference_short_name
            # journal
            if journal_name:
                try:
                    journal_id = self.parse_journal(journal_name)
                except(ParserDataError) as e:
                    logger.warn(str(e))
                del journal_name
            # publication
            publication_id = self.parse_publication(conference_id=conference_id,
                                                    journal_id=journal_id,
                                                    **publication)
            # authors
            if authors:
                for author in authors:
                    try:
                        self.cursor.execute("INSERT INTO core_publicationauthoraffilation (publication_id, author_id) VALUES (%s, %s)", [publication_id, self.parse_author(author)])
                    except(DataError) as e:
                        logger.warn(str(e))
                del authors
            # keywords
            if keywords:
                for keyword in keywords:
                    if keyword and len(keyword) <= 100:
                        self.cursor.execute("INSERT INTO core_publicationkeyword (publication_id, name) VALUES (%s, %s)", [publication_id, keyword])
                del keywords
            # urls
            if urls:
                for url in urls:
                    url_type = None
                    if isinstance(url, dict):
                        url_type = url['type']
                        url = url['value']

                    if url and len(url) <= 200:
                        if url_type and len(url_type) >= 30:
                            url_type = None
                        self.cursor.execute("INSERT INTO core_publicationurl (publication_id, url, type) VALUES (%s, %s, %s)", [publication_id, url, url_type])
                del urls
            # reference
            if reference:
                try:
                    self.parse_reference(reference_id=publication_id, **reference)
                except(ParserDataError) as e:
                    logger.warn(str(e))
                del reference

            return publication_id
        except(ParserDataError) as e:
            #logger.warn(str(e))
            return False
        except(ProgrammingError, DataError) as e:
            logger.error(e, exc_info=True)
            self.conn.rollback()
            raise ParserRollbackError(str(e))

    """
        def check_author_name(self, name):
        try:
            # Milena Mihail, et al.
            name = name.strip().replace(', et al.', '')
            # block: n.n., S., Jr., A.
            if ' ' in name:
                # block: University, Università, Universität, Université
                if not any(extension in name for extension in ('Universit', 'et al.')):
                    return name
        except(AttributeError) as e:
            logger.warn(str(e))

        return False

    def _check_publication_is_valid(self, entry):
        return 'title' in entry and 'authors' in entry
    """
<commit_msg>remove unused code<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import logging
from django.db import connections
from django.db.utils import IntegrityError, DataError, ProgrammingError

logger = logging.getLogger(__name__)

class ParserDataError(Exception):
    pass

class ParserRollbackError(Exception):
    pass

class ParserConnectionError(Exception):
    pass

class Parser:

    PUBLICATION_ATTRIBUTES = [
        'type',
        'title',
        'date',
        'booktitle'
        #'journal',
        'volume',
        'number',
        'pages_from',
        'pages_to',
        'series',
        'publisher',
        'isbn',
        'doi',
        'abstract',
        'copyright'
    ]

    def __init__(self, database):
        self.conn = connections[database]
        self.cursor = self.conn.cursor()
        self.count_publications = 0
        #self.count_citations = 0

    def commit(self):
        '''
        :raise ParserRollbackError When a problems occurred, that required to do a rollback
        '''
        try:
            logger.info('commit')
            self.conn.commit()
        except(IntegrityError) as e:
            logger.error(e, exc_info=True)
            self.conn.rollback()
            raise ParserRollbackError

    def __normalize_string(self, value):
        return value.strip().lower()

    def parse_author(self, name):
        '''
        If the author with the given name already exists, the ID of that author is returned.
        Otherwise a new author gets created.

        :param name: Name of the author
        :return: ID of the author
        :raise ParserDataError: When the name is too long
        '''
        name = self.__normalize_string(name)
        self.cursor.execute("SELECT id FROM core_author WHERE name LIKE %s LIMIT 1", [name])
        result = self.cursor.fetchone()

        if result:
            return result[0]
        else:
            if name and len(name) <= 100:
                self.cursor.execute("INSERT INTO core_author (name) VALUES (%s) RETURNING id", [name])
                return self.cursor.fetchone()[0]
            else:
                raise ParserDataError('Author name is too long')

    def parse_journal(self, name):
        '''
        If the journal with the given name already exists, the ID of that journal is returned.
        Otherwise a new journel gets created.

        :param name: Name of the journal
        :return: ID of the journal
        :raise ParserDataError: When the name is too long
        '''
        name = self.__normalize_string(name)
        self.cursor.execute("SELECT id FROM core_journal WHERE name LIKE %s LIMIT 1", [name])
        result = self.cursor.fetchone()

        if result:
            return result[0]
        else:
            if name and len(name) <= 250:
                self.cursor.execute("INSERT INTO core_journal (name) VALUES (%s) RETURNING id", [name])
                return self.cursor.fetchone()[0]
            else:
                raise ParserDataError('Journal name is too long')

    def parse_conference(self, short_name):
        short_name = self.__normalize_string(short_name)
        self.cursor.execute("SELECT id FROM core_conference WHERE short_name LIKE %s LIMIT 1", [short_name])
        result = self.cursor.fetchone()

        if result:
            return result[0]
        else:
            if short_name and len(short_name) <= 20:
                self.cursor.execute("INSERT INTO core_conference (short_name) VALUES (%s) RETURNING id", [short_name])
                return self.cursor.fetchone()[0]
            else:
                raise ParserDataError('Conference short name is too long: %s' % short_name)

    def parse_publication(self, type=None, title=None, year=None, date=None, booktitle=None, journal_id=None, volume=None, number=None, pages_from=None, pages_to=None, series=None, publisher=None, isbn=None, doi=None, abstract=None, copyright=None, conference_id=None, source=None):
        if title and len(title) <= 250:
            title = self.__normalize_string(title)
            if date and len(date) > 50:
                date = None
            if booktitle and len(booktitle) > 200:
                booktitle = None
            if volume and len(volume) > 20:
                volume = None
            if number and len(number) > 20:
                number = None
            if pages_from and len(pages_from) > 5:
                pages_from = None
            if pages_to and len(pages_to) > 5:
                pages_to = None
            if series and len(series) > 200:
                series = None
            if publisher and len(publisher) > 150:
                publisher = None
            if isbn and len(isbn) > 50:
                isbn = None
            if doi and len(doi) > 50:
                doi = None
            self.cursor.execute("INSERT INTO core_publication (type, title, year, date, booktitle, journal_id, volume, number, pages_from, pages_to, series, publisher, isbn, doi, abstract, copyright, conference_id, source) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id", [type, title, year, date, booktitle, journal_id, volume, number, pages_from, pages_to, series, publisher, isbn, doi, abstract, copyright, conference_id, source])
            return self.cursor.fetchone()[0]
        else:
            raise ParserDataError('Title does not exists or is too long')

    def parse_reference(self, publication_id, reference_id, context=None, source_id=None):
        if publication_id and reference_id:
            self.cursor.execute("INSERT INTO core_publicationreference (publication_id, reference_id, context, source_id) VALUES (%s, %s, %s, %s)", [publication_id, reference_id, context, source_id])
        else:
            raise ParserDataError('publication_id or reference_id does not exists')

    def parse(self, publication, conference_short_name=None, journal_name=None, authors=None, keywords=None, urls=None, reference=None):
        '''

        :param publication: Publication dictionary
        :param conference_short_name: Conference short name
        :param journal_name: Journal name
        :param authors: Authors name array
        :param keywords: Keywords array
        :param urls: Url array
        :param reference: Reference dictionary
        :return: Publication ID, if entry was successfully parsed; False otherwise
        :raise ParserRollbackError: When a problems occurred, that required to do a rollback
        '''

        self.count_publications += 1
        try:

            conference_id = None
            journal_id = None

            # conference
            if conference_short_name:
                try:
                    conference_id = self.parse_conference(conference_short_name)
                except(ParserDataError) as e:
                    logger.warn(str(e))
                del conference_short_name
            # journal
            if journal_name:
                try:
                    journal_id = self.parse_journal(journal_name)
                except(ParserDataError) as e:
                    logger.warn(str(e))
                del journal_name
            # publication
            publication_id = self.parse_publication(conference_id=conference_id,
                                                    journal_id=journal_id,
                                                    **publication)
            # authors
            if authors:
                for author in authors:
                    try:
                        self.cursor.execute("INSERT INTO core_publicationauthoraffilation (publication_id, author_id) VALUES (%s, %s)", [publication_id, self.parse_author(author)])
                    except(DataError) as e:
                        logger.warn(str(e))
                del authors
            # keywords
            if keywords:
                for keyword in keywords:
                    if keyword and len(keyword) <= 100:
                        self.cursor.execute("INSERT INTO core_publicationkeyword (publication_id, name) VALUES (%s, %s)", [publication_id, keyword])
                del keywords
            # urls
            if urls:
                for url in urls:
                    url_type = None
                    if isinstance(url, dict):
                        url_type = url['type']
                        url = url['value']

                    if url and len(url) <= 200:
                        if url_type and len(url_type) >= 30:
                            url_type = None
                        self.cursor.execute("INSERT INTO core_publicationurl (publication_id, url, type) VALUES (%s, %s, %s)", [publication_id, url, url_type])
                del urls
            # reference
            if reference:
                try:
                    self.parse_reference(reference_id=publication_id, **reference)
                except(ParserDataError) as e:
                    logger.warn(str(e))
                del reference

            return publication_id
        except(ParserDataError) as e:
            #logger.warn(str(e))
            return False
        except(ProgrammingError, DataError) as e:
            logger.error(e, exc_info=True)
            self.conn.rollback()
            raise ParserRollbackError(str(e))
<|endoftext|>


<commit_before>"""
Configurable Pipeline for the Analysis of Connectomes
=====================================================

CPAC is a configurable, open-source, Nipype-based, automated processing pipeline for resting state functional MRI (R-fMRI) data, for use by both novice and expert users.
"""

from numpy.testing import nosetester
test = nosetester.NoseTester.test

class _NoseTester(nosetester.NoseTester):
    def test(self, label='fast', verbose=1, extra_argv=['--exe'], doctests = False, coverage=False):
        return super(_NoseTester, self).test(label=label, verbose=verbose, extra_argv=extra_argv, doctests=doctests, coverage=coverage)

test = _NoseTester().test

import anat_preproc, \
       func_preproc, \
       reho, \
       seg_preproc, \
       registration, \
       sca, \
       basc, \
       nuisance, \
       generate_motion_statistics, \
       alff, \
       qc, \
       seg_preproc, \
       vmhc, \
       median_angle, \
       timeseries, \
       network_centrality, \
       scrubbing, \
       group_analysis, \
       easy_thresh,\
       utils, \
       pipeline, \
       cwas, \
       GUI

__all__ = ['GUI', 'pipeline', 'anat_preproc', 'func_preproc', 'registration', 'seg_preproc', 'reho', 'sca', 'basc', 'nuisance', 'alff', 'vmhc', 'median_angle', 'generate_motion_statistics', 'timeseries', 'network_centrality', 'scrubbing', 'utils', 'group_analysis', 'easy_thresh', 'qc', 'cwas']

#__version__ = '0.1-git'

from subprocess import Popen, PIPE
import re

try:
    version = '0.1.9'
#    gitproc = Popen(['git', 'log', '--oneline'], stdout = PIPE)
#    (stdout, stderr) = gitproc.communicate()
#    rows = stdout.split("\n")
#    v_num = re.search( r'(?<=(version_|release_))(.)*', rows[0])
#    if v_num:
#        version = v_num.group(0).strip("'")
#    else:
#        version = 'unknown_version'
except OSError:
    version = 'unknown_version'

__version__ =  str(version)
<commit_msg>Update __init__.py<commit_after>"""
Configurable Pipeline for the Analysis of Connectomes
=====================================================

CPAC is a configurable, open-source, Nipype-based, automated processing pipeline for resting state functional MRI (R-fMRI) data, for use by both novice and expert users.
"""

from numpy.testing import nosetester
test = nosetester.NoseTester.test

class _NoseTester(nosetester.NoseTester):
    def test(self, label='fast', verbose=1, extra_argv=['--exe'], doctests = False, coverage=False):
        return super(_NoseTester, self).test(label=label, verbose=verbose, extra_argv=extra_argv, doctests=doctests, coverage=coverage)

test = _NoseTester().test

import anat_preproc, \
       func_preproc, \
       reho, \
       seg_preproc, \
       registration, \
       sca, \
       basc, \
       nuisance, \
       generate_motion_statistics, \
       alff, \
       qc, \
       seg_preproc, \
       vmhc, \
       median_angle, \
       timeseries, \
       network_centrality, \
       scrubbing, \
       group_analysis, \
       easy_thresh,\
       utils, \
       pipeline, \
       cwas, \
       GUI

__all__ = ['GUI', 'pipeline', 'anat_preproc', 'func_preproc', 'registration', 'seg_preproc', 'reho', 'sca', 'basc', 'nuisance', 'alff', 'vmhc', 'median_angle', 'generate_motion_statistics', 'timeseries', 'network_centrality', 'scrubbing', 'utils', 'group_analysis', 'easy_thresh', 'qc', 'cwas']

#__version__ = '0.1-git'

from subprocess import Popen, PIPE
import re

try:
    version = '0.3.0'
#    gitproc = Popen(['git', 'log', '--oneline'], stdout = PIPE)
#    (stdout, stderr) = gitproc.communicate()
#    rows = stdout.split("\n")
#    v_num = re.search( r'(?<=(version_|release_))(.)*', rows[0])
#    if v_num:
#        version = v_num.group(0).strip("'")
#    else:
#        version = 'unknown_version'
except OSError:
    version = 'unknown_version'

__version__ =  str(version)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import math
import Image
import ImageDraw
import ImageFont
import ImageFilter
from DiagramMetrix import DiagramMetrix, XY

class ImageDrawEx(ImageDraw.ImageDraw):
    def __init__(self, im, mode=None):
        self.image = im
        self.mode = mode
        ImageDraw.ImageDraw.__init__(self, im, mode)

    def thick_rectangle(self, box, thick=1, **kwargs):
        outline = kwargs.get('outline')
        fill = kwargs.get('fill')

        if thick == 1:
            d = 0
        else:
            d = math.ceil(thick / 2.0)

        x1, y1, x2, y2 = box
        self.rectangle(box, fill=fill)
        self.line(((x1, y1 - d), (x1, y2 + d)), fill=outline, width=thick)
        self.line(((x2, y1 - d), (x2, y2 + d)), fill=outline, width=thick)
        self.line(((x1, y1), (x2, y1)), fill=outline, width=thick)
        self.line(((x1, y2), (x2, y2)), fill=outline, width=thick)

    def setupFont(self, font, fontsize):
        if font:
            ttfont = ImageFont.truetype(font, fontsize)
        else:
            ttfont = None

        return ttfont

    def truetypeText(self, xy, string, **kwargs):
        scale_bias = 4
        fill = kwargs.get('fill')
        font = kwargs.get('font')
        fontsize = kwargs.get('fontsize', 11)

        if font is None:
            ttfont = self.setupFont(font, fontsize)
            ImageDraw.ImageDraw.text(self, xy, string,
                                     fill=fill, font=ttfont)
        else:
            ttfont = self.setupFont(font, fontsize * scale_bias)

            size = self.textsize(string, font=ttfont)
            image = Image.new('RGBA', size)
            draw = ImageDraw.Draw(image)
            draw.text((0, 0), string, fill=fill, font=ttfont)
            del draw

            basesize = (size[0] / scale_bias, size[1] / scale_bias)
            text_image = image.resize(basesize, Image.ANTIALIAS)

            self.image.paste(text_image, xy, text_image)
            ImageDraw.ImageDraw.__init__(self, self.image, self.mode)

    def text(self, box, string, **kwargs):
        font = kwargs.get('font')
        fontsize = kwargs.get('fontsize', 11)
        fill = kwargs.get('fill')
        ttfont = self.setupFont(font, fontsize)

        lineSpacing = kwargs.get('lineSpacing', 2)
        size = (box[2] - box[0], box[3] - box[1])

        lines = self._getFoldedText(size, string,
                                    font=ttfont, lineSpacing=lineSpacing)

        height = 0
        for string in lines:
            height += self.textsize(string, font=ttfont)[1] + lineSpacing

        height = (size[1] - (height - lineSpacing)) / 2
        xy = (box[0], box[1])
        for string in lines:
            textsize = self.textsize(string, font=ttfont)
            x = int(math.ceil((size[0] - textsize[0]) / 2.0))

            draw_xy = (xy[0] + x, xy[1] + height)
            self.truetypeText(draw_xy, string, fill=fill,
                              font=font, fontsize=fontsize)

            height += textsize[1] + lineSpacing

    def _getFoldedText(self, size, string, **kwargs):
        ttfont = kwargs.get('font')
        lineSpacing = kwargs.get('lineSpacing', 2)
        lines = []

        height = 0
        truncated = 0
        for line in string.splitlines():
            while line:
                string = string.strip()
                for i in range(0, len(string)):
                    length = len(string) - i
                    metrics = self.textsize(string[0:length], font=ttfont)

                    if metrics[0] <= size[0]:
                        break

                if size[1] < height + metrics[1]:
                    truncated = 1
                    break

                lines.append(line[0:length])
                line = line[length:]

                height += metrics[1] + lineSpacing

        # truncate last line.
        if truncated:
            string = lines.pop()
            for i in range(0, len(string)):
                if i == 0:
                    truncated = string + ' ...'
                else:
                    truncated = string[0:-i] + ' ...'

                size = self.textsize(truncated, font=ttfont)
                if size[0] <= size[0]:
                    lines.append(truncated)
                    break

        return lines

    def loadImage(self, filename, box, scale):
        box_width = box[2] - box[0]
        box_height = box[3] - box[1]

        # resize image.
        image = Image.open(filename)
        w = min([box_width, image.size[0] * scale])
        h = min([box_height, image.size[1] * scale])
        image.thumbnail((w, h), Image.ANTIALIAS)

        # centering image.
        w, h = image.size
        if box_width > w:
            x = box[0] + (box_width - w) / 2
        else:
            x = box[0]

        if box_height > h:
            y = box[1] + (box_height - h) / 2
        else:
            y = box[1]

        self.image.paste(image, (x, y))
        ImageDraw.ImageDraw.__init__(self, self.image, self.mode)

class DiagramDraw(object):
    def __init__(self, mode=None, **kwargs):
        self.mode = None
        self.image = None
        self.imageDraw = None
        self._scale = kwargs.get('scale', 1)
        self.metrix = DiagramMetrix(**kwargs)
        self.fill = kwargs.get('fill', (0, 0, 0))
        self.badgeFill = kwargs.get('badgeFill', 'pink')
        self.shadow = kwargs.get('shadow', (128, 128, 128))
        self.font = kwargs.get('font')
        self.fontsize = kwargs.get('fontsize', 11)

    def scale(self, value):
        if isinstance(value, XY):
            ret = XY(value.x * self._scale, value.y * self._scale)
        elif isinstance(value, tuple):
            ret = tuple([self.scale(x) for x in value])
        elif isinstance(value, list):
            ret = [self.scale(x) for x in value]
        elif isinstance(value, int):
            ret = value * self._scale
        else:
            ret = value

        return ret

    def draw(self, screen, **kwargs):
        self.screen = screen

        paperSize = self.metrix.pageSize(screen.nodes)
        self.image = Image.new('RGB', paperSize, (256, 256, 256))
        self.imageDraw = ImageDrawEx(self.image, self.mode)

        self._prepareEdges()
        self._drawBackground()

        if self._scale > 1:
            self.image = self.image.resize(self.scale(paperSize),
                                           Image.ANTIALIAS)
            self.imageDraw = ImageDrawEx(self.image, self.mode)

        for node in (x for x in self.screen.nodes if x.drawable):
            self.screennode(node, **kwargs)

        for edge in self.screen.edges:
            self.edge(edge)

    def _prepareEdges(self):
        for edge in self.screen.edges:
            m = self.metrix.edge(edge)
            dir = m.direction()
            if dir == 'right':
                r = range(edge.node1.xy[0] + 1, edge.node2.xy[0])
                for x in r:
                    xy = (x, edge.node1.xy[1])
                    nodes = [x for x in self.screen.nodes if x.xy == xy]
                    if len(nodes) > 0:
                        edge.skipped = 1
            elif dir == 'right-down':
                r = range(edge.node1.xy[0] + 1, edge.node2.xy[0])
                for x in r:
                    xy = (x, edge.node2.xy[1])
                    nodes = [x for x in self.screen.nodes if x.xy == xy]
                    if len(nodes) > 0:
                        edge.skipped = 1

    def _drawBackground(self):
        originalScale = self._scale
        self._scale = 1

        # Draw node groups.
        for node in (x for x in self.screen.nodes if x.drawable == 0):
            marginBox = self.metrix.node(node).marginBox()
            self.imageDraw.rectangle(self.scale(marginBox), fill=node.color)

        # Drop node shadows.
        for node in (x for x in self.screen.nodes if x.drawable):
            shadowBox = self.metrix.node(node).shadowBox()
            self.imageDraw.rectangle(self.scale(shadowBox), fill=self.shadow)

        # Smoothing back-ground images.
        for i in range(15):
            self.image = self.image.filter(ImageFilter.SMOOTH_MORE)

        self.imageDraw = ImageDrawEx(self.image, self.mode)
        self._scale = originalScale

    def screennode(self, node, **kwargs):
        m = self.metrix.node(node)

        if node.background:
            self.imageDraw.thick_rectangle(self.scale(m.box()),
                                           outline=self.fill, fill=node.color)
            self.imageDraw.loadImage(node.background, self.scale(m.box()),
                                     scale=self._scale)
            self.imageDraw.thick_rectangle(self.scale(m.box()),
                                           outline=self.fill)
        else:
            self.imageDraw.thick_rectangle(self.scale(m.box()),
                                           outline=self.fill, fill=node.color)

        fontsize = self.scale(self.fontsize)
        lineSpacing = self.scale(self.metrix.lineSpacing)
        self.imageDraw.text(self.scale(m.coreBox()), node.label,
                            fill=self.fill, font=self.font, fontsize=fontsize,
                            lineSpacing=lineSpacing)

        if node.numbered != None:
            xy = self.scale(m.topLeft())
            r = self.scale(self.metrix.cellSize)

            box = [xy.x - r, xy.y - r, xy.x + r, xy.y + r]
            self.imageDraw.ellipse(box, outline=self.fill, fill=self.badgeFill)
            self.imageDraw.text(box, node.numbered, fill=self.fill,
                                font=self.font, fontsize=fontsize)

    def edge(self, edge):
        metrix = self.metrix.edge(edge)

        if edge.color:
            color = edge.color
        else:
            color = self.fill

        shaft = metrix.shaft()
        self.imageDraw.line(self.scale(shaft), fill=color)

        for head in metrix.heads():
            self.imageDraw.polygon(self.scale(head), outline=color, fill=color)

    def save(self, filename, format, size=None):
        if size:
            x, y = size
        else:
            x, y = self.image.size
            x = int(x / self._scale)
            y = int(y / self._scale)

        self.image.thumbnail((x, y), Image.ANTIALIAS)
        self.image.save(filename, format)
<commit_msg> * Fasten rendering truetype'd string; fixed #3<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import math
import Image
import ImageDraw
import ImageFont
import ImageFilter
from DiagramMetrix import DiagramMetrix, XY

class ImageDrawEx(ImageDraw.ImageDraw):
    def __init__(self, im, mode=None):
        self.image = im
        self.mode = mode
        ImageDraw.ImageDraw.__init__(self, im, mode)

    def thick_rectangle(self, box, thick=1, **kwargs):
        outline = kwargs.get('outline')
        fill = kwargs.get('fill')

        if thick == 1:
            d = 0
        else:
            d = math.ceil(thick / 2.0)

        x1, y1, x2, y2 = box
        self.rectangle(box, fill=fill)
        self.line(((x1, y1 - d), (x1, y2 + d)), fill=outline, width=thick)
        self.line(((x2, y1 - d), (x2, y2 + d)), fill=outline, width=thick)
        self.line(((x1, y1), (x2, y1)), fill=outline, width=thick)
        self.line(((x1, y2), (x2, y2)), fill=outline, width=thick)

    def setupFont(self, font, fontsize):
        if font:
            ttfont = ImageFont.truetype(font, fontsize)
        else:
            ttfont = None

        return ttfont

    def truetypeText(self, xy, string, **kwargs):
        fill = kwargs.get('fill')
        font = kwargs.get('font')
        fontsize = kwargs.get('fontsize', 11)
        ttfont = self.setupFont(font, fontsize)

        if font is None:
            ImageDraw.ImageDraw.text(self, xy, string,
                                     fill=fill, font=ttfont)
        else:
            size = self.textsize(string, font=ttfont)

            # Generate mask to support BDF(bitmap font)
            mask = Image.new('1', size)
            draw = ImageDraw.Draw(mask)
            draw.text((0, 0), string, fill='white', font=ttfont)

            # Rendering text
            filler = Image.new('RGB', size, fill)
            self.image.paste(filler, xy, mask)

            ImageDraw.ImageDraw.__init__(self, self.image, self.mode)

    def text(self, box, string, **kwargs):
        font = kwargs.get('font')
        fontsize = kwargs.get('fontsize', 11)
        fill = kwargs.get('fill')
        ttfont = self.setupFont(font, fontsize)

        lineSpacing = kwargs.get('lineSpacing', 2)
        size = (box[2] - box[0], box[3] - box[1])

        lines = self._getFoldedText(size, string,
                                    font=ttfont, lineSpacing=lineSpacing)

        height = 0
        for string in lines:
            height += self.textsize(string, font=ttfont)[1] + lineSpacing

        height = (size[1] - (height - lineSpacing)) / 2
        xy = (box[0], box[1])
        for string in lines:
            textsize = self.textsize(string, font=ttfont)
            x = int(math.ceil((size[0] - textsize[0]) / 2.0))

            draw_xy = (xy[0] + x, xy[1] + height)
            self.truetypeText(draw_xy, string, fill=fill,
                              font=font, fontsize=fontsize)

            height += textsize[1] + lineSpacing

    def _getFoldedText(self, size, string, **kwargs):
        ttfont = kwargs.get('font')
        lineSpacing = kwargs.get('lineSpacing', 2)
        lines = []

        height = 0
        truncated = 0
        for line in string.splitlines():
            while line:
                string = string.strip()
                for i in range(0, len(string)):
                    length = len(string) - i
                    metrics = self.textsize(string[0:length], font=ttfont)

                    if metrics[0] <= size[0]:
                        break

                if size[1] < height + metrics[1]:
                    truncated = 1
                    break

                lines.append(line[0:length])
                line = line[length:]

                height += metrics[1] + lineSpacing

        # truncate last line.
        if truncated:
            string = lines.pop()
            for i in range(0, len(string)):
                if i == 0:
                    truncated = string + ' ...'
                else:
                    truncated = string[0:-i] + ' ...'

                size = self.textsize(truncated, font=ttfont)
                if size[0] <= size[0]:
                    lines.append(truncated)
                    break

        return lines

    def loadImage(self, filename, box, scale):
        box_width = box[2] - box[0]
        box_height = box[3] - box[1]

        # resize image.
        image = Image.open(filename)
        w = min([box_width, image.size[0] * scale])
        h = min([box_height, image.size[1] * scale])
        image.thumbnail((w, h), Image.ANTIALIAS)

        # centering image.
        w, h = image.size
        if box_width > w:
            x = box[0] + (box_width - w) / 2
        else:
            x = box[0]

        if box_height > h:
            y = box[1] + (box_height - h) / 2
        else:
            y = box[1]

        self.image.paste(image, (x, y))
        ImageDraw.ImageDraw.__init__(self, self.image, self.mode)

class DiagramDraw(object):
    def __init__(self, mode=None, **kwargs):
        self.mode = None
        self.image = None
        self.imageDraw = None
        self._scale = kwargs.get('scale', 1)
        self.metrix = DiagramMetrix(**kwargs)
        self.fill = kwargs.get('fill', (0, 0, 0))
        self.badgeFill = kwargs.get('badgeFill', 'pink')
        self.shadow = kwargs.get('shadow', (128, 128, 128))
        self.font = kwargs.get('font')
        self.fontsize = kwargs.get('fontsize', 11)

    def scale(self, value):
        if isinstance(value, XY):
            ret = XY(value.x * self._scale, value.y * self._scale)
        elif isinstance(value, tuple):
            ret = tuple([self.scale(x) for x in value])
        elif isinstance(value, list):
            ret = [self.scale(x) for x in value]
        elif isinstance(value, int):
            ret = value * self._scale
        else:
            ret = value

        return ret

    def draw(self, screen, **kwargs):
        self.screen = screen

        paperSize = self.metrix.pageSize(screen.nodes)
        self.image = Image.new('RGB', paperSize, (256, 256, 256))
        self.imageDraw = ImageDrawEx(self.image, self.mode)

        self._prepareEdges()
        self._drawBackground()

        if self._scale > 1:
            self.image = self.image.resize(self.scale(paperSize),
                                           Image.ANTIALIAS)
            self.imageDraw = ImageDrawEx(self.image, self.mode)

        for node in (x for x in self.screen.nodes if x.drawable):
            self.screennode(node, **kwargs)

        for edge in self.screen.edges:
            self.edge(edge)

    def _prepareEdges(self):
        for edge in self.screen.edges:
            m = self.metrix.edge(edge)
            dir = m.direction()
            if dir == 'right':
                r = range(edge.node1.xy[0] + 1, edge.node2.xy[0])
                for x in r:
                    xy = (x, edge.node1.xy[1])
                    nodes = [x for x in self.screen.nodes if x.xy == xy]
                    if len(nodes) > 0:
                        edge.skipped = 1
            elif dir == 'right-down':
                r = range(edge.node1.xy[0] + 1, edge.node2.xy[0])
                for x in r:
                    xy = (x, edge.node2.xy[1])
                    nodes = [x for x in self.screen.nodes if x.xy == xy]
                    if len(nodes) > 0:
                        edge.skipped = 1

    def _drawBackground(self):
        originalScale = self._scale
        self._scale = 1

        # Draw node groups.
        for node in (x for x in self.screen.nodes if x.drawable == 0):
            marginBox = self.metrix.node(node).marginBox()
            self.imageDraw.rectangle(self.scale(marginBox), fill=node.color)

        # Drop node shadows.
        for node in (x for x in self.screen.nodes if x.drawable):
            shadowBox = self.metrix.node(node).shadowBox()
            self.imageDraw.rectangle(self.scale(shadowBox), fill=self.shadow)

        # Smoothing back-ground images.
        for i in range(15):
            self.image = self.image.filter(ImageFilter.SMOOTH_MORE)

        self.imageDraw = ImageDrawEx(self.image, self.mode)
        self._scale = originalScale

    def screennode(self, node, **kwargs):
        m = self.metrix.node(node)

        if node.background:
            self.imageDraw.thick_rectangle(self.scale(m.box()),
                                           outline=self.fill, fill=node.color)
            self.imageDraw.loadImage(node.background, self.scale(m.box()),
                                     scale=self._scale)
            self.imageDraw.thick_rectangle(self.scale(m.box()),
                                           outline=self.fill)
        else:
            self.imageDraw.thick_rectangle(self.scale(m.box()),
                                           outline=self.fill, fill=node.color)

        fontsize = self.scale(self.fontsize)
        lineSpacing = self.scale(self.metrix.lineSpacing)
        self.imageDraw.text(self.scale(m.coreBox()), node.label,
                            fill=self.fill, font=self.font, fontsize=fontsize,
                            lineSpacing=lineSpacing)

        if node.numbered != None:
            xy = self.scale(m.topLeft())
            r = self.scale(self.metrix.cellSize)

            box = [xy.x - r, xy.y - r, xy.x + r, xy.y + r]
            self.imageDraw.ellipse(box, outline=self.fill, fill=self.badgeFill)
            self.imageDraw.text(box, node.numbered, fill=self.fill,
                                font=self.font, fontsize=fontsize)

    def edge(self, edge):
        metrix = self.metrix.edge(edge)

        if edge.color:
            color = edge.color
        else:
            color = self.fill

        shaft = metrix.shaft()
        self.imageDraw.line(self.scale(shaft), fill=color)

        for head in metrix.heads():
            self.imageDraw.polygon(self.scale(head), outline=color, fill=color)

    def save(self, filename, format, size=None):
        if size:
            x, y = size
        else:
            x, y = self.image.size
            x = int(x / self._scale)
            y = int(y / self._scale)

        self.image.thumbnail((x, y), Image.ANTIALIAS)
        self.image.save(filename, format)
<|endoftext|>