<commit_before>#!/usr/bin/env python3
# Copyright (c) 2016-2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test Hierarchical Deterministic wallet function."""

import os
import shutil

from test_framework.test_framework import PivxTestFramework
from test_framework.util import (
    assert_equal,
    connect_nodes,
    assert_raises_rpc_error
)

class WalletHDTest(PivxTestFramework):
    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 2
        self.extra_args = [['-nuparams=v5_shield:1']] * self.num_nodes
        self.extra_args[1].append('-keypool=0')
        self.supports_cli = False

    def skip_test_if_missing_module(self):
        self.skip_if_no_wallet()

    def generate_and_fund_addresses(self, fShield, fFund, seedid, count):
        add = None
        for i in range(count):
            add = self.nodes[1].getnewshieldaddress() if fShield else self.nodes[1].getnewaddress()
            add_info = self.nodes[1].getaddressinfo(add)
            assert_equal(add_info["hdkeypath"], "m/32'/119'/%d'" % (i + 1) if fShield else "m/44'/119'/0'/0'/%d'" % i)
            assert_equal(add_info["hdseedid"], seedid)
            if fFund:
                self.nodes[0].sendtoaddress(add, 1)
                self.nodes[0].generate(1)
        return add

    def generate_and_fund_transparent_addr(self, seedid, count):
        return self.generate_and_fund_addresses(False, True, seedid, count)

    def generate_and_fund_shield_addr(self, seedid, count):
        return self.generate_and_fund_addresses(True, True, seedid, count)

    def generate_transparent_addr(self, seedid, count):
        return self.generate_and_fund_addresses(False, False, seedid, count)

    def generate_shield_addr(self, seedid, count):
        return self.generate_and_fund_addresses(True, False, seedid, count)

    def start_and_connect_node1(self):
        self.start_node(1)
        connect_nodes(self.nodes[0], 1)
        connect_nodes(self.nodes[1], 0)
        self.sync_all()

    def check_addressbook(self, old_book, new_book):
        assert_equal(len(old_book), len(new_book))
        for add in new_book:
            assert add in old_book
            assert_equal(old_book[add], new_book[add])

    def run_test(self):
        # Make sure we use hd
        if '-legacywallet' in self.nodes[0].extra_args:
            self.log.info("Exiting HD test for non-HD wallets")
            return
        # keep masterkeyid
        masterkeyid = self.nodes[1].getwalletinfo()['hdseedid']
        assert_equal(len(masterkeyid), 40)

        # create an internal key
        change_addr = self.nodes[1].getrawchangeaddress()
        change_addrV= self.nodes[1].getaddressinfo(change_addr)
        assert_equal(change_addrV["hdkeypath"], "m/44'/119'/0'/1'/0'") #first internal child key

        # Import a non-HD private key in the HD wallet
        non_hd_add = self.nodes[0].getnewaddress()
        self.nodes[1].importprivkey(self.nodes[0].dumpprivkey(non_hd_add))

        # This should be enough to keep the master key and the non-HD key
        self.nodes[1].backupwallet(os.path.join(self.nodes[1].datadir, "hd.bak"))

        # Retrieve the hd seed from the dump
        self.nodes[1].dumpwallet(os.path.join(self.nodes[1].datadir, "hd.dump"))
        hdseed = None
        with open(os.path.join(self.nodes[1].datadir, "hd.dump"), 'r', encoding='utf8') as f:
            for line in f:
                if "hdseed=1" in line:
                    hdseed = line.split(' ')[0]
                    break
        assert hdseed is not None

        # Derive some HD addresses and remember the last
        # Also send funds to each add
        self.nodes[0].generate(101)
        NUM_HD_ADDS = 10
        hd_add = self.generate_and_fund_transparent_addr(masterkeyid, NUM_HD_ADDS)
        self.nodes[0].sendtoaddress(non_hd_add, 1)
        self.nodes[0].generate(1)

        # Derive some Shield addresses and remember the last
        # Also send funds to each add
        NUM_SHIELD_ADDS = 10
        z_add = self.generate_and_fund_shield_addr(masterkeyid, NUM_SHIELD_ADDS)

        # create an internal key (again)
        change_addr = self.nodes[1].getrawchangeaddress()
        change_addrV= self.nodes[1].getaddressinfo(change_addr)
        assert_equal(change_addrV["hdkeypath"], "m/44'/119'/0'/1'/1'") #second internal child key

        self.sync_all()
        assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + NUM_SHIELD_ADDS + 1)

        # verify address-book recovery
        self.log.info("Restore backup (with chain)...")
        addrbook_old = self.nodes[1].getaddressesbylabel("")
        self.stop_node(1)
        shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, "regtest", "wallet.dat"))
        self.start_node(1)
        connect_nodes(self.nodes[0], 1)
        self.sync_all()
        self.check_addressbook(addrbook_old, self.nodes[1].getaddressesbylabel(""))

        # now delete the chain and recreate the addresses
        self.log.info("Restore backup (without chain)...")
        self.stop_node(1)
        # we need to delete the complete regtest directory
        # otherwise node1 would auto-recover all funds in flag the keypool keys as used
        shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "blocks"))
        shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "chainstate"))
        shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, "regtest", "wallet.dat"))
        self.start_node(1)

        # Assert that derivation is deterministic
        hd_add_2 = self.generate_transparent_addr(masterkeyid, NUM_HD_ADDS)
        assert_equal(hd_add, hd_add_2)
        z_add_2 = self.generate_shield_addr(masterkeyid, NUM_SHIELD_ADDS)
        assert_equal(z_add, z_add_2)
        # connect and sync
        connect_nodes(self.nodes[0], 1)
        connect_nodes(self.nodes[1], 0)
        self.sync_all()
        assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + NUM_SHIELD_ADDS + 1)

        # Try rescan
        self.stop_node(1)
        self.start_node(1, extra_args=self.extra_args[1] + ['-rescan'])
        assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + NUM_SHIELD_ADDS + 1)

        # Delete chain and resync (without recreating shield addresses)
        self.stop_node(1)
        shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "blocks"))
        shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "chainstate"))
        shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, "regtest", "wallet.dat"))
        self.start_and_connect_node1()
        # Wallet automatically scans blocks older than key on startup (but shielded addresses need to be regenerated)
        assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + 1)

        """ todo: Implement rescanblockchain
        out = self.nodes[1].rescanblockchain(0, 1)
        assert_equal(out['start_height'], 0)
        assert_equal(out['stop_height'], 1)
        out = self.nodes[1].rescanblockchain()
        assert_equal(out['start_height'], 0)
        assert_equal(out['stop_height'], self.nodes[1].getblockcount())
        assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + 1)
        """

        # send a tx and make sure its using the internal chain for the changeoutput
        txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1)
        outs = self.nodes[1].decoderawtransaction(self.nodes[1].gettransaction(txid)['hex'])['vout']
        keypath = ""
        for out in outs:
            if out['value'] != 1:
                keypath = self.nodes[1].getaddressinfo(out['scriptPubKey']['addresses'][0])['hdkeypath']

        assert_equal(keypath[0:16], "m/44'/119'/0'/1'")

        # Generate a new HD seed on node 1 and make sure it is set
        orig_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid']
        self.nodes[1].sethdseed()
        new_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid']
        assert orig_masterkeyid != new_masterkeyid
        addr = self.nodes[1].getnewaddress()
        assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/44\'/119\'/0\'/0\'/0\'') # Make sure the new address is the first from the keypool
        self.nodes[1].keypoolrefill(1) # Fill keypool with 1 key

        # Set a new HD seed on node 1 without flushing the keypool
        new_seed = self.nodes[0].dumpprivkey(self.nodes[0].getnewaddress())
        orig_masterkeyid = new_masterkeyid
        self.nodes[1].sethdseed(False, new_seed)
        new_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid']
        assert orig_masterkeyid != new_masterkeyid
        addr = self.nodes[1].getnewaddress()
        assert_equal(orig_masterkeyid, self.nodes[1].getaddressinfo(addr)['hdseedid'])
        assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/44\'/119\'/0\'/0\'/1\'') # Make sure the new address continues previous keypool

        # Check that the next address is from the new seed
        self.nodes[1].keypoolrefill(1)
        next_addr = self.nodes[1].getnewaddress()
        assert_equal(new_masterkeyid, self.nodes[1].getaddressinfo(next_addr)['hdseedid'])
        assert_equal(self.nodes[1].getaddressinfo(next_addr)['hdkeypath'], 'm/44\'/119\'/0\'/0\'/0\'') # Make sure the new address is not from previous keypool
        assert next_addr != addr

        # Sethdseed parameter validity
        assert_raises_rpc_error(-1, 'sethdseed', self.nodes[0].sethdseed, False, new_seed, 0)
        assert_raises_rpc_error(-5, "Invalid private key", self.nodes[1].sethdseed, False, "not_wif")
        assert_raises_rpc_error(-1, "JSON value is not a boolean as expected", self.nodes[1].sethdseed, "Not_bool")
        assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[1].sethdseed, False, True)
        assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, new_seed)
        assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, self.nodes[1].dumpprivkey(self.nodes[1].getnewaddress()))

        # Delete wallet and recover from first seed
        self.stop_node(1)
        os.remove(os.path.join(self.nodes[1].datadir, "regtest", "wallet.dat"))
        self.start_node(1)
        # Regenerate old shield addresses
        self.log.info("Restore from seed ...")
        self.nodes[1].sethdseed(True, hdseed)
        z_add_3 = self.generate_shield_addr(masterkeyid, NUM_SHIELD_ADDS)
        assert_equal(z_add, z_add_3)
        # Restart, zap, and check balance: 1 PIV * (NUM_HD_ADDS + NUM_SHIELD_ADDS) recovered from seed
        self.stop_node(1)
        self.start_node(1, extra_args=self.extra_args[1] + ['-zapwallettxes=1'])
        assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + NUM_SHIELD_ADDS)

if __name__ == '__main__':
    WalletHDTest().main ()
<commit_msg>test: wallet_hd.py enable RPC based rescan test case<commit_after>#!/usr/bin/env python3
# Copyright (c) 2016-2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test Hierarchical Deterministic wallet function."""

import os
import shutil

from test_framework.test_framework import PivxTestFramework
from test_framework.util import (
    assert_equal,
    connect_nodes,
    assert_raises_rpc_error
)

class WalletHDTest(PivxTestFramework):
    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 2
        self.extra_args = [['-nuparams=v5_shield:1']] * self.num_nodes
        self.extra_args[1].append('-keypool=0')
        self.supports_cli = False

    def skip_test_if_missing_module(self):
        self.skip_if_no_wallet()

    def generate_and_fund_addresses(self, fShield, fFund, seedid, count):
        add = None
        for i in range(count):
            add = self.nodes[1].getnewshieldaddress() if fShield else self.nodes[1].getnewaddress()
            add_info = self.nodes[1].getaddressinfo(add)
            assert_equal(add_info["hdkeypath"], "m/32'/119'/%d'" % (i + 1) if fShield else "m/44'/119'/0'/0'/%d'" % i)
            assert_equal(add_info["hdseedid"], seedid)
            if fFund:
                self.nodes[0].sendtoaddress(add, 1)
                self.nodes[0].generate(1)
        return add

    def generate_and_fund_transparent_addr(self, seedid, count):
        return self.generate_and_fund_addresses(False, True, seedid, count)

    def generate_and_fund_shield_addr(self, seedid, count):
        return self.generate_and_fund_addresses(True, True, seedid, count)

    def generate_transparent_addr(self, seedid, count):
        return self.generate_and_fund_addresses(False, False, seedid, count)

    def generate_shield_addr(self, seedid, count):
        return self.generate_and_fund_addresses(True, False, seedid, count)

    def start_and_connect_node1(self):
        self.start_node(1)
        connect_nodes(self.nodes[0], 1)
        connect_nodes(self.nodes[1], 0)
        self.sync_all()

    def check_addressbook(self, old_book, new_book):
        assert_equal(len(old_book), len(new_book))
        for add in new_book:
            assert add in old_book
            assert_equal(old_book[add], new_book[add])

    def run_test(self):
        # Make sure we use hd
        if '-legacywallet' in self.nodes[0].extra_args:
            self.log.info("Exiting HD test for non-HD wallets")
            return
        # keep masterkeyid
        masterkeyid = self.nodes[1].getwalletinfo()['hdseedid']
        assert_equal(len(masterkeyid), 40)

        # create an internal key
        change_addr = self.nodes[1].getrawchangeaddress()
        change_addrV= self.nodes[1].getaddressinfo(change_addr)
        assert_equal(change_addrV["hdkeypath"], "m/44'/119'/0'/1'/0'") #first internal child key

        # Import a non-HD private key in the HD wallet
        non_hd_add = self.nodes[0].getnewaddress()
        self.nodes[1].importprivkey(self.nodes[0].dumpprivkey(non_hd_add))

        # This should be enough to keep the master key and the non-HD key
        self.nodes[1].backupwallet(os.path.join(self.nodes[1].datadir, "hd.bak"))

        # Retrieve the hd seed from the dump
        self.nodes[1].dumpwallet(os.path.join(self.nodes[1].datadir, "hd.dump"))
        hdseed = None
        with open(os.path.join(self.nodes[1].datadir, "hd.dump"), 'r', encoding='utf8') as f:
            for line in f:
                if "hdseed=1" in line:
                    hdseed = line.split(' ')[0]
                    break
        assert hdseed is not None

        # Derive some HD addresses and remember the last
        # Also send funds to each add
        self.nodes[0].generate(101)
        NUM_HD_ADDS = 10
        hd_add = self.generate_and_fund_transparent_addr(masterkeyid, NUM_HD_ADDS)
        self.nodes[0].sendtoaddress(non_hd_add, 1)
        self.nodes[0].generate(1)

        # Derive some Shield addresses and remember the last
        # Also send funds to each add
        NUM_SHIELD_ADDS = 10
        z_add = self.generate_and_fund_shield_addr(masterkeyid, NUM_SHIELD_ADDS)

        # create an internal key (again)
        change_addr = self.nodes[1].getrawchangeaddress()
        change_addrV= self.nodes[1].getaddressinfo(change_addr)
        assert_equal(change_addrV["hdkeypath"], "m/44'/119'/0'/1'/1'") #second internal child key

        self.sync_all()
        assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + NUM_SHIELD_ADDS + 1)

        # verify address-book recovery
        self.log.info("Restore backup (with chain)...")
        addrbook_old = self.nodes[1].getaddressesbylabel("")
        self.stop_node(1)
        shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, "regtest", "wallet.dat"))
        self.start_node(1)
        connect_nodes(self.nodes[0], 1)
        self.sync_all()
        self.check_addressbook(addrbook_old, self.nodes[1].getaddressesbylabel(""))

        # now delete the chain and recreate the addresses
        self.log.info("Restore backup (without chain)...")
        self.stop_node(1)
        # we need to delete the complete regtest directory
        # otherwise node1 would auto-recover all funds in flag the keypool keys as used
        shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "blocks"))
        shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "chainstate"))
        shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, "regtest", "wallet.dat"))
        self.start_node(1)

        # Assert that derivation is deterministic
        hd_add_2 = self.generate_transparent_addr(masterkeyid, NUM_HD_ADDS)
        assert_equal(hd_add, hd_add_2)
        z_add_2 = self.generate_shield_addr(masterkeyid, NUM_SHIELD_ADDS)
        assert_equal(z_add, z_add_2)
        # connect and sync
        connect_nodes(self.nodes[0], 1)
        connect_nodes(self.nodes[1], 0)
        self.sync_all()
        assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + NUM_SHIELD_ADDS + 1)

        # Try rescan
        self.stop_node(1)
        self.start_node(1, extra_args=self.extra_args[1] + ['-rescan'])
        assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + NUM_SHIELD_ADDS + 1)

        # Try a RPC based rescan
        self.stop_node(1)
        shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "blocks"))
        shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "chainstate"))
        shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, "regtest", "wallet.dat"))
        self.start_and_connect_node1()
        # Wallet automatically scans blocks older than key on startup (but shielded addresses need to be regenerated)
        assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + 1)
        # Wallet automatically scans blocks older than key on startup
        out = self.nodes[1].rescanblockchain(0, 1)
        assert_equal(out['start_height'], 0)
        assert_equal(out['stop_height'], 1)
        out = self.nodes[1].rescanblockchain()
        assert_equal(out['start_height'], 0)
        assert_equal(out['stop_height'], self.nodes[1].getblockcount())
        assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + 1)

        # send a tx and make sure its using the internal chain for the changeoutput
        txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1)
        outs = self.nodes[1].decoderawtransaction(self.nodes[1].gettransaction(txid)['hex'])['vout']
        keypath = ""
        for out in outs:
            if out['value'] != 1:
                keypath = self.nodes[1].getaddressinfo(out['scriptPubKey']['addresses'][0])['hdkeypath']

        assert_equal(keypath[0:16], "m/44'/119'/0'/1'")

        # Generate a new HD seed on node 1 and make sure it is set
        orig_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid']
        self.nodes[1].sethdseed()
        new_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid']
        assert orig_masterkeyid != new_masterkeyid
        addr = self.nodes[1].getnewaddress()
        assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/44\'/119\'/0\'/0\'/0\'') # Make sure the new address is the first from the keypool
        self.nodes[1].keypoolrefill(1) # Fill keypool with 1 key

        # Set a new HD seed on node 1 without flushing the keypool
        new_seed = self.nodes[0].dumpprivkey(self.nodes[0].getnewaddress())
        orig_masterkeyid = new_masterkeyid
        self.nodes[1].sethdseed(False, new_seed)
        new_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid']
        assert orig_masterkeyid != new_masterkeyid
        addr = self.nodes[1].getnewaddress()
        assert_equal(orig_masterkeyid, self.nodes[1].getaddressinfo(addr)['hdseedid'])
        assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/44\'/119\'/0\'/0\'/1\'') # Make sure the new address continues previous keypool

        # Check that the next address is from the new seed
        self.nodes[1].keypoolrefill(1)
        next_addr = self.nodes[1].getnewaddress()
        assert_equal(new_masterkeyid, self.nodes[1].getaddressinfo(next_addr)['hdseedid'])
        assert_equal(self.nodes[1].getaddressinfo(next_addr)['hdkeypath'], 'm/44\'/119\'/0\'/0\'/0\'') # Make sure the new address is not from previous keypool
        assert next_addr != addr

        # Sethdseed parameter validity
        assert_raises_rpc_error(-1, 'sethdseed', self.nodes[0].sethdseed, False, new_seed, 0)
        assert_raises_rpc_error(-5, "Invalid private key", self.nodes[1].sethdseed, False, "not_wif")
        assert_raises_rpc_error(-1, "JSON value is not a boolean as expected", self.nodes[1].sethdseed, "Not_bool")
        assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[1].sethdseed, False, True)
        assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, new_seed)
        assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, self.nodes[1].dumpprivkey(self.nodes[1].getnewaddress()))

        # Delete wallet and recover from first seed
        self.stop_node(1)
        os.remove(os.path.join(self.nodes[1].datadir, "regtest", "wallet.dat"))
        self.start_node(1)
        # Regenerate old shield addresses
        self.log.info("Restore from seed ...")
        self.nodes[1].sethdseed(True, hdseed)
        z_add_3 = self.generate_shield_addr(masterkeyid, NUM_SHIELD_ADDS)
        assert_equal(z_add, z_add_3)
        # Restart, zap, and check balance: 1 PIV * (NUM_HD_ADDS + NUM_SHIELD_ADDS) recovered from seed
        self.stop_node(1)
        self.start_node(1, extra_args=self.extra_args[1] + ['-zapwallettxes=1'])
        assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + NUM_SHIELD_ADDS)

if __name__ == '__main__':
    WalletHDTest().main ()
<|endoftext|>


<commit_before># Lemoneval Project
# Author: Abhabongse Janthong <abhabongse@gmail.com>

class BaseValidator(object):
    """Defines a callable object which checks if the given value is valid."""

    __slots__ = ("name",)

    def __init__(self, *, name=None):
        self.name = name

    def __call__(self, value, target):
        """Checks whether the value is valid. If the validation is successful
        then True is returned. Otherwise, either False is returned or an
        exception describing what went wrong is raised.

        Target is the name of the object requiring the validation and it is
        used for error messages in exceptions.
        """
        pass

class PredicateValidator(BaseValidator):
    """Checks whether the given values passes the predicate test."""

    __slots__ = ("predicate",)

    def __init__(self, predicate, *, name=None):
        if name is None:
            name = getattr(predicate, "__qualname__", None)
        super().__init__(name=name)
        self.predicate = predicate

    def __call__(self, value, target):
        if self.predicate(value):
            return True
        test_name = self.name or "the predicate test"
        raise ValueError(
            f"the given value {value!r} failed {test_name} for the "
            f"parameter '{target}'"
        )
<commit_msg>PredicateValidator raises more verbose errors<commit_after># Lemoneval Project
# Author: Abhabongse Janthong <abhabongse@gmail.com>

class BaseValidator(object):
    """Defines a callable object which checks if the given value is valid."""

    __slots__ = ("name",)

    def __init__(self, *, name=None):
        self.name = name

    def __call__(self, value, target):
        """Checks whether the value is valid. If the validation is successful
        then True is returned. Otherwise, either False is returned or an
        exception describing what went wrong is raised.

        Target is the name of the object requiring the validation and it is
        used for error messages in exceptions.
        """
        pass

class PredicateValidator(BaseValidator):
    """Checks whether the given values passes the predicate test."""

    __slots__ = ("predicate",)

    def __init__(self, predicate, *, name=None):
        if name is None:
            name = getattr(predicate, "__qualname__", None)
        super().__init__(name=name)
        self.predicate = predicate

    def __call__(self, value, target):
        try:
            if self.predicate(value):
                return True
            else:
                raise ValueError
        except Exception as e:
            test_name = self.name or "the predicate test"
            e.args += (
                f"the given value {value!r} failed {test_name} for the "
                f"parameter '{target}'",
            )
            raise
<|endoftext|>


<commit_before>"""This module contains preset settings for defining metric generators"""
from __future__ import absolute_import, print_function, division, unicode_literals

__all__ = [
    'accuracy_from_scores',
    'accuracy_from_preds',
    'recall_from_scores',
    'recall_from_preds',
    'precision_from_scores',
    'precision_from_preds',
    'f1_score_from_scores',
    'f1_score_from_preds',
    'mae',
    'mse',
    'roc_auc_score_from_scores'
]

accuracy_from_scores = {
    'name': 'Accuracy',
    'source':
    """from sklearn.metrics import accuracy_score
import numpy as np

def metric_generator(y_true, y_probas):
    \"\"\"This function computes the accuracy given the true labels array (y_true)
    and the scores/probabilities array (y_probas) with shape (num_samples, num_classes).
    For the function to work correctly, the columns of the probabilities array must
    correspond to a sorted set of the unique values present in y_true.
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) != y_probas.shape[1]:
        raise ValueError('The shape of y_probas does not correspond to the number of unique values in y_true')
    argmax = np.argmax(y_probas, axis=1)
    y_preds = classes_[argmax]
    return accuracy_score(y_true, y_preds)
""",
    'selection_name': 'Accuracy from Scores/Probabilities'
}

accuracy_from_preds = {
    'name': 'Accuracy',
    'source':
    """from sklearn.metrics import accuracy_score

def metric_generator(y_true, y_preds):
    \"\"\"This function computes the accuracy given the true labels array (y_true)
    and the predicted labels array (y_preds).
    \"\"\"
    return accuracy_score(y_true, y_preds)
""",
    'selection_name': 'Accuracy from Predictions'
}

recall_from_scores = {
    'name': 'Recall',
    'source':
    """from sklearn.metrics import recall_score
import numpy as np

def metric_generator(y_true, y_probas):
    \"\"\"This function computes the recall given the true labels array (y_true)
    and the scores/probabilities array (y_probas) with shape (num_samples, num_classes).
    For the function to work correctly, the columns of the probabilities array must
    correspond to a sorted set of the unique values present in y_true. If there are more than
    two classes, micro-averaging is used by default.
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) != np.array(y_probas).shape[1]:
        raise ValueError('The shape of y_probas does not correspond to the number of unique values in y_true')
    argmax = np.argmax(y_probas, axis=1)
    y_preds = classes_[argmax]
    if np.array(y_probas).shape[1] > 2:
        score = recall_score(y_true, y_preds, average='micro')
    else:
        score = recall_score(y_true, y_preds)
    return score
""",
    'selection_name': 'Recall from Scores/Probabilities'
}

recall_from_preds = {
    'name': 'Recall',
    'source':
    """from sklearn.metrics import recall_score
import numpy as np

def metric_generator(y_true, y_preds):
    \"\"\"This function computes the recall given the true labels array (y_true)
    and the predicted labels array (y_preds).
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) > 2:
        score = recall_score(y_true, y_preds, average='micro')
    else:
        score = recall_score(y_true, y_preds)
    return score
""",
    'selection_name': 'Recall from Predictions'
}

precision_from_scores = {
    'name': 'Precision',
    'source':
    """from sklearn.metrics import precision_score
import numpy as np

def metric_generator(y_true, y_probas):
    \"\"\"This function computes the precision given the true labels array (y_true)
    and the scores/probabilities array (y_probas) with shape (num_samples, num_classes).
    For the function to work correctly, the columns of the probabilities array must
    correspond to a sorted set of the unique values present in y_true. If there are more than
    two classes, micro-averaging is used by default.
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) != np.array(y_probas).shape[1]:
        raise ValueError('The shape of y_probas does not correspond to the number of unique values in y_true')
    argmax = np.argmax(y_probas, axis=1)
    y_preds = classes_[argmax]
    if np.array(y_probas).shape[1] > 2:
        score = precision_score(y_true, y_preds, average='micro')
    else:
        score = precision_score(y_true, y_preds)
    return score
""",
    'selection_name': 'Precision from Scores/Probabilities'
}

precision_from_preds = {
    'name': 'Precision',
    'source':
    """from sklearn.metrics import precision_score
import numpy as np

def metric_generator(y_true, y_preds):
    \"\"\"This function computes the precision given the true labels array (y_true)
    and the predicted labels array (y_preds).
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) > 2:
        score = precision_score(y_true, y_preds, average='micro')
    else:
        score = precision_score(y_true, y_preds)
    return score
""",
    'selection_name': 'Precision from Predictions'
}

f1_score_from_scores = {
    'name': 'F1 Score',
    'source':
    """from sklearn.metrics import f1_score
import numpy as np

def metric_generator(y_true, y_probas):
    \"\"\"This function computes the F1 score given the true labels array (y_true)
    and the scores/probabilities array (y_probas) with shape (num_samples, num_classes).
    For the function to work correctly, the columns of the probabilities array must
    correspond to a sorted set of the unique values present in y_true. If there are more than
    two classes, micro-averaging is used by default.
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) != np.array(y_probas).shape[1]:
        raise ValueError('The shape of y_probas does not correspond to the number of unique values in y_true')
    argmax = np.argmax(y_probas, axis=1)
    y_preds = classes_[argmax]
    if np.array(y_probas).shape[1] > 2:
        score = f1_score(y_true, y_preds, average='micro')
    else:
        score = f1_score(y_true, y_preds)
    return score
""",
    'selection_name': 'F1 Score from Scores/Probabilities'
}

f1_score_from_preds = {
    'name': 'F1 Score',
    'source':
    """from sklearn.metrics import f1_score
import numpy as np

def metric_generator(y_true, y_preds):
    \"\"\"This function computes the F1 score given the true labels array (y_true)
    and the predicted labels array (y_preds).
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) > 2:
        score = f1_score(y_true, y_preds, average='micro')
    else:
        score = f1_score(y_true, y_preds)
    return score
""",
    'selection_name': 'F1 Score from Predictions'
}

mae = {
    'name': 'Mean Absolute Error',
    'source':
    """from sklearn.metrics import mean_absolute_error

metric_generator = mean_absolute_error
""",
    'selection_name': 'Mean Absolute Error'
}

mse = {
    'name': 'Mean Squared Error',
    'source':
    """from sklearn.metrics import mean_squared_error

metric_generator = mean_squared_error
""",
    'selection_name': 'Mean Squared Error'
}

roc_auc_score_from_scores = {
    'name': 'ROC AUC Score',
    'source':
    """from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import label_binarize
import numpy as np

def metric_generator(y_true, y_probas):
    \"\"\"This function computes the Area under Curve of the
    Receiver Operating Characteristic given the true labels array (y_true)
    and the scores/probabilities array (y_probas). In a non-binary classification
    task, this will calculate a weighted mean of the AUC for each class. This
    behavior can be changed by passing a different parameter to the
    `average` argument.
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) != np.array(y_probas).shape[1]:
        raise ValueError('The shape of y_probas does not correspond to the number of unique values in y_true')
    binarized = label_binarize(y_true, classes_)
    if len(classes_) == 2:
        binarized = binarized.ravel()
        y_probas = y_probas[:, 1]
    return roc_auc_score(binarized, y_probas, average='weighted')
""",
    'selection_name': 'ROC AUC Score from Scores/Probabilities'
}
<commit_msg>add median absolute error, r2 score, and explained variance score (#24)<commit_after>"""This module contains preset settings for defining metric generators"""
from __future__ import absolute_import, print_function, division, unicode_literals

__all__ = [
    'accuracy_from_scores',
    'accuracy_from_preds',
    'recall_from_scores',
    'recall_from_preds',
    'precision_from_scores',
    'precision_from_preds',
    'f1_score_from_scores',
    'f1_score_from_preds',
    'mae',
    'mse',
    'median_absolute_error',
    'r2_score',
    'explained_variance_score',
    'roc_auc_score_from_scores'
]

accuracy_from_scores = {
    'name': 'Accuracy',
    'source':
    """from sklearn.metrics import accuracy_score
import numpy as np

def metric_generator(y_true, y_probas):
    \"\"\"This function computes the accuracy given the true labels array (y_true)
    and the scores/probabilities array (y_probas) with shape (num_samples, num_classes).
    For the function to work correctly, the columns of the probabilities array must
    correspond to a sorted set of the unique values present in y_true.
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) != y_probas.shape[1]:
        raise ValueError('The shape of y_probas does not correspond to the number of unique values in y_true')
    argmax = np.argmax(y_probas, axis=1)
    y_preds = classes_[argmax]
    return accuracy_score(y_true, y_preds)
""",
    'selection_name': 'Accuracy from Scores/Probabilities'
}

accuracy_from_preds = {
    'name': 'Accuracy',
    'source':
    """from sklearn.metrics import accuracy_score

def metric_generator(y_true, y_preds):
    \"\"\"This function computes the accuracy given the true labels array (y_true)
    and the predicted labels array (y_preds).
    \"\"\"
    return accuracy_score(y_true, y_preds)
""",
    'selection_name': 'Accuracy from Predictions'
}

recall_from_scores = {
    'name': 'Recall',
    'source':
    """from sklearn.metrics import recall_score
import numpy as np

def metric_generator(y_true, y_probas):
    \"\"\"This function computes the recall given the true labels array (y_true)
    and the scores/probabilities array (y_probas) with shape (num_samples, num_classes).
    For the function to work correctly, the columns of the probabilities array must
    correspond to a sorted set of the unique values present in y_true. If there are more than
    two classes, micro-averaging is used by default.
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) != np.array(y_probas).shape[1]:
        raise ValueError('The shape of y_probas does not correspond to the number of unique values in y_true')
    argmax = np.argmax(y_probas, axis=1)
    y_preds = classes_[argmax]
    if np.array(y_probas).shape[1] > 2:
        score = recall_score(y_true, y_preds, average='micro')
    else:
        score = recall_score(y_true, y_preds)
    return score
""",
    'selection_name': 'Recall from Scores/Probabilities'
}

recall_from_preds = {
    'name': 'Recall',
    'source':
    """from sklearn.metrics import recall_score
import numpy as np

def metric_generator(y_true, y_preds):
    \"\"\"This function computes the recall given the true labels array (y_true)
    and the predicted labels array (y_preds).
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) > 2:
        score = recall_score(y_true, y_preds, average='micro')
    else:
        score = recall_score(y_true, y_preds)
    return score
""",
    'selection_name': 'Recall from Predictions'
}

precision_from_scores = {
    'name': 'Precision',
    'source':
    """from sklearn.metrics import precision_score
import numpy as np

def metric_generator(y_true, y_probas):
    \"\"\"This function computes the precision given the true labels array (y_true)
    and the scores/probabilities array (y_probas) with shape (num_samples, num_classes).
    For the function to work correctly, the columns of the probabilities array must
    correspond to a sorted set of the unique values present in y_true. If there are more than
    two classes, micro-averaging is used by default.
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) != np.array(y_probas).shape[1]:
        raise ValueError('The shape of y_probas does not correspond to the number of unique values in y_true')
    argmax = np.argmax(y_probas, axis=1)
    y_preds = classes_[argmax]
    if np.array(y_probas).shape[1] > 2:
        score = precision_score(y_true, y_preds, average='micro')
    else:
        score = precision_score(y_true, y_preds)
    return score
""",
    'selection_name': 'Precision from Scores/Probabilities'
}

precision_from_preds = {
    'name': 'Precision',
    'source':
    """from sklearn.metrics import precision_score
import numpy as np

def metric_generator(y_true, y_preds):
    \"\"\"This function computes the precision given the true labels array (y_true)
    and the predicted labels array (y_preds).
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) > 2:
        score = precision_score(y_true, y_preds, average='micro')
    else:
        score = precision_score(y_true, y_preds)
    return score
""",
    'selection_name': 'Precision from Predictions'
}

f1_score_from_scores = {
    'name': 'F1 Score',
    'source':
    """from sklearn.metrics import f1_score
import numpy as np

def metric_generator(y_true, y_probas):
    \"\"\"This function computes the F1 score given the true labels array (y_true)
    and the scores/probabilities array (y_probas) with shape (num_samples, num_classes).
    For the function to work correctly, the columns of the probabilities array must
    correspond to a sorted set of the unique values present in y_true. If there are more than
    two classes, micro-averaging is used by default.
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) != np.array(y_probas).shape[1]:
        raise ValueError('The shape of y_probas does not correspond to the number of unique values in y_true')
    argmax = np.argmax(y_probas, axis=1)
    y_preds = classes_[argmax]
    if np.array(y_probas).shape[1] > 2:
        score = f1_score(y_true, y_preds, average='micro')
    else:
        score = f1_score(y_true, y_preds)
    return score
""",
    'selection_name': 'F1 Score from Scores/Probabilities'
}

f1_score_from_preds = {
    'name': 'F1 Score',
    'source':
    """from sklearn.metrics import f1_score
import numpy as np

def metric_generator(y_true, y_preds):
    \"\"\"This function computes the F1 score given the true labels array (y_true)
    and the predicted labels array (y_preds).
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) > 2:
        score = f1_score(y_true, y_preds, average='micro')
    else:
        score = f1_score(y_true, y_preds)
    return score
""",
    'selection_name': 'F1 Score from Predictions'
}

mae = {
    'name': 'Mean Absolute Error',
    'source':
    """from sklearn.metrics import mean_absolute_error

metric_generator = mean_absolute_error
""",
    'selection_name': 'Mean Absolute Error'
}

mse = {
    'name': 'Mean Squared Error',
    'source':
    """from sklearn.metrics import mean_squared_error

metric_generator = mean_squared_error
""",
    'selection_name': 'Mean Squared Error'
}

median_absolute_error = {
    'name': 'Median Absolute Error',
    'source':
    """from sklearn.metrics import median_absolute_error

metric_generator = median_absolute_error
""",
    'selection_name': 'Median Absolute Error'
}

r2_score = {
    'name': 'R-Squared Score',
    'source':
    """from sklearn.metrics import r2_score

metric_generator = r2_score
""",
    'selection_name': 'R-Squared Score'
}

explained_variance_score = {
    'name': 'Explained Variance Score',
    'source':
    """from sklearn.metrics import explained_variance_score

metric_generator = explained_variance_score
""",
    'selection_name': 'Explained Variance Score'
}

roc_auc_score_from_scores = {
    'name': 'ROC AUC Score',
    'source':
    """from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import label_binarize
import numpy as np

def metric_generator(y_true, y_probas):
    \"\"\"This function computes the Area under Curve of the
    Receiver Operating Characteristic given the true labels array (y_true)
    and the scores/probabilities array (y_probas). In a non-binary classification
    task, this will calculate a weighted mean of the AUC for each class. This
    behavior can be changed by passing a different parameter to the
    `average` argument.
    \"\"\"
    classes_ = np.unique(y_true)
    if len(classes_) != np.array(y_probas).shape[1]:
        raise ValueError('The shape of y_probas does not correspond to the number of unique values in y_true')
    binarized = label_binarize(y_true, classes_)
    if len(classes_) == 2:
        binarized = binarized.ravel()
        y_probas = y_probas[:, 1]
    return roc_auc_score(binarized, y_probas, average='weighted')
""",
    'selection_name': 'ROC AUC Score from Scores/Probabilities'
}
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys
import os
import getopt;
from twitter import *

# Default values
def_counts = 10
users = []
dir_creds = "~/.twit_cli/"

# input parameters
def usage():
  print "Run: " + sys.argv[0] + " [OPTIONS]"
  print "Where OPTIONS is one the following: "
  print " -h --help"
  print " -c N --counts=N"
  print " -u \"name\" --user=\"name\""
  print " -a --all"

def main(argv):
  global def_counts, users
  try:
    opts, args = getopt.getopt(argv,"hc:u:a",["help","counts=","user=","all"])
  except getopt.GetoptError:
    usage()
    exit(2)
  for opt,arg in opts:
    if opt in ["-h","--help"]:
      usage()
      exit(1)
    elif opt in ["-c","--counts"]:
      print "Retrieving "+str(arg)+" tweets."
      def_counts = arg
    elif opt in ["-u","--user"]:
      print "User: "+arg
      users.append(arg)
    elif opt in ["-a","--all"]:
      print "all users"
      if os.path.exists(os.path.expanduser(dir_creds)):
	token_files = filter(lambda x: x.endswith('.token'), os.listdir(os.path.expanduser(dir_creds)))
	for file in token_files:
	  usr = file[:file.index(".token")]
	  users.append(usr)
      else:
	print "No user to be added, path undefined"
	exit(2)
    else:
      print "Got the following and I don't know what to do with it:"
      print opt + " " + arg
      usage()
      exit(2)

if __name__ == "__main__":
  main(sys.argv[1:])

if len(users) < 1:
  users.append["bilbo_pingouin"]

for user in users:
  print "\n"+user
  # Retrieve the credentials for a given account
  if not os.path.exists(dir_creds):
    os.makedirs(dir_creds) # I just assume there is not race issue here!

  file_creds = dir_creds + user + ".token"
  MY_TWITTER_CREDS = os.path.expanduser(file_creds)

  #TODO: try to setup a page on my server where those values could be obtained!
  if os.path.exists("./app_tokens.dat"):
    cust_token, cust_secret = read_token_file(os.path.expanduser("./app_tokens.dat"))
  else:
    print "ERROR: The app is not identified!"

  if not os.path.exists(MY_TWITTER_CREDS):
    oauth_dance("Twit on CLI",cust_token,cust_secret,
			    MY_TWITTER_CREDS)

  oauth_token, oauth_secret = read_token_file(MY_TWITTER_CREDS)

  # OAuth idnetification
  t = Twitter(auth=OAuth(oauth_token,oauth_secret,cust_token,cust_secret))

  # Get status
  data = t.statuses.home_timeline(count=def_counts)

  # Print lines
  for c in range(len(data)):
    print "* "+data[c]['user']['name']+'('+data[c]['user']['screen_name']+')'+" - "+data[c]['text']+" ## "+data[c]['created_at']

<commit_msg>some C, python and HTML tests<commit_after>#!/usr/bin/python

import sys
import os
import getopt
from twitter import *

# Default values
def_counts = 10
users = []
dir_creds = "~/.twit_cli/"

# input parameters
def usage():
  print "Run: " + sys.argv[0] + " [OPTIONS]"
  print "Where OPTIONS is one the following: "
  print " -h --help"
  print " -c N --counts=N"
  print " -u \"name\" --user=\"name\""
  print " -a --all"

def main(argv):
  global def_counts, users
  try:
    opts, args = getopt.getopt(argv,"hc:u:a",["help","counts=","user=","all"])
  except getopt.GetoptError:
    usage()
    exit(2)
  for opt,arg in opts:
    if opt in ["-h","--help"]:
      usage()
      exit(1)
    elif opt in ["-c","--counts"]:
      print "Retrieving "+str(arg)+" tweets."
      def_counts = arg
    elif opt in ["-u","--user"]:
      print "User: "+arg
      users.append(arg)
    elif opt in ["-a","--all"]:
      print "all users"
      if os.path.exists(os.path.expanduser(dir_creds)):
	token_files = filter(lambda x: x.endswith('.token'), os.listdir(os.path.expanduser(dir_creds)))
	for file in token_files:
	  usr = file[:file.index(".token")]
	  users.append(usr)
      else:
	print "No user to be added, path undefined"
	exit(2)
    else:
      print "Got the following and I don't know what to do with it:"
      print opt + " " + arg
      usage()
      exit(2)

if __name__ == "__main__":
  main(sys.argv[1:])

if len(users) < 1:
  users.append["bilbo_pingouin"]

for user in users:
  print "\n"+user
  # Retrieve the credentials for a given account
  if not os.path.exists(dir_creds):
    os.makedirs(dir_creds) # I just assume there is not race issue here!

  file_creds = dir_creds + user + ".token"
  MY_TWITTER_CREDS = os.path.expanduser(file_creds)

  #TODO: try to setup a page on my server where those values could be obtained!
  if os.path.exists("./app_tokens.dat"):
    cust_token, cust_secret = read_token_file(os.path.expanduser("./app_tokens.dat"))
  else:
    print "ERROR: The app is not identified!"

  if not os.path.exists(MY_TWITTER_CREDS):
    oauth_dance("Twit on CLI",cust_token,cust_secret,
			    MY_TWITTER_CREDS)

  oauth_token, oauth_secret = read_token_file(MY_TWITTER_CREDS)

  # OAuth idnetification
  t = Twitter(auth=OAuth(oauth_token,oauth_secret,cust_token,cust_secret))

  # Get status
  data = t.statuses.home_timeline(count=def_counts)

  # Print lines
  for c in range(len(data)):
    print "* "+data[c]['user']['name']+'('+data[c]['user']['screen_name']+')'+" - "+data[c]['text']+" ## "+data[c]['created_at']

<|endoftext|>


<commit_before>from collections import defaultdict
from django.contrib.auth.decorators import login_required
from django.db import connection
from django.shortcuts import render
from lazysignup.decorators import allow_lazy_user
import colorsys
import math
from collections import namedtuple

NAMES = ('numbers', 'addition', 'subtraction', 'multiplication', 'division')
SKILL_TABLES = {
    'numbers':
    [[str(c + r * 10) for c in range(1, 11)] for r in range(2)],
    'addition':
    [['%s+%s' % (c, r) for c in range(1, 11)] for r in range(1, 21)],
    'subtraction':
    [['%s-%s' % (r, c) for c in range(1, r + 1)] for r in range(1, 11)],
    'multiplication':
    [['%sx%s' % (c, r) for c in range(1, 11)] for r in range(1, 21)],
    'division':
    [['%s/%s' % (a * b, b) for a in range(1, 11)] for b in range(1, 11)],
}

skill_keys = ['pk', 'name', 'note', 'value', 'value_percent', 'style', 'image_name', 'used', 'active']
SkillTuple = namedtuple('Skill', ', '.join(skill_keys))
skill_as_tuple = lambda skill: SkillTuple(*[skill[k] for k in skill_keys])

@allow_lazy_user
@login_required
def my_skills(request, pk=None):
    all_skills = get_all_skills(request.user)

    par = None
    if pk is not None:
        pk = int(pk)
        while pk!=1:
            par, pk = pk, all_skills['id_parid'][int(pk)]
    active = all_skills['id_name'][par] if par is not None else 'numbers'

    data = []
    skills = []
    for name in NAMES:
        skill = get_skill_obj_by_name(name, all_skills)
        skills.append(skill)
        data.append(get_my_skills(name, all_skills))

    ret = render(request, 'model/my_skills.html', {
        "data": data,
        "skills": skills,
        "active": active,
    })

    return ret

def get_all_skills(user):
    cursor = connection.cursor()
    cursor.execute(
        '''SELECT s.id, s.parent_id, s.name, usk.val, s.note, s.active
        FROM model_skill s LEFT JOIN
            (SELECT us.skill_id as sid, us.value as val
             FROM model_userskill us
             WHERE us.user_id={0}) usk
        ON (s.id = usk.sid)
        '''.format(user.pk))
    rows = cursor.fetchall()
    skills = {sid: (parid, name, val, note, active)
              for sid, parid, name, val, note, active in rows}
    user_skills = {}
    name_id = {}
    id_name = {}
    id_parid = {}
    id_note = {}
    id_used = {}
    id_active = {}
    children = defaultdict(list)
    for sid, (parid, name, val, note, active) in skills.iteritems():
        get_skill_value(sid, skills, user_skills)
        name_id[name] = sid
        id_name[sid] = name
        id_parid[sid] = parid
        id_note[sid] = note
        id_used[sid] = val is not None
        id_active[sid] = active
        if parid:
            children[parid].append(sid)
    return {'name_id': name_id,
            'id_name': id_name,
            'id_val': user_skills,
            'id_parid': id_parid,
            'children': children,
            'id_note': id_note,
            'id_used': id_used,
            'id_active': id_active,
            }

def get_children(parent_list, skills):
    parids = [skills['name_id'][p] for p in parent_list]
    children = [ch for parid in parids for ch in skills['children'][parid]]
    ret = {}
    for sid in children:
        obj = get_skill_obj(sid, skills)
        ret[obj.name] = obj
    return ret

def get_skill_obj(sid, skills):
    obj = {'name': skills['id_name'][sid],
           'value': skills['id_val'][sid],
           'image_name': "core/imgs/skill_{}.png".format(skills['id_name'][sid]),
           'pk': sid,
           'note': skills['id_note'][sid],
           'active': skills['id_active'][sid],
           'used': skills['id_used'][sid]}
    obj['value_percent'] = int(100. / (1 + math.exp(-obj['value'])))
    obj['style'] = get_style(obj['value'], used=skills['id_used'][sid])
    return skill_as_tuple(obj)

def get_skill_value(sid, skills, user_skills):
    """
    compute and save absolute skill of user
    """
    if sid not in user_skills:
        parid, name, val, _, _ = skills[sid]
        val = val or 0
        parval = get_skill_value(parid, skills, user_skills) if parid else 0
        user_skills[sid] = val + parval
    return user_skills[sid]

def get_skill_obj_by_name(name, skills):
    sid = skills['name_id'][name]
    return get_skill_obj(sid, skills)

def get_my_skills(name, skills):
    top = get_children([name], skills)
    children = get_children([s.name for s in top.values()], skills)
    return {
        "table": [[get_skill_repr(c, children) for c in r]
                  for r in SKILL_TABLES[name]],
        "skills": top,
        "skill": get_skill_obj_by_name(name, skills),
    }

def get_skill_repr(name, user_skills):
    if name in user_skills:
        return {'name': name, 'style': user_skills[name].style}
    else:
        return {'name': '', 'style': get_style(None)}

def get_style(value, used=True):
    ''' return css style of user skill'''
    if value is None:
        return 'background-color: rgba(127, 127, 0, 0);'

    value = (1 / (1 + math.exp(-value)))
    color = colorsys.hsv_to_rgb(1. / 12 + value * 2 / 9., 1, 0.8)
    color = [int(c * 255) for c in color]
    alpha = 1 if used else 0.2
    return "background-color: rgba({0[0]}, {0[1]}, {0[2]}, {1});".\
        format(color, alpha)
<commit_msg>preselect used skill in my skill when none specifide - related to issue #35<commit_after>from collections import defaultdict
from django.contrib.auth.decorators import login_required
from django.db import connection
from django.shortcuts import render
from lazysignup.decorators import allow_lazy_user
import colorsys
import math
from collections import namedtuple

NAMES = ('numbers', 'addition', 'subtraction', 'multiplication', 'division')
SKILL_TABLES = {
    'numbers':
    [[str(c + r * 10) for c in range(1, 11)] for r in range(2)],
    'addition':
    [['%s+%s' % (c, r) for c in range(1, 11)] for r in range(1, 21)],
    'subtraction':
    [['%s-%s' % (r, c) for c in range(1, r + 1)] for r in range(1, 11)],
    'multiplication':
    [['%sx%s' % (c, r) for c in range(1, 11)] for r in range(1, 21)],
    'division':
    [['%s/%s' % (a * b, b) for a in range(1, 11)] for b in range(1, 11)],
}

skill_keys = ['pk', 'name', 'note', 'value', 'value_percent', 'style', 'image_name', 'used', 'active']
SkillTuple = namedtuple('Skill', ', '.join(skill_keys))
skill_as_tuple = lambda skill: SkillTuple(*[skill[k] for k in skill_keys])

@allow_lazy_user
@login_required
def my_skills(request, pk=None):
    all_skills = get_all_skills(request.user)

    par = None
    if pk is not None:
        pk = int(pk)
        while pk!=1:
            par, pk = pk, all_skills['id_parid'][int(pk)]
    active = all_skills['id_name'][par] if par is not None else None

    data = []
    skills = []
    for name in NAMES:
        skill = get_skill_obj_by_name(name, all_skills)
        skills.append(skill)
        data.append(get_my_skills(name, all_skills))
        if skill.used and active is None:
            active = skill.name

    if active is None:
        active = NAMES[0]

    ret = render(request, 'model/my_skills.html', {
        "data": data,
        "skills": skills,
        "active": active,
    })

    return ret

def get_all_skills(user):
    cursor = connection.cursor()
    cursor.execute(
        '''SELECT s.id, s.parent_id, s.name, usk.val, s.note, s.active
        FROM model_skill s LEFT JOIN
            (SELECT us.skill_id as sid, us.value as val
             FROM model_userskill us
             WHERE us.user_id={0}) usk
        ON (s.id = usk.sid)
        '''.format(user.pk))
    rows = cursor.fetchall()
    skills = {sid: (parid, name, val, note, active)
              for sid, parid, name, val, note, active in rows}
    user_skills = {}
    name_id = {}
    id_name = {}
    id_parid = {}
    id_note = {}
    id_used = {}
    id_active = {}
    children = defaultdict(list)
    for sid, (parid, name, val, note, active) in skills.iteritems():
        get_skill_value(sid, skills, user_skills)
        name_id[name] = sid
        id_name[sid] = name
        id_parid[sid] = parid
        id_note[sid] = note
        id_used[sid] = val is not None
        id_active[sid] = active
        if parid:
            children[parid].append(sid)
    return {'name_id': name_id,
            'id_name': id_name,
            'id_val': user_skills,
            'id_parid': id_parid,
            'children': children,
            'id_note': id_note,
            'id_used': id_used,
            'id_active': id_active,
            }

def get_children(parent_list, skills):
    parids = [skills['name_id'][p] for p in parent_list]
    children = [ch for parid in parids for ch in skills['children'][parid]]
    ret = {}
    for sid in children:
        obj = get_skill_obj(sid, skills)
        ret[obj.name] = obj
    return ret

def get_skill_obj(sid, skills):
    obj = {'name': skills['id_name'][sid],
           'value': skills['id_val'][sid],
           'image_name': "core/imgs/skill_{}.png".format(skills['id_name'][sid]),
           'pk': sid,
           'note': skills['id_note'][sid],
           'active': skills['id_active'][sid],
           'used': skills['id_used'][sid]}
    obj['value_percent'] = int(100. / (1 + math.exp(-obj['value'])))
    obj['style'] = get_style(obj['value'], used=skills['id_used'][sid])
    return skill_as_tuple(obj)

def get_skill_value(sid, skills, user_skills):
    """
    compute and save absolute skill of user
    """
    if sid not in user_skills:
        parid, name, val, _, _ = skills[sid]
        val = val or 0
        parval = get_skill_value(parid, skills, user_skills) if parid else 0
        user_skills[sid] = val + parval
    return user_skills[sid]

def get_skill_obj_by_name(name, skills):
    sid = skills['name_id'][name]
    return get_skill_obj(sid, skills)

def get_my_skills(name, skills):
    top = get_children([name], skills)
    children = get_children([s.name for s in top.values()], skills)
    return {
        "table": [[get_skill_repr(c, children) for c in r]
                  for r in SKILL_TABLES[name]],
        "skills": top,
        "skill": get_skill_obj_by_name(name, skills),
    }

def get_skill_repr(name, user_skills):
    if name in user_skills:
        return {'name': name, 'style': user_skills[name].style}
    else:
        return {'name': '', 'style': get_style(None)}

def get_style(value, used=True):
    ''' return css style of user skill'''
    if value is None:
        return 'background-color: rgba(127, 127, 0, 0);'

    value = (1 / (1 + math.exp(-value)))
    color = colorsys.hsv_to_rgb(1. / 12 + value * 2 / 9., 1, 0.8)
    color = [int(c * 255) for c in color]
    alpha = 1 if used else 0.2
    return "background-color: rgba({0[0]}, {0[1]}, {0[2]}, {1});".\
        format(color, alpha)
<|endoftext|>


<commit_before>'''
class TestTimeSerieTable(object):
    def mock_time(self, hour, min, sec):
        return datetime(2005, 7, 12, hour, min, sec)

    def test_simple_insert(self):
        t = TimeSerieTable(3600)
        d = self.mock_time(14, 12, 30)
        t.insert(d)

        assert_equals(t.start, d)
        assert_equals(len(t.table), 1)

        d = self.mock_time(14, 12, 40)
        t.insert(d)
        assert_equals(len(t.table), 1)
        assert_equals(t.table[len(t.table) - 1]['sum'], 2)

    def test_expand_at_beginning(self):
        t = TimeSerieTable(3600)
        t.insert(self.mock_time(14, 45, 30))
        t.insert(self.mock_time(13, 30, 45))

        assert_equals(t.start, self.mock_time(12, 45, 30))
        assert_equals(len(t.table), 3)
        assert_equals(t.table[0]['sum'], 1)
        assert_equals(t.table[len(t.table) - 1]['sum'], 2)

    def test_expand_and_insert(self):
        t = TimeSerieTable(3600)
        t.insert(self.mock_time(13, 30, 45))
        t.insert(self.mock_time(13, 50, 45))
        t.insert(self.mock_time(15, 45, 30))
        t.insert(self.mock_time(14, 40, 45))

        assert_equals(t.start, self.mock_time(13, 30, 45))
        assert_equals(len(t.table), 3)
        assert_equals(t.table[len(t.table) - 1]['sum'], 4)

    def test_query(self):
        t = TimeSerieTable(3600)
        t.insert(self.mock_time(13, 30, 45))
        t.insert(self.mock_time(13, 50, 45))
        t.insert(self.mock_time(15, 45, 30))
        t.insert(self.mock_time(14, 40, 45))

        res = t.query(self.mock_time(13, 30, 50), self.mock_time(14, 30, 50))
        assert_equals(res, 3)

        res = t.query(self.mock_time(9, 30, 50), self.mock_time(10, 30, 50))
        assert_equals(res, 2)

    def test_copy(self):
        t = TimeSerieTable(3600)
        t.insert(self.mock_time(12, 30, 00))
        copy_t = t.copy()
        assert_equals(len(t.table), len(copy_t.table))
'''<commit_msg>Test time serie table<commit_after>from datetime import datetime
from nose.tools import assert_equals
from server.timeserietable import TimeSerieTable

class TestTimeSerieTable(object):
    def mock_time(self, hour, min, sec):
        return datetime(2005, 7, 12, hour, min, sec)

    def test_simple_insert(self):
        t = TimeSerieTable(3600)
        d = self.mock_time(14, 12, 30)
        t.insert(d)

        assert_equals(t.start, d)
        assert_equals(len(t.table), 1)

        d = self.mock_time(14, 12, 40)
        t.insert(d)
        assert_equals(len(t.table), 1)
        assert_equals(t.table[len(t.table) - 1]['sum'], 2)

    def test_expand_at_beginning(self):
        t = TimeSerieTable(3600)
        t.insert(self.mock_time(14, 45, 30))
        t.insert(self.mock_time(13, 30, 45))

        assert_equals(t.start, self.mock_time(12, 45, 30))
        assert_equals(len(t.table), 3)
        assert_equals(t.table[0]['sum'], 1)
        assert_equals(t.table[len(t.table) - 1]['sum'], 2)

    def test_expand_and_insert(self):
        t = TimeSerieTable(3600)
        t.insert(self.mock_time(13, 30, 45))
        t.insert(self.mock_time(13, 50, 45))
        t.insert(self.mock_time(15, 45, 30))
        t.insert(self.mock_time(14, 40, 45))

        assert_equals(t.start, self.mock_time(13, 30, 45))
        assert_equals(len(t.table), 3)
        assert_equals(t.table[len(t.table) - 1]['sum'], 4)

    def test_query(self):
        t = TimeSerieTable(3600)
        t.insert(self.mock_time(13, 30, 45))
        t.insert(self.mock_time(13, 50, 45))
        t.insert(self.mock_time(15, 45, 30))
        t.insert(self.mock_time(14, 40, 45))

        res = t.query(self.mock_time(13, 30, 50), self.mock_time(14, 30, 50))
        assert_equals(res, 3)

        res = t.query(self.mock_time(9, 30, 50), self.mock_time(10, 30, 50))
        assert_equals(res, 2)

    def test_copy(self):
        t = TimeSerieTable(3600)
        t.insert(self.mock_time(12, 30, 00))
        copy_t = t.copy(1)
        assert_equals(len(t.table), len(copy_t.table))<|endoftext|>


<commit_before>import collections
import os
import threading
import warnings

import numpy

from chainer import _version
from chainer import backends  # NOQA
from chainer import dataset  # NOQA
from chainer import datasets  # NOQA
from chainer import function_hooks  # NOQA
from chainer import functions  # NOQA
from chainer import initializers  # NOQA
from chainer import iterators  # NOQA
from chainer import links  # NOQA
from chainer import optimizers  # NOQA
from chainer import serializers  # NOQA
from chainer import training  # NOQA

# import class and function
# These functions from backends.cuda are kept for backward compatibility
from chainer._runtime_info import print_runtime_info  # NOQA
from chainer.backends.cuda import should_use_cudnn  # NOQA
from chainer.backends.cuda import should_use_cudnn_tensor_core  # NOQA
from chainer.configuration import config  # NOQA
from chainer.configuration import global_config  # NOQA
from chainer.configuration import using_config  # NOQA
from chainer.function import force_backprop_mode  # NOQA
from chainer.function import Function  # NOQA
from chainer.function import FunctionAdapter  # NOQA
from chainer.function import no_backprop_mode  # NOQA
from chainer.function_hook import FunctionHook  # NOQA
from chainer.function_node import FunctionNode  # NOQA
from chainer.function_node import grad  # NOQA
from chainer.functions import array  # NOQA
from chainer.functions.math import basic_math  # NOQA
from chainer.initializer import Initializer  # NOQA
from chainer.link import Chain  # NOQA
from chainer.link import ChainList  # NOQA
from chainer.link import Link  # NOQA
from chainer.optimizer import GradientMethod  # NOQA
from chainer.optimizer import Optimizer  # NOQA
from chainer.optimizer import UpdateRule  # NOQA
from chainer.reporter import DictSummary  # NOQA
from chainer.reporter import get_current_reporter  # NOQA
from chainer.reporter import report  # NOQA
from chainer.reporter import report_scope  # NOQA
from chainer.reporter import Reporter  # NOQA
from chainer.reporter import Summary  # NOQA
from chainer.sequential import Sequential  # NOQA
from chainer.serializer import AbstractSerializer  # NOQA
from chainer.serializer import Deserializer  # NOQA
from chainer.serializer import Serializer  # NOQA
from chainer.variable import as_variable  # NOQA
from chainer.variable import Parameter  # NOQA
from chainer.variable import Variable  # NOQA

# Alias for backward compatibility
from chainer import cuda  # NOQA

from chainer import _environment_check

# Check environment conditions
_environment_check.check()

__version__ = _version.__version__

_thread_local = threading.local()
_array_types = None
_cpu_array_types = None

def get_function_hooks():
    try:
        ret = _thread_local.function_hooks
    except AttributeError:
        ret = collections.OrderedDict()
        _thread_local.function_hooks = ret
    return ret

def _load_array_types():
    # Note: this function may not be protected by GIL because of external
    # calls.
    global _array_types
    global _cpu_array_types
    if _array_types is None:
        array_types = [numpy.ndarray]
        cpu_array_types = [numpy.ndarray]

        if backends.cuda.available:
            array_types.append(backends.cuda.ndarray)

        if backends.intel64.is_ideep_available():
            array_types.append(backends.intel64.mdarray)
            cpu_array_types.append(backends.intel64.mdarray)

        array_types = tuple(array_types)
        cpu_array_types = tuple(cpu_array_types)

        _array_types = array_types
        _cpu_array_types = cpu_array_types

def get_array_types():
    _load_array_types()
    return _array_types

def get_cpu_array_types():
    _load_array_types()
    return _cpu_array_types

def is_arrays_compatible(arrays):
    arrays = [a for a in arrays if a is not None]
    if len(arrays) == 0:
        return True
    if type(arrays[0]) is backends.cuda.ndarray:
        types = backends.cuda.ndarray
    else:
        types = get_cpu_array_types()
    return all([isinstance(a, types) for a in arrays])

global_config.debug = bool(int(os.environ.get('CHAINER_DEBUG', '0')))
global_config.cudnn_deterministic = False
global_config.enable_backprop = True
global_config.keep_graph_on_report = bool(int(
    os.environ.get('CHAINER_KEEP_GRAPH_ON_REPORT', '0')))
global_config.train = True
global_config.type_check = bool(int(os.environ.get('CHAINER_TYPE_CHECK', '1')))
global_config.use_cudnn = os.environ.get('CHAINER_USE_CUDNN', 'auto')
global_config.use_cudnn_tensor_core = 'auto'
global_config.autotune = False
global_config.use_ideep = os.environ.get('CHAINER_USE_IDEEP', 'never')
global_config.lazy_grad_sum = bool(int(
    os.environ.get('CHAINER_LAZY_GRAD_SUM', '0')))
global_config.dtype = numpy.dtype(os.environ.get('CHAINER_DTYPE', 'float32'))

def is_debug():
    """Returns if the debug mode is enabled or not in the current thread.

    Returns:
        bool:  ``True`` if the debug mode is enabled.
    """
    return bool(config.debug)

def set_debug(debug):
    """Enables or disables the debug mode in the current thread.

    .. note::

        ``chainer.set_debug(value)`` is equivalent to
        ``chainer.config.debug = value``.

    Args:
        debug (bool): New debug mode.
    """
    config.debug = debug

class DebugMode(object):
    """Debug mode context.

    This class provides a context manager for debug mode. When entering the
    context, it sets the debug mode to the value of `debug` parameter with
    memorizing its original value. When exiting the context, it sets the debug
    mode back to the original value.

    .. deprecated:: v2.0.0

        Use :func:`chainer.using_config` instead. See :ref:`debug` for details.

    Args:
        debug (bool): Debug mode used in the context.
    """

    def __init__(self, debug):
        warnings.warn('chainer.DebugMode is deprecated. '
                      'Use chainer.using_config("debug", ...) instead.',
                      DeprecationWarning)
        self._using = using_config('debug', debug)

    def __enter__(self):
        self._using.__enter__()

    def __exit__(self, *args):
        self._using.__exit__(*args)

def get_dtype(dtype=None):
    """Resolves Chainer's default dtype.

    Returns:
        If ``dtype`` is not ``None``, it returns the dtype as is. Otherwise, it
        returns ``chainer.config.dtype`` (see :ref:`configuration`).

    """
    if dtype is None:
        return config.dtype
    return dtype

basic_math.install_variable_arithmetics()
array.get_item.install_variable_get_item()

disable_experimental_feature_warning = False
<commit_msg>Raise an error with a kindful message with invalid CHAINER_DTYPE<commit_after>import collections
import os
import threading
import warnings

import numpy

from chainer import _version
from chainer import backends  # NOQA
from chainer import dataset  # NOQA
from chainer import datasets  # NOQA
from chainer import function_hooks  # NOQA
from chainer import functions  # NOQA
from chainer import initializers  # NOQA
from chainer import iterators  # NOQA
from chainer import links  # NOQA
from chainer import optimizers  # NOQA
from chainer import serializers  # NOQA
from chainer import training  # NOQA

# import class and function
# These functions from backends.cuda are kept for backward compatibility
from chainer._runtime_info import print_runtime_info  # NOQA
from chainer.backends.cuda import should_use_cudnn  # NOQA
from chainer.backends.cuda import should_use_cudnn_tensor_core  # NOQA
from chainer.configuration import config  # NOQA
from chainer.configuration import global_config  # NOQA
from chainer.configuration import using_config  # NOQA
from chainer.function import force_backprop_mode  # NOQA
from chainer.function import Function  # NOQA
from chainer.function import FunctionAdapter  # NOQA
from chainer.function import no_backprop_mode  # NOQA
from chainer.function_hook import FunctionHook  # NOQA
from chainer.function_node import FunctionNode  # NOQA
from chainer.function_node import grad  # NOQA
from chainer.functions import array  # NOQA
from chainer.functions.math import basic_math  # NOQA
from chainer.initializer import Initializer  # NOQA
from chainer.link import Chain  # NOQA
from chainer.link import ChainList  # NOQA
from chainer.link import Link  # NOQA
from chainer.optimizer import GradientMethod  # NOQA
from chainer.optimizer import Optimizer  # NOQA
from chainer.optimizer import UpdateRule  # NOQA
from chainer.reporter import DictSummary  # NOQA
from chainer.reporter import get_current_reporter  # NOQA
from chainer.reporter import report  # NOQA
from chainer.reporter import report_scope  # NOQA
from chainer.reporter import Reporter  # NOQA
from chainer.reporter import Summary  # NOQA
from chainer.sequential import Sequential  # NOQA
from chainer.serializer import AbstractSerializer  # NOQA
from chainer.serializer import Deserializer  # NOQA
from chainer.serializer import Serializer  # NOQA
from chainer.variable import as_variable  # NOQA
from chainer.variable import Parameter  # NOQA
from chainer.variable import Variable  # NOQA

# Alias for backward compatibility
from chainer import cuda  # NOQA

from chainer import _environment_check

# Check environment conditions
_environment_check.check()

__version__ = _version.__version__

_thread_local = threading.local()
_array_types = None
_cpu_array_types = None

def get_function_hooks():
    try:
        ret = _thread_local.function_hooks
    except AttributeError:
        ret = collections.OrderedDict()
        _thread_local.function_hooks = ret
    return ret

def _load_array_types():
    # Note: this function may not be protected by GIL because of external
    # calls.
    global _array_types
    global _cpu_array_types
    if _array_types is None:
        array_types = [numpy.ndarray]
        cpu_array_types = [numpy.ndarray]

        if backends.cuda.available:
            array_types.append(backends.cuda.ndarray)

        if backends.intel64.is_ideep_available():
            array_types.append(backends.intel64.mdarray)
            cpu_array_types.append(backends.intel64.mdarray)

        array_types = tuple(array_types)
        cpu_array_types = tuple(cpu_array_types)

        _array_types = array_types
        _cpu_array_types = cpu_array_types

def get_array_types():
    _load_array_types()
    return _array_types

def get_cpu_array_types():
    _load_array_types()
    return _cpu_array_types

def is_arrays_compatible(arrays):
    arrays = [a for a in arrays if a is not None]
    if len(arrays) == 0:
        return True
    if type(arrays[0]) is backends.cuda.ndarray:
        types = backends.cuda.ndarray
    else:
        types = get_cpu_array_types()
    return all([isinstance(a, types) for a in arrays])

global_config.debug = bool(int(os.environ.get('CHAINER_DEBUG', '0')))
global_config.cudnn_deterministic = False
global_config.enable_backprop = True
global_config.keep_graph_on_report = bool(int(
    os.environ.get('CHAINER_KEEP_GRAPH_ON_REPORT', '0')))
global_config.train = True
global_config.type_check = bool(int(os.environ.get('CHAINER_TYPE_CHECK', '1')))
global_config.use_cudnn = os.environ.get('CHAINER_USE_CUDNN', 'auto')
global_config.use_cudnn_tensor_core = 'auto'
global_config.autotune = False
global_config.use_ideep = os.environ.get('CHAINER_USE_IDEEP', 'never')
global_config.lazy_grad_sum = bool(int(
    os.environ.get('CHAINER_LAZY_GRAD_SUM', '0')))

_chainer_dtype = os.environ.get('CHAINER_DTYPE', 'float32')
if _chainer_dtype not in ('float16', 'float32', 'float64'):
    raise TypeError('incorrect dtype name in CHAINER_DTYPE: "{}". '
                    'Only float16/32/64 are allowed.'.format(_chainer_dtype))
global_config.dtype = numpy.dtype(_chainer_dtype)

def is_debug():
    """Returns if the debug mode is enabled or not in the current thread.

    Returns:
        bool:  ``True`` if the debug mode is enabled.
    """
    return bool(config.debug)

def set_debug(debug):
    """Enables or disables the debug mode in the current thread.

    .. note::

        ``chainer.set_debug(value)`` is equivalent to
        ``chainer.config.debug = value``.

    Args:
        debug (bool): New debug mode.
    """
    config.debug = debug

class DebugMode(object):
    """Debug mode context.

    This class provides a context manager for debug mode. When entering the
    context, it sets the debug mode to the value of `debug` parameter with
    memorizing its original value. When exiting the context, it sets the debug
    mode back to the original value.

    .. deprecated:: v2.0.0

        Use :func:`chainer.using_config` instead. See :ref:`debug` for details.

    Args:
        debug (bool): Debug mode used in the context.
    """

    def __init__(self, debug):
        warnings.warn('chainer.DebugMode is deprecated. '
                      'Use chainer.using_config("debug", ...) instead.',
                      DeprecationWarning)
        self._using = using_config('debug', debug)

    def __enter__(self):
        self._using.__enter__()

    def __exit__(self, *args):
        self._using.__exit__(*args)

def get_dtype(dtype=None):
    """Resolves Chainer's default dtype.

    Returns:
        If ``dtype`` is not ``None``, it returns the dtype as is. Otherwise, it
        returns ``chainer.config.dtype`` (see :ref:`configuration`).

    """
    if dtype is None:
        return config.dtype
    return dtype

basic_math.install_variable_arithmetics()
array.get_item.install_variable_get_item()

disable_experimental_feature_warning = False
<|endoftext|>


<commit_before>import base64
import json
import mimetools
import os
import re
from urllib2 import Request, HTTPError, URLError, urlopen
from zipfile import ZipFile

def print_banner(message):
    print ("**************************************************")
    print (message)
    print ("**************************************************")

# TODO do we really want to rely only on basic auth for admin access to external Apigee?
def get_basic_auth(username, password):
    up = '%s:%s' % (username, password)
    return "Basic %s" % base64.b64encode(up).decode()

def create_kvm(secretKey, reg, lambdaARN, host, org, env, username, password):
    print_banner("Creating the KVM for Common-Jazz API Proxy .......")
    payload = {
        "encrypted": "true",
        "entry": [
            {
                "name": "accessKey",
                "value": secretKey
            },
            {
                "name": "reg",
                "value": reg
            },
            {
                "name": "lambdaARN",
                "value": lambdaARN
            }
        ],
        "name": "jzencryptedLambdaMaps"
    }
    data = json.dumps(payload).encode('utf-8')
    headers = {
        'Content-Type': 'application/json',
        'Authorization': get_basic_auth(username, password)
    }
    url = "%s/v1/o/%s/e/%s/keyvaluemaps" % (host, org, env)

    req = Request(url, data, headers)
    try:
        res = urlopen(req)
        if res.getcode() == 201:
            print_banner("KVM created successfully for the Common-Jazz API Proxy")
        else:
            print_banner("KVM creation FAILED for the Common-Jazz API Proxy")
    except HTTPError, e:
        print "HTTP Error:", e.code
        print_banner("KVM already present for the Common-Jazz API Proxy")
        pass

def get_current_deployed_version(host, org, env, flow, username, password):
    print("Getting the current deployed version of %s in %s/%s on %s" % (flow, org, env, host))
    deployedVersion = ''
    try:
        url = "%s/v1/o/%s/sharedflows/%s/deployments" % (host, org, flow)
        req = Request(url)
        req.add_header('Authorization', get_basic_auth(username, password))
        req.add_header('Accept', 'application/json')
        res = urlopen(req)
        apis = json.load(res)
        for e in apis['environment']:
            if (e['name'] == env):
                deployedVersion = e['revision'][0]['name']
                break
        print("Deployed version: %s" % deployedVersion)
    except HTTPError, e:
        # To handle very first deployment
        print "HTTP Error:", e.code
    return deployedVersion

def is_api_deployed(host, org, env, name, revision, username, password):
    url = "%s/v1/o/%s/e/%s/apis/%s/revisions/%s/deployments" % (host, org, env, name, revision)
    req = Request(url)
    req.add_header('Authorization', get_basic_auth(username, password))
    req.add_header('Accept', 'application/json')
    res = urlopen(req)
    return res.getcode() == 200

def import_item(zFile, host, org, name, importType, username, password):
    url = "%s/v1/o/%s/%s?action=import&name=%s" % (host, org, importType, name)
    boundary = mimetools.choose_boundary()
    with open(zFile, 'rb') as fileHandle:
        fileContent = fileHandle.read()
    headers = {
        'Accept': 'application/json',
        'Authorization': get_basic_auth(username, password),
        'Content-Type': "multipart/form-data; boundary=%s" % boundary
    }
    data = (
        ("--%s\r\nContent-Disposition: form-data; name=\"file\"; filename=\"%s\"\r\n" +
            "Content-Type: application/x-zip-compressed\r\n\r\n%s\r\n--%s--")
        % (boundary, zFile, fileContent, boundary))
    req = Request(url, data, headers)
    res = urlopen(req)
    result = json.load(res)
    return result['revision']

def import_api(zFile, host, org, api, username, password):
    print("Importing new api proxy %s in %s on %s" % (api, org, host))
    apiRevision = import_item(zFile, host, org, api, 'apis', username, password)
    print("  New api proxy revision imported: %s" % apiRevision)
    return apiRevision

def import_bundle(zFile, host, org, flow, username, password):
    print("Importing new SharedFlowBundle %s in %s on %s" % (flow, org, host))
    bundleRevision = import_item(zFile, host, org, flow, 'sharedflows', username, password)
    print("  New revision imported: %s" % bundleRevision)
    return bundleRevision

def undeploy(host, org, env, flow, revision, username, password):
    print("Undeploying the current deployed version of %s in %s/%s on %s" % (flow, org, env, host))
    url = "%s/v1/o/%s/e/%s/sharedflows/%s/revisions/%s/deployments" % (host, org, env, flow, revision)
    req = Request(url, headers={'Authorization': get_basic_auth(username, password)})
    req.get_method = lambda: 'DELETE'
    res = urlopen(req)
    print(res.read())

def deploy(host, org, env, flow, revision, username, password):
    print("Deploying the revision %s of %s in %s/%s on %s" % (revision, flow, org, env, host))
    url = "%s/v1/o/%s/e/%s/sharedflows/%s/revisions/%s/deployments?override=true" % (host, org, env, flow, revision)
    req = Request(url, headers={'Authorization': get_basic_auth(username, password)})
    req.get_method = lambda: 'POST'
    res = urlopen(req)
    print(res.read())

def deploy_api(host, org, env, apiName, revision, username, password):
    print("Deploying the revision %s of %s in %s/%s on %s" % (revision, apiName, org, env, host))
    url = (
        "%s/v1/o/%s/apis/%s/revisions/%s/deployments?action=deploy&env=%s&override=true"
        % (host, org, apiName, revision, env))
    req = Request(url, headers={'Authorization': get_basic_auth(username, password)})
    req.get_method = lambda: 'POST'
    res = urlopen(req)
    print(res.read())
    return is_api_deployed(host, org, env, apiName, revision, username, password)

def stamp_build(path, item, build):
    flowFile = os.path.join(path, 'sharedflowbundle', "%s.xml" % item)
    with open(flowFile, 'r') as inFile:
        content = inFile.read()
    # flake8: noqa
    content = re.sub('(?<=\<Description\>).+(?=\</Description\>)', build, content)

    with open(flowFile, 'w') as outFile:
        outFile.write(content)

    print(item + ' stamped with build: ' + build)

def zip_bundle(path, name, build):
    zfPath = os.path.join(path, "%s-%s.zip" % (name, build))
    zf = ZipFile(zfPath, 'w')
    offset = len(path)
    for dir, dirList, fileList in os.walk(os.path.join(path, 'sharedflowbundle')):
        for addFile in fileList:
            sourcePath = os.path.join(dir, addFile)
            zf.write(sourcePath, sourcePath[offset:len(sourcePath)])
    zf.close()
    return zfPath

def deploy_shared_flows(host, org, env, build, username, password):
    print_banner("Deploying Sharedflows now ............")
    flowDir = "apigeeinstaller/sharedflows"
    for item in os.listdir(flowDir):
        itemPath = os.path.join(flowDir, item)
        if os.path.isdir(itemPath):
            stamp_build(itemPath, item, build)
            zfPath = zip_bundle(itemPath, item, build)
            deployedVersion = get_current_deployed_version(host, org, env, item, username, password)
            if deployedVersion:
                undeploy(host, org, env, item, deployedVersion, username, password)
            revision = import_bundle(zfPath, host, org, item, username, password)
            deploy(host, org, env, item, revision, username, password)
    print_banner("Sharedflows deployment Complete")

def get_content(fileName, baseUrl, branch):
    fullUrl = "%s/raw/%s/apigee/%s" % (baseUrl, branch, fileName)
    try:
        print("Downloading %s...." % fullUrl)
        req = urlopen(fullUrl)
        with open(os.path.basename(fileName), "wb") as local_file:
                local_file.write(req.read())
    except HTTPError, e:
        print "HTTP Error:", e.code, fullUrl
    except URLError, e:
        print "URL Error:", e.reason, fullUrl

def deploy_common(host, org, env, username, password, contentUrl, contentBranch):
    print_banner("Importing the Common-Jazz API Proxy ......")
    get_content('Common-Jazz.zip', contentUrl, contentBranch)
    commonRevision = import_api('Common-Jazz.zip', host, org, 'Common-Jazz', username, password)
    success = deploy_api(host, org, env, 'Common-Jazz', commonRevision, username, password)
    if success:
        print_banner("Common-Jazz deployed successfully")
    else:
        print_banner("Common-Jazz NOT deployed successfully")
    return success

def install_proxy(secretKey, reg, lambdaARN, host, org, env,
                  build, username, password,
                  contentUrl='https://github.com/tmobile/jazz-content', contentBranch='master'):
    """Configure the external Apigee proxy and upload shared flows.

    Keyword arguments:
    secretKey -- AWS user secret key
    reg -- AWS userID
    lambdaARN -- ARN of the gateway function Apigee will invoke
    host -- Apigee host
    org -- Apigee org to apply this took
    env -- Apigee env to userName
    username -- Apigee instance basic auth username
    password -- apigee instance basic auth password
    """
    create_kvm(secretKey, reg, lambdaARN, host, org, env, username, password)
    deploy_shared_flows(host, org, env, build, username, password)
    return deploy_common(host, org, env, username, password, contentUrl, contentBranch)
<commit_msg>Updated accesskey and secretkey cred store in kvm (#477)<commit_after>import base64
import json
import mimetools
import os
import re
from urllib2 import Request, HTTPError, URLError, urlopen
from zipfile import ZipFile

def print_banner(message):
    print ("**************************************************")
    print (message)
    print ("**************************************************")

# TODO do we really want to rely only on basic auth for admin access to external Apigee?
def get_basic_auth(username, password):
    up = '%s:%s' % (username, password)
    return "Basic %s" % base64.b64encode(up).decode()

def create_kvm(secretKey, reg, lambdaARN, host, org, env, username, password):
    print_banner("Creating the KVM for Common-Jazz API Proxy .......")
    payload = {
        "encrypted": "true",
        "entry": [
            {
                "name": "accessKey",
                "value": reg
            },
            {
                "name": "secretKey",
                "value": secretKey
            },
            {
                "name": "reg",
                "value": reg
            },
            {
                "name": "lambdaARN",
                "value": lambdaARN
            }
        ],
        "name": "jzencryptedLambdaMaps"
    }
    data = json.dumps(payload).encode('utf-8')
    headers = {
        'Content-Type': 'application/json',
        'Authorization': get_basic_auth(username, password)
    }
    url = "%s/v1/o/%s/e/%s/keyvaluemaps" % (host, org, env)

    req = Request(url, data, headers)
    try:
        res = urlopen(req)
        if res.getcode() == 201:
            print_banner("KVM created successfully for the Common-Jazz API Proxy")
        else:
            print_banner("KVM creation FAILED for the Common-Jazz API Proxy")
    except HTTPError, e:
        print "HTTP Error:", e.code
        print_banner("KVM already present for the Common-Jazz API Proxy")
        pass

def get_current_deployed_version(host, org, env, flow, username, password):
    print("Getting the current deployed version of %s in %s/%s on %s" % (flow, org, env, host))
    deployedVersion = ''
    try:
        url = "%s/v1/o/%s/sharedflows/%s/deployments" % (host, org, flow)
        req = Request(url)
        req.add_header('Authorization', get_basic_auth(username, password))
        req.add_header('Accept', 'application/json')
        res = urlopen(req)
        apis = json.load(res)
        for e in apis['environment']:
            if (e['name'] == env):
                deployedVersion = e['revision'][0]['name']
                break
        print("Deployed version: %s" % deployedVersion)
    except HTTPError, e:
        # To handle very first deployment
        print "HTTP Error:", e.code
    return deployedVersion

def is_api_deployed(host, org, env, name, revision, username, password):
    url = "%s/v1/o/%s/e/%s/apis/%s/revisions/%s/deployments" % (host, org, env, name, revision)
    req = Request(url)
    req.add_header('Authorization', get_basic_auth(username, password))
    req.add_header('Accept', 'application/json')
    res = urlopen(req)
    return res.getcode() == 200

def import_item(zFile, host, org, name, importType, username, password):
    url = "%s/v1/o/%s/%s?action=import&name=%s" % (host, org, importType, name)
    boundary = mimetools.choose_boundary()
    with open(zFile, 'rb') as fileHandle:
        fileContent = fileHandle.read()
    headers = {
        'Accept': 'application/json',
        'Authorization': get_basic_auth(username, password),
        'Content-Type': "multipart/form-data; boundary=%s" % boundary
    }
    data = (
        ("--%s\r\nContent-Disposition: form-data; name=\"file\"; filename=\"%s\"\r\n" +
            "Content-Type: application/x-zip-compressed\r\n\r\n%s\r\n--%s--")
        % (boundary, zFile, fileContent, boundary))
    req = Request(url, data, headers)
    res = urlopen(req)
    result = json.load(res)
    return result['revision']

def import_api(zFile, host, org, api, username, password):
    print("Importing new api proxy %s in %s on %s" % (api, org, host))
    apiRevision = import_item(zFile, host, org, api, 'apis', username, password)
    print("  New api proxy revision imported: %s" % apiRevision)
    return apiRevision

def import_bundle(zFile, host, org, flow, username, password):
    print("Importing new SharedFlowBundle %s in %s on %s" % (flow, org, host))
    bundleRevision = import_item(zFile, host, org, flow, 'sharedflows', username, password)
    print("  New revision imported: %s" % bundleRevision)
    return bundleRevision

def undeploy(host, org, env, flow, revision, username, password):
    print("Undeploying the current deployed version of %s in %s/%s on %s" % (flow, org, env, host))
    url = "%s/v1/o/%s/e/%s/sharedflows/%s/revisions/%s/deployments" % (host, org, env, flow, revision)
    req = Request(url, headers={'Authorization': get_basic_auth(username, password)})
    req.get_method = lambda: 'DELETE'
    res = urlopen(req)
    print(res.read())

def deploy(host, org, env, flow, revision, username, password):
    print("Deploying the revision %s of %s in %s/%s on %s" % (revision, flow, org, env, host))
    url = "%s/v1/o/%s/e/%s/sharedflows/%s/revisions/%s/deployments?override=true" % (host, org, env, flow, revision)
    req = Request(url, headers={'Authorization': get_basic_auth(username, password)})
    req.get_method = lambda: 'POST'
    res = urlopen(req)
    print(res.read())

def deploy_api(host, org, env, apiName, revision, username, password):
    print("Deploying the revision %s of %s in %s/%s on %s" % (revision, apiName, org, env, host))
    url = (
        "%s/v1/o/%s/apis/%s/revisions/%s/deployments?action=deploy&env=%s&override=true"
        % (host, org, apiName, revision, env))
    req = Request(url, headers={'Authorization': get_basic_auth(username, password)})
    req.get_method = lambda: 'POST'
    res = urlopen(req)
    print(res.read())
    return is_api_deployed(host, org, env, apiName, revision, username, password)

def stamp_build(path, item, build):
    flowFile = os.path.join(path, 'sharedflowbundle', "%s.xml" % item)
    with open(flowFile, 'r') as inFile:
        content = inFile.read()
    # flake8: noqa
    content = re.sub('(?<=\<Description\>).+(?=\</Description\>)', build, content)

    with open(flowFile, 'w') as outFile:
        outFile.write(content)

    print(item + ' stamped with build: ' + build)

def zip_bundle(path, name, build):
    zfPath = os.path.join(path, "%s-%s.zip" % (name, build))
    zf = ZipFile(zfPath, 'w')
    offset = len(path)
    for dir, dirList, fileList in os.walk(os.path.join(path, 'sharedflowbundle')):
        for addFile in fileList:
            sourcePath = os.path.join(dir, addFile)
            zf.write(sourcePath, sourcePath[offset:len(sourcePath)])
    zf.close()
    return zfPath

def deploy_shared_flows(host, org, env, build, username, password):
    print_banner("Deploying Sharedflows now ............")
    flowDir = "apigeeinstaller/sharedflows"
    for item in os.listdir(flowDir):
        itemPath = os.path.join(flowDir, item)
        if os.path.isdir(itemPath):
            stamp_build(itemPath, item, build)
            zfPath = zip_bundle(itemPath, item, build)
            deployedVersion = get_current_deployed_version(host, org, env, item, username, password)
            if deployedVersion:
                undeploy(host, org, env, item, deployedVersion, username, password)
            revision = import_bundle(zfPath, host, org, item, username, password)
            deploy(host, org, env, item, revision, username, password)
    print_banner("Sharedflows deployment Complete")

def get_content(fileName, baseUrl, branch):
    fullUrl = "%s/raw/%s/apigee/%s" % (baseUrl, branch, fileName)
    try:
        print("Downloading %s...." % fullUrl)
        req = urlopen(fullUrl)
        with open(os.path.basename(fileName), "wb") as local_file:
                local_file.write(req.read())
    except HTTPError, e:
        print "HTTP Error:", e.code, fullUrl
    except URLError, e:
        print "URL Error:", e.reason, fullUrl

def deploy_common(host, org, env, username, password, contentUrl, contentBranch):
    print_banner("Importing the Common-Jazz API Proxy ......")
    get_content('Common-Jazz.zip', contentUrl, contentBranch)
    commonRevision = import_api('Common-Jazz.zip', host, org, 'Common-Jazz', username, password)
    success = deploy_api(host, org, env, 'Common-Jazz', commonRevision, username, password)
    if success:
        print_banner("Common-Jazz deployed successfully")
    else:
        print_banner("Common-Jazz NOT deployed successfully")
    return success

def install_proxy(secretKey, reg, lambdaARN, host, org, env,
                  build, username, password,
                  contentUrl='https://github.com/tmobile/jazz-content', contentBranch='master'):
    """Configure the external Apigee proxy and upload shared flows.

    Keyword arguments:
    secretKey -- AWS user secret key
    reg -- AWS userID
    lambdaARN -- ARN of the gateway function Apigee will invoke
    host -- Apigee host
    org -- Apigee org to apply this took
    env -- Apigee env to userName
    username -- Apigee instance basic auth username
    password -- apigee instance basic auth password
    """
    create_kvm(secretKey, reg, lambdaARN, host, org, env, username, password)
    deploy_shared_flows(host, org, env, build, username, password)
    return deploy_common(host, org, env, username, password, contentUrl, contentBranch)
<|endoftext|>


<commit_before>from MenuList import MenuList
from Tools.Directories import SCOPE_CURRENT_SKIN, resolveFilename
from enigma import RT_HALIGN_LEFT, eListboxPythonMultiContent, gFont
from Tools.LoadPixmap import LoadPixmap
from Components.config import config
def ChoiceEntryComponent(key = "", text = ["--"]):
	res = [ text ]
	if text[0] == "--":
		if config.skin.primary_skin.value == "Arctic/skin.xml":
			res.append((eListboxPythonMultiContent.TYPE_TEXT, 0, 5, 800, 36, 0, RT_HALIGN_LEFT , "-"*200))
		else:
			res.append((eListboxPythonMultiContent.TYPE_TEXT, 0, 00, 800, 25, 0, RT_HALIGN_LEFT , "-"*200))
	else:
		if config.skin.primary_skin.value != "Arctic/skin.xml":
			res.append((eListboxPythonMultiContent.TYPE_TEXT, 45, 00, 800, 25, 0, RT_HALIGN_LEFT, text[0]))
		else:
			res.append((eListboxPythonMultiContent.TYPE_TEXT, 45, 5, 800, 36, 0, RT_HALIGN_LEFT, text[0]))

		png = LoadPixmap(resolveFilename(SCOPE_CURRENT_SKIN, "skin_default/buttons/key_" + key + ".png"))

		if png is not None:
			if config.skin.primary_skin.value == "Arctic/skin.xml":
				res.append((eListboxPythonMultiContent.TYPE_PIXMAP_ALPHATEST, 5, 10, 35, 25, png))
			else:
				res.append((eListboxPythonMultiContent.TYPE_PIXMAP_ALPHATEST, 5, 0, 35, 25, png))

	return res

class ChoiceList(MenuList):
	def __init__(self, list, selection = 0, enableWrapAround=False):
		MenuList.__init__(self, list, enableWrapAround, eListboxPythonMultiContent)
		if config.skin.primary_skin.value == "Arctic/skin.xml":
			self.l.setFont(0, gFont("Regular", 22))
			self.l.setItemHeight(36)
		else:
			self.l.setFont(0, gFont("Regular", 20))
			self.l.setItemHeight(25)
		self.selection = selection

	def postWidgetCreate(self, instance):
		MenuList.postWidgetCreate(self, instance)
		self.moveToIndex(self.selection)
<commit_msg>Delete ChoiceList.py<commit_after><|endoftext|>