<commit_before>#!/usr/bin/env python3
# Copyright (c) 2014-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

#
# Test for taproot sighash algorithm with pegins and issuances

from test_framework.key import ECKey, ECPubKey, compute_xonly_pubkey, generate_privkey, sign_schnorr, tweak_add_privkey, tweak_add_pubkey, verify_schnorr
from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, CTxOutValue, CTxOutWitness, FromHex, uint256_from_str
from test_framework.test_framework import BitcoinTestFramework, SkipTest
from test_framework.script import TaprootSignatureHash, taproot_construct, taproot_pad_sighash_ty, SIGHASH_DEFAULT, SIGHASH_ALL, SIGHASH_NONE, SIGHASH_SINGLE, SIGHASH_ANYONECANPAY

VALID_SIGHASHES_ECDSA = [
    SIGHASH_ALL,
    SIGHASH_NONE,
    SIGHASH_SINGLE,
    SIGHASH_ANYONECANPAY + SIGHASH_ALL,
    SIGHASH_ANYONECANPAY + SIGHASH_NONE,
    SIGHASH_ANYONECANPAY + SIGHASH_SINGLE
]

VALID_SIGHASHES_TAPROOT = [SIGHASH_DEFAULT] + VALID_SIGHASHES_ECDSA

class TapHashPeginTest(BitcoinTestFramework):

    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 1
        self.extra_args = [
            ["-initialfreecoins=2100000000000000",
             "-anyonecanspendaremine=1",
            "-blindedaddresses=1",
            "-validatepegin=0",
            "-con_parent_chain_signblockscript=51",
            "-parentscriptprefix=75",
            "-parent_bech32_hrp=ert",
            "-minrelaytxfee=0",
            "-maxtxfee=100.0",
        ]]

    def skip_test_if_missing_module(self):
        self.skip_if_no_wallet()

    def setup_network(self, split=False):
        self.setup_nodes()

    def create_taproot_utxo(self):
        # modify the transaction to add one output that should spend previous taproot
        # Create a taproot prevout
        addr = self.nodes[0].getnewaddress()

        sec = generate_privkey()
        pub = compute_xonly_pubkey(sec)[0]
        tap = taproot_construct(pub)
        spk = tap.scriptPubKey
        tweak = tap.tweak

        unconf_addr = self.nodes[0].getaddressinfo(addr)['unconfidential']

        raw_tx = self.nodes[0].createrawtransaction([], [{unconf_addr: 1.2}])
        # edit spk directly, no way to get new address.
        # would need to implement bech32m in python
        tx = FromHex(CTransaction(), raw_tx)
        tx.vout[0].scriptPubKey = spk
        tx.vout[0].nValue = CTxOutValue(12*10**7)
        raw_hex = tx.serialize().hex()

        fund_tx = self.nodes[0].fundrawtransaction(raw_hex, False, )["hex"]
        fund_tx = FromHex(CTransaction(), fund_tx)

        # Createrawtransaction might rearrage txouts
        prev_vout = None
        for i, out in enumerate(fund_tx.vout):
            if spk == out.scriptPubKey:
                prev_vout = i

        tx = self.nodes[0].blindrawtransaction(fund_tx.serialize().hex())
        signed_raw_tx = self.nodes[0].signrawtransactionwithwallet(tx)
        _txid = self.nodes[0].sendrawtransaction(signed_raw_tx['hex'])
        tx = FromHex(CTransaction(), signed_raw_tx['hex'])
        tx.rehash()
        self.nodes[0].generate(1)
        last_blk = self.nodes[0].getblock(self.nodes[0].getbestblockhash())
        assert(tx.hash in last_blk['tx'])

        return tx, prev_vout, spk, sec, pub, tweak

    def pegin_test(self, sighash_ty):

        # Peg-in prep:
        # Hack: since we're not validating peg-ins in parent chain, just make
        # both the funding and claim tx on same chain (printing money)
        fund_info = self.nodes[0].getpeginaddress()
        peg_id = self.nodes[0].sendtoaddress(fund_info["mainchain_address"], 1)
        raw_peg_tx = self.nodes[0].gettransaction(peg_id)["hex"]
        peg_txid = self.nodes[0].sendrawtransaction(raw_peg_tx)
        self.nodes[0].generate(101)
        peg_prf = self.nodes[0].gettxoutproof([peg_txid])
        claim_script = fund_info["claim_script"]

        # Create a pegin transaction
        # We have to manually supply claim script, otherwise the wallet will pick
        raw_claim = self.nodes[0].createrawpegin(raw_peg_tx, peg_prf, claim_script)
        raw_claim = FromHex(CTransaction(), raw_claim['hex'])

        # Create a taproot utxo
        tx, prev_vout, spk, sec, pub, tweak = self.create_taproot_utxo()

        # Spend the pegin and taproot tx together
        raw_claim.vin.append(CTxIn(COutPoint(tx.sha256, prev_vout)))
        raw_claim.vout.append(CTxOut(nValue = CTxOutValue(12 * 10**7), scriptPubKey = spk)) # send back to self

        signed = self.nodes[0].signrawtransactionwithwallet(raw_claim.serialize().hex())
        raw_claim = FromHex(CTransaction(), signed['hex'])
        genesis_hash = uint256_from_str(bytes.fromhex(self.nodes[0].getblockhash(0))[::-1])
        peg_utxo = CTxOut()
        peg_utxo.from_pegin_witness_data(raw_claim.wit.vtxinwit[0].peginWitness)
        msg = TaprootSignatureHash(raw_claim, [peg_utxo, tx.vout[prev_vout]], sighash_ty, genesis_hash, 1)

        # compute the tweak
        tweak_sk = tweak_add_privkey(sec, tweak)
        sig = sign_schnorr(tweak_sk, msg)
        raw_claim.wit.vtxinwit[1].scriptWitness.stack = [taproot_pad_sighash_ty(sig, sighash_ty)]
        pub_tweak = tweak_add_pubkey(pub, tweak)[0]
        assert(verify_schnorr(pub_tweak, sig, msg))
        # Since we add in/outputs the min feerate is no longer maintained.
        self.nodes[0].sendrawtransaction(hexstring = raw_claim.serialize().hex())
        self.nodes[0].generate(1)
        last_blk = self.nodes[0].getblock(self.nodes[0].getbestblockhash())
        raw_claim.rehash()
        assert(raw_claim.hash in last_blk['tx'])

    def issuance_test(self, sighash_ty):
        tx, prev_vout, spk, sec, pub, tweak = self.create_taproot_utxo()

        blind_addr = self.nodes[0].getnewaddress()
        nonblind_addr = self.nodes[0].validateaddress(blind_addr)['unconfidential']
        raw_tx = self.nodes[0].createrawtransaction([], {nonblind_addr: 1})
        raw_tx = FromHex(CTransaction(), raw_tx)

        # Need to taproot outputs later because fundrawtransaction cannot estimate fees
        # prev out has value 1.2 btc
        in_total = tx.vout[prev_vout].nValue.getAmount()
        fees = 100
        raw_tx.vin.append(CTxIn(COutPoint(tx.sha256, prev_vout)))
        raw_tx.vout.append(CTxOut(nValue = CTxOutValue(in_total - fees - 10**8), scriptPubKey = spk)) # send back to self
        raw_tx.vout.append(CTxOut(nValue = CTxOutValue(fees)))

        # issued_tx = raw_tx.serialize().hex()
        blind_addr = self.nodes[0].getnewaddress()
        issue_addr = self.nodes[0].validateaddress(blind_addr)['unconfidential']
        issued_tx = self.nodes[0].rawissueasset(raw_tx.serialize().hex(), [{"asset_amount":2, "asset_address":issue_addr, "blind":False}])[0]["hex"]
        # blind_tx = self.nodes[0].blindrawtransaction(issued_tx) # This is a no-op
        genesis_hash = uint256_from_str(bytes.fromhex(self.nodes[0].getblockhash(0))[::-1])
        issued_tx = FromHex(CTransaction(), issued_tx)
        issued_tx.wit.vtxoutwit = [CTxOutWitness()] * len(issued_tx.vout)
        issued_tx.wit.vtxinwit = [CTxInWitness()] * len(issued_tx.vin)
        msg = TaprootSignatureHash(issued_tx, [tx.vout[prev_vout]], sighash_ty, genesis_hash, 0)

        # compute the tweak
        tweak_sk = tweak_add_privkey(sec, tweak)
        sig = sign_schnorr(tweak_sk, msg)
        issued_tx.wit.vtxinwit[0].scriptWitness.stack = [taproot_pad_sighash_ty(sig, sighash_ty)]
        pub_tweak = tweak_add_pubkey(pub, tweak)[0]
        assert(verify_schnorr(pub_tweak, sig, msg))
        # Since we add in/outputs the min feerate is no longer maintained.
        self.nodes[0].sendrawtransaction(hexstring = issued_tx.serialize().hex())
        self.nodes[0].generate(1)
        last_blk = self.nodes[0].getblock(self.nodes[0].getbestblockhash())
        issued_tx.rehash()
        assert(issued_tx.hash in last_blk['tx'])

    def run_test(self):
        self.nodes[0].generate(101)
        self.wait_until(lambda: self.nodes[0].getblockcount() == 101, timeout=5)
        self.log.info("Testing sighash taproot pegins")
        # Note that this does not test deposit to taproot pegin addresses
        # because there is no support for taproot pegins in rpc. The current rpc assumes
        # to shwsh tweaked address
        for sighash_ty in VALID_SIGHASHES_TAPROOT:
            self.pegin_test(sighash_ty)
        self.log.info("Testing sighash taproot issuances")
        for sighash_ty in VALID_SIGHASHES_TAPROOT:
            self.issuance_test(sighash_ty)

if __name__ == '__main__':
    TapHashPeginTest().main()
<commit_msg>test: update `createrawtransaction` call in feature_taphash_pegins_issuances.py<commit_after>#!/usr/bin/env python3
# Copyright (c) 2014-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

#
# Test for taproot sighash algorithm with pegins and issuances

from test_framework.key import ECKey, ECPubKey, compute_xonly_pubkey, generate_privkey, sign_schnorr, tweak_add_privkey, tweak_add_pubkey, verify_schnorr
from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, CTxOutValue, CTxOutWitness, FromHex, uint256_from_str
from test_framework.test_framework import BitcoinTestFramework, SkipTest
from test_framework.script import TaprootSignatureHash, taproot_construct, taproot_pad_sighash_ty, SIGHASH_DEFAULT, SIGHASH_ALL, SIGHASH_NONE, SIGHASH_SINGLE, SIGHASH_ANYONECANPAY

VALID_SIGHASHES_ECDSA = [
    SIGHASH_ALL,
    SIGHASH_NONE,
    SIGHASH_SINGLE,
    SIGHASH_ANYONECANPAY + SIGHASH_ALL,
    SIGHASH_ANYONECANPAY + SIGHASH_NONE,
    SIGHASH_ANYONECANPAY + SIGHASH_SINGLE
]

VALID_SIGHASHES_TAPROOT = [SIGHASH_DEFAULT] + VALID_SIGHASHES_ECDSA

class TapHashPeginTest(BitcoinTestFramework):

    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 1
        self.extra_args = [
            ["-initialfreecoins=2100000000000000",
             "-anyonecanspendaremine=1",
            "-blindedaddresses=1",
            "-validatepegin=0",
            "-con_parent_chain_signblockscript=51",
            "-parentscriptprefix=75",
            "-parent_bech32_hrp=ert",
            "-minrelaytxfee=0",
            "-maxtxfee=100.0",
        ]]

    def skip_test_if_missing_module(self):
        self.skip_if_no_wallet()

    def setup_network(self, split=False):
        self.setup_nodes()

    def create_taproot_utxo(self):
        # modify the transaction to add one output that should spend previous taproot
        # Create a taproot prevout
        addr = self.nodes[0].getnewaddress()

        sec = generate_privkey()
        pub = compute_xonly_pubkey(sec)[0]
        tap = taproot_construct(pub)
        spk = tap.scriptPubKey
        tweak = tap.tweak

        unconf_addr = self.nodes[0].getaddressinfo(addr)['unconfidential']

        raw_tx = self.nodes[0].createrawtransaction([], [{unconf_addr: 1.2}])
        # edit spk directly, no way to get new address.
        # would need to implement bech32m in python
        tx = FromHex(CTransaction(), raw_tx)
        tx.vout[0].scriptPubKey = spk
        tx.vout[0].nValue = CTxOutValue(12*10**7)
        raw_hex = tx.serialize().hex()

        fund_tx = self.nodes[0].fundrawtransaction(raw_hex, False, )["hex"]
        fund_tx = FromHex(CTransaction(), fund_tx)

        # Createrawtransaction might rearrage txouts
        prev_vout = None
        for i, out in enumerate(fund_tx.vout):
            if spk == out.scriptPubKey:
                prev_vout = i

        tx = self.nodes[0].blindrawtransaction(fund_tx.serialize().hex())
        signed_raw_tx = self.nodes[0].signrawtransactionwithwallet(tx)
        _txid = self.nodes[0].sendrawtransaction(signed_raw_tx['hex'])
        tx = FromHex(CTransaction(), signed_raw_tx['hex'])
        tx.rehash()
        self.nodes[0].generate(1)
        last_blk = self.nodes[0].getblock(self.nodes[0].getbestblockhash())
        assert(tx.hash in last_blk['tx'])

        return tx, prev_vout, spk, sec, pub, tweak

    def pegin_test(self, sighash_ty):

        # Peg-in prep:
        # Hack: since we're not validating peg-ins in parent chain, just make
        # both the funding and claim tx on same chain (printing money)
        fund_info = self.nodes[0].getpeginaddress()
        peg_id = self.nodes[0].sendtoaddress(fund_info["mainchain_address"], 1)
        raw_peg_tx = self.nodes[0].gettransaction(peg_id)["hex"]
        peg_txid = self.nodes[0].sendrawtransaction(raw_peg_tx)
        self.nodes[0].generate(101)
        peg_prf = self.nodes[0].gettxoutproof([peg_txid])
        claim_script = fund_info["claim_script"]

        # Create a pegin transaction
        # We have to manually supply claim script, otherwise the wallet will pick
        raw_claim = self.nodes[0].createrawpegin(raw_peg_tx, peg_prf, claim_script)
        raw_claim = FromHex(CTransaction(), raw_claim['hex'])

        # Create a taproot utxo
        tx, prev_vout, spk, sec, pub, tweak = self.create_taproot_utxo()

        # Spend the pegin and taproot tx together
        raw_claim.vin.append(CTxIn(COutPoint(tx.sha256, prev_vout)))
        raw_claim.vout.append(CTxOut(nValue = CTxOutValue(12 * 10**7), scriptPubKey = spk)) # send back to self

        signed = self.nodes[0].signrawtransactionwithwallet(raw_claim.serialize().hex())
        raw_claim = FromHex(CTransaction(), signed['hex'])
        genesis_hash = uint256_from_str(bytes.fromhex(self.nodes[0].getblockhash(0))[::-1])
        peg_utxo = CTxOut()
        peg_utxo.from_pegin_witness_data(raw_claim.wit.vtxinwit[0].peginWitness)
        msg = TaprootSignatureHash(raw_claim, [peg_utxo, tx.vout[prev_vout]], sighash_ty, genesis_hash, 1)

        # compute the tweak
        tweak_sk = tweak_add_privkey(sec, tweak)
        sig = sign_schnorr(tweak_sk, msg)
        raw_claim.wit.vtxinwit[1].scriptWitness.stack = [taproot_pad_sighash_ty(sig, sighash_ty)]
        pub_tweak = tweak_add_pubkey(pub, tweak)[0]
        assert(verify_schnorr(pub_tweak, sig, msg))
        # Since we add in/outputs the min feerate is no longer maintained.
        self.nodes[0].sendrawtransaction(hexstring = raw_claim.serialize().hex())
        self.nodes[0].generate(1)
        last_blk = self.nodes[0].getblock(self.nodes[0].getbestblockhash())
        raw_claim.rehash()
        assert(raw_claim.hash in last_blk['tx'])

    def issuance_test(self, sighash_ty):
        tx, prev_vout, spk, sec, pub, tweak = self.create_taproot_utxo()

        blind_addr = self.nodes[0].getnewaddress()
        nonblind_addr = self.nodes[0].validateaddress(blind_addr)['unconfidential']
        raw_tx = self.nodes[0].createrawtransaction([], [{nonblind_addr: 1}])
        raw_tx = FromHex(CTransaction(), raw_tx)

        # Need to taproot outputs later because fundrawtransaction cannot estimate fees
        # prev out has value 1.2 btc
        in_total = tx.vout[prev_vout].nValue.getAmount()
        fees = 100
        raw_tx.vin.append(CTxIn(COutPoint(tx.sha256, prev_vout)))
        raw_tx.vout.append(CTxOut(nValue = CTxOutValue(in_total - fees - 10**8), scriptPubKey = spk)) # send back to self
        raw_tx.vout.append(CTxOut(nValue = CTxOutValue(fees)))

        # issued_tx = raw_tx.serialize().hex()
        blind_addr = self.nodes[0].getnewaddress()
        issue_addr = self.nodes[0].validateaddress(blind_addr)['unconfidential']
        issued_tx = self.nodes[0].rawissueasset(raw_tx.serialize().hex(), [{"asset_amount":2, "asset_address":issue_addr, "blind":False}])[0]["hex"]
        # blind_tx = self.nodes[0].blindrawtransaction(issued_tx) # This is a no-op
        genesis_hash = uint256_from_str(bytes.fromhex(self.nodes[0].getblockhash(0))[::-1])
        issued_tx = FromHex(CTransaction(), issued_tx)
        issued_tx.wit.vtxoutwit = [CTxOutWitness()] * len(issued_tx.vout)
        issued_tx.wit.vtxinwit = [CTxInWitness()] * len(issued_tx.vin)
        msg = TaprootSignatureHash(issued_tx, [tx.vout[prev_vout]], sighash_ty, genesis_hash, 0)

        # compute the tweak
        tweak_sk = tweak_add_privkey(sec, tweak)
        sig = sign_schnorr(tweak_sk, msg)
        issued_tx.wit.vtxinwit[0].scriptWitness.stack = [taproot_pad_sighash_ty(sig, sighash_ty)]
        pub_tweak = tweak_add_pubkey(pub, tweak)[0]
        assert(verify_schnorr(pub_tweak, sig, msg))
        # Since we add in/outputs the min feerate is no longer maintained.
        self.nodes[0].sendrawtransaction(hexstring = issued_tx.serialize().hex())
        self.nodes[0].generate(1)
        last_blk = self.nodes[0].getblock(self.nodes[0].getbestblockhash())
        issued_tx.rehash()
        assert(issued_tx.hash in last_blk['tx'])

    def run_test(self):
        self.nodes[0].generate(101)
        self.wait_until(lambda: self.nodes[0].getblockcount() == 101, timeout=5)
        self.log.info("Testing sighash taproot pegins")
        # Note that this does not test deposit to taproot pegin addresses
        # because there is no support for taproot pegins in rpc. The current rpc assumes
        # to shwsh tweaked address
        for sighash_ty in VALID_SIGHASHES_TAPROOT:
            self.pegin_test(sighash_ty)
        self.log.info("Testing sighash taproot issuances")
        for sighash_ty in VALID_SIGHASHES_TAPROOT:
            self.issuance_test(sighash_ty)

if __name__ == '__main__':
    TapHashPeginTest().main()
<|endoftext|>


<commit_before>from optparse import make_option
import urllib
import xml.etree.ElementTree as etree

from django.core.management.base import BaseCommand, CommandError

from speeches.models import Section, Speech, Speaker
from instances.models import Instance

PLAYS = {
    "all_well.xml": "All's Well That Ends Well",
    "as_you.xml": "As You Like It",
    "a_and_c.xml": "Antony and Cleopatra",
    "com_err.xml": "A Comedy of Errors",
    "coriolan.xml": "Coriolanus",
    "cymbelin.xml": "Cymbeline",
    "dream.xml": "A Midsummer Night's Dream",
    "hamlet.xml": "Hamlet",
    "hen_iv_1.xml": "Henry IV, Part I",
    "hen_iv_2.xml": "Henry IV, Part II",
    "hen_v.xml": "Henry V",
    "hen_viii.xml": "Henry VIII",
    "hen_vi_1.xml": "Henry VI, Part 1",
    "hen_vi_2.xml": "Henry VI, Part 2",
    "hen_vi_3.xml": "Henry VI, Part 3",
    "john.xml": "The Life and Death of King John",
    "j_caesar.xml": "Julius Caesar",
    "lear.xml": "King Lear",
    "lll.xml": "Love's Labor's Lost",
    "macbeth.xml": "Macbeth",
    "merchant.xml": "The Merchant of Venice",
    "much_ado.xml": "Much Ado About Nothing",
    "m_for_m.xml": "Measure for Measure",
    "m_wives.xml": "The Merry Wives of Windsor",
    "othello.xml": "Othello",
    "pericles.xml": "Pericles",
    "rich_ii.xml": "Richard II",
    "rich_iii.xml": "Richard III",
    "r_and_j.xml": "Romeo and Juliet",
    "taming.xml": "The Taming of the Shrew",
    "tempest.xml": "The Tempest",
    "timon.xml": "Timon of Athens",
    "titus.xml": "Titus Andronicus",
    "troilus.xml": "Troilus and Cressida",
    "two_gent.xml": "Two Gentlemen of Verona",
    "t_night.xml": "Twelfth Night",
    "win_tale.xml": "A Winter's Tale",
}

class Command(BaseCommand):
    args = '<play>'
    help = 'Import a Shakespeare play into a SayIt instance'
    option_list = BaseCommand.option_list + (
        make_option('--commit', action='store_true', help='Whether to commit to the database or not'),
        make_option('--instance', action='store', default='shakespeare', help='Label of instance to add data to'),
        make_option('--list', action='store_true', help='List the plays available'),
    )

    def make(self, cls, **kwargs):
        s = cls(instance=self.instance, **kwargs)
        if self.commit:
            s.save()
        elif s.title:
            print s.title
        return s

    def handle(self, *args, **options):
        if options['list'] or len(args) != 1:
            self.stdout.write('Plays:\n')
            for play in sorted(PLAYS.values()):
                self.stdout.write('* %s\n' % play)
            if not options['list']:
                raise CommandError("Please specify a play")
            return

        play = args[0]
        file = None
        for f, p in PLAYS.items():
            if play == p:
                file = f
                break

        if not file:
            raise CommandError("No matching play found")

        try:
            self.instance = Instance.objects.get(label=options['instance'])
        except:
            raise CommandError("Instance specified not found")

        self.commit = options['commit']

        xml = urllib.urlopen('http://www.ibiblio.org/xml/examples/shakespeare/%s' % file).read()
        play_xml = etree.fromstring(xml)
        play_section = self.make(Section, title=play)

        for act in play_xml:
            if act.tag != 'ACT': continue
            act_title = act[0].text
            act_section = self.make(Section, title=act_title, parent=play_section)
            scenes = act[1:]
            for scene in scenes:
                scene_title = scene[0].text
                scene_section = self.make(Section, title=scene_title, parent=act_section)
                speeches_xml = scene[1:]
                speeches = []
                for sp in speeches_xml:
                    if sp.tag == 'STAGEDIR' or sp.tag == 'SUBHEAD' or sp.tag == 'SUBTITLE':
                        self.make(Speech, section=scene_section, text='%s' % sp.text)
                        continue

                    if  not sp[0].text:
                        speaker = None
                    elif self.commit:
                        name = sp[0].text.replace('[', '').replace(']', '')
                        speaker, _ = Speaker.objects.get_or_create(name=name, instance=self.instance)
                    else:
                        speaker = Speaker(name=sp[0].text, instance=self.instance)

                    text = ""
                    lines = sp[1:]
                    for line in lines:
                        if len(line):
                            text += '%s' % line[0].text
                            if line[0].tail:
                                text += ' %s' % line[0].tail.strip()
                            text += '\n'
                        elif line.tag == 'LINE':
                            text += '%s\n' % line.text
                        elif line.tag == 'STAGEDIR':
                            text += '%s\n' % line.text

                    self.make(Speech, speaker=speaker, section=scene_section, text=text)

<commit_msg>[Shakespeare] Create new speakers each time.<commit_after>from optparse import make_option
import urllib
import xml.etree.ElementTree as etree

from django.core.management.base import BaseCommand, CommandError

from speeches.models import Section, Speech, Speaker
from instances.models import Instance

PLAYS = {
    "all_well.xml": "All's Well That Ends Well",
    "as_you.xml": "As You Like It",
    "a_and_c.xml": "Antony and Cleopatra",
    "com_err.xml": "A Comedy of Errors",
    "coriolan.xml": "Coriolanus",
    "cymbelin.xml": "Cymbeline",
    "dream.xml": "A Midsummer Night's Dream",
    "hamlet.xml": "Hamlet",
    "hen_iv_1.xml": "Henry IV, Part I",
    "hen_iv_2.xml": "Henry IV, Part II",
    "hen_v.xml": "Henry V",
    "hen_viii.xml": "Henry VIII",
    "hen_vi_1.xml": "Henry VI, Part 1",
    "hen_vi_2.xml": "Henry VI, Part 2",
    "hen_vi_3.xml": "Henry VI, Part 3",
    "john.xml": "The Life and Death of King John",
    "j_caesar.xml": "Julius Caesar",
    "lear.xml": "King Lear",
    "lll.xml": "Love's Labor's Lost",
    "macbeth.xml": "Macbeth",
    "merchant.xml": "The Merchant of Venice",
    "much_ado.xml": "Much Ado About Nothing",
    "m_for_m.xml": "Measure for Measure",
    "m_wives.xml": "The Merry Wives of Windsor",
    "othello.xml": "Othello",
    "pericles.xml": "Pericles",
    "rich_ii.xml": "Richard II",
    "rich_iii.xml": "Richard III",
    "r_and_j.xml": "Romeo and Juliet",
    "taming.xml": "The Taming of the Shrew",
    "tempest.xml": "The Tempest",
    "timon.xml": "Timon of Athens",
    "titus.xml": "Titus Andronicus",
    "troilus.xml": "Troilus and Cressida",
    "two_gent.xml": "Two Gentlemen of Verona",
    "t_night.xml": "Twelfth Night",
    "win_tale.xml": "A Winter's Tale",
}

class Command(BaseCommand):
    args = '<play>'
    help = 'Import a Shakespeare play into a SayIt instance'
    option_list = BaseCommand.option_list + (
        make_option('--commit', action='store_true', help='Whether to commit to the database or not'),
        make_option('--instance', action='store', default='shakespeare', help='Label of instance to add data to'),
        make_option('--list', action='store_true', help='List the plays available'),
    )

    def make(self, cls, **kwargs):
        s = cls(instance=self.instance, **kwargs)
        if self.commit:
            s.save()
        elif s.title:
            print s.title
        return s

    def handle(self, *args, **options):
        if options['list'] or len(args) != 1:
            self.stdout.write('Plays:\n')
            for play in sorted(PLAYS.values()):
                self.stdout.write('* %s\n' % play)
            if not options['list']:
                raise CommandError("Please specify a play")
            return

        play = args[0]
        file = None
        for f, p in PLAYS.items():
            if play == p:
                file = f
                break

        if not file:
            raise CommandError("No matching play found")

        try:
            self.instance = Instance.objects.get(label=options['instance'])
        except:
            raise CommandError("Instance specified not found")

        self.commit = options['commit']

        xml = urllib.urlopen('http://www.ibiblio.org/xml/examples/shakespeare/%s' % file).read()
        play_xml = etree.fromstring(xml)
        play_section = self.make(Section, title=play)

        speakers = {}
        for act in play_xml:
            if act.tag != 'ACT': continue
            act_title = act[0].text
            act_section = self.make(Section, title=act_title, parent=play_section)
            scenes = act[1:]
            for scene in scenes:
                scene_title = scene[0].text
                scene_section = self.make(Section, title=scene_title, parent=act_section)
                speeches_xml = scene[1:]
                speeches = []
                for sp in speeches_xml:
                    if sp.tag == 'STAGEDIR' or sp.tag == 'SUBHEAD' or sp.tag == 'SUBTITLE':
                        self.make(Speech, section=scene_section, text='%s' % sp.text)
                        continue

                    if  not sp[0].text:
                        speaker = None
                    elif self.commit:
                        name = sp[0].text.replace('[', '').replace(']', '')
                        if name in speakers:
                            speaker = speakers[name]
                        else:
                            speaker = Speaker.objects.create(name=name, instance=self.instance)
                            speakers[name] = speaker
                    else:
                        speaker = Speaker(name=sp[0].text, instance=self.instance)

                    text = ""
                    lines = sp[1:]
                    for line in lines:
                        if len(line):
                            text += '%s' % line[0].text
                            if line[0].tail:
                                text += ' %s' % line[0].tail.strip()
                            text += '\n'
                        elif line.tag == 'LINE':
                            text += '%s\n' % line.text
                        elif line.tag == 'STAGEDIR':
                            text += '%s\n' % line.text

                    self.make(Speech, speaker=speaker, section=scene_section, text=text)

<|endoftext|>


<commit_before>import pytest

__author__ = 'thauser'

from pnc_cli import buildconfigurationsets
from pnc_cli import buildconfigurations
from pnc_cli import projects
from pnc_cli import environments
from pnc_cli import utils
from test import testutils
from pnc_cli.swagger_client.apis.buildconfigurationsets_api import BuildconfigurationsetsApi

@pytest.fixture(scope='function', autouse=True)
def get_sets_api():
    global sets_api
    sets_api = BuildconfigurationsetsApi(utils.get_api_client())

@pytest.fixture(scope='module')
def new_set(request):
    set = sets_api.create_new(
        body=buildconfigurationsets._create_build_config_set_object(name=testutils.gen_random_name(),
                                                                    productVersionId=1)).content
    def teardown():
        buildconfigurationsets.delete_build_config_set(id=set.id)
    request.addfinalizer(teardown)
    return set

@pytest.fixture(scope='module')
def new_project():
    project = projects.create_project(name=testutils.gen_random_name() + '_project')
    return project

@pytest.fixture(scope='module')
def new_environment():
    randname = testutils.gen_random_name()
    env = environments.create_environment(name=randname + '_environment', build_type='JAVA', image_id=randname)
    return env

@pytest.fixture(scope='function')
def new_config(request, new_project, new_environment):
    created_bc = buildconfigurations.create_build_configuration(name=testutils.gen_random_name(),
                                                                project=new_project.id,
                                                                environment=new_environment.id,
                                                                build_status="UNKNOWN",
                                                                build_script='mvn clean install',
                                                                product_version_ids=[1],
                                                                scm_repo_url='https://github.com/project-ncl/pnc-simple-test-project.git')
    def teardown():
        buildconfigurations.delete_build_configuration(id=created_bc.id)
    request.addfinalizer(teardown)
    return created_bc

def test_get_all():
    sets = sets_api.get_all().content
    assert sets is not None

def test_create(new_set):
    set_names = [set.name for set in sets_api.get_all(page_size=1000000).content]
    assert new_set.name in set_names

def test_get_specific_no_id():
    testutils.assert_raises_valueerror(sets_api, 'get_specific', id=None)

def test_get_specific_invalid_param():
    testutils.assert_raises_typeerror(sets_api, 'get_specific', id=1)

def test_get_specific(new_set):
    set = sets_api.get_specific(id=new_set.id)
    assert set is not None

def test_update(new_set):
    newname = "newname" + testutils.gen_random_name()
    updated_set = buildconfigurationsets._create_build_config_set_object(name=newname)
    sets_api.update(id=new_set.id, body=updated_set)
    retrieved_set = sets_api.get_specific(id=new_set.id).content
    assert retrieved_set.name == newname

def test_delete_no_id():
    testutils.assert_raises_valueerror(sets_api, 'delete_specific', id=None)

def test_delete_invalid_param():
    testutils.assert_raises_typeerror(sets_api, 'delete_specific', id=1)

def test_delete(new_set):
    sets_api.delete_specific(id=new_set.id)
    response = utils.checked_api_call(sets_api, 'get_specific', id=new_set.id)
    assert response is None

def test_get_configurations_no_id():
    testutils.assert_raises_valueerror(sets_api, 'get_configurations', id=None)

def test_get_configurations_invalid_param():
    testutils.assert_raises_typeerror(sets_api, 'get_configurations', id=1)

def test_get_configurations(new_config, new_set):
    sets_api.add_configuration(id=new_set.id, body=new_config)
    set_configs = sets_api.get_configurations(id=new_set.id)
    assert set_configs is not None

def test_add_configuration(new_config, new_set):
    sets_api.add_configuration(id=new_set.id, body=new_config)
    set_config_ids = [x.id for x in sets_api.get_configurations(id=new_set.id).content]
    assert new_config.id in set_config_ids

def test_remove_configuration_no_set_id():
    testutils.assert_raises_valueerror(sets_api, 'remove_configuration', id=None, config_id=1)

def test_remove_configuration_no_config_id():
    testutils.assert_raises_valueerror(sets_api, 'remove_configuration', id=1, config_id=None)

def test_remove_configuration_invalid_param():
    testutils.assert_raises_typeerror(sets_api, 'remove_configuration', id=1, config_id=1)

def test_remove_configuration(new_config, new_set):
    sets_api.add_configuration(id=new_set.id, body=new_config)
    set_config_ids = [x.id for x in sets_api.get_configurations(id=new_set.id).content]
    # config is added successfully
    assert new_config.id in set_config_ids
    # remove config
    sets_api.remove_configuration(id=new_set.id, config_id=new_config.id)
    #removed successfully
    assert sets_api.get_configurations(id=new_set.id).content is None

def test_get_build_records_no_id():
    testutils.assert_raises_valueerror(sets_api, 'get_build_records', id=None)

def test_get_build_records_invalid_param():
    testutils.assert_raises_typeerror(sets_api, 'get_build_records', id=1)

def test_get_build_records(new_set):
    response = sets_api.get_build_records(id=new_set.id)
    assert response

def test_build_no_id():
    testutils.assert_raises_valueerror(sets_api, 'build', id=None)

def test_build_invalid_param():
    testutils.assert_raises_typeerror(sets_api, 'build', id=1)

def test_build(request, new_set, new_project, new_environment):
    config_one = new_config(request, new_project, new_environment)
    config_two = new_config(request, new_project, new_environment)
    config_three = new_config(request, new_project, new_environment)
    sets_api.add_configuration(id=new_set.id, body=config_one)
    sets_api.add_configuration(id=new_set.id, body=config_two)
    sets_api.add_configuration(id=new_set.id, body=config_three)
    build_records = sets_api.build(id=new_set.id, rebuild_all=False)
    assert build_records is not None

def test_get_all_build_config_set_records_no_id():
    testutils.assert_raises_valueerror(sets_api, 'get_all_build_config_set_records', id=None)

def test_get_all_build_config_set_records_invalid_param():
    testutils.assert_raises_typeerror(sets_api, 'get_all_build_config_set_records', id=1)

def test_get_all_build_config_set_records(new_set):
    response = sets_api.get_all_build_config_set_records(id=new_set.id)
    assert response is not None

<commit_msg>new_set on per/function scope<commit_after>import pytest

__author__ = 'thauser'

from pnc_cli import buildconfigurationsets
from pnc_cli import buildconfigurations
from pnc_cli import projects
from pnc_cli import environments
from pnc_cli import utils
from test import testutils
from pnc_cli.swagger_client.apis.buildconfigurationsets_api import BuildconfigurationsetsApi

@pytest.fixture(scope='function', autouse=True)
def get_sets_api():
    global sets_api
    sets_api = BuildconfigurationsetsApi(utils.get_api_client())

@pytest.fixture(scope='function')
def new_set(request):
    set = sets_api.create_new(
        body=buildconfigurationsets._create_build_config_set_object(name=testutils.gen_random_name(),
                                                                    productVersionId=1)).content
    def teardown():
        buildconfigurationsets.delete_build_config_set(id=set.id)
    request.addfinalizer(teardown)
    return set

@pytest.fixture(scope='module')
def new_project():
    project = projects.create_project(name=testutils.gen_random_name() + '_project')
    return project

@pytest.fixture(scope='module')
def new_environment():
    randname = testutils.gen_random_name()
    env = environments.create_environment(name=randname + '_environment', build_type='JAVA', image_id=randname)
    return env

@pytest.fixture(scope='function')
def new_config(request, new_project, new_environment):
    created_bc = buildconfigurations.create_build_configuration(name=testutils.gen_random_name(),
                                                                project=new_project.id,
                                                                environment=new_environment.id,
                                                                build_status="UNKNOWN",
                                                                build_script='mvn clean install',
                                                                product_version_ids=[1],
                                                                scm_repo_url='https://github.com/project-ncl/pnc-simple-test-project.git')
    def teardown():
        buildconfigurations.delete_build_configuration(id=created_bc.id)
    request.addfinalizer(teardown)
    return created_bc

def test_get_all():
    sets = sets_api.get_all().content
    assert sets is not None

def test_create(new_set):
    set_names = [set.name for set in sets_api.get_all(page_size=1000000).content]
    assert new_set.name in set_names

def test_get_specific_no_id():
    testutils.assert_raises_valueerror(sets_api, 'get_specific', id=None)

def test_get_specific_invalid_param():
    testutils.assert_raises_typeerror(sets_api, 'get_specific', id=1)

def test_get_specific(new_set):
    set = sets_api.get_specific(id=new_set.id)
    assert set is not None

def test_update(new_set):
    newname = "newname" + testutils.gen_random_name()
    updated_set = buildconfigurationsets._create_build_config_set_object(name=newname)
    sets_api.update(id=new_set.id, body=updated_set)
    retrieved_set = sets_api.get_specific(id=new_set.id).content
    assert retrieved_set.name == newname

def test_delete_no_id():
    testutils.assert_raises_valueerror(sets_api, 'delete_specific', id=None)

def test_delete_invalid_param():
    testutils.assert_raises_typeerror(sets_api, 'delete_specific', id=1)

def test_delete(new_set):
    sets_api.delete_specific(id=new_set.id)
    response = utils.checked_api_call(sets_api, 'get_specific', id=new_set.id)
    assert response is None

def test_get_configurations_no_id():
    testutils.assert_raises_valueerror(sets_api, 'get_configurations', id=None)

def test_get_configurations_invalid_param():
    testutils.assert_raises_typeerror(sets_api, 'get_configurations', id=1)

def test_get_configurations(new_config, new_set):
    sets_api.add_configuration(id=new_set.id, body=new_config)
    set_configs = sets_api.get_configurations(id=new_set.id)
    assert set_configs is not None

def test_add_configuration(new_config, new_set):
    sets_api.add_configuration(id=new_set.id, body=new_config)
    set_config_ids = [x.id for x in sets_api.get_configurations(id=new_set.id).content]
    assert new_config.id in set_config_ids

def test_remove_configuration_no_set_id():
    testutils.assert_raises_valueerror(sets_api, 'remove_configuration', id=None, config_id=1)

def test_remove_configuration_no_config_id():
    testutils.assert_raises_valueerror(sets_api, 'remove_configuration', id=1, config_id=None)

def test_remove_configuration_invalid_param():
    testutils.assert_raises_typeerror(sets_api, 'remove_configuration', id=1, config_id=1)

def test_remove_configuration(new_config, new_set):
    sets_api.add_configuration(id=new_set.id, body=new_config)
    set_config_ids = [x.id for x in sets_api.get_configurations(id=new_set.id).content]
    # config is added successfully
    assert new_config.id in set_config_ids
    # remove config
    sets_api.remove_configuration(id=new_set.id, config_id=new_config.id)
    #removed successfully
    assert sets_api.get_configurations(id=new_set.id).content is None

def test_get_build_records_no_id():
    testutils.assert_raises_valueerror(sets_api, 'get_build_records', id=None)

def test_get_build_records_invalid_param():
    testutils.assert_raises_typeerror(sets_api, 'get_build_records', id=1)

def test_get_build_records(new_set):
    response = sets_api.get_build_records(id=new_set.id)
    assert response

def test_build_no_id():
    testutils.assert_raises_valueerror(sets_api, 'build', id=None)

def test_build_invalid_param():
    testutils.assert_raises_typeerror(sets_api, 'build', id=1)

def test_build(request, new_set, new_project, new_environment):
    config_one = new_config(request, new_project, new_environment)
    config_two = new_config(request, new_project, new_environment)
    config_three = new_config(request, new_project, new_environment)
    sets_api.add_configuration(id=new_set.id, body=config_one)
    sets_api.add_configuration(id=new_set.id, body=config_two)
    sets_api.add_configuration(id=new_set.id, body=config_three)
    build_records = sets_api.build(id=new_set.id, rebuild_all=False)
    assert build_records is not None

def test_get_all_build_config_set_records_no_id():
    testutils.assert_raises_valueerror(sets_api, 'get_all_build_config_set_records', id=None)

def test_get_all_build_config_set_records_invalid_param():
    testutils.assert_raises_typeerror(sets_api, 'get_all_build_config_set_records', id=1)

def test_get_all_build_config_set_records(new_set):
    response = sets_api.get_all_build_config_set_records(id=new_set.id)
    assert response is not None

<|endoftext|>


<commit_before>from __future__ import unicode_literals

import frappe

def execute():
	frappe.reload_doc('core', 'doctype', 'communication')

	communications = frappe.db.sql("""
			SELECT
				`tabCommunication`.name, `tabCommunication`.creation, `tabCommunication`.modified,
				`tabCommunication`.modified_by,`tabCommunication`.timeline_doctype, `tabCommunication`.timeline_name,
				`tabCommunication`.link_doctype, `tabCommunication`.link_name
			FROM `tabCommunication`
			WHERE `tabCommunication`.communication_medium='Email'
		""", as_dict=True)

	for count, communication in enumerate(communications):
		counter = 1
		if communication.timeline_doctype and communication.timeline_name:
			values = [
				counter, frappe.generate_hash(length=10), 'timeline_links', 'Communication', communication.name,
				communication.timeline_doctype, communication.timeline_name, communication.creation, communication.modified,
				communication.modified_by
			]
			execute_query(values)
			counter += 1

		if communication.link_doctype and communication.link_name:
			values = [
				counter, frappe.generate_hash(length=10), 'timeline_links', 'Communication', communication.name,
				communication.link_doctype, communication.link_name, communication.creation, communication.modified,
				communication.modified_by
			]
			execute_query(values)

def execute_query(values):
	try:
		frappe.db.sql("""
			INSERT INTO `tabDynamic Link`
				(`idx`, `name`, `parentfield`, `parenttype`, `parent`, `link_doctype`, `link_name`, `creation`,
				`modified`, `modified_by`)
			VALUES ({0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9})
		""".format(values[0], values[1], values[2], values[3], values[4], values[5], values[6], values[7], values[8], values[9]))
	except Exception as e:
		values[1] = frappe.generate_hash(length=10)
		execute_query(values)
<commit_msg>fix(communication): Make optimizations to the patch<commit_after>from __future__ import unicode_literals

import frappe

def execute():
	frappe.reload_doc('core', 'doctype', 'communication')

	communications = frappe.db.sql("""
		SELECT
			`tabCommunication`.name, `tabCommunication`.creation, `tabCommunication`.modified,
			`tabCommunication`.modified_by,`tabCommunication`.timeline_doctype, `tabCommunication`.timeline_name,
			`tabCommunication`.link_doctype, `tabCommunication`.link_name
		FROM `tabCommunication`
		WHERE `tabCommunication`.communication_medium='Email'
	""", as_dict=True)

	name = 10000000000001234567
	values = []

	for count, communication in enumerate(communications):
		counter = 1
		if communication.timeline_doctype and communication.timeline_name:
			name += 1
			values.append("""({0}, "{1}", "timeline_links", "Communication", "{2}", "{3}", "{4}", "{5}", "{6}", "{7}")""".format(
				counter, str(name), communication.name, communication.timeline_doctype,
				communication.timeline_name, communication.creation, communication.modified, communication.modified_by
			))
			counter += 1
		if communication.link_doctype and communication.link_name:
			name += 1
			values.append("""({0}, "{1}", "timeline_links", "Communication", "{2}", "{3}", "{4}", "{5}", "{6}", "{7}")""".format(
				counter, str(name), communication.name, communication.link_doctype,
				communication.link_name, communication.creation, communication.modified, communication.modified_by
			))

		if count % 10000 == 0 or count == len(communications) - 1:
			frappe.db.sql("""
				INSERT INTO `tabDynamic Link`
					(`idx`, `name`, `parentfield`, `parenttype`, `parent`, `link_doctype`, `link_name`, `creation`,
					`modified`, `modified_by`)
				VALUES {0}
			""".format(", ".join([d for d in values])))

			values = []
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Mar 14 09:34:17 2017

@author: lamahamadeh
"""

'''
Case Study about an introduction to kNN classifiction
'''

#Finding the Euclidean Distance Between Two Points
#---------------------------------------------------

import numpy as np

def distance(p1,p2):
    "find the distance betweeen points p1 and p2"
    return np.sqrt(np.sum(np.power(p2 - p1, 2)))

p1 = np.array([1,1])
p2 = np.array([4,4])

print (distance(p1,p2))

#------------------------------------------------------------------------------

#Majority Vote
#-------------

import random

def majority_vote(votes):
    "Return the most common element in votes"
    vote_counts = {}
    for vote in votes:
        if vote in vote_counts:
            vote_counts[vote] += 1
        else:
            vote_counts[vote] = 1
    #items method
    winners = []
    max_counts = max(vote_counts.values()) #ask Python about the frequency the most appearing value has
    for vote, count in vote_counts.items(): #this returns a tuple of (vote,count)
        if count == max_counts:
            winners.append(vote)

    return random.choice(winners) #in case of a tie situation, Python picks a random candidate (element) and returns it.

votes = [1, 2, 3, 1, 2 ,3 ,3 ,3 ,3, 2, 2, 2] #here number 2 and 3 has the same appearing frequency: 5 (tie situation)
winner = majority_vote(votes)
print (winner) #it can be seen after running the code several times that this method picks one of the candidates
#and returns it randomly.

#another method
#--------------
import scipy.stats as ss

def majority_vote_short(votes):
    "Return the most common element in votes "
    mode , count = ss.mstats.mode(votes)
    return mode

votes = [1, 2, 3, 1, 2 ,3 ,3 ,3 ,3]
winner_short = majority_vote_short(votes)
print (winner_short) # it can be seen after running the code several times that this method always
#returns number 3 as the most common candidate and forgets about the other one: 2

#------------------------------------------------------------------------------

#Finding Nearest Neighbors
#--------------------------

points = np.array([[1,1], [1,2], [1,3], [2,1], [2,2], [2,3], [3,1], [3,2], [3,3]]) #points as a vector
p = np.array([2.5,2])#a single point with coordinates: x=2.5 and y=2

#plotting the points and point p
import matplotlib.pyplot as plt

plt.plot(points[:,0], points[:,1],"ro")
plt.plot(p[0],p[1],"bo")
plt.axis([0.5, 3.5, 0.5, 3.5])

distances = np.zeros(points.shape[0])#'.shape[0]' here reutrn a tuple of array dimensions.
for i in range(len(distances)):# loop over all points
    distances[i] = distance(p,points[i]) # compute the distance between point p and every other point using the function 'distnace'

print(distances)#reutrn the distances values between all the points and the point p
print(points[4])#return the point number 4
print(distances[4])#calculate the distnace between point number 4 and point p
print(points[7])#return the point 7
print(distances[7])#calculate the distnace between point number 7 and point p

#construct a 'find_nearest_neighbor' function
def find_nearest_neighbor(p, points, k=5):
    """find the k nearest neighbors of point p nd return their indices """
    distances = np.zeros(points.shape[0])#'.shape[0]' here reutrn a tuple of array dimensions.
    for i in range(len(distances)):# loop over all points
        distances[i] = distance(p,points[i]) # compute the distance between point p and every other point using the function 'distnace'
    ind = np.argsort(distances)#'argsort' is a function that sorts the distnaces indinces from short to long.
    #np.argsort(distances) it sorts an array according to a single argument and returns the sorted indices
    return ind[0:k] # or for short ind[:k]. This step is to return those k points that are nearest to point p

ind = find_nearest_neighbor(p, points, 2); print(points[ind])# return the first 2 nearest neighbors
ind = find_nearest_neighbor(p, points, 3); print(points[ind])# return the first 3 nearest neighbors
ind = find_nearest_neighbor(p, points, 4); print(points[ind])# return the first 4 nearest neighbors

#construct a 'knn_predict' function
def knn_predict(p, points, outcomes, k=5):
    #find k nearest neighbors
    ind = find_nearest_neighbor(p, points, k)
    return majority_vote(outcomes[ind])
    #predict the class/ categroy of p based on majority vote

outcomes = np.array([0,0,0,0,1,1,1,1,1]) # classes definition

print(len(outcomes))

print(knn_predict(np.array([2.5, 2.7]), points, outcomes, k=2)) #class 1
print(knn_predict(np.array([1.0, 2.7]), points, outcomes, k=2)) #class 0

#------------------------------------------------------------------------------

#Generating Synthetic Data
#-------------------------

#------------------------------------------------------------------------------
<commit_msg>Update kNN_Classification.py<commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Mar 14 09:34:17 2017

@author: lamahamadeh
"""

'''
Case Study about an introduction to kNN classifiction
'''

#Finding the Euclidean Distance Between Two Points
#---------------------------------------------------

import numpy as np

def distance(p1,p2):
    "find the distance betweeen points p1 and p2"
    return np.sqrt(np.sum(np.power(p2 - p1, 2)))

p1 = np.array([1,1])
p2 = np.array([4,4])

print (distance(p1,p2))

#------------------------------------------------------------------------------

#Majority Vote
#-------------

import random

def majority_vote(votes):
    "Return the most common element in votes"
    vote_counts = {}
    for vote in votes:
        if vote in vote_counts:
            vote_counts[vote] += 1
        else:
            vote_counts[vote] = 1
    #items method
    winners = []
    max_counts = max(vote_counts.values()) #ask Python about the frequency the most appearing value has
    for vote, count in vote_counts.items(): #this returns a tuple of (vote,count)
        if count == max_counts:
            winners.append(vote)

    return random.choice(winners) #in case of a tie situation, Python picks a random candidate (element) and returns it.

votes = [1, 2, 3, 1, 2 ,3 ,3 ,3 ,3, 2, 2, 2] #here number 2 and 3 has the same appearing frequency: 5 (tie situation)
winner = majority_vote(votes)
print (winner) #it can be seen after running the code several times that this method picks one of the candidates
#and returns it randomly.

#another method
#--------------
import scipy.stats as ss

def majority_vote_short(votes):
    "Return the most common element in votes "
    mode , count = ss.mstats.mode(votes)
    return mode

votes = [1, 2, 3, 1, 2 ,3 ,3 ,3 ,3]
winner_short = majority_vote_short(votes)
print (winner_short) # it can be seen after running the code several times that this method always
#returns number 3 as the most common candidate and forgets about the other one: 2

#------------------------------------------------------------------------------

#Finding Nearest Neighbors
#--------------------------

points = np.array([[1,1], [1,2], [1,3], [2,1], [2,2], [2,3], [3,1], [3,2], [3,3]]) #points as a vector
p = np.array([2.5,2])#a single point with coordinates: x=2.5 and y=2

#plotting the points and point p
import matplotlib.pyplot as plt

plt.plot(points[:,0], points[:,1],"ro")
plt.plot(p[0],p[1],"bo")
plt.axis([0.5, 3.5, 0.5, 3.5])

distances = np.zeros(points.shape[0])#'.shape[0]' here reutrn a tuple of array dimensions.
for i in range(len(distances)):# loop over all points
    distances[i] = distance(p,points[i]) # compute the distance between point p and every other point using the function 'distnace'

print(distances)#reutrn the distances values between all the points and the point p
print(points[4])#return the point number 4
print(distances[4])#calculate the distnace between point number 4 and point p
print(points[7])#return the point 7
print(distances[7])#calculate the distnace between point number 7 and point p

#construct a 'find_nearest_neighbor' function
def find_nearest_neighbor(p, points, k=5):
    """find the k nearest neighbors of point p nd return their indices """
    distances = np.zeros(points.shape[0])#'.shape[0]' here reutrn a tuple of array dimensions.
    for i in range(len(distances)):# loop over all points
        distances[i] = distance(p,points[i]) # compute the distance between point p and every other point using the function 'distnace'
    ind = np.argsort(distances)#'argsort' is a function that sorts the distnaces indinces from short to long.
    #np.argsort(distances) it sorts an array according to a single argument and returns the sorted indices
    return ind[0:k] # or for short ind[:k]. This step is to return those k points that are nearest to point p

ind = find_nearest_neighbor(p, points, 2); print(points[ind])# return the first 2 nearest neighbors
ind = find_nearest_neighbor(p, points, 3); print(points[ind])# return the first 3 nearest neighbors
ind = find_nearest_neighbor(p, points, 4); print(points[ind])# return the first 4 nearest neighbors

#construct a 'knn_predict' function
def knn_predict(p, points, outcomes, k=5):
    #find k nearest neighbors
    ind = find_nearest_neighbor(p, points, k)
    return majority_vote(outcomes[ind])
    #predict the class/ categroy of p based on majority vote

outcomes = np.array([0,0,0,0,1,1,1,1,1]) # classes definition

print(len(outcomes))

print(knn_predict(np.array([2.5, 2.7]), points, outcomes, k=2)) #class 1
print(knn_predict(np.array([1.0, 2.7]), points, outcomes, k=2)) #class 0

#------------------------------------------------------------------------------

#Generating Synthetic Data
#-------------------------

def generate_synthetic_data(n=50):
    """Create two sets of points from bivariate normal distributions."""
    points = np.concatenate((ss.norm(0,1).rvs((n,2)),ss.norm(1,1).rvs((n,2))), axis=0)
    outcomes = np.concatenate((np.repeat(0,n), np.repeat(1,n)))
    return (points, outcomes)

(points, outcomes) = generate_synthetic_data(n=20)

n=20

plt.figure()
plt.plot(points[:n,0], points[:n,1], "ro")
plt.plot(points[n:,0], points[n:,1], "bo")

#------------------------------------------------------------------------------

#Making a Prediction Grid
#-------------------------

#------------------------------------------------------------------------------
<|endoftext|>


<commit_before>import re
from ...lang import number_replace
from ...lang.tokenize import wiki_tokenize_func

ENTITY_MATCH_RE = re.compile(r'<(.+)\|(.+)>')

def parse_types_text(text, dbpedia_redirects, dbpedia_types, numeric=True):
    """
    :type dbpedia_types: dict
    :type dbpedia_redirects: dict
    """
    line = wiki_tokenize_func(text)
    new_line = []
    for word in line:
        match = ENTITY_MATCH_RE.search(word)
        if match:
            uri = match.group(1)
            orig_text = match.group(2)
            orig_text = orig_text.replace('_', ' ')
            for uri in (uri, uri.capitalize()):
                stop = False
                if uri in dbpedia_types:
                    new_line.append(match.expand('<dbpedia:' + dbpedia_types[uri][0]+'>'))
                    stop = True
                elif uri in dbpedia_redirects and dbpedia_redirects[uri] in dbpedia_types:
                    new_line.append(match.expand('<dbpedia:' + dbpedia_types[dbpedia_redirects[uri]][0] + '>'))
                    stop = True
                elif numeric:
                    new_line.extend([number_replace(x) for x in wiki_tokenize_func(orig_text)])
                else:
                    new_line.extend(wiki_tokenize_func(orig_text))
                if stop:
                    break
        elif numeric:
            new_line.append(number_replace(word))
    return ' '.join(new_line)
<commit_msg>Update entities.py<commit_after>import re
from ...lang import number_replace
from ...lang.tokenize import wiki_tokenize_func

ENTITY_MATCH_RE = re.compile(r'<(.+)\|(.+)>')

def parse_types_text(text, dbpedia_redirects, dbpedia_types, numeric=True):
    """
    :type dbpedia_types: dict
    :type dbpedia_redirects: dict
    """
    line = wiki_tokenize_func(text)
    new_line = []
    for word in line:
        match = ENTITY_MATCH_RE.search(word)
        if match:
            uri = match.group(1)
            orig_text = match.group(2)
            orig_text = orig_text.replace('_', ' ')
            for uri in (uri, uri.capitalize()):
                stop = False
                if uri in dbpedia_types:
                    new_line.append(match.expand('<dbpedia:' + dbpedia_types[uri][0]+'>'))
                    stop = True
                elif uri in dbpedia_redirects and dbpedia_redirects[uri] in dbpedia_types:
                    new_line.append(match.expand('<dbpedia:' + dbpedia_types[dbpedia_redirects[uri]][0] + '>'))
                    stop = True
                elif numeric:
                    new_line.extend([number_replace(x) for x in wiki_tokenize_func(orig_text)])
                else:
                    new_line.extend(wiki_tokenize_func(orig_text))
                if stop:
                    break
        elif numeric:
            new_line.append(number_replace(word))
        else:
            new_line.append(word)
    return ' '.join(new_line)
<|endoftext|>


<commit_before># Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from collections import OrderedDict

from paddle import fluid
from paddle.fluid.param_attr import ParamAttr
from paddle.fluid.framework import Variable
from paddle.fluid.regularizer import L2Decay

from ppdet.core.workspace import register, serializable
from numbers import Integral

from .name_adapter import NameAdapter

__all__ = ['ResNet', 'ResNetC5']

@register
@serializable
class ResNet(object):
    """
    Residual Network, see https://arxiv.org/abs/1512.03385
    Args:
        depth (int): ResNet depth, should be 18, 34, 50, 101, 152.
        freeze_at (int): freeze the backbone at which stage
        norm_type (str): normalization type, 'bn', 'sync_bn' or 'affine_channel'
        freeze_norm (bool): freeze normalization layers
        norm_decay (float): weight decay for normalization layer weights
        variant (str): ResNet variant, supports 'a', 'b', 'c', 'd' currently
        feature_maps (list): index of the stages whose feature maps are returned
    """

    def __init__(self,
                 depth=50,
                 freeze_at=2,
                 norm_type='affine_channel',
                 freeze_norm=True,
                 norm_decay=0.,
                 variant='b',
                 feature_maps=[2, 3, 4, 5]):
        super(ResNet, self).__init__()

        assert depth in [18, 34, 50, 101, 152], \
            "depth {} not in [18, 34, 50, 101, 152]"
        assert variant in ['a', 'b', 'c', 'd'], "invalid ResNet variant"
        assert 0 <= freeze_at <= 4, "freeze_at should be 0, 1, 2, 3 or 4"
        assert len(feature_maps) > 0, "need one or more feature maps"
        assert norm_type in ['bn', 'sync_bn', 'affine_channel']

        self.depth = depth
        self.freeze_at = freeze_at
        self.norm_type = norm_type
        self.norm_decay = norm_decay
        self.freeze_norm = freeze_norm
        self.variant = variant
        self._model_type = 'ResNet'
        if isinstance(feature_maps, Integral):
            feature_maps = [feature_maps]
        self.feature_maps = feature_maps
        self.depth_cfg = {
            18: ([2, 2, 2, 2], self.basicblock),
            34: ([3, 4, 6, 3], self.basicblock),
            50: ([3, 4, 6, 3], self.bottleneck),
            101: ([3, 4, 23, 3], self.bottleneck),
            152: ([3, 8, 36, 3], self.bottleneck)
        }
        self.stage_filters = [64, 128, 256, 512]
        self._c1_out_chan_num = 64
        self.na = NameAdapter(self)

    def _conv_norm(self,
                   input,
                   num_filters,
                   filter_size,
                   stride=1,
                   groups=1,
                   act=None,
                   name=None):
        conv = fluid.layers.conv2d(
            input=input,
            num_filters=num_filters,
            filter_size=filter_size,
            stride=stride,
            padding=(filter_size - 1) // 2,
            groups=groups,
            act=None,
            param_attr=ParamAttr(name=name + "_weights"),
            bias_attr=False,
            name=name + '.conv2d.output.1')

        bn_name = self.na.fix_conv_norm_name(name)

        norm_lr = 0. if self.freeze_norm else 1.
        norm_decay = self.norm_decay
        pattr = ParamAttr(
            name=bn_name + '_scale',
            learning_rate=norm_lr,
            regularizer=L2Decay(norm_decay))
        battr = ParamAttr(
            name=bn_name + '_offset',
            learning_rate=norm_lr,
            regularizer=L2Decay(norm_decay))

        if self.norm_type in ['bn', 'sync_bn']:
            out = fluid.layers.batch_norm(
                input=conv,
                act=act,
                name=bn_name + '.output.1',
                param_attr=pattr,
                bias_attr=battr,
                moving_mean_name=bn_name + '_mean',
                moving_variance_name=bn_name + '_variance', )
            scale = fluid.framework._get_var(pattr.name)
            bias = fluid.framework._get_var(battr.name)
        elif self.norm_type == 'affine_channel':
            scale = fluid.layers.create_parameter(
                shape=[conv.shape[1]],
                dtype=conv.dtype,
                attr=pattr,
                default_initializer=fluid.initializer.Constant(1.))
            bias = fluid.layers.create_parameter(
                shape=[conv.shape[1]],
                dtype=conv.dtype,
                attr=battr,
                default_initializer=fluid.initializer.Constant(0.))
            out = fluid.layers.affine_channel(
                x=conv, scale=scale, bias=bias, act=act)
        if self.freeze_norm:
            scale.stop_gradient = True
            bias.stop_gradient = True
        return out

    def _shortcut(self, input, ch_out, stride, is_first, name):
        max_pooling_in_short_cut = self.variant == 'd'
        ch_in = input.shape[1]
        # the naming rule is same as pretrained weight
        name = self.na.fix_shortcut_name(name)

        if ch_in != ch_out or stride != 1:
            if max_pooling_in_short_cut and not is_first:
                input = fluid.layers.pool2d(
                    input=input,
                    pool_size=2,
                    pool_stride=2,
                    pool_padding=0,
                    ceil_mode=True,
                    pool_type='avg')
                return self._conv_norm(input, ch_out, 1, 1, name=name)
            return self._conv_norm(input, ch_out, 1, stride, name=name)
        else:
            return input

    def bottleneck(self, input, num_filters, stride, is_first, name):
        if self.variant == 'a':
            stride1, stride2 = stride, 1
        else:
            stride1, stride2 = 1, stride

        # ResNeXt
        groups = getattr(self, 'groups', 1)
        group_width = getattr(self, 'group_width', -1)
        if groups == 1:
            expand = 4
        elif (groups * group_width) == 256:
            expand = 1
        else:  # FIXME hard code for now, handles 32x4d, 64x4d and 32x8d
            num_filters = num_filters // 2
            expand = 2

        conv_name1, conv_name2, conv_name3, \
            shortcut_name = self.na.fix_bottleneck_name(name)
        conv_def = [[num_filters, 1, stride1, 'relu', 1, conv_name1],
                    [num_filters, 3, stride2, 'relu', groups, conv_name2],
                    [num_filters * expand, 1, 1, None, 1, conv_name3]]

        residual = input
        for (c, k, s, act, g, _name) in conv_def:
            residual = self._conv_norm(
                input=residual,
                num_filters=c,
                filter_size=k,
                stride=s,
                act=act,
                groups=g,
                name=_name)
        short = self._shortcut(
            input,
            num_filters * expand,
            stride,
            is_first=is_first,
            name=shortcut_name)
        # Squeeze-and-Excitation
        if callable(getattr(self, '_squeeze_excitation', None)):
            residual = self._squeeze_excitation(
                input=residual, num_channels=num_filters, name='fc' + name)
        return fluid.layers.elementwise_add(
            x=short, y=residual, act='relu', name=name + ".add.output.5")

    def basicblock(self, input, num_filters, stride, is_first, name):
        conv0 = self._conv_norm(
            input=input,
            num_filters=num_filters,
            filter_size=3,
            act='relu',
            stride=stride,
            name=name + "_branch2a")
        conv1 = self._conv_norm(
            input=conv0,
            num_filters=num_filters,
            filter_size=3,
            act=None,
            name=name + "_branch2b")
        short = self._shortcut(
            input, num_filters, stride, is_first, name=name + "_branch1")
        return fluid.layers.elementwise_add(x=short, y=conv1, act='relu')

    def layer_warp(self, input, stage_num):
        """
        Args:
            input (Variable): input variable.
            stage_num (int): the stage number, should be 2, 3, 4, 5

        Returns:
            The last variable in endpoint-th stage.
        """
        assert stage_num in [2, 3, 4, 5]

        stages, block_func = self.depth_cfg[self.depth]
        count = stages[stage_num - 2]

        ch_out = self.stage_filters[stage_num - 2]
        is_first = False if stage_num != 2 else True
        # Make the layer name and parameter name consistent
        # with ImageNet pre-trained model
        conv = input
        for i in range(count):
            conv_name = self.na.fix_layer_warp_name(stage_num, count, i)
            conv = block_func(
                input=conv,
                num_filters=ch_out,
                stride=2 if i == 0 and stage_num != 2 else 1,
                is_first=is_first,
                name=conv_name)
        return conv

    def c1_stage(self, input):
        out_chan = self._c1_out_chan_num

        conv1_name = self.na.fix_c1_stage_name()

        if self.variant in ['c', 'd']:
            conv_def = [
                [out_chan // 2, 3, 2, "conv1_1"],
                [out_chan // 2, 3, 1, "conv1_2"],
                [out_chan, 3, 1, "conv1_3"],
            ]
        else:
            conv_def = [[out_chan, 7, 2, conv1_name]]

        for (c, k, s, _name) in conv_def:
            input = self._conv_norm(
                input=input,
                num_filters=c,
                filter_size=k,
                stride=s,
                act='relu',
                name=_name)

        output = fluid.layers.pool2d(
            input=input,
            pool_size=3,
            pool_stride=2,
            pool_padding=1,
            pool_type='max')
        return output

    def __call__(self, input):
        assert isinstance(input, Variable)
        assert not (set(self.feature_maps) - set([2, 3, 4, 5])), \
            "feature maps {} not in [2, 3, 4, 5]".format(self.feature_maps)

        res_endpoints = []

        res = input
        feature_maps = self.feature_maps
        severed_head = getattr(self, 'severed_head', False)
        if not severed_head:
            res = self.c1_stage(res)
            feature_maps = range(2, max(self.feature_maps) + 1)

        for i in feature_maps:
            res = self.layer_warp(res, i)
            if i in self.feature_maps:
                res_endpoints.append(res)
            if self.freeze_at >= i:
                res.stop_gradient = True

        return OrderedDict([('res{}_sum'.format(self.feature_maps[idx]), feat)
                            for idx, feat in enumerate(res_endpoints)])

@register
@serializable
class ResNetC5(ResNet):
    __doc__ = ResNet.__doc__

    def __init__(self,
                 depth=50,
                 freeze_at=2,
                 norm_type='affine_channel',
                 freeze_norm=True,
                 norm_decay=0.,
                 variant='b',
                 feature_maps=[5]):
        super(ResNetC5, self).__init__(depth, freeze_at, norm_type, freeze_norm,
                                       norm_decay, variant, feature_maps)
        self.severed_head = True
<commit_msg>Fix ResNet assertion error when `feature_maps` is an integer (#2508)<commit_after># Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from collections import OrderedDict

from paddle import fluid
from paddle.fluid.param_attr import ParamAttr
from paddle.fluid.framework import Variable
from paddle.fluid.regularizer import L2Decay

from ppdet.core.workspace import register, serializable
from numbers import Integral

from .name_adapter import NameAdapter

__all__ = ['ResNet', 'ResNetC5']

@register
@serializable
class ResNet(object):
    """
    Residual Network, see https://arxiv.org/abs/1512.03385
    Args:
        depth (int): ResNet depth, should be 18, 34, 50, 101, 152.
        freeze_at (int): freeze the backbone at which stage
        norm_type (str): normalization type, 'bn', 'sync_bn' or 'affine_channel'
        freeze_norm (bool): freeze normalization layers
        norm_decay (float): weight decay for normalization layer weights
        variant (str): ResNet variant, supports 'a', 'b', 'c', 'd' currently
        feature_maps (list): index of the stages whose feature maps are returned
    """

    def __init__(self,
                 depth=50,
                 freeze_at=2,
                 norm_type='affine_channel',
                 freeze_norm=True,
                 norm_decay=0.,
                 variant='b',
                 feature_maps=[2, 3, 4, 5]):
        super(ResNet, self).__init__()

        if isinstance(feature_maps, Integral):
            feature_maps = [feature_maps]

        assert depth in [18, 34, 50, 101, 152], \
            "depth {} not in [18, 34, 50, 101, 152]"
        assert variant in ['a', 'b', 'c', 'd'], "invalid ResNet variant"
        assert 0 <= freeze_at <= 4, "freeze_at should be 0, 1, 2, 3 or 4"
        assert len(feature_maps) > 0, "need one or more feature maps"
        assert norm_type in ['bn', 'sync_bn', 'affine_channel']

        self.depth = depth
        self.freeze_at = freeze_at
        self.norm_type = norm_type
        self.norm_decay = norm_decay
        self.freeze_norm = freeze_norm
        self.variant = variant
        self._model_type = 'ResNet'
        self.feature_maps = feature_maps
        self.depth_cfg = {
            18: ([2, 2, 2, 2], self.basicblock),
            34: ([3, 4, 6, 3], self.basicblock),
            50: ([3, 4, 6, 3], self.bottleneck),
            101: ([3, 4, 23, 3], self.bottleneck),
            152: ([3, 8, 36, 3], self.bottleneck)
        }
        self.stage_filters = [64, 128, 256, 512]
        self._c1_out_chan_num = 64
        self.na = NameAdapter(self)

    def _conv_norm(self,
                   input,
                   num_filters,
                   filter_size,
                   stride=1,
                   groups=1,
                   act=None,
                   name=None):
        conv = fluid.layers.conv2d(
            input=input,
            num_filters=num_filters,
            filter_size=filter_size,
            stride=stride,
            padding=(filter_size - 1) // 2,
            groups=groups,
            act=None,
            param_attr=ParamAttr(name=name + "_weights"),
            bias_attr=False,
            name=name + '.conv2d.output.1')

        bn_name = self.na.fix_conv_norm_name(name)

        norm_lr = 0. if self.freeze_norm else 1.
        norm_decay = self.norm_decay
        pattr = ParamAttr(
            name=bn_name + '_scale',
            learning_rate=norm_lr,
            regularizer=L2Decay(norm_decay))
        battr = ParamAttr(
            name=bn_name + '_offset',
            learning_rate=norm_lr,
            regularizer=L2Decay(norm_decay))

        if self.norm_type in ['bn', 'sync_bn']:
            out = fluid.layers.batch_norm(
                input=conv,
                act=act,
                name=bn_name + '.output.1',
                param_attr=pattr,
                bias_attr=battr,
                moving_mean_name=bn_name + '_mean',
                moving_variance_name=bn_name + '_variance', )
            scale = fluid.framework._get_var(pattr.name)
            bias = fluid.framework._get_var(battr.name)
        elif self.norm_type == 'affine_channel':
            scale = fluid.layers.create_parameter(
                shape=[conv.shape[1]],
                dtype=conv.dtype,
                attr=pattr,
                default_initializer=fluid.initializer.Constant(1.))
            bias = fluid.layers.create_parameter(
                shape=[conv.shape[1]],
                dtype=conv.dtype,
                attr=battr,
                default_initializer=fluid.initializer.Constant(0.))
            out = fluid.layers.affine_channel(
                x=conv, scale=scale, bias=bias, act=act)
        if self.freeze_norm:
            scale.stop_gradient = True
            bias.stop_gradient = True
        return out

    def _shortcut(self, input, ch_out, stride, is_first, name):
        max_pooling_in_short_cut = self.variant == 'd'
        ch_in = input.shape[1]
        # the naming rule is same as pretrained weight
        name = self.na.fix_shortcut_name(name)

        if ch_in != ch_out or stride != 1:
            if max_pooling_in_short_cut and not is_first:
                input = fluid.layers.pool2d(
                    input=input,
                    pool_size=2,
                    pool_stride=2,
                    pool_padding=0,
                    ceil_mode=True,
                    pool_type='avg')
                return self._conv_norm(input, ch_out, 1, 1, name=name)
            return self._conv_norm(input, ch_out, 1, stride, name=name)
        else:
            return input

    def bottleneck(self, input, num_filters, stride, is_first, name):
        if self.variant == 'a':
            stride1, stride2 = stride, 1
        else:
            stride1, stride2 = 1, stride

        # ResNeXt
        groups = getattr(self, 'groups', 1)
        group_width = getattr(self, 'group_width', -1)
        if groups == 1:
            expand = 4
        elif (groups * group_width) == 256:
            expand = 1
        else:  # FIXME hard code for now, handles 32x4d, 64x4d and 32x8d
            num_filters = num_filters // 2
            expand = 2

        conv_name1, conv_name2, conv_name3, \
            shortcut_name = self.na.fix_bottleneck_name(name)
        conv_def = [[num_filters, 1, stride1, 'relu', 1, conv_name1],
                    [num_filters, 3, stride2, 'relu', groups, conv_name2],
                    [num_filters * expand, 1, 1, None, 1, conv_name3]]

        residual = input
        for (c, k, s, act, g, _name) in conv_def:
            residual = self._conv_norm(
                input=residual,
                num_filters=c,
                filter_size=k,
                stride=s,
                act=act,
                groups=g,
                name=_name)
        short = self._shortcut(
            input,
            num_filters * expand,
            stride,
            is_first=is_first,
            name=shortcut_name)
        # Squeeze-and-Excitation
        if callable(getattr(self, '_squeeze_excitation', None)):
            residual = self._squeeze_excitation(
                input=residual, num_channels=num_filters, name='fc' + name)
        return fluid.layers.elementwise_add(
            x=short, y=residual, act='relu', name=name + ".add.output.5")

    def basicblock(self, input, num_filters, stride, is_first, name):
        conv0 = self._conv_norm(
            input=input,
            num_filters=num_filters,
            filter_size=3,
            act='relu',
            stride=stride,
            name=name + "_branch2a")
        conv1 = self._conv_norm(
            input=conv0,
            num_filters=num_filters,
            filter_size=3,
            act=None,
            name=name + "_branch2b")
        short = self._shortcut(
            input, num_filters, stride, is_first, name=name + "_branch1")
        return fluid.layers.elementwise_add(x=short, y=conv1, act='relu')

    def layer_warp(self, input, stage_num):
        """
        Args:
            input (Variable): input variable.
            stage_num (int): the stage number, should be 2, 3, 4, 5

        Returns:
            The last variable in endpoint-th stage.
        """
        assert stage_num in [2, 3, 4, 5]

        stages, block_func = self.depth_cfg[self.depth]
        count = stages[stage_num - 2]

        ch_out = self.stage_filters[stage_num - 2]
        is_first = False if stage_num != 2 else True
        # Make the layer name and parameter name consistent
        # with ImageNet pre-trained model
        conv = input
        for i in range(count):
            conv_name = self.na.fix_layer_warp_name(stage_num, count, i)
            conv = block_func(
                input=conv,
                num_filters=ch_out,
                stride=2 if i == 0 and stage_num != 2 else 1,
                is_first=is_first,
                name=conv_name)
        return conv

    def c1_stage(self, input):
        out_chan = self._c1_out_chan_num

        conv1_name = self.na.fix_c1_stage_name()

        if self.variant in ['c', 'd']:
            conv_def = [
                [out_chan // 2, 3, 2, "conv1_1"],
                [out_chan // 2, 3, 1, "conv1_2"],
                [out_chan, 3, 1, "conv1_3"],
            ]
        else:
            conv_def = [[out_chan, 7, 2, conv1_name]]

        for (c, k, s, _name) in conv_def:
            input = self._conv_norm(
                input=input,
                num_filters=c,
                filter_size=k,
                stride=s,
                act='relu',
                name=_name)

        output = fluid.layers.pool2d(
            input=input,
            pool_size=3,
            pool_stride=2,
            pool_padding=1,
            pool_type='max')
        return output

    def __call__(self, input):
        assert isinstance(input, Variable)
        assert not (set(self.feature_maps) - set([2, 3, 4, 5])), \
            "feature maps {} not in [2, 3, 4, 5]".format(self.feature_maps)

        res_endpoints = []

        res = input
        feature_maps = self.feature_maps
        severed_head = getattr(self, 'severed_head', False)
        if not severed_head:
            res = self.c1_stage(res)
            feature_maps = range(2, max(self.feature_maps) + 1)

        for i in feature_maps:
            res = self.layer_warp(res, i)
            if i in self.feature_maps:
                res_endpoints.append(res)
            if self.freeze_at >= i:
                res.stop_gradient = True

        return OrderedDict([('res{}_sum'.format(self.feature_maps[idx]), feat)
                            for idx, feat in enumerate(res_endpoints)])

@register
@serializable
class ResNetC5(ResNet):
    __doc__ = ResNet.__doc__

    def __init__(self,
                 depth=50,
                 freeze_at=2,
                 norm_type='affine_channel',
                 freeze_norm=True,
                 norm_decay=0.,
                 variant='b',
                 feature_maps=[5]):
        super(ResNetC5, self).__init__(depth, freeze_at, norm_type, freeze_norm,
                                       norm_decay, variant, feature_maps)
        self.severed_head = True
<|endoftext|>


<commit_before>from couchdbkit.resource import ResourceNotFound
from dimagi.utils.logging import notify_exception

class SyncCouchToSQLMixin(object):
    """
    A mixin to be used for migrating a Couch model
    to a SQL model (see also SyncSQLToCouchMixin).

    Steps to use it:

    1. Create a Django ORM model which inherits from SyncSQLToCouchMixin
    and is identical to the Couch model with one additional field named
    couch_id, for the _id of the Couch doc. Make sure you index couch_id
    and allow it to be null since it can be initially null. Do not include
    this field in _migration_get_fields().

    2. Make the Couch model inherit from SyncCouchToSQLMixin.

    3. Implement the unimplemented methods from the mixins on both models.

    4. Make sure neither of the models overrides the save() method. If
    either does, you will have to manually insert the sync code from
    the corresponding mixin. Also make sure the mixins are the left-most
    classes being inherited from.

    After that, any saves to the Couch model will sync the SQL model, and
    any saves to the SQL model will sync the Couch model. After migrating
    each doc, you can incrementally change the code to use to SQL model
    instead of the Couch model, and eventually remove the Couch model entirely.

    If you have a custom sync process, just override _migration_sync_to_sql.
    """

    @classmethod
    def _migration_get_fields(cls):
        """
        Should return a list of field names to sync. These field names should
        be identical between the Couch and SQL models.
        """
        raise NotImplementedError()

    @classmethod
    def _migration_get_sql_model_class(cls):
        """
        Should return the class of the SQL model.
        """
        raise NotImplementedError()

    def _migration_automatically_handle_dups(self):
        """
        Ideally, you should use a CriticalSection to lock out your get and save
        functionality so that each sync will automatically be atomic. However,
        even after doing so, during periods of high latency the locks may time
        out and you still might end up with a race condition in the sync that
        creates more than one SQL model from this couch model.

        Make this method return True if you want subsequent syncs to delete
        the duplicate records and resync. Otherwise, the default behavior is
        that a notify exception email will go out each time duplicates are found.
        """
        return False

    def _migration_get_sql_object(self):
        cls = self._migration_get_sql_model_class()
        try:
            return cls.objects.get(couch_id=self._id)
        except cls.DoesNotExist:
            return None
        except cls.MultipleObjectsReturned:
            if not self._migration_automatically_handle_dups():
                raise
            cls.objects.filter(couch_id=self._id).delete()
            return None

    def _migration_get_or_create_sql_object(self):
        cls = self._migration_get_sql_model_class()
        obj = self._migration_get_sql_object()
        if obj is None:
            obj = cls(couch_id=self._id)
        return obj

    def _migration_sync_to_sql(self, sql_object):
        for field_name in self._migration_get_fields():
            value = getattr(self, field_name)
            setattr(sql_object, field_name, value)
        sql_object.save(sync_to_couch=False)

    def _migration_do_sync(self):
        sql_object = self._migration_get_or_create_sql_object()
        self._migration_sync_to_sql(sql_object)

    def save(self, *args, **kwargs):
        sync_to_sql = kwargs.pop('sync_to_sql', True)
        super(SyncCouchToSQLMixin, self).save(*args, **kwargs)
        if sync_to_sql:
            try:
                self._migration_do_sync()
            except:
                sql_class_name = self._migration_get_sql_model_class().__name__
                couch_class_name = self.__class__.__name__
                notify_exception(None,
                    message='Could not sync %s SQL object from %s %s' % (sql_class_name,
                        couch_class_name, self._id))

    def delete(self, *args, **kwargs):
        sync_to_sql = kwargs.pop('sync_to_sql', True)
        if sync_to_sql:
            sql_object = self._migration_get_sql_object()
            if sql_object is not None:
                sql_object.delete(sync_to_couch=False)
        super(SyncCouchToSQLMixin, self).delete(*args, **kwargs)

class SyncSQLToCouchMixin(object):
    """
    See SyncCouchToSQLMixin.

    If you have a custom sync process, just override _migration_sync_to_couch.
    """

    @classmethod
    def _migration_get_fields(cls):
        """
        Should return a list of field names to sync. These field names should
        be identical between the Couch and SQL models.
        """
        raise NotImplementedError()

    @classmethod
    def _migration_get_couch_model_class(cls):
        """
        Should return the class of the Couch model.
        """
        raise NotImplementedError()

    def _migration_get_couch_object(self):
        cls = self._migration_get_couch_model_class()
        if not self.couch_id:
            return None
        else:
            obj = cls.get(self.couch_id)

        return obj

    def _migration_get_or_create_couch_object(self):
        cls = self._migration_get_couch_model_class()
        obj = self._migration_get_couch_object()
        if obj is None:
            obj = cls()
            obj.save(sync_to_sql=False)
            self.couch_id = obj._id
            self.save(sync_to_couch=False)
        return obj

    def _migration_sync_to_couch(self, couch_obj):
        for field_name in self._migration_get_fields():
            value = getattr(self, field_name)
            setattr(couch_obj, field_name, value)
        couch_obj.save(sync_to_sql=False)

    def _migration_do_sync(self):
        couch_obj = self._migration_get_or_create_couch_object()
        self._migration_sync_to_couch(couch_obj)

    def save(self, *args, **kwargs):
        sync_to_couch = kwargs.pop('sync_to_couch', True)
        super(SyncSQLToCouchMixin, self).save(*args, **kwargs)
        if sync_to_couch:
            try:
                self._migration_do_sync()
            except:
                couch_class_name = self._migration_get_couch_model_class().__name__
                sql_class_name = self.__class__.__name__
                notify_exception(None,
                    message='Could not sync %s Couch object from %s %s' % (couch_class_name,
                        sql_class_name, self.pk))

    def delete(self, *args, **kwargs):
        sync_to_couch = kwargs.pop('sync_to_couch', True)
        if sync_to_couch:
            couch_object = self._migration_get_couch_object()
            if couch_object is not None:
                couch_object.delete(sync_to_sql=False)
        super(SyncSQLToCouchMixin, self).delete(*args, **kwargs)
<commit_msg>PR feedback<commit_after>from couchdbkit.resource import ResourceNotFound
from dimagi.utils.logging import notify_exception

class SyncCouchToSQLMixin(object):
    """
    A mixin to be used for migrating a Couch model
    to a SQL model (see also SyncSQLToCouchMixin).

    Steps to use it:

    1. Create a Django ORM model which inherits from SyncSQLToCouchMixin
    and is identical to the Couch model with one additional field named
    couch_id, for the _id of the Couch doc. Make sure you index couch_id
    and allow it to be null since it can be initially null. Do not include
    this field in _migration_get_fields().

    2. Make the Couch model inherit from SyncCouchToSQLMixin.

    3. Implement the unimplemented methods from the mixins on both models.

    4. Make sure neither of the models overrides the save() or delete()
    methods. If either does, you will have to manually insert the sync
    code from the corresponding mixin. Also make sure the mixins are the
    left-most classes being inherited from.

    After that, any saves to the Couch model will sync the SQL model, and
    any saves to the SQL model will sync the Couch model. After migrating
    each doc, you can incrementally change the code to use to SQL model
    instead of the Couch model, and eventually remove the Couch model entirely.

    If you have a custom sync process, just override _migration_sync_to_sql.
    """

    @classmethod
    def _migration_get_fields(cls):
        """
        Should return a list of field names to sync. These field names should
        be identical between the Couch and SQL models.
        """
        raise NotImplementedError()

    @classmethod
    def _migration_get_sql_model_class(cls):
        """
        Should return the class of the SQL model.
        """
        raise NotImplementedError()

    def _migration_automatically_handle_dups(self):
        """
        Ideally, you should use a CriticalSection to lock out your get and save
        functionality so that each sync will automatically be atomic. However,
        even after doing so, during periods of high latency the locks may time
        out and you still might end up with a race condition in the sync that
        creates more than one SQL model from this couch model.

        Make this method return True if you want subsequent syncs to delete
        the duplicate records and resync. Otherwise, the default behavior is
        that a notify exception email will go out each time duplicates are found.
        """
        return False

    def _migration_get_sql_object(self):
        cls = self._migration_get_sql_model_class()
        try:
            return cls.objects.get(couch_id=self._id)
        except cls.DoesNotExist:
            return None
        except cls.MultipleObjectsReturned:
            if not self._migration_automatically_handle_dups():
                raise
            cls.objects.filter(couch_id=self._id).delete()
            return None

    def _migration_get_or_create_sql_object(self):
        cls = self._migration_get_sql_model_class()
        obj = self._migration_get_sql_object()
        if obj is None:
            obj = cls(couch_id=self._id)
        return obj

    def _migration_sync_to_sql(self, sql_object):
        for field_name in self._migration_get_fields():
            value = getattr(self, field_name)
            setattr(sql_object, field_name, value)
        sql_object.save(sync_to_couch=False)

    def _migration_do_sync(self):
        sql_object = self._migration_get_or_create_sql_object()
        self._migration_sync_to_sql(sql_object)

    def save(self, *args, **kwargs):
        sync_to_sql = kwargs.pop('sync_to_sql', True)
        super(SyncCouchToSQLMixin, self).save(*args, **kwargs)
        if sync_to_sql:
            try:
                self._migration_do_sync()
            except:
                sql_class_name = self._migration_get_sql_model_class().__name__
                couch_class_name = self.__class__.__name__
                notify_exception(None,
                    message='Could not sync %s SQL object from %s %s' % (sql_class_name,
                        couch_class_name, self._id))

    def delete(self, *args, **kwargs):
        sync_to_sql = kwargs.pop('sync_to_sql', True)
        if sync_to_sql:
            sql_object = self._migration_get_sql_object()
            if sql_object is not None:
                sql_object.delete(sync_to_couch=False)
        super(SyncCouchToSQLMixin, self).delete(*args, **kwargs)

class SyncSQLToCouchMixin(object):
    """
    See SyncCouchToSQLMixin.

    If you have a custom sync process, just override _migration_sync_to_couch.
    """

    @classmethod
    def _migration_get_fields(cls):
        """
        Should return a list of field names to sync. These field names should
        be identical between the Couch and SQL models.
        """
        raise NotImplementedError()

    @classmethod
    def _migration_get_couch_model_class(cls):
        """
        Should return the class of the Couch model.
        """
        raise NotImplementedError()

    def _migration_get_couch_object(self):
        if not self.couch_id:
            return None
        cls = self._migration_get_couch_model_class()
        return cls.get(self.couch_id)

    def _migration_get_or_create_couch_object(self):
        cls = self._migration_get_couch_model_class()
        obj = self._migration_get_couch_object()
        if obj is None:
            obj = cls()
            obj.save(sync_to_sql=False)
            self.couch_id = obj._id
            self.save(sync_to_couch=False)
        return obj

    def _migration_sync_to_couch(self, couch_obj):
        for field_name in self._migration_get_fields():
            value = getattr(self, field_name)
            setattr(couch_obj, field_name, value)
        couch_obj.save(sync_to_sql=False)

    def _migration_do_sync(self):
        couch_obj = self._migration_get_or_create_couch_object()
        self._migration_sync_to_couch(couch_obj)

    def save(self, *args, **kwargs):
        sync_to_couch = kwargs.pop('sync_to_couch', True)
        super(SyncSQLToCouchMixin, self).save(*args, **kwargs)
        if sync_to_couch:
            try:
                self._migration_do_sync()
            except:
                couch_class_name = self._migration_get_couch_model_class().__name__
                sql_class_name = self.__class__.__name__
                notify_exception(None,
                    message='Could not sync %s Couch object from %s %s' % (couch_class_name,
                        sql_class_name, self.pk))

    def delete(self, *args, **kwargs):
        sync_to_couch = kwargs.pop('sync_to_couch', True)
        if sync_to_couch:
            couch_object = self._migration_get_couch_object()
            if couch_object is not None:
                couch_object.delete(sync_to_sql=False)
        super(SyncSQLToCouchMixin, self).delete(*args, **kwargs)
<|endoftext|>