<commit_before>#!/usr/bin/python

from Message import Message
from Action import Action

import socket
import string
import re
import time
#ssl support
from ssl import SSLSocket

class Core:
    def __init__(self,channel="#dut.info",name="twp_bot",port=6667,host='irc.freenode.net'):
        self.m_channel=channel
        self.m_host=host
        self.m_port=port
        self.m_name=name
        self.m_socket=self.createSocket();
    def start(self):
        self.m_socket.connect((self.m_host, self.m_port))
        self.m_socket.sendall('PASS irc\r\n')
        self.m_socket.sendall('NICK '+self.m_name+'\r\n')
        self.m_socket.sendall('USER '+self.m_name+' 127.0.0.1 '+self.m_host+' '+self.m_name+'\r\n')
        self.m_socket.sendall('JOIN '+self.m_channel+'\r\n')
    def createSocket(self):
        return socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    def send2Chan(self,Message):
        time.sleep(1)
        self.m_socket.sendall("PRIVMSG "+Message.getDest()+" : "+Message.getText()+" \r\n")
    def sendAction2Chan(self,Message):
        time.sleep(1)
        self.m_socket.sendall("PRIVMSG "+Message.getDest()+" :"+chr(1)+"ACTION "+Message.getText()+chr(1)+" \r\n")
    def read(self):
        try:
            data = self.m_socket.recv(1024)
            #recuperation des donnees pour le Message
            tabData=data.split(":")
            pseudo=data.split("!")[0]
            prompt="<"+"".join(pseudo)+"> : "
            tabData.pop(0)
            tabData.pop(0)
            msg=":".join(tabData)
            #manque recuperation du destinataire
            regName = ".*(PRIVMSG|NOTICE) "+self.m_name+" :.*"
            if re.match(regName.lower(),data.lower())!=None:
                return Message(self.m_name,pseudo,msg)
            regChan = ".*PRIVMSG "+self.m_channel+" :.*"
            if re.match(regChan.lower(),data.lower())!=None:
                return Message(self.m_channel,pseudo,msg)
            if data.upper().__contains__("PING"):
                self.m_socket.sendall('PONG\r\n')
        except Exception as ex:
            print("Erreur de reception")
            print(ex)
    def quit(self):
        self.m_socket.close();

class secureCore(Core):
    def __init__(self,channel="#dut.info",name="twp_bot",port=6697,host='irc.freenode.net'):
        Core.__init__(self,channel="#dut.info",name="twp_bot",port=6697,host='irc.freenode.net')
        self.m_socket=self.createSocket()
    def createSocket(self):
        return SSLSocket(socket.socket(socket.AF_INET, socket.SOCK_STREAM))

<commit_msg>Fixed issues with args (channel,irc_nickname,port,etc)<commit_after>#!/usr/bin/python

from Message import Message
from Action import Action

import socket
import string
import re
import time
#ssl support
from ssl import SSLSocket

class Core:
    def __init__(self,channel="#dut.info",name="twp_bot",port=6667,host='irc.freenode.net'):
        self.m_channel=channel
        self.m_host=host
        self.m_port=port
        self.m_name=name
        self.m_socket=self.createSocket();
    def start(self):
        self.m_socket.connect((self.m_host, self.m_port))
        self.m_socket.sendall('PASS irc\r\n')
        self.m_socket.sendall('NICK '+self.m_name+'\r\n')
        self.m_socket.sendall('USER '+self.m_name+' 127.0.0.1 '+self.m_host+' '+self.m_name+'\r\n')
        self.m_socket.sendall('JOIN '+self.m_channel+'\r\n')
    def createSocket(self):
        return socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    def send2Chan(self,Message):
        time.sleep(1)
        self.m_socket.sendall("PRIVMSG "+Message.getDest()+" : "+Message.getText()+" \r\n")
    def sendAction2Chan(self,Message):
        time.sleep(1)
        self.m_socket.sendall("PRIVMSG "+Message.getDest()+" :"+chr(1)+"ACTION "+Message.getText()+chr(1)+" \r\n")
    def read(self):
        try:
            data = self.m_socket.recv(1024)
            #recuperation des donnees pour le Message
            tabData=data.split(":")
            pseudo=data.split("!")[0]
            prompt="<"+"".join(pseudo)+"> : "
            tabData.pop(0)
            tabData.pop(0)
            msg=":".join(tabData)
            #manque recuperation du destinataire
            regName = ".*(PRIVMSG|NOTICE) "+self.m_name+" :.*"
            if re.match(regName.lower(),data.lower())!=None:
                return Message(self.m_name,pseudo,msg)
            regChan = ".*PRIVMSG "+self.m_channel+" :.*"
            if re.match(regChan.lower(),data.lower())!=None:
                return Message(self.m_channel,pseudo,msg)
            if data.upper().__contains__("PING"):
                self.m_socket.sendall('PONG\r\n')
        except Exception as ex:
            print("Erreur de reception")
            print(ex)
    def quit(self):
        self.m_socket.close();

class secureCore(Core):
    def __init__(self,channel="#dut.info",name="twp_bot",port=6697,host='irc.freenode.net'):
        Core.__init__(self,channel,name,port,host)
        self.m_socket=self.createSocket()
    def createSocket(self):
        return SSLSocket(socket.socket(socket.AF_INET, socket.SOCK_STREAM))

<|endoftext|>


<commit_before>from rpython.rlib import jit

from topaz.interpreter import Interpreter
from topaz.objects.constraintobject import W_ConstraintObject
from topaz.objects.moduleobject import W_ModuleObject

class ConstraintInterpreter(Interpreter):
    def LOAD_DEREF(self, space, bytecode, frame, pc, idx):
        frame.cells[idx].upgrade_to_closure(space, frame, idx)
        w_res = space.newconstraintvariable(cell=frame.cells[idx])
        if w_res is None:
            w_res = frame.cells[idx].get(space, frame, idx) or space.w_nil
        frame.push(w_res)

    def LOAD_INSTANCE_VAR(self, space, bytecode, frame, pc, idx):
        name = space.symbol_w(bytecode.consts_w[idx])
        w_obj = frame.pop()
        w_res = space.newconstraintvariable(w_owner=w_obj, ivar=name)
        if w_res is None:
            w_res = space.find_instance_var(w_obj, name)
        frame.push(w_res)

    def LOAD_CLASS_VAR(self, space, bytecode, frame, pc, idx):
        name = space.symbol_w(bytecode.consts_w[idx])
        w_module = frame.pop()
        assert isinstance(w_module, W_ModuleObject)
        w_res = space.newconstraintvariable(w_owner=w_module, cvar=name)
        if w_res is None:
            w_res = space.find_class_var(w_module, name)
            if w_res is None:
                raise space.error(space.w_NameError,
                    "uninitialized class variable %s in %s" % (name, w_module.name)
                )
        frame.push(w_res)

    def SEND(self, space, bytecode, frame, pc, meth_idx, num_args):
        args_w = frame.popitemsreverse(num_args)
        w_receiver = frame.pop()
        w_res = None
        if space.is_kind_of(w_receiver, space.w_constraint):
            with space.normal_execution():
                w_res = space.send(w_receiver, bytecode.consts_w[meth_idx], args_w)
        else:
            w_res = space.send(w_receiver, bytecode.consts_w[meth_idx], args_w)
        frame.push(w_res)

class ConstrainedVariable(object):
    _immutable_fields_ = ["cell", "w_owner", "ivar", "cvar", "w_external_variable"]

    def __init__(self, space, cell=None, w_owner=None, ivar=None, cvar=None):
        self.w_external_variable = None
        self.cell = None
        self.w_owner = None
        self.ivar = None
        self.cvar = None
        self.constraint_blocks = []
        if cell:
            self.cell = cell
        elif w_owner and ivar:
            self.w_owner = w_owner
            self.ivar = ivar
        elif w_owner and cvar:
            self.w_owner = w_owner
            self.cvar = cvar
        else:
            raise RuntimeError("Invalid ConstraintVariableObject initialization")

        w_value = self.load_value(space)
        if space.respond_to(w_value, space.newsymbol("for_constraint")):
            with space.normal_execution():
                self.w_external_variable = space.send(
                    w_value,
                    space.newsymbol("for_constraint"),
                    [self.get_name(space)]
                )

    def __del__(self):
        # TODO: remove external variable from solver
        pass

    def is_solveable(self):
        return self.w_external_variable is not None

    def add_constraint_block(self, block):
        self.constraint_blocks.append(block)

    def load_value(self, space):
        if self.cell:
            return self.cell.get(space, None, 0) or space.w_nil
        elif self.ivar is not None:
            return self.w_owner.find_instance_var(space, self.ivar) or space.w_nil
        elif self.cvar is not None:
            return self.w_owner.find_class_var(space, self.cvar) or space.w_nil
        else:
            raise NotImplementedError("inconsistent constraint variable")

    def store_value(self, space, w_value):
        if self.cell:
            self.cell.set(space, None, 0, w_value)
        elif self.ivar is not None:
            self.w_owner.set_instance_var(space, self.ivar, w_value)
        elif self.cvar is not None:
            self.w_owner.set_class_var(space, self.cvar, w_value)
        else:
            raise NotImplementedError("inconsistent constraint variable")

    def get_name(self, space):
        clsname = space.getclass(self.load_value(space)).name
        if self.cell:
            return space.newstr_fromstr("local-%s" % clsname)
        elif self.ivar is not None:
            return space.newstr_fromstr("ivar-%s" % clsname)
        elif self.cvar is not None:
            return space.newstr_fromstr("cvar-%s" % clsname)
        return space.w_nil

    def suggest_value(self, space, w_value):
        assert self.w_external_variable
        with space.constraint_execution():
            space.send(self.w_external_variable, space.newsymbol("suggest_value"), [w_value])

    def set_i(self, space):
        if self.w_external_variable is not None:
            # This variable is part of the solver. Get the solvers
            # interpretation and store it
            w_value = space.send(self.w_external_variable, space.newsymbol("value"))
            if w_value != space.w_nil:
                # w_variable_value may not be the same as w_value,
                # because client code may choose to return a different
                # variables' value in #for_constraint
                w_variable_value = self.load_value(space)
                if space.respond_to(w_variable_value, space.newsymbol("assign_constraint_value")):
                    space.send(w_variable_value, space.newsymbol("assign_constraint_value"), [w_value])
                else:
                    self.store_value(space, w_value)
            return w_value
        return space.w_nil

    def get_i(self, space):
        self.set_i(space)
        return self.load_value(space)

    def recalculate_path(self, space, w_value):
        self.store_value(space, w_value)
        for block in self.constraint_blocks:
            w_constraint = block.get_constraint()
            assert w_constraint
            space.send(w_constraint, space.newsymbol("disable"))
            block.set_constraint(None)
            space.send(space.w_object, space.newsymbol("always"), block=block)
<commit_msg>translation type fix<commit_after>from rpython.rlib import jit

from topaz.interpreter import Interpreter
from topaz.objects.objectobject import W_Root
from topaz.objects.moduleobject import W_ModuleObject

class ConstraintInterpreter(Interpreter):
    def LOAD_DEREF(self, space, bytecode, frame, pc, idx):
        frame.cells[idx].upgrade_to_closure(space, frame, idx)
        w_res = space.newconstraintvariable(cell=frame.cells[idx])
        if w_res is None:
            w_res = frame.cells[idx].get(space, frame, idx) or space.w_nil
        frame.push(w_res)

    def LOAD_INSTANCE_VAR(self, space, bytecode, frame, pc, idx):
        name = space.symbol_w(bytecode.consts_w[idx])
        w_obj = frame.pop()
        w_res = space.newconstraintvariable(w_owner=w_obj, ivar=name)
        if w_res is None:
            w_res = space.find_instance_var(w_obj, name)
        frame.push(w_res)

    def LOAD_CLASS_VAR(self, space, bytecode, frame, pc, idx):
        name = space.symbol_w(bytecode.consts_w[idx])
        w_module = frame.pop()
        assert isinstance(w_module, W_ModuleObject)
        w_res = space.newconstraintvariable(w_owner=w_module, cvar=name)
        if w_res is None:
            w_res = space.find_class_var(w_module, name)
            if w_res is None:
                raise space.error(space.w_NameError,
                    "uninitialized class variable %s in %s" % (name, w_module.name)
                )
        frame.push(w_res)

    def SEND(self, space, bytecode, frame, pc, meth_idx, num_args):
        args_w = frame.popitemsreverse(num_args)
        w_receiver = frame.pop()
        w_res = None
        if space.is_kind_of(w_receiver, space.w_constraint):
            with space.normal_execution():
                w_res = space.send(w_receiver, bytecode.consts_w[meth_idx], args_w)
        else:
            w_res = space.send(w_receiver, bytecode.consts_w[meth_idx], args_w)
        frame.push(w_res)

class ConstrainedVariable(W_Root):
    _immutable_fields_ = ["cell", "w_owner", "ivar", "cvar", "w_external_variable"]

    def __init__(self, space, cell=None, w_owner=None, ivar=None, cvar=None):
        self.w_external_variable = None
        self.cell = None
        self.w_owner = None
        self.ivar = None
        self.cvar = None
        self.constraint_blocks = []
        if cell:
            self.cell = cell
        elif w_owner and ivar:
            self.w_owner = w_owner
            self.ivar = ivar
        elif w_owner and cvar:
            self.w_owner = w_owner
            self.cvar = cvar
        else:
            raise RuntimeError("Invalid ConstraintVariableObject initialization")

        w_value = self.load_value(space)
        if space.respond_to(w_value, space.newsymbol("for_constraint")):
            with space.normal_execution():
                self.w_external_variable = space.send(
                    w_value,
                    space.newsymbol("for_constraint"),
                    [self.get_name(space)]
                )

    def __del__(self):
        # TODO: remove external variable from solver
        pass

    def is_solveable(self):
        return self.w_external_variable is not None

    def add_constraint_block(self, block):
        self.constraint_blocks.append(block)

    def load_value(self, space):
        if self.cell:
            return self.cell.get(space, None, 0) or space.w_nil
        elif self.ivar is not None:
            return self.w_owner.find_instance_var(space, self.ivar) or space.w_nil
        elif self.cvar is not None:
            return self.w_owner.find_class_var(space, self.cvar) or space.w_nil
        else:
            raise NotImplementedError("inconsistent constraint variable")

    def store_value(self, space, w_value):
        if self.cell:
            self.cell.set(space, None, 0, w_value)
        elif self.ivar is not None:
            self.w_owner.set_instance_var(space, self.ivar, w_value)
        elif self.cvar is not None:
            self.w_owner.set_class_var(space, self.cvar, w_value)
        else:
            raise NotImplementedError("inconsistent constraint variable")

    def get_name(self, space):
        clsname = space.getclass(self.load_value(space)).name
        if self.cell:
            return space.newstr_fromstr("local-%s" % clsname)
        elif self.ivar is not None:
            return space.newstr_fromstr("ivar-%s" % clsname)
        elif self.cvar is not None:
            return space.newstr_fromstr("cvar-%s" % clsname)
        return space.w_nil

    def suggest_value(self, space, w_value):
        assert self.w_external_variable
        with space.constraint_execution():
            space.send(self.w_external_variable, space.newsymbol("suggest_value"), [w_value])

    def set_i(self, space):
        if self.w_external_variable is not None:
            # This variable is part of the solver. Get the solvers
            # interpretation and store it
            w_value = space.send(self.w_external_variable, space.newsymbol("value"))
            if w_value != space.w_nil:
                # w_variable_value may not be the same as w_value,
                # because client code may choose to return a different
                # variables' value in #for_constraint
                w_variable_value = self.load_value(space)
                if space.respond_to(w_variable_value, space.newsymbol("assign_constraint_value")):
                    space.send(w_variable_value, space.newsymbol("assign_constraint_value"), [w_value])
                else:
                    self.store_value(space, w_value)
            return w_value
        return space.w_nil

    def get_i(self, space):
        self.set_i(space)
        return self.load_value(space)

    def recalculate_path(self, space, w_value):
        self.store_value(space, w_value)
        for block in self.constraint_blocks:
            w_constraint = block.get_constraint()
            assert w_constraint
            space.send(w_constraint, space.newsymbol("disable"))
            block.set_constraint(None)
            space.send(space.w_object, space.newsymbol("always"), block=block)
<|endoftext|>


<commit_before>from typing import List
import numpy as np
import functools
from operator import itemgetter
from federatedml.ensemble.boosting.hetero.hetero_secureboost_host import HeteroSecureBoostingTreeHost
from federatedml.param.boosting_param import HeteroFastSecureBoostParam
from federatedml.ensemble.basic_algorithms import HeteroFastDecisionTreeHost
from federatedml.ensemble.boosting.hetero import hetero_fast_secureboost_plan as plan
from federatedml.ensemble import HeteroSecureBoostingTreeGuest
from federatedml.protobuf.generated.boosting_tree_model_param_pb2 import FeatureImportanceInfo
from federatedml.util import LOGGER
from federatedml.util import consts

make_readable_feature_importance = HeteroSecureBoostingTreeGuest.make_readable_feature_importance

class HeteroFastSecureBoostingTreeHost(HeteroSecureBoostingTreeHost):

    def __init__(self):
        super(HeteroFastSecureBoostingTreeHost, self).__init__()

        self.tree_num_per_party = 1
        self.guest_depth = 0
        self.host_depth = 0
        self.work_mode = consts.MIX_TREE
        self.init_tree_plan = False
        self.tree_plan = []
        self.model_param = HeteroFastSecureBoostParam()
        self.model_name = 'HeteroFastSecureBoost'

        self.feature_importances_ = {}

    def _init_model(self, param: HeteroFastSecureBoostParam):
        super(HeteroFastSecureBoostingTreeHost, self)._init_model(param)
        self.tree_num_per_party = param.tree_num_per_party
        self.work_mode = param.work_mode
        self.guest_depth = param.guest_depth
        self.host_depth = param.host_depth

    def get_tree_plan(self, idx):

        if not self.init_tree_plan:
            tree_plan = plan.create_tree_plan(self.work_mode, k=self.tree_num_per_party, tree_num=self.boosting_round,
                                              host_list=self.component_properties.host_party_idlist,
                                              complete_secure=self.complete_secure)
            self.tree_plan += tree_plan
            self.init_tree_plan = True

        LOGGER.info('tree plan is {}'.format(self.tree_plan))
        return self.tree_plan[idx]

    def update_feature_importance(self, tree_feature_importance):
        for fid in tree_feature_importance:
            if fid not in self.feature_importances_:
                self.feature_importances_[fid] = tree_feature_importance[fid]
            else:
                self.feature_importances_[fid] += tree_feature_importance[fid]

    def check_host_number(self, tree_type):
        host_num = len(self.component_properties.host_party_idlist)
        LOGGER.info('host number is {}'.format(host_num))
        if tree_type == plan.tree_type_dict['layered_tree']:
            assert host_num == 1, 'only 1 host party is allowed in layered mode'

    def fit_a_booster(self, epoch_idx: int, booster_dim: int):

        tree_type, target_host_id = self.get_tree_plan(epoch_idx)
        self.check_host_number(tree_type)
        self.check_run_sp_opt()
        tree = HeteroFastDecisionTreeHost(tree_param=self.tree_param)
        tree.init(flowid=self.generate_flowid(epoch_idx, booster_dim),
                  valid_features=self.sample_valid_features(),
                  data_bin=self.data_bin, bin_split_points=self.bin_split_points,
                  bin_sparse_points=self.bin_sparse_points,
                  run_sprase_opt=self.run_sparse_opt,
                  data_bin_dense=self.data_bin_dense,
                  runtime_idx=self.component_properties.local_partyid,
                  goss_subsample=self.enable_goss,
                  bin_num=self.bin_num,
                  complete_secure=True if (self.complete_secure and epoch_idx == 0) else False,
                  cipher_compressing=self.cipher_compressing,
                  new_ver=self.new_ver
                  )

        tree.set_tree_work_mode(tree_type, target_host_id)
        tree.set_layered_depth(self.guest_depth, self.host_depth)
        tree.set_self_host_id(self.component_properties.local_partyid)
        tree.set_host_party_idlist(self.component_properties.host_party_idlist)
        LOGGER.debug('tree work mode is {}'.format(tree_type))
        tree.fit()
        self.update_feature_importance(tree.get_feature_importance())
        return tree

    def load_booster(self, model_meta, model_param, epoch_idx, booster_idx):

        tree = HeteroFastDecisionTreeHost(self.tree_param)
        tree.load_model(model_meta, model_param)
        tree.set_flowid(self.generate_flowid(epoch_idx, booster_idx))
        tree.set_runtime_idx(self.component_properties.local_partyid)

        tree_type, target_host_id = self.get_tree_plan(epoch_idx)

        # target_host_id and self_host_id and target_host_id are related to prediction
        tree.set_tree_work_mode(tree_type, target_host_id)
        tree.set_self_host_id(self.component_properties.local_partyid)

        if self.tree_plan[epoch_idx][0] == plan.tree_type_dict['guest_feat_only']:
            tree.use_guest_feat_only_predict_mode()

        return tree

    def generate_summary(self) -> dict:
        summary = super(HeteroFastSecureBoostingTreeHost, self).generate_summary()
        summary['feature_importance'] = make_readable_feature_importance(self.feature_name_fid_mapping,
                                                                         self.feature_importances_)
        return summary

    @staticmethod
    def traverse_host_local_trees(node_pos, sample, trees: List[HeteroFastDecisionTreeHost]):

        """
        in mix mode, a sample can reach leaf directly
        """

        for i in range(len(trees)):

            tree = trees[i]
            if len(tree.tree_node) == 0:  # this tree belongs to other party because it has no tree node
                continue
            leaf_id = tree.host_local_traverse_tree(sample, tree.tree_node, use_missing=tree.use_missing,
                                                    zero_as_missing=tree.zero_as_missing)
            node_pos[i] = leaf_id

        return node_pos

    # this func will be called by super class's predict()
    def boosting_fast_predict(self, data_inst, trees: List[HeteroFastDecisionTreeHost]):

        LOGGER.info('fast sbt running predict')

        if self.work_mode == consts.MIX_TREE:

            LOGGER.info('running mix mode predict')

            tree_num = len(trees)
            node_pos = data_inst.mapValues(lambda x: np.zeros(tree_num, dtype=np.int64))
            local_traverse_func = functools.partial(self.traverse_host_local_trees, trees=trees)
            leaf_pos = node_pos.join(data_inst, local_traverse_func)
            self.predict_transfer_inst.host_predict_data.remote(leaf_pos, idx=0, role=consts.GUEST)

        else:

            LOGGER.info('running layered mode predict')

            super(HeteroFastSecureBoostingTreeHost, self).boosting_fast_predict(data_inst, trees)

    def get_model_meta(self):

        _, model_meta = super(HeteroFastSecureBoostingTreeHost, self).get_model_meta()
        meta_name = consts.HETERO_FAST_SBT_HOST_MODEL + "Meta"
        model_meta.work_mode = self.work_mode

        return meta_name, model_meta

    def get_model_param(self):

        _, model_param = super(HeteroFastSecureBoostingTreeHost, self).get_model_param()
        param_name = consts.HETERO_FAST_SBT_HOST_MODEL + "Param"
        model_param.tree_plan.extend(plan.encode_plan(self.tree_plan))
        model_param.model_name = consts.HETERO_FAST_SBT_MIX if self.work_mode == consts.MIX_TREE else \
                                 consts.HETERO_FAST_SBT_LAYERED
        # in mix mode, host can output feature importance
        feature_importances = list(self.feature_importances_.items())
        feature_importances = sorted(feature_importances, key=itemgetter(1), reverse=True)
        feature_importance_param = []
        LOGGER.debug('host feat importance is {}'.format(feature_importances))
        for fid, importance in feature_importances:
            feature_importance_param.append(FeatureImportanceInfo(sitename=self.role,
                                                                  fid=fid,
                                                                  importance=importance.importance,
                                                                  fullname=self.feature_name_fid_mapping[fid]))
        model_param.feature_importances.extend(feature_importance_param)

        return param_name, model_param

    def set_model_meta(self, model_meta):
        super(HeteroFastSecureBoostingTreeHost, self).set_model_meta(model_meta)
        self.work_mode = model_meta.work_mode

    def set_model_param(self, model_param):
        super(HeteroFastSecureBoostingTreeHost, self).set_model_param(model_param)
        self.tree_plan = plan.decode_plan(model_param.tree_plan)<commit_msg>fix fast sbt feature importance<commit_after>from typing import List
import numpy as np
import functools
from operator import itemgetter
from federatedml.ensemble.boosting.hetero.hetero_secureboost_host import HeteroSecureBoostingTreeHost
from federatedml.param.boosting_param import HeteroFastSecureBoostParam
from federatedml.ensemble.basic_algorithms import HeteroFastDecisionTreeHost
from federatedml.ensemble.boosting.hetero import hetero_fast_secureboost_plan as plan
from federatedml.ensemble import HeteroSecureBoostingTreeGuest
from federatedml.protobuf.generated.boosting_tree_model_param_pb2 import FeatureImportanceInfo
from federatedml.ensemble.basic_algorithms.decision_tree.tree_core.feature_importance import FeatureImportance
from federatedml.util import LOGGER
from federatedml.util import consts

make_readable_feature_importance = HeteroSecureBoostingTreeGuest.make_readable_feature_importance

class HeteroFastSecureBoostingTreeHost(HeteroSecureBoostingTreeHost):

    def __init__(self):
        super(HeteroFastSecureBoostingTreeHost, self).__init__()

        self.tree_num_per_party = 1
        self.guest_depth = 0
        self.host_depth = 0
        self.work_mode = consts.MIX_TREE
        self.init_tree_plan = False
        self.tree_plan = []
        self.model_param = HeteroFastSecureBoostParam()
        self.model_name = 'HeteroFastSecureBoost'

        self.feature_importances_ = {}

    def _init_model(self, param: HeteroFastSecureBoostParam):
        super(HeteroFastSecureBoostingTreeHost, self)._init_model(param)
        self.tree_num_per_party = param.tree_num_per_party
        self.work_mode = param.work_mode
        self.guest_depth = param.guest_depth
        self.host_depth = param.host_depth

    def get_tree_plan(self, idx):

        if not self.init_tree_plan:
            tree_plan = plan.create_tree_plan(self.work_mode, k=self.tree_num_per_party, tree_num=self.boosting_round,
                                              host_list=self.component_properties.host_party_idlist,
                                              complete_secure=self.complete_secure)
            self.tree_plan += tree_plan
            self.init_tree_plan = True

        LOGGER.info('tree plan is {}'.format(self.tree_plan))
        return self.tree_plan[idx]

    def update_feature_importance(self, tree_feature_importance):
        for fid in tree_feature_importance:
            if fid not in self.feature_importances_:
                self.feature_importances_[fid] = tree_feature_importance[fid]
            else:
                self.feature_importances_[fid] += tree_feature_importance[fid]

    def load_feature_importance(self, feat_importance_param):
        param = list(feat_importance_param)
        rs_dict = {}
        for fp in param:
            key = (fp.sitename, fp.fid)
            importance = FeatureImportance()
            importance.from_protobuf(fp)
            rs_dict[key] = importance

        self.feature_importances_ = rs_dict

    def check_host_number(self, tree_type):
        host_num = len(self.component_properties.host_party_idlist)
        LOGGER.info('host number is {}'.format(host_num))
        if tree_type == plan.tree_type_dict['layered_tree']:
            assert host_num == 1, 'only 1 host party is allowed in layered mode'

    def fit_a_booster(self, epoch_idx: int, booster_dim: int):

        tree_type, target_host_id = self.get_tree_plan(epoch_idx)
        self.check_host_number(tree_type)
        self.check_run_sp_opt()
        tree = HeteroFastDecisionTreeHost(tree_param=self.tree_param)
        tree.init(flowid=self.generate_flowid(epoch_idx, booster_dim),
                  valid_features=self.sample_valid_features(),
                  data_bin=self.data_bin, bin_split_points=self.bin_split_points,
                  bin_sparse_points=self.bin_sparse_points,
                  run_sprase_opt=self.run_sparse_opt,
                  data_bin_dense=self.data_bin_dense,
                  runtime_idx=self.component_properties.local_partyid,
                  goss_subsample=self.enable_goss,
                  bin_num=self.bin_num,
                  complete_secure=True if (self.complete_secure and epoch_idx == 0) else False,
                  cipher_compressing=self.cipher_compressing,
                  new_ver=self.new_ver
                  )

        tree.set_tree_work_mode(tree_type, target_host_id)
        tree.set_layered_depth(self.guest_depth, self.host_depth)
        tree.set_self_host_id(self.component_properties.local_partyid)
        tree.set_host_party_idlist(self.component_properties.host_party_idlist)
        LOGGER.debug('tree work mode is {}'.format(tree_type))
        tree.fit()
        self.update_feature_importance(tree.get_feature_importance())
        return tree

    def load_booster(self, model_meta, model_param, epoch_idx, booster_idx):

        tree = HeteroFastDecisionTreeHost(self.tree_param)
        tree.load_model(model_meta, model_param)
        tree.set_flowid(self.generate_flowid(epoch_idx, booster_idx))
        tree.set_runtime_idx(self.component_properties.local_partyid)

        tree_type, target_host_id = self.get_tree_plan(epoch_idx)

        # target_host_id and self_host_id and target_host_id are related to prediction
        tree.set_tree_work_mode(tree_type, target_host_id)
        tree.set_self_host_id(self.component_properties.local_partyid)

        if self.tree_plan[epoch_idx][0] == plan.tree_type_dict['guest_feat_only']:
            tree.use_guest_feat_only_predict_mode()

        return tree

    def generate_summary(self) -> dict:
        summary = super(HeteroFastSecureBoostingTreeHost, self).generate_summary()
        summary['feature_importance'] = make_readable_feature_importance(self.feature_name_fid_mapping,
                                                                         self.feature_importances_)
        return summary

    @staticmethod
    def traverse_host_local_trees(node_pos, sample, trees: List[HeteroFastDecisionTreeHost]):

        """
        in mix mode, a sample can reach leaf directly
        """

        for i in range(len(trees)):

            tree = trees[i]
            if len(tree.tree_node) == 0:  # this tree belongs to other party because it has no tree node
                continue
            leaf_id = tree.host_local_traverse_tree(sample, tree.tree_node, use_missing=tree.use_missing,
                                                    zero_as_missing=tree.zero_as_missing)
            node_pos[i] = leaf_id

        return node_pos

    # this func will be called by super class's predict()
    def boosting_fast_predict(self, data_inst, trees: List[HeteroFastDecisionTreeHost]):

        LOGGER.info('fast sbt running predict')

        if self.work_mode == consts.MIX_TREE:

            LOGGER.info('running mix mode predict')

            tree_num = len(trees)
            node_pos = data_inst.mapValues(lambda x: np.zeros(tree_num, dtype=np.int64))
            local_traverse_func = functools.partial(self.traverse_host_local_trees, trees=trees)
            leaf_pos = node_pos.join(data_inst, local_traverse_func)
            self.predict_transfer_inst.host_predict_data.remote(leaf_pos, idx=0, role=consts.GUEST)

        else:

            LOGGER.info('running layered mode predict')

            super(HeteroFastSecureBoostingTreeHost, self).boosting_fast_predict(data_inst, trees)

    def get_model_meta(self):

        _, model_meta = super(HeteroFastSecureBoostingTreeHost, self).get_model_meta()
        meta_name = consts.HETERO_FAST_SBT_HOST_MODEL + "Meta"
        model_meta.work_mode = self.work_mode

        return meta_name, model_meta

    def get_model_param(self):

        _, model_param = super(HeteroFastSecureBoostingTreeHost, self).get_model_param()
        param_name = consts.HETERO_FAST_SBT_HOST_MODEL + "Param"
        model_param.tree_plan.extend(plan.encode_plan(self.tree_plan))
        model_param.model_name = consts.HETERO_FAST_SBT_MIX if self.work_mode == consts.MIX_TREE else \
                                 consts.HETERO_FAST_SBT_LAYERED
        # in mix mode, host can output feature importance
        feature_importances = list(self.feature_importances_.items())
        feature_importances = sorted(feature_importances, key=itemgetter(1), reverse=True)
        feature_importance_param = []
        LOGGER.debug('host feat importance is {}'.format(feature_importances))
        for fid, importance in feature_importances:
            feature_importance_param.append(FeatureImportanceInfo(sitename=self.role,
                                                                  fid=fid,
                                                                  importance=importance.importance,
                                                                  fullname=self.feature_name_fid_mapping[fid]))
        model_param.feature_importances.extend(feature_importance_param)

        return param_name, model_param

    def set_model_meta(self, model_meta):
        super(HeteroFastSecureBoostingTreeHost, self).set_model_meta(model_meta)
        self.work_mode = model_meta.work_mode

    def set_model_param(self, model_param):
        super(HeteroFastSecureBoostingTreeHost, self).set_model_param(model_param)
        self.tree_plan = plan.decode_plan(model_param.tree_plan)
        self.load_feature_importance(model_param.feature_importances)<|endoftext|>


<commit_before>"""
File saving routines for plots.

Currently unimplemented.  When implemented, will need to be able to save
the Plots in a PlotGroup into files on disk, choosing meaningful names
for each file.

$Id$
"""
__version__='$Revision$'

# (see comments at the end of this file)
plotsaving_classes = {}

def save_plotgroup(name,**params):
    """
    Convenience command for saving a set of plots to disk.  Examples:

      save_plotgroup("Activity")
      save_plotgroup("Orientation Preference")
      save_plotgroup("Projection",projection_name='Afferent',sheet_name='V1')

    Some plotgroups accept optional parameters, which can be passed
    like projection_name and sheet_name above.
    """
    class_ = plotsaving_classes.get(name,TemplatePlotGroupSaver)
    p = class_(name,**params)
    p.plotgroup=p.generate_plotgroup()
    p.plotgroup.update_plots(True)
    p.save_to_disk()

### Currently being written

# (any code written using these classes will have to be altered in the future)

### Examples
#
# Activity:
#./topographica -g examples/cfsom_or.ty -c "topo.sim.run(1); from topo.plotting.plotfilesaver import *; p = TemplatePlotGroupSaver('Activity');p.plotgroup=p.generate_plotgroup();p.plotgroup.update_plots(True);p.save_to_disk()"
#
# Projection:
#./topographica -g examples/cfsom_or.ty -c "from topo.plotting.plotfilesaver import *; p = CFProjectionPlotGroupSaver('Projection');p.projection_name='Afferent';p.sheet_name='V1';p.plotgroup=p.generate_plotgroup();p.plotgroup.update_plots(True);p.save_to_disk()"#
# Orientation Preference:
#./topographica -g examples/cfsom_or.ty -c "topo.sim.run(10); from topo.plotting.plotfilesaver import *; p = TemplatePlotGroupSaver('Orientation Preference');p.plotgroup=p.generate_plotgroup();p.plotgroup.update_plots(True);p.save_to_disk()"
#
# If you have an existing Plot?Saver p, you can use it again:
#  topo.sim.run(10)
#  p.plotgroup.update_plots();p.save_to_disk()

# Plan:
# - clean up
# - add cf

# PIL has ImageFont module

import topo
from topo.base.parameterizedobject import ParameterizedObject,Parameter

from plotgroup import PlotGroup,TemplatePlotGroup,ProjectionPlotGroup
from templates import plotgroup_templates

import ImageOps
import numpy

import Image
from topo.base.parameterclasses import Number

# move elsewhere, and give it some parameters etc
# Alos, maybe call something more specific than compositor, since
# it's jiust for building an array (grid) of images into one big image/
class ImageCompositor(ParameterizedObject):

    # Adapting from:
    # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/412982
    def make_contact_sheet(self,imgs,
                       (marl,mart,marr,marb),
                       padding):
        """\
        Make a contact sheet from a group of filenames:

        fnames       A list of names of the image files

        ncols        Number of columns in the contact sheet
        nrows        Number of rows in the contact sheet
        photow       The width of the photo thumbs in pixels
        photoh       The height of the photo thumbs in pixels

        marl         The left margin in pixels
        mart         The top margin in pixels
        marr         The right margin in pixels
        marl         The left margin in pixels

        padding      The padding between images in pixels

        returns a PIL image object.
        """
        # should make sure imgs is numpy array

        # CB: *** should do this in numpy without the conversion to list and back! ***
        nrows,ncols = imgs.shape
        i_widths = numpy.array([i.size[0] for i in imgs.ravel()]).reshape(nrows,ncols)
        i_heights = numpy.array([i.size[1] for i in imgs.ravel()]).reshape(nrows,ncols)

        col_widths = i_widths.max(axis=0)
        row_heights = i_heights.max(axis=1)

        marw = marl+marr
        marh = mart+ marb

        padw = (ncols-1)*padding
        padh = (nrows-1)*padding

        isize = (col_widths.sum()+marw+padw, row_heights.sum()+marh+padh)

        # Create the new image. The background doesn't have to be white
        white = (255,255,255)
        inew = Image.new('RGB',isize,white)

        # CB: should be replaced with a more numpy technique.
        for irow in range(nrows):
            for icol in range(ncols):
                # if different widths in a col, or different heights in a row,
                # each image will just go at top left defined by largest in col/row
                # (should probably be centered)
                left = marl+col_widths[0:icol].sum()+icol*padding
                right = left+i_widths[irow,icol]
                upper = mart+row_heights[0:irow].sum()+irow*padding
                lower = upper+i_heights[irow,icol]
                inew.paste(imgs[irow,icol],(left,upper,right,lower))
        return inew

class PlotGroupSaver(ParameterizedObject):

    file_format = Parameter(default="png")

    filename_prefix=Parameter(default="")
# filename_format:
#  "topo.sim.name+'.'+topo.sim.time()+'.'+current_sheet+'.'+current_plot+'.'+current_unit'"
# (Eventually we could customize this to allow each template in topo/commands/analysis.py
# to have a nice short filename format.)

    #filename_format = Parameter

    def __init__(self,plotgroup_label,**params):
        super(PlotGroupSaver,self).__init__(**params)
        self.plotgroup_label = plotgroup_label
        #self.plotgroup = self.generate_plotgroup()
        #self.plotgroup.update_plots(True)

    def generate_plotgroup(self):
	return PlotGroup([])

    def save_to_disk(self):
        # need to format e.g. time like it's done elsewhere with some leading 0s
        n = topo.sim.name
        t = topo.sim.time()
        for p,l in zip(self.plotgroup.plots,self.plotgroup.labels):
            name = "%s.%s.%s"%(n,t,l.replace('\n','.'))
            #print "outfile",name
            p.bitmap.image.save(self.filename_prefix+name+".%s"%self.file_format,self.file_format)

class TemplatePlotGroupSaver(PlotGroupSaver):

    def __init__(self,pgt_name,**params):
        self.pgt = plotgroup_templates.get(pgt_name,None)
        super(TemplatePlotGroupSaver,self).__init__(pgt_name,**params)

    def generate_plotgroup(self):
	plotgroup = TemplatePlotGroup([],self.pgt,None)
	return plotgroup

class CFProjectionPlotGroupSaver(TemplatePlotGroupSaver):

    sheet_name = Parameter(default="")
    projection_name = Parameter(default="")
    density = Number(default=10.0)

    def generate_plotgroup(self):
 	plotgroup = ProjectionPlotGroup([],self.pgt,self.sheet_name,
					self.projection_name,
                                        self.density)
        return plotgroup

    def save_to_disk(self):

        imgs = numpy.array([p.bitmap.image for p in self.plotgroup.plots]).reshape(self.plotgroup.proj_plotting_shape)

        i = ImageCompositor().make_contact_sheet(imgs, (3,3,3,3), 3)

        n = topo.sim.name
        t = topo.sim.time()

        sheet = topo.sim[self.sheet_name]
        proj = topo.sim[self.sheet_name].projections()[self.projection_name]

        name = "%s.%s.%s"%(n,t,self.projection_name)

        i.save(self.filename_prefix+name+".%s"%self.file_format,self.file_format)

# plotsaving_classes is like plotpanel_classes in tkgui/topoconsole.py;
# it allows anyone to add their own special plotsaving_classes for any
# particular PlotGroup.  By default, everything uses TemplatePlotGroupSaver
# unless that is overridden explicitly for a particular PlotGroup using
# this data structure.
plotsaving_classes['Projection'] = CFProjectionPlotGroupSaver
#plotsaving_classes['Connection Fields'] = ConnectionFieldsPlotGroupSaver
<commit_msg>Added assertion statements to catch misnamed projections/sheets. (The errors raised by the PlotGroups are too cryptic to realize what's going wrong if you specify a projection that doesn't exist for a sheet.)<commit_after>"""
File saving routines for plots.

Currently unimplemented.  When implemented, will need to be able to save
the Plots in a PlotGroup into files on disk, choosing meaningful names
for each file.

$Id$
"""
__version__='$Revision$'

# (see comments at the end of this file)
plotsaving_classes = {}

def save_plotgroup(name,**params):
    """
    Convenience command for saving a set of plots to disk.  Examples:

      save_plotgroup("Activity")
      save_plotgroup("Orientation Preference")
      save_plotgroup("Projection",projection_name='Afferent',sheet_name='V1')

    Some plotgroups accept optional parameters, which can be passed
    like projection_name and sheet_name above.
    """
    class_ = plotsaving_classes.get(name,TemplatePlotGroupSaver)
    p = class_(name,**params)
    p.plotgroup=p.generate_plotgroup()
    p.plotgroup.update_plots(True)
    p.save_to_disk()

### Currently being written

# (any code written using these classes will have to be altered in the future)

### Examples
#
# Activity:
#./topographica -g examples/cfsom_or.ty -c "topo.sim.run(1); from topo.plotting.plotfilesaver import *; p = TemplatePlotGroupSaver('Activity');p.plotgroup=p.generate_plotgroup();p.plotgroup.update_plots(True);p.save_to_disk()"
#
# Projection:
#./topographica -g examples/cfsom_or.ty -c "from topo.plotting.plotfilesaver import *; p = CFProjectionPlotGroupSaver('Projection');p.projection_name='Afferent';p.sheet_name='V1';p.plotgroup=p.generate_plotgroup();p.plotgroup.update_plots(True);p.save_to_disk()"#
# Orientation Preference:
#./topographica -g examples/cfsom_or.ty -c "topo.sim.run(10); from topo.plotting.plotfilesaver import *; p = TemplatePlotGroupSaver('Orientation Preference');p.plotgroup=p.generate_plotgroup();p.plotgroup.update_plots(True);p.save_to_disk()"
#
# If you have an existing Plot?Saver p, you can use it again:
#  topo.sim.run(10)
#  p.plotgroup.update_plots();p.save_to_disk()

# Plan:
# - clean up
# - add cf

# PIL has ImageFont module

import topo
from topo.base.parameterizedobject import ParameterizedObject,Parameter

from plotgroup import PlotGroup,TemplatePlotGroup,ProjectionPlotGroup
from templates import plotgroup_templates

import ImageOps
import numpy

import Image
from topo.base.parameterclasses import Number

# move elsewhere, and give it some parameters etc
# Alos, maybe call something more specific than compositor, since
# it's jiust for building an array (grid) of images into one big image/
class ImageCompositor(ParameterizedObject):

    # Adapting from:
    # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/412982
    def make_contact_sheet(self,imgs,
                       (marl,mart,marr,marb),
                       padding):
        """\
        Make a contact sheet from a group of filenames:

        fnames       A list of names of the image files

        ncols        Number of columns in the contact sheet
        nrows        Number of rows in the contact sheet
        photow       The width of the photo thumbs in pixels
        photoh       The height of the photo thumbs in pixels

        marl         The left margin in pixels
        mart         The top margin in pixels
        marr         The right margin in pixels
        marl         The left margin in pixels

        padding      The padding between images in pixels

        returns a PIL image object.
        """
        # should make sure imgs is numpy array

        # CB: *** should do this in numpy without the conversion to list and back! ***
        nrows,ncols = imgs.shape
        i_widths = numpy.array([i.size[0] for i in imgs.ravel()]).reshape(nrows,ncols)
        i_heights = numpy.array([i.size[1] for i in imgs.ravel()]).reshape(nrows,ncols)

        col_widths = i_widths.max(axis=0)
        row_heights = i_heights.max(axis=1)

        marw = marl+marr
        marh = mart+ marb

        padw = (ncols-1)*padding
        padh = (nrows-1)*padding

        isize = (col_widths.sum()+marw+padw, row_heights.sum()+marh+padh)

        # Create the new image. The background doesn't have to be white
        white = (255,255,255)
        inew = Image.new('RGB',isize,white)

        # CB: should be replaced with a more numpy technique.
        for irow in range(nrows):
            for icol in range(ncols):
                # if different widths in a col, or different heights in a row,
                # each image will just go at top left defined by largest in col/row
                # (should probably be centered)
                left = marl+col_widths[0:icol].sum()+icol*padding
                right = left+i_widths[irow,icol]
                upper = mart+row_heights[0:irow].sum()+irow*padding
                lower = upper+i_heights[irow,icol]
                inew.paste(imgs[irow,icol],(left,upper,right,lower))
        return inew

class PlotGroupSaver(ParameterizedObject):

    file_format = Parameter(default="png")

    filename_prefix=Parameter(default="")
# filename_format:
#  "topo.sim.name+'.'+topo.sim.time()+'.'+current_sheet+'.'+current_plot+'.'+current_unit'"
# (Eventually we could customize this to allow each template in topo/commands/analysis.py
# to have a nice short filename format.)

    #filename_format = Parameter

    def __init__(self,plotgroup_label,**params):
        super(PlotGroupSaver,self).__init__(**params)
        self.plotgroup_label = plotgroup_label
        #self.plotgroup = self.generate_plotgroup()
        #self.plotgroup.update_plots(True)

    def generate_plotgroup(self):
	return PlotGroup([])

    def save_to_disk(self):
        # need to format e.g. time like it's done elsewhere with some leading 0s
        n = topo.sim.name
        t = topo.sim.time()
        for p,l in zip(self.plotgroup.plots,self.plotgroup.labels):
            name = "%s.%s.%s"%(n,t,l.replace('\n','.'))
            #print "outfile",name
            p.bitmap.image.save(self.filename_prefix+name+".%s"%self.file_format,self.file_format)

class TemplatePlotGroupSaver(PlotGroupSaver):

    def __init__(self,pgt_name,**params):
        self.pgt = plotgroup_templates.get(pgt_name,None)
        super(TemplatePlotGroupSaver,self).__init__(pgt_name,**params)

    def generate_plotgroup(self):
	plotgroup = TemplatePlotGroup([],self.pgt,None)
	return plotgroup

class CFProjectionPlotGroupSaver(TemplatePlotGroupSaver):

    sheet_name = Parameter(default="")
    projection_name = Parameter(default="")
    density = Number(default=10.0)

    def generate_plotgroup(self):

        ## CEBHACKALERT: the PlotGroups should check this stuff
        assert self.sheet_name in topo.sim.objects(), "no sheet %s"%self.sheet_name
        assert self.projection_name in topo.sim[self.sheet_name].projections().keys(), "no proj %s for %s"%(self.projection_name,self.sheet_name)

 	plotgroup = ProjectionPlotGroup([],self.pgt,self.sheet_name,
					self.projection_name,
                                        self.density)
        return plotgroup

    def save_to_disk(self):

        imgs = numpy.array([p.bitmap.image for p in self.plotgroup.plots]).reshape(self.plotgroup.proj_plotting_shape)

        i = ImageCompositor().make_contact_sheet(imgs, (3,3,3,3), 3)

        n = topo.sim.name
        t = topo.sim.time()

        sheet = topo.sim[self.sheet_name]
        proj = topo.sim[self.sheet_name].projections()[self.projection_name]

        name = "%s.%s.%s"%(n,t,self.projection_name)

        i.save(self.filename_prefix+name+".%s"%self.file_format,self.file_format)

# plotsaving_classes is like plotpanel_classes in tkgui/topoconsole.py;
# it allows anyone to add their own special plotsaving_classes for any
# particular PlotGroup.  By default, everything uses TemplatePlotGroupSaver
# unless that is overridden explicitly for a particular PlotGroup using
# this data structure.
plotsaving_classes['Projection'] = CFProjectionPlotGroupSaver
#plotsaving_classes['Connection Fields'] = ConnectionFieldsPlotGroupSaver
<|endoftext|>


<commit_before>"""The models subpackage contains definitions for the following model
architectures:

-  `AlexNet`_
-  `VGG`_
-  `ResNet`_
-  `SqueezeNet`_
-  `DenseNet`_

You can construct a model with random weights by calling its constructor:

.. code:: python

    import torchvision.models as models
    resnet18 = models.resnet18()
    alexnet = models.alexnet()
    squeezenet = models.squeezenet1_0()
    densenet = models.densenet_161()

We provide pre-trained models for the ResNet variants and AlexNet, using the
PyTorch :mod:`torch.utils.model_zoo`. These can  constructed by passing
``pretrained=True``:

.. code:: python

    import torchvision.models as models
    resnet18 = models.resnet18(pretrained=True)
    alexnet = models.alexnet(pretrained=True)

All pre-trained models expect input images normalized in the same way,
i.e. mini-batches of 3-channel RGB images of shape (3 x H x W),
where H and W are expected to be atleast 224.
The images have to be loaded in to a range of [0, 1] and then normalized
using ``mean = [0.485, 0.456, 0.406]`` and ``std = [0.229, 0.224, 0.225]``.
You can use the following transform to normalize::

    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                     std=[0.229, 0.224, 0.225])

An example of such normalization can be found in the imagenet example
`here <https://github.com/pytorch/examples/blob/42e5b996718797e45c46a25c55b031e6768f8440/imagenet/main.py#L89-L101>`_

ImageNet 1-crop error rates (224x224)

================================  =============   =============
Network                           Top-1 error     Top-5 error
================================  =============   =============
ResNet-18                         30.24           10.92
ResNet-34                         26.70           8.58
ResNet-50                         23.85           7.13
ResNet-101                        22.63           6.44
ResNet-152                        21.69           5.94
Inception v3                      22.55           6.44
AlexNet                           43.45           20.91
VGG-11                            30.98           11.37
VGG-13                            30.07           10.75
VGG-16                            28.41           9.62
VGG-19                            27.62           9.12
VGG-11 with batch normalization   29.62           10.19
VGG-13 with batch normalization   28.45           9.63
VGG-16 with batch normalization   26.63           8.50
VGG-19 with batch normalization   25.76           8.15
SqueezeNet 1.0                    41.90           19.58
SqueezeNet 1.1                    41.81           19.38
Densenet-121                      25.35           7.83
Densenet-169                      24.00           7.00
Densenet-201                      22.80           6.43
Densenet-161                      22.35           6.20
================================  =============   =============

.. _AlexNet: https://arxiv.org/abs/1404.5997
.. _VGG: https://arxiv.org/abs/1409.1556
.. _ResNet: https://arxiv.org/abs/1512.03385
.. _SqueezeNet: https://arxiv.org/abs/1602.07360
.. _DenseNet: https://arxiv.org/abs/1608.06993
"""

from .alexnet import *
from .resnet import *
from .vgg import *
from .squeezenet import *
from .inception import *
from .densenet import *
<commit_msg>DOC add all models to torchvision.models docstring<commit_after>"""The models subpackage contains definitions for the following model
architectures:

-  `AlexNet`_
-  `VGG`_
-  `ResNet`_
-  `SqueezeNet`_
-  `DenseNet`_
-  `Inception`_ v3

You can construct a model with random weights by calling its constructor:

.. code:: python

    import torchvision.models as models
    resnet18 = models.resnet18()
    alexnet = models.alexnet()
    vgg16 = models.vgg16()
    squeezenet = models.squeezenet1_0()
    densenet = models.densenet_161()
    inception = models.inception_v3()

We provide pre-trained models, using the PyTorch :mod:`torch.utils.model_zoo`.
These can be constructed by passing ``pretrained=True``:

.. code:: python

    import torchvision.models as models
    resnet18 = models.resnet18(pretrained=True)
    alexnet = models.alexnet(pretrained=True)
    squeezenet = models.squeezenet1_0(pretrained=True)
    vgg16 = models.vgg16(pretrained=True)
    densenet = models.densenet_161(pretrained=True)
    inception = models.inception_v3(pretrained=True)

All pre-trained models expect input images normalized in the same way,
i.e. mini-batches of 3-channel RGB images of shape (3 x H x W),
where H and W are expected to be at least 224.
The images have to be loaded in to a range of [0, 1] and then normalized
using ``mean = [0.485, 0.456, 0.406]`` and ``std = [0.229, 0.224, 0.225]``.
You can use the following transform to normalize::

    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                     std=[0.229, 0.224, 0.225])

An example of such normalization can be found in the imagenet example
`here <https://github.com/pytorch/examples/blob/42e5b996718797e45c46a25c55b031e6768f8440/imagenet/main.py#L89-L101>`_

ImageNet 1-crop error rates (224x224)

================================  =============   =============
Network                           Top-1 error     Top-5 error
================================  =============   =============
ResNet-18                         30.24           10.92
ResNet-34                         26.70           8.58
ResNet-50                         23.85           7.13
ResNet-101                        22.63           6.44
ResNet-152                        21.69           5.94
Inception v3                      22.55           6.44
AlexNet                           43.45           20.91
VGG-11                            30.98           11.37
VGG-13                            30.07           10.75
VGG-16                            28.41           9.62
VGG-19                            27.62           9.12
VGG-11 with batch normalization   29.62           10.19
VGG-13 with batch normalization   28.45           9.63
VGG-16 with batch normalization   26.63           8.50
VGG-19 with batch normalization   25.76           8.15
SqueezeNet 1.0                    41.90           19.58
SqueezeNet 1.1                    41.81           19.38
Densenet-121                      25.35           7.83
Densenet-169                      24.00           7.00
Densenet-201                      22.80           6.43
Densenet-161                      22.35           6.20
================================  =============   =============

.. _AlexNet: https://arxiv.org/abs/1404.5997
.. _VGG: https://arxiv.org/abs/1409.1556
.. _ResNet: https://arxiv.org/abs/1512.03385
.. _SqueezeNet: https://arxiv.org/abs/1602.07360
.. _DenseNet: https://arxiv.org/abs/1608.06993
.. _Inception: https://arxiv.org/abs/1512.00567
"""

from .alexnet import *
from .resnet import *
from .vgg import *
from .squeezenet import *
from .inception import *
from .densenet import *
<|endoftext|>


<commit_before>from django.conf import settings
from django.shortcuts import render
from django.utils import translation
from django.utils.translation import ugettext_lazy as _

from requests import HTTPError

from temba_client.base import TembaAPIError, TembaConnectionError

from dash.orgs import middleware as dash_middleware

class TracproOrgMiddleware(dash_middleware.SetOrgMiddleware):

    def user_has_set_language(self, request):
        """Whether user has requested a specific language via site footer."""
        return any((
            request.session.get(translation.LANGUAGE_SESSION_KEY),
            request.COOKIES.get(settings.LANGUAGE_COOKIE_NAME),
        ))

    def set_language(self, request, org):
        """
        Use the org's default language unless user has selected a specific
        language.
        """
        if org and not self.user_has_set_language(request):
            lang = org.language
            translation.activate(lang)
            request.LANGUAGE_CODE = lang

class HandleTembaAPIError(object):
    """ Catch all Temba exception errors """

    def process_exception(self, request, exception):
        rapidpro_connection_error_string = _(
            "RapidPro appears to be down right now. "
            "Please try again later.")

        if isinstance(exception, TembaAPIError):
            if isinstance(exception.caused_by, HTTPError):
                response = exception.caused_by.response
                if response is not None:
                    if response.status_code == 403:
                        return self.rapidpro_error(
                            request,
                            _("Org does not have a valid API Key. "
                              "Please edit the org through Site Manage or contact your administrator.")
                            )

                    elif response.status_code >= 500:
                        return self.rapidpro_error(request, rapidpro_connection_error_string)

        elif isinstance(exception, TembaConnectionError):
            return self.rapidpro_error(request, rapidpro_connection_error_string)

        return None

    def rapidpro_error(self, request, message):
        return render(request, "rapidpro_error.html", {"error_message": message}, status=400)
<commit_msg>Allow default behavior if org.language is not set<commit_after>from django.conf import settings
from django.shortcuts import render
from django.utils import translation
from django.utils.translation import ugettext_lazy as _

from requests import HTTPError

from temba_client.base import TembaAPIError, TembaConnectionError

from dash.orgs import middleware as dash_middleware

class TracproOrgMiddleware(dash_middleware.SetOrgMiddleware):

    def user_has_set_language(self, request):
        """Whether user has requested a specific language via site footer."""
        return any((
            request.session.get(translation.LANGUAGE_SESSION_KEY),
            request.COOKIES.get(settings.LANGUAGE_COOKIE_NAME),
        ))

    def set_language(self, request, org):
        """
        Use the org's default language unless user has selected a specific
        language.
        """
        if org and org.language and not self.user_has_set_language(request):
            lang = org.language
            translation.activate(lang)
            request.LANGUAGE_CODE = lang

class HandleTembaAPIError(object):
    """ Catch all Temba exception errors """

    def process_exception(self, request, exception):
        rapidpro_connection_error_string = _(
            "RapidPro appears to be down right now. "
            "Please try again later.")

        if isinstance(exception, TembaAPIError):
            if isinstance(exception.caused_by, HTTPError):
                response = exception.caused_by.response
                if response is not None:
                    if response.status_code == 403:
                        return self.rapidpro_error(
                            request,
                            _("Org does not have a valid API Key. "
                              "Please edit the org through Site Manage or contact your administrator.")
                            )

                    elif response.status_code >= 500:
                        return self.rapidpro_error(request, rapidpro_connection_error_string)

        elif isinstance(exception, TembaConnectionError):
            return self.rapidpro_error(request, rapidpro_connection_error_string)

        return None

    def rapidpro_error(self, request, message):
        return render(request, "rapidpro_error.html", {"error_message": message}, status=400)
<|endoftext|>


<commit_before>import json
import logging
import time

from django.conf import settings
from django.http import HttpResponse
from django.shortcuts import render, redirect, render_to_response
from django.template import RequestContext

logger = logging.getLogger(__name__)

def _create_signature(secret, *parts):
    import hmac, hashlib
    hash = hmac.new(secret, digestmod=hashlib.sha1)
    for part in parts:
        hash.update(str(part))
    return hash.hexdigest()

def web_shell(request):
    response = HttpResponse()

    username = request.GET['upn']

    secret = settings.GATE_ONE_API_SECRET
    authobj = {
        'api_key': settings.GATE_ONE_API_KEY,
        'upn': username,
        'timestamp': str(int(time.time() * 1000)),
    }

    sig = _create_signature(secret, authobj['api_key'],
            authobj['upn'], authobj['timestamp'])

    # GATE_ONE_API_AUTH_SETTINGS
    template_params = {
        'WEB_SH_BASE_URL' : 'https://atmo-proxy.iplantcollaborative.org:8443/',
        'WEB_SH_JS_FILE': 'static/gateone.js',
        'GATE_ONE_API_KEY': authobj['api_key'],
        'USERNAME': authobj['upn'],
        'GEN_TIMESTAMP': authobj['timestamp'],
        'GATE_ONE_SIGNATURE': sig,
        'SIGNATURE_METHOD': 'HMAC-SHA1',
        'GATE_ONE_API_VERSION': '1.0'
    }

    if "location" in request.GET:
        template_params['gate_one_location'] = request.GET['location']

    response = render_to_response(
            'web_shell.html',
            template_params,
            context_instance=RequestContext(request)
    )
    return response

<commit_msg>Corrected variable passing issue with web-shell.<commit_after>import json
import logging
import time

from django.conf import settings
from django.http import HttpResponse
from django.shortcuts import render, redirect, render_to_response
from django.template import RequestContext

logger = logging.getLogger(__name__)

def _create_signature(secret, *parts):
    import hmac, hashlib
    hash = hmac.new(secret, digestmod=hashlib.sha1)
    for part in parts:
        hash.update(str(part))
    return hash.hexdigest()

def web_shell(request):
    response = HttpResponse()

    username = request.GET['upn']

    secret = settings.GATE_ONE_API_SECRET
    authobj = {
        'api_key': settings.GATE_ONE_API_KEY,
        'upn': username,
        'timestamp': str(int(time.time() * 1000)),
    }

    sig = _create_signature(secret, authobj['api_key'],
            authobj['upn'], authobj['timestamp'])

    # GATE_ONE_API_AUTH_SETTINGS
    template_params = {
        'WEB_SH_BASE_URL' : settings.WEB_SH_BASE_URL,
        'WEB_SH_JS_FILE': 'static/gateone.js',
        'GATE_ONE_API_KEY': authobj['api_key'],
        'USERNAME': authobj['upn'],
        'GEN_TIMESTAMP': authobj['timestamp'],
        'GATE_ONE_SIGNATURE': sig,
        'SIGNATURE_METHOD': 'HMAC-SHA1',
        'GATE_ONE_API_VERSION': '1.0'
    }

    if "location" in request.GET:
        template_params['gate_one_location'] = request.GET['location']

    response = render_to_response(
            'web_shell.html',
            template_params,
            context_instance=RequestContext(request)
    )
    return response

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Script to mass-rename subtitles"""

from __future__ import print_function
import os
import glob
import itertools
import re

NAME_PATTERNS = [
    re.compile(r'.*(?P<season>\d)x(?P<episode>\d\d).*', re.IGNORECASE),
    re.compile(r'.*s(?P<season>\d\d)e(?P<episode>\d\d).*', re.IGNORECASE)
]

SUB_EXTENSIONS = ['sub', 'srt']

VERBOSE = False

def replace_extension(filename, ext):
    ext = ext[1:] if ext[0] == '.' else ext
    basename, _ = os.path.splitext(filename)
    return '%s.%s' % (basename, ext)

def get_file_info(filename):
    for pattern in NAME_PATTERNS:
        match = pattern.match(filename)
        if match:
            return {k: int(v, base=10)
                    for k, v in match.groupdict().items()}

def find_all_subs(dirpath):
    return itertools.chain(*[glob.glob('%s/*.%s' % (dirpath, ext))
                             for ext in SUB_EXTENSIONS])

def find_sub_for_video(videofile, file_info):
    video_dir = os.path.dirname(os.path.abspath(videofile))
    for subfile in find_all_subs(video_dir):
        sub_info = get_file_info(os.path.basename(subfile))
        if sub_info == file_info:
            return subfile

def find_existing_subtitle_for_video(videofile):
    for sub_ext in SUB_EXTENSIONS:
        sub_file = replace_extension(videofile, sub_ext)
        if os.path.exists(sub_file):
            return sub_file

def debug(mesg):
    if VERBOSE:
        print(mesg)

def find_and_rename_subtitle_for(videofile, skip_rename):
    subtitle = find_existing_subtitle_for_video(videofile)
    if subtitle:
        debug('{0} already exists, skipping...'.format(subtitle))
        return

    file_info = get_file_info(videofile)
    if not file_info:
        debug("Could not find file info for file:  %s" % videofile)
        return

    subtitle = find_sub_for_video(videofile, file_info)
    if subtitle:
        _, sub_ext = os.path.splitext(subtitle)
        newname = replace_extension(videofile, sub_ext)

        if not skip_rename:
            os.rename(subtitle, newname)

        print('{oldname} -> {newname}'.format(oldname=os.path.basename(subtitle),
                                              newname=os.path.basename(newname)))
    else:
        debug("Could not found subs for %s" % videofile)

def run(args):
    global VERBOSE
    if args.verbose:
        VERBOSE = True

    for videofile in args.video_files:
        find_and_rename_subtitle_for(videofile, skip_rename=args.skip_rename)

def parse_args():
    import argparse
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('video_files', nargs='+',
                        help='List of video files to seek subtitles for')
    parser.add_argument('-n', '--skip-rename', action='store_true',
                        help='Just print what it will do, does NOT rename anything')
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='Print verbose information')
    return parser.parse_args()

if '__main__' == __name__:
    run(parse_args())
<commit_msg>fix for python 2.6<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Script to mass-rename subtitles"""

from __future__ import print_function
import os
import glob
import itertools
import re

NAME_PATTERNS = [
    re.compile(r'.*(?P<season>\d)x(?P<episode>\d\d).*', re.IGNORECASE),
    re.compile(r'.*s(?P<season>\d\d)e(?P<episode>\d\d).*', re.IGNORECASE)
]

SUB_EXTENSIONS = ['sub', 'srt']

VERBOSE = False

def replace_extension(filename, ext):
    ext = ext[1:] if ext[0] == '.' else ext
    basename, _ = os.path.splitext(filename)
    return '%s.%s' % (basename, ext)

def get_file_info(filename):
    for pattern in NAME_PATTERNS:
        match = pattern.match(filename)
        if match:
            return dict((k, int(v, base=10))
                        for k, v in match.groupdict().items())

def find_all_subs(dirpath):
    return itertools.chain(*[glob.glob('%s/*.%s' % (dirpath, ext))
                             for ext in SUB_EXTENSIONS])

def find_sub_for_video(videofile, file_info):
    video_dir = os.path.dirname(os.path.abspath(videofile))
    for subfile in find_all_subs(video_dir):
        sub_info = get_file_info(os.path.basename(subfile))
        if sub_info == file_info:
            return subfile

def find_existing_subtitle_for_video(videofile):
    for sub_ext in SUB_EXTENSIONS:
        sub_file = replace_extension(videofile, sub_ext)
        if os.path.exists(sub_file):
            return sub_file

def debug(mesg):
    if VERBOSE:
        print(mesg)

def find_and_rename_subtitle_for(videofile, skip_rename):
    subtitle = find_existing_subtitle_for_video(videofile)
    if subtitle:
        debug('{0} already exists, skipping...'.format(subtitle))
        return

    file_info = get_file_info(videofile)
    if not file_info:
        debug("Could not find file info for file:  %s" % videofile)
        return

    subtitle = find_sub_for_video(videofile, file_info)
    if subtitle:
        _, sub_ext = os.path.splitext(subtitle)
        newname = replace_extension(videofile, sub_ext)

        if not skip_rename:
            os.rename(subtitle, newname)

        print('{oldname} -> {newname}'.format(oldname=os.path.basename(subtitle),
                                              newname=os.path.basename(newname)))
    else:
        debug("Could not found subs for %s" % videofile)

def run(args):
    global VERBOSE
    if args.verbose:
        VERBOSE = True

    for videofile in args.video_files:
        find_and_rename_subtitle_for(videofile, skip_rename=args.skip_rename)

def parse_args():
    import argparse
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('video_files', nargs='+',
                        help='List of video files to seek subtitles for')
    parser.add_argument('-n', '--skip-rename', action='store_true',
                        help='Just print what it will do, does NOT rename anything')
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='Print verbose information')
    return parser.parse_args()

if '__main__' == __name__:
    run(parse_args())
<|endoftext|>


<commit_before>import os
import time
import argparse
import shutil
import subprocess
import warnings
import sys

import numpy as np
import pandas as pd

from src import SPclean
import src.Paths as PATH

def parser():
  '''
  Command-line options
  '''

  parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,description="The program creates a DataBase of the events in the observation.")
  parser.add_argument('id_obs', help='Observation ID')
  parser.add_argument('-conf', help="Run over confirmation observations.",action='store_true')
  parser.add_argument('-load_DB', help="Load events from a DataBase instead of text files.",action='store_true')
  parser.add_argument('--debug', help="Debug mode: shows all the warnings.",action='store_true')
  args = parser.parse_args()

  return args

def main(PATH):
  #warnings.filterwarnings('error', category=FutureWarning)

  args = parser()
  if args.conf:
    PATH.OBS_FOLDER = os.path.join(PATH.OBS_FOLDER, 'confirmations')
    PATH.RAW_FOLDER = os.path.join(PATH.RAW_FOLDER, 'confirmations')

  PATH.WRK_FOLDER = os.path.join(PATH.WRK_FOLDER, args.id_obs)
  PATH.TMP_FOLDER = os.path.join(PATH.TMP_FOLDER, args.id_obs)
  PATH.OBS_FOLDER = os.path.join(PATH.OBS_FOLDER, args.id_obs)

  if os.path.isdir(PATH.WRK_FOLDER): shutil.rmtree(PATH.WRK_FOLDER)
  os.makedirs(PATH.WRK_FOLDER)
  os.makedirs(os.path.join(PATH.WRK_FOLDER, 'sp'))
  os.makedirs(os.path.join(PATH.WRK_FOLDER, 'sp/candidates'))
  if os.path.isdir(PATH.TMP_FOLDER): shutil.rmtree(PATH.TMP_FOLDER)
  os.makedirs(PATH.TMP_FOLDER)
  if os.path.isdir(os.path.join(PATH.OBS_FOLDER, 'sp')): shutil.rmtree(os.path.join(PATH.OBS_FOLDER, 'sp'))

  stdout = open(os.path.join(PATH.WRK_FOLDER, 'sp/log.txt'), 'w')
  sys.stdout = stdout
  stderr = open(os.path.join(PATH.WRK_FOLDER, 'sp/ERROR_log.txt'), 'w')
  sys.stderr = stderr

  scriptFolder = os.path.dirname(os.path.realpath(__file__))
  git_folder = os.path.join(scriptFolder, '.git')
  vers = subprocess.check_output(['git','--git-dir',git_folder,'describe','--tags','--abbrev=0','--always']).strip()
  print "L-SpS version used: {}".format(vers)
  args.vers = vers

  time0 = time.time()
  print "The DataBase is being created."

  try:
    SPclean.main(args)
    print "The DataBase has been created."
    print "Time spent: {:.2f} s.".format(time.time() - time0)

  finally:
    shutil.copytree(os.path.join(PATH.WRK_FOLDER, 'sp'), os.path.join(PATH.OBS_FOLDER, 'sp'))
    shutil.rmtree(PATH.WRK_FOLDER)
    shutil.rmtree(PATH.TMP_FOLDER)
    stdout.close()
    stderr.close()

  return

if __name__ == '__main__':
  main(PATH)
<commit_msg>Changes for Pipeline v2<commit_after>import os
import time
import argparse
import shutil
import subprocess
import warnings
import sys

import numpy as np
import pandas as pd

from src import SPclean
import src.Paths as PATH

def parser():
  '''
  Command-line options
  '''

  parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,description="The program creates a DataBase of the events in the observation.")
  parser.add_argument('id_obs', help='Observation ID')
  parser.add_argument('-conf', help="Run over confirmation observations.",action='store_true')
  parser.add_argument('-load_DB', help="Load events from a DataBase instead of text files.",action='store_true')
  parser.add_argument('--debug', help="Debug mode: shows all the warnings.",action='store_true')
  args = parser.parse_args()

  return args

def main(PATH):
  #warnings.filterwarnings('error', category=FutureWarning)

  args = parser()
  if args.conf:
    PATH.OBS_FOLDER = os.path.join(PATH.OBS_FOLDER, 'confirmations')
    PATH.RAW_FOLDER = os.path.join(PATH.RAW_FOLDER, 'confirmations')

  PATH.WRK_FOLDER = os.path.join(PATH.WRK_FOLDER, args.id_obs)
  PATH.TMP_FOLDER = os.path.join(PATH.TMP_FOLDER, args.id_obs)
  PATH.OBS_FOLDER = os.path.join(PATH.OBS_FOLDER, args.id_obs)

  if os.path.isdir(PATH.WRK_FOLDER): shutil.rmtree(PATH.WRK_FOLDER)
  os.makedirs(PATH.WRK_FOLDER)
  os.makedirs(os.path.join(PATH.WRK_FOLDER, 'sp'))
  os.makedirs(os.path.join(PATH.WRK_FOLDER, 'sp/candidates'))
  if os.path.isdir(PATH.TMP_FOLDER): shutil.rmtree(PATH.TMP_FOLDER)
  os.makedirs(PATH.TMP_FOLDER)
  if os.path.isdir(os.path.join(PATH.OBS_FOLDER, 'sp')): shutil.rmtree(os.path.join(PATH.OBS_FOLDER, 'sp'))

  stdout = open(os.path.join(PATH.WRK_FOLDER, 'sp/log.txt'), 'w')
  sys.stdout = stdout

  scriptFolder = os.path.dirname(os.path.realpath(__file__))
  git_folder = os.path.join(scriptFolder, '.git')
  vers = subprocess.check_output(['git','--git-dir',git_folder,'describe','--tags','--abbrev=0','--always']).strip()
  print "L-SpS version used: {}".format(vers)
  args.vers = vers

  time0 = time.time()
  print "The DataBase is being created."

  try:
    SPclean.main(args)
    print "The DataBase has been created."
    print "Time spent: {:.2f} s.".format(time.time() - time0)

  except Exception as e:
    with open(os.path.join(PATH.WRK_FOLDER, 'sp/ERROR_log.txt'), 'w') as stderr:
      stderr.write(str(e))

  finally:
    shutil.copytree(os.path.join(PATH.WRK_FOLDER, 'sp'), os.path.join(PATH.OBS_FOLDER, 'sp'))
    shutil.rmtree(PATH.WRK_FOLDER)
    shutil.rmtree(PATH.TMP_FOLDER)
    stdout.close()
    stderr.close()

  return

if __name__ == '__main__':
  main(PATH)
<|endoftext|>