<commit_before>'''
RTNetlink: network setup
========================

RTNL is a netlink protocol, used to get and set information
about different network objects -- addresses, routes, interfaces
etc.

RTNL protocol-specific data in messages depends on the object
type. E.g., complete packet with the interface address information::

    nlmsg header:
        + uint32 length
        + uint16 type
        + uint16 flags
        + uint32 sequence number
        + uint32 pid
    ifaddrmsg structure:
        + unsigned char ifa_family
        + unsigned char ifa_prefixlen
        + unsigned char ifa_flags
        + unsigned char ifa_scope
        + uint32 ifa_index
    [ optional NLA tree ]

NLA for this kind of packets can be of type IFA_ADDRESS, IFA_LOCAL
etc. -- please refer to the corresponding source.

Other objects types require different structures, sometimes really
complex. All these structures are described in sources.

---------------------------

Module contents:

'''
from pyroute2.common import map_namespace

#  RTnetlink multicast groups
RTNLGRP_NONE = 0x0
RTNLGRP_LINK = 0x1
RTNLGRP_NOTIFY = 0x2
RTNLGRP_NEIGH = 0x4
RTNLGRP_TC = 0x8
RTNLGRP_IPV4_IFADDR = 0x10
RTNLGRP_IPV4_MROUTE = 0x20
RTNLGRP_IPV4_ROUTE = 0x40
RTNLGRP_IPV4_RULE = 0x80
RTNLGRP_IPV6_IFADDR = 0x100
RTNLGRP_IPV6_MROUTE = 0x200
RTNLGRP_IPV6_ROUTE = 0x400
RTNLGRP_IPV6_IFINFO = 0x800
RTNLGRP_DECnet_IFADDR = 0x1000
RTNLGRP_NOP2 = 0x2000
RTNLGRP_DECnet_ROUTE = 0x4000
RTNLGRP_DECnet_RULE = 0x8000
RTNLGRP_NOP4 = 0x10000
RTNLGRP_IPV6_PREFIX = 0x20000
RTNLGRP_IPV6_RULE = 0x40000
RTNLGRP_MPLS_ROUTE = 0x4000000

# Types of messages
# RTM_BASE = 16
RTM_NEWLINK = 16
RTM_DELLINK = 17
RTM_GETLINK = 18
RTM_SETLINK = 19
RTM_NEWADDR = 20
RTM_DELADDR = 21
RTM_GETADDR = 22
RTM_NEWROUTE = 24
RTM_DELROUTE = 25
RTM_GETROUTE = 26
RTM_NEWNEIGH = 28
RTM_DELNEIGH = 29
RTM_GETNEIGH = 30
RTM_NEWRULE = 32
RTM_DELRULE = 33
RTM_GETRULE = 34
RTM_NEWQDISC = 36
RTM_DELQDISC = 37
RTM_GETQDISC = 38
RTM_NEWTCLASS = 40
RTM_DELTCLASS = 41
RTM_GETTCLASS = 42
RTM_NEWTFILTER = 44
RTM_DELTFILTER = 45
RTM_GETTFILTER = 46
RTM_NEWACTION = 48
RTM_DELACTION = 49
RTM_GETACTION = 50
RTM_NEWPREFIX = 52
RTM_GETMULTICAST = 58
RTM_GETANYCAST = 62
RTM_NEWNEIGHTBL = 64
RTM_GETNEIGHTBL = 66
RTM_SETNEIGHTBL = 67
# custom message types
RTM_GETBRIDGE = 88
RTM_SETBRIDGE = 89
RTM_GETBOND = 90
RTM_SETBOND = 91
(RTM_NAMES, RTM_VALUES) = map_namespace('RTM', globals())

TC_H_INGRESS = 0xfffffff1
TC_H_CLSACT = TC_H_INGRESS
TC_H_ROOT = 0xffffffff

RTNL_GROUPS = RTNLGRP_IPV4_IFADDR |\
    RTNLGRP_IPV6_IFADDR |\
    RTNLGRP_IPV4_ROUTE |\
    RTNLGRP_IPV6_ROUTE |\
    RTNLGRP_NEIGH |\
    RTNLGRP_LINK |\
    RTNLGRP_TC |\
    RTNLGRP_MPLS_ROUTE

encap_type = {'unspec': 0,
              'mpls': 1,
              0: 'unspec',
              1: 'mpls'}

rtypes = {'RTN_UNSPEC': 0,
          'RTN_UNICAST': 1,      # Gateway or direct route
          'RTN_LOCAL': 2,        # Accept locally
          'RTN_BROADCAST': 3,    # Accept locally as broadcast
          #                        send as broadcast
          'RTN_ANYCAST': 4,      # Accept locally as broadcast,
          #                        but send as unicast
          'RTN_MULTICAST': 5,    # Multicast route
          'RTN_BLACKHOLE': 6,    # Drop
          'RTN_UNREACHABLE': 7,  # Destination is unreachable
          'RTN_PROHIBIT': 8,     # Administratively prohibited
          'RTN_THROW': 9,        # Not in this table
          'RTN_NAT': 10,         # Translate this address
          'RTN_XRESOLVE': 11}    # Use external resolver
# normalized
rt_type = dict([(x[0][4:].lower(), x[1]) for x in rtypes.items()] +
               [(x[1], x[0][4:].lower()) for x in rtypes.items()])

rtprotos = {'RTPROT_UNSPEC': 0,
            'RTPROT_REDIRECT': 1,  # Route installed by ICMP redirects;
            #                        not used by current IPv4
            'RTPROT_KERNEL': 2,    # Route installed by kernel
            'RTPROT_BOOT': 3,      # Route installed during boot
            'RTPROT_STATIC': 4,    # Route installed by administrator
            # Values of protocol >= RTPROT_STATIC are not
            # interpreted by kernel;
            # keep in sync with iproute2 !
            'RTPROT_GATED': 8,      # gated
            'RTPROT_RA': 9,         # RDISC/ND router advertisements
            'RTPROT_MRT': 10,       # Merit MRT
            'RTPROT_ZEBRA': 11,     # Zebra
            'RTPROT_BIRD': 12,      # BIRD
            'RTPROT_DNROUTED': 13,  # DECnet routing daemon
            'RTPROT_XORP': 14,      # XORP
            'RTPROT_NTK': 15,       # Netsukuku
            'RTPROT_DHCP': 16}      # DHCP client
# normalized
rt_proto = dict([(x[0][7:].lower(), x[1]) for x in rtprotos.items()] +
                [(x[1], x[0][7:].lower()) for x in rtprotos.items()])

rtscopes = {'RT_SCOPE_UNIVERSE': 0,
            'RT_SCOPE_SITE': 200,
            'RT_SCOPE_LINK': 253,
            'RT_SCOPE_HOST': 254,
            'RT_SCOPE_NOWHERE': 255}
# normalized
rt_scope = dict([(x[0][9:].lower(), x[1]) for x in rtscopes.items()] +
                [(x[1], x[0][9:].lower()) for x in rtscopes.items()])
<commit_msg>netlink: subscribe to rule groups<commit_after>'''
RTNetlink: network setup
========================

RTNL is a netlink protocol, used to get and set information
about different network objects -- addresses, routes, interfaces
etc.

RTNL protocol-specific data in messages depends on the object
type. E.g., complete packet with the interface address information::

    nlmsg header:
        + uint32 length
        + uint16 type
        + uint16 flags
        + uint32 sequence number
        + uint32 pid
    ifaddrmsg structure:
        + unsigned char ifa_family
        + unsigned char ifa_prefixlen
        + unsigned char ifa_flags
        + unsigned char ifa_scope
        + uint32 ifa_index
    [ optional NLA tree ]

NLA for this kind of packets can be of type IFA_ADDRESS, IFA_LOCAL
etc. -- please refer to the corresponding source.

Other objects types require different structures, sometimes really
complex. All these structures are described in sources.

---------------------------

Module contents:

'''
from pyroute2.common import map_namespace

#  RTnetlink multicast groups
RTNLGRP_NONE = 0x0
RTNLGRP_LINK = 0x1
RTNLGRP_NOTIFY = 0x2
RTNLGRP_NEIGH = 0x4
RTNLGRP_TC = 0x8
RTNLGRP_IPV4_IFADDR = 0x10
RTNLGRP_IPV4_MROUTE = 0x20
RTNLGRP_IPV4_ROUTE = 0x40
RTNLGRP_IPV4_RULE = 0x80
RTNLGRP_IPV6_IFADDR = 0x100
RTNLGRP_IPV6_MROUTE = 0x200
RTNLGRP_IPV6_ROUTE = 0x400
RTNLGRP_IPV6_IFINFO = 0x800
RTNLGRP_DECnet_IFADDR = 0x1000
RTNLGRP_NOP2 = 0x2000
RTNLGRP_DECnet_ROUTE = 0x4000
RTNLGRP_DECnet_RULE = 0x8000
RTNLGRP_NOP4 = 0x10000
RTNLGRP_IPV6_PREFIX = 0x20000
RTNLGRP_IPV6_RULE = 0x40000
RTNLGRP_MPLS_ROUTE = 0x4000000

# Types of messages
# RTM_BASE = 16
RTM_NEWLINK = 16
RTM_DELLINK = 17
RTM_GETLINK = 18
RTM_SETLINK = 19
RTM_NEWADDR = 20
RTM_DELADDR = 21
RTM_GETADDR = 22
RTM_NEWROUTE = 24
RTM_DELROUTE = 25
RTM_GETROUTE = 26
RTM_NEWNEIGH = 28
RTM_DELNEIGH = 29
RTM_GETNEIGH = 30
RTM_NEWRULE = 32
RTM_DELRULE = 33
RTM_GETRULE = 34
RTM_NEWQDISC = 36
RTM_DELQDISC = 37
RTM_GETQDISC = 38
RTM_NEWTCLASS = 40
RTM_DELTCLASS = 41
RTM_GETTCLASS = 42
RTM_NEWTFILTER = 44
RTM_DELTFILTER = 45
RTM_GETTFILTER = 46
RTM_NEWACTION = 48
RTM_DELACTION = 49
RTM_GETACTION = 50
RTM_NEWPREFIX = 52
RTM_GETMULTICAST = 58
RTM_GETANYCAST = 62
RTM_NEWNEIGHTBL = 64
RTM_GETNEIGHTBL = 66
RTM_SETNEIGHTBL = 67
# custom message types
RTM_GETBRIDGE = 88
RTM_SETBRIDGE = 89
RTM_GETBOND = 90
RTM_SETBOND = 91
(RTM_NAMES, RTM_VALUES) = map_namespace('RTM', globals())

TC_H_INGRESS = 0xfffffff1
TC_H_CLSACT = TC_H_INGRESS
TC_H_ROOT = 0xffffffff

RTNL_GROUPS = RTNLGRP_IPV4_IFADDR |\
    RTNLGRP_IPV6_IFADDR |\
    RTNLGRP_IPV4_ROUTE |\
    RTNLGRP_IPV6_ROUTE |\
    RTNLGRP_IPV4_RULE |\
    RTNLGRP_IPV6_RULE |\
    RTNLGRP_NEIGH |\
    RTNLGRP_LINK |\
    RTNLGRP_TC |\
    RTNLGRP_MPLS_ROUTE

encap_type = {'unspec': 0,
              'mpls': 1,
              0: 'unspec',
              1: 'mpls'}

rtypes = {'RTN_UNSPEC': 0,
          'RTN_UNICAST': 1,      # Gateway or direct route
          'RTN_LOCAL': 2,        # Accept locally
          'RTN_BROADCAST': 3,    # Accept locally as broadcast
          #                        send as broadcast
          'RTN_ANYCAST': 4,      # Accept locally as broadcast,
          #                        but send as unicast
          'RTN_MULTICAST': 5,    # Multicast route
          'RTN_BLACKHOLE': 6,    # Drop
          'RTN_UNREACHABLE': 7,  # Destination is unreachable
          'RTN_PROHIBIT': 8,     # Administratively prohibited
          'RTN_THROW': 9,        # Not in this table
          'RTN_NAT': 10,         # Translate this address
          'RTN_XRESOLVE': 11}    # Use external resolver
# normalized
rt_type = dict([(x[0][4:].lower(), x[1]) for x in rtypes.items()] +
               [(x[1], x[0][4:].lower()) for x in rtypes.items()])

rtprotos = {'RTPROT_UNSPEC': 0,
            'RTPROT_REDIRECT': 1,  # Route installed by ICMP redirects;
            #                        not used by current IPv4
            'RTPROT_KERNEL': 2,    # Route installed by kernel
            'RTPROT_BOOT': 3,      # Route installed during boot
            'RTPROT_STATIC': 4,    # Route installed by administrator
            # Values of protocol >= RTPROT_STATIC are not
            # interpreted by kernel;
            # keep in sync with iproute2 !
            'RTPROT_GATED': 8,      # gated
            'RTPROT_RA': 9,         # RDISC/ND router advertisements
            'RTPROT_MRT': 10,       # Merit MRT
            'RTPROT_ZEBRA': 11,     # Zebra
            'RTPROT_BIRD': 12,      # BIRD
            'RTPROT_DNROUTED': 13,  # DECnet routing daemon
            'RTPROT_XORP': 14,      # XORP
            'RTPROT_NTK': 15,       # Netsukuku
            'RTPROT_DHCP': 16}      # DHCP client
# normalized
rt_proto = dict([(x[0][7:].lower(), x[1]) for x in rtprotos.items()] +
                [(x[1], x[0][7:].lower()) for x in rtprotos.items()])

rtscopes = {'RT_SCOPE_UNIVERSE': 0,
            'RT_SCOPE_SITE': 200,
            'RT_SCOPE_LINK': 253,
            'RT_SCOPE_HOST': 254,
            'RT_SCOPE_NOWHERE': 255}
# normalized
rt_scope = dict([(x[0][9:].lower(), x[1]) for x in rtscopes.items()] +
                [(x[1], x[0][9:].lower()) for x in rtscopes.items()])
<|endoftext|>


<commit_before>import torch
import numpy as np
from baseline.utils import lookup_sentence, get_version
import torch.autograd
import torch.nn as nn

PYT_MAJOR_VERSION = get_version(torch)

def classify_bt(model, batch_time):
    tensor = torch.from_numpy(batch_time) if type(batch_time) == np.ndarray else batch_time
    probs = model(torch.autograd.Variable(tensor, requires_grad=False).cuda()).exp().data
    probs.div_(torch.sum(probs))
    results = []
    batchsz = probs.size(0)
    for b in range(batchsz):
        outcomes = [(model.labels[id_i], prob_i) for id_i, prob_i in enumerate(probs[b])]
        results.append(outcomes)
    return results

def predict_seq_bt(model, x, xch, lengths):
    x_t = torch.from_numpy(x) if type(x) == np.ndarray else x
    xch_t = torch.from_numpy(xch) if type(xch) == np.ndarray else xch
    len_v = torch.from_numpy(lengths) if type(lengths) == np.ndarray else lengths
    x_v = torch.autograd.Variable(x_t, requires_grad=False).cuda()
    xch_v = torch.autograd.Variable(xch_t, requires_grad=False).cuda()
    #len_v = torch.autograd.Variable(len_t, requires_grad=False)
    results = model((x_v, xch_v, len_v))
    #print(results)
    #if type(x) == np.ndarray:
    #    # results = results.cpu().numpy()
    #    # Fix this to not be greedy
    #    results = np.argmax(results, -1)

    return results

def to_scalar(var):
    # returns a python float
    return var.view(-1).data.tolist()[0]

def argmax(vec):
    # return the argmax as a python int
    _, idx = torch.max(vec, 1)
    return to_scalar(idx)

def log_sum_exp(vec):
    max_score = vec[0, argmax(vec)]
    max_score_broadcast = max_score.view(1, -1).expand(1, vec.size()[1])
    return max_score + torch.log(torch.sum(torch.exp(vec - max_score_broadcast)))

class SequenceCriterion(nn.Module):

    def __init__(self, LossFn=nn.NLLLoss):
        super(SequenceCriterion, self).__init__()
        self.crit = LossFn(ignore_index=0, size_average=False)

    def forward(self, inputs, targets):
        # This is BxT, which is what we want!
        total_sz = targets.nelement()
        loss = self.crit(inputs.view(total_sz, -1), targets.view(total_sz))
        return loss

class StackedLSTMCell(nn.Module):
    def __init__(self, num_layers, input_size, rnn_size, dropout):
        super(StackedLSTMCell, self).__init__()
        self.dropout = nn.Dropout(dropout)
        self.num_layers = num_layers
        self.layers = nn.ModuleList()

        for i in range(num_layers):
            self.layers.append(nn.LSTMCell(input_size=input_size, hidden_size=rnn_size, bias=False))
            input_size = rnn_size

    def forward(self, input, hidden):
        h_0, c_0 = hidden
        hs, cs = [], []
        for i, layer in enumerate(self.layers):
            h_i, c_i = layer(input, (h_0[i], c_0[i]))
            input = h_i
            if i != self.num_layers - 1:
                input = self.dropout(input)
            hs += [h_i]
            cs += [c_i]

        hs = torch.stack(hs)
        cs = torch.stack(cs)

        return input, (hs, cs)

class StackedGRUCell(nn.Module):
    def __init__(self, num_layers, input_size, rnn_size, dropout):
        super(StackedGRUCell, self).__init__()
        self.dropout = nn.Dropout(dropout)
        self.num_layers = num_layers
        self.layers = nn.ModuleList()

        for i in range(num_layers):
            self.layers.append(nn.GRUCell(input_size=input_size, hidden_size=rnn_size))
            input_size = rnn_size

    def forward(self, input, hidden):
        h_0 = hidden
        hs = []
        for i, layer in enumerate(self.layers):
            h_i = layer(input, (h_0[i]))
            input = h_i
            if i != self.num_layers:
                input = self.dropout(input)
            hs += [h_i]

        hs = torch.stack(hs)

        return input, hs

def pytorch_rnn_cell(insz, hsz, rnntype, nlayers, dropout):

    if rnntype == 'gru':
        rnn = StackedGRUCell(nlayers, insz, hsz, dropout)
    else:
        rnn = StackedLSTMCell(nlayers, insz, hsz, dropout)
    return rnn

def pytorch_embedding(x2vec, finetune=True):
    dsz = x2vec.dsz
    lut = nn.Embedding(x2vec.vsz + 1, dsz, padding_idx=0)
    del lut.weight
    lut.weight = nn.Parameter(torch.FloatTensor(x2vec.weights),
                              requires_grad=finetune)
    return lut

def pytorch_activation(name="relu"):
    if name == "tanh":
        return nn.Tanh()
    if name == "prelu":
        return nn.PReLU()
    if name == "sigmoid":
        return nn.Sigmoid()
    if name == "log_sigmoid":
        return nn.LogSigmoid()
    return nn.ReLU()

def pytorch_conv1d(in_channels, out_channels, fsz, unif=0, padding=0):
    c = nn.Conv1d(in_channels, out_channels, fsz, padding=padding)
    if unif > 0:
        c.weight.data.uniform_(-unif, unif)
    else:
        torch.nn.init.orthogonal(c.weight)
    return c

def pytorch_linear(in_sz, out_sz, unif=0):
    l = nn.Linear(in_sz, out_sz)
    if unif > 0:
        l.weight.data.uniform_(-unif, unif)
    else:
        torch.nn.init.orthogonal(l.weight)
    l.bias.data.zero_()
    return l

def pytorch_rnn(insz, hsz, rnntype, nlayers, dropout):

    if rnntype == 'gru':
        rnn = torch.nn.GRU(insz, hsz, nlayers, dropout=dropout)
    else:
        rnn = torch.nn.LSTM(insz, hsz, nlayers, dropout=dropout)

    return rnn

class Highway(nn.Module):

    def __init__(self,
                 input_size):
        super(Highway, self).__init__()
        self.proj = nn.Linear(input_size, input_size)
        self.transform = nn.Linear(input_size, input_size)
        self.transform.bias.data.fill_(-2.0)

    def forward(self, input):
        proj_result = nn.functional.relu(self.proj(input))
        proj_gate = nn.functional.sigmoid(self.transform(input))
        gated = (proj_gate * proj_result) + ((1 - proj_gate) * input)
        return gated

def pytorch_lstm(insz, hsz, rnntype, nlayers, dropout, unif=0, batch_first=False):
    ndir = 2 if rnntype.startswith('b') else 1
    #print('ndir: %d, rnntype: %s, nlayers: %d, dropout: %.2f, unif: %.2f' % (ndir, rnntype, nlayers, dropout, unif))
    rnn = torch.nn.LSTM(insz, hsz, nlayers, dropout=dropout, bidirectional=True if ndir > 1 else False, batch_first=batch_first)#, bias=False)
    if unif > 0:
        for weight in rnn.parameters():
            weight.data.uniform_(-unif, unif)
    else:
        nn.init.orthogonal(rnn.weight_hh_l0)
        nn.init.orthogonal(rnn.weight_ih_l0)

    return rnn, ndir*hsz

def pytorch_prepare_optimizer(model, **kwargs):

    mom = kwargs.get('mom', 0.9)
    optim = kwargs.get('optim', 'sgd')
    eta = kwargs.get('eta', kwargs.get('lr', 0.01))
    decay_rate = float(kwargs.get('decay_rate', 0.0))
    decay_type = kwargs.get('decay_type', None)

    if optim == 'adadelta':
        optimizer = torch.optim.Adadelta(model.parameters(), lr=eta)
    elif optim == 'adam':
        optimizer = torch.optim.Adam(model.parameters(), lr=eta)
    elif optim == 'rmsprop':
        optimizer = torch.optim.RMSprop(model.parameters(), lr=eta)
    elif optim == 'asgd':
        optimizer = torch.optim.ASGD(model.parameters(), lr=eta)
    else:
        optimizer = torch.optim.SGD(model.parameters(), lr=eta, momentum=mom)

    scheduler = None
    if decay_rate > 0.0 and decay_type is not None:
        if decay_type == 'invtime':
            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, gamma=decay_rate)

    return optimizer, scheduler

def append2seq(seq, modules):

    for i, module in enumerate(modules):
        seq.add_module('%s-%d' % (str(module), i), module)

def tensor_max(tensor):
    return tensor.max()

def tensor_shape(tensor):
    return tensor.size()

def tensor_reverse_2nd(tensor):
    idx = torch.LongTensor([i for i in range(tensor.size(1)-1, -1, -1)])
    return tensor.index_select(1, idx)

def long_0_tensor_alloc(dims, dtype=None):
    lt = long_tensor_alloc(dims)
    lt.zero_()
    return lt

def long_tensor_alloc(dims, dtype=None):
    if type(dims) == int or len(dims) == 1:
        return torch.LongTensor(dims)
    return torch.LongTensor(*dims)

# Mashed together from code using numpy only, hacked for th Tensors
# This function should never be used for decoding.  It exists only so that the training model can greedily decode
# It is super slow and doesnt use maintain a beam of hypotheses
def show_examples_pytorch(model, es, rlut1, rlut2, embed2, mxlen, sample, prob_clip, max_examples, reverse):
    si = np.random.randint(0, len(es))

    batch_dict = es[si]

    src_array = batch_dict['src']
    tgt_array = batch_dict['dst']
    src_len = batch_dict['src_len']
    #src_array, tgt_array, src_len, _ = es[si]

    if max_examples > 0:
        max_examples = min(max_examples, src_array.size(0))
        src_array = src_array[0:max_examples]
        tgt_array = tgt_array[0:max_examples]
        src_len = src_len[0:max_examples]

    GO = embed2.vocab['<GO>']
    EOS = embed2.vocab['<EOS>']

    # TODO: fix this, check for GPU first
    src_array = src_array.cuda()

    for src_len,src_i,tgt_i in zip(src_len, src_array, tgt_array):

        print('========================================================================')

        sent = lookup_sentence(rlut1, src_i.cpu().numpy(), reverse=reverse)
        print('[OP] %s' % sent)
        sent = lookup_sentence(rlut2, tgt_i)
        print('[Actual] %s' % sent)
        dst_i = torch.zeros(1, mxlen).long()
        #if use_gpu:
        dst_i = dst_i.cuda()

        next_value = GO
        src_i = src_i.view(1, -1)
        for j in range(mxlen):
            dst_i[0,j] = next_value
            probv = model((torch.autograd.Variable(src_i), torch.autograd.Variable(dst_i)))
            output = probv.squeeze()[j]
            if sample is False:
                _, next_value = torch.max(output, 0)
                next_value = int(next_value.data[0])
            else:
                probs = output.data.exp()
                # This is going to zero out low prob. events so they are not
                # sampled from
                best, ids = probs.topk(prob_clip, 0, largest=True, sorted=True)
                probs.zero_()
                probs.index_copy_(0, ids, best)
                probs.div_(torch.sum(probs))
                fv = torch.multinomial(probs, 1)[0]
                next_value = fv

            if next_value == EOS:
                break

        sent = lookup_sentence(rlut2, dst_i.squeeze())
        print('Guess: %s' % sent)
        print('------------------------------------------------------------------------')
<commit_msg>Revert default back to xavier for now<commit_after>import torch
import numpy as np
from baseline.utils import lookup_sentence, get_version
import torch.autograd
import torch.nn as nn

PYT_MAJOR_VERSION = get_version(torch)

def classify_bt(model, batch_time):
    tensor = torch.from_numpy(batch_time) if type(batch_time) == np.ndarray else batch_time
    probs = model(torch.autograd.Variable(tensor, requires_grad=False).cuda()).exp().data
    probs.div_(torch.sum(probs))
    results = []
    batchsz = probs.size(0)
    for b in range(batchsz):
        outcomes = [(model.labels[id_i], prob_i) for id_i, prob_i in enumerate(probs[b])]
        results.append(outcomes)
    return results

def predict_seq_bt(model, x, xch, lengths):
    x_t = torch.from_numpy(x) if type(x) == np.ndarray else x
    xch_t = torch.from_numpy(xch) if type(xch) == np.ndarray else xch
    len_v = torch.from_numpy(lengths) if type(lengths) == np.ndarray else lengths
    x_v = torch.autograd.Variable(x_t, requires_grad=False).cuda()
    xch_v = torch.autograd.Variable(xch_t, requires_grad=False).cuda()
    #len_v = torch.autograd.Variable(len_t, requires_grad=False)
    results = model((x_v, xch_v, len_v))
    #print(results)
    #if type(x) == np.ndarray:
    #    # results = results.cpu().numpy()
    #    # Fix this to not be greedy
    #    results = np.argmax(results, -1)

    return results

def to_scalar(var):
    # returns a python float
    return var.view(-1).data.tolist()[0]

def argmax(vec):
    # return the argmax as a python int
    _, idx = torch.max(vec, 1)
    return to_scalar(idx)

def log_sum_exp(vec):
    max_score = vec[0, argmax(vec)]
    max_score_broadcast = max_score.view(1, -1).expand(1, vec.size()[1])
    return max_score + torch.log(torch.sum(torch.exp(vec - max_score_broadcast)))

class SequenceCriterion(nn.Module):

    def __init__(self, LossFn=nn.NLLLoss):
        super(SequenceCriterion, self).__init__()
        self.crit = LossFn(ignore_index=0, size_average=False)

    def forward(self, inputs, targets):
        # This is BxT, which is what we want!
        total_sz = targets.nelement()
        loss = self.crit(inputs.view(total_sz, -1), targets.view(total_sz))
        return loss

class StackedLSTMCell(nn.Module):
    def __init__(self, num_layers, input_size, rnn_size, dropout):
        super(StackedLSTMCell, self).__init__()
        self.dropout = nn.Dropout(dropout)
        self.num_layers = num_layers
        self.layers = nn.ModuleList()

        for i in range(num_layers):
            self.layers.append(nn.LSTMCell(input_size=input_size, hidden_size=rnn_size, bias=False))
            input_size = rnn_size

    def forward(self, input, hidden):
        h_0, c_0 = hidden
        hs, cs = [], []
        for i, layer in enumerate(self.layers):
            h_i, c_i = layer(input, (h_0[i], c_0[i]))
            input = h_i
            if i != self.num_layers - 1:
                input = self.dropout(input)
            hs += [h_i]
            cs += [c_i]

        hs = torch.stack(hs)
        cs = torch.stack(cs)

        return input, (hs, cs)

class StackedGRUCell(nn.Module):
    def __init__(self, num_layers, input_size, rnn_size, dropout):
        super(StackedGRUCell, self).__init__()
        self.dropout = nn.Dropout(dropout)
        self.num_layers = num_layers
        self.layers = nn.ModuleList()

        for i in range(num_layers):
            self.layers.append(nn.GRUCell(input_size=input_size, hidden_size=rnn_size))
            input_size = rnn_size

    def forward(self, input, hidden):
        h_0 = hidden
        hs = []
        for i, layer in enumerate(self.layers):
            h_i = layer(input, (h_0[i]))
            input = h_i
            if i != self.num_layers:
                input = self.dropout(input)
            hs += [h_i]

        hs = torch.stack(hs)

        return input, hs

def pytorch_rnn_cell(insz, hsz, rnntype, nlayers, dropout):

    if rnntype == 'gru':
        rnn = StackedGRUCell(nlayers, insz, hsz, dropout)
    else:
        rnn = StackedLSTMCell(nlayers, insz, hsz, dropout)
    return rnn

def pytorch_embedding(x2vec, finetune=True):
    dsz = x2vec.dsz
    lut = nn.Embedding(x2vec.vsz + 1, dsz, padding_idx=0)
    del lut.weight
    lut.weight = nn.Parameter(torch.FloatTensor(x2vec.weights),
                              requires_grad=finetune)
    return lut

def pytorch_activation(name="relu"):
    if name == "tanh":
        return nn.Tanh()
    if name == "hardtanh":
        return nn.Hardtanh()
    if name == "prelu":
        return nn.PReLU()
    if name == "sigmoid":
        return nn.Sigmoid()
    if name == "log_sigmoid":
        return nn.LogSigmoid()
    return nn.ReLU()

def pytorch_conv1d(in_channels, out_channels, fsz, unif=0, padding=0, initializer=None):
    c = nn.Conv1d(in_channels, out_channels, fsz, padding=padding)
    if unif > 0:
        c.weight.data.uniform_(-unif, unif)
    elif initializer == "ortho":
        nn.init.orthogonal(c.weight)
    elif initializer == "he" or initializer == "kaiming":
        nn.init.kaiming_uniform(c.weight)
    else:
        nn.init.xavier_uniform(c.weight)
    return c

def pytorch_linear(in_sz, out_sz, unif=0, initializer=None):
    l = nn.Linear(in_sz, out_sz)
    if unif > 0:
        l.weight.data.uniform_(-unif, unif)
    elif initializer == "ortho":
        nn.init.orthogonal(l.weight)
    elif initializer == "he" or initializer == "kaiming":
        nn.init.kaiming_uniform(l.weight)
    else:
        nn.init.xavier_uniform(l.weight)

    l.bias.data.zero_()
    return l

def pytorch_rnn(insz, hsz, rnntype, nlayers, dropout):

    if rnntype == 'gru':
        rnn = torch.nn.GRU(insz, hsz, nlayers, dropout=dropout)
    else:
        rnn = torch.nn.LSTM(insz, hsz, nlayers, dropout=dropout)

    return rnn

class Highway(nn.Module):

    def __init__(self,
                 input_size):
        super(Highway, self).__init__()
        self.proj = nn.Linear(input_size, input_size)
        self.transform = nn.Linear(input_size, input_size)
        self.transform.bias.data.fill_(-2.0)

    def forward(self, input):
        proj_result = nn.functional.relu(self.proj(input))
        proj_gate = nn.functional.sigmoid(self.transform(input))
        gated = (proj_gate * proj_result) + ((1 - proj_gate) * input)
        return gated

def pytorch_lstm(insz, hsz, rnntype, nlayers, dropout, unif=0, batch_first=False, initializer=None):
    ndir = 2 if rnntype.startswith('b') else 1
    #print('ndir: %d, rnntype: %s, nlayers: %d, dropout: %.2f, unif: %.2f' % (ndir, rnntype, nlayers, dropout, unif))
    rnn = torch.nn.LSTM(insz, hsz, nlayers, dropout=dropout, bidirectional=True if ndir > 1 else False, batch_first=batch_first)#, bias=False)
    if unif > 0:
        for weight in rnn.parameters():
            weight.data.uniform_(-unif, unif)
    elif initializer == "ortho":
        nn.init.orthogonal(rnn.weight_hh_l0)
        nn.init.orthogonal(rnn.weight_ih_l0)
    elif initializer == "he" or initializer == "kaiming":
        nn.init.kaiming_uniform(rnn.weight_hh_l0)
        nn.init.kaiming_uniform(rnn.weight_ih_l0)
    else:
        nn.init.xavier_uniform(rnn.weight_hh_l0)
        nn.init.xavier_uniform(rnn.weight_ih_l0)

    return rnn, ndir*hsz

def pytorch_prepare_optimizer(model, **kwargs):

    mom = kwargs.get('mom', 0.9)
    optim = kwargs.get('optim', 'sgd')
    eta = kwargs.get('eta', kwargs.get('lr', 0.01))
    decay_rate = float(kwargs.get('decay_rate', 0.0))
    decay_type = kwargs.get('decay_type', None)

    if optim == 'adadelta':
        optimizer = torch.optim.Adadelta(model.parameters(), lr=eta)
    elif optim == 'adam':
        optimizer = torch.optim.Adam(model.parameters(), lr=eta)
    elif optim == 'rmsprop':
        optimizer = torch.optim.RMSprop(model.parameters(), lr=eta)
    elif optim == 'asgd':
        optimizer = torch.optim.ASGD(model.parameters(), lr=eta)
    else:
        optimizer = torch.optim.SGD(model.parameters(), lr=eta, momentum=mom)

    scheduler = None
    if decay_rate > 0.0 and decay_type is not None:
        if decay_type == 'invtime':
            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, gamma=decay_rate)

    return optimizer, scheduler

def append2seq(seq, modules):

    for i, module in enumerate(modules):
        seq.add_module('%s-%d' % (str(module), i), module)

def tensor_max(tensor):
    return tensor.max()

def tensor_shape(tensor):
    return tensor.size()

def tensor_reverse_2nd(tensor):
    idx = torch.LongTensor([i for i in range(tensor.size(1)-1, -1, -1)])
    return tensor.index_select(1, idx)

def long_0_tensor_alloc(dims, dtype=None):
    lt = long_tensor_alloc(dims)
    lt.zero_()
    return lt

def long_tensor_alloc(dims, dtype=None):
    if type(dims) == int or len(dims) == 1:
        return torch.LongTensor(dims)
    return torch.LongTensor(*dims)

# Mashed together from code using numpy only, hacked for th Tensors
# This function should never be used for decoding.  It exists only so that the training model can greedily decode
# It is super slow and doesnt use maintain a beam of hypotheses
def show_examples_pytorch(model, es, rlut1, rlut2, embed2, mxlen, sample, prob_clip, max_examples, reverse):
    si = np.random.randint(0, len(es))

    batch_dict = es[si]

    src_array = batch_dict['src']
    tgt_array = batch_dict['dst']
    src_len = batch_dict['src_len']
    #src_array, tgt_array, src_len, _ = es[si]

    if max_examples > 0:
        max_examples = min(max_examples, src_array.size(0))
        src_array = src_array[0:max_examples]
        tgt_array = tgt_array[0:max_examples]
        src_len = src_len[0:max_examples]

    GO = embed2.vocab['<GO>']
    EOS = embed2.vocab['<EOS>']

    # TODO: fix this, check for GPU first
    src_array = src_array.cuda()

    for src_len,src_i,tgt_i in zip(src_len, src_array, tgt_array):

        print('========================================================================')

        sent = lookup_sentence(rlut1, src_i.cpu().numpy(), reverse=reverse)
        print('[OP] %s' % sent)
        sent = lookup_sentence(rlut2, tgt_i)
        print('[Actual] %s' % sent)
        dst_i = torch.zeros(1, mxlen).long()
        #if use_gpu:
        dst_i = dst_i.cuda()

        next_value = GO
        src_i = src_i.view(1, -1)
        for j in range(mxlen):
            dst_i[0,j] = next_value
            probv = model((torch.autograd.Variable(src_i), torch.autograd.Variable(dst_i)))
            output = probv.squeeze()[j]
            if sample is False:
                _, next_value = torch.max(output, 0)
                next_value = int(next_value.data[0])
            else:
                probs = output.data.exp()
                # This is going to zero out low prob. events so they are not
                # sampled from
                best, ids = probs.topk(prob_clip, 0, largest=True, sorted=True)
                probs.zero_()
                probs.index_copy_(0, ids, best)
                probs.div_(torch.sum(probs))
                fv = torch.multinomial(probs, 1)[0]
                next_value = fv

            if next_value == EOS:
                break

        sent = lookup_sentence(rlut2, dst_i.squeeze())
        print('Guess: %s' % sent)
        print('------------------------------------------------------------------------')
<|endoftext|>


<commit_before>#  Copyright 2017-2022 John Snow Labs
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

"""Contains Properties for the Annotator classes.
"""

from pyspark.ml.util import JavaMLWritable
from pyspark.ml.wrapper import JavaModel, JavaEstimator
from pyspark.ml.param.shared import Param, TypeConverters
from pyspark.ml.param import Params
from pyspark import keyword_only

import sparknlp.internal as _internal

class AnnotatorProperties(Params):
    inputCols = Param(Params._dummy(),
                      "inputCols",
                      "previous annotations columns, if renamed",
                      typeConverter=TypeConverters.toListString)
    outputCol = Param(Params._dummy(),
                      "outputCol",
                      "output annotation column. can be left default.",
                      typeConverter=TypeConverters.toString)
    lazyAnnotator = Param(Params._dummy(),
                          "lazyAnnotator",
                          "Whether this AnnotatorModel acts as lazy in RecursivePipelines",
                          typeConverter=TypeConverters.toBoolean
                          )

    def setInputCols(self, *value):
        """Sets column names of input annotations.

        Parameters
        ----------
        *value : str
            Input columns for the annotator
        """
        if len(value) == 1 and type(value[0]) == list:
            return self._set(inputCols=value[0])
        else:
            return self._set(inputCols=list(value))

    def getInputCols(self):
        """Gets current column names of input annotations."""
        return self.getOrDefault(self.inputCols)

    def setOutputCol(self, value):
        """Sets output column name of annotations.

        Parameters
        ----------
        value : str
            Name of output column
        """
        return self._set(outputCol=value)

    def getOutputCol(self):
        """Gets output column name of annotations."""
        return self.getOrDefault(self.outputCol)

    def setLazyAnnotator(self, value):
        """Sets whether Annotator should be evaluated lazily in a
        RecursivePipeline.

        Parameters
        ----------
        value : bool
            Whether Annotator should be evaluated lazily in a
            RecursivePipeline
        """
        return self._set(lazyAnnotator=value)

    def getLazyAnnotator(self):
        """Gets whether Annotator should be evaluated lazily in a
        RecursivePipeline.
        """
        return self.getOrDefault(self.lazyAnnotator)

class AnnotatorModel(JavaModel, _internal.AnnotatorJavaMLReadable, JavaMLWritable, AnnotatorProperties,
                     _internal.ParamsGettersSetters):

    @keyword_only
    def setParams(self):
        kwargs = self._input_kwargs
        return self._set(**kwargs)

    @keyword_only
    def __init__(self, classname, java_model=None):
        super(AnnotatorModel, self).__init__(java_model=java_model)
        if classname and not java_model:
            self.__class__._java_class_name = classname
            self._java_obj = self._new_java_obj(classname, self.uid)
        if java_model is not None:
            self._transfer_params_from_java()
        self._setDefault(lazyAnnotator=False)

class HasEmbeddingsProperties(Params):
    dimension = Param(Params._dummy(),
                      "dimension",
                      "Number of embedding dimensions",
                      typeConverter=TypeConverters.toInt)

    def setDimension(self, value):
        """Sets embeddings dimension.

        Parameters
        ----------
        value : int
            Embeddings dimension
        """
        return self._set(dimension=value)

    def getDimension(self):
        """Gets embeddings dimension."""
        return self.getOrDefault(self.dimension)

class HasStorageRef:
    storageRef = Param(Params._dummy(), "storageRef",
                       "unique reference name for identification",
                       TypeConverters.toString)

    def setStorageRef(self, value):
        """Sets unique reference name for identification.

        Parameters
        ----------
        value : str
            Unique reference name for identification
        """
        return self._set(storageRef=value)

    def getStorageRef(self):
        """Gets unique reference name for identification.

        Returns
        -------
        str
            Unique reference name for identification
        """
        return self.getOrDefault("storageRef")

class HasBatchedAnnotate:
    batchSize = Param(Params._dummy(), "batchSize", "Size of every batch", TypeConverters.toInt)

    def setBatchSize(self, v):
        """Sets batch size.

        Parameters
        ----------
        v : int
            Batch size
        """
        return self._set(batchSize=v)

    def getBatchSize(self):
        """Gets current batch size.

        Returns
        -------
        int
            Current batch size
        """
        return self.getOrDefault("batchSize")

class HasCaseSensitiveProperties:
    caseSensitive = Param(Params._dummy(),
                          "caseSensitive",
                          "whether to ignore case in tokens for embeddings matching",
                          typeConverter=TypeConverters.toBoolean)

    def setCaseSensitive(self, value):
        """Sets whether to ignore case in tokens for embeddings matching.

        Parameters
        ----------
        value : bool
            Whether to ignore case in tokens for embeddings matching
        """
        return self._set(caseSensitive=value)

    def getCaseSensitive(self):
        """Gets whether to ignore case in tokens for embeddings matching.

        Returns
        -------
        bool
            Whether to ignore case in tokens for embeddings matching
        """
        return self.getOrDefault(self.caseSensitive)

class HasExcludableStorage:
    includeStorage = Param(Params._dummy(),
                           "includeStorage",
                           "whether to include indexed storage in trained model",
                           typeConverter=TypeConverters.toBoolean)

    def setIncludeStorage(self, value):
        """Sets whether to include indexed storage in trained model.

        Parameters
        ----------
        value : bool
            Whether to include indexed storage in trained model
        """
        return self._set(includeStorage=value)

    def getIncludeStorage(self):
        """Gets whether to include indexed storage in trained model.

        Returns
        -------
        bool
            Whether to include indexed storage in trained model
        """
        return self.getOrDefault("includeStorage")

class HasStorage(HasStorageRef, HasCaseSensitiveProperties, HasExcludableStorage):
    storagePath = Param(Params._dummy(),
                        "storagePath",
                        "path to file",
                        typeConverter=TypeConverters.identity)

    def setStoragePath(self, path, read_as):
        """Sets path to file.

        Parameters
        ----------
        path : str
            Path to file
        read_as : str
            How to interpret the file

        Notes
        -----
        See :class:`ReadAs <sparknlp.common.ReadAs>` for reading options.
        """
        return self._set(storagePath=ExternalResource(path, read_as, {}))

    def getStoragePath(self):
        """Gets path to file.

        Returns
        -------
        str
            path to file
        """
        return self.getOrDefault("storagePath")

class HasStorageModel(HasStorageRef, HasCaseSensitiveProperties, HasExcludableStorage):

    def saveStorage(self, path, spark):
        """Saves the current model to storage.

        Parameters
        ----------
        path : str
            Path for saving the model.
        spark : :class:`pyspark.sql.SparkSession`
            The current SparkSession
        """
        self._transfer_params_to_java()
        self._java_obj.saveStorage(path, spark._jsparkSession, False)

    @staticmethod
    def loadStorage(path, spark, storage_ref):
        raise NotImplementedError("AnnotatorModel with HasStorageModel did not implement 'loadStorage'")

    @staticmethod
    def loadStorages(path, spark, storage_ref, databases):
        for database in databases:
            _internal._StorageHelper(path, spark, database, storage_ref, within_storage=False)

class AnnotatorApproach(JavaEstimator, JavaMLWritable, _internal.AnnotatorJavaMLReadable, AnnotatorProperties,
                        _internal.ParamsGettersSetters):

    @keyword_only
    def __init__(self, classname):
        _internal.ParamsGettersSetters.__init__(self)
        self.__class__._java_class_name = classname
        self._java_obj = self._new_java_obj(classname, self.uid)
        self._setDefault(lazyAnnotator=False)

    def _create_model(self, java_model):
        raise NotImplementedError('Please implement _create_model in %s' % self)

class RecursiveAnnotatorApproach(_internal.RecursiveEstimator, JavaMLWritable, _internal.AnnotatorJavaMLReadable,
                                 AnnotatorProperties,
                                 _internal.ParamsGettersSetters):
    @keyword_only
    def __init__(self, classname):
        _internal.ParamsGettersSetters.__init__(self)
        self.__class__._java_class_name = classname
        self._java_obj = self._new_java_obj(classname, self.uid)
        self._setDefault(lazyAnnotator=False)

    def _create_model(self, java_model):
        raise NotImplementedError('Please implement _create_model in %s' % self)

def RegexRule(rule, identifier):
    return _internal._RegexRule(rule, identifier).apply()

class ReadAs(object):
    """Object that contains constants for how to read Spark Resources.

    Possible values are:

    ================= =======================================
    Value             Description
    ================= =======================================
    ``ReadAs.TEXT``   Read the resource as text.
    ``ReadAs.SPARK``  Read the resource as a Spark DataFrame.
    ``ReadAs.BINARY`` Read the resource as a binary file.
    ================= =======================================
    """
    TEXT = "TEXT"
    SPARK = "SPARK"
    BINARY = "BINARY"

def ExternalResource(path, read_as=ReadAs.TEXT, options={}):
    """Returns a representation fo an External Resource.

    How the resource is read can be set with `read_as`.

    Parameters
    ----------
    path : str
        Path to the resource
    read_as : str, optional
        How to read the resource, by default ReadAs.TEXT
    options : dict, optional
        Options to read the resource, by default {}
    """
    return _internal._ExternalResource(path, read_as, options).apply()

class CoverageResult:
    def __init__(self, cov_obj):
        self.covered = cov_obj.covered()
        self.total = cov_obj.total()
        self.percentage = cov_obj.percentage()

class HasEnableCachingProperties:
    enableCaching = Param(Params._dummy(),
                          "enableCaching",
                          "Whether to enable caching DataFrames or RDDs during the training",
                          typeConverter=TypeConverters.toBoolean)

    def setEnableCaching(self, value):
        """Sets whether to enable caching DataFrames or RDDs during the training

        Parameters
        ----------
        value : bool
            Whether to enable caching DataFrames or RDDs during the training
        """
        return self._set(enableCaching=value)

    def getEnableCaching(self):
        """Gets whether to enable caching DataFrames or RDDs during the training

        Returns
        -------
        bool
            Whether to enable caching DataFrames or RDDs during the training
        """
        return self.getOrDefault(self.enableCaching)

class HasClassifierActivationProperties:
    activation = Param(Params._dummy(),
                       "activation",
                       "Whether to calculate logits via Softmax or Sigmoid. Default is Softmax",
                       typeConverter=TypeConverters.toString)

    def setActivation(self, value):
        """Sets whether to calculate logits via Softmax or Sigmoid. Default is Softmax

        Parameters
        ----------
        value : str
            Whether to calculate logits via Softmax or Sigmoid. Default is Softmax
        """
        return self._set(activation=value)

    def getActivation(self):
        """Gets whether to calculate logits via Softmax or Sigmoid. Default is Softmax

        Returns
        -------
        str
            Whether to calculate logits via Softmax or Sigmoid. Default is Softmax
        """
        return self.getOrDefault(self.activation)
<commit_msg>Python Refactor: read_as<commit_after>class ReadAs(object):
    """Object that contains constants for how to read Spark Resources.

    Possible values are:

    ================= =======================================
    Value             Description
    ================= =======================================
    ``ReadAs.TEXT``   Read the resource as text.
    ``ReadAs.SPARK``  Read the resource as a Spark DataFrame.
    ``ReadAs.BINARY`` Read the resource as a binary file.
    ================= =======================================
    """
    TEXT = "TEXT"
    SPARK = "SPARK"
    BINARY = "BINARY"

<|endoftext|>


<commit_before>import pytest

@pytest.fixture
def stepwise_testdir(testdir):
    # Rather than having to modify our testfile between tests, we introduce
    # a flag for wether or not the second test should fail.
    testdir.makeconftest(
        """
def pytest_addoption(parser):
    group = parser.getgroup('general')
    group.addoption('--fail', action='store_true', dest='fail')
    group.addoption('--fail-last', action='store_true', dest='fail_last')
"""
    )

    # Create a simple test suite.
    testdir.makepyfile(
        test_a="""
def test_success_before_fail():
    assert 1

def test_fail_on_flag(request):
    assert not request.config.getvalue('fail')

def test_success_after_fail():
    assert 1

def test_fail_last_on_flag(request):
    assert not request.config.getvalue('fail_last')

def test_success_after_last_fail():
    assert 1
"""
    )

    testdir.makepyfile(
        test_b="""
def test_success():
    assert 1
"""
    )

    return testdir

@pytest.fixture
def error_testdir(testdir):
    testdir.makepyfile(
        test_a="""
def test_error(nonexisting_fixture):
    assert 1

def test_success_after_fail():
    assert 1
"""
    )

    return testdir

@pytest.fixture
def broken_testdir(testdir):
    testdir.makepyfile(
        working_testfile="def test_proper(): assert 1", broken_testfile="foobar"
    )
    return testdir

def test_run_without_stepwise(stepwise_testdir):
    result = stepwise_testdir.runpytest("-v", "--strict", "--fail")

    result.stdout.fnmatch_lines(["*test_success_before_fail PASSED*"])
    result.stdout.fnmatch_lines(["*test_fail_on_flag FAILED*"])
    result.stdout.fnmatch_lines(["*test_success_after_fail PASSED*"])

def test_fail_and_continue_with_stepwise(stepwise_testdir):
    # Run the tests with a failing second test.
    result = stepwise_testdir.runpytest("-v", "--strict", "--stepwise", "--fail")
    assert not result.stderr.str()

    stdout = result.stdout.str()
    # Make sure we stop after first failing test.
    assert "test_success_before_fail PASSED" in stdout
    assert "test_fail_on_flag FAILED" in stdout
    assert "test_success_after_fail" not in stdout

    # "Fix" the test that failed in the last run and run it again.
    result = stepwise_testdir.runpytest("-v", "--strict", "--stepwise")
    assert not result.stderr.str()

    stdout = result.stdout.str()
    # Make sure the latest failing test runs and then continues.
    assert "test_success_before_fail" not in stdout
    assert "test_fail_on_flag PASSED" in stdout
    assert "test_success_after_fail PASSED" in stdout

def test_run_with_skip_option(stepwise_testdir):
    result = stepwise_testdir.runpytest(
        "-v", "--strict", "--stepwise", "--stepwise-skip", "--fail", "--fail-last"
    )
    assert not result.stderr.str()

    stdout = result.stdout.str()
    # Make sure first fail is ignore and second fail stops the test run.
    assert "test_fail_on_flag FAILED" in stdout
    assert "test_success_after_fail PASSED" in stdout
    assert "test_fail_last_on_flag FAILED" in stdout
    assert "test_success_after_last_fail" not in stdout

def test_fail_on_errors(error_testdir):
    result = error_testdir.runpytest("-v", "--strict", "--stepwise")

    assert not result.stderr.str()
    stdout = result.stdout.str()

    assert "test_error ERROR" in stdout
    assert "test_success_after_fail" not in stdout

def test_change_testfile(stepwise_testdir):
    result = stepwise_testdir.runpytest(
        "-v", "--strict", "--stepwise", "--fail", "test_a.py"
    )
    assert not result.stderr.str()

    stdout = result.stdout.str()
    assert "test_fail_on_flag FAILED" in stdout

    # Make sure the second test run starts from the beginning, since the
    # test to continue from does not exist in testfile_b.
    result = stepwise_testdir.runpytest("-v", "--strict", "--stepwise", "test_b.py")
    assert not result.stderr.str()

    stdout = result.stdout.str()
    assert "test_success PASSED" in stdout

def test_stop_on_collection_errors(broken_testdir):
    result = broken_testdir.runpytest(
        "-v", "--strict", "--stepwise", "working_testfile.py", "broken_testfile.py"
    )

    stdout = result.stdout.str()
    assert "errors during collection" in stdout
<commit_msg>Use a custom cache directory for test_stepwise tests<commit_after>import pytest

@pytest.fixture
def stepwise_testdir(testdir):
    # Rather than having to modify our testfile between tests, we introduce
    # a flag for wether or not the second test should fail.
    testdir.makeconftest(
        """
def pytest_addoption(parser):
    group = parser.getgroup('general')
    group.addoption('--fail', action='store_true', dest='fail')
    group.addoption('--fail-last', action='store_true', dest='fail_last')
"""
    )

    # Create a simple test suite.
    testdir.makepyfile(
        test_a="""
def test_success_before_fail():
    assert 1

def test_fail_on_flag(request):
    assert not request.config.getvalue('fail')

def test_success_after_fail():
    assert 1

def test_fail_last_on_flag(request):
    assert not request.config.getvalue('fail_last')

def test_success_after_last_fail():
    assert 1
"""
    )

    testdir.makepyfile(
        test_b="""
def test_success():
    assert 1
"""
    )

    # customize cache directory so we don't use the tox's cache directory, which makes tests in this module flaky
    testdir.makeini(
        """
        [pytest]
        cache_dir = .cache
    """
    )

    return testdir

@pytest.fixture
def error_testdir(testdir):
    testdir.makepyfile(
        test_a="""
def test_error(nonexisting_fixture):
    assert 1

def test_success_after_fail():
    assert 1
"""
    )

    return testdir

@pytest.fixture
def broken_testdir(testdir):
    testdir.makepyfile(
        working_testfile="def test_proper(): assert 1", broken_testfile="foobar"
    )
    return testdir

def test_run_without_stepwise(stepwise_testdir):
    result = stepwise_testdir.runpytest("-v", "--strict", "--fail")

    result.stdout.fnmatch_lines(["*test_success_before_fail PASSED*"])
    result.stdout.fnmatch_lines(["*test_fail_on_flag FAILED*"])
    result.stdout.fnmatch_lines(["*test_success_after_fail PASSED*"])

def test_fail_and_continue_with_stepwise(stepwise_testdir):
    # Run the tests with a failing second test.
    result = stepwise_testdir.runpytest("-v", "--strict", "--stepwise", "--fail")
    assert not result.stderr.str()

    stdout = result.stdout.str()
    # Make sure we stop after first failing test.
    assert "test_success_before_fail PASSED" in stdout
    assert "test_fail_on_flag FAILED" in stdout
    assert "test_success_after_fail" not in stdout

    # "Fix" the test that failed in the last run and run it again.
    result = stepwise_testdir.runpytest("-v", "--strict", "--stepwise")
    assert not result.stderr.str()

    stdout = result.stdout.str()
    # Make sure the latest failing test runs and then continues.
    assert "test_success_before_fail" not in stdout
    assert "test_fail_on_flag PASSED" in stdout
    assert "test_success_after_fail PASSED" in stdout

def test_run_with_skip_option(stepwise_testdir):
    result = stepwise_testdir.runpytest(
        "-v", "--strict", "--stepwise", "--stepwise-skip", "--fail", "--fail-last"
    )
    assert not result.stderr.str()

    stdout = result.stdout.str()
    # Make sure first fail is ignore and second fail stops the test run.
    assert "test_fail_on_flag FAILED" in stdout
    assert "test_success_after_fail PASSED" in stdout
    assert "test_fail_last_on_flag FAILED" in stdout
    assert "test_success_after_last_fail" not in stdout

def test_fail_on_errors(error_testdir):
    result = error_testdir.runpytest("-v", "--strict", "--stepwise")

    assert not result.stderr.str()
    stdout = result.stdout.str()

    assert "test_error ERROR" in stdout
    assert "test_success_after_fail" not in stdout

def test_change_testfile(stepwise_testdir):
    result = stepwise_testdir.runpytest(
        "-v", "--strict", "--stepwise", "--fail", "test_a.py"
    )
    assert not result.stderr.str()

    stdout = result.stdout.str()
    assert "test_fail_on_flag FAILED" in stdout

    # Make sure the second test run starts from the beginning, since the
    # test to continue from does not exist in testfile_b.
    result = stepwise_testdir.runpytest("-v", "--strict", "--stepwise", "test_b.py")
    assert not result.stderr.str()

    stdout = result.stdout.str()
    assert "test_success PASSED" in stdout

def test_stop_on_collection_errors(broken_testdir):
    result = broken_testdir.runpytest(
        "-v", "--strict", "--stepwise", "working_testfile.py", "broken_testfile.py"
    )

    stdout = result.stdout.str()
    assert "errors during collection" in stdout
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys
import os
import subprocess
import StringIO
from testconfigurations import TestConfiguration, GetTestConfig
from optparse import OptionParser

# ----------

class Package:
    root = None
    name = None
    version = None

    def __init__(self, root, name, version):
        self.root = root
        self.name = name
        self.version = version

    def GetDescription(self):
        return "%s-%s in %s" % (self.name, self.version, self.root)

    def GetPath(self):
        return os.path.join(self.root, self.name, self.version)

    def GetId(self):
        return "-".join([self.name, self.version])

# ----------

def FindAllPackagesToTest(root):
    """Locate packages that can be tested"""
    tests = []
    for packageName in os.listdir(root):
        if packageName.startswith("."):
            continue
        packageDir = os.path.join(root, packageName)
        if os.path.isdir(packageDir):
            for packageVersion in os.listdir(packageDir):
                if packageVersion.startswith("."):
                    continue
                versionDir = os.path.join(packageDir, packageVersion)
                if os.path.isdir(versionDir):
                    package = Package(root, packageName, packageVersion)
                    tests.append(package)
    return tests

def ExecuteTests(package, configuration, options, outputBuffer):
    if options.verbose:
        print "Package: ", package.GetDescription()
        print "Builders: ", configuration.GetBuilders()
        print "Response files: ", configuration.GetResponseFiles()
    if not options.builder in configuration.GetBuilders():
        outputBuffer.write("Package '%s' does not support the builder, '%s'" % (package.GetDescription(),options.builder))
        return 0
    for config in configuration.GetResponseFiles():
        argList = []
        argList.append("Opus")
        argList.append("@" + os.path.join(os.getcwd(), config))
        argList.append("-buildroot=" + options.buildRoot)
        argList.append("-builder=" + options.builder)
        if sys.platform.startswith("win"):
            argList.append("-platforms=" + ";".join(options.platforms))
            argList.append("-configurations=" + ";".join(options.configurations))
        else:
            argList.append("-platforms=" + ":".join(options.platforms))
            argList.append("-configurations=" + ":".join(options.configurations))
        argList.append("-j=" + str(options.numJobs))
        if options.debugSymbols:
            argList.append("-debugsymbols")
        argList.append("-verbosity=0")
        if options.verbose:
            print "\tExecuting: %s" % " ".join(argList)
        currentDir = os.getcwd()
        try:
            p = subprocess.Popen(argList, stdout=subprocess.PIPE, stderr=sys.stdout, cwd=package.GetPath())
            (output, error) = p.communicate() # this should WAIT
        except Exception, e:
            print "Popen exception: '%s'" % str(e)
            raise
        finally:
            os.chdir(currentDir)
            if not p.returncode:
                outputBuffer.write("Package '%s' succeeded in config '%s'\n" % (package.GetDescription(), config))
                return 0
            else:
                outputBuffer.write("Package '%s' failed in config '%s'\n" % (package.GetDescription(), config))
                outputBuffer.write("Command was: '%s'" % " ".join(argList))
                outputBuffer.write(output)
                return -1

def CleanUp(options):
    argList = []
    if sys.platform.startswith("win"):
        argList.append(os.path.join(os.getcwd(), "removedebugprojects.bat"))
        argList.append("-nopause")
    else:
        argList.append(os.path.join(os.getcwd(), "removedebugprojects.sh"))
    if options.verbose:
        print "Executing: ", argList
    p = subprocess.Popen(argList)
    p.wait()

# ----------

if __name__ == "__main__":
    optParser = OptionParser(description="Opus unittests")
    optParser.add_option("--platform", "-p", dest="platforms", action="append", default=None, help="Platforms to test")
    optParser.add_option("--configuration", "-c", dest="configurations", action="append", default=None, help="Configurations to test")
    optParser.add_option("--test", "-t", dest="tests", action="append", default=None, help="Tests to run")
    optParser.add_option("--buildroot", "-o", dest="buildRoot", action="store", default="build", help="Opus build root")
    optParser.add_option("--builder", "-b", dest="builder", action="store", default="Native", help="Opus builder to test")
    optParser.add_option("--keepfiles", "-k", dest="keepFiles", action="store_true", default=False, help="Keep the Opus build files around")
    optParser.add_option("--jobs", "-j", dest="numJobs", action="store", type="int", default=1, help="Number of jobs to use with Opus builds")
    optParser.add_option("--verbose", "-v", dest="verbose", action="store_true", default=False, help="Verbose output")
    optParser.add_option("--debug", "-d", dest="debugSymbols", action="store_true", default=False, help="Build Opus packages with debug information")
    (options,args) = optParser.parse_args()

    if options.verbose:
        print "Options are ", options
        print "Args    are ", args

    if not options.platforms:
        raise RuntimeError("No platforms were specified")

    if not options.configurations:
        raise RuntimeError("No configurations were specified")

    tests = FindAllPackagesToTest(os.getcwd())
    if not options.tests:
        if options.verbose:
            print "All tests will run"
    else:
        if options.verbose:
            print "Tests to run are: ", options.tests
        filteredTests = []
        for test in options.tests:
            found = False
            for package in tests:
                if package.GetId() == test:
                    filteredTests.append(package)
                    found = True
                    break
            if not found:
                raise RuntimeError("Unrecognized package '%s'" % test)
        tests = filteredTests

    outputBuffer = StringIO.StringIO()
    exitCode = 0
    for package in tests:
        config = GetTestConfig(package.GetId(), options)
        if not config:
            continue
        exitCode += ExecuteTests(package, config, options, outputBuffer)

    print "----------------------------------------"
    print "Results summary"
    print "----------------------------------------"
    print outputBuffer.getvalue()
    if not options.keepFiles:
        # TODO: consider keeping track of all directories created instead
        CleanUp(options)

    sys.exit(exitCode)
<commit_msg>[050dev] Write summary out to disk (on a dated named file), show progress updates in non-verbose mode. General tidy ups of the unit test script. Default to deleting the build outputs before tests are run.<commit_after>#!/usr/bin/python

import sys
import os
import subprocess
import StringIO
import time
from testconfigurations import TestConfiguration, GetTestConfig
from optparse import OptionParser

# ----------

class Package:
    root = None
    name = None
    version = None

    def __init__(self, root, name, version):
        self.root = root
        self.name = name
        self.version = version

    def GetDescription(self):
        return "%s-%s in %s" % (self.name, self.version, self.root)

    def GetPath(self):
        return os.path.join(self.root, self.name, self.version)

    def GetId(self):
        return "-".join([self.name, self.version])

# ----------

def FindAllPackagesToTest(root, options):
    """Locate packages that can be tested"""
    if options.verbose:
        print "Locating packages under '%s'" % root
    tests = []
    for packageName in os.listdir(root):
        if packageName.startswith("."):
            continue
        packageDir = os.path.join(root, packageName)
        if os.path.isdir(packageDir):
            for packageVersion in os.listdir(packageDir):
                if packageVersion.startswith("."):
                    continue
                versionDir = os.path.join(packageDir, packageVersion)
                if os.path.isdir(versionDir):
                    package = Package(root, packageName, packageVersion)
                    if options.verbose:
                        print "\t%s" % package.GetId()
                    tests.append(package)
    return tests

def ExecuteTests(package, configuration, options, outputBuffer):
    print "Package       : ", package.GetId()
    if options.verbose:
        print "Description   : ", package.GetDescription()
        print "Builders      : ", configuration.GetBuilders()
        print "Response files: ", configuration.GetResponseFiles()
    if not options.builder in configuration.GetBuilders():
        outputBuffer.write("Package '%s' does not support the builder, '%s'" % (package.GetDescription(),options.builder))
        return 0
    for responseFile in configuration.GetResponseFiles():
        argList = []
        argList.append("Opus")
        argList.append("@" + os.path.join(os.getcwd(), responseFile))
        argList.append("-buildroot=" + options.buildRoot)
        argList.append("-builder=" + options.builder)
        if sys.platform.startswith("win"):
            argList.append("-platforms=" + ";".join(options.platforms))
            argList.append("-configurations=" + ";".join(options.configurations))
        else:
            argList.append("-platforms=" + ":".join(options.platforms))
            argList.append("-configurations=" + ":".join(options.configurations))
        argList.append("-j=" + str(options.numJobs))
        if options.debugSymbols:
            argList.append("-debugsymbols")
        argList.append("-verbosity=0")
        print "\tExecuting: %s" % " ".join(argList)
        currentDir = os.getcwd()
        try:
            p = subprocess.Popen(argList, stdout=subprocess.PIPE, stderr=sys.stdout, cwd=package.GetPath())
            (output, error) = p.communicate() # this should WAIT
        except Exception, e:
            print "Popen exception: '%s'" % str(e)
            raise
        finally:
            os.chdir(currentDir)
            if not p.returncode:
                outputBuffer.write("Package '%s' succeeded using response file '%s'\n" % (package.GetDescription(), responseFile))
                return 0
            else:
                outputBuffer.write("Package '%s' failed using response file '%s'\n" % (package.GetDescription(), responseFile))
                outputBuffer.write("Command was: '%s'" % " ".join(argList))
                outputBuffer.write(output)
                return -1

def CleanUp(options):
    argList = []
    if sys.platform.startswith("win"):
        argList.append(os.path.join(os.getcwd(), "removedebugprojects.bat"))
        argList.append("-nopause")
    else:
        argList.append(os.path.join(os.getcwd(), "removedebugprojects.sh"))
    if options.verbose:
        print "Executing: ", argList
    p = subprocess.Popen(argList)
    p.wait()

# ----------

if __name__ == "__main__":
    optParser = OptionParser(description="Opus unittests")
    optParser.add_option("--platform", "-p", dest="platforms", action="append", default=None, help="Platforms to test")
    optParser.add_option("--configuration", "-c", dest="configurations", action="append", default=None, help="Configurations to test")
    optParser.add_option("--test", "-t", dest="tests", action="append", default=None, help="Tests to run")
    optParser.add_option("--buildroot", "-o", dest="buildRoot", action="store", default="build", help="Opus build root")
    optParser.add_option("--builder", "-b", dest="builder", action="store", default="Native", help="Opus builder to test")
    optParser.add_option("--keepfiles", "-k", dest="keepFiles", action="store_true", default=False, help="Keep the Opus build files around")
    optParser.add_option("--jobs", "-j", dest="numJobs", action="store", type="int", default=1, help="Number of jobs to use with Opus builds")
    optParser.add_option("--verbose", "-v", dest="verbose", action="store_true", default=False, help="Verbose output")
    optParser.add_option("--debug", "-d", dest="debugSymbols", action="store_true", default=False, help="Build Opus packages with debug information")
    optParser.add_option("--noinitialclean", "-i", dest="noInitialClean", action="store_true", default=False, help="Disable cleaning packages before running tests")
    (options,args) = optParser.parse_args()

    if options.verbose:
        print "Options are ", options
        print "Args    are ", args

    if not options.platforms:
        raise RuntimeError("No platforms were specified")

    if not options.configurations:
        raise RuntimeError("No configurations were specified")

    if not options.noInitialClean:
        CleanUp(options)

    tests = FindAllPackagesToTest(os.getcwd(), options)
    if not options.tests:
        if options.verbose:
            print "All tests will run"
    else:
        if options.verbose:
            print "Tests to run are: ", options.tests
        filteredTests = []
        for test in options.tests:
            found = False
            for package in tests:
                if package.GetId() == test:
                    filteredTests.append(package)
                    found = True
                    break
            if not found:
                raise RuntimeError("Unrecognized package '%s'" % test)
        tests = filteredTests

    outputBuffer = StringIO.StringIO()
    exitCode = 0
    for package in tests:
        config = GetTestConfig(package.GetId(), options)
        if not config:
            continue
        exitCode += ExecuteTests(package, config, options, outputBuffer)

    print "----------------------------------------"
    print "Results summary"
    print "----------------------------------------"
    print outputBuffer.getvalue()

    logFileName = "tests_" + time.strftime("%d-%m-%YT%H-%M-%S") + ".log"
    logFile = os.open(logFileName, 257) # TODO: create and write only?
    os.write(logFile, outputBuffer.getvalue())
    os.close(logFile)
    outputBuffer.close()

    if not options.keepFiles:
        # TODO: consider keeping track of all directories created instead
        CleanUp(options)

    sys.exit(exitCode)
<|endoftext|>


<commit_before>r"""
Tests for QtAwesome.
"""
# Standard library imports
import subprocess

# Test Library imports
import pytest

# Local imports
import qtawesome as qta
from qtawesome.iconic_font import IconicFont

def test_segfault_import():
    output_number = subprocess.call('python -c "import qtawesome '
                                    '; qtawesome.icon()"', shell=True)
    assert output_number == 0

def test_unique_font_family_name(qtbot):
    """
    Test that each font used by qtawesome has a unique name. If this test
    fails, this probably means that you need to rename the family name of
    some fonts. Please see PR #98 for more details on why it is necessary and
    on how to do this.

    Regression test for Issue #107
    """
    resource = qta._instance()
    assert isinstance(resource, IconicFont)

    prefixes = list(resource.fontname.keys())
    assert prefixes

    fontnames = set(resource.fontname.values())
    assert fontnames

    assert len(prefixes) == len(fontnames)

if __name__ == "__main__":
    pytest.main()
<commit_msg>Make the test more comprehensive.<commit_after>r"""
Tests for QtAwesome.
"""
# Standard library imports
import subprocess
import collections

# Test Library imports
import pytest

# Local imports
import qtawesome as qta
from qtawesome.iconic_font import IconicFont

def test_segfault_import():
    output_number = subprocess.call('python -c "import qtawesome '
                                    '; qtawesome.icon()"', shell=True)
    assert output_number == 0

def test_unique_font_family_name(qtbot):
    """
    Test that each font used by qtawesome has a unique name. If this test
    fails, this probably means that you need to rename the family name of
    some fonts. Please see PR #98 for more details on why it is necessary and
    on how to do this.

    Regression test for Issue #107
    """
    resource = qta._instance()
    assert isinstance(resource, IconicFont)

    # Check that the fonts were loaded successfully.
    fontnames = resource.fontname.values()
    assert fontnames

    # Check that qtawesome does not load fonts with duplicate family names.
    duplicates = [fontname for fontname, count in
                  collections.Counter(fontnames).items() if count > 1]
    assert not duplicates

if __name__ == "__main__":
    pytest.main()
<|endoftext|>


<commit_before>import prett
from .. import *

@ui_extension
class TableWidget(QTableWidget, ExcitedSignalInterface,
                  prett.WidgetStringInterface, prett.WidgetIndexInterface, prett.WidgetStringListInterface):
    def set_excited_signal_connection(self):
        # noinspection PyUnresolvedReferences
        self.doubleClicked.connect(st.zero_para(self.excited.emit))

    def set_just_show_mode(self):
        self.verticalHeader().hide()
        self.resizeColumnsToContents()
        self.horizontalHeader().setResizeMode(QHeaderView.Stretch)
        self.setEditTriggers(QAbstractItemView.NoEditTriggers)
        self.setSelectionBehavior(QAbstractItemView.SelectRows)

    def set_headers(self, headers: list):
        self.setColumnCount(len(headers))
        self.setHorizontalHeaderLabels(headers)

    def auto_scaling_column(self):
        self.resizeColumnsToContents()
        col_count = self.columnCount()
        col_width = sum(list([self.columnWidth(i) for i in range(col_count)]))
        total_col_width = self.width() - self.verticalScrollBar().width()
        for i in range(col_count):
            self.setColumnWidth(i, self.columnWidth(i) / col_width * total_col_width)

    def resizeEvent(self, *args, **kwargs):
        self.auto_scaling_column()

    class TableWidgetItem:
        def __init__(self, parent: 'TableWidget'):
            self.parent = parent

        @property
        def row_count(self):
            return self.parent.rowCount()

        @property
        def col_count(self):
            return self.parent.columnCount()

        def item_text(self, row, col):
            return self.parent.item(row, col).text()

    class StringItem(TableWidgetItem, prett.WidgetStringItem):
        """get/set current table row text"""

        def get_value(self):
            if self.parent.index.value >=0 :
                current_row = self.parent.currentRow()
                col_count = self.parent.columnCount()
                value = dict()
                for i in range(col_count):
                    value[self.parent.horizontalHeaderItem(i).text()] = self.item_text(current_row, i)
                return value
            return None

        def set_value(self, value):
            assert isinstance(value, dict)
            if len(value) is not self.col_count:
                raise ValueError('Value length must equal to column count')

            texts = self.parent.string_list.value
            assert isinstance(texts, list)
            if value is None:
                self.parent.index.value = 0
            else:
                for key, v in value.items():
                    value[key] = str(v)
                if value in texts:
                    self.parent.index.value = texts.index(value)
                else:
                    header_labels = list(self.parent.horizontalHeaderItem(i).text() for i in range(self.col_count))
                    self.parent.setRowCount(self.row_count + 1)
                    for i in range(self.col_count):
                        item_text = value[header_labels[i]]
                        if item_text is None:
                            raise ValueError("key value doesn't match header label")
                        table_item = QTableWidgetItem(item_text)
                        table_item.setTextAlignment(Qt.AlignCenter)
                        self.parent.setItem(self.row_count - 1, i, table_item)
                    self.parent.index.value = self.row_count - 1

        def set_changed_connection(self):
            # noinspection PyUnresolvedReferences
            self.parent.currentCellChanged.connect(self.check_change)

    class IndexItem(TableWidgetItem, prett.IndexItem):
        """get/set current select row"""

        def get_value(self):
            return self.parent.currentRow()

        def set_value(self, value):
            value = value or 0
            self.parent.selectRow(value)

        def set_changed_connection(self):
            # noinspection PyUnresolvedReferences
            self.parent.currentCellChanged(self.check_change)

    class StringsItem(TableWidgetItem, prett.StringsItem):
        """ get all tablewidget item text"""

        def get_value(self):
            table_texts = []
            for row in range(self.row_count):
                row_dict = dict()
                for col in range(self.col_count):
                    row_dict[self.parent.horizontalHeaderItem(col).text()] = self.item_text(row, col)
                table_texts.append(row_dict)
            return table_texts

        def set_value(self, value: list):
            value = value or []
            assert isinstance(value, list)

            for i in range(self.col_count):
                self.parent.removeRow(0)
            for row_dict in value:
                self.parent.string.value = row_dict
            self.check_change()
<commit_msg>repaire table—_widget set string_list bug<commit_after>import prett
from .. import *

@ui_extension
class TableWidget(QTableWidget, ExcitedSignalInterface,
                  prett.WidgetStringInterface, prett.WidgetIndexInterface, prett.WidgetStringListInterface):
    def set_excited_signal_connection(self):
        # noinspection PyUnresolvedReferences
        self.doubleClicked.connect(st.zero_para(self.excited.emit))

    def set_just_show_mode(self):
        self.verticalHeader().hide()
        self.resizeColumnsToContents()
        self.horizontalHeader().setResizeMode(QHeaderView.Stretch)
        self.setEditTriggers(QAbstractItemView.NoEditTriggers)
        self.setSelectionBehavior(QAbstractItemView.SelectRows)

    def set_headers(self, headers: list):
        self.setColumnCount(len(headers))
        self.setHorizontalHeaderLabels(headers)

    def auto_scaling_column(self):
        self.resizeColumnsToContents()
        col_count = self.columnCount()
        col_width = sum(list([self.columnWidth(i) for i in range(col_count)]))
        total_col_width = self.width() - self.verticalScrollBar().width()
        for i in range(col_count):
            self.setColumnWidth(i, self.columnWidth(i) / col_width * total_col_width)

    def resizeEvent(self, *args, **kwargs):
        self.auto_scaling_column()

    class TableWidgetItem:
        def __init__(self, parent: 'TableWidget'):
            self.parent = parent

        @property
        def row_count(self):
            return self.parent.rowCount()

        @property
        def col_count(self):
            return self.parent.columnCount()

        def item_text(self, row, col):
            return self.parent.item(row, col).text()

    class StringItem(TableWidgetItem, prett.WidgetStringItem):
        """get/set current table row text"""

        def get_value(self):
            if self.parent.index.value >=0 :
                current_row = self.parent.currentRow()
                col_count = self.parent.columnCount()
                value = dict()
                for i in range(col_count):
                    value[self.parent.horizontalHeaderItem(i).text()] = self.item_text(current_row, i)
                return value
            return None

        def set_value(self, value):
            assert isinstance(value, dict)
            if len(value) is not self.col_count:
                raise ValueError('Value length must equal to column count')

            texts = self.parent.string_list.value
            assert isinstance(texts, list)
            if value is None:
                self.parent.index.value = 0
            else:
                for key, v in value.items():
                    value[key] = str(v)
                if value in texts:
                    self.parent.index.value = texts.index(value)
                else:
                    header_labels = list(self.parent.horizontalHeaderItem(i).text() for i in range(self.col_count))
                    self.parent.setRowCount(self.row_count + 1)
                    for i in range(self.col_count):
                        item_text = value[header_labels[i]]
                        if item_text is None:
                            raise ValueError("key value doesn't match header label")
                        table_item = QTableWidgetItem(item_text)
                        table_item.setTextAlignment(Qt.AlignCenter)
                        self.parent.setItem(self.row_count - 1, i, table_item)
                    self.parent.index.value = self.row_count - 1

        def set_changed_connection(self):
            # noinspection PyUnresolvedReferences
            self.parent.currentCellChanged.connect(self.check_change)

    class IndexItem(TableWidgetItem, prett.IndexItem):
        """get/set current select row"""

        def get_value(self):
            return self.parent.currentRow()

        def set_value(self, value):
            value = value or 0
            self.parent.selectRow(value)

        def set_changed_connection(self):
            # noinspection PyUnresolvedReferences
            self.parent.currentCellChanged(self.check_change)

    class StringsItem(TableWidgetItem, prett.StringsItem):
        """ get all tablewidget item text"""

        def get_value(self):
            table_texts = []
            for row in range(self.row_count):
                row_dict = dict()
                for col in range(self.col_count):
                    row_dict[self.parent.horizontalHeaderItem(col).text()] = self.item_text(row, col)
                table_texts.append(row_dict)
            return table_texts

        def set_value(self, value: list):
            value = value or []
            assert isinstance(value, list)

            for i in range(self.row_count):
                self.parent.removeRow(0)
            for row_dict in value:
                self.parent.string.value = row_dict
            self.check_change()
<|endoftext|>


<commit_before>import sys
import requests
import json
import urllib2

# Backend config
import config

from w1thermsensor import W1ThermSensor

temps = dict()

for sensor in W1ThermSensor.get_available_sensors():
    temp = sensor.get_temperature()
    print("Sensor %s has temperature %.2f" % (sensor.id, temp))
    try:
        log(sensor.id, temp)
    except requests.ConnectionError:
        print("No connection. Could not log temperature.")

def log(sensor_id, temp):
    payload = { 'id': sensor_id, 'temp': temp }

    req = urllib2.Request(config.ENDPOINT)
    req.add_header('Content-Type', 'application/json')
    req.add_header('x-token', config.X_TOKEN)

    response = urllib2.urlopen(req, json.dumps(payload))
<commit_msg>Fix palju script.<commit_after>import sys
import requests
import json
import urllib2

# Backend config
import config

from w1thermsensor import W1ThermSensor

def log(sensor_id, temp):
    payload = { 'id': sensor_id, 'temp': temp }

    req = urllib2.Request(config.ENDPOINT)
    req.add_header('Content-Type', 'application/json')
    req.add_header('x-token', config.X_TOKEN)

    response = urllib2.urlopen(req, json.dumps(payload))

temps = dict()

for sensor in W1ThermSensor.get_available_sensors():
    temp = sensor.get_temperature()
    print("Sensor %s has temperature %.2f" % (sensor.id, temp))
    try:
        log(sensor.id, temp)
    except requests.ConnectionError:
        print("No connection. Could not log temperature.")

<|endoftext|>


<commit_before>from rest_framework.permissions import IsAuthenticated, BasePermission

class PublicDetailPrivateListing(IsAuthenticated):

    """
    Permission class for our custom use case.

    * Always give permission for a ``detail`` request
    * Only give permission for ``listing`` request if user is admin of the project
    * Allow access to ``/projects`` (user's projects listing)
    """

    def has_permission(self, request, view):
        is_authenticated = super().has_permission(request, view)
        if is_authenticated:
            if view.basename == 'projects' and any([
                    view.action == 'list',
                    view.action == 'create',  # used to create Form in BrowsableAPIRenderer
                    view.action is None,  # needed for BrowsableAPIRenderer
            ]):
                # hitting ``/projects/``, allowing
                return True

            # NOTE: ``superproject`` is an action name, defined by the class
            # method under ``ProjectViewSet``. We should apply the same
            # permissions restrictions than for a detail action (since it only
            # returns one superproject if exists). ``list`` and ``retrieve`` are
            # DRF standard action names (same as ``update`` or ``partial_update``).
            if view.detail and view.action in ('list', 'retrieve', 'superproject'):
                # detail view is only allowed on list/retrieve actions (not
                # ``update`` or ``partial_update``).
                return True

            project = view._get_parent_project()
            if view.has_admin_permission(request.user, project):
                return True

        return False

class IsProjectAdmin(BasePermission):

    """Grant permission if user has admin rights on the Project."""

    def has_permission(self, request, view):
        project = view._get_parent_project()
        if view.has_admin_permission(request.user, project):
            return True
<commit_msg>Split PublicDetailPrivateListing permission class<commit_after>from rest_framework.permissions import IsAuthenticated, BasePermission

from readthedocs.core.utils.extend import SettingsOverrideObject

class UserProjectsListing(BasePermission):
    def has_permission(self, request, view):
        if view.basename == 'projects' and any([
                view.action == 'list',
                view.action == 'create',  # used to create Form in BrowsableAPIRenderer
                view.action is None,  # needed for BrowsableAPIRenderer
        ]):
            # hitting ``/projects/``, allowing
            return True

class PublicDetailPrivateListing(IsAuthenticated):

    """
    Permission class for our custom use case.

    * Always give permission for a ``detail`` request
    * Only give permission for ``listing`` request if user is admin of the project
    * Allow access to ``/projects`` (user's projects listing)
    """

    def has_permission(self, request, view):
        # NOTE: ``superproject`` is an action name, defined by the class
        # method under ``ProjectViewSet``. We should apply the same
        # permissions restrictions than for a detail action (since it only
        # returns one superproject if exists). ``list`` and ``retrieve`` are
        # DRF standard action names (same as ``update`` or ``partial_update``).
        if view.detail and view.action in ('list', 'retrieve', 'superproject'):
            # detail view is only allowed on list/retrieve actions (not
            # ``update`` or ``partial_update``).
            return True

        project = view._get_parent_project()
        if view.has_admin_permission(request.user, project):
            return True

        return False

class IsProjectAdmin(BasePermission):

    """Grant permission if user has admin rights on the Project."""

    def has_permission(self, request, view):
        project = view._get_parent_project()
        if view.has_admin_permission(request.user, project):
            return True
<|endoftext|>


<commit_before># Copyright (c) 2015, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import stix
import stix.bindings.stix_common as common_binding

from .datetimewithprecision import DateTimeWithPrecision
from .structured_text import StructuredText

class Activity(stix.Entity):
    _binding = common_binding
    _binding_class = common_binding.ActivityType
    _namespace = 'http://stix.mitre.org/common-1'

    def __init__(self):
        self.date_time = None
        self.description = None

    @property
    def date_time(self):
        return self._date_time

    @date_time.setter
    def date_time(self, value):
        self._set_var(DateTimeWithPrecision, date_time=value)

    @property
    def description(self):
        return self._description

    @description.setter
    def description(self, value):
        self._set_var(StructuredText, description=value)

    def to_obj(self, return_obj=None, ns_info=None):
        super(Activity, self).to_obj(return_obj=return_obj, ns_info=ns_info)

        if not return_obj:
            return_obj = self._binding_class()

        if self.date_time:
            return_obj.Date_Time = self.date_time.to_obj(ns_info=ns_info)
        if self.description:
            return_obj.Description = self.description.to_obj(ns_info=ns_info)

        return return_obj

    @classmethod
    def from_obj(cls, obj, return_obj=None):
        if not obj:
            return None

        if not return_obj:
            return_obj = cls()

        return_obj.date_time = \
                DateTimeWithPrecision.from_obj(obj.Date_Time)
        return_obj.description = StructuredText.from_obj(obj.Description)

        return return_obj

    def to_dict(self):
        return super(Activity, self).to_dict()

    @classmethod
    def from_dict(cls, dict_repr, return_obj=None):
        if not dict_repr:
            return None

        if not return_obj:
            return_obj = cls()

        return_obj.date_time = \
                DateTimeWithPrecision.from_dict(dict_repr.get('date_time'))
        return_obj.description = \
                StructuredText.from_dict(dict_repr.get('description'))

        return return_obj

<commit_msg>Fixed style issues in common/activity.py<commit_after># Copyright (c) 2015, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import stix
import stix.bindings.stix_common as common_binding

from .datetimewithprecision import DateTimeWithPrecision
from .structured_text import StructuredText

class Activity(stix.Entity):
    _binding = common_binding
    _binding_class = common_binding.ActivityType
    _namespace = 'http://stix.mitre.org/common-1'

    def __init__(self):
        self.date_time = None
        self.description = None

    @property
    def date_time(self):
        return self._date_time

    @date_time.setter
    def date_time(self, value):
        self._set_var(DateTimeWithPrecision, date_time=value)

    @property
    def description(self):
        return self._description

    @description.setter
    def description(self, value):
        self._set_var(StructuredText, description=value)

    def to_obj(self, return_obj=None, ns_info=None):
        super(Activity, self).to_obj(return_obj=return_obj, ns_info=ns_info)

        if not return_obj:
            return_obj = self._binding_class()

        if self.date_time:
            return_obj.Date_Time = self.date_time.to_obj(ns_info=ns_info)
        if self.description:
            return_obj.Description = self.description.to_obj(ns_info=ns_info)

        return return_obj

    @classmethod
    def from_obj(cls, obj, return_obj=None):
        if not obj:
            return None

        if not return_obj:
            return_obj = cls()

        return_obj.date_time = DateTimeWithPrecision.from_obj(obj.Date_Time)
        return_obj.description = StructuredText.from_obj(obj.Description)

        return return_obj

    def to_dict(self):
        return super(Activity, self).to_dict()

    @classmethod
    def from_dict(cls, dict_repr, return_obj=None):
        if not dict_repr:
            return None

        if not return_obj:
            return_obj = cls()

        get = dict_repr.get
        return_obj.date_time = DateTimeWithPrecision.from_dict(get('date_time'))
        return_obj.description = StructuredText.from_dict(get('description'))

        return return_obj
<|endoftext|>