<commit_before>from __future__ import division
import numpy as np
from numpy.random import random
na = np.newaxis
import scipy.stats as stats
import scipy.special as special
import scipy.linalg
from numpy.core.umath_tests import inner1d

import general

### data abstraction

def atleast_2d(data):
    # NOTE: can't use np.atleast_2d because if it's 1D we want axis 1 to be the
    # singleton and axis 0 to be the sequence index
    if data.ndim == 1:
        return data.reshape((-1,1))
    return data

def mask_data(data):
    return np.ma.masked_array(np.nan_to_num(data),np.isnan(data),fill_value=0.,hard_mask=True)

def gi(data):
    out = (np.isnan(atleast_2d(data)).sum(1) == 0).ravel()
    return out if len(out) != 0 else None

def getdatasize(data):
    if isinstance(data,np.ma.masked_array):
        return data.shape[0] - data.mask.reshape((data.shape[0],-1))[:,0].sum()
    elif isinstance(data,np.ndarray):
        if len(data) == 0:
            return 0
        return data[gi(data)].shape[0]
    elif isinstance(data,list):
        return sum(getdatasize(d) for d in data)
    else:
        # handle unboxed case for convenience
        assert isinstance(data,int) or isinstance(data,float)
        return 1

def getdatadimension(data):
    if isinstance(data,np.ndarray):
        assert data.ndim > 1
        return data.shape[1]
    elif isinstance(data,list):
        assert len(data) > 0
        return getdatadimension(data[0])
    else:
        # handle unboxed case for convenience
        assert isinstance(data,int) or isinstance(data,float)
        return 1

def combinedata(datas):
    ret = []
    for data in datas:
        if isinstance(data,np.ma.masked_array):
            ret.append(np.ma.compress_rows(data))
        if isinstance(data,np.ndarray):
            ret.append(data)
        elif isinstance(data,list):
            ret.extend(combinedata(data))
        else:
            # handle unboxed case for convenience
            assert isinstance(data,int) or isinstance(data,float)
            ret.append(np.atleast_1d(data))
    return ret

def flattendata(data):
    # data is either an array (possibly a maskedarray) or a list of arrays
    if isinstance(data,np.ndarray):
        return data
    elif isinstance(data,list) or isinstance(data,tuple):
        if any(isinstance(d,np.ma.MaskedArray) for d in data):
            return np.concatenate([np.ma.compress_rows(d) for d in data])
        else:
            return np.concatenate(data)
    else:
        # handle unboxed case for convenience
        assert isinstance(data,int) or isinstance(data,float)
        return np.atleast_1d(data)

### misc

def cov(a):
    # return np.cov(a,rowvar=0,bias=1)
    mu = a.mean(0)
    if isinstance(a,np.ma.MaskedArray):
        return np.ma.dot(a.T,a)/a.count(0)[0] - np.ma.outer(mu,mu)
    else:
        return a.T.dot(a)/a.shape[0] - np.outer(mu,mu)

### Sampling functions

def sample_discrete(distn,size=[],dtype=np.int32):
    'samples from a one-dimensional finite pmf'
    distn = np.atleast_1d(distn)
    assert (distn >=0).all() and distn.ndim == 1
    if (0 == distn).all():
        return np.random.randint(distn.shape[0],size=size)
    cumvals = np.cumsum(distn)
    return np.sum(np.array(random(size))[...,na] * cumvals[-1] > cumvals, axis=-1,dtype=dtype)

def sample_discrete_from_log(p_log,return_lognorms=False,axis=0,dtype=np.int32):
    'samples log probability array along specified axis'
    lognorms = np.logaddexp.reduce(p_log,axis=axis)
    cumvals = np.exp(p_log - np.expand_dims(lognorms,axis)).cumsum(axis)
    thesize = np.array(p_log.shape)
    thesize[axis] = 1
    randvals = random(size=thesize) * \
            np.reshape(cumvals[[slice(None) if i is not axis else -1
                for i in range(p_log.ndim)]],thesize)
    samples = np.sum(randvals > cumvals,axis=axis,dtype=dtype)
    if return_lognorms:
        return samples, lognorms
    else:
        return samples

def sample_markov(T,trans_matrix,init_state_distn):
    out = np.empty(T,dtype=np.int32)
    out[0] = sample_discrete(init_state_distn)
    for t in range(1,T):
        out[t] = sample_discrete(trans_matrix[out[t-1]])
    return out

def sample_niw(mu,lmbda,kappa,nu):
    '''
    Returns a sample from the normal/inverse-wishart distribution, conjugate
    prior for (simultaneously) unknown mean and unknown covariance in a
    Gaussian likelihood model. Returns covariance.
    '''
    # code is based on Matlab's method
    # reference: p. 87 in Gelman's Bayesian Data Analysis
    assert nu > lmbda.shape[0] and kappa > 0

    # first sample Sigma ~ IW(lmbda,nu)
    lmbda = sample_invwishart(lmbda,nu)
    # then sample mu | Lambda ~ N(mu, Lambda/kappa)
    mu = np.random.multivariate_normal(mu,lmbda / kappa)

    return mu, lmbda

def sample_invwishart(S,nu):
    # TODO make a version that returns the cholesky
    # TODO allow passing in chol/cholinv of matrix parameter lmbda
    # TODO lowmem! memoize! dchud (eigen?)
    n = S.shape[0]
    chol = np.linalg.cholesky(S)

    if (nu <= 81+n) and (nu == np.round(nu)):
        x = np.random.randn(nu,n)
    else:
        x = np.diag(np.sqrt(np.atleast_1d(stats.chi2.rvs(nu-np.arange(n)))))
        x[np.triu_indices_from(x,1)] = np.random.randn(n*(n-1)//2)
    R = np.linalg.qr(x,'r')
    T = scipy.linalg.solve_triangular(R.T,chol.T,lower=True).T
    return np.dot(T,T.T)

def sample_wishart(sigma, nu):
    n = sigma.shape[0]
    chol = np.linalg.cholesky(sigma)

    # use matlab's heuristic for choosing between the two different sampling schemes
    if (nu <= 81+n) and (nu == round(nu)):
        # direct
        X = np.dot(chol,np.random.normal(size=(n,nu)))
    else:
        A = np.diag(np.sqrt(np.random.chisquare(nu - np.arange(n))))
        A[np.tri(n,k=-1,dtype=bool)] = np.random.normal(size=(n*(n-1)/2.))
        X = np.dot(chol,A)

    return np.dot(X,X.T)

def sample_mn(M,U=None,Uinv=None,V=None,Vinv=None):
    assert (U is None) ^ (Uinv is None)
    assert (V is None) ^ (Vinv is None)

    G = np.random.normal(size=M.shape)

    if U is not None:
        G = np.dot(np.linalg.cholesky(U),G)
    else:
        G = np.linalg.solve(np.linalg.cholesky(Uinv).T,G)

    if V is not None:
        G = np.dot(G,np.linalg.cholesky(V).T)
    else:
        G = np.linalg.solve(np.linalg.cholesky(Vinv).T,G.T).T

    return M + G

def sample_mniw(nu,S,M,K=None,Kinv=None):
    assert (K is None) ^ (Kinv is None)
    Sigma = sample_invwishart(S,nu)
    if K is not None:
        return sample_mn(M=M,U=Sigma,V=K), Sigma
    else:
        return sample_mn(M=M,U=Sigma,Vinv=Kinv), Sigma

def sample_pareto(x_m,alpha):
    return x_m + np.random.pareto(alpha)

def sample_crp_tablecounts(concentration,customers,colweights):
    m = np.zeros_like(customers)
    tot = customers.sum()
    randseq = np.random.random(tot)

    starts = np.empty_like(customers)
    starts[0,0] = 0
    starts.flat[1:] = np.cumsum(np.ravel(customers)[:customers.size-1])

    for (i,j), n in np.ndenumerate(customers):
        w = colweights[j]
        for k in xrange(n):
            m[i,j] += randseq[starts[i,j]+k] \
                    < (concentration * w) / (k + concentration * w)

    return m

### Entropy
def invwishart_entropy(sigma,nu,chol=None):
    D = sigma.shape[0]
    chol = np.linalg.cholesky(sigma) if chol is None else chol
    Elogdetlmbda = special.digamma((nu-np.arange(D))/2).sum() + D*np.log(2) - 2*np.log(chol.diagonal()).sum()
    return invwishart_log_partitionfunction(sigma,nu,chol)-(nu-D-1)/2*Elogdetlmbda + nu*D/2

def invwishart_log_partitionfunction(sigma,nu,chol=None):
    D = sigma.shape[0]
    chol = np.linalg.cholesky(sigma) if chol is None else chol
    return -1*(nu*np.log(chol.diagonal()).sum() - (nu*D/2*np.log(2) + D*(D-1)/4*np.log(np.pi) \
            + special.gammaln((nu-np.arange(D))/2).sum()))

### Predictive

def multivariate_t_loglik(y,nu,mu,lmbda):
    # returns the log value
    d = len(mu)
    yc = np.array(y-mu,ndmin=2)
    L = np.linalg.cholesky(lmbda)
    ys = scipy.linalg.solve_triangular(L,yc.T,overwrite_b=True,lower=True)
    return scipy.special.gammaln((nu+d)/2.) - scipy.special.gammaln(nu/2.) \
            - (d/2.)*np.log(nu*np.pi) - np.log(L.diagonal()).sum() \
            - (nu+d)/2.*np.log1p(1./nu*inner1d(ys.T,ys.T))

def beta_predictive(priorcounts,newcounts):
    prior_nsuc, prior_nfail = priorcounts
    nsuc, nfail = newcounts

    numer = scipy.special.gammaln(np.array([nsuc+prior_nsuc,
        nfail+prior_nfail, prior_nsuc+prior_nfail])).sum()
    denom = scipy.special.gammaln(np.array([prior_nsuc, prior_nfail,
        prior_nsuc+prior_nfail+nsuc+nfail])).sum()
    return numer - denom

### Statistical tests

def two_sample_t_statistic(pop1, pop2):
    pop1, pop2 = (flattendata(p) for p in (pop1, pop2))
    t = (pop1.mean(0) - pop2.mean(0)) / np.sqrt(pop1.var(0)/pop1.shape[0] + pop2.var(0)/pop2.shape[0])
    p = 2*stats.t.sf(np.abs(t),np.minimum(pop1.shape[0],pop2.shape[0]))
    return t,p

def f_statistic(pop1, pop2): # TODO test
    pop1, pop2 = (flattendata(p) for p in (pop1, pop2))
    var1, var2 = pop1.var(0), pop2.var(0)
    n1, n2 = np.where(var1 >= var2, pop1.shape[0], pop2.shape[0]), \
             np.where(var1 >= var2, pop2.shape[0], pop1.shape[0])
    var1, var2 = np.maximum(var1,var2), np.minimum(var1,var2)
    f = var1 / var2
    p = stats.f.sf(f,n1,n2)
    return f,p

<commit_msg>add mniw_expectedstats<commit_after>from __future__ import division
import numpy as np
from numpy.random import random
na = np.newaxis
import scipy.stats as stats
import scipy.special as special
import scipy.linalg
from numpy.core.umath_tests import inner1d

import general

### data abstraction

# the data type is ndarrays OR lists of ndarrays
# type Data = ndarray | [ndarray]

def atleast_2d(data):
    # NOTE: can't use np.atleast_2d because if it's 1D we want axis 1 to be the
    # singleton and axis 0 to be the sequence index
    if data.ndim == 1:
        return data.reshape((-1,1))
    return data

def mask_data(data):
    return np.ma.masked_array(np.nan_to_num(data),np.isnan(data),fill_value=0.,hard_mask=True)

def gi(data):
    out = (np.isnan(atleast_2d(data)).sum(1) == 0).ravel()
    return out if len(out) != 0 else None

def getdatasize(data):
    if isinstance(data,np.ma.masked_array):
        return data.shape[0] - data.mask.reshape((data.shape[0],-1))[:,0].sum()
    elif isinstance(data,np.ndarray):
        if len(data) == 0:
            return 0
        return data[gi(data)].shape[0]
    elif isinstance(data,list):
        return sum(getdatasize(d) for d in data)
    else:
        # handle unboxed case for convenience
        assert isinstance(data,int) or isinstance(data,float)
        return 1

def getdatadimension(data):
    if isinstance(data,np.ndarray):
        assert data.ndim > 1
        return data.shape[1]
    elif isinstance(data,list):
        assert len(data) > 0
        return getdatadimension(data[0])
    else:
        # handle unboxed case for convenience
        assert isinstance(data,int) or isinstance(data,float)
        return 1

def combinedata(datas):
    ret = []
    for data in datas:
        if isinstance(data,np.ma.masked_array):
            ret.append(np.ma.compress_rows(data))
        if isinstance(data,np.ndarray):
            ret.append(data)
        elif isinstance(data,list):
            ret.extend(combinedata(data))
        else:
            # handle unboxed case for convenience
            assert isinstance(data,int) or isinstance(data,float)
            ret.append(np.atleast_1d(data))
    return ret

def flattendata(data):
    # data is either an array (possibly a maskedarray) or a list of arrays
    if isinstance(data,np.ndarray):
        return data
    elif isinstance(data,list) or isinstance(data,tuple):
        if any(isinstance(d,np.ma.MaskedArray) for d in data):
            return np.concatenate([np.ma.compress_rows(d) for d in data])
        else:
            return np.concatenate(data)
    else:
        # handle unboxed case for convenience
        assert isinstance(data,int) or isinstance(data,float)
        return np.atleast_1d(data)

### misc

def cov(a):
    # return np.cov(a,rowvar=0,bias=1)
    mu = a.mean(0)
    if isinstance(a,np.ma.MaskedArray):
        return np.ma.dot(a.T,a)/a.count(0)[0] - np.ma.outer(mu,mu)
    else:
        return a.T.dot(a)/a.shape[0] - np.outer(mu,mu)

### Sampling functions

def sample_discrete(distn,size=[],dtype=np.int32):
    'samples from a one-dimensional finite pmf'
    distn = np.atleast_1d(distn)
    assert (distn >=0).all() and distn.ndim == 1
    if (0 == distn).all():
        return np.random.randint(distn.shape[0],size=size)
    cumvals = np.cumsum(distn)
    return np.sum(np.array(random(size))[...,na] * cumvals[-1] > cumvals, axis=-1,dtype=dtype)

def sample_discrete_from_log(p_log,return_lognorms=False,axis=0,dtype=np.int32):
    'samples log probability array along specified axis'
    lognorms = np.logaddexp.reduce(p_log,axis=axis)
    cumvals = np.exp(p_log - np.expand_dims(lognorms,axis)).cumsum(axis)
    thesize = np.array(p_log.shape)
    thesize[axis] = 1
    randvals = random(size=thesize) * \
            np.reshape(cumvals[[slice(None) if i is not axis else -1
                for i in range(p_log.ndim)]],thesize)
    samples = np.sum(randvals > cumvals,axis=axis,dtype=dtype)
    if return_lognorms:
        return samples, lognorms
    else:
        return samples

def sample_markov(T,trans_matrix,init_state_distn):
    out = np.empty(T,dtype=np.int32)
    out[0] = sample_discrete(init_state_distn)
    for t in range(1,T):
        out[t] = sample_discrete(trans_matrix[out[t-1]])
    return out

def sample_niw(mu,lmbda,kappa,nu):
    '''
    Returns a sample from the normal/inverse-wishart distribution, conjugate
    prior for (simultaneously) unknown mean and unknown covariance in a
    Gaussian likelihood model. Returns covariance.
    '''
    # code is based on Matlab's method
    # reference: p. 87 in Gelman's Bayesian Data Analysis
    assert nu > lmbda.shape[0] and kappa > 0

    # first sample Sigma ~ IW(lmbda,nu)
    lmbda = sample_invwishart(lmbda,nu)
    # then sample mu | Lambda ~ N(mu, Lambda/kappa)
    mu = np.random.multivariate_normal(mu,lmbda / kappa)

    return mu, lmbda

def sample_invwishart(S,nu):
    # TODO make a version that returns the cholesky
    # TODO allow passing in chol/cholinv of matrix parameter lmbda
    # TODO lowmem! memoize! dchud (eigen?)
    n = S.shape[0]
    chol = np.linalg.cholesky(S)

    if (nu <= 81+n) and (nu == np.round(nu)):
        x = np.random.randn(nu,n)
    else:
        x = np.diag(np.sqrt(np.atleast_1d(stats.chi2.rvs(nu-np.arange(n)))))
        x[np.triu_indices_from(x,1)] = np.random.randn(n*(n-1)//2)
    R = np.linalg.qr(x,'r')
    T = scipy.linalg.solve_triangular(R.T,chol.T,lower=True).T
    return np.dot(T,T.T)

def sample_wishart(sigma, nu):
    n = sigma.shape[0]
    chol = np.linalg.cholesky(sigma)

    # use matlab's heuristic for choosing between the two different sampling schemes
    if (nu <= 81+n) and (nu == round(nu)):
        # direct
        X = np.dot(chol,np.random.normal(size=(n,nu)))
    else:
        A = np.diag(np.sqrt(np.random.chisquare(nu - np.arange(n))))
        A[np.tri(n,k=-1,dtype=bool)] = np.random.normal(size=(n*(n-1)/2.))
        X = np.dot(chol,A)

    return np.dot(X,X.T)

def sample_mn(M,U=None,Uinv=None,V=None,Vinv=None):
    assert (U is None) ^ (Uinv is None)
    assert (V is None) ^ (Vinv is None)

    G = np.random.normal(size=M.shape)

    if U is not None:
        G = np.dot(np.linalg.cholesky(U),G)
    else:
        G = np.linalg.solve(np.linalg.cholesky(Uinv).T,G)

    if V is not None:
        G = np.dot(G,np.linalg.cholesky(V).T)
    else:
        G = np.linalg.solve(np.linalg.cholesky(Vinv).T,G.T).T

    return M + G

def sample_mniw(nu,S,M,K=None,Kinv=None):
    assert (K is None) ^ (Kinv is None)
    Sigma = sample_invwishart(S,nu)
    if K is not None:
        return sample_mn(M=M,U=Sigma,V=K), Sigma
    else:
        return sample_mn(M=M,U=Sigma,Vinv=Kinv), Sigma

def mniw_expectedstats(nu,S,M,K=None,Kinv=None):
    # NOTE: could speed this up with chol factorizing S, not re-solving
    assert (K is None) ^ (Kinv is None)
    m = M.shape[0]
    K = K if K is not None else np.linalg.inv(Kinv)

    E_Sigmainv = nu*np.linalg.inv(S)
    E_Sigmainv_A = nu*np.linalg.solve(S,M)
    E_AT_Sigmainv_A = m*K + nu*M.T.dot(np.linalg.solve(S,M))
    E_logdetSigmainv = special.digamma((nu-np.arange(m))/2.).sum() \
        + m*np.log(2) - np.linalg.slogdet(S)[1]

    return E_Sigmainv, E_Sigmainv_A, E_AT_Sigmainv_A, E_logdetSigmainv

def sample_pareto(x_m,alpha):
    return x_m + np.random.pareto(alpha)

def sample_crp_tablecounts(concentration,customers,colweights):
    m = np.zeros_like(customers)
    tot = customers.sum()
    randseq = np.random.random(tot)

    starts = np.empty_like(customers)
    starts[0,0] = 0
    starts.flat[1:] = np.cumsum(np.ravel(customers)[:customers.size-1])

    for (i,j), n in np.ndenumerate(customers):
        w = colweights[j]
        for k in xrange(n):
            m[i,j] += randseq[starts[i,j]+k] \
                    < (concentration * w) / (k + concentration * w)

    return m

### Entropy
def invwishart_entropy(sigma,nu,chol=None):
    D = sigma.shape[0]
    chol = np.linalg.cholesky(sigma) if chol is None else chol
    Elogdetlmbda = special.digamma((nu-np.arange(D))/2).sum() + D*np.log(2) - 2*np.log(chol.diagonal()).sum()
    return invwishart_log_partitionfunction(sigma,nu,chol)-(nu-D-1)/2*Elogdetlmbda + nu*D/2

def invwishart_log_partitionfunction(sigma,nu,chol=None):
    D = sigma.shape[0]
    chol = np.linalg.cholesky(sigma) if chol is None else chol
    return -1*(nu*np.log(chol.diagonal()).sum() - (nu*D/2*np.log(2) + D*(D-1)/4*np.log(np.pi) \
            + special.gammaln((nu-np.arange(D))/2).sum()))

### Predictive

def multivariate_t_loglik(y,nu,mu,lmbda):
    # returns the log value
    d = len(mu)
    yc = np.array(y-mu,ndmin=2)
    L = np.linalg.cholesky(lmbda)
    ys = scipy.linalg.solve_triangular(L,yc.T,overwrite_b=True,lower=True)
    return scipy.special.gammaln((nu+d)/2.) - scipy.special.gammaln(nu/2.) \
            - (d/2.)*np.log(nu*np.pi) - np.log(L.diagonal()).sum() \
            - (nu+d)/2.*np.log1p(1./nu*inner1d(ys.T,ys.T))

def beta_predictive(priorcounts,newcounts):
    prior_nsuc, prior_nfail = priorcounts
    nsuc, nfail = newcounts

    numer = scipy.special.gammaln(np.array([nsuc+prior_nsuc,
        nfail+prior_nfail, prior_nsuc+prior_nfail])).sum()
    denom = scipy.special.gammaln(np.array([prior_nsuc, prior_nfail,
        prior_nsuc+prior_nfail+nsuc+nfail])).sum()
    return numer - denom

### Statistical tests

def two_sample_t_statistic(pop1, pop2):
    pop1, pop2 = (flattendata(p) for p in (pop1, pop2))
    t = (pop1.mean(0) - pop2.mean(0)) / np.sqrt(pop1.var(0)/pop1.shape[0] + pop2.var(0)/pop2.shape[0])
    p = 2*stats.t.sf(np.abs(t),np.minimum(pop1.shape[0],pop2.shape[0]))
    return t,p

def f_statistic(pop1, pop2): # TODO test
    pop1, pop2 = (flattendata(p) for p in (pop1, pop2))
    var1, var2 = pop1.var(0), pop2.var(0)
    n1, n2 = np.where(var1 >= var2, pop1.shape[0], pop2.shape[0]), \
             np.where(var1 >= var2, pop2.shape[0], pop1.shape[0])
    var1, var2 = np.maximum(var1,var2), np.minimum(var1,var2)
    f = var1 / var2
    p = stats.f.sf(f,n1,n2)
    return f,p

<|endoftext|>


<commit_before># provide support to insight API servers
# see also https://github.com/bitpay/insight-api

import decimal
import json
import io

from .agent import request, urlencode, urlopen

from pycoin.block import Block
from pycoin.convention import btc_to_satoshi
from pycoin.encoding import double_sha256
from pycoin.merkle import merkle
from pycoin.networks.default import get_current_netcode
from pycoin.serialize import b2h, b2h_rev, h2b, h2b_rev
from pycoin.tx.script import tools
from pycoin.tx.Tx import Spendable, Tx, TxIn, TxOut

class InsightProvider(object):
    def __init__(self, base_url="https://insight.bitpay.com", base_path="/api/", netcode=None):
        if netcode is None:
            netcode = get_current_netcode()
        while base_url[-1] == '/':
            base_url = base_url[:-1]
        self.base_url = base_url
        self.base_path = base_path

    def get_blockchain_tip(self):
        URL = "%s%sstatus?q=getLastBlockHash" % (self.base_url, self.base_path)
        d = urlopen(URL).read().decode("utf8")
        r = json.loads(d)
        return h2b_rev(r.get("lastblockhash"))

    def get_blockheader(self, block_hash):
        return self.get_blockheader_with_transaction_hashes(block_hash)[0]

    def get_blockheader_with_transaction_hashes(self, block_hash):
        URL = "%s%sblock/%s" % (self.base_url, self.base_path, b2h_rev(block_hash))
        r = json.loads(urlopen(URL).read().decode("utf8"))
        version = r.get("version")
        previous_block_hash = h2b_rev(r.get("previousblockhash"))
        merkle_root = h2b_rev(r.get("merkleroot"))
        timestamp = r.get("time")
        difficulty = int(r.get("bits"), 16)
        nonce = int(r.get("nonce"))
        tx_hashes = [h2b_rev(tx_hash) for tx_hash in r.get("tx")]
        blockheader = Block(version, previous_block_hash, merkle_root, timestamp, difficulty, nonce)
        if blockheader.hash() != block_hash:
            return None, None
        calculated_hash = merkle(tx_hashes, double_sha256)
        if calculated_hash != merkle_root:
            return None, None
        blockheader.height = r.get("height")
        return blockheader, tx_hashes

    def get_block_height(self, block_hash):
        return self.get_blockheader_with_transaction_hashes(block_hash)[0].height

    def tx_for_tx_hash(self, tx_hash):
        URL = "%s%stx/%s" % (self.base_url, self.base_path, b2h_rev(tx_hash))
        r = json.loads(urlopen(URL).read().decode("utf8"))
        tx = tx_from_json_dict(r)
        if tx.hash() == tx_hash:
            return tx
        return None

    def get_tx_confirmation_block(self, tx_hash):
        return self.get_tx(tx_hash).confirmation_block_hash

    def spendables_for_address(self, bitcoin_address):
        """
        Return a list of Spendable objects for the
        given bitcoin address.
        """
        URL = "%s%saddr/%s/utxo" % (self.base_url, self.base_path, bitcoin_address)
        r = json.loads(urlopen(URL).read().decode("utf8"))
        spendables = []
        for u in r:
            coin_value = btc_to_satoshi(str(u.get("amount")))
            script = h2b(u.get("scriptPubKey"))
            previous_hash = h2b_rev(u.get("txid"))
            previous_index = u.get("vout")
            spendables.append(Spendable(coin_value, script, previous_hash, previous_index))
        return spendables

    def spendables_for_addresses(self, bitcoin_addresses):
        spendables = []
        for addr in bitcoin_addresses:
            spendables.extend(self.spendables_for_address(addr))
        return spendables

    def send_tx(self, tx):
        s = io.BytesIO()
        tx.stream(s)
        tx_as_hex = b2h(s.getvalue())
        data = urlencode(dict(rawtx=tx_as_hex)).encode("utf8")
        URL = "%s%stx/send" % (self.base_url, self.base_path)
        try:
            d = urlopen(URL, data=data).read()
            return d
        except request.HTTPError as err:
            if err.code == 400:
                raise ValueError(err.readline())
            raise err

def tx_from_json_dict(r):
    version = r.get("version")
    lock_time = r.get("locktime")
    txs_in = []
    for vin in r.get("vin"):
        if "coinbase" in vin:
            previous_hash = b'\0' * 32
            script = h2b(vin.get("coinbase"))
            previous_index = 4294967295
        else:
            previous_hash = h2b_rev(vin.get("txid"))
            scriptSig = vin.get("scriptSig")
            if "hex" in scriptSig:
                script = h2b(scriptSig.get("hex"))
            else:
                script = tools.compile(scriptSig.get("asm"))
            previous_index = vin.get("vout")
        sequence = vin.get("sequence")
        txs_in.append(TxIn(previous_hash, previous_index, script, sequence))
    txs_out = []
    for vout in r.get("vout"):
        coin_value = btc_to_satoshi(decimal.Decimal(vout.get("value")))
        script = tools.compile(vout.get("scriptPubKey").get("asm"))
        txs_out.append(TxOut(coin_value, script))
    tx = Tx(version, txs_in, txs_out, lock_time)
    bh = r.get("blockhash")
    if bh:
        bh = h2b_rev(bh)
    tx.confirmation_block_hash = bh
    return tx
<commit_msg>Remove base_path from InsightProvider.<commit_after># provide support to insight API servers
# see also https://github.com/bitpay/insight-api

import decimal
import json
import io

from .agent import request, urlencode, urlopen

from pycoin.block import Block
from pycoin.convention import btc_to_satoshi
from pycoin.encoding import double_sha256
from pycoin.merkle import merkle
from pycoin.networks.default import get_current_netcode
from pycoin.serialize import b2h, b2h_rev, h2b, h2b_rev
from pycoin.tx.script import tools
from pycoin.tx.Tx import Spendable, Tx, TxIn, TxOut

class InsightProvider(object):
    def __init__(self, base_url="https://insight.bitpay.com", netcode=None):
        if netcode is None:
            netcode = get_current_netcode()
        while base_url[-1] == '/':
            base_url = base_url[:-1]
        self.base_url = base_url

    def get_blockchain_tip(self):
        URL = "%s/status?q=getLastBlockHash" % self.base_url
        d = urlopen(URL).read().decode("utf8")
        r = json.loads(d)
        return h2b_rev(r.get("lastblockhash"))

    def get_blockheader(self, block_hash):
        return self.get_blockheader_with_transaction_hashes(block_hash)[0]

    def get_blockheader_with_transaction_hashes(self, block_hash):
        URL = "%s/block/%s" % (self.base_url, b2h_rev(block_hash))
        r = json.loads(urlopen(URL).read().decode("utf8"))
        version = r.get("version")
        previous_block_hash = h2b_rev(r.get("previousblockhash"))
        merkle_root = h2b_rev(r.get("merkleroot"))
        timestamp = r.get("time")
        difficulty = int(r.get("bits"), 16)
        nonce = int(r.get("nonce"))
        tx_hashes = [h2b_rev(tx_hash) for tx_hash in r.get("tx")]
        blockheader = Block(version, previous_block_hash, merkle_root, timestamp, difficulty, nonce)
        if blockheader.hash() != block_hash:
            return None, None
        calculated_hash = merkle(tx_hashes, double_sha256)
        if calculated_hash != merkle_root:
            return None, None
        blockheader.height = r.get("height")
        return blockheader, tx_hashes

    def get_block_height(self, block_hash):
        return self.get_blockheader_with_transaction_hashes(block_hash)[0].height

    def tx_for_tx_hash(self, tx_hash):
        URL = "%s/tx/%s" % (self.base_url, b2h_rev(tx_hash))
        r = json.loads(urlopen(URL).read().decode("utf8"))
        tx = tx_from_json_dict(r)
        if tx.hash() == tx_hash:
            return tx
        return None

    def get_tx_confirmation_block(self, tx_hash):
        return self.get_tx(tx_hash).confirmation_block_hash

    def spendables_for_address(self, bitcoin_address):
        """
        Return a list of Spendable objects for the
        given bitcoin address.
        """
        URL = "%s/addr/%s/utxo" % (self.base_url, bitcoin_address)
        r = json.loads(urlopen(URL).read().decode("utf8"))
        spendables = []
        for u in r:
            coin_value = btc_to_satoshi(str(u.get("amount")))
            script = h2b(u.get("scriptPubKey"))
            previous_hash = h2b_rev(u.get("txid"))
            previous_index = u.get("vout")
            spendables.append(Spendable(coin_value, script, previous_hash, previous_index))
        return spendables

    def spendables_for_addresses(self, bitcoin_addresses):
        spendables = []
        for addr in bitcoin_addresses:
            spendables.extend(self.spendables_for_address(addr))
        return spendables

    def send_tx(self, tx):
        s = io.BytesIO()
        tx.stream(s)
        tx_as_hex = b2h(s.getvalue())
        data = urlencode(dict(rawtx=tx_as_hex)).encode("utf8")
        URL = "%s/tx/send" % self.base_url
        try:
            d = urlopen(URL, data=data).read()
            return d
        except request.HTTPError as err:
            if err.code == 400:
                raise ValueError(err.readline())
            raise err

def tx_from_json_dict(r):
    version = r.get("version")
    lock_time = r.get("locktime")
    txs_in = []
    for vin in r.get("vin"):
        if "coinbase" in vin:
            previous_hash = b'\0' * 32
            script = h2b(vin.get("coinbase"))
            previous_index = 4294967295
        else:
            previous_hash = h2b_rev(vin.get("txid"))
            scriptSig = vin.get("scriptSig")
            if "hex" in scriptSig:
                script = h2b(scriptSig.get("hex"))
            else:
                script = tools.compile(scriptSig.get("asm"))
            previous_index = vin.get("vout")
        sequence = vin.get("sequence")
        txs_in.append(TxIn(previous_hash, previous_index, script, sequence))
    txs_out = []
    for vout in r.get("vout"):
        coin_value = btc_to_satoshi(decimal.Decimal(vout.get("value")))
        script = tools.compile(vout.get("scriptPubKey").get("asm"))
        txs_out.append(TxOut(coin_value, script))
    tx = Tx(version, txs_in, txs_out, lock_time)
    bh = r.get("blockhash")
    if bh:
        bh = h2b_rev(bh)
    tx.confirmation_block_hash = bh
    return tx
<|endoftext|>


<commit_before># provide support to insight API servers
# see also https://github.com/bitpay/insight-api

import decimal
import json
import logging

try:
    from urllib2 import urlopen
except ImportError:
    from urllib.request import urlopen

from pycoin.block import BlockHeader
from pycoin.convention import btc_to_satoshi
from pycoin.encoding import double_sha256
from pycoin.merkle import merkle
from pycoin.serialize import b2h, b2h_rev, h2b, h2b_rev
from pycoin.tx.script import tools
from pycoin.tx import Spendable, Tx, TxIn, TxOut

class InsightService(object):
    def __init__(self, base_url):
        while base_url[-1] == '/':
            base_url = base_url[:-1]
        self.base_url = base_url

    def get_blockchain_tip(self):
        "https://search.bitaccess.ca/api/status?q=getLastBlockHash"
        URL = "%s/api/status?q=getLastBlockHash" % self.base_url
        d = urlopen(URL).read().decode("utf8")
        r = json.loads(d)
        return h2b_rev(r.get("lastblockhash"))

    def get_blockheader(self, block_hash):
        return self.get_blockheader_with_transaction_hashes(block_hash)[0]

    def get_blockheader_with_transaction_hashes(self, block_hash):
        URL = "%s/api/block/%s" % (self.base_url, b2h_rev(block_hash))
        r = json.loads(urlopen(URL).read().decode("utf8"))
        version = r.get("version")
        previous_block_hash = h2b_rev(r.get("previousblockhash"))
        merkle_root = h2b_rev(r.get("merkleroot"))
        timestamp = r.get("time")
        difficulty = int(r.get("bits"), 16)
        nonce = int(r.get("nonce"))
        tx_hashes = [h2b_rev(tx_hash) for tx_hash in r.get("tx")]
        blockheader = BlockHeader(version, previous_block_hash, merkle_root, timestamp, difficulty, nonce)
        if blockheader.hash() != block_hash:
            return None, None
        calculated_hash = merkle(tx_hashes, double_sha256)
        if calculated_hash != merkle_root:
            return None, None
        blockheader.height = r.get("height")
        return blockheader, tx_hashes

    def get_block_height(self, block_hash):
        return self.get_blockheader_with_transaction_hashes(block_hash)[0].height

    def get_tx(self, tx_hash):
        URL = "%s/api/tx/%s" % (self.base_url, b2h_rev(tx_hash))
        r = json.loads(urlopen(URL).read().decode("utf8"))
        tx = tx_from_json_dict(r)
        if tx.hash() == tx_hash:
            return tx
        return None

    def get_tx_confirmation_block(self, tx_hash):
        return self.get_tx(tx_hash).confirmation_block_hash

    def spendables_for_address(self, bitcoin_address):
        """
        Return a list of Spendable objects for the
        given bitcoin address.
        """
        URL = "%s/api/addr/%s/utxo" % (self.base_url, bitcoin_address)
        r = json.loads(urlopen(URL).read().decode("utf8"))
        spendables = []
        for u in r:
            coin_value = btc_to_satoshi(u.get("amount"))
            script = h2b(u.get("scriptPubKey"))
            previous_hash = h2b_rev(u.get("txid"))
            previous_index = u.get("vout")
            spendables.append(Spendable(coin_value, script, previous_hash, previous_index))
        return spendables

    def spendables_for_addresses(self, bitcoin_addresses):
        spendables = []
        for addr in bitcoin_addresses:
            spendables.extend(self.spendables_for_address(addr))
        return spendables

    def send_tx(self, tx):
        # TODO: make this handle errors better
        s = io.BytesIO()
        tx.stream(s)
        tx_as_hex = b2h(s.getvalue())
        data = urlencode(dict(rawtx=tx_as_hex)).encode("utf8")
        URL = "%s/api/tx/send" % self.base_url
        try:
            d = urlopen(URL, data=data).read()
            return d
        except HTTPError as ex:
            logging.exception("problem in send_tx %s", tx.id())
            raise ex

def tx_from_json_dict(r):
    version = r.get("version")
    lock_time = r.get("locktime")
    txs_in = []
    for vin in r.get("vin"):
        if "coinbase" in vin:
            previous_hash = b'\0' * 32
            script = h2b(vin.get("coinbase"))
            previous_index = 4294967295
        else:
            previous_hash = h2b_rev(vin.get("txid"))
            script = tools.compile(vin.get("scriptSig").get("asm"))
            previous_index = vin.get("vout")
        sequence = vin.get("sequence")
        txs_in.append(TxIn(previous_hash, previous_index, script, sequence))
    txs_out = []
    for vout in r.get("vout"):
        coin_value = btc_to_satoshi(decimal.Decimal(vout.get("value")))
        script = tools.compile(vout.get("scriptPubKey").get("asm"))
        txs_out.append(TxOut(coin_value, script))
    tx = Tx(version, txs_in, txs_out, lock_time)
    bh = r.get("blockhash")
    if bh:
        bh = h2b_rev(bh)
    tx.confirmation_block_hash = bh
    return tx
<commit_msg>Fix problem with off-by-one rounding error.<commit_after># provide support to insight API servers
# see also https://github.com/bitpay/insight-api

import decimal
import json
import logging

try:
    from urllib2 import urlopen
except ImportError:
    from urllib.request import urlopen

from pycoin.block import BlockHeader
from pycoin.convention import btc_to_satoshi
from pycoin.encoding import double_sha256
from pycoin.merkle import merkle
from pycoin.serialize import b2h, b2h_rev, h2b, h2b_rev
from pycoin.tx.script import tools
from pycoin.tx import Spendable, Tx, TxIn, TxOut

class InsightService(object):
    def __init__(self, base_url):
        while base_url[-1] == '/':
            base_url = base_url[:-1]
        self.base_url = base_url

    def get_blockchain_tip(self):
        "https://search.bitaccess.ca/api/status?q=getLastBlockHash"
        URL = "%s/api/status?q=getLastBlockHash" % self.base_url
        d = urlopen(URL).read().decode("utf8")
        r = json.loads(d)
        return h2b_rev(r.get("lastblockhash"))

    def get_blockheader(self, block_hash):
        return self.get_blockheader_with_transaction_hashes(block_hash)[0]

    def get_blockheader_with_transaction_hashes(self, block_hash):
        URL = "%s/api/block/%s" % (self.base_url, b2h_rev(block_hash))
        r = json.loads(urlopen(URL).read().decode("utf8"))
        version = r.get("version")
        previous_block_hash = h2b_rev(r.get("previousblockhash"))
        merkle_root = h2b_rev(r.get("merkleroot"))
        timestamp = r.get("time")
        difficulty = int(r.get("bits"), 16)
        nonce = int(r.get("nonce"))
        tx_hashes = [h2b_rev(tx_hash) for tx_hash in r.get("tx")]
        blockheader = BlockHeader(version, previous_block_hash, merkle_root, timestamp, difficulty, nonce)
        if blockheader.hash() != block_hash:
            return None, None
        calculated_hash = merkle(tx_hashes, double_sha256)
        if calculated_hash != merkle_root:
            return None, None
        blockheader.height = r.get("height")
        return blockheader, tx_hashes

    def get_block_height(self, block_hash):
        return self.get_blockheader_with_transaction_hashes(block_hash)[0].height

    def get_tx(self, tx_hash):
        URL = "%s/api/tx/%s" % (self.base_url, b2h_rev(tx_hash))
        r = json.loads(urlopen(URL).read().decode("utf8"))
        tx = tx_from_json_dict(r)
        if tx.hash() == tx_hash:
            return tx
        return None

    def get_tx_confirmation_block(self, tx_hash):
        return self.get_tx(tx_hash).confirmation_block_hash

    def spendables_for_address(self, bitcoin_address):
        """
        Return a list of Spendable objects for the
        given bitcoin address.
        """
        URL = "%s/api/addr/%s/utxo" % (self.base_url, bitcoin_address)
        r = json.loads(urlopen(URL).read().decode("utf8"))
        spendables = []
        for u in r:
            coin_value = btc_to_satoshi(str(u.get("amount")))
            script = h2b(u.get("scriptPubKey"))
            previous_hash = h2b_rev(u.get("txid"))
            previous_index = u.get("vout")
            spendables.append(Spendable(coin_value, script, previous_hash, previous_index))
        return spendables

    def spendables_for_addresses(self, bitcoin_addresses):
        spendables = []
        for addr in bitcoin_addresses:
            spendables.extend(self.spendables_for_address(addr))
        return spendables

    def send_tx(self, tx):
        # TODO: make this handle errors better
        s = io.BytesIO()
        tx.stream(s)
        tx_as_hex = b2h(s.getvalue())
        data = urlencode(dict(rawtx=tx_as_hex)).encode("utf8")
        URL = "%s/api/tx/send" % self.base_url
        try:
            d = urlopen(URL, data=data).read()
            return d
        except HTTPError as ex:
            logging.exception("problem in send_tx %s", tx.id())
            raise ex

def tx_from_json_dict(r):
    version = r.get("version")
    lock_time = r.get("locktime")
    txs_in = []
    for vin in r.get("vin"):
        if "coinbase" in vin:
            previous_hash = b'\0' * 32
            script = h2b(vin.get("coinbase"))
            previous_index = 4294967295
        else:
            previous_hash = h2b_rev(vin.get("txid"))
            script = tools.compile(vin.get("scriptSig").get("asm"))
            previous_index = vin.get("vout")
        sequence = vin.get("sequence")
        txs_in.append(TxIn(previous_hash, previous_index, script, sequence))
    txs_out = []
    for vout in r.get("vout"):
        coin_value = btc_to_satoshi(decimal.Decimal(vout.get("value")))
        script = tools.compile(vout.get("scriptPubKey").get("asm"))
        txs_out.append(TxOut(coin_value, script))
    tx = Tx(version, txs_in, txs_out, lock_time)
    bh = r.get("blockhash")
    if bh:
        bh = h2b_rev(bh)
    tx.confirmation_block_hash = bh
    return tx
<|endoftext|>


<commit_before>from django import forms
from django.contrib.admin.widgets import AdminFileWidget
from django.forms.models import inlineformset_factory, BaseInlineFormSet
from django.utils.translation import ugettext_lazy as _

from multi_email_field.forms import MultiEmailField

from pycon.sponsorship.models import Sponsor, SponsorBenefit, SponsorLevel

class SponsorDetailsForm(forms.ModelForm):
    contact_emails = MultiEmailField(
        help_text=_(u"Please enter one email address per line.")
    )

    class Meta:
        model = Sponsor
        fields = ["name",
                  "contact_name",
                  "contact_emails",
                  "contact_phone",
                  "contact_address",
                  "external_url",
                  "display_url",
                  "web_description",
                  "web_logo",
                  ]
        widgets = {
            'web_description': forms.widgets.Textarea(attrs={'cols': 40, 'rows': 5}),
        }

class SponsorApplicationForm(SponsorDetailsForm):

    class Meta(SponsorDetailsForm.Meta):
        fields = SponsorDetailsForm.Meta.fields + [
            "level",
            "wants_table",
            "wants_booth",
        ]

    def __init__(self, *args, **kwargs):
        self.user = kwargs.pop("user")
        kwargs.update({
            "initial": {
                "contact_name": self.user.get_full_name(),
                "contact_emails": [self.user.email],
            }
        })
        super(SponsorApplicationForm, self).__init__(*args, **kwargs)
        # TODO: there should be a way to turn off each level as it
        # fills, instead of having to edit code. Plus, this should
        # really be a radio button, where the unavailable ones stay
        # visible but grayed out with "Full" or "Out" next to them.
        self.fields['level'].queryset = SponsorLevel.objects.exclude(
            name='Open Source and Community')

    def save(self, commit=True):
        obj = super(SponsorApplicationForm, self).save(commit=False)
        obj.applicant = self.user
        if commit:
            obj.save()
        return obj

class SponsorBenefitsInlineFormSet(BaseInlineFormSet):

    def _construct_form(self, i, **kwargs):
        form = super(SponsorBenefitsInlineFormSet, self)._construct_form(i, **kwargs)

        # only include the relevant data fields for this benefit type
        fields = form.instance.data_fields()
        form.fields = dict((k, v) for (k, v) in form.fields.items() if k in fields + ["id"])
        for field in fields:
            # don't need a label, the form template will label it with the benefit name
            form.fields[field].label = ""

            # provide word limit as help_text
            if form.instance.benefit.type in ["text", "richtext"] and form.instance.max_words:
                form.fields[field].help_text = u"maximum %s words" % form.instance.max_words

            # use admin file widget that shows currently uploaded file
            if field == "upload":
                form.fields[field].widget = AdminFileWidget()

        return form

SponsorBenefitsFormSet = inlineformset_factory(
    Sponsor, SponsorBenefit,
    formset=SponsorBenefitsInlineFormSet,
    can_delete=False, extra=0,
    fields=["text", "upload"]
)

class SponsorEmailForm(forms.Form):
    from_ = forms.EmailField(widget=forms.TextInput(attrs={'class': 'fullwidth-input'}))
    cc = forms.CharField(help_text=_(u"(comma-separated addresses)"),
                         required=False,
                         widget=forms.TextInput(attrs={'class': 'fullwidth-input'}))
    bcc = forms.CharField(help_text=_(u"(comma-separated addresses)"),
                          required=False,
                          widget=forms.TextInput(attrs={'class': 'fullwidth-input'}))
    subject = forms.CharField(widget=forms.TextInput(attrs={'class': 'fullwidth-input'}))
    body = forms.CharField(widget=forms.Textarea(attrs={'class': 'fullwidth-textarea'}))
<commit_msg>Temporarily exclude another form option for 2016<commit_after>from django import forms
from django.contrib.admin.widgets import AdminFileWidget
from django.forms.models import inlineformset_factory, BaseInlineFormSet
from django.utils.translation import ugettext_lazy as _

from multi_email_field.forms import MultiEmailField

from pycon.sponsorship.models import Sponsor, SponsorBenefit, SponsorLevel

class SponsorDetailsForm(forms.ModelForm):
    contact_emails = MultiEmailField(
        help_text=_(u"Please enter one email address per line.")
    )

    class Meta:
        model = Sponsor
        fields = ["name",
                  "contact_name",
                  "contact_emails",
                  "contact_phone",
                  "contact_address",
                  "external_url",
                  "display_url",
                  "web_description",
                  "web_logo",
                  ]
        widgets = {
            'web_description': forms.widgets.Textarea(attrs={'cols': 40, 'rows': 5}),
        }

class SponsorApplicationForm(SponsorDetailsForm):

    class Meta(SponsorDetailsForm.Meta):
        fields = SponsorDetailsForm.Meta.fields + [
            "level",
            "wants_table",
            "wants_booth",
        ]

    def __init__(self, *args, **kwargs):
        self.user = kwargs.pop("user")
        kwargs.update({
            "initial": {
                "contact_name": self.user.get_full_name(),
                "contact_emails": [self.user.email],
            }
        })
        super(SponsorApplicationForm, self).__init__(*args, **kwargs)
        # TODO: there should be a way to turn off each level as it
        # fills, instead of having to edit code. Plus, this should
        # really be a radio button, where the unavailable ones stay
        # visible but grayed out with "Full" or "Out" next to them.
        self.fields['level'].queryset = SponsorLevel.objects.exclude(
            name='Open Source and Community').exclude(
            name='Silver')

    def save(self, commit=True):
        obj = super(SponsorApplicationForm, self).save(commit=False)
        obj.applicant = self.user
        if commit:
            obj.save()
        return obj

class SponsorBenefitsInlineFormSet(BaseInlineFormSet):

    def _construct_form(self, i, **kwargs):
        form = super(SponsorBenefitsInlineFormSet, self)._construct_form(i, **kwargs)

        # only include the relevant data fields for this benefit type
        fields = form.instance.data_fields()
        form.fields = dict((k, v) for (k, v) in form.fields.items() if k in fields + ["id"])
        for field in fields:
            # don't need a label, the form template will label it with the benefit name
            form.fields[field].label = ""

            # provide word limit as help_text
            if form.instance.benefit.type in ["text", "richtext"] and form.instance.max_words:
                form.fields[field].help_text = u"maximum %s words" % form.instance.max_words

            # use admin file widget that shows currently uploaded file
            if field == "upload":
                form.fields[field].widget = AdminFileWidget()

        return form

SponsorBenefitsFormSet = inlineformset_factory(
    Sponsor, SponsorBenefit,
    formset=SponsorBenefitsInlineFormSet,
    can_delete=False, extra=0,
    fields=["text", "upload"]
)

class SponsorEmailForm(forms.Form):
    from_ = forms.EmailField(widget=forms.TextInput(attrs={'class': 'fullwidth-input'}))
    cc = forms.CharField(help_text=_(u"(comma-separated addresses)"),
                         required=False,
                         widget=forms.TextInput(attrs={'class': 'fullwidth-input'}))
    bcc = forms.CharField(help_text=_(u"(comma-separated addresses)"),
                          required=False,
                          widget=forms.TextInput(attrs={'class': 'fullwidth-input'}))
    subject = forms.CharField(widget=forms.TextInput(attrs={'class': 'fullwidth-input'}))
    body = forms.CharField(widget=forms.Textarea(attrs={'class': 'fullwidth-textarea'}))
<|endoftext|>


<commit_before>import errno
import os
import shutil
from girder_worker_utils.transform import Transform
from girder_worker_utils.transforms.girder_io import (
    GirderUploadJobArtifact,
    GirderClientTransform,
    GirderUploadToFolder,
    GirderUploadToItem
)
from . import TemporaryVolume, Connect, NamedOutputPipe, _maybe_transform

class ProgressPipe(Transform):

    def __init__(self, name='.girder_progress', volume=TemporaryVolume.default):
        self.name = name
        self._volume = volume

    def transform(self, task=None, **kwargs):
        from girder_worker.docker.stream_adapter import JobProgressAdapter

        self._volume.transform(**kwargs)

        # TODO What do we do is job_manager is None? When can it me None?
        job_mgr = task.job_manager
        # For now we are using JobProgressAdapter which is part of core, we should
        # probably add a copy to the docker package to break to reference to core.
        return Connect(NamedOutputPipe(self.name, volume=self._volume),
                       JobProgressAdapter(job_mgr)).transform(**kwargs)

class GirderFileIdToStream(GirderClientTransform):
    def __init__(self, _id, **kwargs):
        super(GirderFileIdToStream, self).__init__(**kwargs)
        self.file_id = _id

    def transform(self, **kwargs):
        from girder_worker.docker.io.girder import (
            GirderFileStreamReader
        )
        return GirderFileStreamReader(self.gc, self.file_id)

class GirderFileIdToVolume(GirderClientTransform):
    def __init__(self, _id, volume=TemporaryVolume.default, filename=None, **kwargs):
        super(GirderFileIdToVolume, self).__init__(**kwargs)
        self._file_id = str(_id)
        self._volume = volume
        self._filename = filename
        self._file_path = None

    def _create_file_path(self, root):
        if self._filename is None:
            # If no filename is explicitly passed, we read the filename from Girder
            # and put it in its own directory named by its UUID.
            filename = self.gc.getFile(self._file_id)['name']
            path = os.path.join(root, self._file_id)
            try:
                os.mkdir(path)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise

            return os.path.join(self._file_id, filename), os.path.join(path, filename)
        else:
            return self._filename, os.path.join(root, self._filename)

    def transform(self, **kwargs):
        self._volume.transform(**kwargs)
        dir = self._volume.host_path
        rel_path, self._file_path = self._create_file_path(dir)

        self.gc.downloadFile(self._file_id, self._file_path)

        # Return the path inside the container
        return os.path.join(self._volume.container_path, rel_path)

    def cleanup(self, **kwargs):
        if self._file_path is not None:
            shutil.rmtree(self._file_path, ignore_errors=True)

    def _repr_model_(self):
        if self._filename:
            template = u'<{module}.{cls}: File ID={id} -> "{fname}">'
        else:
            template = u'<{module}.{cls}: File ID={id}>'
        return template.format(
            module=self.__module__, cls=self.__class__.__name__, id=self._file_id,
            fname=self._filename)

class GirderFolderIdToVolume(GirderClientTransform):
    def __init__(self, _id, volume=TemporaryVolume.default, folder_name=None, **kwargs):
        super(GirderFolderIdToVolume, self).__init__(**kwargs)
        self._folder_id = str(_id)
        self._volume = volume
        self._folder_name = folder_name
        self._folder_path = None

    def _create_folder_path(self, root):
        if self._folder_name is None:
            # If no folder name is explicitly passed, we read the filename from Girder
            # and put it in its own directory named by its UUID.
            self._folder_name = self.gc.getFolder(self._folder_id)['name']
        path = os.path.join(root, self._folder_id, self._folder_name)
        try:
            os.makedirs(path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        return os.path.join(self._folder_id, self._folder_name), path

    def transform(self, **kwargs):
        self._volume.transform(**kwargs)
        dir = self._volume.host_path
        rel_path, self._folder_path = self._create_folder_path(dir)

        self.gc.downloadFolderRecursive(self._folder_id, self._folder_path)

        # Return the path inside the container
        return os.path.join(self._volume.container_path, rel_path)

    def cleanup(self, **kwargs):
        if self._folder_path is not None:
            shutil.rmtree(self._folder_path, ignore_errors=True)

    def _repr_model_(self):
        return '<%s.%s: Folder ID=%s -> "%s">' % (
            self.__module__, self.__class__.__name__, self._folder_id, self._folder_path)

class GirderUploadVolumePathToItem(GirderUploadToItem):
    def __init__(self, volumepath, item_id, delete_file=False, **kwargs):
        item_id = str(item_id)
        super(GirderUploadVolumePathToItem, self).__init__(item_id, delete_file, **kwargs)
        self._volumepath = volumepath

    # We ignore the "result"
    def transform(self, *args, **kwargs):
        path = _maybe_transform(self._volumepath, *args, **kwargs)

        return super(GirderUploadVolumePathToItem, self).transform(path)

class GirderUploadVolumePathToFolder(GirderUploadToFolder):
    def __init__(self, volumepath, folder_id, delete_file=False, **kwargs):
        super(GirderUploadVolumePathToFolder, self).__init__(str(folder_id), delete_file, **kwargs)
        self._volumepath = volumepath

    def transform(self, *args, **kwargs):
        path = _maybe_transform(self._volumepath, *args, **kwargs)
        return super(GirderUploadVolumePathToFolder, self).transform(path)

class GirderUploadVolumePathJobArtifact(GirderUploadJobArtifact):
    def __init__(self, volumepath, job_id=None, **kwargs):
        if job_id is not None:
            job_id = str(job_id)
        super(GirderUploadVolumePathJobArtifact, self).__init__(job_id, **kwargs)
        self._volumepath = volumepath

    def transform(self, *args, **kwargs):
        path = _maybe_transform(self._volumepath, *args, **kwargs)
        return super(GirderUploadVolumePathJobArtifact, self).transform(path)
<commit_msg>Add name option for job artifacts<commit_after>import errno
import os
import shutil
from girder_worker_utils.transform import Transform
from girder_worker_utils.transforms.girder_io import (
    GirderUploadJobArtifact,
    GirderClientTransform,
    GirderUploadToFolder,
    GirderUploadToItem
)
from . import TemporaryVolume, Connect, NamedOutputPipe, _maybe_transform

class ProgressPipe(Transform):

    def __init__(self, name='.girder_progress', volume=TemporaryVolume.default):
        self.name = name
        self._volume = volume

    def transform(self, task=None, **kwargs):
        from girder_worker.docker.stream_adapter import JobProgressAdapter

        self._volume.transform(**kwargs)

        # TODO What do we do is job_manager is None? When can it me None?
        job_mgr = task.job_manager
        # For now we are using JobProgressAdapter which is part of core, we should
        # probably add a copy to the docker package to break to reference to core.
        return Connect(NamedOutputPipe(self.name, volume=self._volume),
                       JobProgressAdapter(job_mgr)).transform(**kwargs)

class GirderFileIdToStream(GirderClientTransform):
    def __init__(self, _id, **kwargs):
        super(GirderFileIdToStream, self).__init__(**kwargs)
        self.file_id = _id

    def transform(self, **kwargs):
        from girder_worker.docker.io.girder import (
            GirderFileStreamReader
        )
        return GirderFileStreamReader(self.gc, self.file_id)

class GirderFileIdToVolume(GirderClientTransform):
    def __init__(self, _id, volume=TemporaryVolume.default, filename=None, **kwargs):
        super(GirderFileIdToVolume, self).__init__(**kwargs)
        self._file_id = str(_id)
        self._volume = volume
        self._filename = filename
        self._file_path = None

    def _create_file_path(self, root):
        if self._filename is None:
            # If no filename is explicitly passed, we read the filename from Girder
            # and put it in its own directory named by its UUID.
            filename = self.gc.getFile(self._file_id)['name']
            path = os.path.join(root, self._file_id)
            try:
                os.mkdir(path)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise

            return os.path.join(self._file_id, filename), os.path.join(path, filename)
        else:
            return self._filename, os.path.join(root, self._filename)

    def transform(self, **kwargs):
        self._volume.transform(**kwargs)
        dir = self._volume.host_path
        rel_path, self._file_path = self._create_file_path(dir)

        self.gc.downloadFile(self._file_id, self._file_path)

        # Return the path inside the container
        return os.path.join(self._volume.container_path, rel_path)

    def cleanup(self, **kwargs):
        if self._file_path is not None:
            shutil.rmtree(self._file_path, ignore_errors=True)

    def _repr_model_(self):
        if self._filename:
            template = u'<{module}.{cls}: File ID={id} -> "{fname}">'
        else:
            template = u'<{module}.{cls}: File ID={id}>'
        return template.format(
            module=self.__module__, cls=self.__class__.__name__, id=self._file_id,
            fname=self._filename)

class GirderFolderIdToVolume(GirderClientTransform):
    def __init__(self, _id, volume=TemporaryVolume.default, folder_name=None, **kwargs):
        super(GirderFolderIdToVolume, self).__init__(**kwargs)
        self._folder_id = str(_id)
        self._volume = volume
        self._folder_name = folder_name
        self._folder_path = None

    def _create_folder_path(self, root):
        if self._folder_name is None:
            # If no folder name is explicitly passed, we read the filename from Girder
            # and put it in its own directory named by its UUID.
            self._folder_name = self.gc.getFolder(self._folder_id)['name']
        path = os.path.join(root, self._folder_id, self._folder_name)
        try:
            os.makedirs(path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        return os.path.join(self._folder_id, self._folder_name), path

    def transform(self, **kwargs):
        self._volume.transform(**kwargs)
        dir = self._volume.host_path
        rel_path, self._folder_path = self._create_folder_path(dir)

        self.gc.downloadFolderRecursive(self._folder_id, self._folder_path)

        # Return the path inside the container
        return os.path.join(self._volume.container_path, rel_path)

    def cleanup(self, **kwargs):
        if self._folder_path is not None:
            shutil.rmtree(self._folder_path, ignore_errors=True)

    def _repr_model_(self):
        return '<%s.%s: Folder ID=%s -> "%s">' % (
            self.__module__, self.__class__.__name__, self._folder_id, self._folder_path)

class GirderUploadVolumePathToItem(GirderUploadToItem):
    def __init__(self, volumepath, item_id, delete_file=False, **kwargs):
        item_id = str(item_id)
        super(GirderUploadVolumePathToItem, self).__init__(item_id, delete_file, **kwargs)
        self._volumepath = volumepath

    # We ignore the "result"
    def transform(self, *args, **kwargs):
        path = _maybe_transform(self._volumepath, *args, **kwargs)

        return super(GirderUploadVolumePathToItem, self).transform(path)

class GirderUploadVolumePathToFolder(GirderUploadToFolder):
    def __init__(self, volumepath, folder_id, delete_file=False, **kwargs):
        super(GirderUploadVolumePathToFolder, self).__init__(str(folder_id), delete_file, **kwargs)
        self._volumepath = volumepath

    def transform(self, *args, **kwargs):
        path = _maybe_transform(self._volumepath, *args, **kwargs)
        return super(GirderUploadVolumePathToFolder, self).transform(path)

class GirderUploadVolumePathJobArtifact(GirderUploadJobArtifact):
    def __init__(self, volumepath, job_id=None, name=None, **kwargs):
        if job_id is not None:
            job_id = str(job_id)
        super(GirderUploadVolumePathJobArtifact, self).__init__(job_id, name, **kwargs)
        self._volumepath = volumepath

    def transform(self, *args, **kwargs):
        path = _maybe_transform(self._volumepath, *args, **kwargs)
        return super(GirderUploadVolumePathJobArtifact, self).transform(path)
<|endoftext|>


<commit_before>'''
Created on Feb 18, 2015

@author: jkrzywon
'''

__id__ = "$Id: acknoweldgebox.py 2015-18-02 jkrzywon $"
__revision__ = "$Revision: 1193 $"

import wx
import wx.richtext
import wx.lib.hyperlink
from wx.lib.expando import ExpandoTextCtrl
import random
import os.path
import os
try:
    # Try to find a local config
    import imp
    path = os.getcwd()
    if(os.path.isfile("%s/%s.py" % (path, 'local_config'))) or \
      (os.path.isfile("%s/%s.pyc" % (path, 'local_config'))):
        fObj, path, descr = imp.find_module('local_config', [path])
        config = imp.load_module('local_config', fObj, path, descr)
    else:
        # Try simply importing local_config
        import local_config as config
except:
    # Didn't find local config, load the default
    import config

class DialogAcknowledge(wx.Dialog):
    """
    "Acknowledgement" Dialog Box

    Shows the current method for acknowledging SasView in
    scholarly publications.
    """

    def __init__(self, *args, **kwds):

        kwds["style"] = wx.DEFAULT_DIALOG_STYLE
        wx.Dialog.__init__(self, *args, **kwds)

        self.ack = ExpandoTextCtrl(self, style=wx.TE_LEFT|wx.TE_MULTILINE|wx.TE_BESTWRAP|wx.TE_READONLY|wx.TE_NO_VSCROLL)
        self.ack.SetValue(config._acknowledgement_publications)
        #self.ack.SetMinSize((-1, 55))
        self.citation = ExpandoTextCtrl(self, style=wx.TE_LEFT|wx.TE_MULTILINE|wx.TE_BESTWRAP|wx.TE_READONLY|wx.TE_NO_VSCROLL)
        self.citation.SetValue(config._acknowledgement_citation)
        self.preamble = wx.StaticText(self, -1, config._acknowledgement_preamble)
        items = [config._acknowledgement_preamble_bullet1,
                 config._acknowledgement_preamble_bullet2,
                 config._acknowledgement_preamble_bullet3,
                 config._acknowledgement_preamble_bullet4]
        self.list1 = wx.StaticText(self, -1, "(1) " + items[0])
        self.list2 = wx.StaticText(self, -1, "(2) " + items[1])
        self.list3 = wx.StaticText(self, -1, "(3) " + items[2])
        self.list4 = wx.StaticText(self, -1, "(4) " + items[3])
        self.static_line = wx.StaticLine(self, 0)
        self.__set_properties()
        self.__do_layout()

    def __set_properties(self):
        """
        :TODO - add method documentation
        """
        # begin wxGlade: DialogAbout.__set_properties
        self.preamble.SetFont(wx.Font(11, wx.DEFAULT, wx.NORMAL, wx.NORMAL, 0, ""))
        self.preamble.SetFont(wx.Font(10, wx.DEFAULT, wx.NORMAL, wx.NORMAL, 0, ""))
        self.SetTitle("Acknowledging SasView")
        #Increased size of box from (525, 225), SMK, 04/10/16
        self.SetSize((600, 320))
        # end wxGlade

    def __do_layout(self):
        """
        :TODO - add method documentation
        """
        # begin wxGlade: DialogAbout.__do_layout
        sizer_main = wx.BoxSizer(wx.VERTICAL)
        sizer_titles = wx.BoxSizer(wx.VERTICAL)
        sizer_titles.Add(self.preamble, 0, wx.ALL|wx.EXPAND, 5)
        sizer_titles.Add(self.list1, 0, wx.ALL|wx.EXPAND, 5)
        sizer_titles.Add(self.ack, 0, wx.ALL|wx.EXPAND, 5)
        sizer_titles.Add(self.list2, 0, wx.ALL|wx.EXPAND, 5)
        sizer_titles.Add(self.citation, 0, wx.ALL|wx.EXPAND, 5)
        sizer_titles.Add(self.list3, 0, wx.ALL|wx.EXPAND, 5)
        #sizer_titles.Add(self.static_line, 0, wx.ALL|wx.EXPAND, 0)
        sizer_titles.Add(self.list4, 0, wx.ALL|wx.EXPAND, 5)
        sizer_main.Add(sizer_titles, -1, wx.ALL|wx.EXPAND, 5)
        self.SetAutoLayout(True)
        self.SetSizer(sizer_main)
        self.Layout()
        self.Centre()
        # end wxGlade

##### testing code ############################################################
class MyApp(wx.App):
    """
    Class for running module as stand alone for testing
    """
    def OnInit(self):
        """
        Defines an init when running as standalone
        """
        wx.InitAllImageHandlers()
        dialog = DialogAcknowledge(None, -1, "")
        self.SetTopWindow(dialog)
        dialog.ShowModal()
        dialog.Destroy()
        return 1

# end of class MyApp

if __name__ == "__main__":
    app = MyApp(0)
    app.MainLoop()
<commit_msg>Fettling sizes. Calling this done.<commit_after>'''
Created on Feb 18, 2015

@author: jkrzywon
'''

__id__ = "$Id: acknoweldgebox.py 2015-18-02 jkrzywon $"
__revision__ = "$Revision: 1193 $"

import wx
import wx.richtext
import wx.lib.hyperlink
from wx.lib.expando import ExpandoTextCtrl
import random
import os.path
import os
try:
    # Try to find a local config
    import imp
    path = os.getcwd()
    if(os.path.isfile("%s/%s.py" % (path, 'local_config'))) or \
      (os.path.isfile("%s/%s.pyc" % (path, 'local_config'))):
        fObj, path, descr = imp.find_module('local_config', [path])
        config = imp.load_module('local_config', fObj, path, descr)
    else:
        # Try simply importing local_config
        import local_config as config
except:
    # Didn't find local config, load the default
    import config

class DialogAcknowledge(wx.Dialog):
    """
    "Acknowledgement" Dialog Box

    Shows the current method for acknowledging SasView in
    scholarly publications.
    """

    def __init__(self, *args, **kwds):

        kwds["style"] = wx.DEFAULT_DIALOG_STYLE
        wx.Dialog.__init__(self, *args, **kwds)

        self.ack = ExpandoTextCtrl(self, style=wx.TE_LEFT|wx.TE_MULTILINE|wx.TE_BESTWRAP|wx.TE_READONLY|wx.TE_NO_VSCROLL)
        self.ack.SetValue(config._acknowledgement_publications)
        #self.ack.SetMinSize((-1, 55))
        self.citation = ExpandoTextCtrl(self, style=wx.TE_LEFT|wx.TE_MULTILINE|wx.TE_BESTWRAP|wx.TE_READONLY|wx.TE_NO_VSCROLL)
        self.citation.SetValue(config._acknowledgement_citation)
        self.preamble = wx.StaticText(self, -1, config._acknowledgement_preamble)
        items = [config._acknowledgement_preamble_bullet1,
                 config._acknowledgement_preamble_bullet2,
                 config._acknowledgement_preamble_bullet3,
                 config._acknowledgement_preamble_bullet4]
        self.list1 = wx.StaticText(self, -1, "(1) " + items[0])
        self.list2 = wx.StaticText(self, -1, "(2) " + items[1])
        self.list3 = wx.StaticText(self, -1, "(3) " + items[2])
        self.list4 = wx.StaticText(self, -1, "(4) " + items[3])
        self.static_line = wx.StaticLine(self, 0)
        self.__set_properties()
        self.__do_layout()

    def __set_properties(self):
        """
        :TODO - add method documentation
        """
        # begin wxGlade: DialogAbout.__set_properties
        self.preamble.SetFont(wx.Font(11, wx.DEFAULT, wx.NORMAL, wx.NORMAL, 0, ""))
        self.preamble.SetFont(wx.Font(10, wx.DEFAULT, wx.NORMAL, wx.NORMAL, 0, ""))
        self.SetTitle("Acknowledging SasView")
        #Increased size of box from (525, 225), SMK, 04/10/16
        self.SetClientSize((600, 320))
        # end wxGlade

    def __do_layout(self):
        """
        :TODO - add method documentation
        """
        # begin wxGlade: DialogAbout.__do_layout
        sizer_main = wx.BoxSizer(wx.VERTICAL)
        sizer_titles = wx.BoxSizer(wx.VERTICAL)
        sizer_titles.Add(self.preamble, 0, wx.ALL|wx.EXPAND, 5)
        sizer_titles.Add(self.list1, 0, wx.ALL|wx.EXPAND, 5)
        sizer_titles.Add(self.ack, 0, wx.ALL|wx.EXPAND, 5)
        sizer_titles.Add(self.list2, 0, wx.ALL|wx.EXPAND, 5)
        sizer_titles.Add(self.citation, 0, wx.ALL|wx.EXPAND, 5)
        sizer_titles.Add(self.list3, 0, wx.ALL|wx.EXPAND, 5)
        #sizer_titles.Add(self.static_line, 0, wx.ALL|wx.EXPAND, 0)
        sizer_titles.Add(self.list4, 0, wx.ALL|wx.EXPAND, 5)
        sizer_main.Add(sizer_titles, -1, wx.ALL|wx.EXPAND, 5)
        self.SetAutoLayout(True)
        self.SetSizer(sizer_main)
        self.Layout()
        self.Centre()
        #self.SetClientSize(sizer_main.GetSize())
        # end wxGlade

##### testing code ############################################################
class MyApp(wx.App):
    """
    Class for running module as stand alone for testing
    """
    def OnInit(self):
        """
        Defines an init when running as standalone
        """
        wx.InitAllImageHandlers()
        dialog = DialogAcknowledge(None, -1, "")
        self.SetTopWindow(dialog)
        dialog.ShowModal()
        dialog.Destroy()
        return 1

# end of class MyApp

if __name__ == "__main__":
    app = MyApp(0)
    app.MainLoop()
<|endoftext|>


<commit_before>"""
sentry.web.frontend.projects.rules
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import re

from collections import defaultdict
from django.contrib import messages
from django.core.context_processors import csrf
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.csrf import csrf_protect

from sentry.constants import MEMBER_ADMIN
from sentry.models import Rule
from sentry.utils import json
from sentry.utils.cache import memoize
from sentry.rules import rules
from sentry.web.decorators import has_access
from sentry.web.helpers import render_to_response

class RuleFormValidator(object):
    # XXX(dcramer): please no judgements on any of the rule code, I realize it's
    # all terrible and poorly described
    def __init__(self, project, data=None):
        self.project = project
        self.data = data
        self.errors = {}

    @memoize
    def cleaned_data(self):
        # parse out rules
        rules_by_id = {
            'actions': {},
            'conditions': {},
        }
        # TODO: conditions need to be based on actions
        for rule_type, rule in rules:
            if rule_type.startswith('condition/'):
                rules_by_id['conditions'][rule.id] = rule
            elif rule_type.startswith('action/'):
                rules_by_id['actions'][rule.id] = rule

        key_regexp = r'^(condition|action)\[(\d+)\]\[(.+)\]$'
        raw_data = defaultdict(lambda: defaultdict(dict))
        for key, value in self.data.iteritems():
            match = re.match(key_regexp, key)
            if not match:
                continue
            raw_data[match.group(1)][match.group(2)][match.group(3)] = value

        data = {
            'label': self.data.get('label', '').strip(),
            'action_match': self.data.get('action_match', 'all'),
            'actions': [],
            'conditions': [],
        }

        for num, node in sorted(raw_data['condition'].iteritems()):
            data['conditions'].append(node)
            cls = rules_by_id['conditions'][node['id']]
            if not cls(self.project, node).validate_form():
                self.errors['condition[%s]' % (num,)] = 'Ensure all fields are filled out correctly.'

        for num, node in sorted(raw_data['action'].iteritems()):
            data['actions'].append(node)
            cls = rules_by_id['actions'][node['id']]
            if not cls(self.project, node).validate_form():
                self.errors['action[%s]' % (num,)] = 'Ensure all fields are filled out correctly.'

        if not data['label'] or len(data['label']) > 64:
            self.errors['label'] = 'Value must be less than 64 characters.'

        return data

    def is_valid(self):
        # force validation
        self.cleaned_data
        return not bool(self.errors)

def _generate_rule_label(project, rule, data):
    rule_cls = rules.get(data['id'])
    if rule_cls is None:
        return

    rule_inst = rule_cls(project, data=data, rule=rule)
    return rule_inst.render_label()

@has_access(MEMBER_ADMIN)
def list_rules(request, organization, project):
    rule_list = []
    for rule in Rule.objects.filter(project=project):
        conditions = []
        for data in rule.data['conditions']:
            conditions.append(_generate_rule_label(project, rule, data))
        conditions = filter(bool, conditions)

        actions = []
        for data in rule.data['actions']:
            actions.append(_generate_rule_label(project, rule, data))
        actions = filter(bool, actions)

        rule_list.append({
            'id': rule.id,
            'label': rule.label,
            'match': rule.data.get('action_match', 'all'),
            'actions': actions,
            'conditions': conditions,
        })

    context = csrf(request)
    context.update({
        'organization': organization,
        'team': project.team,
        'page': 'rules',
        'project': project,
        'rule_list': rule_list,
    })

    return render_to_response('sentry/projects/rules/list.html', context, request)

@has_access(MEMBER_ADMIN)
@csrf_protect
def create_or_edit_rule(request, organization, project, rule_id=None):
    if rule_id:
        try:
            rule = Rule.objects.get(project=project, id=rule_id)
        except Rule.DoesNotExist:
            path = reverse('sentry-project-rules', args=[organization.slug, project.slug])
            return HttpResponseRedirect(path)
    else:
        rule = Rule(project=project)

    form_data = {
        'label': rule.label,
        'action_match': rule.data.get('action_match'),
    }

    if request.POST:
        for key, value in request.POST.iteritems():
            form_data[key] = value
    else:
        for num, node in enumerate(rule.data.get('conditions', [])):
            prefix = 'condition[%d]' % (num,)
            for key, value in node.iteritems():
                form_data[prefix + '[' + key + ']'] = value

        for num, node in enumerate(rule.data.get('actions', [])):
            prefix = 'action[%d]' % (num,)
            for key, value in node.iteritems():
                form_data[prefix + '[' + key + ']'] = value

    validator = RuleFormValidator(project, form_data)
    if request.POST and validator.is_valid():
        data = validator.cleaned_data.copy()

        rule.label = data.pop('label')
        rule.data = data
        rule.save()

        messages.add_message(
            request, messages.SUCCESS,
            _('Changes to your rule were saved.'))

        path = reverse('sentry-project-rules', args=[organization.slug, project.slug])
        return HttpResponseRedirect(path)

    action_list = []
    condition_list = []

    # TODO: conditions need to be based on actions
    for rule_type, rule in rules:
        node = rule(project)
        context = {
            'id': node.id,
            'label': node.label,
            'html': node.render_form(),
        }

        if rule_type.startswith('condition/'):
            condition_list.append(context)
        elif rule_type.startswith('action/'):
            action_list.append(context)

    context = csrf(request)
    context.update({
        'rule': rule,
        'form_is_valid': (not request.POST or validator.is_valid()),
        'form_errors': validator.errors,
        'form_data': form_data,
        'organization': organization,
        'team': project.team,
        'page': 'rules',
        'action_list': json.dumps(action_list),
        'condition_list': json.dumps(condition_list),
        'project': project,
    })

    return render_to_response('sentry/projects/rules/new.html', context, request)

@has_access(MEMBER_ADMIN)
@csrf_protect
def remove_rule(request, organization, project, rule_id):
    path = reverse('sentry-project-rules', args=[organization.slug, project.slug])

    try:
        rule = Rule.objects.get(project=project, id=rule_id)
    except Rule.DoesNotExist:
        return HttpResponseRedirect(path)

    rule.delete()

    messages.add_message(request, messages.SUCCESS,
        _('The rule was removed.'))

    return HttpResponseRedirect(path)
<commit_msg>Fix reference to rule<commit_after>"""
sentry.web.frontend.projects.rules
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import re

from collections import defaultdict
from django.contrib import messages
from django.core.context_processors import csrf
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.csrf import csrf_protect

from sentry.constants import MEMBER_ADMIN
from sentry.models import Rule
from sentry.utils import json
from sentry.utils.cache import memoize
from sentry.rules import rules
from sentry.web.decorators import has_access
from sentry.web.helpers import render_to_response

class RuleFormValidator(object):
    # XXX(dcramer): please no judgements on any of the rule code, I realize it's
    # all terrible and poorly described
    def __init__(self, project, data=None):
        self.project = project
        self.data = data
        self.errors = {}

    @memoize
    def cleaned_data(self):
        # parse out rules
        rules_by_id = {
            'actions': {},
            'conditions': {},
        }
        # TODO: conditions need to be based on actions
        for rule_type, rule in rules:
            if rule_type.startswith('condition/'):
                rules_by_id['conditions'][rule.id] = rule
            elif rule_type.startswith('action/'):
                rules_by_id['actions'][rule.id] = rule

        key_regexp = r'^(condition|action)\[(\d+)\]\[(.+)\]$'
        raw_data = defaultdict(lambda: defaultdict(dict))
        for key, value in self.data.iteritems():
            match = re.match(key_regexp, key)
            if not match:
                continue
            raw_data[match.group(1)][match.group(2)][match.group(3)] = value

        data = {
            'label': self.data.get('label', '').strip(),
            'action_match': self.data.get('action_match', 'all'),
            'actions': [],
            'conditions': [],
        }

        for num, node in sorted(raw_data['condition'].iteritems()):
            data['conditions'].append(node)
            cls = rules_by_id['conditions'][node['id']]
            if not cls(self.project, node).validate_form():
                self.errors['condition[%s]' % (num,)] = 'Ensure all fields are filled out correctly.'

        for num, node in sorted(raw_data['action'].iteritems()):
            data['actions'].append(node)
            cls = rules_by_id['actions'][node['id']]
            if not cls(self.project, node).validate_form():
                self.errors['action[%s]' % (num,)] = 'Ensure all fields are filled out correctly.'

        if not data['label'] or len(data['label']) > 64:
            self.errors['label'] = 'Value must be less than 64 characters.'

        return data

    def is_valid(self):
        # force validation
        self.cleaned_data
        return not bool(self.errors)

def _generate_rule_label(project, rule, data):
    rule_cls = rules.get(data['id'])
    if rule_cls is None:
        return

    rule_inst = rule_cls(project, data=data, rule=rule)
    return rule_inst.render_label()

@has_access(MEMBER_ADMIN)
def list_rules(request, organization, project):
    rule_list = []
    for rule in Rule.objects.filter(project=project):
        conditions = []
        for data in rule.data['conditions']:
            conditions.append(_generate_rule_label(project, rule, data))
        conditions = filter(bool, conditions)

        actions = []
        for data in rule.data['actions']:
            actions.append(_generate_rule_label(project, rule, data))
        actions = filter(bool, actions)

        rule_list.append({
            'id': rule.id,
            'label': rule.label,
            'match': rule.data.get('action_match', 'all'),
            'actions': actions,
            'conditions': conditions,
        })

    context = csrf(request)
    context.update({
        'organization': organization,
        'team': project.team,
        'page': 'rules',
        'project': project,
        'rule_list': rule_list,
    })

    return render_to_response('sentry/projects/rules/list.html', context, request)

@has_access(MEMBER_ADMIN)
@csrf_protect
def create_or_edit_rule(request, organization, project, rule_id=None):
    if rule_id:
        try:
            rule = Rule.objects.get(project=project, id=rule_id)
        except Rule.DoesNotExist:
            path = reverse('sentry-project-rules', args=[organization.slug, project.slug])
            return HttpResponseRedirect(path)
    else:
        rule = Rule(project=project)

    form_data = {
        'label': rule.label,
        'action_match': rule.data.get('action_match'),
    }

    if request.POST:
        for key, value in request.POST.iteritems():
            form_data[key] = value
    else:
        for num, node in enumerate(rule.data.get('conditions', [])):
            prefix = 'condition[%d]' % (num,)
            for key, value in node.iteritems():
                form_data[prefix + '[' + key + ']'] = value

        for num, node in enumerate(rule.data.get('actions', [])):
            prefix = 'action[%d]' % (num,)
            for key, value in node.iteritems():
                form_data[prefix + '[' + key + ']'] = value

    validator = RuleFormValidator(project, form_data)
    if request.POST and validator.is_valid():
        data = validator.cleaned_data.copy()

        rule.label = data.pop('label')
        rule.data = data
        rule.save()

        messages.add_message(
            request, messages.SUCCESS,
            _('Changes to your rule were saved.'))

        path = reverse('sentry-project-rules', args=[organization.slug, project.slug])
        return HttpResponseRedirect(path)

    action_list = []
    condition_list = []

    # TODO: conditions need to be based on actions
    for rule_type, rule_cls in rules:
        node = rule_cls(project)
        context = {
            'id': node.id,
            'label': node.label,
            'html': node.render_form(),
        }

        if rule_type.startswith('condition/'):
            condition_list.append(context)
        elif rule_type.startswith('action/'):
            action_list.append(context)

    context = csrf(request)
    context.update({
        'rule': rule,
        'form_is_valid': (not request.POST or validator.is_valid()),
        'form_errors': validator.errors,
        'form_data': form_data,
        'organization': organization,
        'team': project.team,
        'page': 'rules',
        'action_list': json.dumps(action_list),
        'condition_list': json.dumps(condition_list),
        'project': project,
    })

    return render_to_response('sentry/projects/rules/new.html', context, request)

@has_access(MEMBER_ADMIN)
@csrf_protect
def remove_rule(request, organization, project, rule_id):
    path = reverse('sentry-project-rules', args=[organization.slug, project.slug])

    try:
        rule = Rule.objects.get(project=project, id=rule_id)
    except Rule.DoesNotExist:
        return HttpResponseRedirect(path)

    rule.delete()

    messages.add_message(request, messages.SUCCESS,
        _('The rule was removed.'))

    return HttpResponseRedirect(path)
<|endoftext|>


<commit_before>#-*- encoding: utf-8
"""
File object for storing an opened file and editor.
"""

import os
import time

from helpers import *

class File:
    def __init__(self, app = None):
        self.app = app
        self.name = ""
        self.fpath = ""
        self.data = None
        self.read_only = False # Currently unused
        self.last_save = None
        self.opened = time.time()
        self.editor = None
        self.writable = True

    def _path(self):
        """Get the full path of the file."""
        return os.path.join(self.fpath, self.name)

    def path(self):
        """Get the full path of the file."""
        return self._path()

    def parse_path(self, path):
        """Parse a relative path and return full directory and filename as a tuple."""
        if path[:2] == "~" + os.sep:
            p  = os.path.expanduser("~")
            path = os.path.join(p+os.sep, path[2:])
        ab = os.path.abspath(path)
        parts = os.path.split(ab)
        return parts

    def log(self, s, type=None):
        self.app.logger.log(s, type)

    def get_name(self):
        """Get the file name."""
        return self.name

    def get_editor(self):
        """Get the associated editor."""
        return self.editor

    def set_name(self, name):
        """Set the file name."""
        # TODO: sanitize
        self.name = name
        self.update_editor_extension()

    def set_path(self, path):
        """Set the file path. Relative paths are sanitized."""
        self.fpath, self.name = self.parse_path(path)
        self.update_editor_extension()

    def set_data(self, data):
        """Set the file data and apply to editor if it exists."""
        self.data = data
        if self.editor:
            self.editor.set_data(data)

    def set_editor(self, editor):
        """The editor instance set its file extension."""
        self.editor = editor
        self.update_editor_extension()

    def on_load(self):
        """Does checks after file is loaded."""
        self.writable = os.access(self._path(), os.W_OK)
        if not self.writable:
            self.log("File not writable?!")

    def update_editor_extension(self):
        """Set the editor file extension from the current file name."""
        if not self.editor:
            return False
        ext = self.name.split(".")
        if len(ext) > 1:
            self.editor.set_file_extension(ext[-1])

    def save(self):
        """Write the editor data to file."""
        path = self._path()
        data = self.editor.get_data()
        try:
            f = open(self._path(), "w")
            f.write(data)
            f.close()
        except:
            return False
        self.data = data
        self.last_save = time.time()
        return True

    def load(self, read=True):
        """Try to read the actual file and load the data into the editor instance."""
        if not read:
            return True
        path = self._path()
        if not os.path.isfile(path):
            self.log("Given path isn't a file.")
            return False
        data = self._read_text(path)
        if data == False:
            self.log("Normal file read failed.", LOG_WARNING)
            data = self._read_binary(path)
        if data == False:
            self.log("Fallback file read failed.", LOG_WARNING)
            return False
        self.data = data
        self.editor.set_data(data)
        self.on_load()
        return True

    def _read_text(self, file):
        # Read text file
        try:
            f = open(self._path())
            data = f.read()
            f.close()
            return data
        except:
            return False

    def _read_binary(self, file):
        # Read binary file and try to autodetect encoding
        try:
            f = open(self._path(), "rb")
            data = f.read()
            f.close()
            import chardet
            detection = chardet.detect(data)
            charenc = detection['encoding']
            if charenc == None:
                self.log("Failed to detect file encoding.", LOG_WARNING)
                return False
            self.log("Trying to decode with encoding '" + charenc + "'", LOG_INFO)
            return data.decode(charenc)
        except Exception as inst:
            self.log(type(inst))    # the exception instance
            self.log(inst.args)     # arguments stored in .args
            self.log(inst)          # __str__ allows args to be printed directly,
        return False

    def reload(self):
        """Reload file data."""
        return self.load()

    def is_changed(self):
        """Check if the editor data is different from the file."""
        return self.editor.get_data() != self.data

    def is_writable(self):
        """Check if the file is writable."""
        return self.writable
        <commit_msg>Tweaked log message.<commit_after>#-*- encoding: utf-8
"""
File object for storing an opened file and editor.
"""

import os
import time

from helpers import *

class File:
    def __init__(self, app = None):
        self.app = app
        self.name = ""
        self.fpath = ""
        self.data = None
        self.read_only = False # Currently unused
        self.last_save = None
        self.opened = time.time()
        self.editor = None
        self.writable = True

    def _path(self):
        """Get the full path of the file."""
        return os.path.join(self.fpath, self.name)

    def path(self):
        """Get the full path of the file."""
        return self._path()

    def parse_path(self, path):
        """Parse a relative path and return full directory and filename as a tuple."""
        if path[:2] == "~" + os.sep:
            p  = os.path.expanduser("~")
            path = os.path.join(p+os.sep, path[2:])
        ab = os.path.abspath(path)
        parts = os.path.split(ab)
        return parts

    def log(self, s, type=None):
        self.app.logger.log(s, type)

    def get_name(self):
        """Get the file name."""
        return self.name

    def get_editor(self):
        """Get the associated editor."""
        return self.editor

    def set_name(self, name):
        """Set the file name."""
        # TODO: sanitize
        self.name = name
        self.update_editor_extension()

    def set_path(self, path):
        """Set the file path. Relative paths are sanitized."""
        self.fpath, self.name = self.parse_path(path)
        self.update_editor_extension()

    def set_data(self, data):
        """Set the file data and apply to editor if it exists."""
        self.data = data
        if self.editor:
            self.editor.set_data(data)

    def set_editor(self, editor):
        """The editor instance set its file extension."""
        self.editor = editor
        self.update_editor_extension()

    def on_load(self):
        """Does checks after file is loaded."""
        self.writable = os.access(self._path(), os.W_OK)
        if not self.writable:
            self.log("File not writable.")

    def update_editor_extension(self):
        """Set the editor file extension from the current file name."""
        if not self.editor:
            return False
        ext = self.name.split(".")
        if len(ext) > 1:
            self.editor.set_file_extension(ext[-1])

    def save(self):
        """Write the editor data to file."""
        path = self._path()
        data = self.editor.get_data()
        try:
            f = open(self._path(), "w")
            f.write(data)
            f.close()
        except:
            return False
        self.data = data
        self.last_save = time.time()
        return True

    def load(self, read=True):
        """Try to read the actual file and load the data into the editor instance."""
        if not read:
            return True
        path = self._path()
        if not os.path.isfile(path):
            self.log("Given path isn't a file.")
            return False
        data = self._read_text(path)
        if data == False:
            self.log("Normal file read failed.", LOG_WARNING)
            data = self._read_binary(path)
        if data == False:
            self.log("Fallback file read failed.", LOG_WARNING)
            return False
        self.data = data
        self.editor.set_data(data)
        self.on_load()
        return True

    def _read_text(self, file):
        # Read text file
        try:
            f = open(self._path())
            data = f.read()
            f.close()
            return data
        except:
            return False

    def _read_binary(self, file):
        # Read binary file and try to autodetect encoding
        try:
            f = open(self._path(), "rb")
            data = f.read()
            f.close()
            import chardet
            detection = chardet.detect(data)
            charenc = detection['encoding']
            if charenc == None:
                self.log("Failed to detect file encoding.", LOG_WARNING)
                return False
            self.log("Trying to decode with encoding '" + charenc + "'", LOG_INFO)
            return data.decode(charenc)
        except Exception as inst:
            self.log(type(inst))    # the exception instance
            self.log(inst.args)     # arguments stored in .args
            self.log(inst)          # __str__ allows args to be printed directly,
        return False

    def reload(self):
        """Reload file data."""
        return self.load()

    def is_changed(self):
        """Check if the editor data is different from the file."""
        return self.editor.get_data() != self.data

    def is_writable(self):
        """Check if the file is writable."""
        return self.writable
        <|endoftext|>