<commit_before>import numpy as np
from numpy import ma, random
#from scipy.stats import norm, rayleigh, expon, halfnorm, exponpow, exponweib
from scipy.stats import exponweib
#from scipy.stats import kstest

from pydap.client import open_url
import pydap.lib
pydap.lib.CACHE = '.cache'
from scipy.interpolate import RectBivariateSpline, interp1d

# ============================================================================
def get_depth(lat, lon):
    """

    ATENTION, conceptual error on the data near by Greenwich.
    """
    if lat.shape != lon.shape:
        print "lat and lon must have the same size"
    url='http://opendap.ccst.inpe.br/Climatologies/ETOPO/etopo5.cdf'
    dataset = open_url(url)
    etopo = dataset.ROSE
    x = etopo.ETOPO05_X[:]
    if lon.min()<0:
        ind = lon<0
        lon[ind] = lon[ind]+360
    y = etopo.ETOPO05_Y[:]
    iini = max(0, (numpy.absolute(lon.min()-x)).argmin()-2)
    ifin = (numpy.absolute(lon.max()-x)).argmin()+2
    jini = max(0, (numpy.absolute(lat.min()-y)).argmin()-2)
    jfin = (numpy.absolute(lat.max()-y)).argmin()+2
    z = etopo.ROSE[jini:jfin, iini:ifin]
    interpolator = RectBivariateSpline(x[iini:ifin], y[jini:jfin], z.T)
    depth = ma.array([interpolator(xx, yy)[0][0] for xx, yy in zip(lon,lat)])
    return depth

def woa_profile_from_dap(var, doy, lat, lon, depth):
    """
    Monthly Climatologic Mean and Standard Deviation from WOA,
    used either for temperature or salinity.

    INPUTS
        time: [day of the year]
        lat: [-90<lat<90]
        lon: [-180<lon<180]
        depth: [meters]

    Reads the WOA Monthly Climatology NetCDF file and
    returns the corresponding WOA values of salinity or temperature mean and
    standard deviation for the given time, lat, lon, depth.
    """
    if lon<0: lon = lon+360
    if var == 'temperature':
        url ='http://opendap.ccst.inpe.br/Climatologies/WOA/temperature_monthly_1deg.nc'
    elif var == 'salinity':
        url ='http://opendap.ccst.inpe.br/Climatologies/WOA/salinity_monthly_1deg.nc'
    else:
        return

    dataset = open_url(url)

    dn = (numpy.abs(doy-dataset['time'][:])).argmin()
    xn = (numpy.abs(lon-dataset['lon'][:])).argmin()
    yn = (numpy.abs(lat-dataset['lat'][:])).argmin()

    data = {}
    if var == 'temperature':
        an = ma.masked_values(dataset.t_an.t_an[dn,:,yn,xn].reshape(dataset['depth'].shape[0]), dataset.t_an.attributes['_FillValue'])
        #sd = ma.masked_values(dataset.t_sd.t_sd[dn,:,yn,xn].reshape(dataset['depth'].shape[0]), dataset.t_sd.attributes['_FillValue'])
        #se = ma.masked_values(dataset.t_se.t_se[dn,:,yn,xn].reshape(dataset['depth'].shape[0]), dataset.t_se.attributes['_FillValue'])
        #dd = ma.masked_values(dataset.t_dd.t_dd[dn,:,yn,xn].reshape(dataset['depth'].shape[0]), dataset.t_dd.attributes['_FillValue'])
    elif var == 'salinity':
        an = ma.masked_values(dataset.s_an.s_an[dn,:,yn,xn].reshape(dataset['depth'].shape[0]), dataset.s_an.attributes['_FillValue'])
        #sd = ma.masked_values(dataset.s_sd.s_sd[dn,:,yn,xn].reshape(dataset['depth'].shape[0]), dataset.s_sd.attributes['_FillValue'])
    zwoa = ma.array(dataset.depth[:])

    ind=depth<=zwoa.max()
    # Mean value profile
    f = interp1d(zwoa[~ma.getmaskarray(an)].compressed(), an.compressed())
    an_interp = ma.masked_all(depth.shape)
    an_interp[ind] = f(depth[ind])
    # The stdev profile
    #f = interp1d(zwoa[~ma.getmaskarray(sd)].compressed(), sd.compressed())
    #sd_interp = ma.masked_all(depth.shape)
    #sd_interp[ind] = f(depth[ind])

    #return an_interp, sd_interp
    return an_interp

def split_data_groups(ind):
    """ Splits randomly the indices into fit, test and error groups

        Return 3 indices set:
            - ind_fit with 60% of the good
            - ind_test with 20% of the good and 50% of the bad
            - ind_eval with 20% of the good and 50% of the bad
    """
    N = ind.size
    ind_base = np.zeros(N) == 1
    # ==== Good data ==================
    ind_good = np.nonzero(ind == True)[0]
    N_good = ind_good.size
    perm = random.permutation(N_good)
    N_fit = int(round(N_good*.6))
    N_test = int(round(N_good*.2))
    ind_fit = ind_base.copy()
    ind_fit[ind_good[perm[:N_fit]]] = True
    ind_test = ind_base.copy()
    ind_test[ind_good[perm[N_fit:-N_test]]] = True
    ind_err = ind_base.copy()
    ind_err[ind_good[perm[-N_test:]]] = True
    # ==== Bad data ===================
    ind_bad = np.nonzero(ind == False)[0]
    N_bad = ind_bad.size
    perm = random.permutation(N_bad)
    N_test = int(round(N_bad*.5))
    ind_test[ind_bad[perm[:N_test]]] = True
    ind_err[ind_bad[perm[N_test:]]] = True
    output = {'ind_fit': ind_fit, 'ind_test': ind_test, 'ind_err': ind_err}
    return output

# I need to improve this, and include the places where the
#   flags are masked, i.e. only eliminate where the flags
#   could guarantee it was false.

def make_qc_index(flags, criteria, type="anytrue"):
    ind = flags[criteria[0]].copy()
    if type == "anytrue":
        for c in criteria:
            ind[(ind == True) | (flags[c] == True)] = True
        #ind[np.nonzero((ind == True) | (np.array(flags[c]) == True))[0]] = True
    elif type == "alltrue":
        for c in criteria:
            ind[(ind == True) | (flags[c] == True)] = True
    for c in criteria:
        ind[(ind == False) | (flags[c] == False)] = False
        #ind[np.nonzero((ind == False) | (np.array(flags[c]) == False))[0]] = False
    return ind

def fit_tests(aux, qctests, ind=True, q=0.95, verbose=False):
    """
    """
    output = {}
    for teste in qctests:
        samp = aux[teste][ind]
        ind_top = samp>samp.quantile(q)
        param = exponweib.fit(samp[ind_top])
        output[teste] = {'param':param,
                'q95': samp.quantile(q)}

        if verbose == True:
            x = linspace(samp[ind_top].min(), samp[ind_top].max(), 100)[1:]
            pdf_fitted = exponweib.pdf(x, *param[:-2], loc=param[-2], scale=param[-1])
            pylab.plot(x,pdf_fitted,'b-')
            pylab.hist(samp[ind_top], 100, normed=1,alpha=.3)
            pylab.title(teste)
            pylab.show()

    return output

def estimate_anomaly(aux, params):
    prob = ma.ones(aux.shape[0])
    for t in params.keys():
        param = params[t]['param']
        ind = np.isfinite(aux[t])
        prob[ind] = prob[ind] * \
                exponweib.sf(aux[t][ind], *param[:-2], loc=param[-2], scale=param[-1])
    return prob

def estimate_p_optimal(prob, qc, verbose=False):
    err = []
    P = 10.**np.arange(-12, 0, 0.1)
    for p in P:
        false_negative = (prob < p) & (qc == True)
        false_positive = (prob > p) & (qc == False)
        err.append(np.nonzero(false_negative)[0].size + \
                np.nonzero(false_positive)[0].size)
    err = np.array(err)
    if verbose == True:
        pylab.plot(P, err , 'b'); pylab.show()
    return P[err.argmin()], float(err.min())/prob.size#, {'P': P, 'err': err}

def adjust_anomaly_coefficients(ind, qctests, aux, verbose=False):
    indices = split_data_groups(ind)
    output = fit_tests(aux, qctests, indices['ind_fit'], q=0.90,
            verbose=verbose)
    prob = estimate_anomaly(aux, output)
    if verbose == True:
        pylab.hist(prob); pylab.show()
    p_optimal, test_err = estimate_p_optimal(prob[indices['ind_test']],
            ind[indices['ind_test']])
    false_negative = (prob[indices['ind_err']] < p_optimal) & \
        (ind[indices['ind_err']] == True)
    false_positive = (prob[indices['ind_err']] > p_optimal) & \
        (ind[indices['ind_err']] == False)
    err = np.nonzero(false_negative)[0].size + \
            np.nonzero(false_positive)[0].size
    err_ratio = float(err)/prob[indices['ind_err']].size
    false_negative = (prob < p_optimal) & \
        (ind == True)
    false_positive = (prob > p_optimal) & \
        (ind == False)
    output = {'false_negative': false_negative,
            'false_positive': false_positive,
            'p_optimal': p_optimal,
            'err': err,
            'err_ratio': err_ratio}
    return output
<commit_msg>Creating misc.combined_flag() for the new flag system.<commit_after>import numpy as np
from numpy import ma, random
#from scipy.stats import norm, rayleigh, expon, halfnorm, exponpow, exponweib
from scipy.stats import exponweib
#from scipy.stats import kstest

from pydap.client import open_url
import pydap.lib
pydap.lib.CACHE = '.cache'
from scipy.interpolate import RectBivariateSpline, interp1d

# ============================================================================
def get_depth(lat, lon):
    """

    ATENTION, conceptual error on the data near by Greenwich.
    """
    if lat.shape != lon.shape:
        print "lat and lon must have the same size"
    url='http://opendap.ccst.inpe.br/Climatologies/ETOPO/etopo5.cdf'
    dataset = open_url(url)
    etopo = dataset.ROSE
    x = etopo.ETOPO05_X[:]
    if lon.min()<0:
        ind = lon<0
        lon[ind] = lon[ind]+360
    y = etopo.ETOPO05_Y[:]
    iini = max(0, (numpy.absolute(lon.min()-x)).argmin()-2)
    ifin = (numpy.absolute(lon.max()-x)).argmin()+2
    jini = max(0, (numpy.absolute(lat.min()-y)).argmin()-2)
    jfin = (numpy.absolute(lat.max()-y)).argmin()+2
    z = etopo.ROSE[jini:jfin, iini:ifin]
    interpolator = RectBivariateSpline(x[iini:ifin], y[jini:jfin], z.T)
    depth = ma.array([interpolator(xx, yy)[0][0] for xx, yy in zip(lon,lat)])
    return depth

def woa_profile_from_dap(var, doy, lat, lon, depth):
    """
    Monthly Climatologic Mean and Standard Deviation from WOA,
    used either for temperature or salinity.

    INPUTS
        time: [day of the year]
        lat: [-90<lat<90]
        lon: [-180<lon<180]
        depth: [meters]

    Reads the WOA Monthly Climatology NetCDF file and
    returns the corresponding WOA values of salinity or temperature mean and
    standard deviation for the given time, lat, lon, depth.
    """
    if lon<0: lon = lon+360
    if var == 'temperature':
        url ='http://opendap.ccst.inpe.br/Climatologies/WOA/temperature_monthly_1deg.nc'
    elif var == 'salinity':
        url ='http://opendap.ccst.inpe.br/Climatologies/WOA/salinity_monthly_1deg.nc'
    else:
        return

    dataset = open_url(url)

    dn = (numpy.abs(doy-dataset['time'][:])).argmin()
    xn = (numpy.abs(lon-dataset['lon'][:])).argmin()
    yn = (numpy.abs(lat-dataset['lat'][:])).argmin()

    data = {}
    if var == 'temperature':
        an = ma.masked_values(dataset.t_an.t_an[dn,:,yn,xn].reshape(dataset['depth'].shape[0]), dataset.t_an.attributes['_FillValue'])
        #sd = ma.masked_values(dataset.t_sd.t_sd[dn,:,yn,xn].reshape(dataset['depth'].shape[0]), dataset.t_sd.attributes['_FillValue'])
        #se = ma.masked_values(dataset.t_se.t_se[dn,:,yn,xn].reshape(dataset['depth'].shape[0]), dataset.t_se.attributes['_FillValue'])
        #dd = ma.masked_values(dataset.t_dd.t_dd[dn,:,yn,xn].reshape(dataset['depth'].shape[0]), dataset.t_dd.attributes['_FillValue'])
    elif var == 'salinity':
        an = ma.masked_values(dataset.s_an.s_an[dn,:,yn,xn].reshape(dataset['depth'].shape[0]), dataset.s_an.attributes['_FillValue'])
        #sd = ma.masked_values(dataset.s_sd.s_sd[dn,:,yn,xn].reshape(dataset['depth'].shape[0]), dataset.s_sd.attributes['_FillValue'])
    zwoa = ma.array(dataset.depth[:])

    ind=depth<=zwoa.max()
    # Mean value profile
    f = interp1d(zwoa[~ma.getmaskarray(an)].compressed(), an.compressed())
    an_interp = ma.masked_all(depth.shape)
    an_interp[ind] = f(depth[ind])
    # The stdev profile
    #f = interp1d(zwoa[~ma.getmaskarray(sd)].compressed(), sd.compressed())
    #sd_interp = ma.masked_all(depth.shape)
    #sd_interp[ind] = f(depth[ind])

    #return an_interp, sd_interp
    return an_interp

def split_data_groups(ind):
    """ Splits randomly the indices into fit, test and error groups

        Return 3 indices set:
            - ind_fit with 60% of the good
            - ind_test with 20% of the good and 50% of the bad
            - ind_eval with 20% of the good and 50% of the bad
    """
    N = ind.size
    ind_base = np.zeros(N) == 1
    # ==== Good data ==================
    ind_good = np.nonzero(ind == True)[0]
    N_good = ind_good.size
    perm = random.permutation(N_good)
    N_fit = int(round(N_good*.6))
    N_test = int(round(N_good*.2))
    ind_fit = ind_base.copy()
    ind_fit[ind_good[perm[:N_fit]]] = True
    ind_test = ind_base.copy()
    ind_test[ind_good[perm[N_fit:-N_test]]] = True
    ind_err = ind_base.copy()
    ind_err[ind_good[perm[-N_test:]]] = True
    # ==== Bad data ===================
    ind_bad = np.nonzero(ind == False)[0]
    N_bad = ind_bad.size
    perm = random.permutation(N_bad)
    N_test = int(round(N_bad*.5))
    ind_test[ind_bad[perm[:N_test]]] = True
    ind_err[ind_bad[perm[N_test:]]] = True
    output = {'ind_fit': ind_fit, 'ind_test': ind_test, 'ind_err': ind_err}
    return output

# I need to improve this, and include the places where the
#   flags are masked, i.e. only eliminate where the flags
#   could guarantee it was false.

def combined_flag(flags, criteria=None):
    """ Returns the combined flag considering all the criteria

        Collects all flags in the criteria, and for each measurements, it
          return the maximum flag value among the different criteria.

        If criteria is not defined, considers all the flags,
          i.e. flags.keys()
    """
    if criteria is None:
        criteria = flags.keys()

    N = flags[criteria[0]].size
    Nf = len(criteria)
    temp_flag = np.zeros((Nf, N), dtype='i1')
    for i, k in enumerate(flags.keys()):
        temp_flag[i] = flags[k]

    return temp_flag.max(axis=0)

def make_qc_index(flags, criteria, type="anytrue"):
    ind = flags[criteria[0]].copy()
    if type == "anytrue":
        for c in criteria:
            ind[(ind == True) | (flags[c] == True)] = True
        #ind[np.nonzero((ind == True) | (np.array(flags[c]) == True))[0]] = True
    elif type == "alltrue":
        for c in criteria:
            ind[(ind == True) | (flags[c] == True)] = True
    for c in criteria:
        ind[(ind == False) | (flags[c] == False)] = False
        #ind[np.nonzero((ind == False) | (np.array(flags[c]) == False))[0]] = False
    return ind

def fit_tests(aux, qctests, ind=True, q=0.95, verbose=False):
    """
    """
    output = {}
    for teste in qctests:
        samp = aux[teste][ind]
        ind_top = samp>samp.quantile(q)
        param = exponweib.fit(samp[ind_top])
        output[teste] = {'param':param,
                'q95': samp.quantile(q)}

        if verbose == True:
            x = linspace(samp[ind_top].min(), samp[ind_top].max(), 100)[1:]
            pdf_fitted = exponweib.pdf(x, *param[:-2], loc=param[-2], scale=param[-1])
            pylab.plot(x,pdf_fitted,'b-')
            pylab.hist(samp[ind_top], 100, normed=1,alpha=.3)
            pylab.title(teste)
            pylab.show()

    return output

def estimate_anomaly(aux, params):
    prob = ma.ones(aux.shape[0])
    for t in params.keys():
        param = params[t]['param']
        ind = np.isfinite(aux[t])
        prob[ind] = prob[ind] * \
                exponweib.sf(aux[t][ind], *param[:-2], loc=param[-2], scale=param[-1])
    return prob

def estimate_p_optimal(prob, qc, verbose=False):
    err = []
    P = 10.**np.arange(-12, 0, 0.1)
    for p in P:
        false_negative = (prob < p) & (qc == True)
        false_positive = (prob > p) & (qc == False)
        err.append(np.nonzero(false_negative)[0].size + \
                np.nonzero(false_positive)[0].size)
    err = np.array(err)
    if verbose == True:
        pylab.plot(P, err , 'b'); pylab.show()
    return P[err.argmin()], float(err.min())/prob.size#, {'P': P, 'err': err}

def adjust_anomaly_coefficients(ind, qctests, aux, verbose=False):
    indices = split_data_groups(ind)
    output = fit_tests(aux, qctests, indices['ind_fit'], q=0.90,
            verbose=verbose)
    prob = estimate_anomaly(aux, output)
    if verbose == True:
        pylab.hist(prob); pylab.show()
    p_optimal, test_err = estimate_p_optimal(prob[indices['ind_test']],
            ind[indices['ind_test']])
    false_negative = (prob[indices['ind_err']] < p_optimal) & \
        (ind[indices['ind_err']] == True)
    false_positive = (prob[indices['ind_err']] > p_optimal) & \
        (ind[indices['ind_err']] == False)
    err = np.nonzero(false_negative)[0].size + \
            np.nonzero(false_positive)[0].size
    err_ratio = float(err)/prob[indices['ind_err']].size
    false_negative = (prob < p_optimal) & \
        (ind == True)
    false_positive = (prob > p_optimal) & \
        (ind == False)
    output = {'false_negative': false_negative,
            'false_positive': false_positive,
            'p_optimal': p_optimal,
            'err': err,
            'err_ratio': err_ratio}
    return output
<|endoftext|>


<commit_before>import random
from bot.libs.configuration import yaml
from os import path
from bot import Command, StaticConfig
from bot.utils import img_embed

default_pats = 'https://gist.github.com/jkcgs/137a28dc01b3e8538a253652f44eaf09/' \
               'raw/4d202bd98180703b6f4145c8995cdf5b09914b8e/pats.yml'

class Pat(Command):

    def __init__(self, bot):
        super().__init__(bot)
        self.name = 'pat'
        self.help = 'Te envía una imagen de animé de una caricia en la cabeza y algo más'
        self.config = None

    async def handle(self, cmd):
        if self.config is None:
            await cmd.answer('this is not working yet')
            return

        if len(cmd.args) != 1 or len(cmd.message.mentions) != 1:
            await cmd.answer('Formato: !pat <@mención>')
            return

        mention = cmd.message.mentions[0]
        text = '{}, {} te ha dado una palmadita :3'.format(
            mention.display_name, cmd.author_name
        )

        if mention.id == cmd.author.id:
            url = random.choice(self.config['self_pats'])
        elif mention.id == self.bot.user.id:
            url = self.config['bot_pat']
            text = 'oye nuuuu >_<'
        else:
            url = random.choice(self.config['pats'])

        await cmd.answer(embed=img_embed(url, text), withname=False)

    async def task(self):
        self.log.debug('[Pat] Cargando pats...')
        if not StaticConfig.exists('pats'):
            self.log.debug('Loading remote default pats')
            async with self.http.get(default_pats) as r:
                data = await r.text()
                defaults = yaml.load(data)
        else:
            defaults = {
                'pats': [],
                'self_pats': [],
                'bot_pat': 'http://i.imgur.com/tVzapCY.gif'
            }

        self.config = StaticConfig.get_config('pats', defaults)
<commit_msg>Permitir usar !pat sin mención, usar cmd.text para obtener al usuario<commit_after>import random
from bot.libs.configuration import yaml
from bot import Command, StaticConfig
from bot.utils import img_embed

default_pats = 'https://gist.github.com/jkcgs/137a28dc01b3e8538a253652f44eaf09/' \
               'raw/039dfdf76945915ad5d723d0031daa94d2e03e0b/pats.yml'

class Pat(Command):

    def __init__(self, bot):
        super().__init__(bot)
        self.name = 'pat'
        self.help = 'Te envía una imagen de animé de una caricia en la cabeza y algo más'
        self.config = None

    async def handle(self, cmd):
        if self.config is None:
            await cmd.answer('this is not working yet')
            return

        if len(cmd.args) > 0:
            mention = await cmd.get_user(cmd.text)
            if mention is None:
                await cmd.answer('usuario no encontrado. Formato: !pat [@usuario]')
                return

            text = '{}, {} te ha dado una palmadita :3'.format(
                mention.display_name, cmd.author_name
            )

            if mention.id == cmd.author.id:
                url = random.choice(self.config['self_pats'])
            elif mention.id == self.bot.user.id:
                url = self.config['bot_pat']
                text = 'oye nuuuu >_<'
            else:
                url = random.choice(self.config['pats'])
        else:
            url = random.choice(self.config['pats'])
            text = '{}, toma una palmadita :3'.format(cmd.author_name)

        await cmd.answer(embed=img_embed(url, text), withname=False)

    async def task(self):
        self.log.debug('[Pat] Cargando pats...')
        if not StaticConfig.exists('pats'):
            self.log.debug('Loading remote default pats')
            async with self.http.get(default_pats) as r:
                data = await r.text()
                defaults = yaml.load(data)
        else:
            defaults = {
                'pats': [],
                'self_pats': [],
                'bot_pat': 'http://i.imgur.com/tVzapCY.gif'
            }

        self.config = StaticConfig.get_config('pats', defaults)
<|endoftext|>


<commit_before>import angr
import simuvex

import chain_builder
import gadget_analyzer

import pickle
import inspect
import logging
import progressbar

from multiprocessing import Pool

l = logging.getLogger('angrop.rop')

def _str_find_all(a_str, sub):
    start = 0
    while True:
        start = a_str.find(sub, start)
        if start == -1:
            return
        yield start
        start += 1

_global_gadget_analyzer = None

# global initializer for multiprocessing
def _set_global_gadget_analyzer(rop_gadget_analyzer):
    global _global_gadget_analyzer
    _global_gadget_analyzer = rop_gadget_analyzer

def run_worker(addr):
    return _global_gadget_analyzer.analyze_gadget(addr)

# todo what if we have mov eax, [rsp+0x20]; ret (cache would need to know where it is or at least a min/max)
# todo what if we have pop eax; mov ebx, eax; need to encode that we cannot set them to different values
class ROP(angr.Analysis):
    """
    This class is a semantic aware rop gadget finder
    It is a work in progress, so don't be surprised if something doesn't quite work
    """

    def __init__(self, only_check_near_rets=True, max_block_size=20, max_sym_mem_accesses=4, fast_mode=None):
        """
        Initializes the rop gadget finder
        :param only_check_near_rets: If true we skip blocks that are not near rets
        :param max_block_size: limits the size of blocks considered, longer blocks are less likely to be good rop
                               gadgets so we limit the size we consider
        :param fast_mode: if set to True sets options to run fast, if set to False sets options to find more gadgets
                          if set to None makes a decision based on the size of the binary
        :return:
        """

        # params
        self._max_block_size = max_block_size
        self._only_check_near_rets = only_check_near_rets
        self._max_sym_mem_accesses = max_sym_mem_accesses

        a = self.project.arch
        self._sp_reg = a.register_names[a.sp_offset]
        self._ip_reg = a.register_names[a.ip_offset]
        self._base_pointer = a.register_names[a.bp_offset]

        self._reg_list = a.default_symbolic_registers
        # prune the register list of the instruction pointer and the stack pointer
        self._reg_list = filter(lambda r: r != self._sp_reg, self._reg_list)
        self._reg_list = filter(lambda r: r != self._ip_reg, self._reg_list)

        self._syscall_instructions = {"\x0f\x05"}

        if self.project.arch.linux_name == "x86_64":
            self._syscall_instructions = {"\x0f\x05"}
            self._cc = ["rdi", "rsi", "rdx", "rcx", "r8", "r9"]
            self._execve_syscall = 59
        elif self.project.arch.linux_name == "i386":
            self._syscall_instructions = {"\xcd\x80"}
            self._cc = "stack"
            self._execve_syscall = 11

        # get ret locations
        self._ret_locations = self._get_ret_locations()

        # list of gadgets
        self.gadgets = []
        self._duplicates = []

        num_to_check = len(list(self._addresses_to_check()))
        # fast mode
        if fast_mode is None:
            if num_to_check > 20000:
                fast_mode = True
                l.warning("Enabling fast mode for large binary")
            else:
                fast_mode = False
        self._fast_mode = fast_mode

        if self._fast_mode:
            self._max_block_size = 12
            self._max_sym_mem_accesses = 1
            num_to_check = len(list(self._addresses_to_check()))

        l.info("There are %d addresses withing %d bytes of a ret",
               num_to_check, self._max_block_size)

        # gadget analyzer
        self._gadget_analyzer = gadget_analyzer.GadgetAnalyzer(self.project, self._reg_list, self._max_block_size,
                                                               self._fast_mode, self._max_sym_mem_accesses)
        # chain builder
        self._chain_builder = None

        # silence annoying loggers
        simuvex.vex.ccall.l.setLevel("CRITICAL")
        simuvex.vex.expressions.ccall.l.setLevel("CRITICAL")

    def find_gadgets(self, processes=4):
        """
        Finds all the gadgets in the binary by calling analyze_gadget on every address near a ret.
        Saves gadgets in self.gadgets
        :param processes: number of processes to use
        """
        self.gadgets = []

        pool = Pool(processes=processes, initializer=_set_global_gadget_analyzer, initargs=(self._gadget_analyzer,))

        it = pool.imap_unordered(run_worker, self._addresses_to_check_with_caching(), chunksize=5)
        for gadget in it:
            if gadget is not None:
                self.gadgets.append(gadget)

        pool.close()

        # fix up gadgets from cache
        for g in self.gadgets:
            if g.addr in self._cache:
                dups = {g.addr}
                for addr in self._cache[g.addr]:
                    dups.add(addr)
                    g_copy = g.copy()
                    g_copy.addr = addr
                    self.gadgets.append(g_copy)
                self._duplicates.append(dups)
        self.gadgets = sorted(self.gadgets, key=lambda x: x.addr)
        self._reload_chain_funcs()

    def find_gadgets_single_threaded(self):
        """
        Finds all the gadgets in the binary by calling analyze_gadget on every address near a ret.
        Saves gadgets in self.gadgets
        """
        self.gadgets = []

        _set_global_gadget_analyzer(self._gadget_analyzer)
        for _, addr in enumerate(self._ret_locations):
            gadget = _global_gadget_analyzer.analyze_gadget(addr)
            if gadget is not None:
                self.gadgets.append(gadget)

        # fix up gadgets from cache
        for g in self.gadgets:
            if g.addr in self._cache:
                dups = {g.addr}
                for addr in self._cache[g.addr]:
                    dups.add(addr)
                    g_copy = g.copy()
                    g_copy.addr = addr
                    self.gadgets.append(g_copy)
                self._duplicates.append(dups)
        self.gadgets = sorted(self.gadgets, key=lambda x: x.addr)
        self._reload_chain_funcs()

    def save_gadgets(self, path):
        pickle.dump((self.gadgets, self._duplicates), open(path, "wb"))

    def load_gadgets(self, path):
        self.gadgets, self._duplicates = pickle.load(open(path, "rb"))
        self._reload_chain_funcs()

    def _reload_chain_funcs(self):
        for f_name, f in inspect.getmembers(self.chain_builder, predicate=inspect.ismethod):
            if f_name.startswith("_"):
                continue
            setattr(self, f_name, f)

    @property
    def chain_builder(self):
        if self._chain_builder is not None:
            return self._chain_builder
        elif len(self.gadgets) > 0:
            self._chain_builder = chain_builder.ChainBuilder(self.project, self.gadgets, self._duplicates,
                                                             self._reg_list)
            return self._chain_builder
        else:
            raise Exception("No gadgets, call find_gadgets() or load_gadgets() first")

    def _block_has_ip_relative(self, addr, bl):
        string = bl.bytes
        bl2 = self.project.factory.block(0x41414141, insn_bytes=string)
        # TODO: FIXME
        try:
            diff_constants = angr.bindiff.differing_constants(bl, bl2)
        except angr.analyses.bindiff.UnmatchedStatementsException:
            return True
        # check if it changes if we move it
        bl_end = addr + bl.size
        bl2_end = 0x41414141 + bl2.size
        filtered_diffs = []
        for d in diff_constants:
            if d.value_a < addr or d.value_a >= bl_end or \
                    d.value_b < 0x41414141 or d.value_b >= bl2_end:
                filtered_diffs.append(d)
        return len(filtered_diffs) > 0

    def _addresses_to_check_with_caching(self, show_progress=True):
        num_addrs = len(list(self._addresses_to_check()))
        widgets = ['ROP: ', progressbar.Percentage(), ' ',
                   progressbar.Bar(marker=progressbar.RotatingMarker()),
                   ' ', progressbar.ETA(), ' ', progressbar.FileTransferSpeed()]
        progress = progressbar.ProgressBar(widgets=widgets, maxval=num_addrs)
        if show_progress:
            progress.start()
        self._cache = dict()
        seen = dict()
        for i, a in enumerate(self._addresses_to_check()):
            if show_progress:
                progress.update(i)
            try:
                bl = self.project.factory.block(a)
                if bl.size > self._max_block_size:
                    continue
                block_data = bl.bytes
            except angr.AngrTranslationError:
                continue
            if block_data in seen:
                self._cache[seen[block_data]].add(a)
                continue
            else:
                if len(bl.vex.constant_jump_targets) == 0 and not self._block_has_ip_relative(a, bl):
                    seen[block_data] = a
                    self._cache[a] = set()
                yield a
        if show_progress:
            progress.finish()

    def _addresses_to_check(self):
        """
        :return: all the addresses to check
        """
        for loc in self._ret_locations:
            yield self.project.loader.main_bin.rebase_addr + loc

    def _get_ret_locations(self):
        """
        :return: all the locations in the binary with a ret instruction
        """
        addrs = []
        for segment in self.project.loader.main_bin.segments:
            if segment.is_executable:
                min_addr = segment.min_addr + self.project.loader.main_bin.rebase_addr
                num_bytes = segment.max_addr-segment.min_addr

                for offset in xrange(0, num_bytes, self.project.arch.instruction_alignment):
                    block = self.project.factory.block(min_addr+offset)
                    if block.vex.jumpkind.startswith("Ijk_Ret"):
                        addrs.append(min_addr+offset)

        return sorted(addrs)

angr.analysis.register_analysis(ROP, 'ROP')
<commit_msg>Check more addresses for gadgets<commit_after>import angr
import simuvex

import chain_builder
import gadget_analyzer

import pickle
import inspect
import logging
import progressbar

from multiprocessing import Pool

l = logging.getLogger('angrop.rop')

def _str_find_all(a_str, sub):
    start = 0
    while True:
        start = a_str.find(sub, start)
        if start == -1:
            return
        yield start
        start += 1

_global_gadget_analyzer = None

# global initializer for multiprocessing
def _set_global_gadget_analyzer(rop_gadget_analyzer):
    global _global_gadget_analyzer
    _global_gadget_analyzer = rop_gadget_analyzer

def run_worker(addr):
    return _global_gadget_analyzer.analyze_gadget(addr)

# todo what if we have mov eax, [rsp+0x20]; ret (cache would need to know where it is or at least a min/max)
# todo what if we have pop eax; mov ebx, eax; need to encode that we cannot set them to different values
class ROP(angr.Analysis):
    """
    This class is a semantic aware rop gadget finder
    It is a work in progress, so don't be surprised if something doesn't quite work
    """

    def __init__(self, only_check_near_rets=True, max_block_size=20, max_sym_mem_accesses=4, fast_mode=None):
        """
        Initializes the rop gadget finder
        :param only_check_near_rets: If true we skip blocks that are not near rets
        :param max_block_size: limits the size of blocks considered, longer blocks are less likely to be good rop
                               gadgets so we limit the size we consider
        :param fast_mode: if set to True sets options to run fast, if set to False sets options to find more gadgets
                          if set to None makes a decision based on the size of the binary
        :return:
        """

        # params
        self._max_block_size = max_block_size
        self._only_check_near_rets = only_check_near_rets
        self._max_sym_mem_accesses = max_sym_mem_accesses

        a = self.project.arch
        self._sp_reg = a.register_names[a.sp_offset]
        self._ip_reg = a.register_names[a.ip_offset]
        self._base_pointer = a.register_names[a.bp_offset]

        self._reg_list = a.default_symbolic_registers
        # prune the register list of the instruction pointer and the stack pointer
        self._reg_list = filter(lambda r: r != self._sp_reg, self._reg_list)
        self._reg_list = filter(lambda r: r != self._ip_reg, self._reg_list)

        self._syscall_instructions = {"\x0f\x05"}

        if self.project.arch.linux_name == "x86_64":
            self._syscall_instructions = {"\x0f\x05"}
            self._cc = ["rdi", "rsi", "rdx", "rcx", "r8", "r9"]
            self._execve_syscall = 59
        elif self.project.arch.linux_name == "i386":
            self._syscall_instructions = {"\xcd\x80"}
            self._cc = "stack"
            self._execve_syscall = 11

        # get ret locations
        self._ret_locations = self._get_ret_locations()

        # list of gadgets
        self.gadgets = []
        self._duplicates = []

        num_to_check = len(list(self._addresses_to_check()))
        # fast mode
        if fast_mode is None:
            if num_to_check > 20000:
                fast_mode = True
                l.warning("Enabling fast mode for large binary")
            else:
                fast_mode = False
        self._fast_mode = fast_mode

        if self._fast_mode:
            self._max_block_size = 12
            self._max_sym_mem_accesses = 1
            num_to_check = len(list(self._addresses_to_check()))

        l.info("There are %d addresses withing %d bytes of a ret",
               num_to_check, self._max_block_size)

        # gadget analyzer
        self._gadget_analyzer = gadget_analyzer.GadgetAnalyzer(self.project, self._reg_list, self._max_block_size,
                                                               self._fast_mode, self._max_sym_mem_accesses)
        # chain builder
        self._chain_builder = None

        # silence annoying loggers
        simuvex.vex.ccall.l.setLevel("CRITICAL")
        simuvex.vex.expressions.ccall.l.setLevel("CRITICAL")

    def find_gadgets(self, processes=4):
        """
        Finds all the gadgets in the binary by calling analyze_gadget on every address near a ret.
        Saves gadgets in self.gadgets
        :param processes: number of processes to use
        """
        self.gadgets = []

        pool = Pool(processes=processes, initializer=_set_global_gadget_analyzer, initargs=(self._gadget_analyzer,))

        it = pool.imap_unordered(run_worker, self._addresses_to_check_with_caching(), chunksize=5)
        for gadget in it:
            if gadget is not None:
                self.gadgets.append(gadget)

        pool.close()

        # fix up gadgets from cache
        for g in self.gadgets:
            if g.addr in self._cache:
                dups = {g.addr}
                for addr in self._cache[g.addr]:
                    dups.add(addr)
                    g_copy = g.copy()
                    g_copy.addr = addr
                    self.gadgets.append(g_copy)
                self._duplicates.append(dups)
        self.gadgets = sorted(self.gadgets, key=lambda x: x.addr)
        self._reload_chain_funcs()

    def find_gadgets_single_threaded(self):
        """
        Finds all the gadgets in the binary by calling analyze_gadget on every address near a ret.
        Saves gadgets in self.gadgets
        """
        self.gadgets = []

        _set_global_gadget_analyzer(self._gadget_analyzer)
        for _, addr in enumerate(self._addresses_to_check_with_caching()):
            gadget = _global_gadget_analyzer.analyze_gadget(addr)
            if gadget is not None:
                self.gadgets.append(gadget)

        # fix up gadgets from cache
        for g in self.gadgets:
            if g.addr in self._cache:
                dups = {g.addr}
                for addr in self._cache[g.addr]:
                    dups.add(addr)
                    g_copy = g.copy()
                    g_copy.addr = addr
                    self.gadgets.append(g_copy)
                self._duplicates.append(dups)
        self.gadgets = sorted(self.gadgets, key=lambda x: x.addr)
        self._reload_chain_funcs()

    def save_gadgets(self, path):
        pickle.dump((self.gadgets, self._duplicates), open(path, "wb"))

    def load_gadgets(self, path):
        self.gadgets, self._duplicates = pickle.load(open(path, "rb"))
        self._reload_chain_funcs()

    def _reload_chain_funcs(self):
        for f_name, f in inspect.getmembers(self.chain_builder, predicate=inspect.ismethod):
            if f_name.startswith("_"):
                continue
            setattr(self, f_name, f)

    @property
    def chain_builder(self):
        if self._chain_builder is not None:
            return self._chain_builder
        elif len(self.gadgets) > 0:
            self._chain_builder = chain_builder.ChainBuilder(self.project, self.gadgets, self._duplicates,
                                                             self._reg_list)
            return self._chain_builder
        else:
            raise Exception("No gadgets, call find_gadgets() or load_gadgets() first")

    def _block_has_ip_relative(self, addr, bl):
        string = bl.bytes
        test_addr = 0x41414140 + addr % 0x10
        bl2 = self.project.factory.block(test_addr, insn_bytes=string)
        # TODO: FIXME
        try:
            diff_constants = angr.bindiff.differing_constants(bl, bl2)
        except angr.analyses.bindiff.UnmatchedStatementsException:
            return True
        # check if it changes if we move it
        bl_end = addr + bl.size
        bl2_end = test_addr + bl2.size
        filtered_diffs = []
        for d in diff_constants:
            if d.value_a < addr or d.value_a >= bl_end or \
                    d.value_b < test_addr or d.value_b >= bl2_end:
                filtered_diffs.append(d)
        return len(filtered_diffs) > 0

    def _addresses_to_check_with_caching(self, show_progress=True):
        num_addrs = len(list(self._addresses_to_check()))
        widgets = ['ROP: ', progressbar.Percentage(), ' ',
                   progressbar.Bar(marker=progressbar.RotatingMarker()),
                   ' ', progressbar.ETA(), ' ', progressbar.FileTransferSpeed()]
        progress = progressbar.ProgressBar(widgets=widgets, maxval=num_addrs)
        if show_progress:
            progress.start()
        self._cache = dict()
        seen = dict()
        for i, a in enumerate(self._addresses_to_check()):
            if show_progress:
                progress.update(i)
            try:
                bl = self.project.factory.block(a)
                if bl.size > self._max_block_size:
                    continue
                block_data = bl.bytes
            except angr.AngrTranslationError:
                continue
            if block_data in seen:
                self._cache[seen[block_data]].add(a)
                continue
            else:
                if len(bl.vex.constant_jump_targets) == 0 and not self._block_has_ip_relative(a, bl):
                    seen[block_data] = a
                    self._cache[a] = set()
                yield a
        if show_progress:
            progress.finish()

    def _addresses_to_check(self):
        """
        :return: all the addresses to check
        """
        if self._only_check_near_rets:
            # align block size
            alignment = self.project.arch.instruction_alignment
            block_size = (self._max_block_size & ((1 << self.project.arch.bits) - alignment)) + alignment
            slices = [(addr-block_size, addr) for addr in self._ret_locations]
            current_addr = 0
            for st, ed in slices:
                current_addr = max(current_addr, st)
                end_addr = st + block_size + alignment
                for i in range(current_addr, end_addr, alignment):
                    if self.project.loader.main_bin.find_segment_containing(i).is_executable:
                        yield i
                current_addr = max(current_addr, end_addr)
        else:
            raise NotImplementedError("this mode is not implemented in the arch agnostic branch")

    def _get_ret_locations(self):
        """
        :return: all the locations in the binary with a ret instruction
        """
        addrs = [ ]
        seen = set()
        for segment in self.project.loader.main_bin.segments:
            if segment.is_executable:
                min_addr = segment.min_addr + self.project.loader.main_bin.rebase_addr
                num_bytes = segment.max_addr-segment.min_addr

                alignment = self.project.arch.instruction_alignment
                # iterate backwards through the code looking for rets
                # align num_bytes if need
                num_bytes = num_bytes & ((1 << self.project.arch.bits) - alignment)
                for offset in xrange(num_bytes, 0, -alignment):
                    block = self.project.factory.block(min_addr+offset)
                    if block.vex.jumpkind.startswith("Ijk_Ret"):
                        if not block.addr + block.size in seen:
                            addrs.append(min_addr+offset)
                            seen.add(min_addr+offset+block.size)

        return sorted(addrs)

angr.analysis.register_analysis(ROP, 'ROP')
<|endoftext|>


<commit_before>from abc import ABCMeta
import json
import logging
import os

from nativeconfig.exceptions import InitializationError, ValidationError, DeserializationError

LOG = logging.getLogger('nativeconfig')

class BaseOption(property, metaclass=ABCMeta):
    """
    Base class for all options.

    Value of an option can be overridden by an environment variable.

    Setting option to None is equivalent to deleting it.

    @raise Error: Raise if any of the arguments has inappropriate value.
    @raise ValidationError: Raise if default value or any of the choices values is invalid.
    """
    def __init__(self,
                 name,
                 *,
                 getter='get_value',
                 setter='set_value',
                 deleter='del_value',
                 resolver='resolve_value',
                 choices=None,
                 env_name=None,
                 default=None):
        """
        @param name: Name of the property.
        @type name: str

        @param getter: Name of the method of the enclosing class to get value.

        @param setter: Name of the of the enclosing class to set value.

        @param deleter: Name of the deleter of the enclosing class to delete value.

        @param resolver: Name of the resolver of the enclosing class to resolve value that cannot be serialized.

        @param choices: List of allowed choices for the option.

        @param env_name: Name of the environment variable that can override value of the option
        @type env_name: str

        @param default: Default value of the option.

        @raise InitializationError: If any of arguments is incorrect. Only handles most obvious errors.
        @raise ValidationError: if default or any of choices is invalid.
        """
        super(BaseOption, self).__init__(self.fget, self.fset, self.fdel, doc=self.__doc__)

        self._name = name
        self._getter = getter
        self._setter = setter
        self._deleter = deleter
        self._resolver = resolver
        self._choices = choices
        self._env_name = env_name
        self._default = default

        self._one_shot_value = None

        if not name:
            raise InitializationError("'name' cannot be empty.")

        if env_name is not None and len(env_name) == 0:
            raise InitializationError("'env_name' cannot be empty.")

        if choices is not None and len(choices) == 0:
            raise InitializationError("'choices' cannot be empty.")

        if choices is not None:
            for c in choices:
                self.validate(c)

        if default is not None:
            self.validate(default)

    def set_one_shot_value(self, value):
        """
        Set one shot value of the option that overrides value from storage but can be reset by set.

        Useful if you want to allow a user to override the option via CLI.

        @raise ValidationError: Raise if value is not valid.
        """
        self.validate(value)
        self._one_shot_value = None

#{ Validation

    def validate(self, value):
        """
        Validate value. Must raise ValidationError if value is wrong.

        @raise ValidationError: Raise if value is wrong.
        """
        if value is None:
            return

        if self._choices is not None and value not in self._choices:
            raise ValidationError("Value '{}' is not one of the choices {}.".format(value, self._choices), value)

#{ Serialization and deserialization

    def serialize(self, value):
        """
        Serialize value into raw value.

        @see: deserialize

        @raise SerializationError: Raise if value cannot be serialized.
        """
        return str(value)

    def deserialize(self, raw_value):
        """
        Deserialize raw value to value.

        @see: serialize

        @raise DeserializationError: Raise if value cannot be deserialized.
        """
        return str(raw_value)

    def serialize_json(self, value):
        """
        Serialize value into json compatible string.

        @see: deserialize_json

        @rtype: str
        """
        return json.dumps(value)

    def deserialize_json(self, json_value):
        """
        Deserialize json value to value.

        @see: serialize_json
        """
        return json.loads(json_value)

#{ Access backend

    def fget(self, enclosing_self):
        """
        @raise DeserializationError: Raise if value cannot be deserialized or deserialized value isn't one of choices.
        """
        raw_v = None

        if self._env_name:
            raw_v = os.getenv(self._env_name)
            if raw_v is not None:
                LOG.debug("value of '%s' is overridden by environment variable: %s", self._name, raw_v)
                raw_v = self.deserialize_json(raw_v)

        if raw_v is None:
            raw_v = self._one_shot_value

        if raw_v is None:
            raw_v = getattr(enclosing_self, self._getter)(self._name)

        if raw_v is None:
            LOG.debug("No value is set for '%s', use default.", self._name)
            return self._default
        else:
            try:
                value = self.deserialize(raw_v)
                self.validate(value)
                return value
            except (DeserializationError, ValidationError) as e:
                return getattr(enclosing_self, self._resolver)(e, self._name, raw_v)

    def fset(self, enclosing_self, value):
        self._one_shot_value = None

        if value is not None:
            self.validate(value)
            raw_value = self.serialize(value)
            LOG.debug("Value of '%s' is set to '%s'.", value, raw_value)
            getattr(enclosing_self, self._setter)(self._name, raw_value)
        else:
            self.fdel(enclosing_self)

    def fdel(self, enclosing_self):
        LOG.debug("Delete value of '%s'.", self._name)
        getattr(enclosing_self, self._deleter)(self._name)
#}
<commit_msg>Make clear which representations option values can have.<commit_after>from abc import ABCMeta
import json
import logging
import os

from nativeconfig.exceptions import InitializationError, ValidationError, DeserializationError

LOG = logging.getLogger('nativeconfig')

class BaseOption(property, metaclass=ABCMeta):
    """
    Base class for all options.

    Value of each option can be represented by three different versions:

        1. Python Value: used in the code
        2. Raw Value (utf-8 sting): used to store in underlying config
        3. JSON Value: used to interact with the user

    Value of an option can be overridden by an environment variable.

    Setting option to None is equivalent to deleting it.

    @raise Error: Raise if any of the arguments has inappropriate value.
    @raise ValidationError: Raise if default value or any of the choices values is invalid.
    """
    def __init__(self,
                 name,
                 *,
                 getter='get_value',
                 setter='set_value',
                 deleter='del_value',
                 resolver='resolve_value',
                 choices=None,
                 env_name=None,
                 default=None):
        """
        @param name: Name of the property.
        @type name: str

        @param getter: Name of the method of the enclosing class to get value.

        @param setter: Name of the of the enclosing class to set value.

        @param deleter: Name of the deleter of the enclosing class to delete value.

        @param resolver: Name of the resolver of the enclosing class to resolve value that cannot be serialized.

        @param choices: List of allowed Python Values for the option.

        @param env_name: Name of the env variable that contains JSON Value that can override value of the option.
        @type env_name: str

        @param default: Default Python Value of the option.

        @raise InitializationError: If any of arguments is incorrect. Only handles most obvious errors.
        @raise ValidationError: If default or any of choices is invalid.
        """
        super(BaseOption, self).__init__(self.fget, self.fset, self.fdel, doc=self.__doc__)

        self._name = name
        self._getter = getter
        self._setter = setter
        self._deleter = deleter
        self._resolver = resolver
        self._choices = choices
        self._env_name = env_name
        self._default = default

        self._one_shot_value = None

        if not name:
            raise InitializationError("'name' cannot be empty.")

        if env_name is not None and len(env_name) == 0:
            raise InitializationError("'env_name' cannot be empty.")

        if choices is not None and len(choices) == 0:
            raise InitializationError("'choices' cannot be empty.")

        if choices is not None:
            for c in choices:
                self.validate(c)

        if default is not None:
            self.validate(default)

    def set_one_shot_value(self, value):
        """
        Set one shot value of the option that overrides value from storage but can be reset by set.

        Useful if you want to allow a user to override the option via CLI.

        @raise ValidationError: Raise if value is not valid.
        """
        self.validate(value)
        self._one_shot_value = None

#{ Validation

    def validate(self, value):
        """
        Validate Python Value. Must raise ValidationError if value is wrong.

        @raise ValidationError: Raise if value is wrong.
        """
        if value is None:
            return

        if self._choices is not None and value not in self._choices:
            raise ValidationError("Value '{}' is not one of the choices {}.".format(value, self._choices), value)

#{ Serialization and deserialization

    def serialize(self, value):
        """
        Serialize Python Value into Raw Value.

        @see: deserialize

        @raise SerializationError: Raise if value cannot be serialized.
        """
        return str(value)

    def deserialize(self, raw_value):
        """
        Deserialize Raw Value into Python Value.

        @see: serialize

        @raise DeserializationError: Raise if value cannot be deserialized.
        """
        return str(raw_value)

    def serialize_json(self, value):
        """
        Serialize Python Value into JSON Value.

        @see: deserialize_json

        @rtype: str
        """
        return json.dumps(value)

    def deserialize_json(self, json_value):
        """
        Deserialize JSON Value into Python Value.

        @see: serialize_json
        """
        return json.loads(json_value)

#{ Access backend

    def fget(self, enclosing_self):
        """
        Read Raw Value from the storage and deserialized it into Python Value.

        If either DeserializationError or ValidationError occurs in process, they will be forwarded to resolver.

        @param enclosing_self: Instance of class that defines this property.
        """
        raw_v = None

        if self._env_name:
            raw_v = os.getenv(self._env_name)
            if raw_v is not None:
                LOG.debug("value of '%s' is overridden by environment variable: %s", self._name, raw_v)
                raw_v = self.deserialize_json(raw_v)

        if raw_v is None:
            raw_v = self._one_shot_value

        if raw_v is None:
            raw_v = getattr(enclosing_self, self._getter)(self._name)

        if raw_v is None:
            LOG.debug("No value is set for '%s', use default.", self._name)
            return self._default
        else:
            try:
                value = self.deserialize(raw_v)
                self.validate(value)
                return value
            except (DeserializationError, ValidationError) as e:
                return getattr(enclosing_self, self._resolver)(e, self._name, raw_v)

    def fset(self, enclosing_self, value):
        """
        Serialize Python Value into Raw Value and write it to storage.

        Resets One Shot Value.
        Setting None simply deletes Raw Value from storage.

        @param enclosing_self: Instance of class that defines this property.

        @param value: Python Value to be set.
        """
        self._one_shot_value = None

        if value is not None:
            self.validate(value)
            raw_value = self.serialize(value)
            LOG.debug("Value of '%s' is set to '%s'.", self._name, raw_value)
            getattr(enclosing_self, self._setter)(self._name, raw_value)
        else:
            self.fdel(enclosing_self)

    def fdel(self, enclosing_self):
        """
        Delete Raw Value from the storage.

        @param enclosing_self: Instance of class that defines this property.
        """
        LOG.debug("Delete value of '%s'.", self._name)
        getattr(enclosing_self, self._deleter)(self._name)
#}
<|endoftext|>


<commit_before>import json
import threading
from collections.abc import MutableSequence
from pathlib import Path

from robot.board import Board
from robot.markers import Marker

class Camera(Board):
    """
    Object representing an Apriltag camera in robotd

    Polls the robot daemon for new images
    """

    def __init__(self, socket_path):
        self._latest = None
        self._got_image = threading.Event()
        super().__init__(socket_path)
        self._serial = Path(socket_path).stem
        self._stop = threading.Event()
        self._latest_lock = threading.Lock()
        self._start_listening()

    def _start_listening(self):
        """
        Start listening thread
        """
        thread = threading.Thread(target=self._cam_listener_worker)
        self._alive = True
        thread.start()
        self.sock_thread = thread

    def _clean_up(self):
        pass

    def _stop_poll(self):
        """
        Stop polling the camera
        """
        self._alive = False
        self.sock_thread.join()

    @property
    def _alive(self):
        return not self._stop.is_set()

    @_alive.setter
    def _alive(self, value):
        if value:
            self._stop.clear()
        else:
            self._stop.set()

    def _cam_listener_worker(self):
        """
        Worker thread for listening to the camera socket

        Works until `self._running` is set.
        """
        while self._alive:
            data = self._recv()
            if data:
                self._got_image.set()
                with self._latest_lock:
                    self._latest = data

    @staticmethod
    def _see_to_results(data):
        """
        Converts the string output that comes from the camera in robotd to a list of #Marker objects
        :param data: json string data to convert
        :return: #ResultList object (that imitates a list) of markers
        """
        markers = []
        data = json.loads(data.decode('utf-8'))
        for token in data["tokens"]:
            markers.append(Marker(token))
        # Sort by distance
        return ResultList(sorted(markers, key=lambda x: x.distance_metres))

    @property
    def serial(self):
        """
        Serial number for the board
        """
        return self._serial

    def see(self):
        """
        Look for markers
        :return: List of #Marker objects
        """
        self._got_image.wait()
        with self._latest_lock:
            return self._see_to_results(self._latest)

class ResultList(MutableSequence):
    """
    This class pretends to be a list, except it returns
    a much more useful error description if the user indexes an empty array.

    This is to mitigate a common beginners issue where an array is indexed
    without checking that the array has any items
    """

    def __delitem__(self, index):
        del self.data[index]

    def __init__(self, data):
        self.data = data

    def __getitem__(self, item):
        try:
            self.data[item]
        except IndexError as e:
            if len(self.data) == 0:
                raise IndexError("Trying to index an empty list")
            else:
                raise e

    def __setitem__(self, key, value):
        self.data[key] = value

    def insert(self, index, value):
        self.data.insert(index, value)

    def __len__(self):
        return len(self.data)

    def __repr__(self):
        return self.data.__repr__()

    def __eq__(self, other):
        return self.data.__eq__(other)

    def __iter__(self):
        return self.data.__iter__()
<commit_msg>Fix camera race condition<commit_after>import json
import threading
from collections.abc import MutableSequence
from pathlib import Path

from robot.board import Board
from robot.markers import Marker

class Camera(Board):
    """
    Object representing an Apriltag camera in robotd

    Polls the robot daemon for new images
    """

    def __init__(self, socket_path):
        self._latest = None
        self._got_image = threading.Event()
        super().__init__(socket_path)
        self._serial = Path(socket_path).stem
        self._stop = threading.Event()
        self._latest_lock = threading.Lock()
        self._start_listening()

    def _start_listening(self):
        """
        Start listening thread
        """
        thread = threading.Thread(target=self._cam_listener_worker)
        self._alive = True
        thread.start()
        self.sock_thread = thread

    def _clean_up(self):
        pass

    def _stop_poll(self):
        """
        Stop polling the camera
        """
        self._alive = False
        self.sock_thread.join()

    @property
    def _alive(self):
        return not self._stop.is_set()

    @_alive.setter
    def _alive(self, value):
        if value:
            self._stop.clear()
        else:
            self._stop.set()

    def _cam_listener_worker(self):
        """
        Worker thread for listening to the camera socket

        Works until `self._running` is set.
        """
        while self._alive:
            received = self._recv()
            if received:
                data = json.loads(received.decode('utf-8'))
                if data:
                    self._got_image.set()
                    with self._latest_lock:
                        self._latest = data

    @staticmethod
    def _see_to_results(data):
        """
        Converts the string output that comes from the camera in robotd to a list of #Marker objects
        :param data: json string data to convert
        :return: #ResultList object (that imitates a list) of markers
        """
        markers = []
        for token in data["tokens"]:
            markers.append(Marker(token))
        # Sort by distance
        return ResultList(sorted(markers, key=lambda x: x.distance_metres))

    @property
    def serial(self):
        """
        Serial number for the board
        """
        return self._serial

    def see(self):
        """
        Look for markers
        :return: List of #Marker objects
        """
        self._got_image.wait()
        with self._latest_lock:
            return self._see_to_results(self._latest)

class ResultList(MutableSequence):
    """
    This class pretends to be a list, except it returns
    a much more useful error description if the user indexes an empty array.

    This is to mitigate a common beginners issue where an array is indexed
    without checking that the array has any items
    """

    def __delitem__(self, index):
        del self.data[index]

    def __init__(self, data):
        self.data = data

    def __getitem__(self, item):
        try:
            self.data[item]
        except IndexError as e:
            if len(self.data) == 0:
                raise IndexError("Trying to index an empty list")
            else:
                raise e

    def __setitem__(self, key, value):
        self.data[key] = value

    def insert(self, index, value):
        self.data.insert(index, value)

    def __len__(self):
        return len(self.data)

    def __repr__(self):
        return self.data.__repr__()

    def __eq__(self, other):
        return self.data.__eq__(other)

    def __iter__(self):
        return self.data.__iter__()
<|endoftext|>


<commit_before>
import time
import socket
import threading

from random import random

from alerta.common import log as logging
from alerta.common import config

LOG = logging.getLogger(__name__)
CONF = config.CONF

class Carbon(object):

    carbon_opts = {
        'carbon_host': 'localhost',
        'carbon_port': 2003,
        'carbon_protocol': 'tcp',
        'graphite_prefix': 'alerta.%s' % socket.gethostname(),
    }

    def __init__(self, host=None, port=None, protocol=None, prefix=None):

        config.register_opts(Carbon.carbon_opts)

        self.host = host or CONF.carbon_host
        self.port = port or CONF.carbon_port
        self.protocol = protocol or CONF.carbon_protocol
        self.prefix = prefix or CONF.graphite_prefix

        self.lock = threading.Lock()

        if self.protocol not in ['udp', 'tcp']:
            LOG.error("Protocol must be one of: udp, tcp")
            return

        LOG.info('Carbon setup to send %s packets to %s:%s', self.protocol, self.host, self.port)

        self.addr = (self.host, int(self.port))

        if self.protocol == 'udp':
            self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        else:
            self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            try:
                self.socket.settimeout(0.5)
                self.socket.connect(self.addr)
                self._connected = True
            except (socket.error, socket.timeout):
                self._connected = False
                LOG.warning('Carbon server %s not responding on TCP port %s', self.host, self.port)
            finally:
                self.socket.settimeout(None)

    def metric_send(self, name, value, timestamp=None):

        if name is None or value is None:
            LOG.error('Invalid carbon parameters. Must supply name and value.')
            return

        if self.prefix:
            name = '%s.%s' % (self.prefix, name)

        if not timestamp:
            timestamp = int(time.time())

        LOG.debug('Carbon name=%s, value=%s, timestamp=%s', name, value, timestamp)

        if self.protocol == 'udp':
            try:
                count = self.socket.sendto('%s %s %s\n' % (name, value, timestamp), self.addr)
            except socket.error, e:
                LOG.warning('Failed to send metric to UDP Carbon server %s:%s: %s', self.host, self.port, e)
            else:
                LOG.debug('Sent %s UDP metric packets', count)
        else:
            if not self._connected:
                with self.lock:
                    LOG.info('Attempting reconnect to Carbon server %s:%s', self.host, self.port)
                    try:
                        self.socket.settimeout(0.5)
                        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                        self.socket.connect(self.addr)
                        self._connected = True
                        LOG.info('Reconnected to Carbon server %s on TCP port %s', self.host, self.port)
                    except (socket.error, socket.timeout):
                        LOG.warning('Carbon server %s not responding to TCP port %s', self.host, self.port)
                        self._connected = False
                    finally:
                        self.socket.settimeout(None)

            if self._connected:
                with self.lock:
                    try:
                        self.socket.sendall('%s %s %s\n' % (name, value, timestamp))
                    except socket.error, e:
                        LOG.warning('Failed to send metric to TCP Carbon server %s:%s: e', self.host, self.port, e)
                        self._connected = False
                    else:
                        LOG.debug('Sent all TCP metric data')

    def shutdown(self):

        if self.protocol == 'tcp':
            self.socket.close()

class StatsD(object):

    statsd_opts = {
        'statsd_host': 'localhost',
        'statsd_port': 8125,
        'graphite_prefix': 'alerta.%s' % socket.gethostname(),
    }

    def __init__(self, host=None, port=None, rate=1, prefix=None):

        config.register_opts(StatsD.statsd_opts)

        self.host = host or CONF.statsd_host
        self.port = port or CONF.statsd_port
        self.rate = rate
        self.prefix = prefix or CONF.graphite_prefix

        LOG.info('Statsd setup to send packets to %s:%s with sample rate of %d', self.host, self.port, self.rate)

        self.addr = (self.host, int(self.port))
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

    def metric_send(self, name, value, mtype=None, rate=None):
        """
        >>> statsd = StatsD()
        >>> statsd.metric_send('gorets', 1, 'c', 0.1)  # counters  => 'gorets:1|c@0.1'
        >>> statsd.metric_send('glork', 320, 'ms')     # timers    => 'glork:320|ms'
        >>> statsd.metric_send('gaugor', 333, 'g')     # gauge     => 'gaugor:333|g'
        >>> statsd.metric_send('gaugor', -10, 'g')     # gauge dec => 'gaugor:-10|g'
        >>> statsd.metric_send('gaugor', '+4', 'g')    # gauge inc => 'gaugor:+4|g'
        >>> statsd.metric_send('uniques', 765, 's')    # sets      => 'uniques:765|s
        """

        if self.prefix:
            name = '%s.%s' % (self.prefix, name)

        mtype = mtype or 'c'  # default is 'counter'
        rate = rate or self.rate

        LOG.debug('Statsd name=%s, value=%s, type=%s rate=%s', name, value, mtype, rate)

        data = '%s:%s|%s' % (name, value, mtype)

        if rate < 1:
            if random() <= rate:
                data = '%s|@%s' % (data, rate)
            else:
                return

        LOG.debug('Statsd metric send: %s', data)
        try:
            self.socket.sendto(data.encode('utf-8'), self.addr)
        except socket.error, e:
            LOG.warning('Failed to send metric to UDP Statsd server %s:%s: %s', self.host, self.port, e)

<commit_msg>fix graphite error logging<commit_after>
import time
import socket
import threading

from random import random

from alerta.common import log as logging
from alerta.common import config

LOG = logging.getLogger(__name__)
CONF = config.CONF

class Carbon(object):

    carbon_opts = {
        'carbon_host': 'localhost',
        'carbon_port': 2003,
        'carbon_protocol': 'tcp',
        'graphite_prefix': 'alerta.%s' % socket.gethostname(),
    }

    def __init__(self, host=None, port=None, protocol=None, prefix=None):

        config.register_opts(Carbon.carbon_opts)

        self.host = host or CONF.carbon_host
        self.port = port or CONF.carbon_port
        self.protocol = protocol or CONF.carbon_protocol
        self.prefix = prefix or CONF.graphite_prefix

        self.lock = threading.Lock()

        if self.protocol not in ['udp', 'tcp']:
            LOG.error("Protocol must be one of: udp, tcp")
            return

        LOG.info('Carbon setup to send %s packets to %s:%s', self.protocol, self.host, self.port)

        self.addr = (self.host, int(self.port))

        if self.protocol == 'udp':
            self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        else:
            self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            try:
                self.socket.settimeout(0.5)
                self.socket.connect(self.addr)
                self._connected = True
            except (socket.error, socket.timeout):
                self._connected = False
                LOG.warning('Carbon server %s not responding on TCP port %s', self.host, self.port)
            finally:
                self.socket.settimeout(None)

    def metric_send(self, name, value, timestamp=None):

        if name is None or value is None:
            LOG.error('Invalid carbon parameters. Must supply name and value.')
            return

        if self.prefix:
            name = '%s.%s' % (self.prefix, name)

        if not timestamp:
            timestamp = int(time.time())

        LOG.debug('Carbon name=%s, value=%s, timestamp=%s', name, value, timestamp)

        if self.protocol == 'udp':
            try:
                count = self.socket.sendto('%s %s %s\n' % (name, value, timestamp), self.addr)
            except socket.error, e:
                LOG.warning('Failed to send metric to UDP Carbon server %s:%s: %s', self.host, self.port, e)
            else:
                LOG.debug('Sent %s UDP metric packets', count)
        else:
            if not self._connected:
                with self.lock:
                    LOG.info('Attempting reconnect to Carbon server %s:%s', self.host, self.port)
                    try:
                        self.socket.settimeout(0.5)
                        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                        self.socket.connect(self.addr)
                        self._connected = True
                        LOG.info('Reconnected to Carbon server %s on TCP port %s', self.host, self.port)
                    except (socket.error, socket.timeout):
                        LOG.warning('Carbon server %s not responding to TCP port %s', self.host, self.port)
                        self._connected = False
                    finally:
                        self.socket.settimeout(None)

            if self._connected:
                with self.lock:
                    try:
                        self.socket.sendall('%s %s %s\n' % (name, value, timestamp))
                    except socket.error, e:
                        LOG.warning('Failed to send metric to TCP Carbon server %s:%s: %s', self.host, self.port, e)
                        self._connected = False
                    else:
                        LOG.debug('Sent all TCP metric data')

    def shutdown(self):

        if self.protocol == 'tcp':
            self.socket.close()

class StatsD(object):

    statsd_opts = {
        'statsd_host': 'localhost',
        'statsd_port': 8125,
        'graphite_prefix': 'alerta.%s' % socket.gethostname(),
    }

    def __init__(self, host=None, port=None, rate=1, prefix=None):

        config.register_opts(StatsD.statsd_opts)

        self.host = host or CONF.statsd_host
        self.port = port or CONF.statsd_port
        self.rate = rate
        self.prefix = prefix or CONF.graphite_prefix

        LOG.info('Statsd setup to send packets to %s:%s with sample rate of %d', self.host, self.port, self.rate)

        self.addr = (self.host, int(self.port))
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

    def metric_send(self, name, value, mtype=None, rate=None):
        """
        >>> statsd = StatsD()
        >>> statsd.metric_send('gorets', 1, 'c', 0.1)  # counters  => 'gorets:1|c@0.1'
        >>> statsd.metric_send('glork', 320, 'ms')     # timers    => 'glork:320|ms'
        >>> statsd.metric_send('gaugor', 333, 'g')     # gauge     => 'gaugor:333|g'
        >>> statsd.metric_send('gaugor', -10, 'g')     # gauge dec => 'gaugor:-10|g'
        >>> statsd.metric_send('gaugor', '+4', 'g')    # gauge inc => 'gaugor:+4|g'
        >>> statsd.metric_send('uniques', 765, 's')    # sets      => 'uniques:765|s
        """

        if self.prefix:
            name = '%s.%s' % (self.prefix, name)

        mtype = mtype or 'c'  # default is 'counter'
        rate = rate or self.rate

        LOG.debug('Statsd name=%s, value=%s, type=%s rate=%s', name, value, mtype, rate)

        data = '%s:%s|%s' % (name, value, mtype)

        if rate < 1:
            if random() <= rate:
                data = '%s|@%s' % (data, rate)
            else:
                return

        LOG.debug('Statsd metric send: %s', data)
        try:
            self.socket.sendto(data.encode('utf-8'), self.addr)
        except socket.error, e:
            LOG.warning('Failed to send metric to UDP Statsd server %s:%s: %s', self.host, self.port, e)

<|endoftext|>


<commit_before>from django.db import models
from django import forms
from django.forms import ModelForm

class Vendor(models.Model):
    vname = models.CharField(max_length=200)
    vnumber = models.IntegerField(max_length=11, unique=True)
    vmail = models.EmailField()
    vstate = models.CharField(max_length=20)
    vstreet = models.CharField(max_length=70)
    vcity = models.CharField(max_length=30)
    vzip = models.CharField(max_length=10)

    def __unicode__(self):
        return '%s %s %s' %(self.vname, str(self.vnumber), str(self.vmail))

class Engine(models.Model):
    MARKET_OPT = (
        ('FI', 'Finland'),
        ('DK', 'Denmark'),
        ('SE', 'Sweden'),
        ('NO', 'Norway'),
        ('AT', 'Austria'),
        ('FR', 'France'),
        ('DE', 'Germany'),
        ('IT', 'Italy'),
        ('NL', 'Netherland'),
        ('PT', 'Portugal'),
        ('PL', 'Poland'),
        ('GB', 'Great Britain'),
        ('CH', 'Switzerland'),
        ('BE', 'Belgium'),
        ('ES', 'Spain')
        )

    CCODE_OPT = (
        ('10', '10'),
        ('11', '11'),
        ('12', '12'),
        ('13', '13'),
        ('14', '14'),
        ('15', '15'),
        ('16', '16'),
        ('18', '18'),
        ('19', '19'),
        ('20', '21'),
        ('29', '29'),
        ('30', '30'),
        ('31', '31'),
        ('36', '36'),
        ('52', '52'),
        ('55', '55'),
        ('60', '60'),
        ('64', '64'),
        ('66', '66'),
        ('85', '85')
        )

    LEVEL_OPT = (
        ('1', '1st'),
        ('2', '2nd'),
        ('3', '3rd'),
        ('4', '4th'),
        ('5', 'to scan'),
        )

    INVSTATUS_OPT = (
        ('RJ', 'Rejected'),
        ('PO', 'Posted'),
        ('PD', 'Paid on'),
        ('NP', 'Not posted yet'),
        ('BL', 'Blocked'),
        ('NR', 'Not received'),
        ('CA', 'Cancelled'),
        ('RE', 'Reversed')
        )

    REJ_REASONS = (
        ("MPO", "Missing Purchase Order Number"),
        ("BIX", "Bad information"),
        ("DIX", "Duplicate invoice"),
        ("OTH", "Other"),
        ("IDX", "Invalid document"),
        ("WFI", "Wrong system - Route to FI box"),
        ("WSO", "Wrong system - Route to SOS box"),
        ("WCO", "Wrong system - Route to COM box"),
        ("IPO", "Invalid Purchase Order"),
        ("WCN", "Wrong company name"),
        ("WCA", "Wrong company address"),
        ("ICX", "Incorrect currency"),
        ("MCX", "Missing currency"),
        ("WVA", "Wrong VAT amount"),
        ("MVN", "Missing VAT number"),
        ("VMX", "Value on the invoice miscalculated"),
        ("DNV", "Data on the invoice not visible"),
        ("IDM", "Invoice date missing"),
        ("INM", "Invoice number missing"),
        ("MPX", "Missing pages"),
        ("TFM", "Tax free invoice-note missing"),
        ("DEM", "Delivery date / Delivery Note Missing"),
        ("QMX", "Quantity missing"),
        ("SDM", "Service description missing"),
        ("MIA", "Missing Invoice Amount"),
        ("IIX", "Incomplete Invoice"),
        ("MAI", "Missing Accounting Information"),
        ("IAI", "Invalid Accounting Information"),
        ("MAX", "Missing Approval"),
        ("IRA", "Invalid Remit To Address"),
        ("MRA", "Missing Remit To Address"),
        ("RNL", "Remit to address not linked to vendor's record"),
        ("VMX", "Vendor Master"),
        ("MUI", "Multiple invoices"),
        ("RES", "Rescan, poor quality")
        )

    CURR_OPT = (
        ("EUR", "EUR"),
        ("GBP", "GBP"),
        ("CHF", "CHF"),
        ("NOK", "NOK"),
        ("SEK", "SEK"),
        ("PLN", "PLN"),
        ("USD", "USD")
        )

    market = models.CharField(max_length=3, choices=MARKET_OPT)
    ccode = models.CharField(max_length=2, choices=CCODE_OPT)
    level = models.CharField(max_length=8, choices=LEVEL_OPT)
    clerk = models.CharField(max_length=50)
    amount = models.DecimalField(max_digits=11, decimal_places=2, null=True, blank=True)
    currency = models.CharField(max_length=3, choices=CURR_OPT, null=True, blank=True)
    #attachment = models.FileField(upload_to='img', null=True, blank=True)
    reasonother = models.CharField(max_length=500, null=True, blank=True)
    actiondate = models.DateField()
    reminderdate = models.DateField()
    remindernumber = models.CharField(max_length=30)
    vendor = models.CharField(max_length=10)
    mailvendor = models.EmailField()
    invoicenumber = models.CharField(max_length=20)
    invoicestatus = models.CharField(max_length=100, choices=INVSTATUS_OPT)
    actiontaken = models.CharField(max_length=100, blank=True)
    rejectreason = models.CharField(max_length=3, choices=REJ_REASONS, null=True, blank=True)
    paidon = models.DateField(null=True, blank=True)

    def __unicode__(self):
        return '%s, %s, %s, %s, %s, %s, %s, %s, %s %s, %s' % (
            self.market,
            self.ccode,
            self.level,
            self.reminderdate,
            self.remindernumber,
            self.vendor,
            self.mailvendor,
            self.invoicenumber,
            self.invoicestatus,
            self.rejectreason,
            self.paidon
            )

class Login(forms.Form):

    """ Basic login form

    This form renders into a very simple login field, with fields identified
    differently, both for styling and the optional javascript by-name
    handling. Whilst this form is extremely simple, coding one each and
    every time we wish to use one is just dumb and it's much easier to code it
    once and import it into our project.
    """

    uname = forms.CharField(label="Username:")

    uname.widget.attrs.update({'class': 'login-form','id': 'login-user'})

    passw = forms.CharField(label="Password:",widget=forms.PasswordInput())

    passw.widget.attrs.update({'class': 'login-form','id': 'login-password'})

class TrackingForm(ModelForm):

    class Meta:
	model = Engine
	widgets = {
	    'reasonother' : forms.Textarea()
            }
<commit_msg>Turned on the filed attachment to build the function.<commit_after>from django.db import models
from django import forms
from django.forms import ModelForm

class Vendor(models.Model):
    vname = models.CharField(max_length=200)
    vnumber = models.IntegerField(max_length=11, unique=True)
    vmail = models.EmailField()
    vstate = models.CharField(max_length=20)
    vstreet = models.CharField(max_length=70)
    vcity = models.CharField(max_length=30)
    vzip = models.CharField(max_length=10)

    def __unicode__(self):
        return '%s %s %s' %(self.vname, str(self.vnumber), str(self.vmail))

class Engine(models.Model):
    MARKET_OPT = (
        ('FI', 'Finland'),
        ('DK', 'Denmark'),
        ('SE', 'Sweden'),
        ('NO', 'Norway'),
        ('AT', 'Austria'),
        ('FR', 'France'),
        ('DE', 'Germany'),
        ('IT', 'Italy'),
        ('NL', 'Netherland'),
        ('PT', 'Portugal'),
        ('PL', 'Poland'),
        ('GB', 'Great Britain'),
        ('CH', 'Switzerland'),
        ('BE', 'Belgium'),
        ('ES', 'Spain')
        )

    CCODE_OPT = (
        ('10', '10'),
        ('11', '11'),
        ('12', '12'),
        ('13', '13'),
        ('14', '14'),
        ('15', '15'),
        ('16', '16'),
        ('18', '18'),
        ('19', '19'),
        ('20', '21'),
        ('29', '29'),
        ('30', '30'),
        ('31', '31'),
        ('36', '36'),
        ('52', '52'),
        ('55', '55'),
        ('60', '60'),
        ('64', '64'),
        ('66', '66'),
        ('85', '85')
        )

    LEVEL_OPT = (
        ('1', '1st'),
        ('2', '2nd'),
        ('3', '3rd'),
        ('4', '4th'),
        ('5', 'to scan'),
        )

    INVSTATUS_OPT = (
        ('RJ', 'Rejected'),
        ('PO', 'Posted'),
        ('PD', 'Paid on'),
        ('NP', 'Not posted yet'),
        ('BL', 'Blocked'),
        ('NR', 'Not received'),
        ('CA', 'Cancelled'),
        ('RE', 'Reversed')
        )

    REJ_REASONS = (
        ("MPO", "Missing Purchase Order Number"),
        ("BIX", "Bad information"),
        ("DIX", "Duplicate invoice"),
        ("OTH", "Other"),
        ("IDX", "Invalid document"),
        ("WFI", "Wrong system - Route to FI box"),
        ("WSO", "Wrong system - Route to SOS box"),
        ("WCO", "Wrong system - Route to COM box"),
        ("IPO", "Invalid Purchase Order"),
        ("WCN", "Wrong company name"),
        ("WCA", "Wrong company address"),
        ("ICX", "Incorrect currency"),
        ("MCX", "Missing currency"),
        ("WVA", "Wrong VAT amount"),
        ("MVN", "Missing VAT number"),
        ("VMX", "Value on the invoice miscalculated"),
        ("DNV", "Data on the invoice not visible"),
        ("IDM", "Invoice date missing"),
        ("INM", "Invoice number missing"),
        ("MPX", "Missing pages"),
        ("TFM", "Tax free invoice-note missing"),
        ("DEM", "Delivery date / Delivery Note Missing"),
        ("QMX", "Quantity missing"),
        ("SDM", "Service description missing"),
        ("MIA", "Missing Invoice Amount"),
        ("IIX", "Incomplete Invoice"),
        ("MAI", "Missing Accounting Information"),
        ("IAI", "Invalid Accounting Information"),
        ("MAX", "Missing Approval"),
        ("IRA", "Invalid Remit To Address"),
        ("MRA", "Missing Remit To Address"),
        ("RNL", "Remit to address not linked to vendor's record"),
        ("VMX", "Vendor Master"),
        ("MUI", "Multiple invoices"),
        ("RES", "Rescan, poor quality")
        )

    CURR_OPT = (
        ("EUR", "EUR"),
        ("GBP", "GBP"),
        ("CHF", "CHF"),
        ("NOK", "NOK"),
        ("SEK", "SEK"),
        ("PLN", "PLN"),
        ("USD", "USD")
        )

    market = models.CharField(max_length=3, choices=MARKET_OPT)
    ccode = models.CharField(max_length=2, choices=CCODE_OPT)
    level = models.CharField(max_length=8, choices=LEVEL_OPT)
    clerk = models.CharField(max_length=50)
    amount = models.DecimalField(max_digits=11, decimal_places=2, null=True, blank=True)
    currency = models.CharField(max_length=3, choices=CURR_OPT, null=True, blank=True)
    attachment = models.FileField(upload_to='img', null=True, blank=True)
    reasonother = models.CharField(max_length=500, null=True, blank=True)
    actiondate = models.DateField()
    reminderdate = models.DateField()
    remindernumber = models.CharField(max_length=30)
    vendor = models.CharField(max_length=10)
    mailvendor = models.EmailField()
    invoicenumber = models.CharField(max_length=20)
    invoicestatus = models.CharField(max_length=100, choices=INVSTATUS_OPT)
    actiontaken = models.CharField(max_length=100, blank=True)
    rejectreason = models.CharField(max_length=3, choices=REJ_REASONS, null=True, blank=True)
    paidon = models.DateField(null=True, blank=True)

    def __unicode__(self):
        return '%s, %s, %s, %s, %s, %s, %s, %s, %s %s, %s' % (
            self.market,
            self.ccode,
            self.level,
            self.reminderdate,
            self.remindernumber,
            self.vendor,
            self.mailvendor,
            self.invoicenumber,
            self.invoicestatus,
            self.rejectreason,
            self.paidon
            )

class Login(forms.Form):

    """ Basic login form

    This form renders into a very simple login field, with fields identified
    differently, both for styling and the optional javascript by-name
    handling. Whilst this form is extremely simple, coding one each and
    every time we wish to use one is just dumb and it's much easier to code it
    once and import it into our project.
    """

    uname = forms.CharField(label="Username:")

    uname.widget.attrs.update({'class': 'login-form','id': 'login-user'})

    passw = forms.CharField(label="Password:",widget=forms.PasswordInput())

    passw.widget.attrs.update({'class': 'login-form','id': 'login-password'})

class TrackingForm(ModelForm):

    class Meta:
	model = Engine
	widgets = {
	    'reasonother' : forms.Textarea()
            }
<|endoftext|>


<commit_before>#!/usr/bin/env python
from subprocess import check_output

from setuptools import setup, find_packages

PROJECT = 'madcore'

VERSION = '0.3'

def format_version(version):
    fmt = '{tag}.{commitcount}+{gitsha}'
    parts = version.split('-')
    assert len(parts) in (3, 4)
    dirty = len(parts) == 4
    tag, count, sha = parts[:3]
    if count == '0' and not dirty:
        return tag
    return fmt.format(tag=tag, commitcount=count, gitsha=sha)

def get_git_version():
    global VERSION
    git_version_command = 'git describe --tags --long'
    try:
        VERSION = check_output(git_version_command.split()).decode('utf-8').strip()
        VERSION = format_version(VERSION)
    except:
        pass

get_git_version()

try:
    long_description = open('README.md', 'rt').read()
except IOError:
    long_description = ''

with open('requirements.txt') as f:
    required = f.read().splitlines()

setup(
    name=PROJECT,
    version=VERSION,

    description='Madcore Core CLI - Deep Learning & Machine Intelligence Infrastructure Controller',
    long_description=long_description,

    author='Peter Styk',
    author_email='humans@madcore.ai',

    url='https://github.com/madcore-ai/cli',
    download_url='https://github.com/madcore-ai/cli/tarball/master',
    keywords=['aws', 'infrastructure'],
    classifiers=['Development Status :: 3 - Alpha',
                 'License :: OSI Approved :: MIT License',
                 'Programming Language :: Python :: 2.7',
                 'Intended Audience :: Developers',
                 'Environment :: Console',
                 ],

    platforms=['Any'],

    scripts=[],

    provides=[],
    install_requires=required,

    namespace_packages=[],
    packages=find_packages(),
    include_package_data=True,

    entry_points={
        'console_scripts': [
            'madcore = madcore.cli:main'
        ],
        'madcorecli.app': [
            'configure = madcore.cmds.configure:Configure',
            'stacks = madcore.cmds.stacks:Stacks',
            'create = madcore.cmds.create:Create',
            'destroy = madcore.cmds.destroy:Destroy',
            'followme = madcore.cmds.followme:Followme',
            'endpoints = madcore.cmds.endpoints:Endpoints',
            'selftest = madcore.cmds.selftest:SelfTest',
            'registration = madcore.cmds.registration:Registration',
        ],
    },
    zip_safe=False,
)
<commit_msg>Added req to setup.py<commit_after>#!/usr/bin/env python
from subprocess import check_output

from setuptools import setup, find_packages

PROJECT = 'madcore'

VERSION = '0.3.6'

def format_version(version):
    fmt = '{tag}.{commitcount}+{gitsha}'
    parts = version.split('-')
    assert len(parts) in (3, 4)
    dirty = len(parts) == 4
    tag, count, sha = parts[:3]
    if count == '0' and not dirty:
        return tag
    return fmt.format(tag=tag, commitcount=count, gitsha=sha)

def get_git_version():
    global VERSION
    git_version_command = 'git describe --tags --long'
    try:
        VERSION = check_output(git_version_command.split()).decode('utf-8').strip()
        VERSION = format_version(VERSION)
    except:
        pass

get_git_version()

try:
    long_description = open('README.md', 'rt').read()
except IOError:
    long_description = ''

setup(
    name=PROJECT,
    version=VERSION,

    description='Madcore Core CLI - Deep Learning & Machine Intelligence Infrastructure Controller',
    long_description=long_description,

    author='Peter Styk',
    author_email='humans@madcore.ai',

    url='https://github.com/madcore-ai/cli',
    download_url='https://github.com/madcore-ai/cli/tarball/master',
    keywords=['aws', 'infrastructure'],
    classifiers=['Development Status :: 3 - Alpha',
                 'License :: OSI Approved :: MIT License',
                 'Programming Language :: Python :: 2.7',
                 'Intended Audience :: Developers',
                 'Environment :: Console',
                 ],

    platforms=['Any'],

    scripts=[],

    provides=[],
    install_requires=[
        'six>=1.9.0',
        'cliff==2.4.0',
        'boto3==1.4.3',
        'urllib3==1.19.1',
        'python-jenkins==0.4.13',
        'requests==2.12.4',
        'questionnaire==1.1.0',
        'pyfiglet==0.7.5'
    ],

    namespace_packages=[],
    packages=find_packages(),
    include_package_data=True,

    entry_points={
        'console_scripts': [
            'madcore = madcore.cli:main'
        ],
        'madcorecli.app': [
            'configure = madcore.cmds.configure:Configure',
            'stacks = madcore.cmds.stacks:Stacks',
            'create = madcore.cmds.create:Create',
            'destroy = madcore.cmds.destroy:Destroy',
            'followme = madcore.cmds.followme:Followme',
            'endpoints = madcore.cmds.endpoints:Endpoints',
            'selftest = madcore.cmds.selftest:SelfTest',
            'registration = madcore.cmds.registration:Registration',
        ],
    },
    zip_safe=False,
)
<|endoftext|>