<commit_before>#!/usr/bin/python
import plotly
import csv
import math
import time, sched
import SimpleHTTPServer
import SocketServer
import os
import numpy as np
import plotly.plotly as py
import RPi.GPIO as GPIO

from plotly.graph_objs import Scatter, Layout, Figure
from plotly import tools

def isFloat(s):
    try:
        float(s)
        return True
    except ValueError:
        return False

def csvReader(fileNameArr):
    n = len(fileNameArr)
    #Store the time and temp values in a multi-dimentional array
    time_temp = [[] for index in range(0,n*2)]
    for i in range(n):
        #read in the CSV and store it in a huge array
        f = open(fileNameArr[i])
        csv_f = csv.reader(f)
        for row in csv_f:
            if(row != -1):
                if(i>=1):
                    time_temp[i+i].append(row[0])
                    if(isFloat(row[1]) == True):
                        time_temp[i+i+1].append(float(row[1]))
                else:
                    time_temp[i].append(row[0])
                    if(isFloat(row[1]) == True):
                        time_temp[i+1].append(float(row[1]))
    return time_temp

#Graph the temperature data to an HTML file (temperature.html) using the plotly library
def grapher(rate):
    looper = True
    while(looper):
            try:
                time_temp = csvReader(['sensor01.csv','sensor02.csv','sensor03.csv','sensor04.csv'])[:]
                n = len(time_temp)
                traces_arr = [[] for index in range(0,n/2)]
                for val in range(n/2):
                	if(val>=1):
                            #tempname = "Detector" #+= str(val+1)
                         #   tempName = "Trace" + str(val)
                           tempName = Scatter(x=time_temp[val+val], y=time_temp[val+val+1], name = ("Detector" + str(val)))
                           traces_arr[val].append(tempName)
                           #print(time_temp[val+val], time_temp[val+val+1])
                	else:
                            #tempName = "Trace" + str(val)
                            tempName = Scatter(x=time_temp[val], y=time_temp[val+1], name = ("Detector" + str(val)))
                            traces_arr[val].append(tempName)
                            #print ("okay")
                          # print(time_temp[val] ,time_temp[val+1])
               # for j in range(0, n/2):
                   # print(traces_arr[j])

                for i in range(0, n/2):
                    #fig = []
                    #fig.append(traces_arr[i])
                    plotly.offline.plot(traces_arr[i], filename = ("sensor" +str(i+1) + ".html"), auto_open=False, show_link=False)
                    #fig[:]
                    time.sleep(rate)
                traces_arr[:]
            except KeyboardInterrupt:
                print("Terminating graphing ...")
                looper = False
    return

#Config and start the python server to host the files from the directory
def server():
    PORT = 8000
    Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
    Handler.extensions_map.update({
        '.webapp': 'application/x-web-app-manifest+json',
    });
    httpd = SocketServer.TCPServer(("", PORT), Handler)

    print ("Serving at port:", PORT)
    httpd.serve_forever()
    return

def main():
    grapher(5)

if __name__ == "__main__":
    main()
<commit_msg>removed junk comments<commit_after>#!/usr/bin/python
import plotly
import csv
import math
import time, sched
import SimpleHTTPServer
import SocketServer
import os
import numpy as np
import plotly.plotly as py
import RPi.GPIO as GPIO

from plotly.graph_objs import Scatter, Layout, Figure
from plotly import tools

def isFloat(s):
    try:
        float(s)
        return True
    except ValueError:
        return False

def csvReader(fileNameArr):
    n = len(fileNameArr)
    #Store the time and temp values in a multi-dimentional array
    time_temp = [[] for index in range(0,n*2)]
    for i in range(n):
        #read in the CSV and store it in a huge array
        f = open(fileNameArr[i])
        csv_f = csv.reader(f)
        for row in csv_f:
            if(row != -1):
                if(i>=1):
                    time_temp[i+i].append(row[0])
                    if(isFloat(row[1]) == True):
                        time_temp[i+i+1].append(float(row[1]))
                else:
                    time_temp[i].append(row[0])
                    if(isFloat(row[1]) == True):
                        time_temp[i+1].append(float(row[1]))
    return time_temp

#Graph the temperature data to an HTML file (temperature.html) using the plotly library
def grapher(rate):
    looper = True
    while(looper):
            try:
                time_temp = csvReader(['sensor01.csv','sensor02.csv','sensor03.csv','sensor04.csv'])[:]
                n = len(time_temp)
                traces_arr = [[] for index in range(0,n/2)]
                for val in range(n/2):
                	if(val>=1):
                            tempName = Scatter(x=time_temp[val+val], y=time_temp[val+val+1], name = ("Detector" + str(val)))
                            traces_arr[val].append(tempName)
                	else:
                            tempName = Scatter(x=time_temp[val], y=time_temp[val+1], name = ("Detector" + str(val)))
                            traces_arr[val].append(tempName)

                for i in range(0, n/2):
                    print("Writing out sensor " +str(i+1) + " data ...")
                    plotly.offline.plot(traces_arr[i], filename = ("sensor" +str(i+1) + ".html"), auto_open=False, show_link=False)
                    time.sleep(rate)
                traces_arr[:]
            except KeyboardInterrupt:
                print("Terminating graphing ...")
                looper = False
    return

#Config and start the python server to host the files from the directory
def server():
    PORT = 8000
    Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
    Handler.extensions_map.update({
        '.webapp': 'application/x-web-app-manifest+json',
    });
    httpd = SocketServer.TCPServer(("", PORT), Handler)

    print ("Serving at port:", PORT)
    httpd.serve_forever()
    return

def main():
    grapher(5)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from importd import d

from django.core.urlresolvers import reverse, get_mod_func
from django.conf import settings
from django.http import HttpResponse

from dcavenue.utils import generate_order_id, enc_request, dec_response
from dcavenue import POST_URL

@d("/", name="dcavenue-index")
def index(request):
    return HttpResponse(
        "start" % (
            reverse("dcavenue-start") + "?Amount=1&Currency=USD",
        )
    )

@d("/start/", name="dcavenue-start")
def start(request):
    order_id = request.REQUEST.get("Order_Id")
    if not order_id:
        order_id = generate_order_id()
    enc_request_data = enc_request(request, order_id=order_id)
    request.session["dcavenue_order_id"] = order_id

    return HttpResponse(
        """

                <title>Redirecting...</title>

                    <form method="post" name="redirect" action="%s">
                        <input type="hidden" name="encRequest" value="%s">
                        <input type="hidden" name="Merchant_Id" value="%s">
                    </form>

        """ % (POST_URL, enc_request_data, settings.DCAVENUE["MERCHANT_ID"])
    )

@d("/callback/", name="dcavenue-callback")
def callback(request):
    order_id = request.session["dcavenue_order_id"]
    if order_id:
        del request.session["dcavenue_order_id"]
    else:
        raise d.Http404("No order id in session")

    enc_response = request.GET["encResponse"]
    if not enc_response:
        raise d.Http404("No encResponse")

    data = dec_response(request, enc_response)

    if not data:
        raise d.Http404("Checksum Failed")

    cb_module, cb_method = get_mod_func(
        settings.DCAVENUE["CALLBACK", "dcavenue.utils.default_callback"]
    )
    cb = getattr(__import__(cb_module, {}, {}, ['']), cb_method)

    return cb(request, data)

<commit_msg>minor<commit_after>from importd import d

from django.core.urlresolvers import reverse, get_mod_func
from django.conf import settings
from django.http import HttpResponse

from dcavenue.utils import generate_order_id, enc_request, dec_response
from dcavenue import POST_URL

@d("/", name="dcavenue-index")
def index(request):
    return HttpResponse(
        "start" % (
            reverse("dcavenue-start") + "?Amount=10.00",
        )
    )

@d("/start/", name="dcavenue-start")
def start(request):
    order_id = request.REQUEST.get("Order_Id")
    if not order_id:
        order_id = generate_order_id()
    enc_request_data = enc_request(request, order_id=order_id)
    request.session["dcavenue_order_id"] = order_id

    return HttpResponse(
        """

                <title>Redirecting...</title>

                    <form method="post" name="redirect" action="%s">
                        <input type="hidden" name="encRequest" value="%s">
                        <input type="hidden" name="Merchant_Id" value="%s">
                    </form>

        """ % (POST_URL, enc_request_data, settings.DCAVENUE["MERCHANT_ID"])
    )

@d("/callback/", name="dcavenue-callback")
def callback(request):
    order_id = request.session["dcavenue_order_id"]
    if order_id:
        del request.session["dcavenue_order_id"]
    else:
        raise d.Http404("No order id in session")

    enc_response = request.GET["encResponse"]
    if not enc_response:
        raise d.Http404("No encResponse")

    data = dec_response(request, enc_response)

    if not data:
        raise d.Http404("Checksum Failed")

    cb_module, cb_method = get_mod_func(
        settings.DCAVENUE["CALLBACK", "dcavenue.utils.default_callback"]
    )
    cb = getattr(__import__(cb_module, {}, {}, ['']), cb_method)

    return cb(request, data)

<|endoftext|>


<commit_before>VERSION = (8, 0, 0, 'beta', 3)

def get_version():
    "Returns a PEP 386-compliant version number from VERSION."
    assert len(VERSION) == 5
    assert VERSION[3] in ('alpha', 'beta', 'rc', 'final')

    # Now build the two parts of the version number:
    # main = X.Y[.Z]
    # sub = .devN - for pre-alpha releases
    #     | {a|b|c}N - for alpha, beta and rc releases

    parts = 2 if VERSION[2] == 0 else 3
    main = '.'.join(str(x) for x in VERSION[:parts])

    sub = ''

    if VERSION[3] != 'final':
        mapping = {'alpha': 'a', 'beta': 'b', 'rc': 'rc'}
        sub = mapping[VERSION[3]] + str(VERSION[4])

    return str(main + sub)
<commit_msg>Push up version (WorkbenchFS)<commit_after>VERSION = (8, 0, 0, 'beta', 4)

def get_version():
    "Returns a PEP 386-compliant version number from VERSION."
    assert len(VERSION) == 5
    assert VERSION[3] in ('alpha', 'beta', 'rc', 'final')

    # Now build the two parts of the version number:
    # main = X.Y[.Z]
    # sub = .devN - for pre-alpha releases
    #     | {a|b|c}N - for alpha, beta and rc releases

    parts = 2 if VERSION[2] == 0 else 3
    main = '.'.join(str(x) for x in VERSION[:parts])

    sub = ''

    if VERSION[3] != 'final':
        mapping = {'alpha': 'a', 'beta': 'b', 'rc': 'rc'}
        sub = mapping[VERSION[3]] + str(VERSION[4])

    return str(main + sub)
<|endoftext|>


<commit_before>from fo2.models import rows_to_dict_list

from lotes.models import *

def get_lotes(cursor, op='', os='', tam='', cor='', order='',
              oc_ini='', oc_fim='', pula=0, qtd_lotes=100000):
    # Lotes ordenados por OP + OS + referência + estágio
    if oc_ini == '':
        oc_ini = 0
    if oc_fim == '':
        oc_fim = 99999
    if pula is None:
        pula = 0
    if qtd_lotes is None:
        qtd_lotes = 100000
    sql = '''
        WITH Table_qtd_lotes AS (
        SELECT
          COALESCE(
          ( SELECT
              MIN( le.CODIGO_ESTAGIO ) CODIGO_ESTAGIO
            FROM PCPC_040 le
            JOIN MQOP_005 ed
              ON ed.CODIGO_ESTAGIO = le.CODIGO_ESTAGIO
            WHERE le.PERIODO_PRODUCAO = l.PERIODO_PRODUCAO
              AND le.ORDEM_CONFECCAO = l.ORDEM_CONFECCAO
              AND le.QTDE_EM_PRODUCAO_PACOTE <> 0
          )
          , 9999
          ) EST_NUM
        , l.ORDEM_PRODUCAO OP
        , op.SITUACAO OP_SITUACAO
        , CASE WHEN dos.NUMERO_ORDEM IS NULL
          THEN '0'
          ELSE l.NUMERO_ORDEM || ' (' || eos.DESCRICAO || ')'
          END OS
        , l.PROCONF_GRUPO REF
        , l.PROCONF_SUBGRUPO TAM
        , l.PROCONF_ITEM COR
        , COALESCE(
          ( SELECT
              LISTAGG(le.CODIGO_ESTAGIO || ' - ' || ed.DESCRICAO, ' & ')
              WITHIN GROUP (ORDER BY le.CODIGO_ESTAGIO)
            FROM PCPC_040 le
            JOIN MQOP_005 ed
              ON ed.CODIGO_ESTAGIO = le.CODIGO_ESTAGIO
            WHERE le.PERIODO_PRODUCAO = l.PERIODO_PRODUCAO
              AND le.ORDEM_CONFECCAO = l.ORDEM_CONFECCAO
              AND le.QTDE_EM_PRODUCAO_PACOTE <> 0
          )
          , 'FINALIZADO'
          ) EST
        , l.PERIODO_PRODUCAO PERIODO
        , l.ORDEM_CONFECCAO OC
        , l.QTDE_PECAS_PROG QTD
        , l.QTDE_PECAS_PROD PROD1Q
        , l.QTDE_CONSERTO CONSERTO
        , l.QTDE_PECAS_2A PROD2Q
        , l.QTDE_PERDAS PERDA
        , r.NARRATIVA
        , CASE WHEN l.DIVISAO = 0 THEN dp.DIVISAO_PRODUCAO
          ELSE l.DIVISAO END DIVISAO
        , CASE WHEN l.DIVISAO = 0 THEN dp.DESCRICAO
          ELSE di.DESCRICAO END DESCRICAO_DIVISAO
        FROM (
          SELECT
            os.PERIODO_PRODUCAO
          , os.ORDEM_CONFECCAO
          , os.PROCONF_GRUPO
          , os.PROCONF_SUBGRUPO
          , os.PROCONF_ITEM
          , os.ORDEM_PRODUCAO
          , max( os.NUMERO_ORDEM ) NUMERO_ORDEM
          , max( os.QTDE_PECAS_PROG ) QTDE_PECAS_PROG
          , max( os.QTDE_PECAS_PROD ) QTDE_PECAS_PROD
          , max( os.QTDE_CONSERTO ) QTDE_CONSERTO
          , max( os.QTDE_PECAS_2A ) QTDE_PECAS_2A
          , max( os.QTDE_PERDAS ) QTDE_PERDAS
          , max( CASE WHEN os.CODIGO_FAMILIA < 1000
                 THEN os.CODIGO_FAMILIA
                 ELSE 0 END ) DIVISAO
          FROM PCPC_040 os
          WHERE 1=1
            AND (os.ORDEM_PRODUCAO = %s or %s IS NULL)
            AND (os.NUMERO_ORDEM = %s or %s IS NULL)
            AND (os.ORDEM_CONFECCAO >= %s or %s IS NULL)
            AND (os.ORDEM_CONFECCAO <= %s or %s IS NULL)
            AND (os.PROCONF_SUBGRUPO = %s OR %s IS NULL)
            AND (os.PROCONF_ITEM = %s OR %s IS NULL)
          GROUP BY
            os.PERIODO_PRODUCAO
          , os.ORDEM_CONFECCAO
          , os.PROCONF_GRUPO
          , os.PROCONF_SUBGRUPO
          , os.PROCONF_ITEM
          , os.ORDEM_PRODUCAO
        ) l
        LEFT JOIN PCPC_040 dos
          ON l.NUMERO_ORDEM <> 0
         AND dos.PERIODO_PRODUCAO = l.PERIODO_PRODUCAO
         AND dos.ORDEM_CONFECCAO = l.ORDEM_CONFECCAO
         AND dos.NUMERO_ORDEM = l.NUMERO_ORDEM
        LEFT JOIN MQOP_005 eos
          ON eos.CODIGO_ESTAGIO = dos.CODIGO_ESTAGIO
        LEFT JOIN BASI_220 t
          ON t.TAMANHO_REF = l.PROCONF_SUBGRUPO
        JOIN BASI_010 r
          ON r.NIVEL_ESTRUTURA = 1
         AND r.GRUPO_ESTRUTURA = l.PROCONF_GRUPO
         AND r.SUBGRU_ESTRUTURA = l.PROCONF_SUBGRUPO
         AND r.ITEM_ESTRUTURA = l.PROCONF_ITEM
        LEFT JOIN BASI_180 di -- divisao de producao - unidade
          ON di.DIVISAO_PRODUCAO = l.DIVISAO
        LEFT JOIN OBRF_080 osc -- OS capa
          ON l.NUMERO_ORDEM <> 0
         AND osc.NUMERO_ORDEM = l.NUMERO_ORDEM
        LEFT JOIN BASI_180 dp -- divisao de producao - unidade
          ON dp.FACCIONISTA9 = osc.CGCTERC_FORNE9
         AND dp.FACCIONISTA4 = osc.CGCTERC_FORNE4
         AND dp.FACCIONISTA2 = osc.CGCTERC_FORNE2
        JOIN PCPC_020 op -- OP capa
          ON op.ordem_producao = l.ORDEM_PRODUCAO
    '''
    if order == 'o':  # OC
        sql = sql + '''
            ORDER BY
              l.ORDEM_CONFECCAO
        '''
    elif order == 't':  # OS + referência + tamanho + cor + OC
        sql = sql + '''
            ORDER BY
              l.ORDEM_PRODUCAO
            , l.NUMERO_ORDEM
            , l.PROCONF_GRUPO
            , t.ORDEM_TAMANHO
            , l.PROCONF_ITEM
            , l.PERIODO_PRODUCAO
            , l.ORDEM_CONFECCAO
        '''
    elif order == 'e':  # estágio + OS + referência + cor + tamanho + OC
        sql = sql + '''
            ORDER BY
              1
            , l.ORDEM_PRODUCAO
            , l.NUMERO_ORDEM
            , l.PROCONF_GRUPO
            , l.PROCONF_ITEM
            , t.ORDEM_TAMANHO
            , l.PERIODO_PRODUCAO
            , l.ORDEM_CONFECCAO
        '''
    else:  # elif order == '':  # OS + referência + cor + tamanho + OC
        sql = sql + '''
            ORDER BY
              l.ORDEM_PRODUCAO
            , l.NUMERO_ORDEM
            , l.PROCONF_GRUPO
            , l.PROCONF_ITEM
            , t.ORDEM_TAMANHO
            , l.PERIODO_PRODUCAO
            , l.ORDEM_CONFECCAO
        '''
    sql = sql + '''
        )
        select * from Table_qtd_lotes where rownum <= %s
    '''

    qtd_rows = pula + qtd_lotes
    cursor.execute(
        sql, [op, op, os, os, oc_ini, oc_ini, oc_fim, oc_fim,
              tam, tam, cor, cor, qtd_rows])
    data = rows_to_dict_list(cursor)
    for i in range(0, pula):
        if len(data) != 0:
            del(data[0])
    return data

def get_os(cursor, os='', op='', periodo='', oc=''):
    # Informações sobre OS
    sql = """
        SELECT DISTINCT
          os.NUMERO_ORDEM OS
        , os.CODIGO_SERVICO || '-' || s.DESC_TERCEIRO SERV
        , os.CGCTERC_FORNE9 CNPJ9
        , os.CGCTERC_FORNE4 CNPJ4
        , os.CGCTERC_FORNE2 CNPJ2
        , f.NOME_FANTASIA NOME
        , CASE os.SITUACAO_ORDEM
          WHEN 1 THEN '1-Aberta'
          WHEN 2 THEN '2-Em Processo'
          WHEN 3 THEN '3-Baixa Parcial'
          WHEN 4 THEN '4-Baixa Total'
          ELSE '-'
          END SITUACAO
        , os.COD_CANC_ORDEM || '-' || c.DESCR_CANC_ORDEM CANC
        , count(l.ORDEM_CONFECCAO) LOTES
        , sum(
            CASE WHEN l.QTDE_A_PRODUZIR_PACOTE <> 0
            THEN l.QTDE_A_PRODUZIR_PACOTE
            ELSE --l.QTDE_PECAS_PROG
              QTDE_PECAS_PROD
            + QTDE_CONSERTO
            + QTDE_PECAS_2A
            + QTDE_PERDAS
            END
          ) QTD
        , os.DATA_EMISSAO
        , os.DATA_ENTREGA
        , os.OBSERVACAO
        FROM OBRF_080 os
        JOIN OBRF_070 s
          ON s.CODIGO_TERCEIRO = os.CODIGO_SERVICO
        JOIN SUPR_010 f
          ON f.FORNECEDOR9 = os.CGCTERC_FORNE9
         AND f.FORNECEDOR4 = os.CGCTERC_FORNE4
        JOIN OBRF_087 c
          ON c.COD_CANC_ORDEM = os.COD_CANC_ORDEM
        JOIN pcpc_040 l
          ON l.NUMERO_ORDEM = os.NUMERO_ORDEM
        WHERE 1=1
          AND (os.NUMERO_ORDEM = %s or %s IS NULL)
          AND (l.ORDEM_PRODUCAO = %s or %s IS NULL)
          AND (l.PERIODO_PRODUCAO = %s or %s IS NULL)
          AND (l.ORDEM_CONFECCAO = %s or %s IS NULL)
          AND l.NUMERO_ORDEM <> 0
        GROUP BY
          os.NUMERO_ORDEM
        , os.CODIGO_SERVICO
        , s.DESC_TERCEIRO
        , os.CGCTERC_FORNE9
        , os.CGCTERC_FORNE4
        , os.CGCTERC_FORNE2
        , f.NOME_FANTASIA
        , os.SITUACAO_ORDEM
        , os.COD_CANC_ORDEM
        , c.DESCR_CANC_ORDEM
        , os.DATA_EMISSAO
        , os.DATA_ENTREGA
        , os.OBSERVACAO
        ORDER BY
          os.NUMERO_ORDEM
    """
    cursor.execute(sql, [os, os, op, op, periodo, periodo, oc, oc])
    return rows_to_dict_list(cursor)
<commit_msg>get data corte<commit_after>from fo2.models import rows_to_dict_list

from lotes.models import *

def get_lotes(cursor, op='', os='', tam='', cor='', order='',
              oc_ini='', oc_fim='', pula=0, qtd_lotes=100000):
    # Lotes ordenados por OP + OS + referência + estágio
    if oc_ini == '':
        oc_ini = 0
    if oc_fim == '':
        oc_fim = 99999
    if pula is None:
        pula = 0
    if qtd_lotes is None:
        qtd_lotes = 100000
    sql = '''
        WITH Table_qtd_lotes AS (
        SELECT
          COALESCE(
          ( SELECT
              MIN( le.CODIGO_ESTAGIO ) CODIGO_ESTAGIO
            FROM PCPC_040 le
            JOIN MQOP_005 ed
              ON ed.CODIGO_ESTAGIO = le.CODIGO_ESTAGIO
            WHERE le.PERIODO_PRODUCAO = l.PERIODO_PRODUCAO
              AND le.ORDEM_CONFECCAO = l.ORDEM_CONFECCAO
              AND le.QTDE_EM_PRODUCAO_PACOTE <> 0
          )
          , 9999
          ) EST_NUM
        , l.ORDEM_PRODUCAO OP
        , op.SITUACAO OP_SITUACAO
        , CASE WHEN dos.NUMERO_ORDEM IS NULL
          THEN '0'
          ELSE l.NUMERO_ORDEM || ' (' || eos.DESCRICAO || ')'
          END OS
        , l.PROCONF_GRUPO REF
        , l.PROCONF_SUBGRUPO TAM
        , l.PROCONF_ITEM COR
        , COALESCE(
          ( SELECT
              LISTAGG(le.CODIGO_ESTAGIO || ' - ' || ed.DESCRICAO, ' & ')
              WITHIN GROUP (ORDER BY le.CODIGO_ESTAGIO)
            FROM PCPC_040 le
            JOIN MQOP_005 ed
              ON ed.CODIGO_ESTAGIO = le.CODIGO_ESTAGIO
            WHERE le.PERIODO_PRODUCAO = l.PERIODO_PRODUCAO
              AND le.ORDEM_CONFECCAO = l.ORDEM_CONFECCAO
              AND le.QTDE_EM_PRODUCAO_PACOTE <> 0
          )
          , 'FINALIZADO'
          ) EST
        , l.PERIODO_PRODUCAO PERIODO
        , l.ORDEM_CONFECCAO OC
        , l.QTDE_PECAS_PROG QTD
        , l.QTDE_PECAS_PROD PROD1Q
        , l.QTDE_CONSERTO CONSERTO
        , l.QTDE_PECAS_2A PROD2Q
        , l.QTDE_PERDAS PERDA
        , r.NARRATIVA
        , CASE WHEN l.DIVISAO = 0 THEN dp.DIVISAO_PRODUCAO
          ELSE l.DIVISAO END DIVISAO
        , CASE WHEN l.DIVISAO = 0 THEN dp.DESCRICAO
          ELSE di.DESCRICAO END DESCRICAO_DIVISAO
        , op.DATA_ENTRADA_CORTE
        FROM (
          SELECT
            os.PERIODO_PRODUCAO
          , os.ORDEM_CONFECCAO
          , os.PROCONF_GRUPO
          , os.PROCONF_SUBGRUPO
          , os.PROCONF_ITEM
          , os.ORDEM_PRODUCAO
          , max( os.NUMERO_ORDEM ) NUMERO_ORDEM
          , max( os.QTDE_PECAS_PROG ) QTDE_PECAS_PROG
          , max( os.QTDE_PECAS_PROD ) QTDE_PECAS_PROD
          , max( os.QTDE_CONSERTO ) QTDE_CONSERTO
          , max( os.QTDE_PECAS_2A ) QTDE_PECAS_2A
          , max( os.QTDE_PERDAS ) QTDE_PERDAS
          , max( CASE WHEN os.CODIGO_FAMILIA < 1000
                 THEN os.CODIGO_FAMILIA
                 ELSE 0 END ) DIVISAO
          FROM PCPC_040 os
          WHERE 1=1
            AND (os.ORDEM_PRODUCAO = %s or %s IS NULL)
            AND (os.NUMERO_ORDEM = %s or %s IS NULL)
            AND (os.ORDEM_CONFECCAO >= %s or %s IS NULL)
            AND (os.ORDEM_CONFECCAO <= %s or %s IS NULL)
            AND (os.PROCONF_SUBGRUPO = %s OR %s IS NULL)
            AND (os.PROCONF_ITEM = %s OR %s IS NULL)
          GROUP BY
            os.PERIODO_PRODUCAO
          , os.ORDEM_CONFECCAO
          , os.PROCONF_GRUPO
          , os.PROCONF_SUBGRUPO
          , os.PROCONF_ITEM
          , os.ORDEM_PRODUCAO
        ) l
        LEFT JOIN PCPC_040 dos
          ON l.NUMERO_ORDEM <> 0
         AND dos.PERIODO_PRODUCAO = l.PERIODO_PRODUCAO
         AND dos.ORDEM_CONFECCAO = l.ORDEM_CONFECCAO
         AND dos.NUMERO_ORDEM = l.NUMERO_ORDEM
        LEFT JOIN MQOP_005 eos
          ON eos.CODIGO_ESTAGIO = dos.CODIGO_ESTAGIO
        LEFT JOIN BASI_220 t
          ON t.TAMANHO_REF = l.PROCONF_SUBGRUPO
        JOIN BASI_010 r
          ON r.NIVEL_ESTRUTURA = 1
         AND r.GRUPO_ESTRUTURA = l.PROCONF_GRUPO
         AND r.SUBGRU_ESTRUTURA = l.PROCONF_SUBGRUPO
         AND r.ITEM_ESTRUTURA = l.PROCONF_ITEM
        LEFT JOIN BASI_180 di -- divisao de producao - unidade
          ON di.DIVISAO_PRODUCAO = l.DIVISAO
        LEFT JOIN OBRF_080 osc -- OS capa
          ON l.NUMERO_ORDEM <> 0
         AND osc.NUMERO_ORDEM = l.NUMERO_ORDEM
        LEFT JOIN BASI_180 dp -- divisao de producao - unidade
          ON dp.FACCIONISTA9 = osc.CGCTERC_FORNE9
         AND dp.FACCIONISTA4 = osc.CGCTERC_FORNE4
         AND dp.FACCIONISTA2 = osc.CGCTERC_FORNE2
        JOIN PCPC_020 op -- OP capa
          ON op.ordem_producao = l.ORDEM_PRODUCAO
    '''
    if order == 'o':  # OC
        sql = sql + '''
            ORDER BY
              l.ORDEM_CONFECCAO
        '''
    elif order == 't':  # OS + referência + tamanho + cor + OC
        sql = sql + '''
            ORDER BY
              l.ORDEM_PRODUCAO
            , l.NUMERO_ORDEM
            , l.PROCONF_GRUPO
            , t.ORDEM_TAMANHO
            , l.PROCONF_ITEM
            , l.PERIODO_PRODUCAO
            , l.ORDEM_CONFECCAO
        '''
    elif order == 'e':  # estágio + OS + referência + cor + tamanho + OC
        sql = sql + '''
            ORDER BY
              1
            , l.ORDEM_PRODUCAO
            , l.NUMERO_ORDEM
            , l.PROCONF_GRUPO
            , l.PROCONF_ITEM
            , t.ORDEM_TAMANHO
            , l.PERIODO_PRODUCAO
            , l.ORDEM_CONFECCAO
        '''
    else:  # elif order == '':  # OS + referência + cor + tamanho + OC
        sql = sql + '''
            ORDER BY
              l.ORDEM_PRODUCAO
            , l.NUMERO_ORDEM
            , l.PROCONF_GRUPO
            , l.PROCONF_ITEM
            , t.ORDEM_TAMANHO
            , l.PERIODO_PRODUCAO
            , l.ORDEM_CONFECCAO
        '''
    sql = sql + '''
        )
        select * from Table_qtd_lotes where rownum <= %s
    '''

    qtd_rows = pula + qtd_lotes
    cursor.execute(
        sql, [op, op, os, os, oc_ini, oc_ini, oc_fim, oc_fim,
              tam, tam, cor, cor, qtd_rows])
    data = rows_to_dict_list(cursor)
    for i in range(0, pula):
        if len(data) != 0:
            del(data[0])
    return data

def get_os(cursor, os='', op='', periodo='', oc=''):
    # Informações sobre OS
    sql = """
        SELECT DISTINCT
          os.NUMERO_ORDEM OS
        , os.CODIGO_SERVICO || '-' || s.DESC_TERCEIRO SERV
        , os.CGCTERC_FORNE9 CNPJ9
        , os.CGCTERC_FORNE4 CNPJ4
        , os.CGCTERC_FORNE2 CNPJ2
        , f.NOME_FANTASIA NOME
        , CASE os.SITUACAO_ORDEM
          WHEN 1 THEN '1-Aberta'
          WHEN 2 THEN '2-Em Processo'
          WHEN 3 THEN '3-Baixa Parcial'
          WHEN 4 THEN '4-Baixa Total'
          ELSE '-'
          END SITUACAO
        , os.COD_CANC_ORDEM || '-' || c.DESCR_CANC_ORDEM CANC
        , count(l.ORDEM_CONFECCAO) LOTES
        , sum(
            CASE WHEN l.QTDE_A_PRODUZIR_PACOTE <> 0
            THEN l.QTDE_A_PRODUZIR_PACOTE
            ELSE --l.QTDE_PECAS_PROG
              QTDE_PECAS_PROD
            + QTDE_CONSERTO
            + QTDE_PECAS_2A
            + QTDE_PERDAS
            END
          ) QTD
        , os.DATA_EMISSAO
        , os.DATA_ENTREGA
        , os.OBSERVACAO
        FROM OBRF_080 os
        JOIN OBRF_070 s
          ON s.CODIGO_TERCEIRO = os.CODIGO_SERVICO
        JOIN SUPR_010 f
          ON f.FORNECEDOR9 = os.CGCTERC_FORNE9
         AND f.FORNECEDOR4 = os.CGCTERC_FORNE4
        JOIN OBRF_087 c
          ON c.COD_CANC_ORDEM = os.COD_CANC_ORDEM
        JOIN pcpc_040 l
          ON l.NUMERO_ORDEM = os.NUMERO_ORDEM
        WHERE 1=1
          AND (os.NUMERO_ORDEM = %s or %s IS NULL)
          AND (l.ORDEM_PRODUCAO = %s or %s IS NULL)
          AND (l.PERIODO_PRODUCAO = %s or %s IS NULL)
          AND (l.ORDEM_CONFECCAO = %s or %s IS NULL)
          AND l.NUMERO_ORDEM <> 0
        GROUP BY
          os.NUMERO_ORDEM
        , os.CODIGO_SERVICO
        , s.DESC_TERCEIRO
        , os.CGCTERC_FORNE9
        , os.CGCTERC_FORNE4
        , os.CGCTERC_FORNE2
        , f.NOME_FANTASIA
        , os.SITUACAO_ORDEM
        , os.COD_CANC_ORDEM
        , c.DESCR_CANC_ORDEM
        , os.DATA_EMISSAO
        , os.DATA_ENTREGA
        , os.OBSERVACAO
        ORDER BY
          os.NUMERO_ORDEM
    """
    cursor.execute(sql, [os, os, op, op, periodo, periodo, oc, oc])
    return rows_to_dict_list(cursor)
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, print_function

import blessings
import prettytable

from .. import cli
from ..master import CURRENT as MASTER

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

parser = cli.parser(
    description="frameworks list"
)

parser.add_argument(
    "-b", "--task-breakdown", action="store_true",
    help="show task breakdown per framework"
)

parser.add_argument(
    "-i", "--inactive", action="store_true",
    help="show inactive frameworks as well"
)

def task_stats(tasks, state):
    count = 0
    for task in tasks:
        if task['state'] == state:
            count += 1
    return count

@cli.init(parser)
def main(args):
    term = blessings.Terminal()

    table_generator = OrderedDict([
        ("ID", lambda x: x["id"]),
        ("name", lambda x: x["name"]),
        ("host", lambda x: x["hostname"]),
        ("active", lambda x: x["active"]),
    ])

    # TODO: the task breakdown is not done in an efficient way (since we traverse the
    # task list on every framework). This can be optimized to do the traversal only
    # once per framework.
    if args.task_breakdown:
        # active tasks
        table_generator['staging'] = lambda x: task_stats(x["tasks"], "TASK_STAGING")
        table_generator['starting'] = lambda x: task_stats(x["tasks"], "TASK_STARTING")
        table_generator['running'] = lambda x: task_stats(x["tasks"], "TASK_RUNNING")
        # terminal state tasks. The count here is approximate since the API caps the result
        # at a max of 1000 tasks
        table_generator['failed'] = lambda x: task_stats(x["completed_tasks"], "TASK_FAILED")
        table_generator['killed'] = lambda x: task_stats(x["completed_tasks"], "TASK_FAILED")
        table_generator['error'] = lambda x: task_stats(x["completed_tasks"], "TASK_ERROR")
        table_generator['lost'] = lambda x: task_stats(x["completed_tasks"], "TASK_ERROR")
        table_generator['finished'] = lambda x: task_stats(x["completed_tasks"], "TASK_FINISHED")
    else:
        table_generator['active tasks'] = lambda x: len(x["tasks"])

    tb = prettytable.PrettyTable(
        [x.upper() for x in table_generator.keys()],
        border=False,
        max_table_width=term.width,
        hrules=prettytable.NONE,
        vrules=prettytable.NONE,
        left_padding_width=0,
        right_padding_width=1
    )

    def format_row(framework):
        return [fn(framework) for fn in table_generator.values()]

    for framework in MASTER.state['frameworks']:
        if args.inactive or framework['active']:
            tb.add_row(format_row(framework))

    if tb.rowcount == 0:
        # TODO: can this be possible ?
        print("===>{0}You have no frameworks{1}<===".format(
            term.red, term.white))
    else:
        print(tb)
<commit_msg>added caching to the task computation<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, print_function

import blessings
import prettytable

from .. import cli
from ..master import CURRENT as MASTER

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

parser = cli.parser(
    description="frameworks list"
)

parser.add_argument(
    "-t", "--task-breakdown", action="store_true",
    help="show task status breakdown"
)

parser.add_argument(
    "-i", "--inactive", action="store_true",
    help="show inactive frameworks as well"
)

class TaskStats:
    '''
    To avoid going through the task state for every task state, this class acts as
    a cache that generates the stats for the task in one call and returns it if it
    has been computed already
    '''
    def __init__(self):
        self.stats_cache = dict()

    def get(self, framework_id, tasks, state):
        # check if we have things in the cache already
        if framework_id in self.stats_cache.keys():
            return self._lookup(self.stats_cache[framework_id], state)

        state_count = dict()
        for task in tasks:
            task_state = task['state']
            if task_state in state_count.keys():
                state_count[task_state] += 1
            else:
                state_count[task_state] = 1
        # put the stats in the cache
        self.stats_cache[framework_id] = state_count
        return self._lookup(state_count, state)

    def _lookup(self, state_count, state):
        if state in state_count.keys():
            return state_count[state]
        else:
            return 0

def resource_stats(resource, allocated, used):
    r_used = used[resource]
    r_allocated = allocated[resource]
    if r_allocated > 0:
        percent = int(r_used / r_allocated * 100)
        return str(r_used) + "/" + str(r_allocated) + "(" + str(percent) + "%)"
    else:
        # no resources are being used/allocated
        return "-"

@cli.init(parser)
def main(args):
    term = blessings.Terminal()
    active_task_stats = TaskStats()
    terminal_task_stats = TaskStats()

    table_generator = OrderedDict([
        ("ID", lambda x: x["id"]),
        ("name", lambda x: x["name"]),
        ("host", lambda x: x["hostname"]),
        ("active", lambda x: x["active"]),
        ("cpu", lambda x: resource_stats("cpus", x["resources"], x["used_resources"])),
        ("mem", lambda x: resource_stats("mem", x["resources"], x["used_resources"])),
        ("disk", lambda x: resource_stats("disk", x["resources"], x["used_resources"])),
    ])

    # TODO: the task breakdown is not done in an efficient way (since we traverse the
    # task list on every framework). This can be optimized to do the traversal only
    # once per framework.
    if args.task_breakdown:
        # active tasks
        table_generator['staging'] = lambda x: active_task_stats.get(x["id"], x["tasks"], "TASK_STAGING")
        table_generator['starting'] = lambda x: active_task_stats.get(x["id"], x["tasks"], "TASK_STARTING")
        table_generator['running'] = lambda x: active_task_stats.get(x["id"], x["tasks"], "TASK_RUNNING")
        # terminal state tasks. The count here is approximate since the API caps the result
        # at a max of 1000 tasks
        table_generator['failed'] = lambda x: terminal_task_stats.get(x["id"], x["completed_tasks"], "TASK_FAILED")
        table_generator['killed'] = lambda x: terminal_task_stats.get(x["id"], x["completed_tasks"], "TASK_FAILED")
        table_generator['error'] = lambda x: terminal_task_stats.get(x["id"], x["completed_tasks"], "TASK_ERROR")
        table_generator['lost'] = lambda x: terminal_task_stats.get(x["id"], x["completed_tasks"], "TASK_ERROR")
        table_generator['finished'] = lambda x: terminal_task_stats.get(x["id"], x["completed_tasks"], "TASK_FINISHED")
    else:
        table_generator['active tasks'] = lambda x: len(x["tasks"])

    tb = prettytable.PrettyTable(
        [x.upper() for x in table_generator.keys()],
        border=False,
        max_table_width=term.width,
        hrules=prettytable.NONE,
        vrules=prettytable.NONE,
        left_padding_width=0,
        right_padding_width=1
    )

    def format_row(framework):
        return [fn(framework) for fn in table_generator.values()]

    for framework in MASTER.state['frameworks']:
        if args.inactive or framework['active']:
            tb.add_row(format_row(framework))

    if tb.rowcount == 0:
        print("===>{0}You have no frameworks{1}<===".format(
            term.red, term.white))
    else:
        print(tb)
<|endoftext|>


<commit_before>import os
from django.http import HttpResponse, HttpResponseRedirect
from django.views.decorators.csrf import csrf_exempt
from django.contrib.auth.decorators import login_required
from django.template import RequestContext
from django.shortcuts import render_to_response
from settings import ELASTICSEARCH_PROXY, ELASTICSEARCH_REAL,KIBANA_DIR

def login_or_404(func):
    def inner(*args, **karags):
        request = args[0]
        if request.user.is_authenticated():
            return func(*args, **karags)
        else:
            return HttpResponseRedirect("/es/")

    return inner

#@login_or_404
@csrf_exempt
def elasticsearch(request):
    fullpath = request.get_full_path()
    fullpath = fullpath[len(ELASTICSEARCH_PROXY):]
    response = HttpResponse()
    response['X-Accel-Redirect'] = ELASTICSEARCH_REAL + '/' + fullpath
    return response

def home(request):
    return HttpResponseRedirect("index.html")

@login_required
def index(request):
    html = open(os.path.join(KIBANA_DIR,"index.html")).read()
    return HttpResponse(html)

<commit_msg>add django-user header<commit_after>import os
from django.http import HttpResponse, HttpResponseRedirect
from django.views.decorators.csrf import csrf_exempt
from django.contrib.auth.decorators import login_required
from django.template import RequestContext
from django.shortcuts import render_to_response
from settings import ELASTICSEARCH_PROXY, ELASTICSEARCH_REAL, KIBANA_DIR

def login_or_404(func):
    def inner(*args, **karags):
        request = args[0]
        if request.user.is_authenticated():
            return func(*args, **karags)
        else:
            return HttpResponseRedirect("/es/")

    return inner

@login_or_404
@csrf_exempt
def elasticsearch(request):
    fullpath = request.get_full_path()
    fullpath = fullpath[len(ELASTICSEARCH_PROXY):]
    response = HttpResponse()
    response['X-Accel-Redirect'] = ELASTICSEARCH_REAL + '/' + fullpath
    response['Django-User'] = request.user.username
    return response

def home(request):
    return HttpResponseRedirect("index.html")

@login_required
def index(request):
    html = open(os.path.join(KIBANA_DIR, "index.html")).read()
    return HttpResponse(html)
<|endoftext|>


<commit_before>import logging
import flask
import os
import os.path

from datetime import timedelta
from flask.ext.sqlalchemy import SQLAlchemy
from flask_mail import Mail
from kombu import Queue
from raven.contrib.flask import Sentry
from urlparse import urlparse
from werkzeug.contrib.fixers import ProxyFix

from changes.constants import PROJECT_ROOT
from changes.ext.celery import Celery
from changes.ext.pubsub import PubSub
from changes.ext.redis import Redis
from changes.utils.trace import TracerMiddleware

db = SQLAlchemy(session_options={
    'autoflush': True,
    'autocommit': True,
})
mail = Mail()
pubsub = PubSub()
queue = Celery()
redis = Redis()
sentry = Sentry(logging=True, level=logging.WARN)

def create_app(_read_config=True, **config):
    app = flask.Flask(__name__,
                      static_folder=None,
                      template_folder=os.path.join(PROJECT_ROOT, 'templates'))

    app.wsgi_app = ProxyFix(app.wsgi_app)
    app.wsgi_app = TracerMiddleware(app.wsgi_app, app)

    # This key is insecure and you should override it on the server
    app.config['SECRET_KEY'] = 't\xad\xe7\xff%\xd2.\xfe\x03\x02=\xec\xaf\\2+\xb8=\xf7\x8a\x9aLD\xb1'
    app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql:///changes'
    app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True
    app.config['REDIS_URL'] = 'redis://localhost/0'
    app.config['DEBUG'] = True
    app.config['HTTP_PORT'] = 5000
    app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0

    app.config['API_TRACEBACKS'] = True

    app.config['CELERY_ACKS_LATE'] = True
    app.config['CELERY_BROKER_URL'] = 'redis://localhost/0'
    app.config['CELERY_ACCEPT_CONTENT'] = ['json', 'pickle']
    app.config['CELERY_RESULT_SERIALIZER'] = 'json'
    app.config['CELERY_TASK_SERIALIZER'] = 'json'
    app.config['CELERY_DEFAULT_QUEUE'] = 'default'
    app.config['CELERYD_PREFETCH_MULTIPLIER'] = 1
    app.config['CELERY_QUEUES'] = (
        Queue('build.sync', routing_key='build.sync'),
        Queue('build.create', routing_key='build.create'),
        Queue('default', routing_key='default'),
        Queue('repo.sync', routing_key='repo.sync'),
    )
    app.config['CELERY_ROUTES'] = {
        'create_build': {
            'queue': 'build.create',
            'routing_key': 'build.create',
        },
        'sync_build': {
            'queue': 'build.sync',
            'routing_key': 'build.sync',
        },
        'sync_repo': {
            'queue': 'repo.sync',
            'routing_key': 'repo.sync',
        },
    }

    app.config['EVENT_LISTENERS'] = (
        ('changes.listeners.mail.build_finished_handler', 'build.finished'),
    )

    # celerybeat must be running for our cleanup tasks to execute
    # e.g. celery worker -B
    app.config['CELERYBEAT_SCHEDULE'] = {
        'cleanup-builds': {
            'task': 'cleanup_builds',
            'schedule': timedelta(minutes=1),
        },
        # 'check-repos': {
        #     'task': 'check_repos',
        #     'schedule': timedelta(minutes=5),
        # },
    }
    app.config['CELERY_TIMEZONE'] = 'UTC'

    app.config['SENTRY_DSN'] = None

    app.config['JENKINS_URL'] = None
    app.config['JENKINS_TOKEN'] = None
    app.config['JENKINS_SYNC_ARTIFACTS'] = True

    app.config['KOALITY_URL'] = None
    app.config['KOALITY_API_KEY'] = None

    app.config['GOOGLE_CLIENT_ID'] = None
    app.config['GOOGLE_CLIENT_SECRET'] = None
    app.config['GOOGLE_DOMAIN'] = None

    app.config['REPO_ROOT'] = None

    app.config['MAIL_DEFAULT_SENDER'] = 'changes@localhost'
    app.config['BASE_URI'] = None

    app.config.update(config)

    if _read_config:
        if os.environ.get('CHANGES_CONF'):
            # CHANGES_CONF=/etc/changes.conf.py
            app.config.from_envvar('CHANGES_CONF')
        else:
            # Look for ~/.changes/changes.conf.py
            path = os.path.normpath(os.path.expanduser('~/.changes/changes.conf.py'))
            app.config.from_pyfile(path, silent=True)

    if not app.config['BASE_URI']:
        raise ValueError('You must set ``BASE_URI`` in your configuration.')

    parsed_url = urlparse(app.config['BASE_URI'])
    app.config.setdefault('SERVER_NAME', parsed_url.netloc)
    app.config.setdefault('PREFERRED_URL_SCHEME', parsed_url.scheme)

    # init sentry first
    sentry.init_app(app)

    db.init_app(app)
    mail.init_app(app)
    pubsub.init_app(app)
    queue.init_app(app)
    redis.init_app(app)

    from raven.contrib.celery import register_signal, register_logger_signal
    register_signal(sentry.client)
    register_logger_signal(sentry.client)

    # configure debug routes first
    if app.debug:
        configure_debug_routes(app)

    configure_templates(app)

    # TODO: these can be moved to wsgi app entrypoints
    configure_api_routes(app)
    configure_web_routes(app)

    configure_event_listeners(app)
    configure_jobs(app)

    return app

def configure_templates(app):
    from changes.utils.times import duration

    app.jinja_env.filters['duration'] = duration

def configure_api_routes(app):
    from changes.api.auth_index import AuthIndexAPIView
    from changes.api.author_build_index import AuthorBuildIndexAPIView
    from changes.api.build_details import BuildDetailsAPIView
    from changes.api.build_index import BuildIndexAPIView
    from changes.api.build_log_details import BuildLogDetailsAPIView
    from changes.api.build_retry import BuildRetryAPIView
    from changes.api.change_details import ChangeDetailsAPIView
    from changes.api.change_index import ChangeIndexAPIView
    from changes.api.patch_details import PatchDetailsAPIView
    from changes.api.project_build_index import ProjectBuildIndexAPIView
    from changes.api.project_commit_details import ProjectCommitDetailsAPIView
    from changes.api.project_commit_index import ProjectCommitIndexAPIView
    from changes.api.project_index import ProjectIndexAPIView
    from changes.api.project_stats_index import ProjectStatsIndexAPIView
    from changes.api.project_test_details import ProjectTestDetailsAPIView
    from changes.api.project_test_index import ProjectTestIndexAPIView
    from changes.api.project_details import ProjectDetailsAPIView
    from changes.api.testgroup_details import TestGroupDetailsAPIView

    app.add_url_rule(
        '/api/0/auth/', view_func=AuthIndexAPIView.as_view('api-auth'))
    app.add_url_rule(
        '/api/0/authors/<author_id>/builds/', view_func=AuthorBuildIndexAPIView.as_view('api-author-build-list'))
    app.add_url_rule(
        '/api/0/builds/', view_func=BuildIndexAPIView.as_view('api-build-list'))
    app.add_url_rule(
        '/api/0/builds/<build_id>/', view_func=BuildDetailsAPIView.as_view('api-build-details'))
    app.add_url_rule(
        '/api/0/builds/<build_id>/retry/', view_func=BuildRetryAPIView.as_view('api-build-retry'))
    app.add_url_rule(
        '/api/0/builds/<build_id>/logs/<source_id>/', view_func=BuildLogDetailsAPIView.as_view('api-build-log-details'))
    app.add_url_rule(
        '/api/0/changes/', view_func=ChangeIndexAPIView.as_view('api-change-list'))
    app.add_url_rule(
        '/api/0/changes/<change_id>/', view_func=ChangeDetailsAPIView.as_view('api-change-details'))
    app.add_url_rule(
        '/api/0/changes/<change_id>/builds/', view_func=BuildIndexAPIView.as_view('api-change-build-list'))
    app.add_url_rule(
        '/api/0/patches/<patch_id>/', view_func=PatchDetailsAPIView.as_view('api-patch-details'))
    app.add_url_rule(
        '/api/0/projects/', view_func=ProjectIndexAPIView.as_view('api-project-list'))
    app.add_url_rule(
        '/api/0/projects/<project_id>/', view_func=ProjectDetailsAPIView.as_view('api-project-details'))
    app.add_url_rule(
        '/api/0/projects/<project_id>/builds/', view_func=ProjectBuildIndexAPIView.as_view('api-project-build-list'))
    app.add_url_rule(
        '/api/0/projects/<project_id>/commits/', view_func=ProjectCommitIndexAPIView.as_view('api-project-commit-list'))
    app.add_url_rule(
        '/api/0/projects/<project_id>/commits/<commit_id>/', view_func=ProjectCommitDetailsAPIView.as_view('api-project-commit-details'))
    app.add_url_rule(
        '/api/0/projects/<project_id>/stats/', view_func=ProjectStatsIndexAPIView.as_view('api-project-stats'))
    app.add_url_rule(
        '/api/0/projects/<project_id>/tests/', view_func=ProjectTestIndexAPIView.as_view('api-project-test-list'))
    app.add_url_rule(
        '/api/0/projects/<project_id>/tests/<test_id>/', view_func=ProjectTestDetailsAPIView.as_view('api-project-test-details'))
    app.add_url_rule(
        '/api/0/testgroups/<testgroup_id>/', view_func=TestGroupDetailsAPIView.as_view('api-change-testgroup-details'))

def configure_web_routes(app):
    from changes.web.auth import AuthorizedView, LoginView, LogoutView
    from changes.web.index import IndexView
    from changes.web.static import StaticView

    app.add_url_rule(
        '/static/<path:filename>',
        view_func=StaticView.as_view('static', root=os.path.join(PROJECT_ROOT, 'static')))
    app.add_url_rule(
        '/partials/<path:filename>',
        view_func=StaticView.as_view('partials', root=os.path.join(PROJECT_ROOT, 'partials')))

    app.add_url_rule(
        '/auth/login/', view_func=LoginView.as_view('login', authorized_url='authorized'))
    app.add_url_rule(
        '/auth/logout/', view_func=LogoutView.as_view('logout', complete_url='index'))
    app.add_url_rule(
        '/auth/complete/', view_func=AuthorizedView.as_view('authorized', authorized_url='authorized', complete_url='index'))

    app.add_url_rule(
        '/<path:path>', view_func=IndexView.as_view('index-path'))
    app.add_url_rule(
        '/', view_func=IndexView.as_view('index'))

def configure_debug_routes(app):
    from changes.debug.reports.build import BuildReportMailView

    app.add_url_rule(
        '/debug/mail/report/build/', view_func=BuildReportMailView.as_view('debug-build-report'))

def configure_jobs(app):
    from changes.jobs.check_repos import check_repos
    from changes.jobs.cleanup_builds import cleanup_builds
    from changes.jobs.create_build import create_build
    from changes.jobs.notify_listeners import notify_listeners
    from changes.jobs.sync_artifact import sync_artifact
    from changes.jobs.sync_build import sync_build
    from changes.jobs.sync_repo import sync_repo
    from changes.jobs.update_project_stats import update_project_stats

    queue.register('check_repos', check_repos)
    queue.register('cleanup_builds', cleanup_builds)
    queue.register('create_build', create_build)
    queue.register('notify_listeners', notify_listeners)
    queue.register('sync_artifact', sync_artifact)
    queue.register('sync_build', sync_build)
    queue.register('sync_repo', sync_repo)
    queue.register('update_project_stats', update_project_stats)

def configure_event_listeners(app):
    from changes.signals import register_listener
    from changes.utils.imports import import_string

    for func_path, signal_name in app.config['EVENT_LISTENERS']:
        func = import_string(func_path)
        register_listener(func, signal_name)
<commit_msg>Disable autocommit as it causes errors<commit_after>import logging
import flask
import os
import os.path

from datetime import timedelta
from flask.ext.sqlalchemy import SQLAlchemy
from flask_mail import Mail
from kombu import Queue
from raven.contrib.flask import Sentry
from urlparse import urlparse
from werkzeug.contrib.fixers import ProxyFix

from changes.constants import PROJECT_ROOT
from changes.ext.celery import Celery
from changes.ext.pubsub import PubSub
from changes.ext.redis import Redis
from changes.utils.trace import TracerMiddleware

db = SQLAlchemy(session_options={
    'autoflush': True,
})
mail = Mail()
pubsub = PubSub()
queue = Celery()
redis = Redis()
sentry = Sentry(logging=True, level=logging.WARN)

def create_app(_read_config=True, **config):
    app = flask.Flask(__name__,
                      static_folder=None,
                      template_folder=os.path.join(PROJECT_ROOT, 'templates'))

    app.wsgi_app = ProxyFix(app.wsgi_app)
    app.wsgi_app = TracerMiddleware(app.wsgi_app, app)

    # This key is insecure and you should override it on the server
    app.config['SECRET_KEY'] = 't\xad\xe7\xff%\xd2.\xfe\x03\x02=\xec\xaf\\2+\xb8=\xf7\x8a\x9aLD\xb1'
    app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql:///changes'
    app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True
    app.config['REDIS_URL'] = 'redis://localhost/0'
    app.config['DEBUG'] = True
    app.config['HTTP_PORT'] = 5000
    app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0

    app.config['API_TRACEBACKS'] = True

    app.config['CELERY_ACKS_LATE'] = True
    app.config['CELERY_BROKER_URL'] = 'redis://localhost/0'
    app.config['CELERY_ACCEPT_CONTENT'] = ['json', 'pickle']
    app.config['CELERY_RESULT_SERIALIZER'] = 'json'
    app.config['CELERY_TASK_SERIALIZER'] = 'json'
    app.config['CELERY_DEFAULT_QUEUE'] = 'default'
    app.config['CELERYD_PREFETCH_MULTIPLIER'] = 1
    app.config['CELERY_QUEUES'] = (
        Queue('build.sync', routing_key='build.sync'),
        Queue('build.create', routing_key='build.create'),
        Queue('default', routing_key='default'),
        Queue('repo.sync', routing_key='repo.sync'),
    )
    app.config['CELERY_ROUTES'] = {
        'create_build': {
            'queue': 'build.create',
            'routing_key': 'build.create',
        },
        'sync_build': {
            'queue': 'build.sync',
            'routing_key': 'build.sync',
        },
        'sync_repo': {
            'queue': 'repo.sync',
            'routing_key': 'repo.sync',
        },
    }

    app.config['EVENT_LISTENERS'] = (
        ('changes.listeners.mail.build_finished_handler', 'build.finished'),
    )

    # celerybeat must be running for our cleanup tasks to execute
    # e.g. celery worker -B
    app.config['CELERYBEAT_SCHEDULE'] = {
        'cleanup-builds': {
            'task': 'cleanup_builds',
            'schedule': timedelta(minutes=1),
        },
        # 'check-repos': {
        #     'task': 'check_repos',
        #     'schedule': timedelta(minutes=5),
        # },
    }
    app.config['CELERY_TIMEZONE'] = 'UTC'

    app.config['SENTRY_DSN'] = None

    app.config['JENKINS_URL'] = None
    app.config['JENKINS_TOKEN'] = None
    app.config['JENKINS_SYNC_ARTIFACTS'] = True

    app.config['KOALITY_URL'] = None
    app.config['KOALITY_API_KEY'] = None

    app.config['GOOGLE_CLIENT_ID'] = None
    app.config['GOOGLE_CLIENT_SECRET'] = None
    app.config['GOOGLE_DOMAIN'] = None

    app.config['REPO_ROOT'] = None

    app.config['MAIL_DEFAULT_SENDER'] = 'changes@localhost'
    app.config['BASE_URI'] = None

    app.config.update(config)

    if _read_config:
        if os.environ.get('CHANGES_CONF'):
            # CHANGES_CONF=/etc/changes.conf.py
            app.config.from_envvar('CHANGES_CONF')
        else:
            # Look for ~/.changes/changes.conf.py
            path = os.path.normpath(os.path.expanduser('~/.changes/changes.conf.py'))
            app.config.from_pyfile(path, silent=True)

    if not app.config['BASE_URI']:
        raise ValueError('You must set ``BASE_URI`` in your configuration.')

    parsed_url = urlparse(app.config['BASE_URI'])
    app.config.setdefault('SERVER_NAME', parsed_url.netloc)
    app.config.setdefault('PREFERRED_URL_SCHEME', parsed_url.scheme)

    # init sentry first
    sentry.init_app(app)

    db.init_app(app)
    mail.init_app(app)
    pubsub.init_app(app)
    queue.init_app(app)
    redis.init_app(app)

    from raven.contrib.celery import register_signal, register_logger_signal
    register_signal(sentry.client)
    register_logger_signal(sentry.client)

    # configure debug routes first
    if app.debug:
        configure_debug_routes(app)

    configure_templates(app)

    # TODO: these can be moved to wsgi app entrypoints
    configure_api_routes(app)
    configure_web_routes(app)

    configure_event_listeners(app)
    configure_jobs(app)

    return app

def configure_templates(app):
    from changes.utils.times import duration

    app.jinja_env.filters['duration'] = duration

def configure_api_routes(app):
    from changes.api.auth_index import AuthIndexAPIView
    from changes.api.author_build_index import AuthorBuildIndexAPIView
    from changes.api.build_details import BuildDetailsAPIView
    from changes.api.build_index import BuildIndexAPIView
    from changes.api.build_log_details import BuildLogDetailsAPIView
    from changes.api.build_retry import BuildRetryAPIView
    from changes.api.change_details import ChangeDetailsAPIView
    from changes.api.change_index import ChangeIndexAPIView
    from changes.api.patch_details import PatchDetailsAPIView
    from changes.api.project_build_index import ProjectBuildIndexAPIView
    from changes.api.project_commit_details import ProjectCommitDetailsAPIView
    from changes.api.project_commit_index import ProjectCommitIndexAPIView
    from changes.api.project_index import ProjectIndexAPIView
    from changes.api.project_stats_index import ProjectStatsIndexAPIView
    from changes.api.project_test_details import ProjectTestDetailsAPIView
    from changes.api.project_test_index import ProjectTestIndexAPIView
    from changes.api.project_details import ProjectDetailsAPIView
    from changes.api.testgroup_details import TestGroupDetailsAPIView

    app.add_url_rule(
        '/api/0/auth/', view_func=AuthIndexAPIView.as_view('api-auth'))
    app.add_url_rule(
        '/api/0/authors/<author_id>/builds/', view_func=AuthorBuildIndexAPIView.as_view('api-author-build-list'))
    app.add_url_rule(
        '/api/0/builds/', view_func=BuildIndexAPIView.as_view('api-build-list'))
    app.add_url_rule(
        '/api/0/builds/<build_id>/', view_func=BuildDetailsAPIView.as_view('api-build-details'))
    app.add_url_rule(
        '/api/0/builds/<build_id>/retry/', view_func=BuildRetryAPIView.as_view('api-build-retry'))
    app.add_url_rule(
        '/api/0/builds/<build_id>/logs/<source_id>/', view_func=BuildLogDetailsAPIView.as_view('api-build-log-details'))
    app.add_url_rule(
        '/api/0/changes/', view_func=ChangeIndexAPIView.as_view('api-change-list'))
    app.add_url_rule(
        '/api/0/changes/<change_id>/', view_func=ChangeDetailsAPIView.as_view('api-change-details'))
    app.add_url_rule(
        '/api/0/changes/<change_id>/builds/', view_func=BuildIndexAPIView.as_view('api-change-build-list'))
    app.add_url_rule(
        '/api/0/patches/<patch_id>/', view_func=PatchDetailsAPIView.as_view('api-patch-details'))
    app.add_url_rule(
        '/api/0/projects/', view_func=ProjectIndexAPIView.as_view('api-project-list'))
    app.add_url_rule(
        '/api/0/projects/<project_id>/', view_func=ProjectDetailsAPIView.as_view('api-project-details'))
    app.add_url_rule(
        '/api/0/projects/<project_id>/builds/', view_func=ProjectBuildIndexAPIView.as_view('api-project-build-list'))
    app.add_url_rule(
        '/api/0/projects/<project_id>/commits/', view_func=ProjectCommitIndexAPIView.as_view('api-project-commit-list'))
    app.add_url_rule(
        '/api/0/projects/<project_id>/commits/<commit_id>/', view_func=ProjectCommitDetailsAPIView.as_view('api-project-commit-details'))
    app.add_url_rule(
        '/api/0/projects/<project_id>/stats/', view_func=ProjectStatsIndexAPIView.as_view('api-project-stats'))
    app.add_url_rule(
        '/api/0/projects/<project_id>/tests/', view_func=ProjectTestIndexAPIView.as_view('api-project-test-list'))
    app.add_url_rule(
        '/api/0/projects/<project_id>/tests/<test_id>/', view_func=ProjectTestDetailsAPIView.as_view('api-project-test-details'))
    app.add_url_rule(
        '/api/0/testgroups/<testgroup_id>/', view_func=TestGroupDetailsAPIView.as_view('api-change-testgroup-details'))

def configure_web_routes(app):
    from changes.web.auth import AuthorizedView, LoginView, LogoutView
    from changes.web.index import IndexView
    from changes.web.static import StaticView

    app.add_url_rule(
        '/static/<path:filename>',
        view_func=StaticView.as_view('static', root=os.path.join(PROJECT_ROOT, 'static')))
    app.add_url_rule(
        '/partials/<path:filename>',
        view_func=StaticView.as_view('partials', root=os.path.join(PROJECT_ROOT, 'partials')))

    app.add_url_rule(
        '/auth/login/', view_func=LoginView.as_view('login', authorized_url='authorized'))
    app.add_url_rule(
        '/auth/logout/', view_func=LogoutView.as_view('logout', complete_url='index'))
    app.add_url_rule(
        '/auth/complete/', view_func=AuthorizedView.as_view('authorized', authorized_url='authorized', complete_url='index'))

    app.add_url_rule(
        '/<path:path>', view_func=IndexView.as_view('index-path'))
    app.add_url_rule(
        '/', view_func=IndexView.as_view('index'))

def configure_debug_routes(app):
    from changes.debug.reports.build import BuildReportMailView

    app.add_url_rule(
        '/debug/mail/report/build/', view_func=BuildReportMailView.as_view('debug-build-report'))

def configure_jobs(app):
    from changes.jobs.check_repos import check_repos
    from changes.jobs.cleanup_builds import cleanup_builds
    from changes.jobs.create_build import create_build
    from changes.jobs.notify_listeners import notify_listeners
    from changes.jobs.sync_artifact import sync_artifact
    from changes.jobs.sync_build import sync_build
    from changes.jobs.sync_repo import sync_repo
    from changes.jobs.update_project_stats import update_project_stats

    queue.register('check_repos', check_repos)
    queue.register('cleanup_builds', cleanup_builds)
    queue.register('create_build', create_build)
    queue.register('notify_listeners', notify_listeners)
    queue.register('sync_artifact', sync_artifact)
    queue.register('sync_build', sync_build)
    queue.register('sync_repo', sync_repo)
    queue.register('update_project_stats', update_project_stats)

def configure_event_listeners(app):
    from changes.signals import register_listener
    from changes.utils.imports import import_string

    for func_path, signal_name in app.config['EVENT_LISTENERS']:
        func = import_string(func_path)
        register_listener(func, signal_name)
<|endoftext|>


<commit_before>import json
import datetime
import traceback
from pprint import pprint
from collections import defaultdict

from django.core.serializers import json as djangojson
from django.db.models import Model, ManyToManyField
from django.db.models.query import RawQuerySet
from django.db.models.sql.constants import QUERY_TERMS
from django.db.models.options import Options
from django.db.models.fields import AutoField, FieldDoesNotExist
from django.db import connection

from django.forms.models import model_to_dict
from django.utils.encoding import smart_str
from django.contrib.auth.models import User, Group
from django.core import serializers

from datapoints.models import *
from source_data.models import *

class v2Request(object):

    def __init__(self, request, content_type):

        self.request = request
        self.content_type = content_type
        self.user_id = request.user.id
        self.data = None
        self.meta = None
        self.err = None

        # Tells the API which models are avail for GET / POST / META requests #
        self.orm_mapping = {
            'campaign': {'orm_obj':Campaign,
                'permission_function':self.apply_campaign_permissions},
            'region': {'orm_obj':Region,
                'permission_function':self.apply_region_permissions},
            'document_review' : {'orm_obj':DocumentDetail,
                'permission_function': self.group_document_metadata},
            'indicator': {'orm_obj':IndicatorAbstracted,
                'permission_function':None},
            'group': {'orm_obj':Group,
                'permission_function':None},
            'user': {'orm_obj':User,
                'permission_function':None},
            'region_permission': {'orm_obj':RegionPermission,
                'permission_function':None},
            'user_group': {'orm_obj':UserGroup,
                'permission_function':None},
            'document': {'orm_obj':Document,
                'permission_function':None},
            'office': {'orm_obj':Office,
                'permission_function':None},
            'indicator_map': {'orm_obj':IndicatorMap,
                'permission_function':None},
            'region_map': {'orm_obj':RegionMap,
                'permission_function':None},
            'campaign_map': {'orm_obj':CampaignMap,
                'permission_function':None}
        }

    def main(self):

        response_data = {
            'objects':self.data,
            'meta':self.meta,
            'error':self.err,
        }

        return response_data

    ## permissions functions ##

    def apply_region_permissions(self, list_of_object_ids):
        '''
        This returns a raw queryset, that is the query itself isn't actually
        executed until the data is unpacked in the serialize method.

        For more information on how region permissions work, take a look
        at the definition of the stored proc called below.
        '''

        data = Region.objects.raw("SELECT * FROM\
            fn_get_authorized_regions_by_user(%s,%s,%s)",[self.request.user.id,
            list_of_object_ids,self.read_write])

        return None, data

    def apply_campaign_permissions(self, list_of_object_ids):
        '''
        As in above, this returns a raw queryset, and will be executed in the
        serialize method.

        The below query reads: "show me all campaigns that have data for
        regions that I am permitted to see."

        No need to do recursion here, because the data is already aggregated
         regionally when ingested into the datapoint_abstracted table.
        '''

        data = Campaign.objects.raw("""
            SELECT c.* FROM campaign c
            INNER JOIN datapoint_abstracted da
                ON c.id = da.campaign_id
            INNER JOIN region_permission rm
                ON da.region_id = rm.region_id
                AND rm.user_id = %s
            WHERE c.id = ANY(COALESCE(%s,ARRAY[c.id]))
        """, [self.user_id, list_of_object_ids])

        return None, data

    def group_document_metadata(self,list_of_object_ids):
        '''
        This function is not actually about permissions, but rather data
        manipulation needed for the front end.  Here i create three nodes
        (region, campaign, indicator) and add all metadata here.
        '''

        raw_data = DocumentDetail.objects.raw("""
            SELECT * FROM
            document_detail dd
            WHERE dd.id = ANY(COALESCE(%s,ARRAY[dd.id]))
        """,[list_of_object_ids]
        )

        cleaned_data = {
            'region':[],
            'campaign':[],
            'indicator':[],
        }

        for row in raw_data:

            row_dict = dict(row.__dict__)
            del row_dict['_state']

            cleaned_data[row.db_model].append(row_dict)

        return None, cleaned_data

class v2PostRequest(v2Request):

    def __init__(self, request, content_type):

        super(v2PostRequest, self).__init__(request, content_type)

        ## DB obj can be different between GET and POST requests ##
        self.orm_mapping['indicator']['orm_obj'] = Indicator

        ## find the DB obj we want to POST to
        self.db_obj = self.orm_mapping[content_type]['orm_obj']

        ## klean URL parameters
        self.kwargs = self.clean_kwargs(request.POST)

    def clean_kwargs(self,query_dict):

        cleaned_kwargs = {}

        for k,v in query_dict.iteritems():
            cleaned_kwargs[k] = v

        ## add mapped_by id for mapping endpoints ##
        if self.content_type in ['region_map','indicator_map','campaign_map']:
            cleaned_kwargs['mapped_by_id'] = self.user_id

        return cleaned_kwargs

    def main(self):
        '''
        Return error if not implemented

        if method is create:
        Create an object in accordance to the URL kwargs and return the new ID

        if delete:
            query for objects taht match
        '''

        if self.content_type == 'user':

            self.err = 'User POST not implemented in v2 api.'
            return super(v2PostRequest, self).main()

        ## Create, Update or Delete ##
        request_type = self.determined_request_type()

        try:

            if request_type == 'CREATE':

                new_obj = self.db_obj.objects.create(**self.kwargs)
                self.data = {'new_id':new_obj.id }

            elif request_type == 'DELETE':

                old_obj = self.db_obj.objects.get(**self.kwargs)
                self.data = {'deleted_id': old_obj.id }

                old_obj.delete()

            elif request_type == 'UPDATE':

                obj_to_update = self.db_obj(id=self.id_param, **self.kwargs)
                obj_to_update.save()

                self.data = data = {'updated_id': obj_to_update.id
                , 'updated_values': self.kwargs }

        except Exception, e:
            self.err = traceback.format_exc()

        return super(v2PostRequest, self).main()

    def determined_request_type(self):
        '''
        POST can be create, update, or delete.
        '''

        try:
            self.id_param = self.kwargs['id']
            del self.kwargs['id']

            if self.id_param == '':
                return 'DELETE'
            else:
                return 'UPDATE'

        except KeyError:
            self.id_param = None
            return 'CREATE'

class v2MetaRequest(v2Request):

    def __init__(self, request, content_type):

        super(v2MetaRequest, self).__init__(request, content_type)
        self.db_obj = self.orm_mapping[content_type]['orm_obj']

    def main(self):
        '''
        Use information about the django model in order to send meta data
        about the resource to the API.  This is used by the front end to
        dynamically generate table views and forms to interact with these
        models.
        '''

        self.data = {}

        self.all_field_meta = []
        self.meta_data = {
                'slug':self.content_type,
                'name':self.content_type,
                'primary_key':'id',
                'search_field':'slug',
                'defaultSortField':'id',
                'defaultSortDirection':'asc',
        }

        db_table = self.db_obj._meta.db_table
        ca_dct = ColumnAttributes.objects.filter(table_name = \
            db_table).values('column_name','display_name',\
            'display_on_table_flag')

        self.column_lookup = {}
        for row in ca_dct:
            column_name = row['column_name']
            del row['column_name']
            self.column_lookup[column_name] = row

        ## BUILD METADATA FOR EACH FIELD ##
        for ix,(field) in enumerate(self.db_obj._meta.get_all_field_names()):

            try:
                self.build_field_meta_dict(field,ix)
            except KeyError:
                pass

        self.data['fields'] = self.all_field_meta

        return super(v2MetaRequest, self).main()

    def build_field_meta_dict(self, field, ix):
        '''
        Examine model instance to find meta data
        Query the Column Attributes table to find metadata on what django model
        does not store.
        '''
        try:
            field_object = self.db_obj._meta.get_field(field)
        except FieldDoesNotExist:
            return None

        ## DICT TO MAP DJANNGO FIELD DEFINITION TO THE TYPES THE FE EXPECTS ##
        field_type_mapper = {'AutoField':'number','FloatField':'number',
            'ForeignKey':'array','CharField':'string','ManyToManyField':'array',
            'DateTimeField':'datetime','DateField':'datetime','BooleanField':
            'boolean','SlugField':'string','TextField':'string'}

        ## BUILD A DICTIONARY FOR EACH FIELD ##
        field_object_dict = {
            'name': field_object.name,
            'title': self.column_lookup[field_object.name]['display_name'],
            'type': field_type_mapper[field_object.get_internal_type()],
            'max_length': field_object.max_length,
            'editable' : field_object.editable,
            'default_value' : str(field_object.get_default()),
                'display' : {
                    'on_table':self.column_lookup[field_object.name]['display_on_table_flag'],
                    'weightTable':ix,
                    'weightForm':ix,
                },
            'constraints': self.build_field_constraints(field_object)
            }

        self.all_field_meta.append(field_object_dict)

    def build_field_constraints(self,field_object):

        field_constraints = {
            'unique':field_object.unique
            }

        try:
            field_constraints['required'] = field_object.required
        except AttributeError:
            field_constraints['required'] = False

        if field_object.name == 'groups':
        # if isinstance(field_object,ManyToManyField) and field_object.name == 'groups':

            ## HACK FOR USERS ##
            dict_list = [{'value':1,'label':'UNICEF HQ'}]
            field_constraints['items'] = {'oneOf':dict_list}

        return field_constraints

class v2GetRequest(v2Request):

    def __init__(self, request, content_type):

        super(v2GetRequest, self).__init__(request, content_type)

        self.db_obj = self.orm_mapping[content_type]['orm_obj']
        self.permission_function = self.orm_mapping[content_type]\
            ['permission_function']

        self.kwargs = self.clean_kwargs(request.GET)

        print self.db_obj

    def main(self):
        '''
        Get the list of database objects ( ids ) by applying the URL kwargs to
        the filter method of the djanog ORM.
        '''
        ## IF THERE ARE NO FILTERS, THE API DOES NOT NEED TO ##
        ## QUERY THE DATABASE BEFORE APPLYING PERMISSIONS ##
        if not self.kwargs and self.content_type in ['campaign','region']:
            qset = None
        else:
            qset = list(self.db_obj.objects.filter(**self.kwargs).values())

        err, filtered_data = self.apply_permissions(qset)
        err, data = self.serialize(filtered_data)

        ## apply limit and offset.  Not ideal that this does happen at the
        ## data base level, but applying limit/offset at here and querying for
        ## all the data is fine for now as these endpoints are fast.

        try:
            self.data = data[self.offset:self.limit + self.offset]
        except TypeError:
            self.data = data

        self.full_data_length = len(data)
        self.err = err
        self.meta = self.build_meta()

        return super(v2GetRequest, self).main()

    def clean_kwargs(self,query_dict):
        '''
        When passing filters make sure that what is in the URL string is
        actually a field of the model.

        This includes parsing the query terms out of the parameter key.  i.e.
        when the API receives id__gte=50, we need to check to see if "id"
        is a field ,not id__gte=50.
        '''

        cleaned_kwargs = {}
        operator_lookup = {}

        ## MAP THE QUERY PARAMETER WITH ITS OPERATOR, TO THE DB MODEL ##
        for param in query_dict.keys():

            try:
                operator_lookup[param[0:param.index('__')]] = param
            except ValueError:
                operator_lookup[param] = param

        ## ONLY WANT TO CLEAN KWARGS FOR COLUMNS THAT EXISTS FOR THIS MODEL ##
        db_model_keys = list(set(self.db_obj._meta.get_all_field_names()).\
            intersection(set(k for k in operator_lookup.keys())))

        ## FINALLY CREATE ONE DICT (cleaned_kwargs) WITH THE ORIGINAL K,V ##
        ## IN THE URL, BUT FILTERED ON COLUMNS AVAILABLE TO THE MODEL ##
        for k in db_model_keys:

            query_key = operator_lookup[k]
            query_value = query_dict[operator_lookup[k]]#[]

            if "," in query_value:
                cleaned_kwargs[query_key] = query_value.split(',')
            else:
                cleaned_kwargs[query_key] = query_value

        ## FIND THE LIMIT AND OFFSET AND STORE AS CLASS ATTRIBUETS ##
        try:
            self.limit = int(query_dict['limit'])
        except KeyError:
            self.limit = 1000000000

        try:
            self.offset = int(query_dict['offset'])
        except KeyError:
            self.offset = 0

        ## Find the Read/Write param for regions ( see POLIO-779 ) ##

        try:
            self.read_write = query_dict['read_write']
        except KeyError:
            self.read_write = 'r'

        return cleaned_kwargs

    def build_meta(self):
        '''
        '''

        meta_dict = {
            'limit': self.limit,
            'offset': self.offset,
            'total_count': self.full_data_length,
        }

        return meta_dict

    def apply_permissions(self, queryset):
        '''
        Right now this is only for regions and Datapoints.

        Returns a Raw Queryset
        '''

        if not self.permission_function:
            return None, queryset

        ## if filters then create that list of IDs, otherwise pass ##
        ## None, and the permissions function wont filter on an ID list ##
        if not queryset:
            list_of_object_ids = None
        else:
            list_of_object_ids = [x['id'] for x in queryset]

        err, data = self.permission_function(list_of_object_ids)

        return err, data

    def serialize(self, data):
        '''
        document_review has a custom serialization in which 'region', 'campaign'
        and 'indicator' are keys and the value is a list of the cooresponding
        mappings.  This needs to be cleaned up, but for now that content
        type skips through the serialization method.

        '''

        if self.content_type != 'document_review':
            serialized = [self.clean_row_result(row) for row in data]

            return None, serialized

        else:
            return None, data

    def clean_row_result(self, row_data):
        '''
        When Serializing, everything but Int and List are converted to string.
        In this case the List (in the case of indicators), is a json array.

        If it is a raw queryset, first convert the row to a dict using the
        built in __dict__ method.

        This just returns a list of dict.  The JsonResponse in the view
        does the actual json conversion.

        Also get rid of the _state attribute ( see POLIO-801)

        '''

        cleaned_row_data = {}

        # if raw queryset, convert to dict
        if isinstance(row_data,Model):
            row_data = dict(row_data.__dict__)
            del row_data['_state']

        for k,v in row_data.iteritems():
            if isinstance(v, int):
                cleaned_row_data[k] = v
            elif 'json' in k: # if k == 'bound_json':
                cleaned_row_data[k] = json.loads(v)
            else:
                cleaned_row_data[k] = smart_str(v)

        return cleaned_row_data
<commit_msg>merging<commit_after>import json
import datetime
import traceback
from pprint import pprint
from collections import defaultdict

from django.core.serializers import json as djangojson
from django.db.models import Model, ManyToManyField
from django.db.models.query import RawQuerySet
from django.db.models.sql.constants import QUERY_TERMS
from django.db.models.options import Options
from django.db.models.fields import AutoField, FieldDoesNotExist
from django.db import connection

from django.forms.models import model_to_dict
from django.utils.encoding import smart_str
from django.contrib.auth.models import User, Group
from django.core import serializers

from datapoints.models import *
from source_data.models import *

class v2Request(object):

    def __init__(self, request, content_type):

        self.request = request
        self.content_type = content_type
        self.user_id = request.user.id
        self.data = None
        self.meta = None
        self.err = None

        # Tells the API which models are avail for GET / POST / META requests #
        self.orm_mapping = {
            'campaign': {'orm_obj':Campaign,
                'permission_function':self.apply_campaign_permissions},
            'region': {'orm_obj':Region,
                'permission_function':self.apply_region_permissions},
            'document_review' : {'orm_obj':DocumentDetail,
                'permission_function': self.group_document_metadata},
            'indicator': {'orm_obj':IndicatorAbstracted,
                'permission_function':None},
            'group': {'orm_obj':Group,
                'permission_function':None},
            'user': {'orm_obj':User,
                'permission_function':None},
            'region_permission': {'orm_obj':RegionPermission,
                'permission_function':None},
            'user_group': {'orm_obj':UserGroup,
                'permission_function':None},
            'document': {'orm_obj':Document,
                'permission_function':None},
            'office': {'orm_obj':Office,
                'permission_function':None},
            'indicator_map': {'orm_obj':IndicatorMap,
                'permission_function':None},
            'region_map': {'orm_obj':RegionMap,
                'permission_function':None},
            'campaign_map': {'orm_obj':CampaignMap,
                'permission_function':None}
        }

    def main(self):

        response_data = {
            'objects':self.data,
            'meta':self.meta,
            'error':self.err,
        }

        return response_data

    ## permissions functions ##

    def apply_region_permissions(self, list_of_object_ids):
        '''
        This returns a raw queryset, that is the query itself isn't actually
        executed until the data is unpacked in the serialize method.

        For more information on how region permissions work, take a look
        at the definition of the stored proc called below.
        '''

        data = Region.objects.raw("SELECT * FROM\
            fn_get_authorized_regions_by_user(%s,%s,%s)",[self.request.user.id,
            list_of_object_ids,self.read_write])

        return None, data

    def apply_campaign_permissions(self, list_of_object_ids):
        '''
        As in above, this returns a raw queryset, and will be executed in the
        serialize method.

        The below query reads: "show me all campaigns that have data for
        regions that I am permitted to see."

        No need to do recursion here, because the data is already aggregated
         regionally when ingested into the datapoint_abstracted table.
        '''

        data = Campaign.objects.raw("""
            SELECT c.* FROM campaign c
            INNER JOIN datapoint_abstracted da
                ON c.id = da.campaign_id
            INNER JOIN region_permission rm
                ON da.region_id = rm.region_id
                AND rm.user_id = %s
            WHERE c.id = ANY(COALESCE(%s,ARRAY[c.id]))
        """, [self.user_id, list_of_object_ids])

        return None, data

    def group_document_metadata(self,list_of_object_ids):
        '''
        This function is not actually about permissions, but rather data
        manipulation needed for the front end.  Here i create three nodes
        (region, campaign, indicator) and add all metadata here.
        '''

        raw_data = DocumentDetail.objects.raw("""
            SELECT * FROM
            document_detail dd
            WHERE dd.id = ANY(COALESCE(%s,ARRAY[-1]))
        """,[list_of_object_ids]
        )

        cleaned_data = {
            'region':[],
            'campaign':[],
            'indicator':[],
        }

        for row in raw_data:

            row_dict = dict(row.__dict__)
            del row_dict['_state']

            cleaned_data[row.db_model].append(row_dict)

        return None, cleaned_data

class v2PostRequest(v2Request):

    def __init__(self, request, content_type):

        super(v2PostRequest, self).__init__(request, content_type)

        ## DB obj can be different between GET and POST requests ##
        self.orm_mapping['indicator']['orm_obj'] = Indicator

        ## find the DB obj we want to POST to
        self.db_obj = self.orm_mapping[content_type]['orm_obj']

        ## klean URL parameters
        self.kwargs = self.clean_kwargs(request.POST)

    def clean_kwargs(self,query_dict):

        cleaned_kwargs = {}

        for k,v in query_dict.iteritems():
            cleaned_kwargs[k] = v

        ## add mapped_by id for mapping endpoints ##
        if self.content_type in ['region_map','indicator_map','campaign_map']:
            cleaned_kwargs['mapped_by_id'] = self.user_id

        return cleaned_kwargs

    def main(self):
        '''
        Return error if not implemented

        if method is create:
        Create an object in accordance to the URL kwargs and return the new ID

        if delete:
            query for objects taht match
        '''

        if self.content_type == 'user':

            self.err = 'User POST not implemented in v2 api.'
            return super(v2PostRequest, self).main()

        ## Create, Update or Delete ##
        request_type = self.determined_request_type()

        try:

            if request_type == 'CREATE':

                new_obj = self.db_obj.objects.create(**self.kwargs)
                self.data = {'new_id':new_obj.id }

            elif request_type == 'DELETE':

                old_obj = self.db_obj.objects.get(**self.kwargs)
                self.data = {'deleted_id': old_obj.id }

                old_obj.delete()

            elif request_type == 'UPDATE':

                obj_to_update = self.db_obj(id=self.id_param, **self.kwargs)
                obj_to_update.save()

                self.data = data = {'updated_id': obj_to_update.id
                , 'updated_values': self.kwargs }

        except Exception, e:
            self.err = traceback.format_exc()

        return super(v2PostRequest, self).main()

    def determined_request_type(self):
        '''
        POST can be create, update, or delete.
        '''

        try:
            self.id_param = self.kwargs['id']
            del self.kwargs['id']

            if self.id_param == '':
                return 'DELETE'
            else:
                return 'UPDATE'

        except KeyError:
            self.id_param = None
            return 'CREATE'

class v2MetaRequest(v2Request):

    def __init__(self, request, content_type):

        super(v2MetaRequest, self).__init__(request, content_type)
        self.db_obj = self.orm_mapping[content_type]['orm_obj']

    def main(self):
        '''
        Use information about the django model in order to send meta data
        about the resource to the API.  This is used by the front end to
        dynamically generate table views and forms to interact with these
        models.
        '''

        self.data = {}

        self.all_field_meta = []
        self.meta_data = {
                'slug':self.content_type,
                'name':self.content_type,
                'primary_key':'id',
                'search_field':'slug',
                'defaultSortField':'id',
                'defaultSortDirection':'asc',
        }

        db_table = self.db_obj._meta.db_table
        ca_dct = ColumnAttributes.objects.filter(table_name = \
            db_table).values('column_name','display_name',\
            'display_on_table_flag')

        self.column_lookup = {}
        for row in ca_dct:
            column_name = row['column_name']
            del row['column_name']
            self.column_lookup[column_name] = row

        ## BUILD METADATA FOR EACH FIELD ##
        for ix,(field) in enumerate(self.db_obj._meta.get_all_field_names()):

            try:
                self.build_field_meta_dict(field,ix)
            except KeyError:
                pass

        self.data['fields'] = self.all_field_meta

        return super(v2MetaRequest, self).main()

    def build_field_meta_dict(self, field, ix):
        '''
        Examine model instance to find meta data
        Query the Column Attributes table to find metadata on what django model
        does not store.
        '''
        try:
            field_object = self.db_obj._meta.get_field(field)
        except FieldDoesNotExist:
            return None

        ## DICT TO MAP DJANNGO FIELD DEFINITION TO THE TYPES THE FE EXPECTS ##
        field_type_mapper = {'AutoField':'number','FloatField':'number',
            'ForeignKey':'array','CharField':'string','ManyToManyField':'array',
            'DateTimeField':'datetime','DateField':'datetime','BooleanField':
            'boolean','SlugField':'string','TextField':'string'}

        ## BUILD A DICTIONARY FOR EACH FIELD ##
        field_object_dict = {
            'name': field_object.name,
            'title': self.column_lookup[field_object.name]['display_name'],
            'type': field_type_mapper[field_object.get_internal_type()],
            'max_length': field_object.max_length,
            'editable' : field_object.editable,
            'default_value' : str(field_object.get_default()),
                'display' : {
                    'on_table':self.column_lookup[field_object.name]['display_on_table_flag'],
                    'weightTable':ix,
                    'weightForm':ix,
                },
            'constraints': self.build_field_constraints(field_object)
            }

        self.all_field_meta.append(field_object_dict)

    def build_field_constraints(self,field_object):

        field_constraints = {
            'unique':field_object.unique
            }

        try:
            field_constraints['required'] = field_object.required
        except AttributeError:
            field_constraints['required'] = False

        if field_object.name == 'groups':
        # if isinstance(field_object,ManyToManyField) and field_object.name == 'groups':

            ## HACK FOR USERS ##
            dict_list = [{'value':1,'label':'UNICEF HQ'}]
            field_constraints['items'] = {'oneOf':dict_list}

        return field_constraints

class v2GetRequest(v2Request):

    def __init__(self, request, content_type):

        super(v2GetRequest, self).__init__(request, content_type)

        self.db_obj = self.orm_mapping[content_type]['orm_obj']
        self.permission_function = self.orm_mapping[content_type]\
            ['permission_function']

        self.kwargs = self.clean_kwargs(request.GET)

    def main(self):
        '''
        Get the list of database objects ( ids ) by applying the URL kwargs to
        the filter method of the djanog ORM.
        '''
        ## IF THERE ARE NO FILTERS, THE API DOES NOT NEED TO ##
        ## QUERY THE DATABASE BEFORE APPLYING PERMISSIONS ##
        if not self.kwargs and self.content_type in ['campaign','region']:
            qset = None
        else:
            qset = list(self.db_obj.objects.filter(**self.kwargs).values())

        err, filtered_data = self.apply_permissions(qset)
        err, data = self.serialize(filtered_data)

        ## apply limit and offset.  Not ideal that this does happen at the
        ## data base level, but applying limit/offset at here and querying for
        ## all the data is fine for now as these endpoints are fast.

        try:
            self.data = data[self.offset:self.limit + self.offset]
        except TypeError:
            self.data = data

        self.full_data_length = len(data)
        self.err = err
        self.meta = self.build_meta()

        return super(v2GetRequest, self).main()

    def clean_kwargs(self,query_dict):
        '''
        When passing filters make sure that what is in the URL string is
        actually a field of the model.

        This includes parsing the query terms out of the parameter key.  i.e.
        when the API receives id__gte=50, we need to check to see if "id"
        is a field ,not id__gte=50.
        '''

        cleaned_kwargs = {}
        operator_lookup = {}

        ## MAP THE QUERY PARAMETER WITH ITS OPERATOR, TO THE DB MODEL ##
        for param in query_dict.keys():

            try:
                operator_lookup[param[0:param.index('__')]] = param
            except ValueError:
                operator_lookup[param] = param

        ## ONLY WANT TO CLEAN KWARGS FOR COLUMNS THAT EXISTS FOR THIS MODEL ##
        db_model_keys = list(set(self.db_obj._meta.get_all_field_names()).\
            intersection(set(k for k in operator_lookup.keys())))

        ## FINALLY CREATE ONE DICT (cleaned_kwargs) WITH THE ORIGINAL K,V ##
        ## IN THE URL, BUT FILTERED ON COLUMNS AVAILABLE TO THE MODEL ##
        for k in db_model_keys:

            query_key = operator_lookup[k]
            query_value = query_dict[operator_lookup[k]]#[]

            if "," in query_value:
                cleaned_kwargs[query_key] = query_value.split(',')
            else:
                cleaned_kwargs[query_key] = query_value

        ## FIND THE LIMIT AND OFFSET AND STORE AS CLASS ATTRIBUETS ##
        try:
            self.limit = int(query_dict['limit'])
        except KeyError:
            self.limit = 1000000000

        try:
            self.offset = int(query_dict['offset'])
        except KeyError:
            self.offset = 0

        ## Find the Read/Write param for regions ( see POLIO-779 ) ##

        try:
            self.read_write = query_dict['read_write']
        except KeyError:
            self.read_write = 'r'

        return cleaned_kwargs

    def build_meta(self):
        '''
        '''

        meta_dict = {
            'limit': self.limit,
            'offset': self.offset,
            'total_count': self.full_data_length,
        }

        return meta_dict

    def apply_permissions(self, queryset):
        '''
        Right now this is only for regions and Datapoints.

        Returns a Raw Queryset
        '''

        if not self.permission_function:
            return None, queryset

        ## if filters then create that list of IDs, otherwise pass ##
        ## None, and the permissions function wont filter on an ID list ##
        if not queryset:
            list_of_object_ids = None
        else:
            list_of_object_ids = [x['id'] for x in queryset]

        err, data = self.permission_function(list_of_object_ids)

        return err, data

    def serialize(self, data):
        '''
        document_review has a custom serialization in which 'region', 'campaign'
        and 'indicator' are keys and the value is a list of the cooresponding
        mappings.  This needs to be cleaned up, but for now that content
        type skips through the serialization method.

        '''

        if self.content_type != 'document_review':
            serialized = [self.clean_row_result(row) for row in data]

            return None, serialized

        else:
            return None, data

    def clean_row_result(self, row_data):
        '''
        When Serializing, everything but Int and List are converted to string.
        In this case the List (in the case of indicators), is a json array.

        If it is a raw queryset, first convert the row to a dict using the
        built in __dict__ method.

        This just returns a list of dict.  The JsonResponse in the view
        does the actual json conversion.

        Also get rid of the _state attribute ( see POLIO-801)

        '''

        cleaned_row_data = {}

        # if raw queryset, convert to dict
        if isinstance(row_data,Model):
            row_data = dict(row_data.__dict__)
            del row_data['_state']

        for k,v in row_data.iteritems():
            if isinstance(v, int):
                cleaned_row_data[k] = v
            elif 'json' in k: # if k == 'bound_json':
                cleaned_row_data[k] = json.loads(v)
            else:
                cleaned_row_data[k] = smart_str(v)

        return cleaned_row_data
<|endoftext|>