<commit_before>import tempfile
import re
import sys
import os
import subprocess
import platform
import argparse
import multiprocessing
import shutil
import fileinput
import re
from enum import Enum

# vs help
win_sdk_default = '10.0.16299.0'
win_toolset_default = '142'
qt_version_default = '5.15.0'

vs_where_path = os.path.join(
    os.environ['ProgramFiles(x86)'], 'Microsoft Visual Studio', 'Installer', 'vswhere.exe'
)

host_is_64bit = (False, True)[platform.machine().endswith('64')]
this_dir = os.path.dirname(os.path.realpath(__file__))
build_dir = this_dir + '\\build'

class QtVerison(Enum):
    Major = 0
    Minor = 1
    Micro = 2

def getLatestVSVersion():
    args = [
        '-latest',
        '-products *',
        '-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64',
        '-property installationVersion'
    ]
    cmd = [vs_where_path] + args
    output = subprocess.check_output(' '.join(cmd)).decode('utf-8')
    if output:
        return output.splitlines()[0].split('.')[0]
    else:
        return

def findVSLatestDir():
    args = [
        '-latest',
        '-products *',
        '-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64',
        '-property installationPath'
    ]
    cmd = [vs_where_path] + args
    output = subprocess.check_output(' '.join(cmd)).decode('utf-8')
    if output:
        return output.splitlines()[0]
    else:
        return

def findMSBuild():
    filename = 'MSBuild.exe'
    for root, _, files in os.walk(findVSLatestDir() + r'\\MSBuild'):
        if filename in files:
            return os.path.join(root, filename)

def getVSEnv(arch='x64', platform='', version=''):
    env_cmd = 'set path=%path:"=% && ' + \
        getVSEnvCmd(arch, platform, version) + ' && set'
    p = subprocess.Popen(env_cmd,
                         shell=True,
                         stdout=subprocess.PIPE)
    stdout, _ = p.communicate()
    out = stdout.decode('utf-8', 'ignore').split("\r\n")[5:-1]
    return dict(s.split('=', 1) for s in out)

def getCMakeGenerator(vs_version):
    if vs_version == '15':
        return 'Visual Studio 15 2017 Win64'
    else:
        return 'Visual Studio ' + vs_version + ' 2019'

def getQtVersionNumber(qt_version, version_type):
    version_list = qt_version.split('.')
    return version_list[version_type.value]

def getVSEnvCmd(arch='x64', platform='', version=''):
    vcEnvInit = [findVSLatestDir() + r'\VC\Auxiliary\Build\"vcvarsall.bat']
    if platform != '':
        args = [arch, platform, version]
    else:
        args = [arch, version]
    if args:
        vcEnvInit.extend(args)
    vcEnvInit = 'call \"' + ' '.join(vcEnvInit)
    return vcEnvInit

def build_project(msbuild, msbuild_args, proj, env_vars):
    args = []
    args.extend(msbuild_args)
    args.append(proj)
    cmd = [msbuild]
    cmd.extend(args)
    p = subprocess.Popen(cmd, shell=True,
                         stdout=sys.stdout,
                         env=env_vars)
    _, _ = p.communicate()
    if p.returncode:
        print("Build failed when building ", proj)
        sys.exit(1)

def purge_dir(build_dir):
    if os.path.exists(build_dir):
        try:
            shutil.rmtree(build_dir)
        except Exception as e:
            print('Error while removing directory: ' + str(e))
            sys.exit(1)

def generate(force, qtver, sdk, toolset, arch):
    # check build dir
    if os.path.exists(build_dir):
        if not force:
            print("Skipping generate, build directory already exists!")
            return
        purge_dir(build_dir)
    print('Generating lrc with Qt-' + qtver + ' ' +
          arch + ' ' + sdk + ' ' + toolset)
    daemon_dir = os.path.dirname(this_dir) + '\\daemon'
    daemon_bin = daemon_dir + '\\build\\x64\\ReleaseLib_win32\\bin\\dring.lib'
    if not os.path.exists(daemon_bin):
        print("Daemon library not found!")
        sys.exit(1)
    # we just assume Qt is installed in the default folder
    qt_dir = 'C:\\Qt\\' + qtver
    cmake_gen = getCMakeGenerator(getLatestVSVersion())
    qt_minor_version = getQtVersionNumber(qtver, QtVerison.Minor)

    qt_cmake_dir = qt_dir + (
        '\\msvc2017_64' if int(qt_minor_version) <= 14 else '\\msvc2019_64') + '\\lib\\cmake\\'
    cmake_options = [
        '-DQt5_DIR=' + qt_cmake_dir + 'Qt5',
        '-DQt5Core_DIR=' + qt_cmake_dir + 'Qt5Core',
        '-DQt5Sql_DIR=' + qt_cmake_dir + 'Qt5Sql',
        '-DQt5LinguistTools_DIR=' + qt_cmake_dir + 'Qt5LinguistTools',
        '-DQt5Concurrent_DIR=' + qt_cmake_dir + 'Qt5Concurrent',
        '-DQt5Gui_DIR=' + qt_cmake_dir + 'Qt5Gui',
        '-Dring_BIN=' + daemon_bin,
        '-DRING_INCLUDE_DIR=' + daemon_dir + '\\src\\dring',
        '-DCMAKE_SYSTEM_VERSION=' + sdk
    ]
    if not os.path.exists(build_dir):
        os.makedirs(build_dir)
    os.chdir(build_dir)
    cmd = ['cmake', '..', '-G', cmake_gen]
    cmd.extend(cmake_options)
    p = subprocess.Popen(cmd, shell=True,
                         stdout=subprocess.PIPE)
    _, _ = p.communicate()
    if p.returncode:
        print("Couldn't generate!")
        sys.exit(1)

    qtwrapper_proj_path = build_dir + '\\src\\qtwrapper\\qtwrapper.vcxproj'
    lrc_proj_path = build_dir + '\\ringclient_static.vcxproj'

    # force toolset
    replace_vs_prop(qtwrapper_proj_path,
                    'PlatformToolset',
                    toolset)
    replace_vs_prop(lrc_proj_path,
                    'PlatformToolset',
                    toolset)
    # force unicode
    replace_vs_prop(qtwrapper_proj_path,
                    'CharacterSet',
                    'Unicode')
    replace_vs_prop(lrc_proj_path,
                    'CharacterSet',
                    'Unicode')
    os.chdir(this_dir)

def replace_vs_prop(filename, prop, val):
    p = re.compile(r'(?s)<' + prop + r'\s?.*?>(.*?)<\/' + prop + r'>')
    val = r'<' + prop + r'>' + val + r'</' + prop + r'>'
    with fileinput.FileInput(filename, inplace=True) as file:
        for line in file:
            print(re.sub(p, val, line), end='')

def build(arch, toolset):
    print('Building lrc Release|' + arch)
    vs_env_vars = {}
    vs_env_vars.update(getVSEnv())
    qtwrapper_proj_path = build_dir + '\\src\\qtwrapper\\qtwrapper.vcxproj'
    lrc_proj_path = build_dir + '\\ringclient_static.vcxproj'
    msbuild = findMSBuild()
    if not os.path.isfile(msbuild):
        raise IOError('msbuild.exe not found. path=' + msbuild)
    msbuild_args = [
        '/nologo',
        '/verbosity:minimal',
        '/maxcpucount:' + str(multiprocessing.cpu_count()),
        '/p:Platform=' + arch,
        '/p:Configuration=' + 'Release',
        '/p:useenv=true',
        '/p:PlatformToolset=' + toolset]
    build_project(msbuild, msbuild_args, qtwrapper_proj_path, vs_env_vars)
    build_project(msbuild, msbuild_args, lrc_proj_path, vs_env_vars)

def parse_args():
    ap = argparse.ArgumentParser(description="Windows Jami-lrc build tool")
    ap.add_argument(
        '-b', '--build', action='store_true',
        help='Build lrc')
    ap.add_argument(
        '-f', '--force', action='store_true',
        help='Force action')
    ap.add_argument(
        '-p', '--purge', action='store_true',
        help='Purges the build directory')
    ap.add_argument(
        '-q', '--qtver', default=qt_version_default,
        help='Sets the Qt version to build with')
    ap.add_argument(
        '-g', '--gen', action='store_true',
        help='Generates vs project files for lrc using CMake')
    ap.add_argument(
        '-a', '--arch', default='x64',
        help='Sets the build architecture')
    ap.add_argument(
        '-s', '--sdk', default=win_sdk_default, type=str,
        help='Use specified windows sdk version')
    ap.add_argument(
        '-t', '--toolset', default=win_toolset_default, type=str,
        help='Use specified platform toolset version')

    parsed_args = ap.parse_args()

    if parsed_args.toolset:
        if parsed_args.toolset[0] != 'v':
            parsed_args.toolset = 'v' + parsed_args.toolset

    return parsed_args

def main():
    if not host_is_64bit:
        print('These scripts will only run on a 64-bit Windows system for now!')
        sys.exit(1)

    if int(getLatestVSVersion()) < 15:
        print('These scripts require at least Visual Studio v15 2017!')
        sys.exit(1)

    parsed_args = parse_args()

    if parsed_args.purge:
        print('Purging build dir')
        purge_dir(build_dir)

    if parsed_args.gen:
        generate(parsed_args.force, parsed_args.qtver,
                 parsed_args.sdk, parsed_args.toolset, parsed_args.arch)

    if parsed_args.build:
        build(parsed_args.arch, parsed_args.toolset)

if __name__ == '__main__':
    main()
<commit_msg>script: change daemon lib check from dring.lib to jami.lib<commit_after>import tempfile
import re
import sys
import os
import subprocess
import platform
import argparse
import multiprocessing
import shutil
import fileinput
import re
from enum import Enum

# vs help
win_sdk_default = '10.0.16299.0'
win_toolset_default = '142'
qt_version_default = '5.15.0'

vs_where_path = os.path.join(
    os.environ['ProgramFiles(x86)'], 'Microsoft Visual Studio', 'Installer', 'vswhere.exe'
)

host_is_64bit = (False, True)[platform.machine().endswith('64')]
this_dir = os.path.dirname(os.path.realpath(__file__))
build_dir = this_dir + '\\build'

class QtVerison(Enum):
    Major = 0
    Minor = 1
    Micro = 2

def getLatestVSVersion():
    args = [
        '-latest',
        '-products *',
        '-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64',
        '-property installationVersion'
    ]
    cmd = [vs_where_path] + args
    output = subprocess.check_output(' '.join(cmd)).decode('utf-8')
    if output:
        return output.splitlines()[0].split('.')[0]
    else:
        return

def findVSLatestDir():
    args = [
        '-latest',
        '-products *',
        '-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64',
        '-property installationPath'
    ]
    cmd = [vs_where_path] + args
    output = subprocess.check_output(' '.join(cmd)).decode('utf-8')
    if output:
        return output.splitlines()[0]
    else:
        return

def findMSBuild():
    filename = 'MSBuild.exe'
    for root, _, files in os.walk(findVSLatestDir() + r'\\MSBuild'):
        if filename in files:
            return os.path.join(root, filename)

def getVSEnv(arch='x64', platform='', version=''):
    env_cmd = 'set path=%path:"=% && ' + \
        getVSEnvCmd(arch, platform, version) + ' && set'
    p = subprocess.Popen(env_cmd,
                         shell=True,
                         stdout=subprocess.PIPE)
    stdout, _ = p.communicate()
    out = stdout.decode('utf-8', 'ignore').split("\r\n")[5:-1]
    return dict(s.split('=', 1) for s in out)

def getCMakeGenerator(vs_version):
    if vs_version == '15':
        return 'Visual Studio 15 2017 Win64'
    else:
        return 'Visual Studio ' + vs_version + ' 2019'

def getQtVersionNumber(qt_version, version_type):
    version_list = qt_version.split('.')
    return version_list[version_type.value]

def getVSEnvCmd(arch='x64', platform='', version=''):
    vcEnvInit = [findVSLatestDir() + r'\VC\Auxiliary\Build\"vcvarsall.bat']
    if platform != '':
        args = [arch, platform, version]
    else:
        args = [arch, version]
    if args:
        vcEnvInit.extend(args)
    vcEnvInit = 'call \"' + ' '.join(vcEnvInit)
    return vcEnvInit

def build_project(msbuild, msbuild_args, proj, env_vars):
    args = []
    args.extend(msbuild_args)
    args.append(proj)
    cmd = [msbuild]
    cmd.extend(args)
    p = subprocess.Popen(cmd, shell=True,
                         stdout=sys.stdout,
                         env=env_vars)
    _, _ = p.communicate()
    if p.returncode:
        print("Build failed when building ", proj)
        sys.exit(1)

def purge_dir(build_dir):
    if os.path.exists(build_dir):
        try:
            shutil.rmtree(build_dir)
        except Exception as e:
            print('Error while removing directory: ' + str(e))
            sys.exit(1)

def generate(force, qtver, sdk, toolset, arch):
    # check build dir
    if os.path.exists(build_dir):
        if not force:
            print("Skipping generate, build directory already exists!")
            return
        purge_dir(build_dir)
    print('Generating lrc with Qt-' + qtver + ' ' +
          arch + ' ' + sdk + ' ' + toolset)
    daemon_dir = os.path.dirname(this_dir) + '\\daemon'
    daemon_bin = daemon_dir + '\\build\\x64\\ReleaseLib_win32\\bin\\jami.lib'
    if not os.path.exists(daemon_bin):
        print("Daemon library not found!")
        sys.exit(1)
    # we just assume Qt is installed in the default folder
    qt_dir = 'C:\\Qt\\' + qtver
    cmake_gen = getCMakeGenerator(getLatestVSVersion())
    qt_minor_version = getQtVersionNumber(qtver, QtVerison.Minor)

    qt_cmake_dir = qt_dir + (
        '\\msvc2017_64' if int(qt_minor_version) <= 14 else '\\msvc2019_64') + '\\lib\\cmake\\'
    cmake_options = [
        '-DQt5_DIR=' + qt_cmake_dir + 'Qt5',
        '-DQt5Core_DIR=' + qt_cmake_dir + 'Qt5Core',
        '-DQt5Sql_DIR=' + qt_cmake_dir + 'Qt5Sql',
        '-DQt5LinguistTools_DIR=' + qt_cmake_dir + 'Qt5LinguistTools',
        '-DQt5Concurrent_DIR=' + qt_cmake_dir + 'Qt5Concurrent',
        '-DQt5Gui_DIR=' + qt_cmake_dir + 'Qt5Gui',
        '-Dring_BIN=' + daemon_bin,
        '-DRING_INCLUDE_DIR=' + daemon_dir + '\\src\\dring',
        '-DCMAKE_SYSTEM_VERSION=' + sdk
    ]
    if not os.path.exists(build_dir):
        os.makedirs(build_dir)
    os.chdir(build_dir)
    cmd = ['cmake', '..', '-G', cmake_gen]
    cmd.extend(cmake_options)
    p = subprocess.Popen(cmd, shell=True,
                         stdout=subprocess.PIPE)
    _, _ = p.communicate()
    if p.returncode:
        print("Couldn't generate!")
        sys.exit(1)

    qtwrapper_proj_path = build_dir + '\\src\\qtwrapper\\qtwrapper.vcxproj'
    lrc_proj_path = build_dir + '\\ringclient_static.vcxproj'

    # force toolset
    replace_vs_prop(qtwrapper_proj_path,
                    'PlatformToolset',
                    toolset)
    replace_vs_prop(lrc_proj_path,
                    'PlatformToolset',
                    toolset)
    # force unicode
    replace_vs_prop(qtwrapper_proj_path,
                    'CharacterSet',
                    'Unicode')
    replace_vs_prop(lrc_proj_path,
                    'CharacterSet',
                    'Unicode')
    os.chdir(this_dir)

def replace_vs_prop(filename, prop, val):
    p = re.compile(r'(?s)<' + prop + r'\s?.*?>(.*?)<\/' + prop + r'>')
    val = r'<' + prop + r'>' + val + r'</' + prop + r'>'
    with fileinput.FileInput(filename, inplace=True) as file:
        for line in file:
            print(re.sub(p, val, line), end='')

def build(arch, toolset):
    print('Building lrc Release|' + arch)
    vs_env_vars = {}
    vs_env_vars.update(getVSEnv())
    qtwrapper_proj_path = build_dir + '\\src\\qtwrapper\\qtwrapper.vcxproj'
    lrc_proj_path = build_dir + '\\ringclient_static.vcxproj'
    msbuild = findMSBuild()
    if not os.path.isfile(msbuild):
        raise IOError('msbuild.exe not found. path=' + msbuild)
    msbuild_args = [
        '/nologo',
        '/verbosity:minimal',
        '/maxcpucount:' + str(multiprocessing.cpu_count()),
        '/p:Platform=' + arch,
        '/p:Configuration=' + 'Release',
        '/p:useenv=true',
        '/p:PlatformToolset=' + toolset]
    build_project(msbuild, msbuild_args, qtwrapper_proj_path, vs_env_vars)
    build_project(msbuild, msbuild_args, lrc_proj_path, vs_env_vars)

def parse_args():
    ap = argparse.ArgumentParser(description="Windows Jami-lrc build tool")
    ap.add_argument(
        '-b', '--build', action='store_true',
        help='Build lrc')
    ap.add_argument(
        '-f', '--force', action='store_true',
        help='Force action')
    ap.add_argument(
        '-p', '--purge', action='store_true',
        help='Purges the build directory')
    ap.add_argument(
        '-q', '--qtver', default=qt_version_default,
        help='Sets the Qt version to build with')
    ap.add_argument(
        '-g', '--gen', action='store_true',
        help='Generates vs project files for lrc using CMake')
    ap.add_argument(
        '-a', '--arch', default='x64',
        help='Sets the build architecture')
    ap.add_argument(
        '-s', '--sdk', default=win_sdk_default, type=str,
        help='Use specified windows sdk version')
    ap.add_argument(
        '-t', '--toolset', default=win_toolset_default, type=str,
        help='Use specified platform toolset version')

    parsed_args = ap.parse_args()

    if parsed_args.toolset:
        if parsed_args.toolset[0] != 'v':
            parsed_args.toolset = 'v' + parsed_args.toolset

    return parsed_args

def main():
    if not host_is_64bit:
        print('These scripts will only run on a 64-bit Windows system for now!')
        sys.exit(1)

    if int(getLatestVSVersion()) < 15:
        print('These scripts require at least Visual Studio v15 2017!')
        sys.exit(1)

    parsed_args = parse_args()

    if parsed_args.purge:
        print('Purging build dir')
        purge_dir(build_dir)

    if parsed_args.gen:
        generate(parsed_args.force, parsed_args.qtver,
                 parsed_args.sdk, parsed_args.toolset, parsed_args.arch)

    if parsed_args.build:
        build(parsed_args.arch, parsed_args.toolset)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from pprint import pprint

from utils.functions.models import rows_to_dict_list_lower

def pedido_faturavel_sortimento(cursor, deposito, data_de, data_ate):

    filtro_deposito = ''
    if deposito is not None and data_de != '':
        filtro_deposito = f''' --
            AND i.CODIGO_DEPOSITO = {deposito} '''

    filtro_data_de = ''
    if data_de is not None and data_de != '':
        filtro_data_de = ''' --
            AND ped.DATA_ENTR_VENDA >= '{}' '''.format(data_de)

    filtro_data_ate = ''
    if data_ate is not None and data_ate != '':
        filtro_data_ate = ''' --
            AND ped.DATA_ENTR_VENDA <= '{}' '''.format(data_ate)

    sql = f"""
        WITH ped_data_entr AS -- pedidos pré-filtrados
        (
          SELECT
            ped.PEDIDO_VENDA
          FROM PEDI_100 ped -- pedido de venda
          WHERE ped.STATUS_PEDIDO <> 5 -- 5=cancelado
            {filtro_data_de} -- filtro_data_de
            {filtro_data_ate} -- filtro_data_ate
            -- filtro_embarque_de
            -- AND ped.DATA_ENTR_VENDA >= TO_DATE('2019-02-25','YYYY-MM-DD')
            -- filtro_embarque_ate
            -- AND ped.DATA_ENTR_VENDA <= TO_DATE('2020-03-29','YYYY-MM-DD')
          GROUP BY
            ped.PEDIDO_VENDA
        )
        , ped_faturas AS -- pedidos e suas faturas
        (
          SELECT
            ped.PEDIDO_VENDA PEDIDO
          , fok.NUM_NOTA_FISCAL NFOK
          , fe.DOCUMENTO NFDEV
          FROM ped_data_entr ped -- pedido de venda filtrado
          LEFT JOIN FATU_050 fok -- fatura
            ON fok.PEDIDO_VENDA = ped.PEDIDO_VENDA
           AND fok.SITUACAO_NFISC = 1  -- 1=emitida; 2=cancelada
          LEFT JOIN OBRF_010 fe -- nota fiscal de entrada/devolução
            ON fe.NOTA_DEV = fok.NUM_NOTA_FISCAL
           AND fe.SITUACAO_ENTRADA = 1 -- 1=ativa
        )
        , it_ped_qtd AS -- itens de pedidos com qtd
        (
          SELECT
            pf.PEDIDO
          , pf.NFOK
          , pf.NFDEV
          , i.CD_IT_PE_GRUPO REF
          , i.CD_IT_PE_SUBGRUPO TAM
          , i.CD_IT_PE_ITEM COR
          , sum(i.QTDE_PEDIDA) QTD
          FROM ped_faturas pf -- pedidos e suas faturas
          JOIN PEDI_110 i -- item de pedido de venda
            ON i.PEDIDO_VENDA = pf.PEDIDO
          WHERE 1=1
            {filtro_deposito} -- filtro_deposito
          GROUP BY
            pf.PEDIDO
          , pf.NFOK
          , pf.NFDEV
          , i.CD_IT_PE_NIVEL99
          , i.CD_IT_PE_GRUPO
          , i.CD_IT_PE_SUBGRUPO
          , i.CD_IT_PE_ITEM
        )
        , it_ped_q_qfat AS -- itens de pedidos com qtd e qtd faturada
        (
          SELECT
            pq.PEDIDO
          , pq.NFOK
          , pq.NFDEV
          , pq.REF
          , pq.TAM
          , pq.COR
          , pq.QTD
          , sum(COALESCE(inf.QTDE_ITEM_FATUR, 0)) QTD_FAT
          FROM it_ped_qtd pq -- itens de pedidos com qtd
          LEFT JOIN fatu_060 inf -- item de nf de saída
            ON inf.CH_IT_NF_NUM_NFIS = pq.NFOK
           AND inf.PEDIDO_VENDA = pq.PEDIDO
           AND inf.COD_CANCELAMENTO = 0
           AND inf.NIVEL_ESTRUTURA = 1
           AND inf.GRUPO_ESTRUTURA = pq.REF
           AND inf.SUBGRU_ESTRUTURA = pq.TAM
           AND inf.ITEM_ESTRUTURA = pq.COR
          GROUP BY
            pq.PEDIDO
          , pq.NFOK
          , pq.NFDEV
          , pq.REF
          , pq.TAM
          , pq.COR
          , pq.QTD
        )
        , it_ped_qtd_final AS -- itens de pedidos com qtd e qtd fat e devolvida
        (
          SELECT
            pq.PEDIDO
          , pq.NFOK
          , pq.NFDEV
          , pq.REF
          , pq.TAM
          , pq.COR
          , pq.QTD
          , pq.QTD_FAT
          , sum(COALESCE(inf.QUANTIDADE, 0)) QTD_DEV
          , pq.QTD - pq.QTD_FAT + sum(COALESCE(inf.QUANTIDADE, 0)) QTD_FINAL
          FROM it_ped_q_qfat pq -- itens de pedidos com qtd
          LEFT JOIN OBRF_015 inf -- item de nf de devolução
            ON inf.CAPA_ENT_NRDOC = pq.NFDEV
           AND inf.NUM_NOTA_ORIG = pq.NFOK
           AND inf.CODITEM_NIVEL99 = 1
           AND inf.CODITEM_GRUPO = pq.REF
           AND inf.CODITEM_SUBGRUPO = pq.TAM
           AND inf.CODITEM_ITEM = pq.COR
          GROUP BY
            pq.PEDIDO
          , pq.NFOK
          , pq.NFDEV
          , pq.REF
          , pq.TAM
          , pq.COR
          , pq.QTD
          , pq.QTD_FAT
          HAVING
            pq.QTD - pq.QTD_FAT + sum(COALESCE(inf.QUANTIDADE, 0)) > 0
        )
        , it_qtd AS -- itens de qtd final
        (
          SELECT
            pq.REF
          , pq.COR
          , pq.TAM
          , sum(pq.QTD_FINAL) QTD
          FROM it_ped_qtd_final pq -- itens de pedidos com qtd e qtd fat e dev
          LEFT JOIN BASI_220 t -- tamanhos
            ON t.TAMANHO_REF = pq.TAM
          GROUP BY
            pq.REF
          , pq.COR
          , t.ORDEM_TAMANHO
          , pq.TAM
          ORDER BY
            pq.REF
          , pq.COR
          , t.ORDEM_TAMANHO
          , pq.TAM
        )
        SELECT
          ped.*
        FROM it_qtd ped
    """
    cursor.execute(sql)
    return rows_to_dict_list_lower(cursor)
<commit_msg>opção para listar pedidos<commit_after>from pprint import pprint

from utils.functions.models import rows_to_dict_list_lower

def pedido_faturavel_sortimento(
        cursor, deposito, data_de, data_ate, retorno='r'):

    filtro_deposito = ''
    if deposito is not None and data_de != '':
        filtro_deposito = f''' --
            AND i.CODIGO_DEPOSITO = {deposito} '''

    filtro_data_de = ''
    if data_de is not None and data_de != '':
        filtro_data_de = ''' --
            AND ped.DATA_ENTR_VENDA >= '{}' '''.format(data_de)

    filtro_data_ate = ''
    if data_ate is not None and data_ate != '':
        filtro_data_ate = ''' --
            AND ped.DATA_ENTR_VENDA <= '{}' '''.format(data_ate)

    sql = f"""
        WITH ped_data_entr AS -- pedidos pré-filtrados
        (
          SELECT
            ped.PEDIDO_VENDA
          FROM PEDI_100 ped -- pedido de venda
          WHERE ped.STATUS_PEDIDO <> 5 -- 5=cancelado
            {filtro_data_de} -- filtro_data_de
            {filtro_data_ate} -- filtro_data_ate
            -- filtro_embarque_de
            -- AND ped.DATA_ENTR_VENDA >= TO_DATE('2019-02-25','YYYY-MM-DD')
            -- filtro_embarque_ate
            -- AND ped.DATA_ENTR_VENDA <= TO_DATE('2020-03-29','YYYY-MM-DD')
          GROUP BY
            ped.PEDIDO_VENDA
        )
        , ped_faturas AS -- pedidos e suas faturas
        (
          SELECT
            ped.PEDIDO_VENDA PEDIDO
          , fok.NUM_NOTA_FISCAL NFOK
          , fe.DOCUMENTO NFDEV
          FROM ped_data_entr ped -- pedido de venda filtrado
          LEFT JOIN FATU_050 fok -- fatura
            ON fok.PEDIDO_VENDA = ped.PEDIDO_VENDA
           AND fok.SITUACAO_NFISC = 1  -- 1=emitida; 2=cancelada
          LEFT JOIN OBRF_010 fe -- nota fiscal de entrada/devolução
            ON fe.NOTA_DEV = fok.NUM_NOTA_FISCAL
           AND fe.SITUACAO_ENTRADA = 1 -- 1=ativa
        )
        , it_ped_qtd AS -- itens de pedidos com qtd
        (
          SELECT
            pf.PEDIDO
          , pf.NFOK
          , pf.NFDEV
          , i.CD_IT_PE_GRUPO REF
          , i.CD_IT_PE_SUBGRUPO TAM
          , i.CD_IT_PE_ITEM COR
          , sum(i.QTDE_PEDIDA) QTD
          FROM ped_faturas pf -- pedidos e suas faturas
          JOIN PEDI_110 i -- item de pedido de venda
            ON i.PEDIDO_VENDA = pf.PEDIDO
          WHERE 1=1
            {filtro_deposito} -- filtro_deposito
          GROUP BY
            pf.PEDIDO
          , pf.NFOK
          , pf.NFDEV
          , i.CD_IT_PE_NIVEL99
          , i.CD_IT_PE_GRUPO
          , i.CD_IT_PE_SUBGRUPO
          , i.CD_IT_PE_ITEM
        )
        , it_ped_q_qfat AS -- itens de pedidos com qtd e qtd faturada
        (
          SELECT
            pq.PEDIDO
          , pq.NFOK
          , pq.NFDEV
          , pq.REF
          , pq.TAM
          , pq.COR
          , pq.QTD
          , sum(COALESCE(inf.QTDE_ITEM_FATUR, 0)) QTD_FAT
          FROM it_ped_qtd pq -- itens de pedidos com qtd
          LEFT JOIN fatu_060 inf -- item de nf de saída
            ON inf.CH_IT_NF_NUM_NFIS = pq.NFOK
           AND inf.PEDIDO_VENDA = pq.PEDIDO
           AND inf.COD_CANCELAMENTO = 0
           AND inf.NIVEL_ESTRUTURA = 1
           AND inf.GRUPO_ESTRUTURA = pq.REF
           AND inf.SUBGRU_ESTRUTURA = pq.TAM
           AND inf.ITEM_ESTRUTURA = pq.COR
          GROUP BY
            pq.PEDIDO
          , pq.NFOK
          , pq.NFDEV
          , pq.REF
          , pq.TAM
          , pq.COR
          , pq.QTD
        )
        , it_ped_qtd_final AS -- itens de pedidos com qtd e qtd fat e devolvida
        (
          SELECT
            pq.PEDIDO
          , pq.NFOK
          , pq.NFDEV
          , pq.REF
          , pq.TAM
          , pq.COR
          , pq.QTD
          , pq.QTD_FAT
          , sum(COALESCE(inf.QUANTIDADE, 0)) QTD_DEV
          , pq.QTD - pq.QTD_FAT + sum(COALESCE(inf.QUANTIDADE, 0)) QTD_FINAL
          FROM it_ped_q_qfat pq -- itens de pedidos com qtd
          LEFT JOIN OBRF_015 inf -- item de nf de devolução
            ON inf.CAPA_ENT_NRDOC = pq.NFDEV
           AND inf.NUM_NOTA_ORIG = pq.NFOK
           AND inf.CODITEM_NIVEL99 = 1
           AND inf.CODITEM_GRUPO = pq.REF
           AND inf.CODITEM_SUBGRUPO = pq.TAM
           AND inf.CODITEM_ITEM = pq.COR
          GROUP BY
            pq.PEDIDO
          , pq.NFOK
          , pq.NFDEV
          , pq.REF
          , pq.TAM
          , pq.COR
          , pq.QTD
          , pq.QTD_FAT
          HAVING
            pq.QTD - pq.QTD_FAT + sum(COALESCE(inf.QUANTIDADE, 0)) > 0
        )
    """
    if retorno == 'r':
        sql += """
            , it_qtd AS -- itens de qtd final
            (
              SELECT
                pq.REF
              , pq.COR
              , pq.TAM
              , sum(pq.QTD_FINAL) QTD
              FROM it_ped_qtd_final pq -- itens de ped com qtd e qtd fat e dev
              LEFT JOIN BASI_220 t -- tamanhos
                ON t.TAMANHO_REF = pq.TAM
              GROUP BY
                pq.REF
              , pq.COR
              , t.ORDEM_TAMANHO
              , pq.TAM
              ORDER BY
                pq.REF
              , pq.COR
              , t.ORDEM_TAMANHO
              , pq.TAM
            )
            SELECT
              ped.*
            FROM it_qtd ped
        """
    elif retorno == 'p':
        sql += """
            , it_qtd AS -- itens de qtd final
            (
              SELECT
                pq.PEDIDO
              , sum(pq.QTD_FINAL) QTD
              FROM it_ped_qtd_final pq -- itens de ped com qtd e qtd fat e dev
              GROUP BY
                pq.PEDIDO
              ORDER BY
                pq.PEDIDO
            )
            SELECT
              ped.*
            FROM it_qtd ped
        """
    cursor.execute(sql)
    return rows_to_dict_list_lower(cursor)
<|endoftext|>


<commit_before>"""
Base settings for the CMS.

Individual sites may override these settings by creating their own settings
file.  It is recommended that you import these base settings into your custom
settings file using the following syntax:

>>> from cms.settings import *

An example site-specific settings file is included with this distribution.
Please see `example_site.settings` for more information.

For an explanation of these settings, please see the Django documentation at:

<http://docs.djangoproject.com/en/dev/>
"""

import os

# Absolute path to the directory containing the CMS.

CMS_ROOT = os.path.dirname(__file__)

# Absolute path to the directory containing the CMS media files.

CMS_MEDIA_ROOT = os.path.join(CMS_ROOT, "media")

# Static media serving.

MEDIA_DEBUG = False

# Template settings.

TEMPLATE_LOADERS = ("django.template.loaders.filesystem.load_template_source",
                    "django.template.loaders.app_directories.load_template_source",)

TEMPLATE_DIRS = (os.path.join(CMS_ROOT, "templates",),)

TEMPLATE_CONTEXT_PROCESSORS = ("django.contrib.auth.context_processors.auth",
                               "django.core.context_processors.debug",
                               "django.core.context_processors.i18n",
                               "django.core.context_processors.media",
                               "django.contrib.messages.context_processors.messages",
                               "django.core.context_processors.request",
                               "cms.core.context_processors.site",
                               "cms.core.context_processors.media",
                               "cms.core.context_processors.conf",
                               "cms.apps.pages.context_processors.page",)

# Dispatch settings.

MIDDLEWARE_CLASSES = ("django.middleware.transaction.TransactionMiddleware",
                      "django.middleware.common.CommonMiddleware",
                      "django.contrib.sessions.middleware.SessionMiddleware",
                      "django.contrib.messages.middleware.MessageMiddleware",
                      "django.contrib.auth.middleware.AuthenticationMiddleware",
                      "cms.apps.pages.middleware.PageMiddleware",)

ROOT_URLCONF = "cms.urls"

# Application settings.

INSTALLED_APPS = ("django.contrib.contenttypes",
                  "django.contrib.sessions",
                  "django.contrib.auth",
                  "django.contrib.admin",
                  "django.contrib.sitemaps",
                  "django.contrib.sites",
                  "django.contrib.messages",
                  "reversion",
                  "cms.core",
                  "cms.apps.pages",
                  "cms.apps.media",)

# Multi-site settings.

SITE_ID = 1

# Publication settings.

PUBLICATION_PREVIEW_KEY = "preview"

# SEO settings.

SEO_PRIORITIES = ((1.0, "Very high"),
                  (0.8, "High"),
                  (0.5, "Medium"),
                  (0.3, "Low"),
                  (0.0, "Very low"),)

# Internationalization settings.

USE_I18N = False

# Staff management settings.

DEFAULT_GROUP_IDS = (1,)<commit_msg>Updating to use new Django settings<commit_after>"""
Base settings for the CMS.

Individual sites may override these settings by creating their own settings
file.  It is recommended that you import these base settings into your custom
settings file using the following syntax:

>>> from cms.settings import *

An example site-specific settings file is included with this distribution.
Please see `example_site.settings` for more information.

For an explanation of these settings, please see the Django documentation at:

<http://docs.djangoproject.com/en/dev/>
"""

import os

# Absolute path to the directory containing the CMS.

CMS_ROOT = os.path.dirname(__file__)

# Absolute path to the directory containing the CMS media files.

CMS_MEDIA_ROOT = os.path.join(CMS_ROOT, "media")

# Static media serving.

MEDIA_DEBUG = False

# Template settings.

TEMPLATE_LOADERS = (
    "django.template.loaders.filesystem.load_template_source",
    "django.template.loaders.app_directories.load_template_source",
)

TEMPLATE_DIRS = (os.path.join(CMS_ROOT, "templates",),)

TEMPLATE_CONTEXT_PROCESSORS = (
    "django.contrib.auth.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.contrib.messages.context_processors.messages",
    "django.core.context_processors.request",
    "cms.core.context_processors.site",
    "cms.core.context_processors.media",
    "cms.core.context_processors.conf",
    "cms.apps.pages.context_processors.page",
)

# Dispatch settings.

MIDDLEWARE_CLASSES = ("django.middleware.transaction.TransactionMiddleware",
                      "django.middleware.common.CommonMiddleware",
                      "django.contrib.sessions.middleware.SessionMiddleware",
                      "django.contrib.messages.middleware.MessageMiddleware",
                      "django.contrib.auth.middleware.AuthenticationMiddleware",
                      "cms.apps.pages.middleware.PageMiddleware",)

ROOT_URLCONF = "cms.urls"

# Application settings.

INSTALLED_APPS = ("django.contrib.contenttypes",
                  "django.contrib.sessions",
                  "django.contrib.auth",
                  "django.contrib.admin",
                  "django.contrib.sitemaps",
                  "django.contrib.sites",
                  "django.contrib.messages",
                  "reversion",
                  "cms.core",
                  "cms.apps.pages",
                  "cms.apps.media",)

# Multi-site settings.

SITE_ID = 1

# Publication settings.

PUBLICATION_PREVIEW_KEY = "preview"

# SEO settings.

SEO_PRIORITIES = ((1.0, "Very high"),
                  (0.8, "High"),
                  (0.5, "Medium"),
                  (0.3, "Low"),
                  (0.0, "Very low"),)

# Internationalization settings.

USE_I18N = False

USE_L10N = True

# Staff management settings.

DEFAULT_GROUP_IDS = (1,)

# Logging settings.

LOGGING = {
    "version": 1,
    "disable_existing_loggers": False,
    "handlers": {
        "mail_admins": {
            "level": "ERROR",
            "class": "django.utils.log.AdminEmailHandler"
        }
    },
    "loggers": {
        "django.request": {
            "handlers": ["mail_admins"],
            "level": "ERROR",
            "propagate": True,
        },
    }
}<|endoftext|>


<commit_before>#!/usr/bin/python
"""
    Based on Mako-[version]/examples/wsgi/run_wsgi.py.
    Changes marked with # S13:...
"""
import cgi, re, os, posixpath, sys
from mako.lookup import TemplateLookup
from mako import exceptions
from mimetypes import guess_type

root = os.getcwd()
host = "0.0.0.0"          # S13: added this so it can be changed.
port = 5000               # S13: used to be 8000, also changeable.
temp = "/tmp/makoserv"    # S13: where to store cached templates.
dis_ext = ["py"]          # S13: a list of file extensions that should 404.
error_style = 'html'      # select 'text' for plaintext error reporting

lookup = TemplateLookup(directories=[root],
    filesystem_checks=True, module_directory=temp,
    # S13: because we'll be using some UTF-8 characters.
    input_encoding="utf-8", output_encoding="utf-8")

def serve(environ, start_response):
    """serves requests using the WSGI callable interface."""
    fieldstorage = cgi.FieldStorage(
        fp = environ['wsgi.input'],
        environ = environ,
        keep_blank_values = True
    )
    # S13: template context has to be a bit more sensible.
    d = dict()
    d["args"] = dict([(k, getfield(fieldstorage[k])) for k in fieldstorage])
    d["env"] = environ

    uri = environ.get('PATH_INFO', '/')
    if uri.endswith("/"):  # S13: use the index.html if no filename is given.
        uri += 'index.html'
    else:
        uri = re.sub(r'^/$', '/index.html', uri)

    if re.match(r'.*\.(html|json)$', uri):    # S13: accept JSON requests.
        try:
            template = lookup.get_template(uri)
            if uri.endswith(".json"):         # S13: include the corrent type.
                start_response("200 OK", [('Content-Type','application/json')])
            else:
                start_response("200 OK", [('Content-Type','text/html')])
            return [template.render(**d)]
        except exceptions.TopLevelLookupException:
            start_response("404 Not Found", [])
            return ["Cant find template '%s'" % uri]
        except:
            if error_style == 'text':
                start_response("200 OK", [('Content-Type','text/plain')])
                return [exceptions.text_error_template().render()]
            else:
                start_response("200 OK", [('Content-Type','text/html')])
                return [exceptions.html_error_template().render()]
    elif re.match(r'.*\.(%s)$' % "|".join(dis_ext), uri):
        start_response("404 Not Found", [("Content-Type", "text/plain")])
        return ["File not found."]
    else:
        u = re.sub(r'^\/+', '', uri)
        filename = os.path.join(root, u)
        # S13: if the request is a file, proceed as usual.
        if os.path.isfile(filename):
            start_response("200 OK", [('Content-Type',
                # S13: use mimetypes.guess_type.
                guess_type(filename)[0])])
            return [file(filename).read()]
        # S13: if the request is a directory, redirect to the index.
        elif os.path.isdir(filename):
            uri += "/"
            start_response("302 Moved Permanently", [("Location", uri)])
            return []
        # S13: if it's neither, respond with a 404.
        else:
            start_response("404 Not Found", [("Content-Type", "text/plain")])
            return ["File not found."]

def getfield(f):
    """convert values from cgi.Field objects to plain values."""
    if isinstance(f, list):
        return [getfield(x) for x in f]
    else:
        return f.value

# S13: there used to be a function guess_type here. Removed that
# and simply replaced it with mimetype.guess_type (see imports).

if __name__ == '__main__':
    import wsgiref.simple_server
    # S13: Allow us to change the host (name or IP) and port.
    if len(sys.argv) > 1:
        ipp = sys.argv[1].split(":")
        if len(ipp) == 2:
            host = ipp[0]
            port = int(ipp[1])
    server = wsgiref.simple_server.make_server(host, port, serve)
    print "Server listening on port %d" % port
    server.serve_forever()

<commit_msg>Bugfix: fixed error for files whose mimetype cannot be guessed.<commit_after>#!/usr/bin/python
"""
    Based on Mako-[version]/examples/wsgi/run_wsgi.py.
    Changes marked with # S13:...
"""
import cgi, re, os, posixpath, sys
from mako.lookup import TemplateLookup
from mako import exceptions
from mimetypes import guess_type

root = os.getcwd()
host = "0.0.0.0"          # S13: added this so it can be changed.
port = 5000               # S13: used to be 8000, also changeable.
temp = "/tmp/makoserv"    # S13: where to store cached templates.
dis_ext = ["py"]          # S13: a list of file extensions that should 404.
error_style = 'html'      # select 'text' for plaintext error reporting

lookup = TemplateLookup(directories=[root],
    filesystem_checks=True, module_directory=temp,
    # S13: because we'll be using some UTF-8 characters.
    input_encoding="utf-8", output_encoding="utf-8")

def serve(environ, start_response):
    """serves requests using the WSGI callable interface."""
    fieldstorage = cgi.FieldStorage(
        fp = environ['wsgi.input'],
        environ = environ,
        keep_blank_values = True
    )
    # S13: template context has to be a bit more sensible.
    d = dict()
    d["args"] = dict([(k, getfield(fieldstorage[k])) for k in fieldstorage])
    d["env"] = environ

    uri = environ.get('PATH_INFO', '/')
    if uri.endswith("/"):  # S13: use the index.html if no filename is given.
        uri += 'index.html'
    else:
        uri = re.sub(r'^/$', '/index.html', uri)

    if re.match(r'.*\.(html|json)$', uri):    # S13: accept JSON requests.
        try:
            template = lookup.get_template(uri)
            if uri.endswith(".json"):         # S13: include the corrent type.
                start_response("200 OK", [('Content-Type','application/json')])
            else:
                start_response("200 OK", [('Content-Type','text/html')])
            return [template.render(**d)]
        except exceptions.TopLevelLookupException:
            start_response("404 Not Found", [])
            return ["Cant find template '%s'" % uri]
        except:
            if error_style == 'text':
                start_response("200 OK", [('Content-Type','text/plain')])
                return [exceptions.text_error_template().render()]
            else:
                start_response("200 OK", [('Content-Type','text/html')])
                return [exceptions.html_error_template().render()]
    elif re.match(r'.*\.(%s)$' % "|".join(dis_ext), uri):
        start_response("404 Not Found", [("Content-Type", "text/plain")])
        return ["File not found."]
    else:
        u = re.sub(r'^\/+', '', uri)
        filename = os.path.join(root, u)
        # S13: if the request is a file, proceed as usual.
        if os.path.isfile(filename):
            # S13: use mimetypes.guess_type.
            mt = guess_type(filename)[0]
            if mt: start_response("200 OK", [('Content-Type', mt)])
            else: start_response("200 OK", [])
            return [file(filename).read()]
        # S13: if the request is a directory, redirect to the index.
        elif os.path.isdir(filename):
            uri += "/"
            start_response("302 Moved Permanently", [("Location", uri)])
            return []
        # S13: if it's neither, respond with a 404.
        else:
            start_response("404 Not Found", [("Content-Type", "text/plain")])
            return ["File not found."]

def getfield(f):
    """convert values from cgi.Field objects to plain values."""
    if isinstance(f, list):
        return [getfield(x) for x in f]
    else:
        return f.value

# S13: there used to be a function guess_type here. Removed that
# and simply replaced it with mimetype.guess_type (see imports).

if __name__ == '__main__':
    import wsgiref.simple_server
    # S13: Allow us to change the host (name or IP) and port.
    if len(sys.argv) > 1:
        ipp = sys.argv[1].split(":")
        if len(ipp) == 2:
            host = ipp[0]
            port = int(ipp[1])
    server = wsgiref.simple_server.make_server(host, port, serve)
    print "Server listening on port %d" % port
    server.serve_forever()

<|endoftext|>


<commit_before>import os
import sys
import csv
import json
import traceback

from stix.core import STIXPackage
from stix.utils import set_id_namespace
from stix.indicator import Indicator
from cybox.core import Observable
from cybox.objects.address_object import Address
from cybox.objects.file_object import File
from cybox.common import Hash
from cybox.objects.uri_object import URI
from cybox.objects.win_registry_key_object import WinRegistryKey

#from cybox.objects.domain_name_object import DomainName
OUTPUT_FORMATS = ('csv', 'json', 'yara', 'autofocus', 'stix')

NAMESPACE = {"url.co.uk" : "namespace"} # Add appropriate namespace here
set_id_namespace(NAMESPACE) # new ids will be prefixed by "certuk"

stix_package = STIXPackage()

ind_ip = Indicator()
ind_ip.add_indicator_type("IP Watchlist")

ind_file = Indicator()
ind_file.add_indicator_type("File Hash Watchlist")

ind_url = Indicator()
ind_url.add_indicator_type("URL Watchlist")

ind_domain = Indicator()
ind_domain.add_indicator_type("Domain Watchlist")

ind_email = Indicator()
ind_email.add_indicator_type("Malicious E-mail")

ind_registrykey = Indicator()
ind_registrykey.add_indicator_type("Host Characteristics")

#ind_type = Indicator()
#ind_type.add_indicator_type("<stix_indicator_type>")

ind_dict = {
    'IP': ind_ip,
    'MD5': ind_file,
    'SHA1': ind_file,
    'SHA256': ind_file,

    'URL': ind_url,
    'Host': ind_domain,
    'Email': ind_email,
    'Registry': ind_registrykey,
    #filename to be added,
    #filepath to be added,
    #CVE to be added

}

def getHandler(output_format):
    output_format = output_format.lower()
    if output_format not in OUTPUT_FORMATS:
        print("[WARNING] Invalid output format specified.. using CSV")
        output_format = 'csv'

    handler_format = "OutputHandler_" + output_format
    handler_class = getattr(sys.modules[__name__], handler_format)

    return handler_class()

class OutputHandler(object):
    def print_match(self, fpath, page, name, match, last = False):
        pass

    def print_header(self, fpath):
        pass

    def print_footer(self, fpath):
        pass

    def print_error(self, fpath, exception):
        traceback.print_exc()
        print("[ERROR] %s" % (exception))

class OutputHandler_stix(OutputHandler):
    global stix_package
    global ind_dict
    def print_match(self, fpath, page, name, match):
        if name in ind_dict:
            indicator = ind_dict[name]
            indicator.title = fpath
            #===========
            # Add new object handlers here:
            if name == 'IP':
                new_obj = Address(address_value=match, category=Address.CAT_IPV4)
            elif name == 'MD5' or name == 'SHA1' or name == 'SHA256':
                new_obj = File()
                new_obj.add_hash(Hash(match))

            elif name == 'URL':
                new_obj = URI(type_=URI.TYPE_URL, value=match)

            elif name == 'Host':
                new_obj = URI(type_=URI.TYPE_DOMAIN, value=match)

            elif name == 'Email':
                new_obj = Address(address_value=match, category=Address.CAT_EMAIL) #Not sure if this is right - should this be using the email_message_object?

            elif name == 'Registry':
                new_obj = WinRegistryKey(values=match)

            #elif name == <type_from_parser>:
                #new_obj = STIX_Object()
            #===========

            new_obs = Observable(new_obj)
            new_obs.title = "Page Ref: " + str(page)
            indicator.add_observable(new_obs)

    def print_footer(self, fpath):
        add_ind_list = []
        for key in ind_dict:
            if ind_dict[key] not in add_ind_list:
                add_ind_list.append(ind_dict[key])
                stix_package.add_indicator(ind_dict[key])
        print stix_package.to_xml()
        data = stix_package.to_xml()
        newstixfile = open("test.xml", "w")
        newstixfile.write(data)
        newstixfile.close()

class OutputHandler_csv(OutputHandler):
    def __init__(self):
        self.csv_writer = csv.writer(sys.stdout, delimiter = '\t')

    def print_match(self, fpath, page, name, match):
        self.csv_writer.writerow((fpath, page, name, match))

    def print_error(self, fpath, exception):
        self.csv_writer.writerow((fpath, '0', 'error', exception))

class OutputHandler_json(OutputHandler):
    def print_match(self, fpath, page, name, match):
        data = {
            'path' : fpath,
            'file' : os.path.basename(fpath),
            'page' : page,
            'type' : name,
            'match': match
        }

        print(json.dumps(data))

    def print_error(self, fpath, exception):
        data = {
            'path'      : fpath,
            'file'      : os.path.basename(fpath),
            'type'      : 'error',
            'exception' : exception
        }

        print(json.dumps(data))

class OutputHandler_yara(OutputHandler):
    def __init__(self):
        self.rule_enc = ''.join(chr(c) if chr(c).isupper() or chr(c).islower() or chr(c).isdigit() else '_' for c in range(256))

    def print_match(self, fpath, page, name, match):
        if name in self.cnt:
            self.cnt[name] += 1
        else:
            self.cnt[name] = 1

        string_id = "$%s%d" % (name, self.cnt[name])
        self.sids.append(string_id)
        string_value = match.replace('\\', '\\\\')
        print("\t\t%s = \"%s\"" % (string_id, string_value))

    def print_header(self, fpath):
        rule_name = os.path.splitext(os.path.basename(fpath))[0].translate(self.rule_enc)

        print("rule %s" % (rule_name))
        print("{")
        print("\tstrings:")

        self.cnt = {}
        self.sids = []

    def print_footer(self, fpath):
        cond = ' or '.join(self.sids)

        print("\tcondition:")
        print("\t\t" + cond)
        print("}")

class OutputHandler_autofocus(OutputHandler):
    def __init__(self):
        self.rule_enc = ''.join(chr(c) if chr(c).isupper() or chr(c).islower() or chr(c).isdigit() else '_' for c in range(256))

    def print_match(self, fpath, page, name, match):
        string_value = match.replace('hxxp', 'http').replace('\\', '\\\\')

        if name == "MD5":
            auto_focus_query = '{"field":"sample.md5","operator":"is","value":\"%s\"},' % (string_value)
        elif name == "SHA1":
            auto_focus_query = '{"field":"sample.sha1","operator":"is","value":\"%s\"},' % (string_value)
        elif name == "SHA256":
            auto_focus_query = '{"field":"sample.sha256","operator":"is","value":\"%s\"},' % (string_value)
        elif name == "URL":
            auto_focus_query = '{"field":"sample.tasks.connection","operator":"contains","value":\"%s\"},' % (string_value)
        elif name == "Host":
            auto_focus_query = '{"field":"sample.tasks.dns","operator":"contains","value":\"%s\"},' % (string_value)
        elif name == "Registry":
            #auto_focus_query = '{"field":"sample.tasks.registry","operator":"is","value":\"%s\"},' % (string_value)
            return
        elif name == "Filepath":
            #auto_focus_query = '{"field":"sample.tasks.file","operator":"is","value":\"%s\"},' % (string_value)
            return
        elif name == "Filename":
            #auto_focus_query = '{"field":"alias.filename","operator":"is","value":\"%s\"},' % (string_value)
            return
        elif name == "Email":
            #auto_focus_query = '{"field":"alias.email","operator":"is","value":\"%s\"},' % (string_value)
            return
        elif name == "IP":
            auto_focus_query = '{"field":"sample.tasks.connection","operator":"contains","value":\"%s\"},' % (string_value)
        elif name == "CVE":
            return
        print(auto_focus_query)

    def print_header(self, fpath):
        rule_name = os.path.splitext(os.path.basename(fpath))[0].translate(self.rule_enc)

        print("AutoFocus Search for: %s" % (rule_name))
        print('{"operator":"Any","children":[')

    def print_footer(self, fpath):
        rule_name = os.path.splitext(os.path.basename(fpath))[0].translate(self.rule_enc)
        print('{"field":"sample.tag","operator":"is in the list","value":[\"%s\"]}]}' % (rule_name))

<commit_msg>Amended stix printing function.<commit_after>import os
import sys
import csv
import json
import traceback

from stix.core import STIXPackage
from stix.utils import set_id_namespace
from stix.indicator import Indicator
from cybox.core import Observable
from cybox.objects.address_object import Address
from cybox.objects.file_object import File
from cybox.common import Hash
from cybox.objects.uri_object import URI
from cybox.objects.win_registry_key_object import WinRegistryKey

#from cybox.objects.domain_name_object import DomainName
OUTPUT_FORMATS = ('csv', 'json', 'yara', 'autofocus', 'stix')

NAMESPACE = {"http://www.cert.gov.uk" : "certuk"}
set_id_namespace(NAMESPACE)

stix_package = STIXPackage()
add_ind_list = []

ind_ip = Indicator()
ind_ip.add_indicator_type("IP Watchlist")

ind_file = Indicator()
ind_file.add_indicator_type("File Hash Watchlist")

ind_url = Indicator()
ind_url.add_indicator_type("URL Watchlist")

ind_domain = Indicator()
ind_domain.add_indicator_type("Domain Watchlist")

ind_email = Indicator()
ind_email.add_indicator_type("Malicious E-mail")

ind_registrykey = Indicator()
ind_registrykey.add_indicator_type("Host Characteristics")

ind_dict = {
    'IP': ind_ip,
    'MD5': ind_file,
    'SHA1': ind_file,
    'SHA256': ind_file,

    'URL': ind_url,
    'Host': ind_domain,
    'Email': ind_email,
    'Registry': ind_registrykey,
    #filename to be added,
    #filepath to be added,
    #CVE to be added

}

def getHandler(output_format):
    output_format = output_format.lower()
    if output_format not in OUTPUT_FORMATS:
        print("[WARNING] Invalid output format specified.. using CSV")
        output_format = 'csv'

    handler_format = "OutputHandler_" + output_format
    handler_class = getattr(sys.modules[__name__], handler_format)

    return handler_class()

class OutputHandler(object):
    def print_match(self, fpath, page, name, match, last = False):
        pass

    def print_header(self, fpath):
        pass

    def print_footer(self, fpath):
        pass

    def print_error(self, fpath, exception):
        traceback.print_exc()
        print("[ERROR] %s" % (exception))

class OutputHandler_stix(OutputHandler):
    # Add all entities to stix indicator files (note: must collate objects before printing - unlike other formats!)
    # - It is noted that this isn't a very stix-ish way of doing things
    # - Expect an analyst to go through the output and re-link associated iocs where necessary
    # - eg: Email addresses are creted as a list of email address objects, should be re-built to email objects if other relevant indicators (such as file attachements) exist
    global stix_package
    global ind_dict
    global add_ind_list
    def print_match(self, fpath, page, name, match):
        if name in ind_dict:
            indicator = ind_dict[name]
            add_ind_list.append(name)
            indicator.title = fpath
            #===========
            # Add new object handlers here:
            if name == 'IP':
                new_obj = Address(address_value=match, category=Address.CAT_IPV4)
            elif name == 'MD5' or name == 'SHA1' or name == 'SHA256':
                new_obj = File()
                new_obj.add_hash(Hash(match))

            elif name == 'URL':
                new_obj = URI(type_=URI.TYPE_URL, value=match)

            elif name == 'Host':
                new_obj = URI(type_=URI.TYPE_DOMAIN, value=match)

            elif name == 'Email':
                new_obj = Address(address_value=match, category=Address.CAT_EMAIL) #Not sure if this is right - should this be using the email_message_object?

            elif name == 'Registry':
                new_obj = WinRegistryKey(values=match)
            #===========

            new_obs = Observable(new_obj)
            new_obs.title = "Page Ref: " + str(page)
            indicator.add_observable(new_obs)

    def print_footer(self, fpath):
        global add_ind_list
        for key in ind_dict:
            if key in add_ind_list:
                stix_package.add_indicator(ind_dict[key])
        print stix_package.to_xml()

class OutputHandler_csv(OutputHandler):
    def __init__(self):
        self.csv_writer = csv.writer(sys.stdout, delimiter = '\t')

    def print_match(self, fpath, page, name, match):
        self.csv_writer.writerow((fpath, page, name, match))

    def print_error(self, fpath, exception):
        self.csv_writer.writerow((fpath, '0', 'error', exception))

class OutputHandler_json(OutputHandler):
    def print_match(self, fpath, page, name, match):
        data = {
            'path' : fpath,
            'file' : os.path.basename(fpath),
            'page' : page,
            'type' : name,
            'match': match
        }

        print(json.dumps(data))

    def print_error(self, fpath, exception):
        data = {
            'path'      : fpath,
            'file'      : os.path.basename(fpath),
            'type'      : 'error',
            'exception' : exception
        }

        print(json.dumps(data))

class OutputHandler_yara(OutputHandler):
    def __init__(self):
        self.rule_enc = ''.join(chr(c) if chr(c).isupper() or chr(c).islower() or chr(c).isdigit() else '_' for c in range(256))

    def print_match(self, fpath, page, name, match):
        if name in self.cnt:
            self.cnt[name] += 1
        else:
            self.cnt[name] = 1

        string_id = "$%s%d" % (name, self.cnt[name])
        self.sids.append(string_id)
        string_value = match.replace('\\', '\\\\')
        print("\t\t%s = \"%s\"" % (string_id, string_value))

    def print_header(self, fpath):
        rule_name = os.path.splitext(os.path.basename(fpath))[0].translate(self.rule_enc)

        print("rule %s" % (rule_name))
        print("{")
        print("\tstrings:")

        self.cnt = {}
        self.sids = []

    def print_footer(self, fpath):
        cond = ' or '.join(self.sids)

        print("\tcondition:")
        print("\t\t" + cond)
        print("}")

class OutputHandler_autofocus(OutputHandler):
    def __init__(self):
        self.rule_enc = ''.join(chr(c) if chr(c).isupper() or chr(c).islower() or chr(c).isdigit() else '_' for c in range(256))

    def print_match(self, fpath, page, name, match):
        string_value = match.replace('hxxp', 'http').replace('\\', '\\\\')

        if name == "MD5":
            auto_focus_query = '{"field":"sample.md5","operator":"is","value":\"%s\"},' % (string_value)
        elif name == "SHA1":
            auto_focus_query = '{"field":"sample.sha1","operator":"is","value":\"%s\"},' % (string_value)
        elif name == "SHA256":
            auto_focus_query = '{"field":"sample.sha256","operator":"is","value":\"%s\"},' % (string_value)
        elif name == "URL":
            auto_focus_query = '{"field":"sample.tasks.connection","operator":"contains","value":\"%s\"},' % (string_value)
        elif name == "Host":
            auto_focus_query = '{"field":"sample.tasks.dns","operator":"contains","value":\"%s\"},' % (string_value)
        elif name == "Registry":
            #auto_focus_query = '{"field":"sample.tasks.registry","operator":"is","value":\"%s\"},' % (string_value)
            return
        elif name == "Filepath":
            #auto_focus_query = '{"field":"sample.tasks.file","operator":"is","value":\"%s\"},' % (string_value)
            return
        elif name == "Filename":
            #auto_focus_query = '{"field":"alias.filename","operator":"is","value":\"%s\"},' % (string_value)
            return
        elif name == "Email":
            #auto_focus_query = '{"field":"alias.email","operator":"is","value":\"%s\"},' % (string_value)
            return
        elif name == "IP":
            auto_focus_query = '{"field":"sample.tasks.connection","operator":"contains","value":\"%s\"},' % (string_value)
        elif name == "CVE":
            return
        print(auto_focus_query)

    def print_header(self, fpath):
        rule_name = os.path.splitext(os.path.basename(fpath))[0].translate(self.rule_enc)

        print("AutoFocus Search for: %s" % (rule_name))
        print('{"operator":"Any","children":[')

    def print_footer(self, fpath):
        rule_name = os.path.splitext(os.path.basename(fpath))[0].translate(self.rule_enc)
        print('{"field":"sample.tag","operator":"is in the list","value":[\"%s\"]}]}' % (rule_name))

<|endoftext|>


<commit_before>'''
warcprox/kafkafeed.py - support for publishing information about archived
urls to apache kafka

Copyright (C) 2015-2016 Internet Archive

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
USA.
'''

import os
import time
import kafka
import datetime
import json
import logging
from hanzo import warctools
import threading
import requests
from io import StringIO
import warcprox
from typing import List

def cdx_line(entry):
    out = StringIO()
    out.write(entry['url'])
    out.write(' ')
    out.write(entry['start_time_plus_duration'][0:14])
    out.write(' ')
    out.write(entry['url'])
    out.write(' ')
    out.write(entry['content_type'])
    out.write(' ')
    out.write(str(entry['status_code']))
    out.write(' ')
    out.write(entry['content_digest'])
    out.write(' - - ')
    out.write(str(entry['warc_length']))
    out.write(' ')
    out.write(str(entry['warc_offset']))
    out.write(' ')
    out.write(entry['warc_filename'])
    out.write('\n')
    line = out.getvalue()
    out.close()
    return line

def to_json(recorded_url: warcprox.warcproxy.RecordedUrl, records: List[warctools.WarcRecord]):
    # {"status_code":200,"content_digest":"sha1:3VU56HI3BTMDZBL2TP7SQYXITT7VEAJQ","host":"www.kaosgl.com","via":"http://www.kaosgl.com/sayfa.php?id=4427","account_id":"877","seed":"http://www.kaosgl.com/","warc_filename":"ARCHIVEIT-6003-WEEKLY-JOB171310-20150903100014694-00002.warc.gz","url":"http://www.kaosgl.com/resim/HomofobiKarsitiBulusma/trabzon05.jpg","size":29700,"start_time_plus_duration":"20150903175709637+1049","timestamp":"2015-09-03T17:57:10.707Z","mimetype":"image/jpeg","collection_id":"6003","is_test_crawl":"false","job_name":"6003-20150902172136074","warc_offset":856320200,"thread":6,"hop_path":"RLLLLLE","extra_info":{},"annotations":"duplicate:digest","content_length":29432}

    # Normal recorded response:
    if recorded_url.response_recorder:
        content_length = recorded_url.response_recorder.len - recorded_url.response_recorder.payload_offset
        payload_digest = warcprox.digest_str(
            recorded_url.payload_digest, True)
    else:
        # WARCPROX_WRITE_RECORD request
        content_length = len(recorded_url.request_data)
        payload_digest = records[0].get_header(b'WARC-Payload-Digest')

    now = datetime.datetime.utcnow()
    d = {
        'url': recorded_url.url.decode('utf-8'),
        'host': recorded_url.host,
        'http_method': recorded_url.method,
        'status_code': recorded_url.status,
        'wire_bytes': recorded_url.size,
        'content_type': recorded_url.mimetype,
        'content_digest': payload_digest.decode("utf-8"),
        'content_length': content_length,
        'start_time_plus_duration': '{:%Y%m%d%H%M%S}{:03d}+{}'.format(
            recorded_url.timestamp, recorded_url.timestamp.microsecond // 1000,
            int(recorded_url.duration.total_seconds() * 1000)),
        'annotations': 'duplicate:digest' if records[0].type == 'revisit' else '',
        'warc_filename': records[0].warc_filename,
        'warc_offset': records[0].offset,
        'warc_length': records[0].length,
        "warc_content_type": records[0].content_type.decode("utf-8"),
        "warc_type": records[0].type.decode("utf-8"),
        "warc_id": records[0].id.decode("utf-8"),
        'timestamp': '{:%Y-%m-%dT%H:%M:%S}.{:03d}Z'.format(now, now.microsecond // 1000)
    }

    # fields expected to be populated here are (for archive-it):
    # account_id, collection_id, is_test_crawl, seed, job_name
    if recorded_url.warcprox_meta and 'capture-feed-extra-fields' in recorded_url.warcprox_meta:
        for (k, v) in recorded_url.warcprox_meta['capture-feed-extra-fields'].items():
            d[k] = v

    print(d)

    return d

class KafkaCaptureFeed:
    logger = logging.getLogger('warcprox-plugins.listeners.CaptureFeed')

    def __init__(self):
        self.broker_list = os.environ["KAFKA_BOOTSTRAP_SERVERS"]
        self.topic = os.environ["KAFKA_CRAWLED_TOPIC"]
        self.acks = int(os.environ.get("KAFKA_ACKS", "0"))
        self.__producer = None
        self._connection_exception = None
        self._lock = threading.RLock()

    def _producer(self):
        with self._lock:
            if not self.__producer:
                try:
                    # acks=0 to avoid ever blocking
                    self.__producer = kafka.KafkaProducer(
                            bootstrap_servers=self.broker_list, acks=self.acks)
                    if self._connection_exception:
                        logging.info('connected to kafka successfully!')
                        self._connection_exception = None
                except Exception as e:
                    if not self._connection_exception:
                        self._connection_exception = e
                        logging.error('problem connecting to kafka', exc_info=1)

        return self.__producer

    def notify(self, recorded_url: warcprox.warcproxy.RecordedUrl, records: List[warctools.WarcRecord]):

        if records[0].type not in (b'revisit', b'response', b'resource'):
            return

        topic = recorded_url.warcprox_meta.get('capture-feed-topic', self.topic)
        if not topic:
            return

        d = to_json(recorded_url,records)

        msg = json.dumps(d, separators=(',', ':')).encode('utf-8')
        self.logger.debug('feeding kafka topic=%r msg=%r', topic, msg)
        p = self._producer()
        if p:
            p.send(topic, msg)
        else:
            self.logger.warning("Could not find producer! Failed to send this message: " + msg)

class UpdateOutbackCDX:
    logger = logging.getLogger('warcprox-plugins.listeners.UpdateOutbackCDX')

    def __init__(self):
        self.endpoint = os.environ["CDXSERVER_ENDPOINT"]
        self.session = requests.Session()

    def notify(self, recorded_url, records):

        if records[0].type not in (b'revisit', b'response', b'resource'):
            return

        # Convert the record to the right form:
        d = to_json(recorded_url,records)
        cdx_11 = cdx_line(d)

        # POST it to the CDX server:
        sent = False
        while not sent:
            r = self.session.post(self.endpoint, data=cdx_11.encode('utf-8'))
            if r.status_code == 200:
                sent = True
                # logger.info("POSTed to cdxserver: %s" % cdx_11)
            else:
                self.logger.error("Failed with %s %s\n%s" % (r.status_code, r.reason, r.text))
                self.logger.error("Failed submission was: %s" % cdx_11.encode('utf-8'))
                time.sleep(10)
<commit_msg>Don't print every line.<commit_after>'''
warcprox/kafkafeed.py - support for publishing information about archived
urls to apache kafka

Copyright (C) 2015-2016 Internet Archive

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
USA.
'''

import os
import time
import kafka
import datetime
import json
import logging
from hanzo import warctools
import threading
import requests
from io import StringIO
import warcprox
from typing import List

def cdx_line(entry):
    out = StringIO()
    out.write(entry['url'])
    out.write(' ')
    out.write(entry['start_time_plus_duration'][0:14])
    out.write(' ')
    out.write(entry['url'])
    out.write(' ')
    out.write(entry['content_type'])
    out.write(' ')
    out.write(str(entry['status_code']))
    out.write(' ')
    out.write(entry['content_digest'])
    out.write(' - - ')
    out.write(str(entry['warc_length']))
    out.write(' ')
    out.write(str(entry['warc_offset']))
    out.write(' ')
    out.write(entry['warc_filename'])
    out.write('\n')
    line = out.getvalue()
    out.close()
    return line

def to_json(recorded_url: warcprox.warcproxy.RecordedUrl, records: List[warctools.WarcRecord]):
    # {"status_code":200,"content_digest":"sha1:3VU56HI3BTMDZBL2TP7SQYXITT7VEAJQ","host":"www.kaosgl.com","via":"http://www.kaosgl.com/sayfa.php?id=4427","account_id":"877","seed":"http://www.kaosgl.com/","warc_filename":"ARCHIVEIT-6003-WEEKLY-JOB171310-20150903100014694-00002.warc.gz","url":"http://www.kaosgl.com/resim/HomofobiKarsitiBulusma/trabzon05.jpg","size":29700,"start_time_plus_duration":"20150903175709637+1049","timestamp":"2015-09-03T17:57:10.707Z","mimetype":"image/jpeg","collection_id":"6003","is_test_crawl":"false","job_name":"6003-20150902172136074","warc_offset":856320200,"thread":6,"hop_path":"RLLLLLE","extra_info":{},"annotations":"duplicate:digest","content_length":29432}

    # Normal recorded response:
    if recorded_url.response_recorder:
        content_length = recorded_url.response_recorder.len - recorded_url.response_recorder.payload_offset
        payload_digest = warcprox.digest_str(
            recorded_url.payload_digest, True)
    else:
        # WARCPROX_WRITE_RECORD request
        content_length = len(recorded_url.request_data)
        payload_digest = records[0].get_header(b'WARC-Payload-Digest')

    now = datetime.datetime.utcnow()
    d = {
        'url': recorded_url.url.decode('utf-8'),
        'host': recorded_url.host,
        'http_method': recorded_url.method,
        'status_code': recorded_url.status,
        'wire_bytes': recorded_url.size,
        'content_type': recorded_url.mimetype,
        'content_digest': payload_digest.decode("utf-8"),
        'content_length': content_length,
        'start_time_plus_duration': '{:%Y%m%d%H%M%S}{:03d}+{}'.format(
            recorded_url.timestamp, recorded_url.timestamp.microsecond // 1000,
            int(recorded_url.duration.total_seconds() * 1000)),
        'annotations': 'duplicate:digest' if records[0].type == 'revisit' else '',
        'warc_filename': records[0].warc_filename,
        'warc_offset': records[0].offset,
        'warc_length': records[0].length,
        "warc_content_type": records[0].content_type.decode("utf-8"),
        "warc_type": records[0].type.decode("utf-8"),
        "warc_id": records[0].id.decode("utf-8"),
        'timestamp': '{:%Y-%m-%dT%H:%M:%S}.{:03d}Z'.format(now, now.microsecond // 1000)
    }

    # fields expected to be populated here are (for archive-it):
    # account_id, collection_id, is_test_crawl, seed, job_name
    if recorded_url.warcprox_meta and 'capture-feed-extra-fields' in recorded_url.warcprox_meta:
        for (k, v) in recorded_url.warcprox_meta['capture-feed-extra-fields'].items():
            d[k] = v

    return d

class KafkaCaptureFeed:
    logger = logging.getLogger('warcprox-plugins.listeners.CaptureFeed')

    def __init__(self):
        self.broker_list = os.environ["KAFKA_BOOTSTRAP_SERVERS"]
        self.topic = os.environ["KAFKA_CRAWLED_TOPIC"]
        self.acks = int(os.environ.get("KAFKA_ACKS", "0"))
        self.__producer = None
        self._connection_exception = None
        self._lock = threading.RLock()

    def _producer(self):
        with self._lock:
            if not self.__producer:
                try:
                    # acks=0 to avoid ever blocking
                    self.__producer = kafka.KafkaProducer(
                            bootstrap_servers=self.broker_list, acks=self.acks)
                    if self._connection_exception:
                        logging.info('connected to kafka successfully!')
                        self._connection_exception = None
                except Exception as e:
                    if not self._connection_exception:
                        self._connection_exception = e
                        logging.error('problem connecting to kafka', exc_info=1)

        return self.__producer

    def notify(self, recorded_url: warcprox.warcproxy.RecordedUrl, records: List[warctools.WarcRecord]):

        if records[0].type not in (b'revisit', b'response', b'resource'):
            return

        topic = recorded_url.warcprox_meta.get('capture-feed-topic', self.topic)
        if not topic:
            return

        d = to_json(recorded_url,records)

        msg = json.dumps(d, separators=(',', ':')).encode('utf-8')
        self.logger.debug('feeding kafka topic=%r msg=%r', topic, msg)
        p = self._producer()
        if p:
            p.send(topic, msg)
        else:
            self.logger.warning("Could not find producer! Failed to send this message: " + msg)

class UpdateOutbackCDX:
    logger = logging.getLogger('warcprox-plugins.listeners.UpdateOutbackCDX')

    def __init__(self):
        self.endpoint = os.environ["CDXSERVER_ENDPOINT"]
        self.session = requests.Session()

    def notify(self, recorded_url, records):

        if records[0].type not in (b'revisit', b'response', b'resource'):
            return

        # Convert the record to the right form:
        d = to_json(recorded_url,records)
        cdx_11 = cdx_line(d)

        # POST it to the CDX server:
        sent = False
        while not sent:
            r = self.session.post(self.endpoint, data=cdx_11.encode('utf-8'))
            if r.status_code == 200:
                sent = True
                # logger.info("POSTed to cdxserver: %s" % cdx_11)
            else:
                self.logger.error("Failed with %s %s\n%s" % (r.status_code, r.reason, r.text))
                self.logger.error("Failed submission was: %s" % cdx_11.encode('utf-8'))
                time.sleep(10)
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to synthesize generated parts of this library."""
import os

import synthtool as s
import synthtool.gcp as gcp
from synthtool.languages import python

common = gcp.CommonTemplates()

default_version = "v1"

for library in s.get_staging_dirs(default_version):
    # rename to google-cloud-os-config
    s.replace([library / "google/**/*.py", library / "tests/**/*.py"],
               "google-cloud-osconfig",
               "google-cloud-os-config")

    # Add newline after last item in list
    s.replace(library / "google/cloud/**/*client.py",
    "(-  Must be unique within the project\.)",
    "\g<1>\n")

    s.move(library, excludes=["nox.py", "setup.py", "README.rst", "docs/index.rst"])

s.remove_staging_dirs()

# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
templated_files = common.py_library(
    samples=False,
    microgenerator=True,
    cov_level=98,
)
python.py_samples(skip_readmes=True)
s.move(
    templated_files, excludes=[".coveragerc"]
)  # the microgenerator has a good coveragerc file

s.shell.run(["nox", "-s", "blacken"], hide_output=False)

# ----------------------------------------------------------------------------
# Main Branch migration
# ----------------------------------------------------------------------------

s.replace(
  "*.rst",
  "master",
  "main"
)

s.replace(
  "*.rst",
  "google-cloud-python/blob/main",
  "google-cloud-python/blob/master"
)

s.replace(
  "CONTRIBUTING.rst",
  "kubernetes/community/blob/main",
  "kubernetes/community/blob/master"
)

s.replace(
  ".kokoro/*",
  "master",
  "main"
)

<commit_msg>chore: revert owlbot main branch templates (#117)<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to synthesize generated parts of this library."""
import os

import synthtool as s
import synthtool.gcp as gcp
from synthtool.languages import python

common = gcp.CommonTemplates()

default_version = "v1"

for library in s.get_staging_dirs(default_version):
    # rename to google-cloud-os-config
    s.replace([library / "google/**/*.py", library / "tests/**/*.py"],
               "google-cloud-osconfig",
               "google-cloud-os-config")

    # Add newline after last item in list
    s.replace(library / "google/cloud/**/*client.py",
    "(-  Must be unique within the project\.)",
    "\g<1>\n")

    s.move(library, excludes=["nox.py", "setup.py", "README.rst", "docs/index.rst"])

s.remove_staging_dirs()

# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
templated_files = common.py_library(
    samples=False,
    microgenerator=True,
    cov_level=98,
)
python.py_samples(skip_readmes=True)
s.move(
    templated_files, excludes=[".coveragerc"]
)  # the microgenerator has a good coveragerc file

s.shell.run(["nox", "-s", "blacken"], hide_output=False)

<|endoftext|>


<commit_before># coding: utf-8
"""
    weasyprint.layout
    -----------------

    Transform a "before layout" box tree into an "after layout" tree.
    (Surprising, hu?)

    Break boxes across lines and pages; determine the size and dimension
    of each box fragement.

    Boxes in the new tree have *used values* in their ``position_x``,
    ``position_y``, ``width`` and ``height`` attributes, amongst others.

    See http://www.w3.org/TR/CSS21/cascade.html#used-value

    :copyright: Copyright 2011-2014 Simon Sapin and contributors, see AUTHORS.
    :license: BSD, see LICENSE for details.

"""
from __future__ import division, unicode_literals
from collections import defaultdict
from ..compat import xrange

from .absolute import absolute_box_layout
from .pages import make_all_pages, make_margin_boxes
from .backgrounds import layout_backgrounds

def layout_fixed_boxes(context, pages):
    """Lay out and yield the fixed boxes of ``pages``."""
    for page in pages:
        for box in page.fixed_boxes:
            # Use an empty list as last argument because the fixed boxes in the
            # fixed box has already been added to page.fixed_boxes, we don't
            # want to get them again
            yield absolute_box_layout(context, box, page, [])

def layout_document(enable_hinting, style_for, get_image_from_uri, root_box,
                    font_config):
    """Lay out the whole document.

    This includes line breaks, page breaks, absolute size and position for all
    boxes.

    :param context: a LayoutContext object.
    :returns: a list of laid out Page objects.

    """
    context = LayoutContext(
        enable_hinting, style_for, get_image_from_uri, font_config)
    pages = list(make_all_pages(context, root_box))
    page_counter = [1]
    counter_values = {'page': page_counter, 'pages': [len(pages)]}
    for i, page in enumerate(pages):
        root_children = []
        root, = page.children
        root_children.extend(layout_fixed_boxes(context, pages[:i]))
        root_children.extend(root.children)
        root_children.extend(layout_fixed_boxes(context, pages[i + 1:]))
        root.children = root_children
        context.current_page = page_counter[0]
        page.children = (root,) + tuple(
            make_margin_boxes(context, page, counter_values))
        layout_backgrounds(page, get_image_from_uri)
        yield page
        page_counter[0] += 1

class LayoutContext(object):
    def __init__(self, enable_hinting, style_for, get_image_from_uri,
                 font_config):
        self.enable_hinting = enable_hinting
        self.style_for = style_for
        self.get_image_from_uri = get_image_from_uri
        self.font_config = font_config
        self._excluded_shapes_lists = []
        self.excluded_shapes = None  # Not initialized yet
        self.string_set = defaultdict(lambda: defaultdict(lambda: list()))
        self.current_page = None
        self.strut_layouts = {}

    def create_block_formatting_context(self):
        self.excluded_shapes = []
        self._excluded_shapes_lists.append(self.excluded_shapes)

    def finish_block_formatting_context(self, root_box):
        # See http://www.w3.org/TR/CSS2/visudet.html#root-height
        if root_box.style.height == 'auto' and self.excluded_shapes:
            box_bottom = root_box.content_box_y() + root_box.height
            max_shape_bottom = max([
                shape.position_y + shape.margin_height()
                for shape in self.excluded_shapes] + [box_bottom])
            root_box.height += max_shape_bottom - box_bottom
        self._excluded_shapes_lists.pop()
        if self._excluded_shapes_lists:
            self.excluded_shapes = self._excluded_shapes_lists[-1]
        else:
            self.excluded_shapes = None

    def get_string_set_for(self, name, keyword=None):
        """Resolve value of string function (as set by string set).

        We'll have something like this that represents all assignments on a
        given page:

        {1: [u'First Header'], 3: [u'Second Header'],
         4: [u'Third Header', u'3.5th Header']}

        Value depends on current page.
        http://dev.w3.org/csswg/css-gcpm/#funcdef-string

        :param name: the name of the named string.
        :param keyword: indicates which value of the named string to use.
                        Default is the first assignment on the current page
                        else the most recent assignment (entry value)
        :returns: text

        """
        last = keyword == "last"
        if self.current_page in self.string_set[name]:
            # a value was assigned on this page
            if keyword == 'first-except':
                # 'first-except' excludes the page it was assinged on
                return ""
            elif last:
                # use the most recent assignment
                return self.string_set[name][self.current_page][-1]
            return self.string_set[name][self.current_page][0]
        else:
            # search backwards through previous pages
            for previous_page in xrange(self.current_page, 0, -1):
                if previous_page in self.string_set[name]:
                    return self.string_set[name][previous_page][-1]
        return ""
<commit_msg>Fix import order<commit_after># coding: utf-8
"""
    weasyprint.layout
    -----------------

    Transform a "before layout" box tree into an "after layout" tree.
    (Surprising, hu?)

    Break boxes across lines and pages; determine the size and dimension
    of each box fragement.

    Boxes in the new tree have *used values* in their ``position_x``,
    ``position_y``, ``width`` and ``height`` attributes, amongst others.

    See http://www.w3.org/TR/CSS21/cascade.html#used-value

    :copyright: Copyright 2011-2014 Simon Sapin and contributors, see AUTHORS.
    :license: BSD, see LICENSE for details.

"""
from __future__ import division, unicode_literals
from collections import defaultdict

from .absolute import absolute_box_layout
from .pages import make_all_pages, make_margin_boxes
from .backgrounds import layout_backgrounds
from ..compat import xrange

def layout_fixed_boxes(context, pages):
    """Lay out and yield the fixed boxes of ``pages``."""
    for page in pages:
        for box in page.fixed_boxes:
            # Use an empty list as last argument because the fixed boxes in the
            # fixed box has already been added to page.fixed_boxes, we don't
            # want to get them again
            yield absolute_box_layout(context, box, page, [])

def layout_document(enable_hinting, style_for, get_image_from_uri, root_box,
                    font_config):
    """Lay out the whole document.

    This includes line breaks, page breaks, absolute size and position for all
    boxes.

    :param context: a LayoutContext object.
    :returns: a list of laid out Page objects.

    """
    context = LayoutContext(
        enable_hinting, style_for, get_image_from_uri, font_config)
    pages = list(make_all_pages(context, root_box))
    page_counter = [1]
    counter_values = {'page': page_counter, 'pages': [len(pages)]}
    for i, page in enumerate(pages):
        root_children = []
        root, = page.children
        root_children.extend(layout_fixed_boxes(context, pages[:i]))
        root_children.extend(root.children)
        root_children.extend(layout_fixed_boxes(context, pages[i + 1:]))
        root.children = root_children
        context.current_page = page_counter[0]
        page.children = (root,) + tuple(
            make_margin_boxes(context, page, counter_values))
        layout_backgrounds(page, get_image_from_uri)
        yield page
        page_counter[0] += 1

class LayoutContext(object):
    def __init__(self, enable_hinting, style_for, get_image_from_uri,
                 font_config):
        self.enable_hinting = enable_hinting
        self.style_for = style_for
        self.get_image_from_uri = get_image_from_uri
        self.font_config = font_config
        self._excluded_shapes_lists = []
        self.excluded_shapes = None  # Not initialized yet
        self.string_set = defaultdict(lambda: defaultdict(lambda: list()))
        self.current_page = None
        self.strut_layouts = {}

    def create_block_formatting_context(self):
        self.excluded_shapes = []
        self._excluded_shapes_lists.append(self.excluded_shapes)

    def finish_block_formatting_context(self, root_box):
        # See http://www.w3.org/TR/CSS2/visudet.html#root-height
        if root_box.style.height == 'auto' and self.excluded_shapes:
            box_bottom = root_box.content_box_y() + root_box.height
            max_shape_bottom = max([
                shape.position_y + shape.margin_height()
                for shape in self.excluded_shapes] + [box_bottom])
            root_box.height += max_shape_bottom - box_bottom
        self._excluded_shapes_lists.pop()
        if self._excluded_shapes_lists:
            self.excluded_shapes = self._excluded_shapes_lists[-1]
        else:
            self.excluded_shapes = None

    def get_string_set_for(self, name, keyword=None):
        """Resolve value of string function (as set by string set).

        We'll have something like this that represents all assignments on a
        given page:

        {1: [u'First Header'], 3: [u'Second Header'],
         4: [u'Third Header', u'3.5th Header']}

        Value depends on current page.
        http://dev.w3.org/csswg/css-gcpm/#funcdef-string

        :param name: the name of the named string.
        :param keyword: indicates which value of the named string to use.
                        Default is the first assignment on the current page
                        else the most recent assignment (entry value)
        :returns: text

        """
        last = keyword == "last"
        if self.current_page in self.string_set[name]:
            # a value was assigned on this page
            if keyword == 'first-except':
                # 'first-except' excludes the page it was assinged on
                return ""
            elif last:
                # use the most recent assignment
                return self.string_set[name][self.current_page][-1]
            return self.string_set[name][self.current_page][0]
        else:
            # search backwards through previous pages
            for previous_page in xrange(self.current_page, 0, -1):
                if previous_page in self.string_set[name]:
                    return self.string_set[name][previous_page][-1]
        return ""
<|endoftext|>