<commit_before>import re
import sys

import ir
import spirv

def output_instruction(stream, module, inst, is_raw_mode, indent='  '):
    """Output one instruction."""
    line = indent
    if inst.result_id is not None:
        result_id = inst.result_id
        if result_id in module.id_to_symbol_name:
            result_id = module.id_to_symbol_name[result_id]
        line = line + result_id + ' = '
    line = line + inst.op_name
    if inst.type_id is not None:
        line = line + ' ' + module.type_id_to_name[inst.type_id]

    if not is_raw_mode:
        line = line + format_decorations_for_inst(module, inst)

    if inst.operands:
        line = line + ' '
        for operand in inst.operands:
            if operand in module.id_to_symbol_name:
                operand = module.id_to_symbol_name[operand]
            if operand in module.type_id_to_name:
                operand = module.type_id_to_name[operand]
            line = line + operand + ', '
        line = line[:-2]

    stream.write(line + '\n')

def get_decorations(module, inst_id):
    decorations = []
    for inst in module.global_insts:
        if inst.op_name == 'OpDecorate' and inst.operands[0] == inst_id:
            decorations.append(inst)
    return decorations

def get_symbol_name(module, symbol_id):
    if symbol_id in module.id_to_symbol_name:
        return module.id_to_symbol_name[symbol_id]

    for inst in module.global_insts:
        if inst.op_name == 'OpName' and inst.operands[0] == symbol_id:
            name = inst.operands[1]
            name = name[1:-1]

            # glslang tend to add type information to function names.
            # E.g. "foo(vec4)" get the symbol name "foo(vf4;"
            # Truncate such names to fit our IL.
            regex = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*')
            match = regex.match(name)
            new_name = match.group(0)
            if new_name != name:
                sys.stderr.write('warning: truncated symbol name "'
                                 + name + '" to "' + new_name + '"\n')

            symbol_name = '%' + new_name
            break
    else:
        symbol_name = '%' + symbol_id[1:]

    module.id_to_symbol_name[symbol_id] = symbol_name

    return symbol_name

def format_decoration(decoration_inst):
    res = decoration_inst.operands[1]
    if decoration_inst.operands[2:]:
        res = res + '('
        for param in decoration_inst.operands[2:]:
            res = res + param + ', '
        res = res[:-2] + ')'
    return res

def format_decorations_for_inst(module, inst):
    line = ''
    decorations = get_decorations(module, inst.result_id)
    for decoration in decorations:
        line = line + ' ' + format_decoration(decoration)
    return line

def add_type_if_needed(module, inst, needed_types):
    if inst.op_name in spirv.TYPE_DECLARATION_INSTRUCTIONS:
        if inst.op_name != 'OpTypeFunction':
            if module.type_id_to_name[inst.result_id] == inst.result_id:
                needed_types.add(inst.result_id)
        for operand in inst.operands:
            if operand[0] == '%':
                type_inst = module.id_to_inst[operand]
                add_type_if_needed(module, type_inst, needed_types)
    if inst.type_id is not None:
        if module.type_id_to_name[inst.type_id] == inst.type_id:
            needed_types.add(inst.type_id)

def get_needed_types(module):
    needed_types = set()
    for inst in module.instructions():
        if inst.op_name not in spirv.TYPE_DECLARATION_INSTRUCTIONS:
            add_type_if_needed(module, inst, needed_types)
    return needed_types

def output_global_instructions(stream, module, is_raw_mode, names, newline=True):
    for inst in module.global_insts:
        if inst.op_name in names:
            if newline:
                stream.write('\n')
                newline = False
            output_instruction(stream, module, inst, is_raw_mode, indent='')

def output_basic_block(stream, module, basic_block):
    """Output one basic block."""
    stream.write(basic_block.inst.result_id + ':\n')
    for inst in basic_block.insts:
        output_instruction(stream, module, inst, False)

def output_function_raw(stream, module, func):
    """Output one function (raw mode)."""
    stream.write('\n')
    noindent_names = ['OpFunction', 'OpLabel', 'OpFunctionParameter',
                      'OpFunctionEnd']
    for inst in func.instructions():
        if inst.op_name in noindent_names:
            indent = ''
        else:
            indent = '  '
        output_instruction(stream, module, inst, True, indent=indent)

def output_function(stream, module, func):
    """Output one function (pretty-printed mode)."""
    stream.write('\n')
    symbol_name = get_symbol_name(module, func.inst.result_id)
    line = 'define ' + module.type_id_to_name[func.inst.type_id] + ' '
    line = line + symbol_name + '('
    for inst in func.arguments:
        line = line + module.type_id_to_name[inst.type_id]
        line = line + ' ' + inst.result_id + ', '
    if line[-2:] == ', ':
        line = line[:-2]
    line = line + ') {\n'
    stream.write(line)

    for basic_block in func.basic_blocks:
        if basic_block != func.basic_blocks[0]:
            stream.write('\n')
        output_basic_block(stream, module, basic_block)

    stream.write('}\n')

def output_functions(stream, module, is_raw_mode):
    """Output all functions."""
    for func in module.functions:
        if is_raw_mode:
            output_function_raw(stream, module, func)
        else:
            output_function(stream, module, func)

def generate_global_symbols(module):
    """Add function/global varible names to the symbol table."""
    for func in module.functions:
        get_symbol_name(module, func.inst.result_id)
    for inst in module.global_insts:
        if inst.op_name == 'OpVariable':
            get_symbol_name(module, inst.result_id)

def add_type_name(module, inst):
    if inst.op_name == 'OpTypeVoid':
        type_name = 'void'
    elif inst.op_name == 'OpTypeBool':
        type_name = 'bool'
    elif inst.op_name == 'OpTypeInt':
        width = inst.operands[0]
        if width not in ['8', '16', '32', '64']:
            raise ir.IRError("Invalid OpTypeInt width " + width)
        signedness = inst.operands[1]
        if not signedness in ['0', '1']:
            error = "Invalid OpTypeInt signedness " + str(signedness)
            raise ir.IRError(error)
        type_name = 's' if signedness else 'u'
        type_name = type_name + width
    elif inst.op_name == 'OpTypeFloat':
        width = inst.operands[0]
        if width not in ['16', '32', '64']:
            raise ir.IRError("Invalid OpTypeFloat width " + width)
        type_name = 'f' + width
    elif inst.op_name == 'OpTypeVector':
        component_type = module.type_id_to_name[inst.operands[0]]
        count = inst.operands[1]
        if int(count) not in range(2, 16):
            error = "Invalid OpTypeVector component count " + str(count)
            raise ir.IRError(error)
        type_name = '<' + str(count) + ' x ' + component_type + '>'
    else:
        type_name = inst.result_id

    module.type_id_to_name[inst.result_id] = type_name

def add_type_names(module):
    for inst in module.global_insts:
        if inst.op_name in spirv.TYPE_DECLARATION_INSTRUCTIONS:
            add_type_name(module, inst)

def write_module(stream, module, is_raw_mode):
    module.id_to_symbol_name = {}
    module.type_id_to_name = {}

    add_type_names(module)
    if not is_raw_mode:
        generate_global_symbols(module)

    for name in spirv.INITIAL_INSTRUCTIONS:
        if name != 'OpExtInstImport':
            output_global_instructions(stream, module, is_raw_mode, [name],
                                       newline=False)
    output_global_instructions(stream, module, is_raw_mode,
                               ['OpExtInstImport'])

    if is_raw_mode:
        output_global_instructions(stream, module, is_raw_mode,
                                   spirv.DEBUG_INSTRUCTIONS)
        output_global_instructions(stream, module, is_raw_mode,
                                   spirv.DECORATION_INSTRUCTIONS)
        output_global_instructions(stream, module, is_raw_mode,
                                   spirv.TYPE_DECLARATION_INSTRUCTIONS)
    else:
        needed_types = get_needed_types(module)
        if needed_types:
            stream.write('\n')
            for inst in module.global_insts:
                if inst.result_id in needed_types:
                    output_instruction(stream, module, inst, is_raw_mode,
                                       indent='')

    output_global_instructions(stream, module, is_raw_mode,
                               spirv.CONSTANT_INSTRUCTIONS)
    output_global_instructions(stream, module, is_raw_mode,
                               spirv.GLOBAL_VARIABLE_INSTRUCTIONS)
    output_functions(stream, module, is_raw_mode)

    del module.type_id_to_name
    del module.id_to_symbol_name
<commit_msg>Make is_raw_mode=False default in write_module()<commit_after>import re
import sys

import ir
import spirv

def output_instruction(stream, module, inst, is_raw_mode, indent='  '):
    """Output one instruction."""
    line = indent
    if inst.result_id is not None:
        result_id = inst.result_id
        if result_id in module.id_to_symbol_name:
            result_id = module.id_to_symbol_name[result_id]
        line = line + result_id + ' = '
    line = line + inst.op_name
    if inst.type_id is not None:
        line = line + ' ' + module.type_id_to_name[inst.type_id]

    if not is_raw_mode:
        line = line + format_decorations_for_inst(module, inst)

    if inst.operands:
        line = line + ' '
        for operand in inst.operands:
            if operand in module.id_to_symbol_name:
                operand = module.id_to_symbol_name[operand]
            if operand in module.type_id_to_name:
                operand = module.type_id_to_name[operand]
            line = line + operand + ', '
        line = line[:-2]

    stream.write(line + '\n')

def get_decorations(module, inst_id):
    decorations = []
    for inst in module.global_insts:
        if inst.op_name == 'OpDecorate' and inst.operands[0] == inst_id:
            decorations.append(inst)
    return decorations

def get_symbol_name(module, symbol_id):
    if symbol_id in module.id_to_symbol_name:
        return module.id_to_symbol_name[symbol_id]

    for inst in module.global_insts:
        if inst.op_name == 'OpName' and inst.operands[0] == symbol_id:
            name = inst.operands[1]
            name = name[1:-1]

            # glslang tend to add type information to function names.
            # E.g. "foo(vec4)" get the symbol name "foo(vf4;"
            # Truncate such names to fit our IL.
            regex = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*')
            match = regex.match(name)
            new_name = match.group(0)
            if new_name != name:
                sys.stderr.write('warning: truncated symbol name "'
                                 + name + '" to "' + new_name + '"\n')

            symbol_name = '%' + new_name
            break
    else:
        symbol_name = '%' + symbol_id[1:]

    module.id_to_symbol_name[symbol_id] = symbol_name

    return symbol_name

def format_decoration(decoration_inst):
    res = decoration_inst.operands[1]
    if decoration_inst.operands[2:]:
        res = res + '('
        for param in decoration_inst.operands[2:]:
            res = res + param + ', '
        res = res[:-2] + ')'
    return res

def format_decorations_for_inst(module, inst):
    line = ''
    decorations = get_decorations(module, inst.result_id)
    for decoration in decorations:
        line = line + ' ' + format_decoration(decoration)
    return line

def add_type_if_needed(module, inst, needed_types):
    if inst.op_name in spirv.TYPE_DECLARATION_INSTRUCTIONS:
        if inst.op_name != 'OpTypeFunction':
            if module.type_id_to_name[inst.result_id] == inst.result_id:
                needed_types.add(inst.result_id)
        for operand in inst.operands:
            if operand[0] == '%':
                type_inst = module.id_to_inst[operand]
                add_type_if_needed(module, type_inst, needed_types)
    if inst.type_id is not None:
        if module.type_id_to_name[inst.type_id] == inst.type_id:
            needed_types.add(inst.type_id)

def get_needed_types(module):
    needed_types = set()
    for inst in module.instructions():
        if inst.op_name not in spirv.TYPE_DECLARATION_INSTRUCTIONS:
            add_type_if_needed(module, inst, needed_types)
    return needed_types

def output_global_instructions(stream, module, is_raw_mode, names, newline=True):
    for inst in module.global_insts:
        if inst.op_name in names:
            if newline:
                stream.write('\n')
                newline = False
            output_instruction(stream, module, inst, is_raw_mode, indent='')

def output_basic_block(stream, module, basic_block):
    """Output one basic block."""
    stream.write(basic_block.inst.result_id + ':\n')
    for inst in basic_block.insts:
        output_instruction(stream, module, inst, False)

def output_function_raw(stream, module, func):
    """Output one function (raw mode)."""
    stream.write('\n')
    noindent_names = ['OpFunction', 'OpLabel', 'OpFunctionParameter',
                      'OpFunctionEnd']
    for inst in func.instructions():
        if inst.op_name in noindent_names:
            indent = ''
        else:
            indent = '  '
        output_instruction(stream, module, inst, True, indent=indent)

def output_function(stream, module, func):
    """Output one function (pretty-printed mode)."""
    stream.write('\n')
    symbol_name = get_symbol_name(module, func.inst.result_id)
    line = 'define ' + module.type_id_to_name[func.inst.type_id] + ' '
    line = line + symbol_name + '('
    for inst in func.arguments:
        line = line + module.type_id_to_name[inst.type_id]
        line = line + ' ' + inst.result_id + ', '
    if line[-2:] == ', ':
        line = line[:-2]
    line = line + ') {\n'
    stream.write(line)

    for basic_block in func.basic_blocks:
        if basic_block != func.basic_blocks[0]:
            stream.write('\n')
        output_basic_block(stream, module, basic_block)

    stream.write('}\n')

def output_functions(stream, module, is_raw_mode):
    """Output all functions."""
    for func in module.functions:
        if is_raw_mode:
            output_function_raw(stream, module, func)
        else:
            output_function(stream, module, func)

def generate_global_symbols(module):
    """Add function/global varible names to the symbol table."""
    for func in module.functions:
        get_symbol_name(module, func.inst.result_id)
    for inst in module.global_insts:
        if inst.op_name == 'OpVariable':
            get_symbol_name(module, inst.result_id)

def add_type_name(module, inst):
    if inst.op_name == 'OpTypeVoid':
        type_name = 'void'
    elif inst.op_name == 'OpTypeBool':
        type_name = 'bool'
    elif inst.op_name == 'OpTypeInt':
        width = inst.operands[0]
        if width not in ['8', '16', '32', '64']:
            raise ir.IRError("Invalid OpTypeInt width " + width)
        signedness = inst.operands[1]
        if not signedness in ['0', '1']:
            error = "Invalid OpTypeInt signedness " + str(signedness)
            raise ir.IRError(error)
        type_name = 's' if signedness else 'u'
        type_name = type_name + width
    elif inst.op_name == 'OpTypeFloat':
        width = inst.operands[0]
        if width not in ['16', '32', '64']:
            raise ir.IRError("Invalid OpTypeFloat width " + width)
        type_name = 'f' + width
    elif inst.op_name == 'OpTypeVector':
        component_type = module.type_id_to_name[inst.operands[0]]
        count = inst.operands[1]
        if int(count) not in range(2, 16):
            error = "Invalid OpTypeVector component count " + str(count)
            raise ir.IRError(error)
        type_name = '<' + str(count) + ' x ' + component_type + '>'
    else:
        type_name = inst.result_id

    module.type_id_to_name[inst.result_id] = type_name

def add_type_names(module):
    for inst in module.global_insts:
        if inst.op_name in spirv.TYPE_DECLARATION_INSTRUCTIONS:
            add_type_name(module, inst)

def write_module(stream, module, is_raw_mode=False):
    module.id_to_symbol_name = {}
    module.type_id_to_name = {}

    add_type_names(module)
    if not is_raw_mode:
        generate_global_symbols(module)

    for name in spirv.INITIAL_INSTRUCTIONS:
        if name != 'OpExtInstImport':
            output_global_instructions(stream, module, is_raw_mode, [name],
                                       newline=False)
    output_global_instructions(stream, module, is_raw_mode,
                               ['OpExtInstImport'])

    if is_raw_mode:
        output_global_instructions(stream, module, is_raw_mode,
                                   spirv.DEBUG_INSTRUCTIONS)
        output_global_instructions(stream, module, is_raw_mode,
                                   spirv.DECORATION_INSTRUCTIONS)
        output_global_instructions(stream, module, is_raw_mode,
                                   spirv.TYPE_DECLARATION_INSTRUCTIONS)
    else:
        needed_types = get_needed_types(module)
        if needed_types:
            stream.write('\n')
            for inst in module.global_insts:
                if inst.result_id in needed_types:
                    output_instruction(stream, module, inst, is_raw_mode,
                                       indent='')

    output_global_instructions(stream, module, is_raw_mode,
                               spirv.CONSTANT_INSTRUCTIONS)
    output_global_instructions(stream, module, is_raw_mode,
                               spirv.GLOBAL_VARIABLE_INSTRUCTIONS)
    output_functions(stream, module, is_raw_mode)

    del module.type_id_to_name
    del module.id_to_symbol_name
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import requests
import threading
import time
import curses
import argparse
import datetime
import yaml

__PRODUCT_ID = "Web URL Fuzzer 1.0 (c) 4k1/wufuzzer"

fuzzdb_dirs  = ["/"]
fuzzdb_files = []

stdscr       = None
confyml      = None

class Logging():

    def __init__(self):
        self.__lock = threading.Lock()
        self.__fn   = datetime.datetime.today().strftime("%Y%m%d_%H%M%S_#")

    def get_basename(self):
        return self.__fn

    def log(self, tid, value):
        try:
            f = open(self.__fn + str(tid) + ".log", "a")
            f.write(value + '\r\n')
            f.close()
        except:
            None

    def warn(self, value):
        with self.__lock:
            try:
                f = open(self.__fn + "!.log", "a")
                f.write(value + '\r\n')
                f.close()
            except:
                None

def vsplit(iterable, n):
    return [iterable[x:x + n] for x in range(0, len(iterable), n)]

def load_config():
    confs = []
    try:
        f = open("wufuzzer.yml")
        global confyml
        confyml = yaml.load(f)
        f.close()
    except:
        print ("[-] Missing config to read 'wufuzzer.yml'.")
        exit(-1)

def get_config(key, default=None):
    if key in confyml["core"]:
        if type(confyml["core"][key]) is int:
            return int(confyml["core"][key])
        else:
            return str(confyml["core"][key])
    elif default == None:
        raise
    else:
        return default

def passed(p):
    if p == "/":
        return ""
    if p[0:1] == "/":
        p = p[1:]
    if p[-1:] != "/":
        return p + "/"
    return p

def filed(p):
    if p[0:1] == "/":
        p = p[1:]
    return p

def fuzzdb():
    global fuzzdb_dirs
    global fuzzdb_files

    default_db = ""
    try:
        default_db = confyml["core"]["default_db"]
    except:
        print ("[-] Missing config 'core.default_db'.")
        exit(-1)

    base = ""
    try:
        base = passed(confyml[default_db]["base"])
    except:
        print ("[*] Missing config '" + default_db + ".base'.")
        print ("[*] No default database.")
        return

    for deffile in confyml[default_db]["files"]:
        try:
            if deffile["option"] not in ["dirs","fixed"]:
                print ("[-] Unknown option '" + deffile["option"] + "'.")
                raise()
            if deffile["type"] not in ["dironly","mixed"]:
                print ("[-] Unknown type '" + deffile["type"] + "'.")
                raise()

            f = open("/" + base + deffile["file"])
            dbrecords = f.readlines()
            f.close()

            for row in dbrecords:
                e = row.strip()

                if (deffile["type"] == "dironly"):
                    fuzzdb_dirs.append(passed(e))
                elif(deffile["type"] == "mixed"):
                        if (e[-1:] == "/"):
                            fuzzdb_dirs.append(passed(e))
                        else:
                            fuzzdb_files.append(e)
                        else:
                            fuzzdb_dirs.append(e)
                else:
                    print ("[-] Mismatch options.")
                    raise()

        except:
            print ("[*] Error has occurred to read '" + "/" + base + deffile["file"] + "'. Skipped.")

    fuzzdb_dirs = list(set(fuzzdb_dirs))
    fuzzdb_files = list(set(fuzzdb_files))

class FuzzerRunner(threading.Thread):

    __target    = ""
    __dirs      = []
    __files     = []
    __stop      = False
    __stopped   = True
    __tid       = -1
    __status    = ""
    __log       = ""
    __logger    = None
    __nowcnt    = 0

    def __init__(self, dirs, files, tid, target, logger):
        super(FuzzerRunner, self).__init__()
        self.__dirs     = dirs
        self.__files    = files
        self.__tid      = tid
        self.__target   = target
        self.__logger   = logger

    def get_status(self):
        return self.__status

    def get_log(self):
        return self.__log

    def get_stopped(self):
        return self.__stopped

    def get_max(self):
        return len(self.__dirs) * len(self.__files) + len(self.__dirs)

    def get_now(self):
        return self.__nowcnt

    def stop(self):
        self.__stop = True

    def __scan(self, cpath):
        err_times = 0
        while True:
            if (self.__stop):
                self.__status = "[PAUSE]"
                self.__stopped = True
                return False     # Abort

            try:
                res = requests.head(passed(self.__target) + cpath, allow_redirects=False, headers={'Connection':'close'})
                rcode = res.status_code
                res.connection.close()

                self.__log = "#" + str(self.__tid) + " " + str(rcode) + " '" + passed(self.__target) + cpath + "'"
                if str(rcode) not in str(get_config("except_http_codes", 404)):
                    self.__logger.warn(self.__log)
                else:
                    self.__logger.log(self.__tid, self.__log)

                time.sleep(get_config("request_interval", 0))
                break

            except:
                err_times += 1
                self.__log = "Warning : Request failed (" + str(err_times) + " time(s).). '" + passed(self.__target) + cpath + "' - waiting " + str(get_config("retry_interval", 10)) + " sec..."
                self.__logger.log(self.__tid, self.__log)
                time.sleep(get_config("retry_interval", 10))

        return True     # Continue

    def run(self):
        self.__stop     = False
        self.__stopped  = False
        self.__status   = "[ RUN ]"
        self.__nowcnt   = 0

        for dir in sorted(self.__dirs):
            cpath = passed(dir)
            if not self.__scan(cpath):
                return
            self.__nowcnt += 1

            for file in sorted(self.__files):
                cpath = passed(dir) + filed(file)
                if not self.__scan(cpath):
                    return
                self.__nowcnt += 1

        self.__status = "[FINSH]"
        self.__stopped = True

def output_status(cnt, th):
    for i in range(0, cnt):

        (maxy, maxx) = stdscr.getmaxyx()

        if maxy > i + 4:
            stdscr.addstr(i + 2, 0, th[i].get_status() + " " + th[i].get_log(), curses.A_DIM)
            stdscr.clrtoeol()

        indic = "( " + str(th[i].get_now()) + " / " + str(th[i].get_max()) + " )"

        if maxy > i + 4:
            stdscr.addstr(i + 2, maxx - len(indic), indic, curses.A_DIM)

        stdscr.refresh()

def fuzzer(target, logger):

    cnt = int(get_config('max_threads', 1))

    scnt_dir = int(len(fuzzdb_dirs) / cnt) + 1
    sdirs = vsplit(fuzzdb_dirs, scnt_dir)
    cnt = len(sdirs)

    th = {}

    for i in range(0, cnt):
        th[i] = FuzzerRunner(sdirs[i], fuzzdb_files, i, target, logger)
        th[i].start()

    output_status(cnt, th)

    while True:
        stdscr.timeout(1)
        try:
            c = stdscr.getch()
            if c == ord('p'):
                for i in range(0, cnt):
                    th[i].stop()
            elif c == ord('q'):
                all_stopped = True
                for i in range(0, cnt):
                    all_stopped = all_stopped and th[i].get_stopped()
                if (all_stopped):
                    break
                else:
                    for i in range(0, cnt):
                        th[i].stop()
        except:
            None
        output_status(cnt, th)
        time.sleep(0.1)

if __name__ == '__main__':

    print (__PRODUCT_ID)
    parser = argparse.ArgumentParser()
    parser.add_argument("target")
    args = parser.parse_args()

    load_config()
    print ("[+] Loaded config.")
    fuzzdb()
    print ("[+] Loaded databases. ")
    print ("    " + str(len(fuzzdb_dirs)) + " dir(s), " + str(len(fuzzdb_files)) + " file(s).")
    logger = Logging()
    print ("[+] Logging started. basename=./" + logger.get_basename() )

    stdscr = curses.initscr()
    (maxy, maxx) = stdscr.getmaxyx()
    curses.noecho()
    curses.cbreak()

    stdscr.addstr(0, 0, __PRODUCT_ID, curses.A_DIM)
    stdscr.refresh()
    stdscr.addstr(maxy-1,  0, "q",     curses.A_REVERSE)
    stdscr.addstr(maxy-1,  2, "Quit",  curses.A_DIM)
    stdscr.addstr(maxy-1, 10, "p",     curses.A_REVERSE)
    stdscr.addstr(maxy-1, 12, "Pause", curses.A_DIM)
    stdscr.refresh()

    try:
        fuzzer(args.target, logger)
    finally:
        curses.nocbreak()
        stdscr.keypad(0)
        curses.echo()
        curses.endwin()
<commit_msg>Avoided using reserved words for variable names<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import requests
import threading
import time
import curses
import argparse
import datetime
import yaml

__PRODUCT_ID = "Web URL Fuzzer 1.0 (c) 4k1/wufuzzer"

fuzzdb_dirs  = ["/"]
fuzzdb_files = []

stdscr       = None
confyml      = None

class Logging():

    def __init__(self):
        self.__lock = threading.Lock()
        self.__fn   = datetime.datetime.today().strftime("%Y%m%d_%H%M%S_#")

    def get_basename(self):
        return self.__fn

    def log(self, tid, value):
        try:
            f = open(self.__fn + str(tid) + ".log", "a")
            f.write(value + '\r\n')
            f.close()
        except:
            None

    def warn(self, value):
        with self.__lock:
            try:
                f = open(self.__fn + "!.log", "a")
                f.write(value + '\r\n')
                f.close()
            except:
                None

def vsplit(iterable, n):
    return [iterable[x:x + n] for x in range(0, len(iterable), n)]

def load_config():
    confs = []
    try:
        f = open("wufuzzer.yml")
        global confyml
        confyml = yaml.load(f)
        f.close()
    except:
        print ("[-] Missing config to read 'wufuzzer.yml'.")
        exit(-1)

def get_config(key, default=None):
    if key in confyml["core"]:
        if type(confyml["core"][key]) is int:
            return int(confyml["core"][key])
        else:
            return str(confyml["core"][key])
    elif default == None:
        raise
    else:
        return default

def passed(p):
    if p == "/":
        return ""
    if p[0:1] == "/":
        p = p[1:]
    if p[-1:] != "/":
        return p + "/"
    return p

def filed(p):
    if p[0:1] == "/":
        p = p[1:]
    return p

def fuzzdb():
    global fuzzdb_dirs
    global fuzzdb_files

    default_db = ""
    try:
        default_db = confyml["core"]["default_db"]
    except:
        print ("[-] Missing config 'core.default_db'.")
        exit(-1)

    base = ""
    try:
        base = passed(confyml[default_db]["base"])
    except:
        print ("[*] Missing config '" + default_db + ".base'.")
        print ("[*] No default database.")
        return

    for deffile in confyml[default_db]["files"]:
        try:
            if deffile["option"] not in ["dirs","fixed"]:
                print ("[-] Unknown option '" + deffile["option"] + "'.")
                raise()
            if deffile["type"] not in ["dironly","mixed"]:
                print ("[-] Unknown type '" + deffile["type"] + "'.")
                raise()

            f = open("/" + base + deffile["file"])
            dbrecords = f.readlines()
            f.close()

            for row in dbrecords:
                e = row.strip()

                if (deffile["type"] == "dironly"):
                    fuzzdb_dirs.append(passed(e))
                elif(deffile["type"] == "mixed"):
                        if (e[-1:] == "/"):
                            fuzzdb_dirs.append(passed(e))
                        else:
                            fuzzdb_files.append(e)
                        else:
                            fuzzdb_dirs.append(e)
                else:
                    print ("[-] Mismatch options.")
                    raise()

        except:
            print ("[*] Error has occurred to read '" + "/" + base + deffile["file"] + "'. Skipped.")

    fuzzdb_dirs = list(set(fuzzdb_dirs))
    fuzzdb_files = list(set(fuzzdb_files))

class FuzzerRunner(threading.Thread):

    __target    = ""
    __dirs      = []
    __files     = []
    __stop      = False
    __stopped   = True
    __tid       = -1
    __status    = ""
    __log       = ""
    __logger    = None
    __nowcnt    = 0

    def __init__(self, dirs, files, tid, target, logger):
        super(FuzzerRunner, self).__init__()
        self.__dirs     = dirs
        self.__files    = files
        self.__tid      = tid
        self.__target   = target
        self.__logger   = logger

    def get_status(self):
        return self.__status

    def get_log(self):
        return self.__log

    def get_stopped(self):
        return self.__stopped

    def get_max(self):
        return len(self.__dirs) * len(self.__files) + len(self.__dirs)

    def get_now(self):
        return self.__nowcnt

    def stop(self):
        self.__stop = True

    def __scan(self, cpath):
        err_times = 0
        while True:
            if (self.__stop):
                self.__status = "[PAUSE]"
                self.__stopped = True
                return False     # Abort

            try:
                res = requests.head(passed(self.__target) + cpath, allow_redirects=False, headers={'Connection':'close'})
                rcode = res.status_code
                res.connection.close()

                self.__log = "#" + str(self.__tid) + " " + str(rcode) + " '" + passed(self.__target) + cpath + "'"
                if str(rcode) not in str(get_config("except_http_codes", 404)):
                    self.__logger.warn(self.__log)
                else:
                    self.__logger.log(self.__tid, self.__log)

                time.sleep(get_config("request_interval", 0))
                break

            except:
                err_times += 1
                self.__log = "Warning : Request failed (" + str(err_times) + " time(s).). '" + passed(self.__target) + cpath + "' - waiting " + str(get_config("retry_interval", 10)) + " sec..."
                self.__logger.log(self.__tid, self.__log)
                time.sleep(get_config("retry_interval", 10))

        return True     # Continue

    def run(self):
        self.__stop     = False
        self.__stopped  = False
        self.__status   = "[ RUN ]"
        self.__nowcnt   = 0

        for dv in sorted(self.__dirs):
            cpath = passed(dv)
            if not self.__scan(cpath):
                return
            self.__nowcnt += 1

            for fv in sorted(self.__files):
                cpath = passed(dv) + filed(fv)
                if not self.__scan(cpath):
                    return
                self.__nowcnt += 1

        self.__status = "[FINSH]"
        self.__stopped = True

def output_status(cnt, th):
    for i in range(0, cnt):

        (maxy, maxx) = stdscr.getmaxyx()

        if maxy > i + 4:
            stdscr.addstr(i + 2, 0, th[i].get_status() + " " + th[i].get_log(), curses.A_DIM)
            stdscr.clrtoeol()

        indic = "( " + str(th[i].get_now()) + " / " + str(th[i].get_max()) + " )"

        if maxy > i + 4:
            stdscr.addstr(i + 2, maxx - len(indic), indic, curses.A_DIM)

        stdscr.refresh()

def fuzzer(target, logger):

    cnt = int(get_config('max_threads', 1))

    scnt_dir = int(len(fuzzdb_dirs) / cnt) + 1
    sdirs = vsplit(fuzzdb_dirs, scnt_dir)
    cnt = len(sdirs)

    th = {}

    for i in range(0, cnt):
        th[i] = FuzzerRunner(sdirs[i], fuzzdb_files, i, target, logger)
        th[i].start()

    output_status(cnt, th)

    while True:
        stdscr.timeout(1)
        try:
            c = stdscr.getch()
            if c == ord('p'):
                for i in range(0, cnt):
                    th[i].stop()
            elif c == ord('q'):
                all_stopped = True
                for i in range(0, cnt):
                    all_stopped = all_stopped and th[i].get_stopped()
                if (all_stopped):
                    break
                else:
                    for i in range(0, cnt):
                        th[i].stop()
        except:
            None
        output_status(cnt, th)
        time.sleep(0.1)

if __name__ == '__main__':

    print (__PRODUCT_ID)
    parser = argparse.ArgumentParser()
    parser.add_argument("target")
    args = parser.parse_args()

    load_config()
    print ("[+] Loaded config.")
    fuzzdb()
    print ("[+] Loaded databases. ")
    print ("    " + str(len(fuzzdb_dirs)) + " dir(s), " + str(len(fuzzdb_files)) + " file(s).")
    logger = Logging()
    print ("[+] Logging started. basename=./" + logger.get_basename() )

    stdscr = curses.initscr()
    (maxy, maxx) = stdscr.getmaxyx()
    curses.noecho()
    curses.cbreak()

    stdscr.addstr(0, 0, __PRODUCT_ID, curses.A_DIM)
    stdscr.refresh()
    stdscr.addstr(maxy-1,  0, "q",     curses.A_REVERSE)
    stdscr.addstr(maxy-1,  2, "Quit",  curses.A_DIM)
    stdscr.addstr(maxy-1, 10, "p",     curses.A_REVERSE)
    stdscr.addstr(maxy-1, 12, "Pause", curses.A_DIM)
    stdscr.refresh()

    try:
        fuzzer(args.target, logger)
    finally:
        curses.nocbreak()
        stdscr.keypad(0)
        curses.echo()
        curses.endwin()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Phil Adams http://philadams.net

Python wrapper around the Habitica (http://habitica.com) API
http://github.com/philadams/habitica
"""

import json

import requests

API_URI_BASE = 'api/v3'
API_CONTENT_TYPE = 'application/json'

class Habitica(object):
    """
    A minimalist Habitica API class.
    """

    def __init__(self, auth=None, resource=None, aspect=None):
        self.auth = auth
        self.resource = resource
        self.aspect = aspect
        self.headers = auth if auth else {}
        self.headers.update({'content-type': API_CONTENT_TYPE})

    def __getattr__(self, m):
        try:
            return object.__getattr__(self, m)
        except AttributeError:
            if not self.resource:
                return Habitica(auth=self.auth, resource=m)
            else:
                return Habitica(auth=self.auth, resource=self.resource,
                                aspect=m)

    def __call__(self, **kwargs):
        method = kwargs.pop('_method', 'get')

        # build up URL... Habitica's api is the *teeniest* bit annoying
        # so either i need to find a cleaner way here, or i should
        # get involved in the API itself and... help it.
        if self.aspect:
            aspect_id = kwargs.pop('_id', None)
            direction = kwargs.pop('_direction', None)
            uri = '%s/%s' % (self.auth['url'],
                             API_URI_BASE)
            if aspect_id is not None:
                uri = '%s/%s/%s' % (uri,
                                    self.aspect,
                                    str(aspect_id))
            elif self.aspect == 'tasks':
                uri = '%s/%s/%s' % (uri,
                                    self.aspect,
                                    self.resource)
            else:
                uri = '%s/%s/%s' % (uri,
                                    self.resource,
                                    self.aspect)
            if direction is not None:
                uri = '%s/score/%s' % (uri, direction)
        else:
            uri = '%s/%s/%s' % (self.auth['url'],
                                API_URI_BASE,
                                self.resource)

        # actually make the request of the API
        if method in ['put', 'post']:
            res = getattr(requests, method)(uri, headers=self.headers,
                                            data=json.dumps(kwargs))
        else:
            res = getattr(requests, method)(uri, headers=self.headers,
                                            params=kwargs)

        # print(res.url)  # debug...
        if res.status_code == requests.codes.ok:
            return res.json()["data"]
        else:
            res.raise_for_status()
<commit_msg>Fix party quest 404 when run `habitica status` command<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Phil Adams http://philadams.net

Python wrapper around the Habitica (http://habitica.com) API
http://github.com/philadams/habitica
"""

import json

import requests

API_URI_BASE = 'api/v3'
API_CONTENT_TYPE = 'application/json'

class Habitica(object):
    """
    A minimalist Habitica API class.
    """

    def __init__(self, auth=None, resource=None, aspect=None):
        self.auth = auth
        self.resource = resource
        self.aspect = aspect
        self.headers = auth if auth else {}
        self.headers.update({'content-type': API_CONTENT_TYPE})

    def __getattr__(self, m):
        try:
            return object.__getattr__(self, m)
        except AttributeError:
            if not self.resource:
                return Habitica(auth=self.auth, resource=m)
            else:
                return Habitica(auth=self.auth, resource=self.resource,
                                aspect=m)

    def __call__(self, **kwargs):
        method = kwargs.pop('_method', 'get')

        # build up URL... Habitica's api is the *teeniest* bit annoying
        # so either i need to find a cleaner way here, or i should
        # get involved in the API itself and... help it.
        if self.aspect:
            aspect_id = kwargs.pop('_id', None)
            direction = kwargs.pop('_direction', None)
            uri = '%s/%s' % (self.auth['url'],
                             API_URI_BASE)
            if aspect_id is not None:
                uri = '%s/%s/%s' % (uri,
                                    self.aspect,
                                    str(aspect_id))
            elif self.aspect == 'tasks':
                uri = '%s/%s/%s' % (uri,
                                    self.aspect,
                                    self.resource)
            else:
                uri = '%s/%s/%s' % (uri,
                                    self.resource,
                                    self.aspect)
            if direction is not None:
                uri = '%s/score/%s' % (uri, direction)
        else:
            uri = '%s/%s/%s' % (self.auth['url'],
                                API_URI_BASE,
                                self.resource)

        # actually make the request of the API
        if method in ['put', 'post']:
            res = getattr(requests, method)(uri, headers=self.headers,
                                            data=json.dumps(kwargs))
        else:
            res = getattr(requests, method)(uri, headers=self.headers,
                                            params=kwargs)

        # print(res.url)  # debug...
        if res.status_code == requests.codes.ok:
            return res.json()["data"]
        elif res.status_code == 404:
            return None
        else:
            res.raise_for_status()
<|endoftext|>


<commit_before>#
# Copyright (c) 2013 Pavlo Lavrenenko
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

bl_info = {
    "name": "Graphene entity format",
    "author": "Pavlo Lavrenenko",
    "version": (0, 1, 5),
    "blender": (2, 6, 0),
    "location": "File > Import-Export",
    "description": "Export entity for Graphene library",
    "category": "Import-Export"
}

import struct
from collections import namedtuple
import bmesh
import bpy
from bpy.types import Operator
from bpy.props import StringProperty, EnumProperty
from bpy_extras.io_utils import ExportHelper, axis_conversion

Material = namedtuple("Material", """
    ambient
    diffuse_intensity
    diffuse_color
    specular_intensity
    specular_hardness
    specular_color""")

def triangulate(mesh):
    bm = bmesh.new()
    bm.from_mesh(mesh)
    bmesh.ops.triangulate(bm, faces=bm.faces)
    bm.to_mesh(mesh)
    bm.free()

def write_entity(context, filepath, global_matrix):
    if bpy.ops.object.mode_set.poll():
        bpy.ops.object.mode_set(mode='OBJECT')

    meshes = []

    for obj in context.scene.objects:
        if obj.parent and obj.parent.dupli_type in {'VERTS', 'FACES'}:
            continue

        if obj.dupli_type != 'NONE':
            obj.dupli_list_create(context.scene)
            derived_objects = [(dupli.object, dupli.matrix) for dupli in obj.dupli_list]
        else:
            derived_objects = [(obj, obj.matrix_world)]

        for derived_object, local_matrix in derived_objects:
            try:
                mesh = derived_object.to_mesh(context.scene, True, 'PREVIEW')
            except RuntimeError:
                pass
            else:
                triangulate(mesh)
                meshes.append(mesh)

                # Modified global_matrix keeps X vector pointing right (see below), face winding
                # becomes Counter Clock Wise with normals pointing the opposite direction.
                # flip_normals() changes the face winding back to Clock Wise and fix normals direction.
                mesh.transform(global_matrix * local_matrix)
                mesh.flip_normals()

                # Calculate split vertex normals, which preserve sharp edges.
                mesh.calc_normals_split()

        if obj.dupli_type != 'NONE':
            obj.dupli_list_clear()

    with open(filepath, "wb") as f:
        f.write(struct.pack("<4s", bytearray("GPHN", 'ASCII')))
        f.write(struct.pack("<4b", *(bl_info["version"] + (len(meshes),))))

        for mesh in meshes:
            loops = mesh.loops[:]
            loops_number = len(loops)

            vertices = [(0.0, 0.0, 0.0)] * loops_number
            normals = [(0.0, 0.0, 0.0)] * loops_number
            uvs = [(0.0, 0.0)] * loops_number

            polygons = mesh.polygons[:]
            faces = [p.loop_indices for p in polygons]

            mesh_vertices = mesh.vertices[:]
            materials = mesh.materials[:]  # TODO: Write more than one material

            try:
                uv_loops = mesh.uv_layers.active.data[:]
            except AttributeError:
                uv_loops = None

            for face_index in [i for p in polygons for i in p.loop_indices]:
                vertex_index = loops[face_index].vertex_index

                vertex = mesh_vertices[vertex_index]
                vertices[face_index] = vertex.co
                normals[face_index] = vertex.normal

                if uv_loops is not None:
                    uv = uv_loops[face_index].uv
                    uvs[face_index] = (uv[0], uv[1])

            try:
                material = materials[0]
            except IndexError:  # Create default material
                material = Material(1.0, 0.8, (0.8, 0.8, 0.8), 0.5, 50, (1.0, 1.0, 1.0))

            f.write(struct.pack("<2f", material.ambient, material.diffuse_intensity))
            f.write(struct.pack("<3f", *material.diffuse_color[:]))
            f.write(struct.pack("<1f1i", material.specular_intensity, material.specular_hardness))
            f.write(struct.pack("<3f", *material.specular_color[:]))

            try:
                diffuse_texture = material.active_texture.image.filepath[2:257]
            except AttributeError:
                diffuse_texture = ''

            diffuse_texture += '\0' * (255 - len(diffuse_texture))
            f.write(struct.pack("<255s1b", bytearray(diffuse_texture, 'ASCII'), 0))

            f.write(struct.pack("<1i1i", len(vertices), len(polygons)))
            for vertex in vertices:
                f.write(struct.pack("<3f", vertex[0], vertex[1], vertex[2]))
            for normal in normals:
                f.write(struct.pack("<3f", normal[0], normal[1], normal[2]))
            for uv in uvs:
                f.write(struct.pack("<2f", uv[0], uv[1]))
            for face in faces:
                f.write(struct.pack("<3i", face[0], face[1], face[2]))

            bpy.data.meshes.remove(mesh)

    return {'FINISHED'}

class ExportEntity(Operator, ExportHelper):

    bl_idname = "export.graphene_entity"
    bl_label = "Export Graphene entity"

    filename_ext = ".entity"
    filter_glob = StringProperty(default="*.entity", options={'HIDDEN'})

    def execute(self, context):
        # Translate from Blender X(right), Y(forward), Z(up)
        # to Graphene X(right), Y(up), Z(forward) coordinate system
        global_matrix = axis_conversion(to_forward='Z', to_up='Y').to_4x4()

        # axis_conversion() issued matrix also flips X(right) to X(left),
        # flip back manually
        global_matrix[0][0] = 1.0

        return write_entity(context, self.filepath, global_matrix)

def menu_func_export(self, context):
    self.layout.operator(ExportEntity.bl_idname, text="Graphene (.entity)")

def register():
    bpy.utils.register_class(ExportEntity)
    bpy.types.INFO_MT_file_export.append(menu_func_export)

def unregister():
    bpy.utils.unregister_class(ExportEntity)
    bpy.types.INFO_MT_file_export.remove(menu_func_export)

if __name__ == "__main__":
    register()
<commit_msg>Refactor Blender export plugin<commit_after>#
# Copyright (c) 2013 Pavlo Lavrenenko
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

bl_info = {
    "name": "Graphene entity format",
    "author": "Pavlo Lavrenenko",
    "version": (0, 1, 5),
    "blender": (2, 6, 0),
    "location": "File > Import-Export",
    "description": "Export entity for Graphene library",
    "category": "Import-Export"
}

import struct
import os
from collections import namedtuple
import bmesh
import bpy
from bpy.types import Operator, Material
from bpy.props import StringProperty, EnumProperty
from bpy_extras.io_utils import ExportHelper, axis_conversion

def triangulate(mesh):
    bm = bmesh.new()
    bm.from_mesh(mesh)
    bmesh.ops.triangulate(bm, faces=bm.faces)
    bm.to_mesh(mesh)
    bm.free()

def write_entity(context, filepath, global_matrix):
    if bpy.ops.object.mode_set.poll():
        bpy.ops.object.mode_set(mode='OBJECT')

    meshes = []

    for obj in context.scene.objects:
        object_parent = obj.parent
        if object_parent is not None and object_parent.dupli_type in {'VERTS', 'FACES'}:
            continue

        if obj.dupli_type != 'NONE':
            obj.dupli_list_create(context.scene)
            derived_objects = [(dupli.object, dupli.matrix) for dupli in obj.dupli_list]
        else:
            derived_objects = [(obj, obj.matrix_world)]

        for derived_object, local_matrix in derived_objects:
            mesh = derived_object.to_mesh(context.scene, True, 'PREVIEW')

            # Renderer requires 3 vertex faces
            triangulate(mesh)
            meshes.append(mesh)

            # Modified global_matrix keeps X vector pointing right (see below), face winding
            # becomes Counter Clock Wise with normals pointing the opposite direction.
            # flip_normals() changes the face winding back to Clock Wise and fix normals direction.
            mesh.transform(global_matrix * local_matrix)
            mesh.flip_normals()

            # Calculate split vertex normals, which preserve sharp edges.
            mesh.calc_normals_split()

        if obj.dupli_type != 'NONE':
            obj.dupli_list_clear()

    with open(filepath, "wb") as f:
        f.write(struct.pack("<4s", bytearray("GPHN", 'ASCII')))
        f.write(struct.pack("<4b", *(bl_info["version"] + (len(meshes),))))

        for mesh in meshes:
            loops = mesh.loops[:]
            polygons = mesh.polygons[:]
            vertices = mesh.vertices[:]
            materials = mesh.materials[:]

            if mesh.uv_layers.active is not None:
                uv_loop = mesh.uv_layers.active.data[:]
            else:
                uv_loop = None

            export_vertices = [(0.0, 0.0, 0.0)] * len(loops)
            export_normals = [(0.0, 0.0, 0.0)] * len(loops)
            export_uvs = [(0.0, 0.0)] * len(loops)
            export_faces = [p.loop_indices for p in polygons]

            for face_index in [index for face in export_faces for index in face]:
                vertex_index = loops[face_index].vertex_index
                vertex = vertices[vertex_index]

                export_vertices[face_index] = vertex.co
                export_normals[face_index] = vertex.normal
                if uv_loop is not None:
                    export_uvs[face_index] = uv_loop[face_index].uv

            # Materials list remains non empty after materials are removed
            if materials and materials[0] is not None:
                # TODO: Write more than one material
                material = materials[0]
            else:
                material = bpy.data.materials.new(name="default")
                material.ambient = 1.0
                material.diffuse_intensity = 1.0
                material.diffuse_color = (1.0, 0.0, 0.0)
                material.specular_intensity = 0.5
                material.specular_hardness = 50
                material.specular_color = (1.0, 1.0, 1.0)

            f.write(struct.pack("<2f", material.ambient, material.diffuse_intensity))
            f.write(struct.pack("<3f", *material.diffuse_color))
            f.write(struct.pack("<1f1i", material.specular_intensity, material.specular_hardness))
            f.write(struct.pack("<3f", *material.specular_color))

            diffuse_texture = material.active_texture
            if diffuse_texture is not None and diffuse_texture.type == 'IMAGE':
                # Per Blender Manual: Files & Data System - Blender File - Blend Files
                # When relative paths are supported, the File Browser provides a Relative Path checkbox,
                # when entering the path into a text field, use a double slash prefix (//) to make it so.
                texture_filepath = diffuse_texture.image.filepath
                if texture_filepath[0:2] == "//":
                    blend_filedir = os.path.dirname(context.blend_data.filepath)
                    texture_filepath = os.path.join(blend_filedir, texture_filepath[2:])

                export_filedir = os.path.dirname(filepath)
                export_filename = os.path.relpath(texture_filepath, export_filedir)
            else:
                export_filename = ""

            export_filename += "\0" * (255 - len(export_filename))
            f.write(struct.pack("<255s1b", bytearray(export_filename, 'ASCII'), 0))

            f.write(struct.pack("<1i1i", len(export_vertices), len(export_faces)))
            for vertex in export_vertices:
                f.write(struct.pack("<3f", vertex[0], vertex[1], vertex[2]))
            for normal in export_normals:
                f.write(struct.pack("<3f", normal[0], normal[1], normal[2]))
            for uv in export_uvs:
                f.write(struct.pack("<2f", uv[0], uv[1]))
            for face in export_faces:
                f.write(struct.pack("<3i", face[0], face[1], face[2]))

            bpy.data.meshes.remove(mesh)

    return {'FINISHED'}

class ExportEntity(Operator, ExportHelper):

    bl_idname = "export.graphene_entity"
    bl_label = "Export Graphene entity"

    filename_ext = ".entity"
    filter_glob = StringProperty(default="*.entity", options={'HIDDEN'})

    def execute(self, context):
        # Translate from Blender X(right), Y(forward), Z(up)
        # to Graphene X(right), Y(up), Z(forward) coordinate system
        global_matrix = axis_conversion(to_forward='Z', to_up='Y').to_4x4()

        # axis_conversion() issued matrix also flips X(right) to X(left),
        # flip back manually
        global_matrix[0][0] = 1.0

        return write_entity(context, self.filepath, global_matrix)

def menu_func_export(self, context):
    self.layout.operator(ExportEntity.bl_idname, text="Graphene (.entity)")

def register():
    bpy.utils.register_class(ExportEntity)
    bpy.types.INFO_MT_file_export.append(menu_func_export)

def unregister():
    bpy.utils.unregister_class(ExportEntity)
    bpy.types.INFO_MT_file_export.remove(menu_func_export)

if __name__ == "__main__":
    register()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# The MIT License (MIT)
#
# Copyright (c) 2015 Shigeru Kitazaki
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""Simple command line script.
TODO: Write description on your own.
"""

import argparse
import configparser
import json
import logging
import logging.config
import os
import sys
import traceback

__version__ = '0.1.0'
__author__ = 'Shigeru Kitazaki'

APPNAME = os.path.splitext(os.path.basename(__file__))[0]
BASEDIR = os.path.dirname(os.path.abspath(__file__))

logging.config.dictConfig({
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'standard': {
            'datefmt': '%Y-%m-%d %H:%M:%S',
            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
        },
        'detailed': {
            'datefmt': '%Y-%m-%d %H:%M:%S',
            'format': '%(asctime)s [%(levelname)s] %(name)s '
                      '%(filename)s:L%(lineno)-4d: %(message)s'
        }
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
            'level': 'DEBUG',
            'stream': 'ext://sys.stderr',
            'formatter': 'standard'
        },
        'file': {
            'class': 'logging.handlers.RotatingFileHandler',
            'level': 'INFO',
            'formatter': 'detailed',
            'filename': APPNAME + '.log',
            'mode': 'a',
            'maxBytes': 10485760,
            'backupCount': 5,
            'encoding': 'utf8'
        }
    },
    'loggers': {
        '': {
            'handlers': ['console'],
            'level': 'INFO',
            'propagate': True
        },
        APPNAME: {
            'handlers': ['console'],
            'level': 'WARN',
            'propagate': False
        }
    }
})

DEFAULT_INPUT_FILE_ENCODING = 'utf8'
DEFAULT_OUTPUT_FILE_ENCODING = 'utf8'

def parse_arguments():
    """Parse arguments and set up logging verbosity.

    :rtype: parsed arguments as Namespace object.
    """
    parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument('-V', '--version', action='version',
                        version='%(prog)s ' + __version__)

    parser.add_argument('-c', '--config', dest='config',
                        help='configuration file', metavar='FILE')
    parser.add_argument('-o', '--output', dest='output',
                        help='output path', metavar='FILE')
    parser.add_argument('-n', '--dryrun', dest='dryrun',
                        help='dry run', default=False, action='store_true')
    parser.add_argument('-e', '--encoding', dest='encoding',
                        help='input file encoding',
                        default=DEFAULT_INPUT_FILE_ENCODING)
    parser.add_argument('-E', '--output-encoding', dest='encoding_out',
                        help='output file encoding',
                        default=DEFAULT_OUTPUT_FILE_ENCODING)
    parser.add_argument('-r', '--recursive', dest='recursive', default=False,
                        help='search recursive', action='store_true')
    parser.add_argument('files', nargs='*',
                        help='input files', metavar='FILE')

    group = parser.add_mutually_exclusive_group()

    group.add_argument('-v', '--verbose', dest='verbose',
                       action='count', default=0,
                       help='increase logging verbosity')

    group.add_argument('-q', '--quiet', dest='quiet',
                       default=False, action='store_true',
                       help='set logging to quiet mode')

    try:
        args = parser.parse_args()
    except IOError:
        e = sys.exc_info()[1]
        parser.error('File not found: %s' % (e, ))

    # Set up logging verbosity level.
    logger = logging.getLogger(APPNAME)
    if args.quiet:
        logger.setLevel(logging.CRITICAL)
    elif args.verbose >= 3:
        logger.setLevel(logging.DEBUG)
    elif args.verbose >= 2:
        logger.setLevel(logging.INFO)
    elif args.verbose >= 1:
        logger.setLevel(logging.WARN)
    else:
        logger.setLevel(logging.ERROR)

    return args

def collect_files(inputs, recursive=False):
    '''Check input argument files path.
    '''
    logger = logging.getLogger(APPNAME + '.setup')
    if inputs is None or len(inputs) == 0:
        return
    files = []
    for path in inputs:
        if os.path.isfile(path):
            logger.debug('Target file exists: %s', path)
            files.append(path)
        elif os.path.isdir(path) and recursive:
            logger.debug('Target directory exists: %s', path)
            for root, ds, fs in os.walk(path):
                # Prune hidden directory.
                ds[:] = [d for d in ds if not d.startswith('.')]
                for f in sorted(filter(lambda f: not f.endswith('~'), fs)):
                    p = os.path.join(root, f)
                    files.append(p)
        else:
            logger.fatal('File not found: %s', path)
            sys.exit(1)
    logger.debug('Collect {:,} files.'.format(len(files)))
    return files

class ConfigLoader(object):
    """Configuration file loader to support multiple file types.

    Supported file types are:
    * ini/cfg
    * json

    :param fp: file pointer to load
    :param filetype: either of 'ini|cfg', 'json', or 'yml|yaml' file.
        If nothing specified, detect by file extension automatically.
    """

    def __init__(self, path):
        self.path = path
        self.logger = logging.getLogger(APPNAME + '.config')

    def _load(self, extension):
        self.logger.debug('Config file extension is "%s".', extension)
        if extension == '.json':
            with open(self.path) as fp:
                return json.load(fp)
        elif extension in (".ini", ".cfg"):
            parser = configparser.SafeConfigParser()
            with open(self.path) as fp:
                parser.readfp(fp)
            config = {}
            for s in parser.sections():
                config[s] = dict(parser.items(s))
            return config
        else:
            self.logger.warn('Unknown file type extension: %s', extension)

    def load(self, env=None):
        """ Load a section values of given environment.
        If nothing to specified, use environmental variable.
        If unknown environment was specified, warn it on logger.
        :param env: environment key to load in a coercive manner
        :type env: string
        :rtype: dict
        """
        self.logger.debug('Loading config "%s" ...', self.path)
        ext = os.path.splitext(self.path)[-1].lower()
        if len(ext) == 0:
            self.logger.warn("Missing file extension: %s", self.path)
            return
        elif not ext.startswith('.'):
            self.logger.warn("Extension doesn't start with dot: %s", self.path)
            return
        return self._load(ext)

class MainProcess(object):

    """Main process class.
    # TODO: Implement your logic.
    """

    def __init__(self, dryrun):
        self.dryrun = dryrun
        self.logger = logging.getLogger(APPNAME + '.main')

    def configure(self, configfile):
        loader = ConfigLoader(configfile)
        config = loader.load()
        for k in sorted(config):
            self.logger.debug('config key: %s', k)
        # TODO: Implement your logic.

    def run(self, files, encoding, output):
        if files:
            for fname in files:
                self.logger.info('Start processing: {}'.format(fname))
                with open(fname, 'r', encoding=encoding) as fp:
                    _ = self.process(fp, output)
                self.logger.info('Finish processing: {}'.format(fname))
        else:
            _ = self.process(sys.stdin, output)

    def process(self, fp, output):
        # TODO: Implement your logic.
        pass

CONFIGURATION = """Start running with following configurations.
==============================================================================
  Base directory     : {basedir}
  Current working dir: {cwd}
  Configuration file : {configfile}
  Dry-run            : {dryrun}
  Input encoding     : {encoding}
  Input #files       : {nfiles}
  Search recursive   : {recursive}
  Output path        : {output}
  Output encoding    : {encoding_out}
==============================================================================
""".rstrip()

def main():
    logger = logging.getLogger(APPNAME)
    # Parse command line arguments.
    args = parse_arguments()
    files = collect_files(args.files, args.recursive)
    encoding = args.encoding
    processor = MainProcess(args.dryrun)
    # Check and set configuration file.
    if args.config:
        configfile = args.config
        if os.path.isfile(configfile):
            processor.configure(configfile)
        else:
            logger.fatal('Configuration file is not found: %s', configfile)
            sys.exit(1)
    if args.output:
        outputpath = args.output
        if os.path.isfile(outputpath):
            logger.warn('Overwrite output file: %s', outputpath)
        output = open(outputpath, 'w', encoding=args.encoding_out)
    else:
        output = sys.stdout
    logger.info(CONFIGURATION.format(basedir=BASEDIR, cwd=os.getcwd(),
                configfile=args.config, dryrun=args.dryrun,
                encoding=encoding, nfiles=len(files or []),
                recursive=args.recursive,
                output=args.output, encoding_out=args.encoding_out))
    # Dispatch main process., and catch unknown error.
    try:
        processor.run(files, encoding, output)
    except Exception:
        e = sys.exc_info()[1]
        logger.error(e)
        traceback.print_exc(file=sys.stderr)
    finally:
        logger.info('Finish running.')

    if not output.isatty():
        output.close()

def test():
    # TODO: Implement test code.
    pass

if __name__ == '__main__':
    main()

# vim: set et ts=4 sw=4 cindent fileencoding=utf-8 :
<commit_msg>Implement progress monitor on boilerplate for Python.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# The MIT License (MIT)
#
# Copyright (c) 2015 Shigeru Kitazaki
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""Simple command line script.
TODO: Write description on your own.
"""

import argparse
import configparser
import csv
import datetime
import hashlib
import json
import logging
import logging.config
import os
import sqlite3
import sys
import time
import traceback
from functools import partial

__version__ = '0.1.0'
__author__ = 'Shigeru Kitazaki'

APPNAME = os.path.splitext(os.path.basename(__file__))[0]
BASEDIR = os.path.dirname(os.path.abspath(__file__))

logging.config.dictConfig({
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'standard': {
            'datefmt': '%Y-%m-%d %H:%M:%S',
            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
        },
        'detailed': {
            'datefmt': '%Y-%m-%d %H:%M:%S',
            'format': '%(asctime)s [%(levelname)s] %(name)s '
                      '%(filename)s:L%(lineno)-4d: %(message)s'
        }
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
            'level': 'DEBUG',
            'stream': 'ext://sys.stderr',
            'formatter': 'standard'
        },
        'file': {
            'class': 'logging.handlers.RotatingFileHandler',
            'level': 'INFO',
            'formatter': 'detailed',
            'filename': APPNAME + '.log',
            'mode': 'a',
            'maxBytes': 10485760,
            'backupCount': 5,
            'encoding': 'utf8'
        }
    },
    'loggers': {
        '': {
            'handlers': ['console'],
            'level': 'INFO',
            'propagate': True
        },
        APPNAME: {
            'handlers': ['console'],
            'level': 'WARN',
            'propagate': False
        }
    }
})

DEFAULT_INPUT_FILE_ENCODING = 'utf8'
DEFAULT_OUTPUT_FILE_ENCODING = 'utf8'

def parse_arguments():
    """Parse arguments and set up logging verbosity.

    :rtype: parsed arguments as Namespace object.
    """
    parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument('-V', '--version', action='version',
                        version='%(prog)s ' + __version__)

    parser.add_argument('-c', '--config', dest='config', required=False,
                        help='configuration file', metavar='FILE')
    parser.add_argument('-m', '--monitor', dest='monitor', required=False,
                        help='process monitor file', metavar='FILE')
    parser.add_argument('-o', '--output', dest='output',
                        help='output path', metavar='FILE')
    parser.add_argument('-n', '--dryrun', dest='dryrun',
                        help='dry run', default=False, action='store_true')
    parser.add_argument('-e', '--encoding', dest='encoding',
                        help='input file encoding',
                        default=DEFAULT_INPUT_FILE_ENCODING)
    parser.add_argument('-E', '--output-encoding', dest='encoding_out',
                        help='output file encoding',
                        default=DEFAULT_OUTPUT_FILE_ENCODING)
    parser.add_argument('-r', '--recursive', dest='recursive', default=False,
                        help='search recursive', action='store_true')
    parser.add_argument('files', nargs='*',
                        help='input files', metavar='FILE')

    group = parser.add_mutually_exclusive_group()

    group.add_argument('-v', '--verbose', dest='verbose',
                       action='count', default=0,
                       help='increase logging verbosity')

    group.add_argument('-q', '--quiet', dest='quiet',
                       default=False, action='store_true',
                       help='set logging to quiet mode')

    try:
        args = parser.parse_args()
    except IOError:
        e = sys.exc_info()[1]
        parser.error('File not found: %s' % (e, ))

    # Set up logging verbosity level.
    logger = logging.getLogger(APPNAME)
    if args.quiet:
        logger.setLevel(logging.CRITICAL)
    elif args.verbose >= 3:
        logger.setLevel(logging.DEBUG)
    elif args.verbose >= 2:
        logger.setLevel(logging.INFO)
    elif args.verbose >= 1:
        logger.setLevel(logging.WARN)
    else:
        logger.setLevel(logging.ERROR)

    return args

def collect_files(inputs, recursive=False):
    '''Check input argument files path.
    '''
    logger = logging.getLogger(APPNAME + '.setup')
    if inputs is None or len(inputs) == 0:
        return
    files = []
    for path in inputs:
        if os.path.isfile(path):
            logger.debug('Target file exists: %s', path)
            files.append(path)
        elif os.path.isdir(path) and recursive:
            logger.debug('Target directory exists: %s', path)
            for root, ds, fs in os.walk(path):
                # Prune hidden directory.
                ds[:] = [d for d in ds if not d.startswith('.')]
                for f in sorted(filter(lambda f: not f.endswith('~'), fs)):
                    p = os.path.join(root, f)
                    files.append(p)
        else:
            logger.fatal('File not found: %s', path)
            sys.exit(1)
    logger.debug('Collect {:,} files.'.format(len(files)))
    return files

def md5sum(path):
    # Calculate MD5 sum value.
    chunk_size = 4096
    md5 = hashlib.md5()
    with open(path, 'rb') as fp:
        for buf in iter(partial(fp.read, chunk_size), b''):
            md5.update(buf)
    return md5.hexdigest()

class ConfigLoader(object):
    """Configuration file loader to support multiple file types.

    Supported file types are:
    * ini/cfg
    * json

    :param fp: file pointer to load
    :param filetype: either of 'ini|cfg', 'json', or 'yml|yaml' file.
        If nothing specified, detect by file extension automatically.
    """

    def __init__(self, path):
        self.path = path
        self.logger = logging.getLogger(APPNAME + '.config')

    def _load(self, extension):
        self.logger.debug('Config file extension is "%s".', extension)
        if extension == '.json':
            with open(self.path) as fp:
                return json.load(fp)
        elif extension in (".ini", ".cfg"):
            parser = configparser.SafeConfigParser()
            with open(self.path) as fp:
                parser.readfp(fp)
            config = {}
            for s in parser.sections():
                config[s] = dict(parser.items(s))
            return config
        else:
            self.logger.warn('Unknown file type extension: %s', extension)

    def load(self, env=None):
        """ Load a section values of given environment.
        If nothing to specified, use environmental variable.
        If unknown environment was specified, warn it on logger.
        :param env: environment key to load in a coercive manner
        :type env: string
        :rtype: dict
        """
        self.logger.debug('Loading config "%s" ...', self.path)
        ext = os.path.splitext(self.path)[-1].lower()
        if len(ext) == 0:
            self.logger.warn("Missing file extension: %s", self.path)
            return
        elif not ext.startswith('.'):
            self.logger.warn("Extension doesn't start with dot: %s", self.path)
            return
        return self._load(ext)

class ProgressMonitor(object):

    DEFAULT_MONITOR_FILE = ':memory:'
    TABLE_NAME = '_monitor'
    SCHEMA = (
        {'id': 'seq', 'type': 'integer', 'primary': True},
        {'id': 'path', 'type': 'string', 'required': True},
        {'id': 'start_at', 'type': 'float', 'required': True},
        {'id': 'finish_at', 'type': 'float'},
        {'id': 'digest', 'type': 'string', 'required': True, 'unique': True},
        {'id': 'result', 'type': 'string'},  # Anything encoded by JSON
    )

    def __init__(self, fname):
        self.logger = logging.getLogger(APPNAME + '.monitor')
        if fname and os.path.isfile(fname):
            self.logger.info('Reuse monitor file: %s', fname)
        self.db = sqlite3.connect(fname or ProgressMonitor.DEFAULT_MONITOR_FILE)
        self.create_table()
        self.current = None

    def create_table(self):
        # Check whether monitor table already exists.
        r = self.fetch_one('sql',
                           (('type', '=', 'table'),
                            ('tbl_name', '=', ProgressMonitor.TABLE_NAME)),
                           'sqlite_master')
        if r:
            self.logger.info('Monitor table is already created.')
            self.logger.debug(r[0])
            return
        d = []
        # Pattern mapping against JSON Table Schema
        for s in ProgressMonitor.SCHEMA:
            t = 'TEXT'
            if s['type'] == 'integer':
                t = 'INTEGER'
            elif s['type'] == 'float':
                t = 'REAL'
            f = '{} {}'.format(s['id'], t)
            if s.get('primary'):
                f += ' PRIMARY KEY'
            if s.get('required'):
                f += ' NOT NULL'
            if s.get('unique'):
                f += ' UNIQUE'
            d.append(f)
        ddl = """CREATE TABLE {} ({})""".format(
            ProgressMonitor.TABLE_NAME, ','.join(d))
        self.logger.debug('Create monitor table: %s', ddl)
        cur = self.db.cursor()
        cur.execute(ddl)
        cur.close()
        self.logger.info('Created monitor table.')

    def terminate(self, dump=None):
        self.db.commit()
        if dump is None:
            return
        # Dump monitor records as tab-delimited values.
        writer = csv.writer(dump, delimiter='\t', lineterminator='\n')
        header = ('seq', 'path', 'start_at', 'finish_at', 'elapsed', 'digest')
        writer.writerow(header)
        q = 'SELECT * FROM {} ORDER BY seq'.format(ProgressMonitor.TABLE_NAME)
        cur = self.db.cursor()
        cur.execute(q)
        for r in cur:
            t = [r[0], r[1]]
            start = datetime.datetime.fromtimestamp(r[2])
            t.append(start.strftime('%Y-%m-%d %H:%M:%S'))
            if r[3]:
                finish = datetime.datetime.fromtimestamp(r[3])
                t.append(finish.strftime('%Y-%m-%d %H:%M:%S'))
                t.append(r[3] - r[2])
            else:
                t.append(None)
                t.append(None)
            t.append(r[4])
            writer.writerow(list(map(lambda s: str(s) if s else '', t)))
        cur.close()

    def fetch_one(self, columns, conditions, table=None):
        # It's okay `columns` is string or list.
        if type(columns) == str:
            columns = [columns, ]
        d = []
        values = []
        # Column name, Operator, Value tuples.
        for c, o, v in conditions:
            # "?" is placement holder.
            d.append("{} {} ?".format(c, o))
            values.append(v)
        q = """SELECT {} FROM {} WHERE {}""".format(
            ','.join(columns),
            table or ProgressMonitor.TABLE_NAME,
            ' AND '.join(d))
        self.logger.debug('Fetch one record: %s; %s', q, values)
        cur = self.db.cursor()
        cur.execute(q, values)
        r = cur.fetchone()
        cur.close()
        return r

    def start(self, path):
        md5 = md5sum(path)
        r = self.fetch_one(['seq', 'path', 'start_at', 'finish_at'], (
            ('digest', '=', md5),
        ))
        if r:
            msg = 'Already processed "{}": [{}] {} -> {}'
            self.logger.info(msg.format(r[1], r[0], r[2], r[3]))
            return r
        self.logger.info('Start monitoring: %s (%s)', path, md5)
        columns = ('path', 'start_at', 'digest')
        values = (path, time.time(), md5)
        q = """INSERT INTO {} ({}) VALUES ({})""".format(
            ProgressMonitor.TABLE_NAME,
            ','.join(columns),
            ','.join(['?' for i in range(len(values))])
            )
        self.logger.debug('Insert one record: %s; %s', q, values)
        cur = self.db.cursor()
        cur.execute(q, values)
        cur.close()
        self.current = md5

    def finish(self, result=None):
        if self.current is None:
            self.logger.fatal('Monitor nothing, but `finish()` is called.')
            return
        r = self.fetch_one(['seq', 'path', 'start_at'], (
            ('digest', '=', self.current),
        ))
        if r is None:
            self.logger.fatal('Monitor "%s", but removed.', self.current)
            return
        now = time.time()
        if result:
            values = (now, json.dumps(result), r[0])
            columns = ('finish_at = ?', 'result = ?')
        else:
            values = (now, r[0])
            columns = ('finish_at = ?', )
        q = """UPDATE {} SET {} WHERE seq = ?""".format(
            ProgressMonitor.TABLE_NAME,
            ','.join(columns)
            )
        self.logger.debug('Update one record: %s; %s', q, values)
        cur = self.db.cursor()
        cur.execute(q, values)
        cur.close()
        self.current = None
        self.logger.info('Finish processing: {} [{}] {:,.03f}sec'.format(
            r[1], r[0], now - r[2]))

class MainProcess(object):

    """Main process class.
    # TODO: Implement your logic.
    """

    def __init__(self, dryrun, monitor_file=None):
        self.dryrun = dryrun
        self.logger = logging.getLogger(APPNAME + '.main')
        self.monitor = ProgressMonitor(monitor_file)

    def configure(self, configfile):
        loader = ConfigLoader(configfile)
        config = loader.load()
        for k in sorted(config):
            self.logger.debug('config key: %s', k)
        # TODO: Implement your logic.

    def setup_writer(self, output):
        # TODO: Implement your logic.
        pass

    def run(self, files, encoding):
        if files:
            for fname in files:
                canskip = self.monitor.start(fname)
                if canskip:
                    continue
                with open(fname, 'r', encoding=encoding) as fp:
                    r = self.process(fp)
                self.monitor.finish({'lines': r})
            # TODO: Set output file name for monitor dump.
            self.monitor.terminate(sys.stdout)
        else:
            self.process(sys.stdin)

    def process(self, fp):
        # TODO: Implement your logic.
        lines = 0
        reader = map(str.rstrip, fp)
        for l in reader:
            lines += 1
        return lines

CONFIGURATION = """Start running with following configurations.
==============================================================================
  Base directory     : {basedir}
  Current working dir: {cwd}
  Configuration file : {configfile}
  Monitor file       : {monitorfile}
  Dry-run            : {dryrun}
  Input encoding     : {encoding}
  Input #files       : {nfiles}
  Search recursive   : {recursive}
  Output path        : {output}
  Output encoding    : {encoding_out}
==============================================================================
""".rstrip()

def main():
    logger = logging.getLogger(APPNAME)
    # Parse command line arguments.
    args = parse_arguments()
    files = collect_files(args.files, args.recursive)
    encoding = args.encoding
    processor = MainProcess(args.dryrun, args.monitor)
    # Check and set configuration file.
    if args.config:
        configfile = args.config
        if os.path.isfile(configfile):
            processor.configure(configfile)
        else:
            logger.fatal('Configuration file is not found: %s', configfile)
            sys.exit(1)
    if args.output:
        outputpath = args.output
        if os.path.isfile(outputpath):
            logger.warn('Overwrite output file: %s', outputpath)
        output = open(outputpath, 'w', encoding=args.encoding_out)
    else:
        output = sys.stdout
    processor.setup_writer(output)
    logger.info(CONFIGURATION.format(basedir=BASEDIR, cwd=os.getcwd(),
                configfile=args.config, dryrun=args.dryrun,
                encoding=encoding, nfiles=len(files or []),
                recursive=args.recursive, monitorfile=args.monitor,
                output=args.output, encoding_out=args.encoding_out))
    # Dispatch main process, and catch unknown error.
    try:
        processor.run(files, encoding)
    except Exception:
        e = sys.exc_info()[1]
        logger.error(e)
        traceback.print_exc(file=sys.stderr)
    finally:
        logger.info('Finish running.')

    if not output.isatty():
        output.close()

def test():
    # TODO: Implement test code.
    pass

if __name__ == '__main__':
    main()

# vim: set et ts=4 sw=4 cindent fileencoding=utf-8 :
<|endoftext|>