<commit_before>#!/usr/bin/env python
import argparse
import os
import sys

import binaryninja as binja
import magic  # pip install python-magic

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
REMILL_DIR = os.path.dirname(SCRIPT_DIR)

sys.path.append(REMILL_DIR)
from generated.CFG import CFG_pb2

DEBUG = False

INDIRECT_TERMINATORS = [
    binja.core.LLIL_CALL,
    binja.core.LLIL_SYSCALL,
    binja.core.LLIL_TRAP,
    binja.core.LLIL_BP
]

UNKNOWN_IL = [
    binja.core.LLIL_UNDEF,
    binja.core.LLIL_UNIMPL,
    binja.core.LLIL_UNIMPL_MEM
]

def debug(s):
    if DEBUG:
        sys.stdout.write('{}\n'.format(str(s)))

def add_indirect_blocks(pb_mod, indirects):
    # type: (CFG_pb2.Module, set) -> None
    for addr in indirects:
        iblock = pb_mod.indirect_blocks.add()
        iblock.address = addr

def is_cpuid(bv, il):
    # type: (binja.BinaryView, binja.LowLevelILInstruction) -> bool
    txt = get_inst_text(bv, il.address)
    return 'cpuid' == txt[0].text.strip()

def is_interrupt(bv, il):
    # type: (binja.BinaryView, binja.LowLevelILInstruction) -> bool
    txt = get_inst_text(bv, il.address)
    return 'int' == txt[0].text.strip()

def get_inst_text(bv, addr):
    # type: (binja.BinaryView, int) -> list
    data = bv.read(addr, 16)
    return bv.arch.get_instruction_text(data, addr)[0]

def read_inst_bytes(bv, il):
    # type: (binja.BinaryView, binja.LowLevelILInstruction) -> str
    inst_data = bv.read(il.address, 16)
    inst_info = bv.arch.get_instruction_info(inst_data, il.address)
    return inst_data[:inst_info.length]

def process_inst(bv, pb_block, il, indirects):
    # type: (binja.BinaryView, CFG_pb2.Block, binja.LowLevelILInstruction, set) -> (CFG_pb2.Instr, bool)
    pb_inst = pb_block.instructions.add()
    pb_inst.address = il.address
    pb_inst.bytes = read_inst_bytes(bv, il)
    pb_inst.size = len(pb_inst.bytes)

    op = il.operation
    if op in INDIRECT_TERMINATORS:
        debug('Found indirect terminator: {} @ {:x}'.format(il.operation_name, il.address))
        return pb_inst, True

    elif op in UNKNOWN_IL:
        if is_cpuid(bv, il):
            debug('Found indirect terminator: cpuid @ {:x}'.format(il.address))
            return pb_inst, True
        if is_interrupt(bv, il):
            debug('Found indirect terminator: int @ {:x}'.format(il.address))
            return pb_inst, True

    return pb_inst, False

def create_block(pb_mod, addr):
    # type: (CFG_pb2.Module, int) -> CFG_pb2.Block
    pb_block = pb_mod.blocks.add()
    pb_block.address = addr
    return pb_block

def process_blocks(bv, pb_mod, func, indirects):
    # type: (binja.BinaryView, CFG_pb2.Module, binja.Function, set) -> None
    ilfunc = func.lifted_il

    for block in func:
        pb_block = create_block(pb_mod, block.start)

        # Keep track of the current address in the block
        inst_idx = block.start
        end_block = False
        while inst_idx < block.end:
            # Check if the block should be split early
            if end_block:
                indirects.add(inst_idx)
                pb_block = create_block(pb_mod, inst_idx)

            # Get the IL at the current address
            il = ilfunc[func.get_lifted_il_at(bv.arch, inst_idx)]

            # Add the instruction data
            pb_inst, end_block = process_inst(bv, pb_block, il, indirects)
            inst_idx += pb_inst.size

def is_export(func):
    # type: (binja.Function) -> bool
    sym_type = binja.core.BNSymbolType_by_name[func.symbol.type]
    return sym_type == binja.core.FunctionSymbol and not func.auto

def is_import(func):
    # type: (binja.Function) -> bool
    sym_type = binja.core.BNSymbolType_by_name[func.symbol.type]
    return sym_type == binja.core.ImportedFunctionSymbol

def is_internal(func):
    # type: (binja.Function) -> bool
    sym_type = binja.core.BNSymbolType_by_name[func.symbol.type]
    return sym_type == binja.core.FunctionSymbol and func.auto

def analyze_exports(bv, pb_mod, indirects):
    # type: (binja.BinaryView, CFG_pb2.Module, set) -> None
    for func in bv.functions:
        if is_export(func):
            pb_func = pb_mod.functions.add()
            pb_func.name = func.symbol.short_name
            pb_func.address = func.start
            pb_func.is_imported = False
            pb_func.is_exported = True
            pb_func.is_weak = False

            debug('Adding export: {} @ {:x}'.format(pb_func.name, pb_func.address))
            process_blocks(bv, pb_mod, func, indirects)

def analyze_imports(bv, pb_mod):
    # type: (binja.BinaryView, CFG_pb2.Module) -> None
    for func in bv.functions:
        if is_import(func):
            pb_func = pb_mod.functions.add()
            pb_func.name = func.symbol.short_name
            pb_func.address = func.start
            pb_func.is_imported = True
            pb_func.is_exported = False
            pb_func.is_weak = False  # TODO: see if this can be figured out

            debug('Adding import: {} @ {:x}'.format(pb_func.name, pb_func.address))

def analyze_internal_functions(bv, pb_mod, indirects):
    # type: (binja.BinaryView, CFG_pb2.Module, set) -> None
    for func in bv.functions:
        if is_internal(func):
            pb_func = pb_mod.functions.add()
            pb_func.name = func.symbol.short_name  # TODO: should this be different?
            pb_func.address = func.start
            pb_func.is_imported = False
            pb_func.is_exported = False
            pb_func.is_weak = False

            debug('Adding function: {} @ {:x}'.format(pb_func.name, pb_func.address))
            process_blocks(bv, pb_mod, func, indirects)

def recover_cfg(bv, outf):
    # type: (binja.BinaryView, file) -> None
    pb_mod = CFG_pb2.Module()
    pb_mod.binary_path = bv.file.filename
    indirects = set()

    debug('Analyzing exports...')
    analyze_exports(bv, pb_mod, indirects)

    debug('Analyzing imports...')
    analyze_imports(bv, pb_mod)

    debug('Analyzing internal functions...')
    analyze_internal_functions(bv, pb_mod, indirects)

    debug('Adding indirect blocks...')
    add_indirect_blocks(pb_mod, indirects)

    debug('Saving CFG to {}'.format(outf.name))
    outf.write(pb_mod.SerializeToString())
    outf.close()

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument('-d', '--debug', action='store_true',
                        help='Enable verbose debugging mode')

    parser.add_argument('-o', '--output',
                        type=str, default=None,
                        help='The output control flow graph recovered from this file')

    parser.add_argument('file', help='Binary to recover control flow graph from')

    args = parser.parse_args(sys.argv[1:])

    # Enable debugging
    if args.debug:
        global DEBUG
        DEBUG = True

    # Get path to input file
    fpath = os.path.abspath(args.file)
    fdir = os.path.dirname(fpath)

    # Resolve path to output file
    if args.output:
        # Attempt to create directories to the output file
        try:
            os.mkdir(os.path.dirname(args.output))
        except OSError:
            pass

        outf = open(args.output, 'wb')
    else:
        # Default output file is "{basename}.cfg"
        outpath = os.path.join(fdir, '{}.cfg'.format(os.path.basename(fpath)))
        outf = open(outpath, 'wb')

    # Look at magic bytes to choose the right BinaryViewType
    magic_type = magic.from_file(fpath)
    if 'ELF' in magic_type:
        bv_type = binja.BinaryViewType['ELF']
    elif 'PE32' in magic_type:
        bv_type = binja.BinaryViewType['PE']
    elif 'Mach-O' in magic_type:
        bv_type = binja.BinaryViewType['Mach-O']
    else:
        # "Raw" type, can't be used for anything, quitting
        debug('Unknown binary type: "{}"'.format(magic_type))
        return 1

    # Load and analyze the binary
    bv = bv_type.open(fpath)
    bv.update_analysis_and_wait()

    # Binja will not load a binary with no entry point
    if len(bv) == 0:
        debug('Binary could not be loaded in binja, is it linked?')
        return 1

    recover_cfg(bv, outf)

if __name__ == '__main__':
    exit(main())
<commit_msg>Add jump table entries as indirects<commit_after>#!/usr/bin/env python
import argparse
import os
import sys

import binaryninja as binja
import magic  # pip install python-magic

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
REMILL_DIR = os.path.dirname(SCRIPT_DIR)

sys.path.append(REMILL_DIR)
from generated.CFG import CFG_pb2

DEBUG = False

INDIRECT_TERMINATORS = [
    binja.core.LLIL_CALL,
    binja.core.LLIL_SYSCALL,
    binja.core.LLIL_TRAP,
    binja.core.LLIL_BP
]

UNKNOWN_IL = [
    binja.core.LLIL_UNDEF,
    binja.core.LLIL_UNIMPL,
    binja.core.LLIL_UNIMPL_MEM
]

def debug(s):
    if DEBUG:
        sys.stdout.write('{}\n'.format(str(s)))

def add_indirect_blocks(pb_mod, indirects):
    # type: (CFG_pb2.Module, set) -> None
    for addr in indirects:
        iblock = pb_mod.indirect_blocks.add()
        iblock.address = addr

def is_cpuid(bv, il):
    # type: (binja.BinaryView, binja.LowLevelILInstruction) -> bool
    txt = get_inst_text(bv, il.address)
    return 'cpuid' == txt[0].text.strip()

def is_interrupt(bv, il):
    # type: (binja.BinaryView, binja.LowLevelILInstruction) -> bool
    txt = get_inst_text(bv, il.address)
    return 'int' == txt[0].text.strip()

def get_inst_text(bv, addr):
    # type: (binja.BinaryView, int) -> list
    data = bv.read(addr, 16)
    return bv.arch.get_instruction_text(data, addr)[0]

def read_inst_bytes(bv, il):
    # type: (binja.BinaryView, binja.LowLevelILInstruction) -> str
    inst_data = bv.read(il.address, 16)
    inst_info = bv.arch.get_instruction_info(inst_data, il.address)
    return inst_data[:inst_info.length]

def process_inst(bv, pb_block, il, indirects):
    # type: (binja.BinaryView, CFG_pb2.Block, binja.LowLevelILInstruction, set) -> (CFG_pb2.Instr, bool)
    pb_inst = pb_block.instructions.add()
    pb_inst.address = il.address
    pb_inst.bytes = read_inst_bytes(bv, il)
    pb_inst.size = len(pb_inst.bytes)

    op = il.operation
    if op in INDIRECT_TERMINATORS:
        debug('Found indirect terminator: {} @ {:x}'.format(il.operation_name, il.address))
        return pb_inst, True

    elif op in UNKNOWN_IL:
        if is_cpuid(bv, il):
            debug('Found indirect terminator: cpuid @ {:x}'.format(il.address))
            return pb_inst, True
        if is_interrupt(bv, il):
            debug('Found indirect terminator: int @ {:x}'.format(il.address))
            return pb_inst, True

    elif op == binja.core.LLIL_JUMP_TO:
        # Add all jump table entries as indirect blocks
        ilfunc = il.function
        for idx in il.targets:
            indirects.add(ilfunc[idx].address)

    return pb_inst, False

def create_block(pb_mod, addr):
    # type: (CFG_pb2.Module, int) -> CFG_pb2.Block
    pb_block = pb_mod.blocks.add()
    pb_block.address = addr
    return pb_block

def process_blocks(bv, pb_mod, func, indirects):
    # type: (binja.BinaryView, CFG_pb2.Module, binja.Function, set) -> None
    ilfunc = func.lifted_il

    for block in func:
        pb_block = create_block(pb_mod, block.start)

        # Keep track of the current address in the block
        inst_idx = block.start
        end_block = False
        while inst_idx < block.end:
            # Check if the block should be split early
            if end_block:
                indirects.add(inst_idx)
                pb_block = create_block(pb_mod, inst_idx)

            # Get the IL at the current address
            il = ilfunc[func.get_lifted_il_at(bv.arch, inst_idx)]

            # Add the instruction data
            pb_inst, end_block = process_inst(bv, pb_block, il, indirects)
            inst_idx += pb_inst.size

def is_export(func):
    # type: (binja.Function) -> bool
    sym_type = binja.core.BNSymbolType_by_name[func.symbol.type]
    return sym_type == binja.core.FunctionSymbol and not func.auto

def is_import(func):
    # type: (binja.Function) -> bool
    sym_type = binja.core.BNSymbolType_by_name[func.symbol.type]
    return sym_type == binja.core.ImportedFunctionSymbol

def is_internal(func):
    # type: (binja.Function) -> bool
    sym_type = binja.core.BNSymbolType_by_name[func.symbol.type]
    return sym_type == binja.core.FunctionSymbol and func.auto

def analyze_exports(bv, pb_mod, indirects):
    # type: (binja.BinaryView, CFG_pb2.Module, set) -> None
    for func in bv.functions:
        if is_export(func):
            pb_func = pb_mod.functions.add()
            pb_func.name = func.symbol.short_name
            pb_func.address = func.start
            pb_func.is_imported = False
            pb_func.is_exported = True
            pb_func.is_weak = False

            debug('Adding export: {} @ {:x}'.format(pb_func.name, pb_func.address))
            process_blocks(bv, pb_mod, func, indirects)

def analyze_imports(bv, pb_mod):
    # type: (binja.BinaryView, CFG_pb2.Module) -> None
    for func in bv.functions:
        if is_import(func):
            pb_func = pb_mod.functions.add()
            pb_func.name = func.symbol.short_name
            pb_func.address = func.start
            pb_func.is_imported = True
            pb_func.is_exported = False
            pb_func.is_weak = False  # TODO: see if this can be figured out

            debug('Adding import: {} @ {:x}'.format(pb_func.name, pb_func.address))

def analyze_internal_functions(bv, pb_mod, indirects):
    # type: (binja.BinaryView, CFG_pb2.Module, set) -> None
    for func in bv.functions:
        if is_internal(func):
            pb_func = pb_mod.functions.add()
            pb_func.name = func.symbol.short_name  # TODO: should this be different?
            pb_func.address = func.start
            pb_func.is_imported = False
            pb_func.is_exported = False
            pb_func.is_weak = False

            debug('Adding function: {} @ {:x}'.format(pb_func.name, pb_func.address))
            process_blocks(bv, pb_mod, func, indirects)

def recover_cfg(bv, outf):
    # type: (binja.BinaryView, file) -> None
    pb_mod = CFG_pb2.Module()
    pb_mod.binary_path = bv.file.filename
    indirects = set()

    debug('Analyzing exports...')
    analyze_exports(bv, pb_mod, indirects)

    debug('Analyzing imports...')
    analyze_imports(bv, pb_mod)

    debug('Analyzing internal functions...')
    analyze_internal_functions(bv, pb_mod, indirects)

    debug('Adding indirect blocks...')
    add_indirect_blocks(pb_mod, indirects)

    debug('Saving CFG to {}'.format(outf.name))
    outf.write(pb_mod.SerializeToString())
    outf.close()

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument('-d', '--debug', action='store_true',
                        help='Enable verbose debugging mode')

    parser.add_argument('-o', '--output',
                        type=str, default=None,
                        help='The output control flow graph recovered from this file')

    parser.add_argument('file', help='Binary to recover control flow graph from')

    args = parser.parse_args(sys.argv[1:])

    # Enable debugging
    if args.debug:
        global DEBUG
        DEBUG = True

    # Get path to input file
    fpath = os.path.abspath(args.file)
    fdir = os.path.dirname(fpath)

    # Resolve path to output file
    if args.output:
        # Attempt to create directories to the output file
        try:
            os.mkdir(os.path.dirname(args.output))
        except OSError:
            pass

        outf = open(args.output, 'wb')
    else:
        # Default output file is "{basename}.cfg"
        outpath = os.path.join(fdir, '{}.cfg'.format(os.path.basename(fpath)))
        outf = open(outpath, 'wb')

    # Look at magic bytes to choose the right BinaryViewType
    magic_type = magic.from_file(fpath)
    if 'ELF' in magic_type:
        bv_type = binja.BinaryViewType['ELF']
    elif 'PE32' in magic_type:
        bv_type = binja.BinaryViewType['PE']
    elif 'Mach-O' in magic_type:
        bv_type = binja.BinaryViewType['Mach-O']
    else:
        # "Raw" type, can't be used for anything, quitting
        debug('Unknown binary type: "{}"'.format(magic_type))
        return 1

    # Load and analyze the binary
    bv = bv_type.open(fpath)
    bv.update_analysis_and_wait()

    # Binja will not load a binary with no entry point
    if len(bv) == 0:
        debug('Binary could not be loaded in binja, is it linked?')
        return 1

    recover_cfg(bv, outf)

if __name__ == '__main__':
    exit(main())
<|endoftext|>


<commit_before>import json
import requests
from requests.auth import HTTPBasicAuth

class Currency(object):
    RUB = 'RUB'
    USD = 'USD'
    EUR = 'EUR'
    GBP = 'GBP'

class Timezone(object):
    HST = 'HST'
    AKST = 'AKST'
    PST = 'PST'
    MST = 'MST'
    CST = 'CST'
    EST = 'EST'
    AST = 'AST'
    BRT = 'BRT'
    UTC = 'UTC'
    GMT = 'GMT'
    CET = 'CET'
    EET = 'EET'
    MSK = 'MSK'
    AZT = 'AZT'
    AMT = 'AMT'
    SAMT = 'SAMT'
    GET = 'GET'
    TJT = 'TJT'
    YEKT = 'YEKT'
    ALMT = 'ALMT'
    NOVT = 'NOVT'
    KRAT = 'KRAT'
    HKT = 'HKT'
    IRKT = 'IRKT'
    SGT = 'SGT'
    ULAT = 'ULAT'
    YAKT = 'YAKT'
    VLAT = 'VLAT'
    SAKT = 'SAKT'
    ANAT = 'ANAT'

class CloudPaymentsError(Exception):
    def __init__(self, response, message=None):
        self.response = response
        super(CloudPaymentsError, self).__init__(message or
                                                 response.get('Message'))

class Secure3dAuthenticationRequiredException(CloudPaymentsError):
    def __init__(self, response):
        self.transaction_id = response['Model']['TransactionId']
        self.pa_req = response['Model']['PaReq']
        super(Secure3dAuthenticationRequiredException, self).__init__(response)

class PaymentError(CloudPaymentsError):
    def __init__(self, response):
        self.reason = response['Model']['Reason']
        self.reason_code = response['Model']['ReasonCode']
        super(PaymentError, self).__init__(response)

class CloudPayments(object):
    URL = 'https://api.cloudpayments.ru/'

    def __init__(self, public_id, api_secret):
        self.public_id = public_id
        self.api_secret = api_secret

    def _send_request(self, endpoint, params=None):
        auth = HTTPBasicAuth(self.public_id, self.api_secret)
        response = requests.post(self.URL + endpoint, data=params, auth=auth)
        return response.json()

    def test(self):
        response = self._send_request('test')
        if not response['Success']:
            raise CloudPaymentsError(response)

    def charge_card(self, cryptogram, amount, currency, name, ip_address,
                    invoice_id=None, description=None, account_id=None,
                    email=None, data=None, with_auth=False):
        params = {
            'Amount': amount,
            'Currency': currency,
            'IpAddress': ip_address,
            'Name': name,
            'CardCryptogramPacket': cryptogram
        }
        if invoice_id is not None:
            params['InvoiceId'] = invoice_id
        if description is not None:
            params['Description'] = description
        if account_id is not None:
            params['AccountId'] = account_id
        if email is not None:
            params['Email'] = email
        if data is not None:
            params['JsonData'] = json.dumps(data)

        endpoint = ('payments/cards/auth' if with_auth else
                    'payments/cards/charge')
        response = self._send_request(endpoint, params)

        if response['Success']:
            return response['Model']
        if response['Message']:
            raise CloudPaymentsError(response)
        if 'ReasonCode' in response['Model']:
            raise PaymentError(response)
        raise Secure3dAuthenticationRequiredException(response)

    def finish_3d_secure_authentication(self, transaction_id, pa_res):
        params = {
            'TransactionId': transaction_id,
            'PaRes': pa_res
        }
        response = self._send_request('payments/cards/post3ds', params)

        if response['Success']:
            return response['Model']
        raise CloudPaymentsError(response)

    def charge_token(self, token, account_id, amount, currency,
                     ip_address=None, invoice_id=None, description=None,
                     email=None, data=None, with_auth=False):
        params = {
            'Amount': amount,
            'Currency': currency,
            'AccountId': account_id,
            'Token': token
        }
        if invoice_id is not None:
            params['InvoiceId'] = invoice_id
        if description is not None:
            params['Description'] = description
        if ip_address is not None:
            params['IpAddress'] = ip_address
        if email is not None:
            params['Email'] = email
        if data is not None:
            params['JsonData'] = json.dumps(data)

        endpoint = ('payments/tokens/auth' if with_auth else
                    'payments/tokens/charge')
        response = self._send_request(endpoint, params)

        if response['Success']:
            return response['Model']
        if 'Model' in response and 'ReasonCode' in response['Model']:
            raise PaymentError(response)
        raise CloudPaymentsError(response)

    def confirm_payment(self, transaction_id, amount):
        params = {
            'Amount': amount,
            'TransactionId': transaction_id
        }
        response = self._send_request('payments/confirm', params)

        if not response['Success']:
            raise CloudPaymentsError(response)

    def void_payment(self, transaction_id):
        params = {'TransactionId': transaction_id}
        response = self._send_request('payments/void', params)

        if not response['Success']:
            raise CloudPaymentsError(response)

    def refund(self, transaction_id, amount):
        params = {
            'Amount': amount,
            'TransactionId': transaction_id
        }
        response = self._send_request('payments/refund', params)

        if not response['Success']:
            raise CloudPaymentsError(response)

    def find_payment(self, invoice_id):
        params = {'InvoiceId': invoice_id}
        response = self._send_request('payments/find', params)

        if response['Success']:
            return response['Model']
        raise CloudPaymentsError(response)

    def list_payments(self, date, timezone=None):
        params = {'Date': date.isoformat()}
        if timezone is not None:
            params['Timezone'] = timezone

        response = self._send_request('payments/list', params)

        if response['Success']:
            return response['Model']
        raise CloudPaymentsError(response)

    def create_subscription(self, token, account_id, amount, currency,
                            description, email, start_date, interval, period,
                            require_confirmation=False, max_periods=None):
        params = {
            'Token': token,
            'AccountId': account_id,
            'Description': description,
            'Email': email,
            'Amount': amount,
            'Currency': currency,
            'RequireConfiramtion': require_confirmation,
            'StartDate': start_date.isoformat(),
            'Interval': interval,
            'Period': period,
        }
        if max_periods is not None:
            params['MaxPeriods'] = max_periods

        response = self._send_request('subscriptions/create', params)

        if response['Success']:
            return response['Model']
        raise CloudPaymentsError(response)

    def update_subscription(self, subscription_id, amount=None, currency=None,
                            description=None, start_date=None, interval=None,
                            period=None, require_confirmation=None,
                            max_periods=None):
        params  = {
            'Id': subscription_id
        }
        if description is not None:
            params['Description'] = description
        if amount is not None:
            params['Amount'] = amount
        if currency is not None:
            params['Currency'] = currency
        if require_confirmation is not None:
            params['RequireConfirmation'] = require_confirmation
        if start_date is not None:
            params['StartDate'] = start_date.isoformat()
        if interval is not None:
            params['Interval'] = interval
        if period is not None:
            params['Period'] = period
        if max_periods is not None:
            params['MaxPeriods'] = max_periods

        response = self._send_request('subscriptions/update', params)

        if response['Success']:
            return response['Model']
        raise CloudPaymentsError(response)

    def cancel_subscription(self, subscription_id):
        params = {'SubscriptionId': subscription_id}

        response = self._send_request('subscriptions/cancel', params)

        if not response['Success']:
            raise CloudPaymentsError(response)

<commit_msg>Добавил метод для получения информации о подписке<commit_after>import json
import requests
from requests.auth import HTTPBasicAuth

class Currency(object):
    RUB = 'RUB'
    USD = 'USD'
    EUR = 'EUR'
    GBP = 'GBP'

class Timezone(object):
    HST = 'HST'
    AKST = 'AKST'
    PST = 'PST'
    MST = 'MST'
    CST = 'CST'
    EST = 'EST'
    AST = 'AST'
    BRT = 'BRT'
    UTC = 'UTC'
    GMT = 'GMT'
    CET = 'CET'
    EET = 'EET'
    MSK = 'MSK'
    AZT = 'AZT'
    AMT = 'AMT'
    SAMT = 'SAMT'
    GET = 'GET'
    TJT = 'TJT'
    YEKT = 'YEKT'
    ALMT = 'ALMT'
    NOVT = 'NOVT'
    KRAT = 'KRAT'
    HKT = 'HKT'
    IRKT = 'IRKT'
    SGT = 'SGT'
    ULAT = 'ULAT'
    YAKT = 'YAKT'
    VLAT = 'VLAT'
    SAKT = 'SAKT'
    ANAT = 'ANAT'

class CloudPaymentsError(Exception):
    def __init__(self, response, message=None):
        self.response = response
        super(CloudPaymentsError, self).__init__(message or
                                                 response.get('Message'))

class Secure3dAuthenticationRequiredException(CloudPaymentsError):
    def __init__(self, response):
        self.transaction_id = response['Model']['TransactionId']
        self.pa_req = response['Model']['PaReq']
        super(Secure3dAuthenticationRequiredException, self).__init__(response)

class PaymentError(CloudPaymentsError):
    def __init__(self, response):
        self.reason = response['Model']['Reason']
        self.reason_code = response['Model']['ReasonCode']
        super(PaymentError, self).__init__(response)

class CloudPayments(object):
    URL = 'https://api.cloudpayments.ru/'

    def __init__(self, public_id, api_secret):
        self.public_id = public_id
        self.api_secret = api_secret

    def _send_request(self, endpoint, params=None):
        auth = HTTPBasicAuth(self.public_id, self.api_secret)
        response = requests.post(self.URL + endpoint, data=params, auth=auth)
        return response.json()

    def test(self):
        response = self._send_request('test')
        if not response['Success']:
            raise CloudPaymentsError(response)

    def charge_card(self, cryptogram, amount, currency, name, ip_address,
                    invoice_id=None, description=None, account_id=None,
                    email=None, data=None, with_auth=False):
        params = {
            'Amount': amount,
            'Currency': currency,
            'IpAddress': ip_address,
            'Name': name,
            'CardCryptogramPacket': cryptogram
        }
        if invoice_id is not None:
            params['InvoiceId'] = invoice_id
        if description is not None:
            params['Description'] = description
        if account_id is not None:
            params['AccountId'] = account_id
        if email is not None:
            params['Email'] = email
        if data is not None:
            params['JsonData'] = json.dumps(data)

        endpoint = ('payments/cards/auth' if with_auth else
                    'payments/cards/charge')
        response = self._send_request(endpoint, params)

        if response['Success']:
            return response['Model']
        if response['Message']:
            raise CloudPaymentsError(response)
        if 'ReasonCode' in response['Model']:
            raise PaymentError(response)
        raise Secure3dAuthenticationRequiredException(response)

    def finish_3d_secure_authentication(self, transaction_id, pa_res):
        params = {
            'TransactionId': transaction_id,
            'PaRes': pa_res
        }
        response = self._send_request('payments/cards/post3ds', params)

        if response['Success']:
            return response['Model']
        raise CloudPaymentsError(response)

    def charge_token(self, token, account_id, amount, currency,
                     ip_address=None, invoice_id=None, description=None,
                     email=None, data=None, with_auth=False):
        params = {
            'Amount': amount,
            'Currency': currency,
            'AccountId': account_id,
            'Token': token
        }
        if invoice_id is not None:
            params['InvoiceId'] = invoice_id
        if description is not None:
            params['Description'] = description
        if ip_address is not None:
            params['IpAddress'] = ip_address
        if email is not None:
            params['Email'] = email
        if data is not None:
            params['JsonData'] = json.dumps(data)

        endpoint = ('payments/tokens/auth' if with_auth else
                    'payments/tokens/charge')
        response = self._send_request(endpoint, params)

        if response['Success']:
            return response['Model']
        if 'Model' in response and 'ReasonCode' in response['Model']:
            raise PaymentError(response)
        raise CloudPaymentsError(response)

    def confirm_payment(self, transaction_id, amount):
        params = {
            'Amount': amount,
            'TransactionId': transaction_id
        }
        response = self._send_request('payments/confirm', params)

        if not response['Success']:
            raise CloudPaymentsError(response)

    def void_payment(self, transaction_id):
        params = {'TransactionId': transaction_id}
        response = self._send_request('payments/void', params)

        if not response['Success']:
            raise CloudPaymentsError(response)

    def refund(self, transaction_id, amount):
        params = {
            'Amount': amount,
            'TransactionId': transaction_id
        }
        response = self._send_request('payments/refund', params)

        if not response['Success']:
            raise CloudPaymentsError(response)

    def find_payment(self, invoice_id):
        params = {'InvoiceId': invoice_id}
        response = self._send_request('payments/find', params)

        if response['Success']:
            return response['Model']
        raise CloudPaymentsError(response)

    def list_payments(self, date, timezone=None):
        params = {'Date': date.isoformat()}
        if timezone is not None:
            params['Timezone'] = timezone

        response = self._send_request('payments/list', params)

        if response['Success']:
            return response['Model']
        raise CloudPaymentsError(response)

    def create_subscription(self, token, account_id, amount, currency,
                            description, email, start_date, interval, period,
                            require_confirmation=False, max_periods=None):
        params = {
            'Token': token,
            'AccountId': account_id,
            'Description': description,
            'Email': email,
            'Amount': amount,
            'Currency': currency,
            'RequireConfiramtion': require_confirmation,
            'StartDate': start_date.isoformat(),
            'Interval': interval,
            'Period': period,
        }
        if max_periods is not None:
            params['MaxPeriods'] = max_periods

        response = self._send_request('subscriptions/create', params)

        if response['Success']:
            return response['Model']
        raise CloudPaymentsError(response)

    def get_subscription(self, subscription_id):
        params = {'Id': subscription_id}
        response = self._send_request('subscriptions/get', params)

        if response['Success']:
            return response['Model']
        raise CloudPaymentsError(response)

    def update_subscription(self, subscription_id, amount=None, currency=None,
                            description=None, start_date=None, interval=None,
                            period=None, require_confirmation=None,
                            max_periods=None):
        params  = {
            'Id': subscription_id
        }
        if description is not None:
            params['Description'] = description
        if amount is not None:
            params['Amount'] = amount
        if currency is not None:
            params['Currency'] = currency
        if require_confirmation is not None:
            params['RequireConfirmation'] = require_confirmation
        if start_date is not None:
            params['StartDate'] = start_date.isoformat()
        if interval is not None:
            params['Interval'] = interval
        if period is not None:
            params['Period'] = period
        if max_periods is not None:
            params['MaxPeriods'] = max_periods

        response = self._send_request('subscriptions/update', params)

        if response['Success']:
            return response['Model']
        raise CloudPaymentsError(response)

    def cancel_subscription(self, subscription_id):
        params = {'SubscriptionId': subscription_id}

        response = self._send_request('subscriptions/cancel', params)

        if not response['Success']:
            raise CloudPaymentsError(response)

<|endoftext|>


<commit_before>from itertools import ifilter, product, starmap
from operator import mul

def is_palyndrome(n):
    return str(n) == str(n)[::-1]

print reduce(max, ifilter(is_palyndrome, starmap(mul, product(xrange(100, 1000), repeat=2))))
<commit_msg>Fixed indents.<commit_after>from itertools import ifilter, product, starmap
from operator import mul

def is_palyndrome(n):
    return str(n) == str(n)[::-1]

print reduce(max, ifilter(
    is_palyndrome,
    starmap(mul, product(xrange(100, 1000), repeat=2))))
<|endoftext|>


<commit_before>import boto3
import logging
import json
import pprint
import jmespath
from itertools import izip_longest

# Setup simple logging for INFO
logger = logging.getLogger()
logger.setLevel(logging.INFO)

# Setup PrettyPrinter
pp = pprint.PrettyPrinter(indent=2)

# Global variables to adjust behaviour. Change to fit your setup.
cd_app_name = "DemoApplication"
cd_dst_dg_name = "Demo-Tag-Ubuntu"
cd_dg_name = "Demo-Tag-Ubuntu-PreProd"
# Replace the following tag Key and Value for the one used in your initial Deployment Group
cd_dg_tag = {'Key': 'CodeDeploy', 'Value': 'PreProd'}

# Function to deal with paginated results.
def get_all_results(client_function, jmes_query=None, flatten=False, **params):
    results = list()
    tmpResults = [client_function(**params)]

    while tmpResults[-1].get('nextToken'):
        tmpResults.append(client_function(nextToken=tmpResults[-1].get('nextToken'), **params))

    if jmes_query:
        for r in tmpResults:
            results.append(jmespath.search(jmes_query, r))
        if flatten:
            return [item for sublist in results for item in sublist]
        else:
            return results
    else:
        if flatten:
            return [item for sublist in tmpResults for item in sublist]
        else:
            return tmpResults

def autodeploy_handler(event, context):
    # Define the connections to the correct region
    ec2 = boto3.resource('ec2', region_name=event['region'])
    cd = boto3.client('codedeploy', region_name=event['region'])

    print "Event Region: {0}".format(event['region'])
    print "Event Time: {0}".format(event['time'])
    print "Instance ID: {0}".format(event['detail']['instance-id'])
    print "Instance Status: {0}".format(event['detail']['state'])

    # Obtain the EC2 object of the instance from the event
    instance = ec2.Instance(event['detail']['instance-id'])

    # Print a dict of the Tags (see https://github.com/boto/boto3/issues/264)
    print "Tags: {0}".format(dict(map(lambda x: (x['Key'], x['Value']), instance.tags or [])))

    # Print a dict of the filtering Tag
    print "Filter: {{'{0}': '{1}'}}".format(cd_dg_tag['Key'], cd_dg_tag['Value'])

    if instance.tags is not None:
        for t in instance.tags:
            if t['Key'] == cd_dg_tag['Key'] and t['Value'] == cd_dg_tag['Value']:
                print "Instance {0} is a target for AutoDeploy!".format(instance.id)

    # Get all successful deployments from the destination Deployment Group
    deployments = get_all_results(
        cd.list_deployments,
        'deployments',
        flatten=True,
        applicationName = cd_app_name,
        deploymentGroupName = cd_dst_dg_name,
        includeOnlyStatuses = ['Succeeded']
        )

    # Get details about the previously found deployments, in batches (max 100)
    deployments_info = list()
    chunk_size = [iter(deployments)] * 100 # batch_get_deployments has a max of 100 IDs as input

    for chunk in izip_longest(*chunk_size):
        chunk = filter(lambda x: x!=None, chunk) # Remove None fillings from list
        deployments_info.extend(get_all_results(
            cd.batch_get_deployments,
            'deploymentsInfo[].[deploymentId,createTime]',
            flatten=True,
            deploymentIds=chunk)
            )

    deployments_info.sort(key=lambda x: x[1])
    latest_deployment = deployments_info[-1][0]

    deployment_template = jmespath.search(
        'deploymentInfo.{applicationName:applicationName,deploymentGroupName:deploymentGroupName,revision:revision,deploymentConfigName:deploymentConfigName}',
        cd.get_deployment(deploymentId=latest_deployment)
        )
    deployment_template['deploymentGroupName'] = cd_dg_name # Overwrite the Deployment Group name
    deployment_template['description'] = "Created from Lambda"

    resp = cd.create_deployment(**deployment_template)
    pp.pprint(resp)
<commit_msg>Obtain successful revision straight from the DG and remove all unnecessary code<commit_after>import boto3
import logging
import json
import pprint
import jmespath
from itertools import izip_longest

# Setup simple logging for INFO
logger = logging.getLogger()
logger.setLevel(logging.INFO)

# Setup PrettyPrinter
pp = pprint.PrettyPrinter(indent=2)

# Global variables to adjust behaviour. Change to fit your setup.
cd_app_name = "DemoApplication"
cd_dst_dg_name = "Demo-Tag-Ubuntu"
cd_dg_name = "Demo-Tag-Ubuntu-PreProd"
# Replace the following tag Key and Value for the one used in your initial Deployment Group
cd_dg_tag = {'Key': 'CodeDeploy', 'Value': 'PreProd'}

def autodeploy_handler(event, context):
    # Define the connections to the correct region
    ec2 = boto3.resource('ec2', region_name=event['region'])
    cd = boto3.client('codedeploy', region_name=event['region'])

    print "Event Region: {0}".format(event['region'])
    print "Event Time: {0}".format(event['time'])
    print "Instance ID: {0}".format(event['detail']['instance-id'])
    print "Instance Status: {0}".format(event['detail']['state'])

    # Obtain the EC2 object of the instance from the event
    instance = ec2.Instance(event['detail']['instance-id'])

    # Print a dict of the Tags (see https://github.com/boto/boto3/issues/264)
    print "Tags: {0}".format(dict(map(lambda x: (x['Key'], x['Value']), instance.tags or [])))

    # Print a dict of the filtering Tag
    print "Filter: {{'{0}': '{1}'}}".format(cd_dg_tag['Key'], cd_dg_tag['Value'])

    if instance.tags is not None:
        for t in instance.tags:
            if t['Key'] == cd_dg_tag['Key'] and t['Value'] == cd_dg_tag['Value']:
                print "Instance {0} is a target for AutoDeploy!".format(instance.id)

    # Get latest successful Revision from the destination Deployment Group
    revision = jmespath.search(
        'deploymentGroupInfo.{revision:targetRevision}',
        cd.get_deployment_group(
            applicationName = cd_app_name,
            deploymentGroupName = cd_dst_dg_name,
            )
        )

    # Create Deployment for the inital Deployment Group
    resp = cd.create_deployment(
        applicationName = cd_app_name,
        deploymentGroupName = cd_dg_name,
        **revision
        )
    pp.pprint(resp)
<|endoftext|>


<commit_before>import datetime
from fabric.api import env
from fabric.contrib import django
django.project('openbudgets')
from django.conf import settings

env.user = 'robot'
env.roledefs = {
    'demo': ['162.243.15.220'],
    'openmuni-budgets': [''],
}

CONFIG = {
    'debug': True,
    'sentry_dsn': '',
    'user': env.user,
    'machine_location': env.roledefs['demo'][0],
    'machine_port': 80,
    'project_name': 'openbudgets',
    'project_root': '/srv/projects/openbudgets',
    'project_env': '/srv/environments/openbudgets',
    'dataset_root': settings.OPENBUDGETS_DATA['directory'],
    'dataset_repo': settings.OPENBUDGETS_DATA['repo'],
    'dataset_branch': settings.OPENBUDGETS_DATA['branch'],
    'db_name': 'openbudgets',
    'db_user': env.user,
    'db_dump_file': settings.OPENBUDGETS_DATA['db_dump'],
    'app_location': '127.0.0.1',
    'app_port': 9000,
    'app_workers': 4,
    'app_timeout': 45,
    'app_wsgi': 'openbudgets.wsgi:application',
    'repo': 'https://github.com/prjts/openbudgets',
    'branch': 'develop',
    'allowed_hosts': ['openbudgets.io', 'en.openbudgets.io', 'he.openbudgets.io',
                      'ar.openbudgets.io','ru.openbudgets.io'],
    'cookie_domain': '.openbudgets.io',
    'nginx_access_log': '/srv/logs/openbudgets_nginx_access.log',
    'nginx_error_log': '/srv/logs/openbudgets_nginx_error.log',
    'gunicorn_access_log': '/srv/logs/openbudgets_gunicorn_access.log',
    'gunicorn_error_log': '/srv/logs/openbudgets_gunicorn_error.log',
    'redis_access_log': '/srv/logs/openbudgets_redis_access.log',
    'redis_error_log': '/srv/logs/openbudgets_redis_error.log',
    'timestamp': datetime.datetime.now(),
}

WORKON = 'workon ' + CONFIG['project_name']

DEACTIVATE = 'deactivate'
<commit_msg>Extending app timeout temporarily for imports<commit_after>import datetime
from fabric.api import env
from fabric.contrib import django
django.project('openbudgets')
from django.conf import settings

env.user = 'robot'
env.roledefs = {
    'demo': ['162.243.15.220'],
    'openmuni-budgets': [''],
}

CONFIG = {
    'debug': True,
    'sentry_dsn': '',
    'user': env.user,
    'machine_location': env.roledefs['demo'][0],
    'machine_port': 80,
    'project_name': 'openbudgets',
    'project_root': '/srv/projects/openbudgets',
    'project_env': '/srv/environments/openbudgets',
    'dataset_root': settings.OPENBUDGETS_DATA['directory'],
    'dataset_repo': settings.OPENBUDGETS_DATA['repo'],
    'dataset_branch': settings.OPENBUDGETS_DATA['branch'],
    'db_name': 'openbudgets',
    'db_user': env.user,
    'db_dump_file': settings.OPENBUDGETS_DATA['db_dump'],
    'app_location': '127.0.0.1',
    'app_port': 9000,
    'app_workers': 4,
    'app_timeout': 150,
    'app_wsgi': 'openbudgets.wsgi:application',
    'repo': 'https://github.com/prjts/openbudgets',
    'branch': 'develop',
    'allowed_hosts': ['openbudgets.io', 'en.openbudgets.io', 'he.openbudgets.io',
                      'ar.openbudgets.io','ru.openbudgets.io'],
    'cookie_domain': '.openbudgets.io',
    'nginx_access_log': '/srv/logs/openbudgets_nginx_access.log',
    'nginx_error_log': '/srv/logs/openbudgets_nginx_error.log',
    'gunicorn_access_log': '/srv/logs/openbudgets_gunicorn_access.log',
    'gunicorn_error_log': '/srv/logs/openbudgets_gunicorn_error.log',
    'redis_access_log': '/srv/logs/openbudgets_redis_access.log',
    'redis_error_log': '/srv/logs/openbudgets_redis_error.log',
    'timestamp': datetime.datetime.now(),
}

WORKON = 'workon ' + CONFIG['project_name']

DEACTIVATE = 'deactivate'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""

flask_couchdb.manager
~~~~~~~~~~~~~~~~~~~~~

This package provides utilities to make using Flask with a
CouchDB database server easier.

:copyright: 2010 Matthew "LeafStorm" Frazier
:license:   MIT/X11, see LICENSE for details

"""

import itertools
import couchdb
from couchdb.design import ViewDefinition as CouchDBViewDefinition
from flask import g, current_app
from flask import _app_ctx_stack as stack

__all__ = ['CouchDB']

### The manager class

class CouchDB(object):
    """
    This manages connecting to the database every request and synchronizing
    the view definitions to it.

    :param auto_sync: Whether to automatically sync the database every
                      request. (Defaults to `False`.)
    """
    def __init__(self, app=None, server=None, db=None):
        self.doc_viewdefs = {}
        self.general_viewdefs = []
        self.sync_callbacks = []
        self.db = db
        self.server = server
        self.app = app
        if self.app is not None:
           self.init_app(app)

    def init_app(self, app):
        app.before_request(self.request_start)

    def request_start(self):
        g.couch = self

    def all_viewdefs(self):
        """
        This iterates through all the view definitions registered generally
        and the ones on specific document classes.
        """
        return itertools.chain(self.general_viewdefs,
                               *self.doc_viewdefs.itervalues())

    def add_document(self, dc):
        """
        This adds all the view definitions from a document class so they will
        be added to the database when it is synced.

        :param dc: The class to add. It should be a subclass of `Document`.
        """
        viewdefs = []
        for name in dir(dc):
            item = getattr(dc, name)
            if isinstance(item, CouchDBViewDefinition):
                viewdefs.append(item)
        if viewdefs:
            self.doc_viewdefs[dc] = viewdefs

    def add_viewdef(self, viewdef):
        """
        This adds standalone view definitions (it should be a `ViewDefinition`
        instance or list thereof) to the manager. It will be added to the
        design document when it it is synced.

        :param viewdef: The view definition to add. It can also be a tuple or
                        list.
        """
        if isinstance(viewdef, CouchDBViewDefinition):
            self.general_viewdefs.append(viewdef)
        else:
            self.general_viewdefs.extend(viewdef)

    def on_sync(self, fn):
        """
        This adds a callback to run when the database is synced. The callbacks
        are passed the live database (so they should use that instead of
        relying on the thread locals), and can do pretty much whatever they
        want. The design documents have already been synchronized. Callbacks
        are called in the order they are added, but you shouldn't rely on
        that.

        If you can reliably detect whether it is necessary, this may be a good
        place to add default data. However, the callbacks could theoretically
        be run on every request, so it is a bad idea to insert the default
        data every time.

        :param fn: The callback function to add.
        """
        self.sync_callbacks.append(fn)

    def connect_db(self, app=None):
        """
        This connects to the database for the given app. It presupposes that
        the database has already been synced, and as such an error will be
        raised if the database does not exist.

        :param app: The app to get the settings from.
        """
        if self.db: return self.db
        self.server = couchdb.Server( app.config['COUCHDB_SERVER'] )
        self.db = self.get_or_create_db( app.config['COUCHDB_DATABASE'] )
        return self.db

    def get_or_create_db(self, db_name, server=None):
        server = server or self.server
        if db_name not in server:
           db = server.create(db_name)
        else:
           db = server[db_name]
        return db

    def sync(self, app=None):
        """
        This syncs the database for the given app. It will first make sure the
        database exists, then synchronize all the views and run all the
        callbacks with the connected database.

        It will run any callbacks registered with `on_sync`, and when the
        views are being synchronized, if a method called `update_design_doc`
        exists on the manager, it will be called before every design document
        is updated.

        :param app: The application to synchronize with.
        """
        db = self.db
        CouchDBViewDefinition.sync_many(
            db, tuple(self.all_viewdefs()),
            callback=getattr(self, 'update_design_doc', None)
        )
        for callback in self.sync_callbacks:
            callback(db)

<commit_msg>serializable properties do not work well with getattr<commit_after># -*- coding: utf-8 -*-
"""

flask_couchdb.manager
~~~~~~~~~~~~~~~~~~~~~

This package provides utilities to make using Flask with a
CouchDB database server easier.

:copyright: 2010 Matthew "LeafStorm" Frazier
:license:   MIT/X11, see LICENSE for details

"""

import itertools
import couchdb
from couchdb.design import ViewDefinition as CouchDBViewDefinition
from flask import g, current_app
from flask import _app_ctx_stack as stack

__all__ = ['CouchDB']

### The manager class

class CouchDB(object):
    """
    This manages connecting to the database every request and synchronizing
    the view definitions to it.

    :param auto_sync: Whether to automatically sync the database every
                      request. (Defaults to `False`.)
    """
    def __init__(self, app=None, server=None, db=None):
        self.doc_viewdefs = {}
        self.general_viewdefs = []
        self.sync_callbacks = []
        self.db = db
        self.server = server
        self.app = app
        if self.app is not None:
           self.init_app(app)

    def init_app(self, app):
        app.before_request(self.request_start)

    def request_start(self):
        g.couch = self

    def all_viewdefs(self):
        """
        This iterates through all the view definitions registered generally
        and the ones on specific document classes.
        """
        return itertools.chain(self.general_viewdefs,
                               *self.doc_viewdefs.itervalues())

    def add_document(self, dc):
        """
        This adds all the view definitions from a document class so they will
        be added to the database when it is synced.

        :param dc: The class to add. It should be a subclass of `Document`.
        """
        viewdefs = []
        for name in dir(dc):
            try:
                item = getattr(dc, name)
                if isinstance(item, CouchDBViewDefinition):
                    viewdefs.append(item)
            except:
                pass
        if viewdefs:
            self.doc_viewdefs[dc] = viewdefs

    def add_viewdef(self, viewdef):
        """
        This adds standalone view definitions (it should be a `ViewDefinition`
        instance or list thereof) to the manager. It will be added to the
        design document when it it is synced.

        :param viewdef: The view definition to add. It can also be a tuple or
                        list.
        """
        if isinstance(viewdef, CouchDBViewDefinition):
            self.general_viewdefs.append(viewdef)
        else:
            self.general_viewdefs.extend(viewdef)

    def on_sync(self, fn):
        """
        This adds a callback to run when the database is synced. The callbacks
        are passed the live database (so they should use that instead of
        relying on the thread locals), and can do pretty much whatever they
        want. The design documents have already been synchronized. Callbacks
        are called in the order they are added, but you shouldn't rely on
        that.

        If you can reliably detect whether it is necessary, this may be a good
        place to add default data. However, the callbacks could theoretically
        be run on every request, so it is a bad idea to insert the default
        data every time.

        :param fn: The callback function to add.
        """
        self.sync_callbacks.append(fn)

    def connect_db(self, app=None):
        """
        This connects to the database for the given app. It presupposes that
        the database has already been synced, and as such an error will be
        raised if the database does not exist.

        :param app: The app to get the settings from.
        """
        if self.db: return self.db
        self.server = couchdb.Server( app.config['COUCHDB_SERVER'] )
        self.db = self.get_or_create_db( app.config['COUCHDB_DATABASE'] )
        return self.db

    def get_or_create_db(self, db_name, server=None):
        server = server or self.server
        if db_name not in server:
           db = server.create(db_name)
        else:
           db = server[db_name]
        return db

    def sync(self, app=None):
        """
        This syncs the database for the given app. It will first make sure the
        database exists, then synchronize all the views and run all the
        callbacks with the connected database.

        It will run any callbacks registered with `on_sync`, and when the
        views are being synchronized, if a method called `update_design_doc`
        exists on the manager, it will be called before every design document
        is updated.

        :param app: The application to synchronize with.
        """
        db = self.db
        CouchDBViewDefinition.sync_many(
            db, tuple(self.all_viewdefs()),
            callback=getattr(self, 'update_design_doc', None)
        )
        for callback in self.sync_callbacks:
            callback(db)

<|endoftext|>


<commit_before># Copyright 2021 DeepMind Technologies Limited. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""The CLRS Algorithmic Reasoning Benchmark."""

from clrs import models
from clrs._src import algorithms
from clrs._src import decoders
from clrs._src import processors
from clrs._src.dataset import CLRSDataset
from clrs._src.dataset import create_chunked_dataset
from clrs._src.dataset import create_dataset
from clrs._src.dataset import get_clrs_folder
from clrs._src.dataset import get_dataset_gcp_url
from clrs._src.model import evaluate
from clrs._src.model import evaluate_hints
from clrs._src.model import Model
from clrs._src.probing import DataPoint
from clrs._src.processors import get_processor_factory
from clrs._src.samplers import build_sampler
from clrs._src.samplers import CLRS30
from clrs._src.samplers import Features
from clrs._src.samplers import Feedback
from clrs._src.samplers import Sampler
from clrs._src.samplers import Trajectory
from clrs._src.specs import Location
from clrs._src.specs import OutputClass
from clrs._src.specs import Spec
from clrs._src.specs import SPECS
from clrs._src.specs import Stage
from clrs._src.specs import Type

__version__ = "0.0.2"

__all__ = (
    "build_sampler",
    "CLRS30",
    "create_chunked_dataset",
    "create_dataset",
    "get_clrs_folder",
    "get_dataset_gcp_url",
    "get_processor_factory",
    "DataPoint",
    "evaluate",
    "evaluate_hints",
    "Features",
    "Feedback",
    "Location",
    "Model",
    "Sampler",
    "Spec",
    "SPECS",
    "Stage",
    "Trajectory",
    "Type",
)
<commit_msg>Increase version number for PyPI release.<commit_after># Copyright 2021 DeepMind Technologies Limited. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""The CLRS Algorithmic Reasoning Benchmark."""

from clrs import models
from clrs._src import algorithms
from clrs._src import decoders
from clrs._src import processors
from clrs._src.dataset import CLRSDataset
from clrs._src.dataset import create_chunked_dataset
from clrs._src.dataset import create_dataset
from clrs._src.dataset import get_clrs_folder
from clrs._src.dataset import get_dataset_gcp_url
from clrs._src.model import evaluate
from clrs._src.model import evaluate_hints
from clrs._src.model import Model
from clrs._src.probing import DataPoint
from clrs._src.processors import get_processor_factory
from clrs._src.samplers import build_sampler
from clrs._src.samplers import CLRS30
from clrs._src.samplers import Features
from clrs._src.samplers import Feedback
from clrs._src.samplers import Sampler
from clrs._src.samplers import Trajectory
from clrs._src.specs import Location
from clrs._src.specs import OutputClass
from clrs._src.specs import Spec
from clrs._src.specs import SPECS
from clrs._src.specs import Stage
from clrs._src.specs import Type

__version__ = "1.0.0"

__all__ = (
    "build_sampler",
    "CLRS30",
    "create_chunked_dataset",
    "create_dataset",
    "get_clrs_folder",
    "get_dataset_gcp_url",
    "get_processor_factory",
    "DataPoint",
    "evaluate",
    "evaluate_hints",
    "Features",
    "Feedback",
    "Location",
    "Model",
    "Sampler",
    "Spec",
    "SPECS",
    "Stage",
    "Trajectory",
    "Type",
)
<|endoftext|>


<commit_before>import logging
import virtool.routes
import virtool.sessions

from aiohttp import web
from motor import motor_asyncio

logger = logging.getLogger(__name__)

def create_app(loop, db_name="virtool"):
    app = web.Application(loop=loop, middlewares=[virtool.sessions.middleware_factory])

    app["db"] = motor_asyncio.AsyncIOMotorClient(io_loop=loop)[db_name]

    virtool.routes.setup_routes(app)

    return app
<commit_msg>Put some old functions into the new web module<commit_after>import os
import sys
import logging
import subprocess
import virtool.routes
import virtool.sessions

from aiohttp import web
from motor import motor_asyncio

logger = logging.getLogger(__name__)

def create_app(loop, db_name="virtool"):
    app = web.Application(loop=loop, middlewares=[virtool.sessions.middleware_factory])

    app["db"] = motor_asyncio.AsyncIOMotorClient(io_loop=loop)[db_name]

    virtool.routes.setup_routes(app)

    return app

def reload():
    exe = sys.executable

    if exe.endswith("python"):
        os.execl(exe, exe, *sys.argv)

    if exe.endswith("run"):
        os.execv(exe, sys.argv)

    raise SystemError("Could not determine executable type")

def shutdown(app, exit_code=0):
    """
    Shutdown the server by cancelling all jobs and calling :func:`sys.exit`.

    :param exit_code: the exit code to return.
    :type exit_code: int

    """
    logging.info("Shutting down")

    job_manager = app.get("job_manager")

    if job_manager:
        for job_id in job_manager.jobs_dict:
            job_manager.cancel(job_id)

    logging.info("Exiting")

    sys.exit(exit_code)

def find_server_version(install_path="."):
    try:
        return subprocess.check_output(["git", "describe", "--tags"]).decode().rstrip()
    except (subprocess.CalledProcessError, FileNotFoundError):
        try:
            version_file_path = os.path.join(install_path, "VERSION")

            with open(version_file_path, "r") as version_file:
                return version_file.read().rstrip()

        except FileNotFoundError:
            logger.critical("Could not determine software version.")
            return "Unknown"
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import binascii

# Module specific Exception
class ConnectionError(Exception):
    pass

class TransmissionError(Exception):
    pass

class ReceptionError(Exception):
    pass

class TimeoutError(Exception):
    pass

class ConfigurationError(Exception):
    pass

class LoRa(object):

    # Helper Function
    def _set_parent_dev(self):
        try:
            import pyb
        except:
            self._parent_dev = 'pc'
        else:
            self._parent_dev = 'pyb'

    # Create connection to LoRa module from parent device
    def _open(self, serPort=1):
        if self._parent_dev == 'pyb':
            # via micropython board
            import pyb

            # try to connect
            try:
                ser = pyb.UART(serPort, 57600)
                ser.init(baudrate=57600,
                         bits=8,
                         parity=None,
                         stop=1,
                         timeout=self._read_timeout_serial,
                         timeout_char=self._read_timeout_serial)
            # error handling
            except:
                raise ConnectionError("No LoRa module found.")

        else:
            # via computer
            import serial

            # try to connect
            try:
                ser = serial.Serial(serPort,
                                    baudrate=57600,
                                    bytesize=serial.EIGHTBITS,
                                    parity=serial.PARITY_NONE,
                                    stopbits=serial.STOPBITS_ONE,
                                    timeout=(self._read_timeout_serial / 1000))
            # error handling
            except serial.SerialException:
                raise ConnectionError("No LoRa module at given port.")
            except ValueError:
                raise ConnectionError("Wrong configuration parameter given.")

        self.port = serPort
        self._parent_ser = ser

    # End connection to parent device
    def _close(self):
        if self._parent_dev == 'pc':
            self._parent_ser.close()

    # Communicate with LoRa module via parent device
    def _ser_write(self, strIn):
        # reset error handling flags
        transmit_failed = False
        transmit_incomplete = False

        if self._parent_dev == 'pc':
            # format input string
            data = "{}\r\n".format(strIn)
            data = data.encode('utf-8')

            # try to send data (blocking)
            try:
                transmitted_bytes = self._parent_ser.write(data)
            except serial.SerialTimeoutException:
                transmit_failed = True
            else:
                if transmitted_bytes < len(data):
                    transmit_incomplete = True
        else:
            # format input string
            data = "{}\r\n".format(strIn)

            # send data (blocking)
            transmitted_bytes = self._parent_ser.write(data)

            # verify transmit
            if transmitted_bytes == 0:
                transmit_failed = True
            elif transmitted_bytes < len(data):
                transmit_incomplete = True

        # error handling
        if transmit_failed:
            raise TransmissionError("Write failed due to timeout.")
        if transmit_incomplete:
            raise TransmissionError("Write was incomplete.")

        # debugging
        if self._debug:
            print("LoRa:_ser_write : {}".format(data))

    def _ser_read(self):
        # read one line from parent device (blocking)
        received_bytes = self._parent_ser.readline()

        # verification of timeout
        if received_bytes is None or len(received_bytes) == 0:
            raise TimeoutError("Timeout occurred during reception.")

        # format output string
        if self._parent_dev == 'pc':
            received_bytes = received_bytes.decode('utf-8')

        # debugging
        if self._debug:
            print("LoRa:_ser_read : {}".format(received_bytes.strip()))

        return received_bytes

    def _ser_write_read_verify(self, strIn, strOut=0):
        # transmit data
        self._ser_write(strIn)

        # wait for answer
        answer = self._ser_read()

        # if requested, verify return
        if strOut is not 0:
            if answer.strip() != strOut:
                # exception if LoRa module is already receiving and set to be
                # receiving again it will respond "busy" instead of "ok"
                if strIn.startswith('radio rx ') and answer.strip() == 'busy':
                    pass
                else:
                    raise ReceptionError("Transmitted and received string " +
                                         "are not corresponding.")

        return answer

    # LoRa communication functions
    def recv(self):
        if self._debug:
            print("LoRa:recv : Prepare for reception.")

        # try to configure device as receiver
        try:
            self._ser_write_read_verify("mac pause")
            self._ser_write_read_verify("radio rx 0", "ok")
        except (TransmissionError, ReceptionError):
            raise ConfigurationError("Configuration as receiver failed.")
        else:
            # obtain SNR value
            if self._debug:
                try:
                    self._ser_write("radio get snr")
                    snr = self._ser_read()
                    # range -128 to 127
                    print("LoRa:recv : SNR = %d".format(snr))
                except ReceptionError:
                    raise ConfigurationError("Could not obtain SNR value.")

            # wait for incoming data
            if self._debug:
                print("LoRa:recv : Receiving...")
            try:
                received_data = self._ser_read()
            except (ReceptionError, TimeoutError) as e:
                raise e
            else:
                # verify if no LoRa timeout occurred
                if received_data.strip() == 'radio_err':
                    raise TimeoutError("Timeout occurred during reception.")

                # strip data
                received_data = received_data[8:].strip()

                if self._debug:
                    print("LoRa:recv : {}".format(received_data))

                return received_data

    def send(self, tx_data):
        if self._debug:
            print("LoRa:send : Prepare for transmission.")

        # try to configure device as transmitter
        try:
            self._ser_write_read_verify("mac pause")
        except (TransmissionError, ReceptionError):
            raise ConfigurationError("Configuration as transmitter failed.")

        if self._debug:
            print("LoRa:send : Sending...")
        try:
            # send data
            self._ser_write_read_verify("radio tx " + str(tx_data), "ok")

            # read out transmission verification
            ret = self._ser_read()
            if ret.strip() != "radio_tx_ok":
                raise TransmissionError("No transmission confirmation " +
                                        "received.")
        except (TransmissionError, ReceptionError):
            raise TransmissionError("Error while sending.")
        else:
            if self._debug:
                print("LoRa:send : " + str(tx_data))

    def send_str(self, tx_str):
        try:
            tx_data = binascii.hexlify(tx_str)
        except TypeError:
            raise TransmissionError("Transmitted string cannot be converted.")
        else:
            self.send(tx_data)

    def recv_str(self):
        # receive data
        rx_data = self.recv()

        # try to convert data to string
        try:
            text = binascii.unhexlify(rx_data)
        except TypeError:
            raise ReceptionError("Received data has odd length")
        else:
            # TODO: comment
            if(len(rx_data) > 2):
                text = binascii.unhexlify(rx_data[:-1])

        return text

    # Constructor
    def __init__(self, port, timeout_serial=2000, timeout_lora=2000,
                 debug=False):
        # set internal parameters
        self._read_timeout_serial = timeout_serial  # in ms
        self._read_timeout_lora = timeout_lora      # in ms, 0 = endless
        self._debug = debug

        try:
            if self._debug:
                print("LoRa:__init__ : Start initialization.")

            # get parent device
            self._set_parent_dev()

            # open serial connection on parent device
            self._open(port)

            # configure LoRa module via parent device
            self._ser_write_read_verify("radio set mod lora", "ok")
            self._ser_write_read_verify("radio set freq 868000000", "ok")
            self._ser_write_read_verify("radio set pwr 14", "ok")
            self._ser_write_read_verify("radio set sf sf12", "ok")
            self._ser_write_read_verify("radio set afcbw 125", "ok")
            self._ser_write_read_verify("radio set rxbw 250", "ok")
            self._ser_write_read_verify("radio set fdev 5000", "ok")
            self._ser_write_read_verify("radio set prlen 8", "ok")
            self._ser_write_read_verify("radio set crc on", "ok")
            self._ser_write_read_verify("radio set cr 4/8", "ok")
            self._ser_write_read_verify("radio set wdt 0", "ok")
            self._ser_write_read_verify("radio set sync 12", "ok")
            self._ser_write_read_verify("radio set bw 250", "ok")

            # set reception timeout
            if self._read_timeout_lora > 0:
                self._ser_write_read_verify("radio set wdt {}".format(
                    self._read_timeout_lora), "ok")

        except (TransmissionError, ReceptionError):
            raise ConfigurationError("Initial configuration failed.")
        else:
            if self._debug:
                print("LoRa:__init__ : Connection established and " +
                      "configuration completed successfully.")

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self._close()

    def __repr__(self):
        return "<LoRa module connection on port %d>".format(self.port)

# execute only if run as a script
if __name__ == "__main__":
    import argparse

    # command line arguments and parsing
    parser = argparse.ArgumentParser(description='LoRa Script Arguments')
    parser.add_argument('-d', '--debug',
                        action='store_true', help='Debug mode')
    parser.add_argument('-t', '--transmit',
                        default='',
                        help='Data to transmit')
    parser.add_argument('-r', '--receive',
                        default='2000',
                        help='Timeout upon reception (in ms)')
    parser.add_argument('-p', '--port',
                        default='ttyUSB0',
                        help='Serial Port ' +
                        'e.g. pyb, ttyUSB1, /dev/tty.usbserial-A5046HZ5')
    args = parser.parse_args()

    debug = args.debug
    serial_port = args.port
    tx_data = args.transmit
    timeout = int(args.receive)

    # create connection
    try:
        # set same timeout for serial connection and LoRa receiver
        lora = LoRa(serial_port, timeout, timeout, debug)
    except ConfigurationError as e:
        print("Error occurred: " + str(e))

    # start operation
    if tx_data != "":
        try:
            lora.send_str(tx_data)
        except (ConfigurationError, TransmissionError) as e:
            print("Error occurred: " + str(e))
    else:
        while True:
            try:
                lora.recv_str()
            except TimeoutError:
                print("Reception timeout occurred, continuing.")
            except (ConfigurationError, ReceptionError) as e:
                print("Error occurred: " + str(e))
                break
<commit_msg>added docstrings<commit_after>#!/usr/bin/env python3
import binascii

# Module specific Exception
class ConnectionError(Exception):
    '''Parent device (PC or pyb) failing to connect to LoRa module'''
    pass

class TransmissionError(Exception):
    '''Error during transmission (in LoRa or serial communication)'''
    pass

class ReceptionError(Exception):
    '''Error during reception (in LoRa or serial communication)'''
    pass

class TimeoutError(Exception):
    '''LoRa module reception timeout error'''
    pass

class ConfigurationError(Exception):
    '''Parent device (PC or pyb) failing to configure LoRa module for task'''
    pass

class LoRa(object):
    '''Representing a physical connection to a RN2483 LoRa module'''

    # Helper Function
    def _set_parent_dev(self):
        '''Set parent device (PC or micropython board)'''
        try:
            import pyb
        except:
            self._parent_dev = 'pc'
        else:
            self._parent_dev = 'pyb'

    # Create connection to LoRa module from parent device
    def _open(self, serPort=1):
        '''Open serial connection to LoRa module'''
        if self._parent_dev == 'pyb':
            # via micropython board
            import pyb

            # try to connect
            try:
                ser = pyb.UART(serPort, 57600)
                ser.init(baudrate=57600,
                         bits=8,
                         parity=None,
                         stop=1,
                         timeout=self._read_timeout_serial,
                         timeout_char=self._read_timeout_serial)
            # error handling
            except:
                raise ConnectionError("No LoRa module found.")

        else:
            # via computer
            import serial

            # try to connect
            try:
                # set serial write timeout to same value as read timeout
                # to avoid being blocking for too long
                timeout = self._read_timeout_serial / 1000

                ser = serial.Serial(serPort,
                                    baudrate=57600,
                                    bytesize=serial.EIGHTBITS,
                                    parity=serial.PARITY_NONE,
                                    stopbits=serial.STOPBITS_ONE,
                                    timeout=timeout,
                                    write_timeout=timeout)
            # error handling
            except serial.SerialException:
                raise ConnectionError("No LoRa module at given port.")
            except ValueError:
                raise ConnectionError("Wrong configuration parameter given.")

        self.port = serPort
        self._parent_ser = ser

    # End connection to parent device
    def _close(self):
        '''Close serial connection to LoRa module'''
        if self._parent_dev == 'pc':
            self._parent_ser.close()

    # Communicate with LoRa module via parent device
    def _ser_write(self, strIn):
        '''Write via serial connection to LoRa module'''
        # reset error handling flags
        transmit_failed = False
        transmit_incomplete = False

        if self._parent_dev == 'pc':
            # format input string
            data = "{}\r\n".format(strIn)
            data = data.encode('utf-8')

            # try to send data (blocking)
            try:
                transmitted_bytes = self._parent_ser.write(data)
            except serial.SerialTimeoutException:
                transmit_failed = True
            else:
                if transmitted_bytes < len(data):
                    transmit_incomplete = True
        else:
            # format input string
            data = "{}\r\n".format(strIn)

            # send data (blocking)
            transmitted_bytes = self._parent_ser.write(data)

            # verify transmit
            if transmitted_bytes == 0:
                transmit_failed = True
            elif transmitted_bytes < len(data):
                transmit_incomplete = True

        # error handling
        if transmit_failed:
            raise TransmissionError("Write failed due to timeout.")
        if transmit_incomplete:
            raise TransmissionError("Write was incomplete.")

        # debugging
        if self._debug:
            print("LoRa:_ser_write : {}".format(data))

    def _ser_read(self):
        '''Read via serial connection from LoRa module'''
        # read one line from parent device (blocking)
        received_bytes = self._parent_ser.readline()

        # verification of timeout
        if received_bytes is None or len(received_bytes) == 0:
            raise TimeoutError("Timeout occurred during reception.")

        # format output string
        if self._parent_dev == 'pc':
            received_bytes = received_bytes.decode('utf-8')

        # debugging
        if self._debug:
            print("LoRa:_ser_read : {}".format(received_bytes.strip()))

        return received_bytes

    def _ser_write_read_verify(self, strIn, strOut=0):
        '''Perform serial write and read and verify the read output'''
        # transmit data
        self._ser_write(strIn)

        # wait for answer
        answer = self._ser_read()

        # if requested, verify return
        if strOut is not 0:
            if answer.strip() != strOut:
                # exception if LoRa module is already receiving and set to be
                # receiving again it will respond "busy" instead of "ok"
                if strIn.startswith('radio rx ') and answer.strip() == 'busy':
                    pass
                else:
                    raise ReceptionError("Transmitted and received string " +
                                         "are not corresponding.")

        return answer

    # LoRa communication functions
    def recv(self):
        '''Set LoRa module in receiver mode and read received data'''
        if self._debug:
            print("LoRa:recv : Prepare for reception.")

        # try to configure device as receiver
        try:
            self._ser_write_read_verify("mac pause")
            self._ser_write_read_verify("radio rx 0", "ok")
        except (TransmissionError, ReceptionError):
            raise ConfigurationError("Configuration as receiver failed.")
        else:
            # obtain SNR value
            if self._debug:
                try:
                    self._ser_write("radio get snr")
                    snr = self._ser_read()
                    # range -128 to 127
                    print("LoRa:recv : SNR = %d".format(snr))
                except ReceptionError:
                    raise ConfigurationError("Could not obtain SNR value.")

            # wait for incoming data
            if self._debug:
                print("LoRa:recv : Receiving...")
            try:
                received_data = self._ser_read()
            except (ReceptionError, TimeoutError) as e:
                raise e
            else:
                # verify if no LoRa timeout occurred
                if received_data.strip() == 'radio_err':
                    raise TimeoutError("Timeout occurred during reception.")

                # strip data
                received_data = received_data[8:].strip()

                if self._debug:
                    print("LoRa:recv : {}".format(received_data))

                return received_data

    def send(self, tx_data):
        '''Set LoRa module in transmitter mode and send data'''
        if self._debug:
            print("LoRa:send : Prepare for transmission.")

        # try to configure device as transmitter
        try:
            self._ser_write_read_verify("mac pause")
        except (TransmissionError, ReceptionError):
            raise ConfigurationError("Configuration as transmitter failed.")

        if self._debug:
            print("LoRa:send : Sending...")
        try:
            # send data
            self._ser_write_read_verify("radio tx " + str(tx_data), "ok")

            # read out transmission verification
            ret = self._ser_read()
            if ret.strip() != "radio_tx_ok":
                raise TransmissionError("No transmission confirmation " +
                                        "received.")
        except (TransmissionError, ReceptionError):
            raise TransmissionError("Error while sending.")
        else:
            if self._debug:
                print("LoRa:send : " + str(tx_data))

    def recv_str(self):
        '''Set LoRa module in receiver mode and read received ASCI string'''
        # receive data
        rx_data = self.recv()

        # try to convert data to string
        try:
            text = binascii.unhexlify(rx_data)
        except TypeError:
            raise ReceptionError("Received data has odd length.")
        else:
            # TODO: comment
            if(len(rx_data) > 2):
                text = binascii.unhexlify(rx_data[:-1])

        return text

    def send_str(self, tx_str):
        '''Set LoRa module in transmitter mode and send ASCI string'''
        try:
            tx_data = binascii.hexlify(tx_str)
        except TypeError:
            raise TransmissionError("Transmitted string cannot be converted.")
        else:
            self.send(tx_data)

    # Constructor
    def __init__(self, port, timeout_serial=2000, timeout_lora=2000,
                 debug=False):
        '''Open serial connection to LoRa module and configure it'''

        # set internal parameters
        self._read_timeout_serial = timeout_serial  # in ms
        self._read_timeout_lora = timeout_lora      # in ms, 0 = endless
        self._debug = debug

        try:
            if self._debug:
                print("LoRa:__init__ : Start initialization.")

            # get parent device
            self._set_parent_dev()

            # open serial connection on parent device
            self._open(port)

            # configure LoRa module via parent device
            self._ser_write_read_verify("radio set mod lora", "ok")
            self._ser_write_read_verify("radio set freq 868000000", "ok")
            self._ser_write_read_verify("radio set pwr 14", "ok")
            self._ser_write_read_verify("radio set sf sf12", "ok")
            self._ser_write_read_verify("radio set afcbw 125", "ok")
            self._ser_write_read_verify("radio set rxbw 250", "ok")
            self._ser_write_read_verify("radio set fdev 5000", "ok")
            self._ser_write_read_verify("radio set prlen 8", "ok")
            self._ser_write_read_verify("radio set crc on", "ok")
            self._ser_write_read_verify("radio set cr 4/8", "ok")
            self._ser_write_read_verify("radio set wdt 0", "ok")
            self._ser_write_read_verify("radio set sync 12", "ok")
            self._ser_write_read_verify("radio set bw 250", "ok")

            # set reception timeout
            if self._read_timeout_lora > 0:
                self._ser_write_read_verify("radio set wdt {}".format(
                    self._read_timeout_lora), "ok")

        except (TransmissionError, ReceptionError):
            raise ConfigurationError("Initial configuration failed.")
        else:
            if self._debug:
                print("LoRa:__init__ : Connection established and " +
                      "configuration completed successfully.")

    # Destructor
    def __exit__(self, exc_type, exc_value, traceback):
        '''Close serial connection to LoRa module'''
        self._close()

    def __enter__(self):
        return self

    def __repr__(self):
        return "<LoRa module connection on port %d>".format(self.port)

# execute only if run as a script
if __name__ == "__main__":
    import argparse

    # command line arguments and parsing
    parser = argparse.ArgumentParser(description='LoRa Script Arguments')
    parser.add_argument('-d', '--debug',
                        action='store_true', help='Debug mode')
    parser.add_argument('-t', '--transmit',
                        default='',
                        help='Data to transmit')
    parser.add_argument('-r', '--receive',
                        default='2000',
                        help='Timeout upon reception (in ms)')
    parser.add_argument('-p', '--port',
                        default='ttyUSB0',
                        help='Serial Port ' +
                        'e.g. pyb, ttyUSB1, /dev/tty.usbserial-A5046HZ5')
    args = parser.parse_args()

    debug = args.debug
    serial_port = args.port
    tx_data = args.transmit
    timeout = int(args.receive)

    # create connection
    try:
        # set same timeout for serial connection and LoRa receiver
        lora = LoRa(serial_port, timeout, timeout, debug)
    except ConfigurationError as e:
        print("Error occurred: " + str(e))

    # start operation
    if tx_data != "":
        try:
            lora.send_str(tx_data)
        except (ConfigurationError, TransmissionError) as e:
            print("Error occurred: " + str(e))
    else:
        while True:
            try:
                lora.recv_str()
            except TimeoutError:
                print("Reception timeout occurred, continuing.")
            except (ConfigurationError, ReceptionError) as e:
                print("Error occurred: " + str(e))
                break
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2015 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# from twisted.internet.task import react
from twisted.internet import reactor
from twisted.web.client import Agent, readBody
from twisted.web.http_headers import Headers
from twisted.internet.ssl import ClientContextFactory
from twisted.web.client import FileBodyProducer

from StringIO import StringIO
from urllib import urlencode

# from calvin.utilities.calvinlogger import get_logger
from calvin.utilities.calvin_callback import CalvinCBClass

# _log = get_logger(__name__)

class HTTPRequest(object):
    def __init__(self):
        self._response = {}

    def parse_headers(self, response):
        self._response = {}
        self._response['version'] = "%s/%d.%d" % (response.version)
        self._response['status'] = response.code
        self._response['phrase'] = response.phrase
        self._response['headers'] = {}
        for hdr, val in response.headers.getAllRawHeaders():
            self._response['headers'][hdr.lower()] = val[0] if isinstance(val, list) and len(val) > 0 else val

    def parse_body(self, body):
        self._response['body'] = body

    def body(self):
        return self._response.get('body', None)

    def headers(self):
        return self._response.get('headers', None)

    def status(self):
        return self._response.get('status', None)

    def version(self):
        return self._response.get('version', None)

    def phrase(self):
        return self._response.get('phrase', None)

def encode_params(params):
    if params:
        return "?" + urlencode(params)
    return ""

def encode_headers(headers):
    twisted_headers = Headers()
    for k, v in headers.items():
        key = k.encode('ascii', 'ignore')
        val = v.encode('ascii', 'ignore')
        twisted_headers.addRawHeader(key, val)
    return twisted_headers

def encode_body(data):
    if not data:
        return None
    if not isinstance(data, str):
        return None
    return FileBodyProducer(StringIO(data))

class HTTPClient(CalvinCBClass):

    class WebClientContextFactory(ClientContextFactory):
        """TODO: enable certificate verification, hostname checking"""
        def getContext(self, hostname, port):
            return ClientContextFactory.getContext(self)

    def __init__(self, callbacks=None):
        super(HTTPClient, self).__init__(callbacks)
        self._agent = Agent(reactor, self.WebClientContextFactory())

    def _receive_headers(self, response, request):
        request.parse_headers(response)
        self._callback_execute('receive-headers', request)
        deferred = readBody(response)
        deferred.addCallback(self._receive_body, request)

    def _receive_body(self, response, request):
        request.parse_body(response)
        self._callback_execute('receive-body', request)

    def request(self, command, url, params, headers, data):
        url += encode_params(params)
        twisted_headers = encode_headers(headers)
        body = encode_body(data)
        deferred = self._agent.request(command, url, headers=twisted_headers, bodyProducer=body)
        request = HTTPRequest()
        deferred.addCallback(self._receive_headers, request)
        return request
<commit_msg>http_client: Use deliverBody to get response body<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2015 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# from twisted.internet.task import react
from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
from twisted.internet.ssl import ClientContextFactory
from twisted.web.client import FileBodyProducer
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol

from StringIO import StringIO
from urllib import urlencode

# from calvin.utilities.calvinlogger import get_logger
from calvin.utilities.calvin_callback import CalvinCBClass

# _log = get_logger(__name__)

class HTTPRequest(object):
    def __init__(self):
        self._response = {}

    def parse_headers(self, response):
        self._response = {}
        self._response['version'] = "%s/%d.%d" % (response.version)
        self._response['status'] = response.code
        self._response['phrase'] = response.phrase
        self._response['headers'] = {}
        for hdr, val in response.headers.getAllRawHeaders():
            self._response['headers'][hdr.lower()] = val[0] if isinstance(val, list) and len(val) > 0 else val

    def parse_body(self, body):
        self._response['body'] = body

    def body(self):
        return self._response.get('body', None)

    def headers(self):
        return self._response.get('headers', None)

    def status(self):
        return self._response.get('status', None)

    def version(self):
        return self._response.get('version', None)

    def phrase(self):
        return self._response.get('phrase', None)

def encode_params(params):
    if params:
        return "?" + urlencode(params)
    return ""

def encode_headers(headers):
    twisted_headers = Headers()
    for k, v in headers.items():
        key = k.encode('ascii', 'ignore')
        val = v.encode('ascii', 'ignore')
        twisted_headers.addRawHeader(key, val)
    return twisted_headers

def encode_body(data):
    if not data:
        return None
    if not isinstance(data, str):
        return None
    return FileBodyProducer(StringIO(data))

class BodyReader(Protocol):

    def __init__(self, deferred, cb, request):
        self.deferred = deferred
        self.data = ""
        self.cb = cb
        self.request = request

    def dataReceived(self, bytes):
        self.data += bytes

    def connectionLost(self, reason):
        self.deferred.callback(None)
        self.cb(self.data, self.request)

class HTTPClient(CalvinCBClass):

    class WebClientContextFactory(ClientContextFactory):
        """TODO: enable certificate verification, hostname checking"""
        def getContext(self, hostname, port):
            return ClientContextFactory.getContext(self)

    def __init__(self, callbacks=None):
        super(HTTPClient, self).__init__(callbacks)
        self._agent = Agent(reactor, self.WebClientContextFactory())

    def _receive_headers(self, response, request):
        request.parse_headers(response)
        self._callback_execute('receive-headers', request)
        finished = Deferred()
        response.deliverBody(BodyReader(finished, self._receive_body, request))
        return finished

    def _receive_body(self, response, request):
        request.parse_body(response)
        self._callback_execute('receive-body', request)

    def request(self, command, url, params, headers, data):
        url += encode_params(params)
        twisted_headers = encode_headers(headers)
        body = encode_body(data)
        deferred = self._agent.request(command, url, headers=twisted_headers, bodyProducer=body)
        request = HTTPRequest()
        deferred.addCallback(self._receive_headers, request)
        return request
<|endoftext|>


<commit_before>import uuid
import zstdandard as zstd
import ujson

from Typing import Dict
from Typing import object

__all__ = ['compiler']

class Job():
    def __init__(self):
        self.id = str(uuid.uuid4())
        self.expected_responses = []

    def _uncompress_payload(self, compressed: bytes) -> Dict[str, object]:
        dctx = zstd.ZstdDecompressor()
        decompressed = dctx.decompress(compressed)
        return ujson.reads(decompressed)
<commit_msg>Of course there is no ujson.reads<commit_after>import uuid
import zstdandard as zstd
import ujson

from Typing import Dict
from Typing import object

__all__ = ['jobmanager']

class Job():
    def __init__(self):
        self.id = str(uuid.uuid4())
        self.expected_responses = []

    def _uncompress_payload(self, compressed: bytes) -> Dict[str, object]:
        dctx = zstd.ZstdDecompressor()
        decompressed = dctx.decompress(compressed)
        return ujson.loads(decompressed)
<|endoftext|>