<commit_before>from __future__ import print_function
from array import array
import sys

import ops
import term
import vterm
import ops_decode
from zmath import to_signed_word
from debug import DBG, warn, err

def b16_setter(base):
    def setter(self, val):
        val &= 0xffff
        self.env.mem[base] = val >> 8
        self.env.mem[base+1] = val & 0xff
    return setter

def u16_prop(base):
    def getter(self): return self.env.u16(base)
    return property(fget=getter, fset=b16_setter(base))

def u8_prop(base):
    def getter(self): return self.env.mem[base]
    def setter(self, val): self.env.mem[base] = val & 0xff
    return property(fget=getter, fset=setter)

class Header(object):

    # use *_prop for dyn values
    # and just set others from mem
    # in __init__

    def __init__(self, env):
        self.env = env

        self.version = env.mem[0x0]

        self.release = env.u16(0x2)
        self.high_mem_base = env.u16(0x4)
        self.pc = env.u16(0x6)
        self.dict_base = env.u16(0x8)
        self.obj_tab_base = env.u16(0xA)
        self.global_var_base = env.u16(0xC)
        self.static_mem_base = env.u16(0xE)

        self.serial = list(env.mem[0x12:0x18])

        self.abbrev_base = env.u16(0x18)
        self.file_len = env.u16(0x1A)
        self.checksum = env.u16(0x1C)

        #everything after 1C is after v3

        self.routine_offset = env.u16(0x28) #div'd by 8
        self.string_offset = env.u16(0x2A) #div'd by 8

        self.term_chars_base = env.u16(0x2E)
        self.alpha_tab_base = env.u16(0x34)
        self.hdr_ext_tab_base = env.u16(0x36)

        self.unicode_tab_base = 0
        self.hdr_ext_tab_length = 0
        if self.hdr_ext_tab_base:
            self.hdr_ext_tab_length = env.u16(self.hdr_ext_tab_base)
            if self.hdr_ext_tab_length >= 3:
                self.unicode_tab_base = env.u16(self.hdr_ext_tab_base+3)

    flags1 = u8_prop(0x1)
    flags2 = u16_prop(0x10)

    #everything after 1C is after v3

    interp_number = u8_prop(0x1E)
    interp_version = u8_prop(0x1F)

    screen_height_lines = u8_prop(0x20)
    screen_width_chars = u8_prop(0x21)
    screen_width_units = u16_prop(0x22)
    screen_height_units = u16_prop(0x24)

    font_width_units = u8_prop(0x26)
    font_height_units = u8_prop(0x27)

    default_bg_color = u8_prop(0x2C)
    default_fg_color = u8_prop(0x2D)

    std_rev_number = u16_prop(0x32)

def set_standard_flags(hdr):
    if hdr.version < 4:
        # no variable-spaced font (bit 6 = 0)
        # no status line (bit 4 = 0)
        hdr.flags1 &= 0b10101111
        # screen splitting available (bit 5 = 1)
        hdr.flags1 |= 0b00100000
    else:
        # no timed keyboard events available (bit 7 = 0)
        # no sound effects available (bit 5 = 0)
        # no italic available (bit 3 = 0)
        # no boldface available (bit 2 = 0)
        # no picture display available (bit 1 = 0)
        hdr.flags1 &= 0b01010000
        # fixed-space font available (bit 4 = 1)
        # color available (bit 0 = 1)
        hdr.flags1 |= 0b00010001

    # uncheck stuff we don't support in flags 2
    # menus (bit 8)
    # sound effects (bit 7)
    # mouse (bit 5)
    # undo (bit 4)
    # and pictures (bit 3)
    hdr.flags2 &= 0b1111111001000111

    # use the apple 2e interp # to fix Beyond Zork compat
    hdr.interp_number = 2

    MAXIMUM_WIDTH = 80
    term_w, term_h = term.get_size()
    if term_w > 1:
        # FIXME: (?) inform games or bash or something (me?) seems to have
        # trouble pushing all the way to the edge of terminals. Saving a
        # column is also handy for having a spot for the scroll-pause symbol,
        # which, take NOTE, *doesn't* have trouble being drawn at the edge.
        term_w -= 1

    hdr.screen_width_units = min(MAXIMUM_WIDTH, term_w)
    hdr.screen_height_units = term_h

    hdr.screen_width_chars = hdr.screen_width_units
    hdr.screen_height_lines = hdr.screen_height_units

    hdr.font_width_units = 1
    hdr.font_height_units = 1

    hdr.default_fg_color = 9
    hdr.default_bg_color = 2

    # clear flags 3 (S 11.1.7.4.1)
    if hdr.hdr_ext_tab_length >= 4:
        env.mem[self.hdr_ext_tab_base+4] = 0

class Env:
    def __init__(self, mem):
        self.orig_mem = mem
        self.mem = array('B', map(ord, mem))

        self.hdr = Header(self)
        set_standard_flags(self.hdr)

        self.pc = self.hdr.pc
        self.callstack = [ops.Frame(0)]
        self.icache = {}
        self.fncache = {}

        self.fg_color = self.hdr.default_fg_color
        self.bg_color = self.hdr.default_bg_color

        # to make quetzal saves easier
        self.last_pc_branch_var = None
        self.last_pc_store_var = None

        self.output_buffer = {
            1: vterm.Screen(self),
            2: '', # transcript
            3: '', # mem
            4: ''  # player input (not impld atm)
        }
        self.screen = self.output_buffer[1]
        self.cursor = {
            0:(0,0),
            1:(0,0)
        }
        if self.hdr.version <= 4:
            self.cursor[0] = (self.hdr.screen_height_units-1, 0)

        self.text_style = 'normal'

        self.selected_ostreams = set([1])
        self.memory_ostream_stack = []
        self.use_buffered_output = True

        self.current_window = 0
        self.top_window_height = 0

    def fixup_after_restore(self):
        # make sure our standard flags are set after load
        set_standard_flags(self.hdr)

    def u16(self, i):
        return (self.mem[i] << 8) | self.mem[i+1]
    def check_dyn_mem(self, i):
        if i >= self.hdr.static_mem_base:
            err('game tried to write in static mem: '+str(i))
        elif i <= 0x36 and i != 0x10:
            err('game tried to write in non-dyn header bytes: '+str(i))
    def write16(self, i, val):
        self.check_dyn_mem(i)
        self.mem[i] = (val >> 8) & 0xff
        self.mem[i+1] = val & 0xff
    def write8(self, i, val):
        self.check_dyn_mem(i)
        self.mem[i] = val & 0xff
    def reset(self):
        # only the bottom two bits of flags2 survive reset
        # (transcribe to printer & fixed pitch font)
        bits_to_save = self.hdr.flags2 & 3
        self.__init__(self.orig_mem)
        self.hdr.flags2 &= ~3
        self.hdr.flags2 |= bits_to_save
    def quit(self):
        self.screen.flush()
        sys.exit()

def step(env):

    pc, icache = env.pc, env.icache
    if pc in icache:
        op, opinfo, env.pc = icache[pc]
    else:
        op, opinfo, env.pc = ops_decode.decode(env, pc)
        if pc >= env.hdr.static_mem_base:
            icache[pc] = op, opinfo, env.pc

    # fixup dynamic operands
    if opinfo.has_dynamic_operands:
        for i, var_num in opinfo.var_op_info:
            opinfo.operands[i] = ops.get_var(env, var_num)

    # for Quetzal
    if opinfo.last_pc_branch_var:
        env.last_pc_branch_var = opinfo.last_pc_branch_var
    if opinfo.last_pc_store_var:
        env.last_pc_store_var = opinfo.last_pc_store_var

    if DBG:
        warn(hex(pc))
        warn('op:', op.__name__)
        warn('    operands', dbg_decode_operands(env, op.__name__, opinfo.operands))
        if opinfo.branch_offset != None:
            warn('    branch_offset', opinfo.branch_offset)
            warn('    branch_to', dbg_decode_branch(env, opinfo.branch_offset))
            warn('    branch_on', opinfo.branch_on)

        op(env, opinfo)

        if opinfo.store_var:
            warn(    'store_var', ops.get_var_name(opinfo.store_var))
            warn(    'stored_result', dbg_decode_result(env, op.__name__, opinfo.store_var))
    else:
        op(env, opinfo)

def dbg_decode_branch(env, offset):
    if offset == 0 or offset == 1:
        return env.callstack[-1].return_addr
    return env.pc + offset - 2

def dbg_decode_operands(env, opname, operands):
    if opname in ['add', 'sub', 'mul', 'div', 'mod', 'jump', 'random_', 'print_num']:
        return map(to_signed_word, operands)
    elif opname in ['loadw', 'loadb', 'storew', 'storeb', 'inc_chk', 'dec_chk']:
        return [operands[0], to_signed_word(operands[1])] + operands[2:]
    elif opname in ['print_', 'print_ret']:
        result = ops.unpack_string(env, operands)
        if len(result) > 40:
            return repr(result[:40] + '...')
        return result
    return operands

def dbg_decode_result(env, opname, store_var):
    if 'call' in opname:
        return 'unknown (just called)'
    var = ops.get_var(env, store_var, pop_stack=False)
    if opname in ['add', 'sub', 'mul', 'div', 'mod']:
        return to_signed_word(var)
    return var

<commit_msg>Fix mis-refactored hdr ext table flags clear<commit_after>from __future__ import print_function
from array import array
import sys

import ops
import term
import vterm
import ops_decode
from zmath import to_signed_word
from debug import DBG, warn, err

def b16_setter(base):
    def setter(self, val):
        val &= 0xffff
        self.env.mem[base] = val >> 8
        self.env.mem[base+1] = val & 0xff
    return setter

def u16_prop(base):
    def getter(self): return self.env.u16(base)
    return property(fget=getter, fset=b16_setter(base))

def u8_prop(base):
    def getter(self): return self.env.mem[base]
    def setter(self, val): self.env.mem[base] = val & 0xff
    return property(fget=getter, fset=setter)

class Header(object):

    # use *_prop for dyn values
    # and just set others from mem
    # in __init__

    def __init__(self, env):
        self.env = env

        self.version = env.mem[0x0]

        self.release = env.u16(0x2)
        self.high_mem_base = env.u16(0x4)
        self.pc = env.u16(0x6)
        self.dict_base = env.u16(0x8)
        self.obj_tab_base = env.u16(0xA)
        self.global_var_base = env.u16(0xC)
        self.static_mem_base = env.u16(0xE)

        self.serial = list(env.mem[0x12:0x18])

        self.abbrev_base = env.u16(0x18)
        self.file_len = env.u16(0x1A)
        self.checksum = env.u16(0x1C)

        #everything after 1C is after v3

        self.routine_offset = env.u16(0x28) #div'd by 8
        self.string_offset = env.u16(0x2A) #div'd by 8

        self.term_chars_base = env.u16(0x2E)
        self.alpha_tab_base = env.u16(0x34)
        self.hdr_ext_tab_base = env.u16(0x36)

        self.unicode_tab_base = 0
        self.hdr_ext_tab_length = 0
        if self.hdr_ext_tab_base:
            self.hdr_ext_tab_length = env.u16(self.hdr_ext_tab_base)
            if self.hdr_ext_tab_length >= 3:
                self.unicode_tab_base = env.u16(self.hdr_ext_tab_base+3)

    flags1 = u8_prop(0x1)
    flags2 = u16_prop(0x10)

    #everything after 1C is after v3

    interp_number = u8_prop(0x1E)
    interp_version = u8_prop(0x1F)

    screen_height_lines = u8_prop(0x20)
    screen_width_chars = u8_prop(0x21)
    screen_width_units = u16_prop(0x22)
    screen_height_units = u16_prop(0x24)

    font_width_units = u8_prop(0x26)
    font_height_units = u8_prop(0x27)

    default_bg_color = u8_prop(0x2C)
    default_fg_color = u8_prop(0x2D)

    std_rev_number = u16_prop(0x32)

def set_standard_flags(env):
    hdr = env.hdr

    if hdr.version < 4:
        # no variable-spaced font (bit 6 = 0)
        # no status line (bit 4 = 0)
        hdr.flags1 &= 0b10101111
        # screen splitting available (bit 5 = 1)
        hdr.flags1 |= 0b00100000
    else:
        # no timed keyboard events available (bit 7 = 0)
        # no sound effects available (bit 5 = 0)
        # no italic available (bit 3 = 0)
        # no boldface available (bit 2 = 0)
        # no picture display available (bit 1 = 0)
        hdr.flags1 &= 0b01010000
        # fixed-space font available (bit 4 = 1)
        # color available (bit 0 = 1)
        hdr.flags1 |= 0b00010001

    # uncheck stuff we don't support in flags 2
    # menus (bit 8)
    # sound effects (bit 7)
    # mouse (bit 5)
    # undo (bit 4)
    # and pictures (bit 3)
    hdr.flags2 &= 0b1111111001000111

    # use the apple 2e interp # to fix Beyond Zork compat
    hdr.interp_number = 2

    MAXIMUM_WIDTH = 80
    term_w, term_h = term.get_size()
    if term_w > 1:
        # FIXME: (?) inform games or bash or something (me?) seems to have
        # trouble pushing all the way to the edge of terminals. Saving a
        # column is also handy for having a spot for the scroll-pause symbol,
        # which, take NOTE, *doesn't* have trouble being drawn at the edge.
        term_w -= 1

    hdr.screen_width_units = min(MAXIMUM_WIDTH, term_w)
    hdr.screen_height_units = term_h

    hdr.screen_width_chars = hdr.screen_width_units
    hdr.screen_height_lines = hdr.screen_height_units

    hdr.font_width_units = 1
    hdr.font_height_units = 1

    hdr.default_fg_color = 9
    hdr.default_bg_color = 2

    # clear flags 3 (S 11.1.7.4.1)
    if hdr.hdr_ext_tab_length >= 4:
        env.mem[hdr.hdr_ext_tab_base + 4] = 0

class Env:
    def __init__(self, mem):
        self.orig_mem = mem
        self.mem = array('B', map(ord, mem))

        self.hdr = Header(self)
        set_standard_flags(self)

        self.pc = self.hdr.pc
        self.callstack = [ops.Frame(0)]
        self.icache = {}
        self.fncache = {}

        self.fg_color = self.hdr.default_fg_color
        self.bg_color = self.hdr.default_bg_color

        # to make quetzal saves easier
        self.last_pc_branch_var = None
        self.last_pc_store_var = None

        self.output_buffer = {
            1: vterm.Screen(self),
            2: '', # transcript
            3: '', # mem
            4: ''  # player input (not impld atm)
        }
        self.screen = self.output_buffer[1]
        self.cursor = {
            0:(0,0),
            1:(0,0)
        }
        if self.hdr.version <= 4:
            self.cursor[0] = (self.hdr.screen_height_units-1, 0)

        self.text_style = 'normal'

        self.selected_ostreams = set([1])
        self.memory_ostream_stack = []
        self.use_buffered_output = True

        self.current_window = 0
        self.top_window_height = 0

    def fixup_after_restore(self):
        # make sure our standard flags are set after load
        set_standard_flags(self)

    def u16(self, i):
        return (self.mem[i] << 8) | self.mem[i+1]
    def check_dyn_mem(self, i):
        if i >= self.hdr.static_mem_base:
            err('game tried to write in static mem: '+str(i))
        elif i <= 0x36 and i != 0x10:
            err('game tried to write in non-dyn header bytes: '+str(i))
    def write16(self, i, val):
        self.check_dyn_mem(i)
        self.mem[i] = (val >> 8) & 0xff
        self.mem[i+1] = val & 0xff
    def write8(self, i, val):
        self.check_dyn_mem(i)
        self.mem[i] = val & 0xff
    def reset(self):
        # only the bottom two bits of flags2 survive reset
        # (transcribe to printer & fixed pitch font)
        bits_to_save = self.hdr.flags2 & 3
        self.__init__(self.orig_mem)
        self.hdr.flags2 &= ~3
        self.hdr.flags2 |= bits_to_save
    def quit(self):
        self.screen.flush()
        sys.exit()

def step(env):

    pc, icache = env.pc, env.icache
    if pc in icache:
        op, opinfo, env.pc = icache[pc]
    else:
        op, opinfo, env.pc = ops_decode.decode(env, pc)
        if pc >= env.hdr.static_mem_base:
            icache[pc] = op, opinfo, env.pc

    # fixup dynamic operands
    if opinfo.has_dynamic_operands:
        for i, var_num in opinfo.var_op_info:
            opinfo.operands[i] = ops.get_var(env, var_num)

    # for Quetzal
    if opinfo.last_pc_branch_var:
        env.last_pc_branch_var = opinfo.last_pc_branch_var
    if opinfo.last_pc_store_var:
        env.last_pc_store_var = opinfo.last_pc_store_var

    if DBG:
        warn(hex(pc))
        warn('op:', op.__name__)
        warn('    operands', dbg_decode_operands(env, op.__name__, opinfo.operands))
        if opinfo.branch_offset != None:
            warn('    branch_offset', opinfo.branch_offset)
            warn('    branch_to', dbg_decode_branch(env, opinfo.branch_offset))
            warn('    branch_on', opinfo.branch_on)

        op(env, opinfo)

        if opinfo.store_var:
            warn(    'store_var', ops.get_var_name(opinfo.store_var))
            warn(    'stored_result', dbg_decode_result(env, op.__name__, opinfo.store_var))
    else:
        op(env, opinfo)

def dbg_decode_branch(env, offset):
    if offset == 0 or offset == 1:
        return env.callstack[-1].return_addr
    return env.pc + offset - 2

def dbg_decode_operands(env, opname, operands):
    if opname in ['add', 'sub', 'mul', 'div', 'mod', 'jump', 'random_', 'print_num']:
        return map(to_signed_word, operands)
    elif opname in ['loadw', 'loadb', 'storew', 'storeb', 'inc_chk', 'dec_chk']:
        return [operands[0], to_signed_word(operands[1])] + operands[2:]
    elif opname in ['print_', 'print_ret']:
        result = ops.unpack_string(env, operands)
        if len(result) > 40:
            return repr(result[:40] + '...')
        return result
    return operands

def dbg_decode_result(env, opname, store_var):
    if 'call' in opname:
        return 'unknown (just called)'
    var = ops.get_var(env, store_var, pop_stack=False)
    if opname in ['add', 'sub', 'mul', 'div', 'mod']:
        return to_signed_word(var)
    return var

<|endoftext|>


<commit_before>import os
import subprocess
import types
import unittest
try:
    from unittest import mock
except ImportError:
    import mock

from django.core.urlresolvers import reverse
from django.http import QueryDict
from django.test import SimpleTestCase
import responses
from six.moves.urllib.parse import urljoin

DEFAULT_SETTINGS = dict(
    DEBUG=True,
    SECRET_KEY='a' * 24,
    ROOT_URLCONF='tests.urls',
    INSTALLED_APPS=[
        'django.contrib.sessions',
    ],
    MIDDLEWARE_CLASSES=[
        'django.contrib.sessions.middleware.SessionMiddleware'
    ],
    SESSION_ENGINE='django.contrib.sessions.backends.signed_cookies',
    TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': False,
        'OPTIONS': {
            'context_processors': [],
            'loaders': ['tests.utils.DummyTemplateLoader']
        },
    }],
    GOOGLE_ANALYTICS_ID=os.environ.get('GOOGLE_ANALYTICS_ID', 'UA-12345678-0'),
)

class FormErrorReportingTestCase(SimpleTestCase):
    @classmethod
    def setUpClass(cls):
        import django
        from django.conf import settings

        if not settings.configured:
            settings.configure(**DEFAULT_SETTINGS)
            django.setup()

        super(FormErrorReportingTestCase, cls).setUpClass()

    def submit_simple_form(self, data):
        from tests.forms import SimpleReportedForm

        form = SimpleReportedForm(data=data)
        form.ga_client_id = form.get_ga_client_id()  # freeze a generated client id
        return form

    def assertResponseErrorsReported(self, rsps, expected_error_dicts):  # noqa
        for expected_error_dict in expected_error_dicts:
            expected_error_dict.update({
                'v': '1',
                'tid': DEFAULT_SETTINGS['GOOGLE_ANALYTICS_ID'],
                't': 'event',
            })
        reported_error_dicts = []
        for error_line in rsps.calls[0].request.body.splitlines():
            if not error_line:
                continue
            error_params = QueryDict(error_line)
            error_params = {k: v for k, v in error_params.items()}
            reported_error_dicts.append(error_params)
        self.assertEqual(len(reported_error_dicts), len(expected_error_dicts))
        error_dict_pairs = zip(reported_error_dicts, expected_error_dicts)
        for reported_error_dict, expected_error_dict in error_dict_pairs:
            self.assertDictEqual(reported_error_dict, expected_error_dict)

    def assertFormErrorsReported(self, form, expected_error_dicts):  # noqa
        with responses.RequestsMock() as rsps:
            rsps.add(rsps.POST, form.get_ga_batch_endpoint())
            self.assertFalse(form.is_valid(), 'Form should be invalid')
            self.assertResponseErrorsReported(rsps, expected_error_dicts)

    def test_no_errors_send_no_reports(self):
        from tests.forms import SimpleReportedForm

        form = SimpleReportedForm(data={
            'required_number': 4,
            'required_text': 'abc',
        })
        with mock.patch.object(SimpleReportedForm, 'report_errors_to_ga', return_value=None) as method:
            self.assertTrue(form.is_valid())
            self.assertFalse(method.called)

    def test_single_field_error_reported(self):
        form = self.submit_simple_form({
            'required_number': 1,
            'required_text': 'abc',
        })
        self.assertFormErrorsReported(form, [
            {
                'cid': form.ga_client_id,
                'ec': 'tests.forms.SimpleReportedForm',
                'ea': 'required_number',
                'el': 'Ensure this value is greater than or equal to 3.',
            },
        ])

    def test_non_field_error_reported(self):
        form = self.submit_simple_form({
            'required_number': 4,
            'required_text': 'abc',
        })
        form.raise_non_field_error = True
        self.assertFormErrorsReported(form, [
            {
                'cid': form.ga_client_id,
                'ec': 'tests.forms.SimpleReportedForm',
                'ea': '__all__',
                'el': 'This form is invalid.',
            },
        ])

    def test_multiple_errors_reported(self):
        form = self.submit_simple_form({
            'required_number': 1,
            'required_text': '',
        })
        form.raise_non_field_error = True
        self.assertFormErrorsReported(form, [
            {
                'cid': form.ga_client_id,
                'ec': 'tests.forms.SimpleReportedForm',
                'ea': '__all__',
                'el': 'This form is invalid.',
            },
            {
                'cid': form.ga_client_id,
                'ec': 'tests.forms.SimpleReportedForm',
                'ea': 'required_number',
                'el': 'Ensure this value is greater than or equal to 3.',
            },
            {
                'cid': form.ga_client_id,
                'ec': 'tests.forms.SimpleReportedForm',
                'ea': 'required_text',
                'el': 'This field is required.',
            },
        ])

    @mock.patch('tests.utils.get_template_source')
    @mock.patch('tests.urls.get_context')
    def test_form_errors_with_session(self, mocked_context, mocked_template_source):
        from tests.forms import RequestReportedForm

        def get_context(request):
            form = RequestReportedForm(request, data=request.POST)
            return {
                'form': form,
            }

        mocked_context.side_effect = get_context
        mocked_template_source.return_value = '''
        "{{ form.is_valid }}"
        '''

        with responses.RequestsMock() as rsps:
            rsps.add(rsps.POST, urljoin(RequestReportedForm.ga_endpoint_base, '/batch'))
            response = self.client.post(reverse('dummy'), data={
                'required_number': 1,
                'required_text': '',
            }, HTTP_USER_AGENT='Mozilla/5.0')
            self.assertContains(response, '"False"')
            self.assertResponseErrorsReported(rsps, [
                {
                    'cid': response.client.session.get('ga_client_id'),
                    'ec': 'tests.forms.RequestReportedForm',
                    'ea': 'required_number',
                    'el': 'Ensure this value is greater than or equal to 3.',
                    'uip': '127.0.0.1',
                    'ua': 'Mozilla/5.0',
                },
                {
                    'cid': response.client.session.get('ga_client_id'),
                    'ec': 'tests.forms.RequestReportedForm',
                    'ea': 'required_text',
                    'el': 'This field is required.',
                    'uip': '127.0.0.1',
                    'ua': 'Mozilla/5.0',
                },
            ])

    @unittest.skipIf('GOOGLE_ANALYTICS_ID' not in os.environ,
                     'Provide a valid GOOGLE_ANALYTICS_ID environment variable')
    def test_validate_reporting_format(self):
        from tests.forms import SimpleReportedForm

        report_errors_to_ga = SimpleReportedForm.report_errors_to_ga

        def report_errors(self_, errors):
            super_responses = report_errors_to_ga(self_, errors)
            self.assertEqual(len(super_responses), 1)
            response = super_responses[0].json()['hitParsingResult']
            valid = all(result['valid'] for result in response)
            try:
                error_message = []
                for result in response:
                    result = map(lambda message: '%(messageType)s (%(parameter)s): %(description)s' % message,
                                 result['parserMessage'])
                    error_message.append('\n'.join(result))
                error_message = '----------\n'.join(error_message)
            except KeyError:
                error_message = 'Parser message not readable :('
            self.assertTrue(valid, error_message)
            return super_responses

        form = self.submit_simple_form({
            'required_number': 1,
            'required_text': 'abc',
        })
        form.ga_endpoint_base = 'https://ssl.google-analytics.com/debug/'
        form.ga_batch_hits = False
        form.report_errors_to_ga = types.MethodType(report_errors, form)
        form.is_valid()

class CodeStyleTestCase(unittest.TestCase):
    def test_code_style(self):
        try:
            subprocess.check_output(['flake8'])
        except subprocess.CalledProcessError as e:
            self.fail('Code style checks failed\n%s' % e.output.decode('utf-8'))
<commit_msg>Use longer user agent for test<commit_after>import os
import subprocess
import types
import unittest
try:
    from unittest import mock
except ImportError:
    import mock

from django.core.urlresolvers import reverse
from django.http import QueryDict
from django.test import SimpleTestCase
import responses
from six.moves.urllib.parse import urljoin

DEFAULT_SETTINGS = dict(
    DEBUG=True,
    SECRET_KEY='a' * 24,
    ROOT_URLCONF='tests.urls',
    INSTALLED_APPS=[
        'django.contrib.sessions',
    ],
    MIDDLEWARE_CLASSES=[
        'django.contrib.sessions.middleware.SessionMiddleware'
    ],
    SESSION_ENGINE='django.contrib.sessions.backends.signed_cookies',
    TEMPLATES=[{
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': False,
        'OPTIONS': {
            'context_processors': [],
            'loaders': ['tests.utils.DummyTemplateLoader']
        },
    }],
    GOOGLE_ANALYTICS_ID=os.environ.get('GOOGLE_ANALYTICS_ID', 'UA-12345678-0'),
)

class FormErrorReportingTestCase(SimpleTestCase):
    @classmethod
    def setUpClass(cls):
        import django
        from django.conf import settings

        if not settings.configured:
            settings.configure(**DEFAULT_SETTINGS)
            django.setup()

        super(FormErrorReportingTestCase, cls).setUpClass()

    def submit_simple_form(self, data):
        from tests.forms import SimpleReportedForm

        form = SimpleReportedForm(data=data)
        form.ga_client_id = form.get_ga_client_id()  # freeze a generated client id
        return form

    def assertResponseErrorsReported(self, rsps, expected_error_dicts):  # noqa
        for expected_error_dict in expected_error_dicts:
            expected_error_dict.update({
                'v': '1',
                'tid': DEFAULT_SETTINGS['GOOGLE_ANALYTICS_ID'],
                't': 'event',
            })
        reported_error_dicts = []
        for error_line in rsps.calls[0].request.body.splitlines():
            if not error_line:
                continue
            error_params = QueryDict(error_line)
            error_params = {k: v for k, v in error_params.items()}
            reported_error_dicts.append(error_params)
        self.assertEqual(len(reported_error_dicts), len(expected_error_dicts))
        error_dict_pairs = zip(reported_error_dicts, expected_error_dicts)
        for reported_error_dict, expected_error_dict in error_dict_pairs:
            self.assertDictEqual(reported_error_dict, expected_error_dict)

    def assertFormErrorsReported(self, form, expected_error_dicts):  # noqa
        with responses.RequestsMock() as rsps:
            rsps.add(rsps.POST, form.get_ga_batch_endpoint())
            self.assertFalse(form.is_valid(), 'Form should be invalid')
            self.assertResponseErrorsReported(rsps, expected_error_dicts)

    def test_no_errors_send_no_reports(self):
        from tests.forms import SimpleReportedForm

        form = SimpleReportedForm(data={
            'required_number': 4,
            'required_text': 'abc',
        })
        with mock.patch.object(SimpleReportedForm, 'report_errors_to_ga', return_value=None) as method:
            self.assertTrue(form.is_valid())
            self.assertFalse(method.called)

    def test_single_field_error_reported(self):
        form = self.submit_simple_form({
            'required_number': 1,
            'required_text': 'abc',
        })
        self.assertFormErrorsReported(form, [
            {
                'cid': form.ga_client_id,
                'ec': 'tests.forms.SimpleReportedForm',
                'ea': 'required_number',
                'el': 'Ensure this value is greater than or equal to 3.',
            },
        ])

    def test_non_field_error_reported(self):
        form = self.submit_simple_form({
            'required_number': 4,
            'required_text': 'abc',
        })
        form.raise_non_field_error = True
        self.assertFormErrorsReported(form, [
            {
                'cid': form.ga_client_id,
                'ec': 'tests.forms.SimpleReportedForm',
                'ea': '__all__',
                'el': 'This form is invalid.',
            },
        ])

    def test_multiple_errors_reported(self):
        form = self.submit_simple_form({
            'required_number': 1,
            'required_text': '',
        })
        form.raise_non_field_error = True
        self.assertFormErrorsReported(form, [
            {
                'cid': form.ga_client_id,
                'ec': 'tests.forms.SimpleReportedForm',
                'ea': '__all__',
                'el': 'This form is invalid.',
            },
            {
                'cid': form.ga_client_id,
                'ec': 'tests.forms.SimpleReportedForm',
                'ea': 'required_number',
                'el': 'Ensure this value is greater than or equal to 3.',
            },
            {
                'cid': form.ga_client_id,
                'ec': 'tests.forms.SimpleReportedForm',
                'ea': 'required_text',
                'el': 'This field is required.',
            },
        ])

    @mock.patch('tests.utils.get_template_source')
    @mock.patch('tests.urls.get_context')
    def test_form_errors_with_session(self, mocked_context, mocked_template_source):
        from tests.forms import RequestReportedForm

        def get_context(request):
            form = RequestReportedForm(request, data=request.POST)
            return {
                'form': form,
            }

        mocked_context.side_effect = get_context
        mocked_template_source.return_value = '''
        "{{ form.is_valid }}"
        '''

        user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) ' \
                     'Chrome/55.0.2883.95 Safari/537.36'
        with responses.RequestsMock() as rsps:
            rsps.add(rsps.POST, urljoin(RequestReportedForm.ga_endpoint_base, '/batch'))
            response = self.client.post(reverse('dummy'), data={
                'required_number': 1,
                'required_text': '',
            }, HTTP_USER_AGENT=user_agent)
            client_id = response.client.session.get('ga_client_id')
            self.assertContains(response, '"False"')
            self.assertResponseErrorsReported(rsps, [
                {
                    'cid': client_id,
                    'ec': 'tests.forms.RequestReportedForm',
                    'ea': 'required_number',
                    'el': 'Ensure this value is greater than or equal to 3.',
                    'uip': '127.0.0.1',
                    'ua': user_agent,
                },
                {
                    'cid': client_id,
                    'ec': 'tests.forms.RequestReportedForm',
                    'ea': 'required_text',
                    'el': 'This field is required.',
                    'uip': '127.0.0.1',
                    'ua': user_agent,
                },
            ])

    @unittest.skipIf('GOOGLE_ANALYTICS_ID' not in os.environ,
                     'Provide a valid GOOGLE_ANALYTICS_ID environment variable')
    def test_validate_reporting_format(self):
        from tests.forms import SimpleReportedForm

        report_errors_to_ga = SimpleReportedForm.report_errors_to_ga

        def report_errors(self_, errors):
            super_responses = report_errors_to_ga(self_, errors)
            self.assertEqual(len(super_responses), 1)
            response = super_responses[0].json()['hitParsingResult']
            valid = all(result['valid'] for result in response)
            try:
                error_message = []
                for result in response:
                    result = map(lambda message: '%(messageType)s (%(parameter)s): %(description)s' % message,
                                 result['parserMessage'])
                    error_message.append('\n'.join(result))
                error_message = '----------\n'.join(error_message)
            except KeyError:
                error_message = 'Parser message not readable :('
            self.assertTrue(valid, error_message)
            return super_responses

        form = self.submit_simple_form({
            'required_number': 1,
            'required_text': 'abc',
        })
        form.ga_endpoint_base = 'https://ssl.google-analytics.com/debug/'
        form.ga_batch_hits = False
        form.report_errors_to_ga = types.MethodType(report_errors, form)
        form.is_valid()

class CodeStyleTestCase(unittest.TestCase):
    def test_code_style(self):
        try:
            subprocess.check_output(['flake8'])
        except subprocess.CalledProcessError as e:
            self.fail('Code style checks failed\n%s' % e.output.decode('utf-8'))
<|endoftext|>


<commit_before>""" Tests karr_lab_build_utils.

:Author: Jonathan Karr <karr@mssm.edu>
:Date: 2016-08-03
:Copyright: 2016, Karr Lab
:License: MIT
"""

from glob import glob
from karr_lab_build_utils.__main__ import App as KarrLabBuildUtilsCli
from karr_lab_build_utils.core import BuildHelper
import shutil
import os
import sys
import tempfile
import unittest

if sys.version_info >= (3, 0, 0):
    from test.support import EnvironmentVarGuard
else:
    from test.test_support import EnvironmentVarGuard

class TestKarrLabBuildUtils(unittest.TestCase):
    COVERALLS_REPO_TOKEN = ''
    CODECLIMATE_REPO_TOKEN = ''
    DUMMY_TEST = 'tests/test_karr_lab_build_utils.py:TestKarrLabBuildUtils.test_dummy_test'

    @staticmethod
    def construct_environment():
        env = EnvironmentVarGuard()
        env.set('CIRCLE_ARTIFACTS', tempfile.mkdtemp())
        env.set('CIRCLE_TEST_REPORTS', tempfile.mkdtemp())
        env.set('COVERALLS_REPO_TOKEN', TestKarrLabBuildUtils.COVERALLS_REPO_TOKEN)
        env.set('CODECLIMATE_REPO_TOKEN', TestKarrLabBuildUtils.CODECLIMATE_REPO_TOKEN)
        env.set('CIRCLE_BUILD_NUM', '0')
        if not os.getenv('CIRCLECI'):
            with open('tests/fixtures/TEST_SERVER_TOKEN', 'r') as file:
                env.set('TEST_SERVER_TOKEN', file.read().rstrip())
            with open('tests/fixtures/CIRCLE_PROJECT_REPONAME', 'r') as file:
                env.set('CIRCLE_PROJECT_REPONAME', file.read().rstrip())
            with open('tests/fixtures/CIRCLE_PROJECT_USERNAME', 'r') as file:
                env.set('CIRCLE_PROJECT_USERNAME', file.read().rstrip())
            with open('tests/fixtures/CIRCLE_BRANCH', 'r') as file:
                env.set('CIRCLE_BRANCH', file.read().rstrip())
            with open('tests/fixtures/CIRCLE_SHA1', 'r') as file:
                env.set('CIRCLE_SHA1', file.read().rstrip())
            with open('tests/fixtures/GITHUB_USERNAME', 'r') as file:
                env.set('GITHUB_USERNAME', file.read().rstrip())
            with open('tests/fixtures/GITHUB_PASSWORD', 'r') as file:
                env.set('GITHUB_PASSWORD', file.read().rstrip())
            with open('tests/fixtures/CIRCLE_API_TOKEN', 'r') as file:
                env.set('CIRCLE_API_TOKEN', file.read().rstrip())

        return env

    @staticmethod
    def construct_build_helper():
        with TestKarrLabBuildUtils.construct_environment():
            buildHelper = BuildHelper()

        return buildHelper

    def test_create_circleci_build(self):
        buildHelper = self.construct_build_helper()

        """ test API """
        buildHelper.create_circleci_build()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['create-circleci-build']) as app:
                app.run()

    def test_create_codeclimate_github_webhook(self):
        buildHelper = self.construct_build_helper()

        """ test API """
        try:
            buildHelper.create_codeclimate_github_webhook()
        except ValueError as err:
            pass

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['create-codeclimate-github-webhook']) as app:
                try:
                    app.run()
                except ValueError as err:
                    pass

    def test_install_requirements(self):
        buildHelper = self.construct_build_helper()

        """ test API """
        buildHelper.install_requirements()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['install-requirements']) as app:
                app.run()

    def test_run_tests(self):
        self.help_run('pytest')
        self.help_run('nose')

    def help_run(self, test_runner):
        buildHelper = self.construct_build_helper()
        buildHelper.test_runner = test_runner
        py_v = buildHelper.get_python_version()

        """ test API """
        latest_results_filename = os.path.join(buildHelper.proj_tests_xml_dir, '{0:s}.{1:s}.xml'.format(
            buildHelper.proj_tests_xml_latest_filename, py_v))
        lastest_cov_filename = '.coverage.{}'.format(py_v)
        if os.path.isfile(latest_results_filename):
            os.remove(latest_results_filename)
        if os.path.isfile(lastest_cov_filename):
            os.remove(lastest_cov_filename)

        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)

        self.assertTrue(os.path.isfile(latest_results_filename))
        self.assertTrue(os.path.isfile(lastest_cov_filename))

        """ test CLI """
        argv = ['run-tests', TestKarrLabBuildUtils.DUMMY_TEST, '--with-xunit', '--with-coverage']
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=argv) as app:
                app.run()
                self.assertEqual(TestKarrLabBuildUtils.DUMMY_TEST, app.pargs.test_path)
                self.assertTrue(app.pargs.with_xunit)
                self.assertTrue(app.pargs.with_coverage)

    def test_make_and_archive_reports(self):
        buildHelper = self.construct_build_helper()
        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)

        py_v = buildHelper.get_python_version()
        shutil.copyfile(
            os.path.join(buildHelper.proj_tests_xml_dir, '{0:s}.{1:s}.xml'.format(
                buildHelper.proj_tests_xml_latest_filename, py_v)),
            os.path.join(buildHelper.proj_tests_xml_dir, '{0:d}.{1:s}.xml'.format(10000000000000001, py_v))
        )
        shutil.copyfile(
            os.path.join(buildHelper.proj_tests_xml_dir, '{0:s}.{1:s}.xml'.format(
                buildHelper.proj_tests_xml_latest_filename, py_v)),
            os.path.join(buildHelper.proj_tests_xml_dir, '{0:d}.{1:s}.xml'.format(10000000000000002, py_v))
        )

        """ test API """
        buildHelper.make_and_archive_reports()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['make-and-archive-reports']) as app:
                app.run()

    def test_archive_test_report(self):
        buildHelper = self.construct_build_helper()
        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)

        """ test API """
        buildHelper.archive_test_report()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['archive-test-report']) as app:
                app.run()

    def test_combine_coverage_reports(self):
        buildHelper = self.construct_build_helper()
        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)
        shutil.move('.coverage.{}'.format(buildHelper.get_python_version()), '.coverage.1')
        shutil.copyfile('.coverage.1', '.coverage.2')

        """ test API """
        if os.path.isfile('.coverage'):
            os.remove('.coverage')
        self.assertTrue(os.path.isfile('.coverage.1'))
        self.assertTrue(os.path.isfile('.coverage.2'))

        buildHelper.combine_coverage_reports()

        self.assertTrue(os.path.isfile('.coverage'))
        self.assertTrue(os.path.isfile('.coverage.1'))
        self.assertTrue(os.path.isfile('.coverage.2'))

        """ test CLI """
        if os.path.isfile('.coverage'):
            os.remove('.coverage')
        self.assertTrue(os.path.isfile('.coverage.1'))
        self.assertTrue(os.path.isfile('.coverage.2'))

        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['combine-coverage-reports']) as app:
                app.run()

        self.assertTrue(os.path.isfile('.coverage'))
        self.assertTrue(os.path.isfile('.coverage.1'))
        self.assertTrue(os.path.isfile('.coverage.2'))

        # cleanup
        os.remove('.coverage.1')
        os.remove('.coverage.2')

    def test_archive_coverage_report(self):
        buildHelper = self.construct_build_helper()
        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)

        buildHelper.combine_coverage_reports()

        """ test API """
        buildHelper.archive_coverage_report()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['archive-coverage-report']) as app:
                app.run()

    def test_copy_coverage_report_to_artifacts_directory(self):
        buildHelper = self.construct_build_helper()
        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)

        abs_cov_filename = os.path.join(buildHelper.build_artifacts_dir, '.coverage')
        abs_cov_filename_v = os.path.join(buildHelper.build_artifacts_dir,
                                          '.coverage.{}'.format(buildHelper.get_python_version()))

        """ test API """
        if os.path.isfile(abs_cov_filename):
            os.remove(abs_cov_filename)
        if os.path.isfile(abs_cov_filename_v):
            os.remove(abs_cov_filename_v)

        buildHelper.copy_coverage_report_to_artifacts_directory()

        self.assertFalse(os.path.isfile(abs_cov_filename))
        self.assertTrue(os.path.isfile(abs_cov_filename_v))

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['copy-coverage-report-to-artifacts-directory']) as app:
                app.run()

    def test_upload_coverage_report_to_coveralls(self):
        buildHelper = self.construct_build_helper()
        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)

        shutil.move('.coverage.{}'.format(buildHelper.get_python_version()), '.coverage')

        """ test API """
        buildHelper.upload_coverage_report_to_coveralls()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['upload-coverage-report-to-coveralls']) as app:
                app.run()

    def test_upload_coverage_report_to_code_climate(self):
        buildHelper = self.construct_build_helper()
        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)

        shutil.move('.coverage.{}'.format(buildHelper.get_python_version()), '.coverage')

        """ test API """
        buildHelper.upload_coverage_report_to_code_climate()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['upload-coverage-report-to-code-climate']) as app:
                app.run()

    def test_make_documentation(self):
        buildHelper = self.construct_build_helper()

        """ test API """
        if os.path.isdir(buildHelper.proj_docs_build_html_dir):
            shutil.rmtree(buildHelper.proj_docs_build_html_dir)

        buildHelper.make_documentation()

        self.assertTrue(os.path.isfile(os.path.join(buildHelper.proj_docs_build_html_dir, 'index.html')))

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['make-documentation']) as app:
                app.run()

    def test_archive_documentation(self):
        """ setup """
        buildHelper = self.construct_build_helper()
        buildHelper.make_documentation()

        """ test API """
        artifacts_docs_dir = os.path.join(buildHelper.build_artifacts_dir, buildHelper.artifacts_docs_build_html_dir)
        self.assertFalse(os.path.isfile(os.path.join(artifacts_docs_dir, 'index.html')))
        buildHelper.archive_documentation()
        self.assertTrue(os.path.isfile(os.path.join(artifacts_docs_dir, 'index.html')))

        """ test CLI """
        with self.construct_environment():
            artifacts_docs_dir = os.path.join(os.getenv('CIRCLE_ARTIFACTS'), buildHelper.artifacts_docs_build_html_dir)
            self.assertFalse(os.path.isfile(os.path.join(artifacts_docs_dir, 'index.html')))
            with KarrLabBuildUtilsCli(argv=['archive-documentation']) as app:
                app.run()
            self.assertTrue(os.path.isfile(os.path.join(artifacts_docs_dir, 'index.html')))

    def test_get_version(self):
        """ setup """
        buildHelper = self.construct_build_helper()

        """ test API """
        buildHelper.get_version()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['get-version']) as app:
                app.run()

    def test_dummy_test(self):
        buildHelper = self.construct_build_helper()
<commit_msg>making tests more flexible<commit_after>""" Tests karr_lab_build_utils.

:Author: Jonathan Karr <karr@mssm.edu>
:Date: 2016-08-03
:Copyright: 2016, Karr Lab
:License: MIT
"""

from glob import glob
from karr_lab_build_utils.__main__ import App as KarrLabBuildUtilsCli
from karr_lab_build_utils.core import BuildHelper
import shutil
import os
import sys
import tempfile
import unittest

if sys.version_info >= (3, 0, 0):
    from test.support import EnvironmentVarGuard
else:
    from test.test_support import EnvironmentVarGuard

class TestKarrLabBuildUtils(unittest.TestCase):
    COVERALLS_REPO_TOKEN = ''
    CODECLIMATE_REPO_TOKEN = ''
    DUMMY_TEST = 'tests/test_karr_lab_build_utils.py:TestKarrLabBuildUtils.test_dummy_test'

    @staticmethod
    def construct_environment():
        env = EnvironmentVarGuard()
        env.set('CIRCLE_ARTIFACTS', tempfile.mkdtemp())
        env.set('CIRCLE_TEST_REPORTS', tempfile.mkdtemp())
        env.set('COVERALLS_REPO_TOKEN', TestKarrLabBuildUtils.COVERALLS_REPO_TOKEN)
        env.set('CODECLIMATE_REPO_TOKEN', TestKarrLabBuildUtils.CODECLIMATE_REPO_TOKEN)
        env.set('CIRCLE_BUILD_NUM', '0')
        env.set('CIRCLE_BRANCH', 'master')
        env.set('CIRCLE_SHA1', '--test--')

        if not os.getenv('CIRCLE_PROJECT_REPONAME'):
            with open('tests/fixtures/CIRCLE_PROJECT_REPONAME', 'r') as file:
                env.set('CIRCLE_PROJECT_REPONAME', file.read().rstrip())

        if not os.getenv('CIRCLE_PROJECT_USERNAME'):
            with open('tests/fixtures/CIRCLE_PROJECT_USERNAME', 'r') as file:
                env.set('CIRCLE_PROJECT_USERNAME', file.read().rstrip())

        if not os.getenv('GITHUB_USERNAME'):
            with open('tests/fixtures/GITHUB_USERNAME', 'r') as file:
                env.set('GITHUB_USERNAME', file.read().rstrip())

        if not os.getenv('GITHUB_PASSWORD'):
            with open('tests/fixtures/GITHUB_PASSWORD', 'r') as file:
                env.set('GITHUB_PASSWORD', file.read().rstrip())

        if not os.getenv('CIRCLE_API_TOKEN'):
            with open('tests/fixtures/CIRCLE_API_TOKEN', 'r') as file:
                env.set('CIRCLE_API_TOKEN', file.read().rstrip())

        if not os.getenv('TEST_SERVER_TOKEN'):
            with open('tests/fixtures/TEST_SERVER_TOKEN', 'r') as file:
                env.set('TEST_SERVER_TOKEN', file.read().rstrip())

        return env

    @staticmethod
    def construct_build_helper():
        with TestKarrLabBuildUtils.construct_environment():
            buildHelper = BuildHelper()

        return buildHelper

    def test_create_circleci_build(self):
        buildHelper = self.construct_build_helper()

        """ test API """
        buildHelper.create_circleci_build()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['create-circleci-build']) as app:
                app.run()

    def test_create_codeclimate_github_webhook(self):
        buildHelper = self.construct_build_helper()

        """ test API """
        try:
            buildHelper.create_codeclimate_github_webhook()
        except ValueError as err:
            pass

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['create-codeclimate-github-webhook']) as app:
                try:
                    app.run()
                except ValueError as err:
                    pass

    def test_install_requirements(self):
        buildHelper = self.construct_build_helper()

        """ test API """
        buildHelper.install_requirements()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['install-requirements']) as app:
                app.run()

    def test_run_tests(self):
        self.help_run('pytest')
        self.help_run('nose')

    def help_run(self, test_runner):
        buildHelper = self.construct_build_helper()
        buildHelper.test_runner = test_runner
        py_v = buildHelper.get_python_version()

        """ test API """
        latest_results_filename = os.path.join(buildHelper.proj_tests_xml_dir, '{0:s}.{1:s}.xml'.format(
            buildHelper.proj_tests_xml_latest_filename, py_v))
        lastest_cov_filename = '.coverage.{}'.format(py_v)
        if os.path.isfile(latest_results_filename):
            os.remove(latest_results_filename)
        if os.path.isfile(lastest_cov_filename):
            os.remove(lastest_cov_filename)

        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)

        self.assertTrue(os.path.isfile(latest_results_filename))
        self.assertTrue(os.path.isfile(lastest_cov_filename))

        """ test CLI """
        argv = ['run-tests', TestKarrLabBuildUtils.DUMMY_TEST, '--with-xunit', '--with-coverage']
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=argv) as app:
                app.run()
                self.assertEqual(TestKarrLabBuildUtils.DUMMY_TEST, app.pargs.test_path)
                self.assertTrue(app.pargs.with_xunit)
                self.assertTrue(app.pargs.with_coverage)

    def test_make_and_archive_reports(self):
        buildHelper = self.construct_build_helper()
        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)

        py_v = buildHelper.get_python_version()
        shutil.copyfile(
            os.path.join(buildHelper.proj_tests_xml_dir, '{0:s}.{1:s}.xml'.format(
                buildHelper.proj_tests_xml_latest_filename, py_v)),
            os.path.join(buildHelper.proj_tests_xml_dir, '{0:d}.{1:s}.xml'.format(10000000000000001, py_v))
        )
        shutil.copyfile(
            os.path.join(buildHelper.proj_tests_xml_dir, '{0:s}.{1:s}.xml'.format(
                buildHelper.proj_tests_xml_latest_filename, py_v)),
            os.path.join(buildHelper.proj_tests_xml_dir, '{0:d}.{1:s}.xml'.format(10000000000000002, py_v))
        )

        """ test API """
        buildHelper.make_and_archive_reports()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['make-and-archive-reports']) as app:
                app.run()

    def test_archive_test_report(self):
        buildHelper = self.construct_build_helper()
        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)

        """ test API """
        buildHelper.archive_test_report()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['archive-test-report']) as app:
                app.run()

    def test_combine_coverage_reports(self):
        buildHelper = self.construct_build_helper()
        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)
        shutil.move('.coverage.{}'.format(buildHelper.get_python_version()), '.coverage.1')
        shutil.copyfile('.coverage.1', '.coverage.2')

        """ test API """
        if os.path.isfile('.coverage'):
            os.remove('.coverage')
        self.assertTrue(os.path.isfile('.coverage.1'))
        self.assertTrue(os.path.isfile('.coverage.2'))

        buildHelper.combine_coverage_reports()

        self.assertTrue(os.path.isfile('.coverage'))
        self.assertTrue(os.path.isfile('.coverage.1'))
        self.assertTrue(os.path.isfile('.coverage.2'))

        """ test CLI """
        if os.path.isfile('.coverage'):
            os.remove('.coverage')
        self.assertTrue(os.path.isfile('.coverage.1'))
        self.assertTrue(os.path.isfile('.coverage.2'))

        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['combine-coverage-reports']) as app:
                app.run()

        self.assertTrue(os.path.isfile('.coverage'))
        self.assertTrue(os.path.isfile('.coverage.1'))
        self.assertTrue(os.path.isfile('.coverage.2'))

        # cleanup
        os.remove('.coverage.1')
        os.remove('.coverage.2')

    def test_archive_coverage_report(self):
        buildHelper = self.construct_build_helper()
        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)

        buildHelper.combine_coverage_reports()

        """ test API """
        buildHelper.archive_coverage_report()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['archive-coverage-report']) as app:
                app.run()

    def test_copy_coverage_report_to_artifacts_directory(self):
        buildHelper = self.construct_build_helper()
        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)

        abs_cov_filename = os.path.join(buildHelper.build_artifacts_dir, '.coverage')
        abs_cov_filename_v = os.path.join(buildHelper.build_artifacts_dir,
                                          '.coverage.{}'.format(buildHelper.get_python_version()))

        """ test API """
        if os.path.isfile(abs_cov_filename):
            os.remove(abs_cov_filename)
        if os.path.isfile(abs_cov_filename_v):
            os.remove(abs_cov_filename_v)

        buildHelper.copy_coverage_report_to_artifacts_directory()

        self.assertFalse(os.path.isfile(abs_cov_filename))
        self.assertTrue(os.path.isfile(abs_cov_filename_v))

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['copy-coverage-report-to-artifacts-directory']) as app:
                app.run()

    def test_upload_coverage_report_to_coveralls(self):
        buildHelper = self.construct_build_helper()
        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)

        shutil.move('.coverage.{}'.format(buildHelper.get_python_version()), '.coverage')

        """ test API """
        buildHelper.upload_coverage_report_to_coveralls()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['upload-coverage-report-to-coveralls']) as app:
                app.run()

    def test_upload_coverage_report_to_code_climate(self):
        buildHelper = self.construct_build_helper()
        buildHelper.run_tests(test_path=self.DUMMY_TEST,
                              with_xunit=True, with_coverage=True, exit_on_failure=False)

        shutil.move('.coverage.{}'.format(buildHelper.get_python_version()), '.coverage')

        """ test API """
        buildHelper.upload_coverage_report_to_code_climate()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['upload-coverage-report-to-code-climate']) as app:
                app.run()

    def test_make_documentation(self):
        buildHelper = self.construct_build_helper()

        """ test API """
        if os.path.isdir(buildHelper.proj_docs_build_html_dir):
            shutil.rmtree(buildHelper.proj_docs_build_html_dir)

        buildHelper.make_documentation()

        self.assertTrue(os.path.isfile(os.path.join(buildHelper.proj_docs_build_html_dir, 'index.html')))

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['make-documentation']) as app:
                app.run()

    def test_archive_documentation(self):
        """ setup """
        buildHelper = self.construct_build_helper()
        buildHelper.make_documentation()

        """ test API """
        artifacts_docs_dir = os.path.join(buildHelper.build_artifacts_dir, buildHelper.artifacts_docs_build_html_dir)
        self.assertFalse(os.path.isfile(os.path.join(artifacts_docs_dir, 'index.html')))
        buildHelper.archive_documentation()
        self.assertTrue(os.path.isfile(os.path.join(artifacts_docs_dir, 'index.html')))

        """ test CLI """
        with self.construct_environment():
            artifacts_docs_dir = os.path.join(os.getenv('CIRCLE_ARTIFACTS'), buildHelper.artifacts_docs_build_html_dir)
            self.assertFalse(os.path.isfile(os.path.join(artifacts_docs_dir, 'index.html')))
            with KarrLabBuildUtilsCli(argv=['archive-documentation']) as app:
                app.run()
            self.assertTrue(os.path.isfile(os.path.join(artifacts_docs_dir, 'index.html')))

    def test_get_version(self):
        """ setup """
        buildHelper = self.construct_build_helper()

        """ test API """
        buildHelper.get_version()

        """ test CLI """
        with self.construct_environment():
            with KarrLabBuildUtilsCli(argv=['get-version']) as app:
                app.run()

    def test_dummy_test(self):
        buildHelper = self.construct_build_helper()
<|endoftext|>


<commit_before># Copyright (c) 2015 Yubico AB
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#    1. Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    2. Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import os
import re
import struct
import six
import time
from base64 import b64encode
from functools import total_ordering
from enum import IntEnum, unique
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
from cryptography.hazmat.primitives import hmac, hashes
from cryptography.hazmat.backends import default_backend
from six.moves.urllib.parse import unquote, urlparse, parse_qs
from .driver_ccid import APDUError, SW_OK
from .util import (
    AID, Tlv, parse_tlvs, time_challenge, parse_b32_key,
    format_code, parse_truncated, hmac_shorten_key)

@unique
class TAG(IntEnum):
    NAME = 0x71
    NAME_LIST = 0x72
    KEY = 0x73
    CHALLENGE = 0x74
    RESPONSE = 0x75
    TRUNCATED_RESPONSE = 0x76
    NO_RESPONSE = 0x77
    PROPERTY = 0x78
    VERSION = 0x79
    IMF = 0x7a
    ALGORITHM = 0x7b
    TOUCH = 0x7c

@unique
class ALGO(IntEnum):
    SHA1 = 0x01
    SHA256 = 0x02
    SHA512 = 0x03

@unique
class OATH_TYPE(IntEnum):
    HOTP = 0x10
    TOTP = 0x20

@unique
class PROPERTIES(IntEnum):
    REQUIRE_TOUCH = 0x02

@unique
class INS(IntEnum):
    PUT = 0x01
    DELETE = 0x02
    SET_CODE = 0x03
    RESET = 0x04
    LIST = 0xa1
    CALCULATE = 0xa2
    VALIDATE = 0xa3
    CALCULATE_ALL = 0xa4
    SEND_REMAINING = 0xa5

@unique
class MASK(IntEnum):
    ALGO = 0x0f
    TYPE = 0xf0

@unique
class SW(IntEnum):
    NO_SPACE = 0x6a84
    COMMAND_ABORTED = 0x6f00
    MORE_DATA = 0x61
    INVALID_INSTRUCTION = 0x6d00

class CredentialData(object):

    def __init__(self, secret, issuer, name, oath_type=OATH_TYPE.TOTP,
                 algorithm=ALGO.SHA1, digits=6, period=30, counter=0,
                 touch=False):
        self.secret = secret
        self.issuer = issuer
        self.name = name
        self.oath_type = oath_type
        self.algorithm = algorithm
        self.digits = digits
        self.period = period
        self.counter = counter
        self.touch = touch

    @classmethod
    def from_uri(cls, uri):
        parsed = urlparse(uri.strip())
        if parsed.scheme != 'otpauth':
            raise ValueError('Invalid URI scheme')

        params = dict((k, v[0]) for k, v in parse_qs(parsed.query).items())
        params['secret'] = parse_b32_key(params['secret'])
        params['algorithm'] = ALGO[params.get('algorithm', 'SHA1').upper()]
        issuer = None
        name = unquote(parsed.path)[1:]  # Unquote and strip leading /
        if ':' in name:
            issuer, name = name.split(':', 1)
        params['issuer'] = params.get('issuer', issuer)
        params['name'] = name
        params['oath_type'] = OATH_TYPE[parsed.hostname.upper()]
        params['digits'] = int(params.get('digits', 6))
        params['period'] = int(params.get('period', 30))
        params['counter'] = int(params.get('counter', 0))
        return cls(**params)

    def make_key(self):
        key = self.name
        if self.issuer:
            key = '%s:%s' % (self.issuer, key)
        if self.oath_type == OATH_TYPE.TOTP and self.period != 30:
            key = '%d/%s' % (self.period, key)
        return key.encode('utf-8')

@total_ordering
class Credential(object):

    def __init__(self, key, oath_type=OATH_TYPE.TOTP, touch=False):
        self.key = key
        self.oath_type = oath_type
        self.touch = touch
        self.issuer, self.name, self.period = Credential.parse_key(key)

    def __lt__(self, other):
        a = ((self.issuer or self.name).lower(), self.name.lower())
        b = ((other.issuer or other.name).lower(), other.name.lower())
        return a < b

    @property
    def is_steam(self):
        return self.issuer == 'Steam'

    @property
    def is_hidden(self):
        return self.issuer == '_hidden'

    @staticmethod
    def parse_key(data):
        if re.match(br'^\d+/', data):
            period, data = data.split(b'/', 1)
            period = int(period)
        else:
            period = 30

        if b':' in data:
            issuer, data = data.split(b':', 1)
            issuer = issuer.decode('utf-8')
        else:
            issuer = None
        return issuer, data.decode('utf-8'), period

def _derive_key(salt, passphrase):
    kdf = PBKDF2HMAC(hashes.SHA1(), 16, salt, 1000, default_backend())
    return kdf.derive(passphrase.encode('utf-8'))

def _get_device_id(device_salt):
    h = hashes.Hash(hashes.SHA256(), default_backend())
    h.update(device_salt)
    d = h.finalize()[:16]
    return b64encode(d).replace(b'=', b'').decode()

class Code(object):

    def __init__(self, value, valid_from, valid_to):
        self.value = value
        self.valid_from = valid_from
        self.valid_to = valid_to

    def __str__(self):
        return self.value

class OathController(object):

    def __init__(self, driver):
        resp = driver.select(AID.OATH)
        tags = dict((x.tag, x.value) for x in parse_tlvs(resp))
        self._version = tuple(six.iterbytes(tags[TAG.VERSION]))
        self._salt = tags[TAG.NAME]
        self._id = _get_device_id(self._salt)
        self._challenge = tags.get(TAG.CHALLENGE)
        self._driver = driver

    @property
    def version(self):
        return self._version

    @property
    def id(self):
        return self._id

    @property
    def locked(self):
        return self._challenge is not None

    @property
    def _426device(self):
        return (4, 2, 0) <= self.version <= (4, 2, 6)

    def derive_key(self, password):
        return _derive_key(self._salt, password)

    def send_apdu(self, ins, p1, p2, data=b''):
        resp, sw = self._driver.send_apdu(0, ins, p1, p2, data, check=None)
        while (sw >> 8) == SW.MORE_DATA:
            more, sw = self._driver.send_apdu(
                0, INS.SEND_REMAINING, 0, 0, b'', check=None)
            resp += more

        if sw != SW_OK:
            raise APDUError(resp, sw)

        return resp

    def reset(self):
        self.send_apdu(INS.RESET, 0xde, 0xad)
        resp = self._driver.select(AID.OATH)
        tags = dict((x.tag, x.value) for x in parse_tlvs(resp))
        self._salt = tags[TAG.NAME]
        self._id = _get_device_id(self._salt)

    def put(self, credential_data):
        d = credential_data
        key = d.make_key()
        secret_header = bytearray([d.oath_type | d.algorithm, d.digits])
        secret = hmac_shorten_key(d.secret, d.algorithm.name)
        data = Tlv(TAG.NAME, key) + Tlv(TAG.KEY, secret_header + secret)

        properties = 0

        if d.touch:
            properties |= PROPERTIES.REQUIRE_TOUCH

        if properties:
            data += bytearray([TAG.PROPERTY, properties])

        if d.counter > 0:
            data += Tlv(TAG.IMF, struct.pack('>I', d.counter))

        self.send_apdu(INS.PUT, 0, 0, bytes(data))
        return Credential(key, d.oath_type, d.touch)

    def list(self):
        def _gen_creds():
            resp = self.send_apdu(INS.LIST, 0, 0)
            while resp:
                length = six.indexbytes(resp, 1) - 1
                oath_type = OATH_TYPE(MASK.TYPE & six.indexbytes(resp, 2))
                key = resp[3:3 + length]
                yield Credential(key, oath_type)
                resp = resp[3 + length:]

        return list(_gen_creds())

    def calculate(self, cred, timestamp=None):

        # The 4.2.0-4.2.6 firmwares have a known issue with credentials that
        # require touch: If this action is performed within 2 seconds of a
        # command resulting in a long response (over 54 bytes),
        # the command will hang. A workaround is to send an invalid command
        # (resulting in a short reply) prior to the "calculate" command.
        if self._426device and cred.touch:
            self._driver.send_apdu(0, 0, 0, 0, '', check=SW.INVALID_INSTRUCTION)

        if timestamp is None:
            timestamp = int(time.time())
        challenge = time_challenge(timestamp, period=cred.period)
        data = Tlv(TAG.NAME, cred.key) + Tlv(TAG.CHALLENGE, challenge)
        resp = self.send_apdu(INS.CALCULATE, 0, 0, data)
        resp = parse_tlvs(resp)[0].value
        # Manual dynamic truncation is required
        # for Steam entries, so let's do it for all.
        digits = six.indexbytes(resp, 0)
        resp = resp[1:]
        offset = six.indexbytes(resp, -1) & 0xF
        code_data = resp[offset:offset + 4]
        code_data = parse_truncated(code_data)
        code_value = format_code(code_data, digits, steam=cred.is_steam)
        if cred.oath_type == OATH_TYPE.TOTP:
            valid_from = timestamp - (timestamp % cred.period)
            valid_to = valid_from + cred.period
        else:
            valid_from = timestamp
            valid_to = float('Inf')
        return Code(code_value, valid_from, valid_to)

    def delete(self, cred):
        data = Tlv(TAG.NAME, cred.key)
        self.send_apdu(INS.DELETE, 0, 0, data)

    def calculate_all(self, timestamp=None):
        if timestamp is None:
            timestamp = int(time.time())

        def _gen_all():
            valid_from = timestamp - (timestamp % 30)
            valid_to = valid_from + 30
            data = Tlv(TAG.CHALLENGE, time_challenge(timestamp))
            resp = self.send_apdu(INS.CALCULATE_ALL, 0, 0x01, data)
            tlvs = parse_tlvs(resp)
            while tlvs:
                key = tlvs.pop(0).value
                resp = tlvs.pop(0)
                oath_type = OATH_TYPE.HOTP if resp.tag == TAG.NO_RESPONSE else \
                    OATH_TYPE.TOTP
                touch = resp.tag == TAG.TOUCH
                cred = Credential(key, oath_type, touch)

                if resp.tag == TAG.TRUNCATED_RESPONSE:
                    if cred.period != 30 or cred.is_steam:
                        code = self.calculate(cred, timestamp)
                    else:
                        digits = six.indexbytes(resp.value, 0)
                        code_value = parse_truncated(resp.value[1:])
                        code_value = format_code(code_value, digits)
                        code = Code(code_value, valid_from, valid_to)
                else:
                    code = None

                yield cred, code
        return list(_gen_all())

    def set_password(self, password):
        key = self.derive_key(password)
        keydata = bytearray([OATH_TYPE.TOTP | ALGO.SHA1]) + key
        challenge = os.urandom(8)
        h = hmac.HMAC(key, hashes.SHA1(), default_backend())
        h.update(challenge)
        response = h.finalize()
        data = Tlv(TAG.KEY, keydata) + Tlv(TAG.CHALLENGE, challenge) + Tlv(
            TAG.RESPONSE, response)
        self.send_apdu(INS.SET_CODE, 0, 0, data)
        return key

    def clear_password(self):
        self.send_apdu(INS.SET_CODE, 0, 0, Tlv(TAG.KEY, b''))

    def validate(self, key):
        h = hmac.HMAC(key, hashes.SHA1(), default_backend())
        h.update(self._challenge)
        response = h.finalize()
        challenge = os.urandom(8)
        h = hmac.HMAC(key, hashes.SHA1(), default_backend())
        h.update(challenge)
        verification = h.finalize()
        data = Tlv(TAG.RESPONSE, response) + Tlv(TAG.CHALLENGE, challenge)
        resp = self.send_apdu(INS.VALIDATE, 0, 0, data)
        if parse_tlvs(resp)[0].value != verification:
            raise ValueError(
                'Response from validation does not match verification!')
        self._challenge = None
<commit_msg>Set period to None for non-TOTP.<commit_after># Copyright (c) 2015 Yubico AB
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#    1. Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    2. Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import os
import re
import struct
import six
import time
from base64 import b64encode
from functools import total_ordering
from enum import IntEnum, unique
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
from cryptography.hazmat.primitives import hmac, hashes
from cryptography.hazmat.backends import default_backend
from six.moves.urllib.parse import unquote, urlparse, parse_qs
from .driver_ccid import APDUError, SW_OK
from .util import (
    AID, Tlv, parse_tlvs, time_challenge, parse_b32_key,
    format_code, parse_truncated, hmac_shorten_key)

@unique
class TAG(IntEnum):
    NAME = 0x71
    NAME_LIST = 0x72
    KEY = 0x73
    CHALLENGE = 0x74
    RESPONSE = 0x75
    TRUNCATED_RESPONSE = 0x76
    NO_RESPONSE = 0x77
    PROPERTY = 0x78
    VERSION = 0x79
    IMF = 0x7a
    ALGORITHM = 0x7b
    TOUCH = 0x7c

@unique
class ALGO(IntEnum):
    SHA1 = 0x01
    SHA256 = 0x02
    SHA512 = 0x03

@unique
class OATH_TYPE(IntEnum):
    HOTP = 0x10
    TOTP = 0x20

@unique
class PROPERTIES(IntEnum):
    REQUIRE_TOUCH = 0x02

@unique
class INS(IntEnum):
    PUT = 0x01
    DELETE = 0x02
    SET_CODE = 0x03
    RESET = 0x04
    LIST = 0xa1
    CALCULATE = 0xa2
    VALIDATE = 0xa3
    CALCULATE_ALL = 0xa4
    SEND_REMAINING = 0xa5

@unique
class MASK(IntEnum):
    ALGO = 0x0f
    TYPE = 0xf0

@unique
class SW(IntEnum):
    NO_SPACE = 0x6a84
    COMMAND_ABORTED = 0x6f00
    MORE_DATA = 0x61
    INVALID_INSTRUCTION = 0x6d00

class CredentialData(object):

    def __init__(self, secret, issuer, name, oath_type=OATH_TYPE.TOTP,
                 algorithm=ALGO.SHA1, digits=6, period=30, counter=0,
                 touch=False):
        self.secret = secret
        self.issuer = issuer
        self.name = name
        self.oath_type = oath_type
        self.algorithm = algorithm
        self.digits = digits
        self.period = period
        self.counter = counter
        self.touch = touch

    @classmethod
    def from_uri(cls, uri):
        parsed = urlparse(uri.strip())
        if parsed.scheme != 'otpauth':
            raise ValueError('Invalid URI scheme')

        params = dict((k, v[0]) for k, v in parse_qs(parsed.query).items())
        params['secret'] = parse_b32_key(params['secret'])
        params['algorithm'] = ALGO[params.get('algorithm', 'SHA1').upper()]
        issuer = None
        name = unquote(parsed.path)[1:]  # Unquote and strip leading /
        if ':' in name:
            issuer, name = name.split(':', 1)
        params['issuer'] = params.get('issuer', issuer)
        params['name'] = name
        params['oath_type'] = OATH_TYPE[parsed.hostname.upper()]
        params['digits'] = int(params.get('digits', 6))
        params['period'] = int(params.get('period', 30))
        params['counter'] = int(params.get('counter', 0))
        return cls(**params)

    def make_key(self):
        key = self.name
        if self.issuer:
            key = '%s:%s' % (self.issuer, key)
        if self.oath_type == OATH_TYPE.TOTP and self.period != 30:
            key = '%d/%s' % (self.period, key)
        return key.encode('utf-8')

@total_ordering
class Credential(object):

    def __init__(self, key, oath_type=OATH_TYPE.TOTP, touch=False):
        self.key = key
        self.oath_type = oath_type
        self.touch = touch
        self.issuer, self.name, period = Credential.parse_key(key)
        self.period = period if oath_type == OATH_TYPE.TOTP else None

    def __lt__(self, other):
        a = ((self.issuer or self.name).lower(), self.name.lower())
        b = ((other.issuer or other.name).lower(), other.name.lower())
        return a < b

    @property
    def is_steam(self):
        return self.issuer == 'Steam'

    @property
    def is_hidden(self):
        return self.issuer == '_hidden'

    @staticmethod
    def parse_key(data):
        if re.match(br'^\d+/', data):
            period, data = data.split(b'/', 1)
            period = int(period)
        else:
            period = 30

        if b':' in data:
            issuer, data = data.split(b':', 1)
            issuer = issuer.decode('utf-8')
        else:
            issuer = None
        return issuer, data.decode('utf-8'), period

def _derive_key(salt, passphrase):
    kdf = PBKDF2HMAC(hashes.SHA1(), 16, salt, 1000, default_backend())
    return kdf.derive(passphrase.encode('utf-8'))

def _get_device_id(device_salt):
    h = hashes.Hash(hashes.SHA256(), default_backend())
    h.update(device_salt)
    d = h.finalize()[:16]
    return b64encode(d).replace(b'=', b'').decode()

class Code(object):

    def __init__(self, value, valid_from, valid_to):
        self.value = value
        self.valid_from = valid_from
        self.valid_to = valid_to

    def __str__(self):
        return self.value

class OathController(object):

    def __init__(self, driver):
        resp = driver.select(AID.OATH)
        tags = dict((x.tag, x.value) for x in parse_tlvs(resp))
        self._version = tuple(six.iterbytes(tags[TAG.VERSION]))
        self._salt = tags[TAG.NAME]
        self._id = _get_device_id(self._salt)
        self._challenge = tags.get(TAG.CHALLENGE)
        self._driver = driver

    @property
    def version(self):
        return self._version

    @property
    def id(self):
        return self._id

    @property
    def locked(self):
        return self._challenge is not None

    @property
    def _426device(self):
        return (4, 2, 0) <= self.version <= (4, 2, 6)

    def derive_key(self, password):
        return _derive_key(self._salt, password)

    def send_apdu(self, ins, p1, p2, data=b''):
        resp, sw = self._driver.send_apdu(0, ins, p1, p2, data, check=None)
        while (sw >> 8) == SW.MORE_DATA:
            more, sw = self._driver.send_apdu(
                0, INS.SEND_REMAINING, 0, 0, b'', check=None)
            resp += more

        if sw != SW_OK:
            raise APDUError(resp, sw)

        return resp

    def reset(self):
        self.send_apdu(INS.RESET, 0xde, 0xad)
        resp = self._driver.select(AID.OATH)
        tags = dict((x.tag, x.value) for x in parse_tlvs(resp))
        self._salt = tags[TAG.NAME]
        self._id = _get_device_id(self._salt)

    def put(self, credential_data):
        d = credential_data
        key = d.make_key()
        secret_header = bytearray([d.oath_type | d.algorithm, d.digits])
        secret = hmac_shorten_key(d.secret, d.algorithm.name)
        data = Tlv(TAG.NAME, key) + Tlv(TAG.KEY, secret_header + secret)

        properties = 0

        if d.touch:
            properties |= PROPERTIES.REQUIRE_TOUCH

        if properties:
            data += bytearray([TAG.PROPERTY, properties])

        if d.counter > 0:
            data += Tlv(TAG.IMF, struct.pack('>I', d.counter))

        self.send_apdu(INS.PUT, 0, 0, bytes(data))
        return Credential(key, d.oath_type, d.touch)

    def list(self):
        def _gen_creds():
            resp = self.send_apdu(INS.LIST, 0, 0)
            while resp:
                length = six.indexbytes(resp, 1) - 1
                oath_type = OATH_TYPE(MASK.TYPE & six.indexbytes(resp, 2))
                key = resp[3:3 + length]
                yield Credential(key, oath_type)
                resp = resp[3 + length:]

        return list(_gen_creds())

    def calculate(self, cred, timestamp=None):

        # The 4.2.0-4.2.6 firmwares have a known issue with credentials that
        # require touch: If this action is performed within 2 seconds of a
        # command resulting in a long response (over 54 bytes),
        # the command will hang. A workaround is to send an invalid command
        # (resulting in a short reply) prior to the "calculate" command.
        if self._426device and cred.touch:
            self._driver.send_apdu(0, 0, 0, 0, '', check=SW.INVALID_INSTRUCTION)

        if timestamp is None:
            timestamp = int(time.time())
        if cred.oath_type == OATH_TYPE.TOTP:
            valid_from = timestamp - (timestamp % cred.period)
            valid_to = valid_from + cred.period
            challenge = time_challenge(timestamp, period=cred.period)
        else:
            valid_from = timestamp
            valid_to = float('Inf')
            challenge = b''
        data = Tlv(TAG.NAME, cred.key) + Tlv(TAG.CHALLENGE, challenge)
        resp = self.send_apdu(INS.CALCULATE, 0, 0, data)
        resp = parse_tlvs(resp)[0].value
        # Manual dynamic truncation is required
        # for Steam entries, so let's do it for all.
        digits = six.indexbytes(resp, 0)
        resp = resp[1:]
        offset = six.indexbytes(resp, -1) & 0xF
        code_data = resp[offset:offset + 4]
        code_data = parse_truncated(code_data)
        code_value = format_code(code_data, digits, steam=cred.is_steam)
        return Code(code_value, valid_from, valid_to)

    def delete(self, cred):
        data = Tlv(TAG.NAME, cred.key)
        self.send_apdu(INS.DELETE, 0, 0, data)

    def calculate_all(self, timestamp=None):
        if timestamp is None:
            timestamp = int(time.time())

        def _gen_all():
            valid_from = timestamp - (timestamp % 30)
            valid_to = valid_from + 30
            data = Tlv(TAG.CHALLENGE, time_challenge(timestamp))
            resp = self.send_apdu(INS.CALCULATE_ALL, 0, 0x01, data)
            tlvs = parse_tlvs(resp)
            while tlvs:
                key = tlvs.pop(0).value
                resp = tlvs.pop(0)
                oath_type = OATH_TYPE.HOTP if resp.tag == TAG.NO_RESPONSE else \
                    OATH_TYPE.TOTP
                touch = resp.tag == TAG.TOUCH
                cred = Credential(key, oath_type, touch)

                if resp.tag == TAG.TRUNCATED_RESPONSE:
                    if cred.period != 30 or cred.is_steam:
                        code = self.calculate(cred, timestamp)
                    else:
                        digits = six.indexbytes(resp.value, 0)
                        code_value = parse_truncated(resp.value[1:])
                        code_value = format_code(code_value, digits)
                        code = Code(code_value, valid_from, valid_to)
                else:
                    code = None

                yield cred, code
        return list(_gen_all())

    def set_password(self, password):
        key = self.derive_key(password)
        keydata = bytearray([OATH_TYPE.TOTP | ALGO.SHA1]) + key
        challenge = os.urandom(8)
        h = hmac.HMAC(key, hashes.SHA1(), default_backend())
        h.update(challenge)
        response = h.finalize()
        data = Tlv(TAG.KEY, keydata) + Tlv(TAG.CHALLENGE, challenge) + Tlv(
            TAG.RESPONSE, response)
        self.send_apdu(INS.SET_CODE, 0, 0, data)
        return key

    def clear_password(self):
        self.send_apdu(INS.SET_CODE, 0, 0, Tlv(TAG.KEY, b''))

    def validate(self, key):
        h = hmac.HMAC(key, hashes.SHA1(), default_backend())
        h.update(self._challenge)
        response = h.finalize()
        challenge = os.urandom(8)
        h = hmac.HMAC(key, hashes.SHA1(), default_backend())
        h.update(challenge)
        verification = h.finalize()
        data = Tlv(TAG.RESPONSE, response) + Tlv(TAG.CHALLENGE, challenge)
        resp = self.send_apdu(INS.VALIDATE, 0, 0, data)
        if parse_tlvs(resp)[0].value != verification:
            raise ValueError(
                'Response from validation does not match verification!')
        self._challenge = None
<|endoftext|>


<commit_before>from __future__ import print_function

__author__ = "John Kirkham <kirkhamj@janelia.hhmi.org>"
__date__ = "$May 18, 2015 22:08:21 EDT$"

import os
import shutil
import tempfile
import time

from splauncher.core import main

class TestCore(object):
    def setup(self):
        self.cwd = os.getcwd()
        self.tempdir = ""
        self.tempdir = tempfile.mkdtemp()
        os.chdir(self.tempdir)

        print("tempdir = \"%s\"" % self.tempdir)

    def teardown(self):
        os.chdir(self.cwd)
        shutil.rmtree(self.tempdir)
        self.tempdir = ""
        self.cwd = ""

    def test_main_0(self):
        main("",
            "python",
            "-c",
            "from __future__ import print_function;" +
            "import sys;" +
            "print(\"output\", file=sys.stdout)"
        )
        time.sleep(5)

        filenames = []
        for each_filename in os.listdir(self.tempdir):
            filenames.append(os.path.join(self.tempdir, each_filename))

        assert ".err" in filenames[0]
        assert ".out" in filenames[1]

        with open(filenames[0], "r") as f:
            s = f.read().strip()
            print("File \"%s\" contains \"%s\"." % (f.name, s))
            assert s == ""

        with open(filenames[1], "r") as f:
            s = f.read().strip()
            print("File \"%s\" contains \"%s\"." % (f.name, s))
            assert s == "output"

    def test_main_1(self):
        main("",
            "python",
            "-c",
            "from __future__ import print_function;" +
            "import sys;" +
            "print(\"error\", file=sys.stderr)"
        )
        time.sleep(5)

        filenames = []
        for each_filename in os.listdir(self.tempdir):
            filenames.append(os.path.join(self.tempdir, each_filename))

        assert ".err" in filenames[0]
        assert ".out" in filenames[1]

        with open(filenames[0], "r") as f:
            s = f.read().strip()
            print("File \"%s\" contains \"%s\"." % (f.name, s))
            assert s == "error"

        with open(filenames[1], "r") as f:
            s = f.read().strip()
            print("File \"%s\" contains \"%s\"." % (f.name, s))
            assert s == ""
<commit_msg>tests/test_splauncher/test_core.py: Wait until both the out and error files have been added.<commit_after>from __future__ import print_function

__author__ = "John Kirkham <kirkhamj@janelia.hhmi.org>"
__date__ = "$May 18, 2015 22:08:21 EDT$"

import os
import shutil
import tempfile
import time

from splauncher.core import main

class TestCore(object):
    def setup(self):
        self.cwd = os.getcwd()
        self.tempdir = ""
        self.tempdir = tempfile.mkdtemp()
        os.chdir(self.tempdir)

        print("tempdir = \"%s\"" % self.tempdir)

    def teardown(self):
        os.chdir(self.cwd)
        shutil.rmtree(self.tempdir)
        self.tempdir = ""
        self.cwd = ""

    def test_main_0(self):
        main("",
            "python",
            "-c",
            "from __future__ import print_function;" +
            "import sys;" +
            "print(\"output\", file=sys.stdout)"
        )

        while len(os.listdir(self.tempdir)) < 2:
           time.sleep(1)

        filenames = []
        for each_filename in os.listdir(self.tempdir):
            filenames.append(os.path.join(self.tempdir, each_filename))

        assert ".err" in filenames[0]
        assert ".out" in filenames[1]

        with open(filenames[0], "r") as f:
            s = f.read().strip()
            print("File \"%s\" contains \"%s\"." % (f.name, s))
            assert s == ""

        with open(filenames[1], "r") as f:
            s = f.read().strip()
            print("File \"%s\" contains \"%s\"." % (f.name, s))
            assert s == "output"

    def test_main_1(self):
        main("",
            "python",
            "-c",
            "from __future__ import print_function;" +
            "import sys;" +
            "print(\"error\", file=sys.stderr)"
        )

        while len(os.listdir(self.tempdir)) < 2:
           time.sleep(1)

        filenames = []
        for each_filename in os.listdir(self.tempdir):
            filenames.append(os.path.join(self.tempdir, each_filename))

        assert ".err" in filenames[0]
        assert ".out" in filenames[1]

        with open(filenames[0], "r") as f:
            s = f.read().strip()
            print("File \"%s\" contains \"%s\"." % (f.name, s))
            assert s == "error"

        with open(filenames[1], "r") as f:
            s = f.read().strip()
            print("File \"%s\" contains \"%s\"." % (f.name, s))
            assert s == ""
<|endoftext|>