<commit_before>from functools import partial
import os
from stat import S_IFREG
import subprocess
import sys
import time

import click

from .devices import Device
from .size import ByteSize

# this will possibly have to be adjusted in the future with huge usb-sticks
G = 1024**3

def candidate_for_writing(dev, max_size):
    if not dev.is_device:
        return False, 'not a device'

    if not dev.major == 8:
        return False, 'major device number is not 8 (SCSI)'

    if dev.read_only:
        return False, 'device is read-only'

    if not dev.removable:
        return False, 'device is not a removable device'

    if dev.size == 0:
        return False, 'device has 0 size'

    if dev.size > max_size:
        return False, 'device has a size larger than {} bytes'.format(max_size)

    return True, None

def dev_info(dev):
    return '{0.path} ({0.size:.1fH}): {0.model}'.format(dev)

class Reader(object):
    # FIXME: replace with reader that encompasses complete pipe and can
    #        count bytes in as well as bytes out, for more accurate
    #        progressbars
    BUFFER_SIZE_MIN = 1024  # 1 KB
    BUFFER_SIZE_MAX = 16 * 1024 * 1024  # 16 M
    BUFFER_SIZE_DEFAULT = 512 * 1024  # 512 KB

    def __init__(self, src, buffer_size, limit=None):
        self.src = src
        self.bytes_read = 0
        self.buffer_size = buffer_size or self.BUFFER_SIZE_DEFAULT
        assert self.BUFFER_SIZE_MIN <= self.buffer_size <= self.BUFFER_SIZE_MAX
        self.limit = limit

    def increase_buffer(self):
        self.buffer_size = min(self.buffer_size * 2, self.BUFFER_SIZE_MAX)

    def decreate_buffer(self):
        self.buffer_size = max(self.buffer_size // 2, self.BUFFER_SIZE_MIN)

    def __iter__(self):
        return self

    def __next__(self):
        if self.limit is not None:
            read_size = min(self.buffer_size, self.limit - self.bytes_read)
        else:
            read_size = self.buffer_size

        if read_size <= 0:
            raise StopIteration

        buf = self.src.read(read_size)

        if not buf:
            raise StopIteration

        self.bytes_read += len(buf)

        return buf

    next = __next__

@click.command()
@click.argument('image-file',
                type=click.Path(readable=True,
                                dir_okay=False,
                                exists=True))
@click.option('--chunk-size',
              '-C',
              type=ByteSize,
              default=None,
              help='Read-buffer size (default: auto-adjust)')
@click.option('-d',
              '--decompress',
              default='auto',
              type=click.Choice(['auto', 'none', 'xz']),
              help='On-the-fly decompression (default: auto-detect)')
@click.option('-e/-E',
              '--eject/--no-eject',
              default=True,
              help='Eject medium after writing (default: True).')
@click.option('--limit',
              '-l',
              type=ByteSize,
              help='Write no more than this many bytes to device.')
@click.option(
    '--max-size',
    default=32 * G,
    help='Maximum size in bytes before rejecting to write to device.')
@click.option('--target',
              '-t',
              type=click.Path(dir_okay=False,
                              writable=True),
              help='The target to write to. If none is given, a menu is shown'
              ' to select one.')
@click.option('--verbose',
              '-v',
              is_flag=True,
              default=False,
              help='Output more detailed information.')
@click.option('--i-know-what-im-doing',
              is_flag=True,
              default=False,
              help='Disable all safety checks.')
def wrimg(image_file, target, verbose, i_know_what_im_doing, limit, chunk_size,
          decompress, max_size, eject):
    if verbose:
        info = click.echo
    else:
        info = lambda *args, **kwargs: None

    error = partial(click.echo, err=True)

    if not target:
        candidates = []
        for dev in sorted(Device.iter_block_devices(), key=lambda d: d.path):
            ok, msg = candidate_for_writing(dev, max_size)
            if not ok:
                info('skipping {}: {}'.format(dev.path, msg))
                continue
            candidates.append(dev)

        if not candidates:
            error('No device given on command-line and no suitable candidate '
                  'found')
            sys.exit(1)
        elif len(candidates) == 1:
            target = candidates[0]
        else:
            # display menu
            for i, c in enumerate(candidates):
                click.echo('[{}] {}'.format(i, dev_info(c)))

            idx = int(click.prompt('Select a device',
                                   type=click.Choice(map(str, range(len(
                                       candidates))))))

            target = candidates[idx]
    else:
        target = Device(target)

    # sanity-check target-device
    ok, msg = candidate_for_writing(target)

    if not ok:
        if not i_know_what_im_doing:
            error('{.path}: {}.\nAdd --i-know-what-im-doing to disable this '
                  'check.'.format(target, msg))
            sys.exit(0)
        error('WARNING: {}: {}'.format(dev_info(target), msg))

    # determine compression type
    verb = 'Write'

    if decompress == 'auto':
        if image_file.endswith('.xz'):
            decompress = 'xz'
        else:
            error('Could not determine compression type from file-ending')
            sys.exit(1)
    elif decompress == 'none':
        decompress = None

    if decompress:
        info('Decompression enabled, using {}'.format(decompress))
        verb = 'Decompress ({}) and write'.format(decompress)

    # confirm start, featuring the defensive assert!
    assert click.confirm('{} to {}?'.format(verb, dev_info(target)),
                         err=True,
                         abort=True)

    # determine number of bytes to write
    img_st = os.stat(image_file)

    # start with the target device size
    total = target.size

    if not decompress and img_st.st_mode & S_IFREG:
        # regular file, limit by its size
        total = min(total, img_st.st_size)

    # apply limit
    if limit is not None:
        total = min(total, limit)

    info('Copying {:.0f} bytes from {} to {.path}'.format(total, image_file,
                                                          target))

    with open(image_file, 'rb') as src, target.open('wb') as dst:
        # compression hooks in here
        if decompress:
            if decompress == 'xz':
                decompress = subprocess.Popen(
                    ['xz', '--decompress', '--stdout'],
                    stdout=subprocess.PIPE,
                    stdin=src, )
            else:
                raise NotImplementedError('Unknown compressoin type {}'
                                          .format(decompress))

            src = decompress.stdout

        reader = Reader(src, chunk_size, limit)

        if not decompress:
            pbar = click.progressbar(length=total,
                                     label='writing',
                                     info_sep='| ')
        else:

            def show_bytes_written(item):
                return 'total: {:.1f H}'.format(ByteSize(reader.bytes_read))

            pbar = click.progressbar(reader,
                                     label='writing',
                                     info_sep='| ',
                                     item_show_func=show_bytes_written)

        with pbar as bar:
            chunk_source = pbar if decompress else reader

            for chunk in chunk_source:
                # measure time
                start = time.time()

                dst.write(chunk)
                dst.flush()
                os.fsync(dst.fileno())

                end = time.time()

                speed = ByteSize(len(chunk) / (end - start))

                # adjust chunk size if needed
                if chunk_size is None:
                    if end - start > 2:
                        # took longer then two seconds, halve bufsize
                        reader.decrease_buffer()
                    elif end - start < 0.5:
                        reader.increase_buffer()

                bar.label = '{:.1f H}/s'.format(speed)

                if not decompress:
                    bar.update(len(chunk))

    # we're done writing, call eject
    info('Ejecting {}'.format(target.path))
    subprocess.check_open(['eject', target.path])
<commit_msg>Fix candidate_for_writing() callsite.<commit_after>from functools import partial
import os
from stat import S_IFREG
import subprocess
import sys
import time

import click

from .devices import Device
from .size import ByteSize

# this will possibly have to be adjusted in the future with huge usb-sticks
G = 1024**3

def candidate_for_writing(dev, max_size):
    if not dev.is_device:
        return False, 'not a device'

    if not dev.major == 8:
        return False, 'major device number is not 8 (SCSI)'

    if dev.read_only:
        return False, 'device is read-only'

    if not dev.removable:
        return False, 'device is not a removable device'

    if dev.size == 0:
        return False, 'device has 0 size'

    if dev.size > max_size:
        return False, 'device has a size larger than {} bytes'.format(max_size)

    return True, None

def dev_info(dev):
    return '{0.path} ({0.size:.1fH}): {0.model}'.format(dev)

class Reader(object):
    # FIXME: replace with reader that encompasses complete pipe and can
    #        count bytes in as well as bytes out, for more accurate
    #        progressbars
    BUFFER_SIZE_MIN = 1024  # 1 KB
    BUFFER_SIZE_MAX = 16 * 1024 * 1024  # 16 M
    BUFFER_SIZE_DEFAULT = 512 * 1024  # 512 KB

    def __init__(self, src, buffer_size, limit=None):
        self.src = src
        self.bytes_read = 0
        self.buffer_size = buffer_size or self.BUFFER_SIZE_DEFAULT
        assert self.BUFFER_SIZE_MIN <= self.buffer_size <= self.BUFFER_SIZE_MAX
        self.limit = limit

    def increase_buffer(self):
        self.buffer_size = min(self.buffer_size * 2, self.BUFFER_SIZE_MAX)

    def decreate_buffer(self):
        self.buffer_size = max(self.buffer_size // 2, self.BUFFER_SIZE_MIN)

    def __iter__(self):
        return self

    def __next__(self):
        if self.limit is not None:
            read_size = min(self.buffer_size, self.limit - self.bytes_read)
        else:
            read_size = self.buffer_size

        if read_size <= 0:
            raise StopIteration

        buf = self.src.read(read_size)

        if not buf:
            raise StopIteration

        self.bytes_read += len(buf)

        return buf

    next = __next__

@click.command()
@click.argument('image-file',
                type=click.Path(readable=True,
                                dir_okay=False,
                                exists=True))
@click.option('--chunk-size',
              '-C',
              type=ByteSize,
              default=None,
              help='Read-buffer size (default: auto-adjust)')
@click.option('-d',
              '--decompress',
              default='auto',
              type=click.Choice(['auto', 'none', 'xz']),
              help='On-the-fly decompression (default: auto-detect)')
@click.option('-e/-E',
              '--eject/--no-eject',
              default=True,
              help='Eject medium after writing (default: True).')
@click.option('--limit',
              '-l',
              type=ByteSize,
              help='Write no more than this many bytes to device.')
@click.option(
    '--max-size',
    default=32 * G,
    help='Maximum size in bytes before rejecting to write to device.')
@click.option('--target',
              '-t',
              type=click.Path(dir_okay=False,
                              writable=True),
              help='The target to write to. If none is given, a menu is shown'
              ' to select one.')
@click.option('--verbose',
              '-v',
              is_flag=True,
              default=False,
              help='Output more detailed information.')
@click.option('--i-know-what-im-doing',
              is_flag=True,
              default=False,
              help='Disable all safety checks.')
def wrimg(image_file, target, verbose, i_know_what_im_doing, limit, chunk_size,
          decompress, max_size, eject):
    if verbose:
        info = click.echo
    else:
        info = lambda *args, **kwargs: None

    error = partial(click.echo, err=True)

    if not target:
        candidates = []
        for dev in sorted(Device.iter_block_devices(), key=lambda d: d.path):
            ok, msg = candidate_for_writing(dev, max_size)
            if not ok:
                info('skipping {}: {}'.format(dev.path, msg))
                continue
            candidates.append(dev)

        if not candidates:
            error('No device given on command-line and no suitable candidate '
                  'found')
            sys.exit(1)
        elif len(candidates) == 1:
            target = candidates[0]
        else:
            # display menu
            for i, c in enumerate(candidates):
                click.echo('[{}] {}'.format(i, dev_info(c)))

            idx = int(click.prompt('Select a device',
                                   type=click.Choice(map(str, range(len(
                                       candidates))))))

            target = candidates[idx]
    else:
        target = Device(target)

    # sanity-check target-device
    ok, msg = candidate_for_writing(target, max_size)

    if not ok:
        if not i_know_what_im_doing:
            error('{.path}: {}.\nAdd --i-know-what-im-doing to disable this '
                  'check.'.format(target, msg))
            sys.exit(0)
        error('WARNING: {}: {}'.format(dev_info(target), msg))

    # determine compression type
    verb = 'Write'

    if decompress == 'auto':
        if image_file.endswith('.xz'):
            decompress = 'xz'
        else:
            error('Could not determine compression type from file-ending')
            sys.exit(1)
    elif decompress == 'none':
        decompress = None

    if decompress:
        info('Decompression enabled, using {}'.format(decompress))
        verb = 'Decompress ({}) and write'.format(decompress)

    # confirm start, featuring the defensive assert!
    assert click.confirm('{} to {}?'.format(verb, dev_info(target)),
                         err=True,
                         abort=True)

    # determine number of bytes to write
    img_st = os.stat(image_file)

    # start with the target device size
    total = target.size

    if not decompress and img_st.st_mode & S_IFREG:
        # regular file, limit by its size
        total = min(total, img_st.st_size)

    # apply limit
    if limit is not None:
        total = min(total, limit)

    info('Copying {:.0f} bytes from {} to {.path}'.format(total, image_file,
                                                          target))

    with open(image_file, 'rb') as src, target.open('wb') as dst:
        # compression hooks in here
        if decompress:
            if decompress == 'xz':
                decompress = subprocess.Popen(
                    ['xz', '--decompress', '--stdout'],
                    stdout=subprocess.PIPE,
                    stdin=src, )
            else:
                raise NotImplementedError('Unknown compressoin type {}'
                                          .format(decompress))

            src = decompress.stdout

        reader = Reader(src, chunk_size, limit)

        if not decompress:
            pbar = click.progressbar(length=total,
                                     label='writing',
                                     info_sep='| ')
        else:

            def show_bytes_written(item):
                return 'total: {:.1f H}'.format(ByteSize(reader.bytes_read))

            pbar = click.progressbar(reader,
                                     label='writing',
                                     info_sep='| ',
                                     item_show_func=show_bytes_written)

        with pbar as bar:
            chunk_source = pbar if decompress else reader

            for chunk in chunk_source:
                # measure time
                start = time.time()

                dst.write(chunk)
                dst.flush()
                os.fsync(dst.fileno())

                end = time.time()

                speed = ByteSize(len(chunk) / (end - start))

                # adjust chunk size if needed
                if chunk_size is None:
                    if end - start > 2:
                        # took longer then two seconds, halve bufsize
                        reader.decrease_buffer()
                    elif end - start < 0.5:
                        reader.increase_buffer()

                bar.label = '{:.1f H}/s'.format(speed)

                if not decompress:
                    bar.update(len(chunk))

    # we're done writing, call eject
    info('Ejecting {}'.format(target.path))
    subprocess.check_open(['eject', target.path])
<|endoftext|>


<commit_before>import six, sys, os
from core.services import PDFGeneratorView
from django.conf import settings
from django.http import HttpResponse
from .forms import HousingCounselorForm

if six.PY2:
    try:
        sys.path.append(os.environ.get('PDFREACTOR_LIB'))
        from PDFreactor import *
    except:
        PDFreactor = None

class HousingCounselorPDFView(PDFGeneratorView):
    def get_render_url(self):
        request = self.request

        form = HousingCounselorForm(request.GET)
        if form.is_valid():
            zip = form.cleaned_data['zip']
            api_url = 'hud-api-replace/%s.html/' % zip
            url = '%s://%s/%s' % (request.scheme, 'localhost', api_url)
            return url
        else:
            raise Exception(form.errors)

    def get(self, request):

        self.request = request
        if settings.DEBUG and PDFreactor is None:
            return HttpResponse("PDF Reactor is not configured, can not render %s" % self.get_render_url())

        return self.generate_pdf()

    def get_filename(self):
        return '%s.pdf' % self.request.GET['zip']

    def generate_pdf(self):
        url = self.get_render_url()

        if self.license is None:
            raise Exception("PDFGeneratorView requires a license")

        try:
            pdf_reactor = PDFreactor()
        except:
            raise PDFReactorNotConfigured('PDFreactor python library path needs to be configured.')

        pdf_reactor.setLogLevel(PDFreactor.LOG_LEVEL_WARN)
        pdf_reactor.setLicenseKey(str(self.license))
        pdf_reactor.setAuthor('CFPB')
        pdf_reactor.setAddTags(True)
        pdf_reactor.setAddBookmarks(True)

        result = pdf_reactor.renderDocumentFromURL(url)

        # Check if successful
        if result is None:
            # Not successful, return 500
            raise Exception('Error while rendering PDF: {}'.format(
                pdf_reactor.getError()))
        else:
            # Set the correct header for PDF output
            response = HttpResponse(result, content_type='application/pdf')
            response['Content-Disposition'] = 'attachment; filename={0}'.format(
                self.get_filename())
            return response
<commit_msg>when bad zip codes are encountered in production, *intentionally*<commit_after>import six, sys, os
from core.services import PDFGeneratorView
from django.conf import settings
from django.http import HttpResponse
from .forms import HousingCounselorForm

if six.PY2:
    try:
        sys.path.append(os.environ.get('PDFREACTOR_LIB'))
        from PDFreactor import *
    except:
        PDFreactor = None

class InvalidZipException(Exception):
    pass

class HousingCounselorPDFView(PDFGeneratorView):
    def get_render_url(self):
        request = self.request

        form = HousingCounselorForm(request.GET)
        if form.is_valid():
            zip = form.cleaned_data['zip']
            api_url = 'hud-api-replace/%s.html/' % zip
            url = '%s://%s/%s' % (request.scheme, 'localhost', api_url)
            return url
        else:
            raise InvalidZipException(form.errors)

    def get(self, request):

        self.request = request
        if settings.DEBUG and PDFreactor is None:
            return HttpResponse("PDF Reactor is not configured, can not render %s" % self.get_render_url())

        try:
            return self.generate_pdf()
        except InvalidZipException:
            if not settings.DEBUG:
                return HttpResponse(status=500)
            raise

    def get_filename(self):
        return '%s.pdf' % self.request.GET['zip']

    def generate_pdf(self):
        url = self.get_render_url()

        if self.license is None:
            raise Exception("PDFGeneratorView requires a license")

        try:
            pdf_reactor = PDFreactor()
        except:
            raise PDFReactorNotConfigured('PDFreactor python library path needs to be configured.')

        pdf_reactor.setLogLevel(PDFreactor.LOG_LEVEL_WARN)
        pdf_reactor.setLicenseKey(str(self.license))
        pdf_reactor.setAuthor('CFPB')
        pdf_reactor.setAddTags(True)
        pdf_reactor.setAddBookmarks(True)

        result = pdf_reactor.renderDocumentFromURL(url)

        # Check if successful
        if result is None:
            # Not successful, return 500
            raise Exception('Error while rendering PDF: {}'.format(
                pdf_reactor.getError()))
        else:
            # Set the correct header for PDF output
            response = HttpResponse(result, content_type='application/pdf')
            response['Content-Disposition'] = 'attachment; filename={0}'.format(
                self.get_filename())
            return response
<|endoftext|>


<commit_before># Copyright 2019 The Meson development team

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This class contains the basic functionality needed to run any interpreter
# or an interpreter-based tool.

import subprocess as S
from pathlib import Path
from threading import Thread
import typing as T
import re
import os

from .. import mlog
from ..environment import Environment
from ..mesonlib import PerMachine, Popen_safe, version_compare, MachineChoice, is_windows, OptionKey
from ..programs import find_external_program, NonExistingExternalProgram

if T.TYPE_CHECKING:
    from ..programs import ExternalProgram

TYPE_result    = T.Tuple[int, T.Optional[str], T.Optional[str]]
TYPE_cache_key = T.Tuple[str, T.Tuple[str, ...], str, T.FrozenSet[T.Tuple[str, str]]]

class CMakeExecutor:
    # The class's copy of the CMake path. Avoids having to search for it
    # multiple times in the same Meson invocation.
    class_cmakebin = PerMachine(None, None)   # type: PerMachine[T.Optional[ExternalProgram]]
    class_cmakevers = PerMachine(None, None)  # type: PerMachine[T.Optional[str]]
    class_cmake_cache = {}  # type: T.Dict[T.Any, TYPE_result]

    def __init__(self, environment: 'Environment', version: str, for_machine: MachineChoice, silent: bool = False):
        self.min_version = version
        self.environment = environment
        self.for_machine = for_machine
        self.cmakebin, self.cmakevers = self.find_cmake_binary(self.environment, silent=silent)
        self.always_capture_stderr = True
        self.print_cmout = False
        self.prefix_paths = []      # type: T.List[str]
        self.extra_cmake_args = []  # type: T.List[str]

        if self.cmakebin is None:
            return

        if not version_compare(self.cmakevers, self.min_version):
            mlog.warning(
                'The version of CMake', mlog.bold(self.cmakebin.get_path()),
                'is', mlog.bold(self.cmakevers), 'but version', mlog.bold(self.min_version),
                'is required')
            self.cmakebin = None
            return

        self.prefix_paths = self.environment.coredata.options[OptionKey('cmake_prefix_path', machine=self.for_machine)].value
        if self.prefix_paths:
            self.extra_cmake_args += ['-DCMAKE_PREFIX_PATH={}'.format(';'.join(self.prefix_paths))]

    def find_cmake_binary(self, environment: Environment, silent: bool = False) -> T.Tuple[T.Optional['ExternalProgram'], T.Optional[str]]:
        # Only search for CMake the first time and store the result in the class
        # definition
        if isinstance(CMakeExecutor.class_cmakebin[self.for_machine], NonExistingExternalProgram):
            mlog.debug(f'CMake binary for {self.for_machine} is cached as not found')
            return None, None
        elif CMakeExecutor.class_cmakebin[self.for_machine] is not None:
            mlog.debug(f'CMake binary for {self.for_machine} is cached.')
        else:
            assert CMakeExecutor.class_cmakebin[self.for_machine] is None

            mlog.debug(f'CMake binary for {self.for_machine} is not cached')
            for potential_cmakebin in find_external_program(
                    environment, self.for_machine, 'cmake', 'CMake',
                    environment.default_cmake, allow_default_for_cross=False):
                version_if_ok = self.check_cmake(potential_cmakebin)
                if not version_if_ok:
                    continue
                if not silent:
                    mlog.log('Found CMake:', mlog.bold(potential_cmakebin.get_path()),
                             f'({version_if_ok})')
                CMakeExecutor.class_cmakebin[self.for_machine] = potential_cmakebin
                CMakeExecutor.class_cmakevers[self.for_machine] = version_if_ok
                break
            else:
                if not silent:
                    mlog.log('Found CMake:', mlog.red('NO'))
                # Set to False instead of None to signify that we've already
                # searched for it and not found it
                CMakeExecutor.class_cmakebin[self.for_machine] = NonExistingExternalProgram()
                CMakeExecutor.class_cmakevers[self.for_machine] = None
                return None, None

        return CMakeExecutor.class_cmakebin[self.for_machine], CMakeExecutor.class_cmakevers[self.for_machine]

    def check_cmake(self, cmakebin: 'ExternalProgram') -> T.Optional[str]:
        if not cmakebin.found():
            mlog.log(f'Did not find CMake {cmakebin.name!r}')
            return None
        try:
            p, out = Popen_safe(cmakebin.get_command() + ['--version'])[0:2]
            if p.returncode != 0:
                mlog.warning('Found CMake {!r} but couldn\'t run it'
                             ''.format(' '.join(cmakebin.get_command())))
                return None
        except FileNotFoundError:
            mlog.warning('We thought we found CMake {!r} but now it\'s not there. How odd!'
                         ''.format(' '.join(cmakebin.get_command())))
            return None
        except PermissionError:
            msg = 'Found CMake {!r} but didn\'t have permissions to run it.'.format(' '.join(cmakebin.get_command()))
            if not is_windows():
                msg += '\n\nOn Unix-like systems this is often caused by scripts that are not executable.'
            mlog.warning(msg)
            return None
        cmvers = re.search(r'(cmake|cmake3)\s*version\s*([\d.]+)', out).group(2)
        return cmvers

    def set_exec_mode(self, print_cmout: T.Optional[bool] = None, always_capture_stderr: T.Optional[bool] = None) -> None:
        if print_cmout is not None:
            self.print_cmout = print_cmout
        if always_capture_stderr is not None:
            self.always_capture_stderr = always_capture_stderr

    def _cache_key(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_cache_key:
        fenv = frozenset(env.items()) if env is not None else frozenset()
        targs = tuple(args)
        return (self.cmakebin.get_path(), targs, build_dir.as_posix(), fenv)

    def _call_cmout_stderr(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_result:
        cmd = self.cmakebin.get_command() + args
        proc = S.Popen(cmd, stdout=S.PIPE, stderr=S.PIPE, cwd=str(build_dir), env=env)  # TODO [PYTHON_37]: drop Path conversion

        # stdout and stderr MUST be read at the same time to avoid pipe
        # blocking issues. The easiest way to do this is with a separate
        # thread for one of the pipes.
        def print_stdout() -> None:
            while True:
                line = proc.stdout.readline()
                if not line:
                    break
                mlog.log(line.decode(errors='ignore').strip('\n'))
            proc.stdout.close()

        t = Thread(target=print_stdout)
        t.start()

        try:
            # Read stderr line by line and log non trace lines
            raw_trace = ''
            tline_start_reg = re.compile(r'^\s*(.*\.(cmake|txt))\(([0-9]+)\):\s*(\w+)\(.*$')
            inside_multiline_trace = False
            while True:
                line_raw = proc.stderr.readline()
                if not line_raw:
                    break
                line = line_raw.decode(errors='ignore')
                if tline_start_reg.match(line):
                    raw_trace += line
                    inside_multiline_trace = not line.endswith(' )\n')
                elif inside_multiline_trace:
                    raw_trace += line
                else:
                    mlog.warning(line.strip('\n'))

        finally:
            proc.stderr.close()
            t.join()
            proc.wait()

        return proc.returncode, None, raw_trace

    def _call_cmout(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_result:
        cmd = self.cmakebin.get_command() + args
        proc = S.Popen(cmd, stdout=S.PIPE, stderr=S.STDOUT, cwd=str(build_dir), env=env)  # TODO [PYTHON_37]: drop Path conversion
        while True:
            line = proc.stdout.readline()
            if not line:
                break
            mlog.log(line.decode(errors='ignore').strip('\n'))
        proc.stdout.close()
        proc.wait()
        return proc.returncode, None, None

    def _call_quiet(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_result:
        build_dir.mkdir(parents=True, exist_ok=True)
        cmd = self.cmakebin.get_command() + args
        ret = S.run(cmd, env=env, cwd=str(build_dir), close_fds=False,
                    stdout=S.PIPE, stderr=S.PIPE, universal_newlines=False)   # TODO [PYTHON_37]: drop Path conversion
        rc = ret.returncode
        out = ret.stdout.decode(errors='ignore')
        err = ret.stderr.decode(errors='ignore')
        return rc, out, err

    def _call_impl(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_result:
        mlog.debug(f'Calling CMake ({self.cmakebin.get_command()}) in {build_dir} with:')
        for i in args:
            mlog.debug(f'  - "{i}"')
        if not self.print_cmout:
            return self._call_quiet(args, build_dir, env)
        else:
            if self.always_capture_stderr:
                return self._call_cmout_stderr(args, build_dir, env)
            else:
                return self._call_cmout(args, build_dir, env)

    def call(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]] = None, disable_cache: bool = False) -> TYPE_result:
        if env is None:
            env = os.environ.copy()

        args = args + self.extra_cmake_args
        if disable_cache:
            return self._call_impl(args, build_dir, env)

        # First check if cached, if not call the real cmake function
        cache = CMakeExecutor.class_cmake_cache
        key = self._cache_key(args, build_dir, env)
        if key not in cache:
            cache[key] = self._call_impl(args, build_dir, env)
        return cache[key]

    def found(self) -> bool:
        return self.cmakebin is not None

    def version(self) -> str:
        return self.cmakevers

    def executable_path(self) -> str:
        return self.cmakebin.get_path()

    def get_command(self) -> T.List[str]:
        return self.cmakebin.get_command()

    def get_cmake_prefix_paths(self) -> T.List[str]:
        return self.prefix_paths

    def machine_choice(self) -> MachineChoice:
        return self.for_machine
<commit_msg>remove double import<commit_after># Copyright 2019 The Meson development team

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This class contains the basic functionality needed to run any interpreter
# or an interpreter-based tool.

import subprocess as S
from pathlib import Path
from threading import Thread
import typing as T
import re
import os

from .. import mlog
from ..mesonlib import PerMachine, Popen_safe, version_compare, MachineChoice, is_windows, OptionKey
from ..programs import find_external_program, NonExistingExternalProgram

if T.TYPE_CHECKING:
    from ..environment import Environment
    from ..programs import ExternalProgram

TYPE_result    = T.Tuple[int, T.Optional[str], T.Optional[str]]
TYPE_cache_key = T.Tuple[str, T.Tuple[str, ...], str, T.FrozenSet[T.Tuple[str, str]]]

class CMakeExecutor:
    # The class's copy of the CMake path. Avoids having to search for it
    # multiple times in the same Meson invocation.
    class_cmakebin = PerMachine(None, None)   # type: PerMachine[T.Optional[ExternalProgram]]
    class_cmakevers = PerMachine(None, None)  # type: PerMachine[T.Optional[str]]
    class_cmake_cache = {}  # type: T.Dict[T.Any, TYPE_result]

    def __init__(self, environment: 'Environment', version: str, for_machine: MachineChoice, silent: bool = False):
        self.min_version = version
        self.environment = environment
        self.for_machine = for_machine
        self.cmakebin, self.cmakevers = self.find_cmake_binary(self.environment, silent=silent)
        self.always_capture_stderr = True
        self.print_cmout = False
        self.prefix_paths = []      # type: T.List[str]
        self.extra_cmake_args = []  # type: T.List[str]

        if self.cmakebin is None:
            return

        if not version_compare(self.cmakevers, self.min_version):
            mlog.warning(
                'The version of CMake', mlog.bold(self.cmakebin.get_path()),
                'is', mlog.bold(self.cmakevers), 'but version', mlog.bold(self.min_version),
                'is required')
            self.cmakebin = None
            return

        self.prefix_paths = self.environment.coredata.options[OptionKey('cmake_prefix_path', machine=self.for_machine)].value
        if self.prefix_paths:
            self.extra_cmake_args += ['-DCMAKE_PREFIX_PATH={}'.format(';'.join(self.prefix_paths))]

    def find_cmake_binary(self, environment: 'Environment', silent: bool = False) -> T.Tuple[T.Optional['ExternalProgram'], T.Optional[str]]:
        # Only search for CMake the first time and store the result in the class
        # definition
        if isinstance(CMakeExecutor.class_cmakebin[self.for_machine], NonExistingExternalProgram):
            mlog.debug(f'CMake binary for {self.for_machine} is cached as not found')
            return None, None
        elif CMakeExecutor.class_cmakebin[self.for_machine] is not None:
            mlog.debug(f'CMake binary for {self.for_machine} is cached.')
        else:
            assert CMakeExecutor.class_cmakebin[self.for_machine] is None

            mlog.debug(f'CMake binary for {self.for_machine} is not cached')
            for potential_cmakebin in find_external_program(
                    environment, self.for_machine, 'cmake', 'CMake',
                    environment.default_cmake, allow_default_for_cross=False):
                version_if_ok = self.check_cmake(potential_cmakebin)
                if not version_if_ok:
                    continue
                if not silent:
                    mlog.log('Found CMake:', mlog.bold(potential_cmakebin.get_path()),
                             f'({version_if_ok})')
                CMakeExecutor.class_cmakebin[self.for_machine] = potential_cmakebin
                CMakeExecutor.class_cmakevers[self.for_machine] = version_if_ok
                break
            else:
                if not silent:
                    mlog.log('Found CMake:', mlog.red('NO'))
                # Set to False instead of None to signify that we've already
                # searched for it and not found it
                CMakeExecutor.class_cmakebin[self.for_machine] = NonExistingExternalProgram()
                CMakeExecutor.class_cmakevers[self.for_machine] = None
                return None, None

        return CMakeExecutor.class_cmakebin[self.for_machine], CMakeExecutor.class_cmakevers[self.for_machine]

    def check_cmake(self, cmakebin: 'ExternalProgram') -> T.Optional[str]:
        if not cmakebin.found():
            mlog.log(f'Did not find CMake {cmakebin.name!r}')
            return None
        try:
            p, out = Popen_safe(cmakebin.get_command() + ['--version'])[0:2]
            if p.returncode != 0:
                mlog.warning('Found CMake {!r} but couldn\'t run it'
                             ''.format(' '.join(cmakebin.get_command())))
                return None
        except FileNotFoundError:
            mlog.warning('We thought we found CMake {!r} but now it\'s not there. How odd!'
                         ''.format(' '.join(cmakebin.get_command())))
            return None
        except PermissionError:
            msg = 'Found CMake {!r} but didn\'t have permissions to run it.'.format(' '.join(cmakebin.get_command()))
            if not is_windows():
                msg += '\n\nOn Unix-like systems this is often caused by scripts that are not executable.'
            mlog.warning(msg)
            return None
        cmvers = re.search(r'(cmake|cmake3)\s*version\s*([\d.]+)', out).group(2)
        return cmvers

    def set_exec_mode(self, print_cmout: T.Optional[bool] = None, always_capture_stderr: T.Optional[bool] = None) -> None:
        if print_cmout is not None:
            self.print_cmout = print_cmout
        if always_capture_stderr is not None:
            self.always_capture_stderr = always_capture_stderr

    def _cache_key(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_cache_key:
        fenv = frozenset(env.items()) if env is not None else frozenset()
        targs = tuple(args)
        return (self.cmakebin.get_path(), targs, build_dir.as_posix(), fenv)

    def _call_cmout_stderr(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_result:
        cmd = self.cmakebin.get_command() + args
        proc = S.Popen(cmd, stdout=S.PIPE, stderr=S.PIPE, cwd=str(build_dir), env=env)  # TODO [PYTHON_37]: drop Path conversion

        # stdout and stderr MUST be read at the same time to avoid pipe
        # blocking issues. The easiest way to do this is with a separate
        # thread for one of the pipes.
        def print_stdout() -> None:
            while True:
                line = proc.stdout.readline()
                if not line:
                    break
                mlog.log(line.decode(errors='ignore').strip('\n'))
            proc.stdout.close()

        t = Thread(target=print_stdout)
        t.start()

        try:
            # Read stderr line by line and log non trace lines
            raw_trace = ''
            tline_start_reg = re.compile(r'^\s*(.*\.(cmake|txt))\(([0-9]+)\):\s*(\w+)\(.*$')
            inside_multiline_trace = False
            while True:
                line_raw = proc.stderr.readline()
                if not line_raw:
                    break
                line = line_raw.decode(errors='ignore')
                if tline_start_reg.match(line):
                    raw_trace += line
                    inside_multiline_trace = not line.endswith(' )\n')
                elif inside_multiline_trace:
                    raw_trace += line
                else:
                    mlog.warning(line.strip('\n'))

        finally:
            proc.stderr.close()
            t.join()
            proc.wait()

        return proc.returncode, None, raw_trace

    def _call_cmout(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_result:
        cmd = self.cmakebin.get_command() + args
        proc = S.Popen(cmd, stdout=S.PIPE, stderr=S.STDOUT, cwd=str(build_dir), env=env)  # TODO [PYTHON_37]: drop Path conversion
        while True:
            line = proc.stdout.readline()
            if not line:
                break
            mlog.log(line.decode(errors='ignore').strip('\n'))
        proc.stdout.close()
        proc.wait()
        return proc.returncode, None, None

    def _call_quiet(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_result:
        build_dir.mkdir(parents=True, exist_ok=True)
        cmd = self.cmakebin.get_command() + args
        ret = S.run(cmd, env=env, cwd=str(build_dir), close_fds=False,
                    stdout=S.PIPE, stderr=S.PIPE, universal_newlines=False)   # TODO [PYTHON_37]: drop Path conversion
        rc = ret.returncode
        out = ret.stdout.decode(errors='ignore')
        err = ret.stderr.decode(errors='ignore')
        return rc, out, err

    def _call_impl(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_result:
        mlog.debug(f'Calling CMake ({self.cmakebin.get_command()}) in {build_dir} with:')
        for i in args:
            mlog.debug(f'  - "{i}"')
        if not self.print_cmout:
            return self._call_quiet(args, build_dir, env)
        else:
            if self.always_capture_stderr:
                return self._call_cmout_stderr(args, build_dir, env)
            else:
                return self._call_cmout(args, build_dir, env)

    def call(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]] = None, disable_cache: bool = False) -> TYPE_result:
        if env is None:
            env = os.environ.copy()

        args = args + self.extra_cmake_args
        if disable_cache:
            return self._call_impl(args, build_dir, env)

        # First check if cached, if not call the real cmake function
        cache = CMakeExecutor.class_cmake_cache
        key = self._cache_key(args, build_dir, env)
        if key not in cache:
            cache[key] = self._call_impl(args, build_dir, env)
        return cache[key]

    def found(self) -> bool:
        return self.cmakebin is not None

    def version(self) -> str:
        return self.cmakevers

    def executable_path(self) -> str:
        return self.cmakebin.get_path()

    def get_command(self) -> T.List[str]:
        return self.cmakebin.get_command()

    def get_cmake_prefix_paths(self) -> T.List[str]:
        return self.prefix_paths

    def machine_choice(self) -> MachineChoice:
        return self.for_machine
<|endoftext|>


<commit_before>import MiniBotFramework.Actuation.Actuator
import MiniBotFramework.Actuation.GpioMotor
<commit_msg>update initpy<commit_after>import MiniBotFramework.Actuation.Actuator
import MiniBotFramework.Actuation.GpioMotor
import MiniBotFramework.Actuation.TwoWheelMovement
<|endoftext|>


<commit_before># This software and any associated files are copyright 2010 Brian Carver and
# Michael Lissner.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from djangosphinx.models import SphinxSearch
from django.db import models
import alert

# a tuple, which we'll pass to the choices argument in various places
PACER_CODES = (
    ('ca1',  'Court of Appeals for the First Circuit'),
    ('ca2',  'Court of Appeals for the Second Circuit'),
    ('ca3',  'Court of Appeals for the Third Circuit'),
    ('ca4',  'Court of Appeals for the Fourth Circuit'),
    ('ca5',  'Court of Appeals for the Fifth Circuit'),
    ('ca6',  'Court of Appeals for the Sixth Circuit'),
    ('ca7',  'Court of Appeals for the Seventh Circuit'),
    ('ca8',  'Court of Appeals for the Eighth Circuit'),
    ('ca9',  'Court of Appeals for the Ninth Circuit'),
    ('ca10', 'Court of Appeals for the Tenth Circuit'),
    ('ca11', 'Court of Appeals for the Eleventh Circuit'),
    ('cadc', 'Court of Appeals for the D.C. Circuit'),
    ('cafc', 'Court of Appeals for the Federal Circuit'),
    ('scotus', 'Supreme Court of the United States'),
)

# changes here need to be mirrored in the coverage page view and the exceptions
# list for sphinx
DOCUMENT_STATUSES = (
    ('P', 'Published/Precedential'),
    ('U', 'Unpublished/Non-Precedential'),
    ('E', 'Errata'),
    ('I', 'In-chambers'),
    ('R', 'Relating-to orders'),
)

DOCUMENT_SOURCES = (
    ('C', 'court website'),
    ('R', 'bulk.resource.org'),
    ('M', 'manual input'),
)

# A class to represent some information about each court, can be extended as needed.
class Court(models.Model):
    courtUUID = models.CharField("a unique ID for each court",
        max_length=100,
        primary_key=True,
        choices=PACER_CODES)
    courtURL = models.URLField("the homepage for each court")
    courtShortName = models.CharField("the shortname for the court",
        max_length=100,
        blank=True)

    # uses the choices argument in courtUUID to create a good display of the object.
    def __unicode__(self):
        return self.get_courtUUID_display()

    class Meta:
        db_table = "Court"
        ordering = ["courtUUID"] #this reinforces the default

# A class to represent each party that is extracted from a document
class Party(models.Model):
    partyUUID = models.AutoField("a unique ID for each party", primary_key=True)
    partyExtracted = models.CharField("a party name", max_length=100)

    def __unicode__(self):
        return self.partyExtracted

    class Meta:
        verbose_name_plural = "parties"
        db_table = "Party"
        ordering = ["partyExtracted"]

# A class to represent each judge that is extracted from a document
class Judge(models.Model):
    judgeUUID = models.AutoField("a unique ID for each judge", primary_key=True)
    court = models.ForeignKey(Court, verbose_name="the court where the judge served during this time period")
    canonicalName = models.CharField("the official name of the judge: fname, mname, lname",
        max_length=150)
    judgeAvatar = models.ImageField("the judge's face",
        upload_to="avatars/judges/%Y/%m/%d",
        blank=True)
    startDate = models.DateField("the start date that the judge is on the bench")
    endDate = models.DateField("the end date that the judge is on the bench")

    def __unicode__(self):
        return self.canonicalName

    class Meta:
        db_table = "Judge"
        ordering = ["court", "canonicalName"]

# A class to hold the various aliases that a judge may have, such as M. Lissner,
# Michael Jay Lissner, Michael Lissner, etc.
class JudgeAlias (models.Model):
    aliasUUID = models.AutoField("a unique ID for each alias", primary_key=True)
    judgeUUID = models.ForeignKey(Judge, verbose_name="the judge for whom we are assigning an alias")
    alias = models.CharField("a name under which the judge appears in a document", max_length=100)

    # should return something like 'Mike is mapped to Michael Lissner'
    def __unicode__(self):
        return u'%s is mapped to %s' % (self.alias, self.judgeUUID.canonicalName)

    class Meta:
        verbose_name = "judge alias"
        verbose_name_plural = "judge aliases"
        db_table = "JudgeAlias"
        ordering = ["alias"]

class Citation(models.Model):
    search = SphinxSearch()
    citationUUID = models.AutoField("a unique ID for each citation",
        primary_key=True)
    caseNameShort = models.CharField("short name, as it is usually found on the court website",
        max_length=100,
        blank=True,
        db_index=True)
    caseNameFull =  models.TextField("full name of the case, as found on the first page of the PDF",
        blank=True)
    caseNumber = models.CharField("the case number",
        blank=True,
        max_length=50,
        db_index=True)
    officialCitationWest = models.CharField("the citation number, as described by WestLaw",
        max_length=50,
        blank=True)
    officialCitationLexis = models.CharField("the citation number, as described by LexisNexis",
        max_length=50,
        blank=True)

    def __unicode__(self):
        if self.caseNameShort:
            return self.caseNameShort
        else:
            return self.citationUUID

    class Meta:
        db_table = "Citation"
        ordering = ["caseNameFull"]

class ExcerptSummary(models.Model):
    excerptUUID = models.AutoField("a unique ID for each excerpt",
        primary_key=True)
    autoExcerpt = models.TextField("the first 100 words of the PDF file",
        blank=True)
    courtSummary = models.TextField("a summary of the document, as provided by the court itself",
        blank=True)

    def __unicode__(self):
        return self.excerptUUID

    class Meta:
        verbose_name = "excerpt summary"
        verbose_name_plural = "excerpt summaries"
        db_table = "ExcerptSummary"

# A class which holds the bulk of the information regarding documents. This must
# go last, since it references the above classes
class Document(models.Model):
    search = SphinxSearch()
    documentUUID = models.AutoField("a unique ID for each document",
        primary_key=True)
    source = models.CharField("the source of the document",
        max_length=3,
        choices=DOCUMENT_SOURCES,
        blank=True)
    documentSHA1 = models.CharField("unique ID for the document, as generated via sha1 on the PDF",
        max_length=40,
        db_index=True)
    dateFiled = models.DateField("the date filed by the court",
        blank=True,
        null=True,
        db_index=True)
    court = models.ForeignKey(Court,
        verbose_name="the court where the document was filed",
        db_index=True)
    judge = models.ManyToManyField(Judge,
        verbose_name="the judges that heard the case",
        blank=True,
        null=True)
    party = models.ManyToManyField(Party,
        verbose_name="the parties that were in the case",
        blank=True,
        null=True)
    citation = models.ForeignKey(Citation,
        verbose_name="the citation information for the document",
        blank=True,
        null=True)
    excerptSummary = models.ForeignKey(ExcerptSummary,
        verbose_name="the excerpt information for the document",
        blank=True,
        null=True)
    download_URL = models.URLField("the URL on the court website where the document was originally scraped",
        verify_exists=False)
    time_retrieved = models.DateTimeField("the exact date and time stamp that the document was placed into our database",
        auto_now_add=True,
        editable=False)
    local_path = models.FileField("the location, relative to MEDIA_ROOT, where the files are stored",
        upload_to='pdf/%Y/%m/%d',
        blank=True)
    documentPlainText = models.TextField("plain text of the document after extraction from the PDF",
        blank=True)
    documentHTML = models.TextField("HTML of the document",
        blank=True)
    documentType = models.CharField("the type of document, as described by document_types.txt",
        max_length=50,
        blank=True,
        choices=DOCUMENT_STATUSES)

    def __unicode__(self):
        try:
            return self.citation.caseNameShort
        except:
            return documentSHA1

    @models.permalink
    def get_absolute_url(self):
        try:
            return ('viewCases', [str(self.court.courtUUID),
                str(self.citation.caseNameShort).replace('-', '_').replace(' ', '-')])
        except:
            return ('viewCases', [str(self.court.courtUUID),
                str(self.documentSHA1)])

    class Meta:
        db_table = "Document"
        ordering = ["-time_retrieved"]
<commit_msg>monkeying with the model.<commit_after># This software and any associated files are copyright 2010 Brian Carver and
# Michael Lissner.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from djangosphinx.models import SphinxSearch
from django.db import models
import alert

# a tuple, which we'll pass to the choices argument in various places
PACER_CODES = (
    ('ca1',  'Court of Appeals for the First Circuit'),
    ('ca2',  'Court of Appeals for the Second Circuit'),
    ('ca3',  'Court of Appeals for the Third Circuit'),
    ('ca4',  'Court of Appeals for the Fourth Circuit'),
    ('ca5',  'Court of Appeals for the Fifth Circuit'),
    ('ca6',  'Court of Appeals for the Sixth Circuit'),
    ('ca7',  'Court of Appeals for the Seventh Circuit'),
    ('ca8',  'Court of Appeals for the Eighth Circuit'),
    ('ca9',  'Court of Appeals for the Ninth Circuit'),
    ('ca10', 'Court of Appeals for the Tenth Circuit'),
    ('ca11', 'Court of Appeals for the Eleventh Circuit'),
    ('cadc', 'Court of Appeals for the D.C. Circuit'),
    ('cafc', 'Court of Appeals for the Federal Circuit'),
    ('scotus', 'Supreme Court of the United States'),
)

# changes here need to be mirrored in the coverage page view and the exceptions
# list for sphinx
DOCUMENT_STATUSES = (
    ('P', 'Published/Precedential'),
    ('U', 'Unpublished/Non-Precedential'),
    ('E', 'Errata'),
    ('I', 'In-chambers'),
    ('R', 'Relating-to orders'),
)

DOCUMENT_SOURCES = (
    ('C', 'court website'),
    ('R', 'bulk.resource.org'),
    ('M', 'manual input'),
)

# A class to represent some information about each court, can be extended as needed.
class Court(models.Model):
    courtUUID = models.CharField("a unique ID for each court",
        max_length=100,
        primary_key=True,
        choices=PACER_CODES)
    courtURL = models.URLField("the homepage for each court")
    courtShortName = models.CharField("the shortname for the court",
        max_length=100,
        blank=True)

    # uses the choices argument in courtUUID to create a good display of the object.
    def __unicode__(self):
        return self.get_courtUUID_display()

    class Meta:
        db_table = "Court"
        ordering = ["courtUUID"] #this reinforces the default

# A class to represent each party that is extracted from a document
class Party(models.Model):
    partyUUID = models.AutoField("a unique ID for each party", primary_key=True)
    partyExtracted = models.CharField("a party name", max_length=100)

    def __unicode__(self):
        return self.partyExtracted

    class Meta:
        verbose_name_plural = "parties"
        db_table = "Party"
        ordering = ["partyExtracted"]

# A class to represent each judge that is extracted from a document
class Judge(models.Model):
    judgeUUID = models.AutoField("a unique ID for each judge", primary_key=True)
    court = models.ForeignKey(Court, verbose_name="the court where the judge served during this time period")
    canonicalName = models.CharField("the official name of the judge: fname, mname, lname",
        max_length=150)
    judgeAvatar = models.ImageField("the judge's face",
        upload_to="avatars/judges/%Y/%m/%d",
        blank=True)
    startDate = models.DateField("the start date that the judge is on the bench")
    endDate = models.DateField("the end date that the judge is on the bench")

    def __unicode__(self):
        return self.canonicalName

    class Meta:
        db_table = "Judge"
        ordering = ["court", "canonicalName"]

# A class to hold the various aliases that a judge may have, such as M. Lissner,
# Michael Jay Lissner, Michael Lissner, etc.
class JudgeAlias (models.Model):
    aliasUUID = models.AutoField("a unique ID for each alias", primary_key=True)
    judgeUUID = models.ForeignKey(Judge, verbose_name="the judge for whom we are assigning an alias")
    alias = models.CharField("a name under which the judge appears in a document", max_length=100)

    # should return something like 'Mike is mapped to Michael Lissner'
    def __unicode__(self):
        return u'%s is mapped to %s' % (self.alias, self.judgeUUID.canonicalName)

    class Meta:
        verbose_name = "judge alias"
        verbose_name_plural = "judge aliases"
        db_table = "JudgeAlias"
        ordering = ["alias"]

class Citation(models.Model):
    search = SphinxSearch()
    citationUUID = models.AutoField("a unique ID for each citation",
        primary_key=True)
    caseNameShort = models.CharField("short name, as it is usually found on the court website",
        max_length=100,
        blank=True,
        db_index=True)
    caseNameFull =  models.TextField("full name of the case, as found on the first page of the PDF",
        blank=True)
    caseNumber = models.CharField("the case number",
        blank=True,
        max_length=50,
        db_index=True)
    officialCitationWest = models.CharField("the citation number, as described by WestLaw",
        max_length=50,
        blank=True)
    officialCitationLexis = models.CharField("the citation number, as described by LexisNexis",
        max_length=50,
        blank=True)

    def __unicode__(self):
        if self.caseNameShort:
            return self.caseNameShort
        else:
            return self.citationUUID

    class Meta:
        db_table = "Citation"
        ordering = ["caseNameFull"]

class ExcerptSummary(models.Model):
    excerptUUID = models.AutoField("a unique ID for each excerpt",
        primary_key=True)
    autoExcerpt = models.TextField("the first 100 words of the PDF file",
        blank=True)
    courtSummary = models.TextField("a summary of the document, as provided by the court itself",
        blank=True)

    def __unicode__(self):
        return self.excerptUUID

    class Meta:
        verbose_name = "excerpt summary"
        verbose_name_plural = "excerpt summaries"
        db_table = "ExcerptSummary"

# A class which holds the bulk of the information regarding documents. This must
# go last, since it references the above classes
class Document(models.Model):
    search = SphinxSearch()
    documentUUID = models.AutoField("a unique ID for each document",
        primary_key=True)
    source = models.CharField("the source of the document",
        max_length=3,
        choices=DOCUMENT_SOURCES,
        blank=True)
    documentSHA1 = models.CharField("unique ID for the document, as generated via sha1 on the PDF",
        max_length=40,
        db_index=True)
    dateFiled = models.DateField("the date filed by the court",
        blank=True,
        null=True,
        db_index=True)
    court = models.ForeignKey(Court,
        verbose_name="the court where the document was filed",
        db_index=True)
    judge = models.ManyToManyField(Judge,
        verbose_name="the judges that heard the case",
        blank=True,
        null=True)
    party = models.ManyToManyField(Party,
        verbose_name="the parties that were in the case",
        blank=True,
        null=True)
    citation = models.ForeignKey(Citation,
        verbose_name="the citation information for the document",
        blank=True,
        null=True)
    excerptSummary = models.ForeignKey(ExcerptSummary,
        verbose_name="the excerpt information for the document",
        blank=True,
        null=True)
    download_URL = models.URLField("the URL on the court website where the document was originally scraped",
        verify_exists=False)
    time_retrieved = models.DateTimeField("the exact date and time stamp that the document was placed into our database",
        auto_now_add=True,
        editable=False)
    local_path = models.FileField("the location, relative to MEDIA_ROOT, where the files are stored",
        upload_to='pdf/%Y/%m/%d',
        blank=True)
    documentPlainText = models.TextField("plain text of the document after extraction from the PDF",
        blank=True)
    documentHTML = models.TextField("HTML of the document",
        blank=True)
    documentType = models.CharField("the type of document, as described by document_types.txt",
        max_length=50,
        blank=True,
        choices=DOCUMENT_STATUSES)

    def __unicode__(self):
        try:
            tempvar = self.citation.caseNameShort
            return tempvar
        except:
            return documentSHA1

    @models.permalink
    def get_absolute_url(self):
        try:
            return ('viewCases', [str(self.court.courtUUID),
                str(self.citation.caseNameShort).replace('-', '_').replace(' ', '-')])
        except:
            return ('viewCases', [str(self.court.courtUUID),
                str(self.documentSHA1)])

    class Meta:
        db_table = "Document"
        ordering = ["-time_retrieved"]
<|endoftext|>


<commit_before>#!/usr/bin/python3

"""
Sample program that uses a generated GRIP pipeline to detect red areas in an image and publish them to NetworkTables.
"""

import cv2
from ntcore import NetworkTables
from grip import GripPipeline

def extra_processing(pipeline):
    """
    Performs extra processing on the pipeline's outputs and publishes data to NetworkTables.
    :param pipeline: the pipeline that just processed an image
    :return: None
    """
    center_x_positions = []
    center_y_positions = []
    widths = []
    heights = []

    # Find the bounding boxes of the contours to get x, y, width, and height
    for contour in pipeline.filter_contours_output:
        x, y, w, h = cv2.boundingRect(contour)
        center_x_positions.append(x + w / 2)  # X and Y are coordinates of the top-left corner of the bounding box
        center_y_positions.append(y + h / w)
        widths.append(w)
        heights.append(y)

    # Publish to the '/vision/red_areas' network table
    table = NetworkTables.getTable('/vision/red_areas')
    table.putNumberArray('x', center_x_positions)
    table.putNumberArray('y', center_y_positions)
    table.putNumberArray('width', widths)
    table.putNumberArray('height', heights)

def main():
    print('Initializing NetworkTables')
    NetworkTables.setClientMode()
    NetworkTables.setIPAddress('localhost')
    NetworkTables.initialize()

    print('Creating video capture')
    cap = cv2.VideoCapture(0)

    print('Creating pipeline')
    pipeline = GripPipeline()

    print('Running pipeline')
    while cap.isOpened():
        have_frame, frame = cap.read()
        if have_frame:
            pipeline.process(frame)
            extra_processing(pipeline)

    print('Capture closed')

if __name__ == '__main__':
    main()
<commit_msg>Fix networktables import (#6)<commit_after>#!/usr/bin/python3

"""
Sample program that uses a generated GRIP pipeline to detect red areas in an image and publish them to NetworkTables.
"""

import cv2
from networktables import NetworkTables
from grip import GripPipeline

def extra_processing(pipeline):
    """
    Performs extra processing on the pipeline's outputs and publishes data to NetworkTables.
    :param pipeline: the pipeline that just processed an image
    :return: None
    """
    center_x_positions = []
    center_y_positions = []
    widths = []
    heights = []

    # Find the bounding boxes of the contours to get x, y, width, and height
    for contour in pipeline.filter_contours_output:
        x, y, w, h = cv2.boundingRect(contour)
        center_x_positions.append(x + w / 2)  # X and Y are coordinates of the top-left corner of the bounding box
        center_y_positions.append(y + h / w)
        widths.append(w)
        heights.append(y)

    # Publish to the '/vision/red_areas' network table
    table = NetworkTables.getTable('/vision/red_areas')
    table.putNumberArray('x', center_x_positions)
    table.putNumberArray('y', center_y_positions)
    table.putNumberArray('width', widths)
    table.putNumberArray('height', heights)

def main():
    print('Initializing NetworkTables')
    NetworkTables.setClientMode()
    NetworkTables.setIPAddress('localhost')
    NetworkTables.initialize()

    print('Creating video capture')
    cap = cv2.VideoCapture(0)

    print('Creating pipeline')
    pipeline = GripPipeline()

    print('Running pipeline')
    while cap.isOpened():
        have_frame, frame = cap.read()
        if have_frame:
            pipeline.process(frame)
            extra_processing(pipeline)

    print('Capture closed')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run specific test on specific environment."""

import json
import logging
import os
import re
import shutil
import string
import tempfile
import time
import zipfile

from devil.utils import zip_utils
from pylib.base import base_test_result
from pylib.base import test_run
from pylib.remote.device import appurify_constants
from pylib.remote.device import appurify_sanitized
from pylib.remote.device import remote_device_helper

_DEVICE_OFFLINE_RE = re.compile('error: device not found')
_LONG_MSG_RE = re.compile('longMsg=')
_SHORT_MSG_RE = re.compile('shortMsg=')

class RemoteDeviceTestRun(test_run.TestRun):
  """Run tests on a remote device."""

  _TEST_RUN_KEY = 'test_run'
  _TEST_RUN_ID_KEY = 'test_run_id'

  WAIT_TIME = 5
  COMPLETE = 'complete'
  HEARTBEAT_INTERVAL = 300

  def __init__(self, env, test_instance):
    """Constructor.

    Args:
      env: Environment the tests will run in.
      test_instance: The test that will be run.
    """
    super(RemoteDeviceTestRun, self).__init__(env, test_instance)
    self._env = env
    self._test_instance = test_instance
    self._app_id = ''
    self._test_id = ''
    self._results = ''
    self._test_run_id = ''
    self._results_temp_dir = None

  #override
  def SetUp(self):
    """Set up a test run."""
    if self._env.trigger:
      self._TriggerSetUp()
    elif self._env.collect:
      assert isinstance(self._env.collect, basestring), (
                        'File for storing test_run_id must be a string.')
      with open(self._env.collect, 'r') as persisted_data_file:
        persisted_data = json.loads(persisted_data_file.read())
        self._env.LoadFrom(persisted_data)
        self.LoadFrom(persisted_data)

  def _TriggerSetUp(self):
    """Set up the triggering of a test run."""
    raise NotImplementedError

  #override
  def RunTests(self):
    """Run the test."""
    if self._env.trigger:
      with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                              logging.WARNING):
        test_start_res = appurify_sanitized.api.tests_run(
            self._env.token, self._env.device_type_id, self._app_id,
            self._test_id)
      remote_device_helper.TestHttpResponse(
        test_start_res, 'Unable to run test.')
      self._test_run_id = test_start_res.json()['response']['test_run_id']
      logging.info('Test run id: %s', self._test_run_id)

    if self._env.collect:
      current_status = ''
      timeout_counter = 0
      heartbeat_counter = 0
      while self._GetTestStatus(self._test_run_id) != self.COMPLETE:
        if self._results['detailed_status'] != current_status:
          logging.info('Test status: %s', self._results['detailed_status'])
          current_status = self._results['detailed_status']
          timeout_counter = 0
          heartbeat_counter = 0
        if heartbeat_counter > self.HEARTBEAT_INTERVAL:
          logging.info('Test status: %s', self._results['detailed_status'])
          heartbeat_counter = 0

        timeout = self._env.timeouts.get(
            current_status, self._env.timeouts['unknown'])
        if timeout_counter > timeout:
          raise remote_device_helper.RemoteDeviceError(
              'Timeout while in %s state for %s seconds'
              % (current_status, timeout),
              is_infra_error=True)
        time.sleep(self.WAIT_TIME)
        timeout_counter += self.WAIT_TIME
        heartbeat_counter += self.WAIT_TIME
      self._DownloadTestResults(self._env.results_path)

      if self._results['results']['exception']:
        raise remote_device_helper.RemoteDeviceError(
            self._results['results']['exception'], is_infra_error=True)

      return self._ParseTestResults()

  #override
  def TearDown(self):
    """Tear down the test run."""
    if self._env.collect:
      self._CollectTearDown()
    elif self._env.trigger:
      assert isinstance(self._env.trigger, basestring), (
                        'File for storing test_run_id must be a string.')
      with open(self._env.trigger, 'w') as persisted_data_file:
        persisted_data = {}
        self.DumpTo(persisted_data)
        self._env.DumpTo(persisted_data)
        persisted_data_file.write(json.dumps(persisted_data))

  def _CollectTearDown(self):
    if self._GetTestStatus(self._test_run_id) != self.COMPLETE:
      with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                              logging.WARNING):
        test_abort_res = appurify_sanitized.api.tests_abort(
            self._env.token, self._test_run_id, reason='Test runner exiting.')
      remote_device_helper.TestHttpResponse(test_abort_res,
                                            'Unable to abort test.')
    if self._results_temp_dir:
      shutil.rmtree(self._results_temp_dir)

  def __enter__(self):
    """Set up the test run when used as a context manager."""
    self.SetUp()
    return self

  def __exit__(self, exc_type, exc_val, exc_tb):
    """Tear down the test run when used as a context manager."""
    self.TearDown()

  def DumpTo(self, persisted_data):
    test_run_data = {
      self._TEST_RUN_ID_KEY: self._test_run_id,
    }
    persisted_data[self._TEST_RUN_KEY] = test_run_data

  def LoadFrom(self, persisted_data):
    test_run_data = persisted_data[self._TEST_RUN_KEY]
    self._test_run_id = test_run_data[self._TEST_RUN_ID_KEY]

  def _ParseTestResults(self):
    raise NotImplementedError

  def _GetTestByName(self, test_name):
    """Gets test_id for specific test.

    Args:
      test_name: Test to find the ID of.
    """
    with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                            logging.WARNING):
      test_list_res = appurify_sanitized.api.tests_list(self._env.token)
    remote_device_helper.TestHttpResponse(test_list_res,
                                          'Unable to get tests list.')
    for test in test_list_res.json()['response']:
      if test['test_type'] == test_name:
        return test['test_id']
    raise remote_device_helper.RemoteDeviceError(
        'No test found with name %s' % (test_name))

  def _DownloadTestResults(self, results_path):
    """Download the test results from remote device service.

    Downloads results in temporary location, and then copys results
    to results_path if results_path is not set to None.

    Args:
      results_path: Path to download appurify results zipfile.

    Returns:
      Path to downloaded file.
    """

    if self._results_temp_dir is None:
      self._results_temp_dir = tempfile.mkdtemp()
      logging.info('Downloading results to %s.', self._results_temp_dir)
      with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                              logging.WARNING):
        appurify_sanitized.utils.wget(self._results['results']['url'],
                                      self._results_temp_dir + '/results')
    if results_path:
      logging.info('Copying results to %s', results_path)
      if not os.path.exists(os.path.dirname(results_path)):
        os.makedirs(os.path.dirname(results_path))
      shutil.copy(self._results_temp_dir + '/results', results_path)
    return self._results_temp_dir + '/results'

  def _GetTestStatus(self, test_run_id):
    """Checks the state of the test, and sets self._results

    Args:
      test_run_id: Id of test on on remote service.
    """

    with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                            logging.WARNING):
      test_check_res = appurify_sanitized.api.tests_check_result(
          self._env.token, test_run_id)
    remote_device_helper.TestHttpResponse(test_check_res,
                                          'Unable to get test status.')
    self._results = test_check_res.json()['response']
    return self._results['status']

  def _AmInstrumentTestSetup(self, app_path, test_path, runner_package,
                             environment_variables, extra_apks=None):
    config = {'runner': runner_package}
    if environment_variables:
      config['environment_vars'] = ','.join(
          '%s=%s' % (k, v) for k, v in environment_variables.iteritems())

    self._app_id = self._UploadAppToDevice(app_path)

    data_deps = self._test_instance.GetDataDependencies()
    if data_deps:
      with tempfile.NamedTemporaryFile(suffix='.zip') as test_with_deps:
        sdcard_files = []
        additional_apks = []
        host_test = os.path.basename(test_path)
        with zipfile.ZipFile(test_with_deps.name, 'w') as zip_file:
          zip_file.write(test_path, host_test, zipfile.ZIP_DEFLATED)
          for h, _ in data_deps:
            if os.path.isdir(h):
              zip_utils.WriteToZipFile(zip_file, h, '.')
              sdcard_files.extend(os.listdir(h))
            else:
              zip_utils.WriteToZipFile(zip_file, h, os.path.basename(h))
              sdcard_files.append(os.path.basename(h))
          for a in extra_apks or ():
            zip_utils.WriteToZipFile(zip_file, a, os.path.basename(a))
            additional_apks.append(os.path.basename(a))

        config['sdcard_files'] = ','.join(sdcard_files)
        config['host_test'] = host_test
        if additional_apks:
          config['additional_apks'] = ','.join(additional_apks)
        self._test_id = self._UploadTestToDevice(
            'robotium', test_with_deps.name, app_id=self._app_id)
    else:
      self._test_id = self._UploadTestToDevice('robotium', test_path)

    logging.info('Setting config: %s', config)
    appurify_configs = {}
    if self._env.network_config:
      appurify_configs['network'] = self._env.network_config
    self._SetTestConfig('robotium', config, **appurify_configs)

  def _UploadAppToDevice(self, app_path):
    """Upload app to device."""
    logging.info('Uploading %s to remote service as %s.', app_path,
                 self._test_instance.suite)
    with open(app_path, 'rb') as apk_src:
      with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                              logging.WARNING):
        upload_results = appurify_sanitized.api.apps_upload(
            self._env.token, apk_src, 'raw', name=self._test_instance.suite)
      remote_device_helper.TestHttpResponse(
          upload_results, 'Unable to upload %s.' % app_path)
      return upload_results.json()['response']['app_id']

  def _UploadTestToDevice(self, test_type, test_path, app_id=None):
    """Upload test to device
    Args:
      test_type: Type of test that is being uploaded. Ex. uirobot, gtest..
    """
    logging.info('Uploading %s to remote service.', test_path)
    with open(test_path, 'rb') as test_src:
      with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                              logging.WARNING):
        upload_results = appurify_sanitized.api.tests_upload(
            self._env.token, test_src, 'raw', test_type, app_id=app_id)
      remote_device_helper.TestHttpResponse(upload_results,
          'Unable to upload %s.' % test_path)
      return upload_results.json()['response']['test_id']

  def _SetTestConfig(self, runner_type, runner_configs,
                     network=appurify_constants.NETWORK.WIFI_1_BAR,
                     pcap=0, profiler=0, videocapture=0):
    """Generates and uploads config file for test.
    Args:
      runner_configs: Configs specific to the runner you are using.
      network: Config to specify the network environment the devices running
          the tests will be in.
      pcap: Option to set the recording the of network traffic from the device.
      profiler: Option to set the recording of CPU, memory, and network
          transfer usage in the tests.
      videocapture: Option to set video capture during the tests.

    """
    logging.info('Generating config file for test.')
    with tempfile.TemporaryFile() as config:
      config_data = [
          '[appurify]',
          'network=%s' % network,
          'pcap=%s' % pcap,
          'profiler=%s' % profiler,
          'videocapture=%s' % videocapture,
          '[%s]' % runner_type
      ]
      config_data.extend(
          '%s=%s' % (k, v) for k, v in runner_configs.iteritems())
      config.write(''.join('%s\n' % l for l in config_data))
      config.flush()
      config.seek(0)
      with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                              logging.WARNING):
        config_response = appurify_sanitized.api.config_upload(
            self._env.token, config, self._test_id)
      remote_device_helper.TestHttpResponse(
          config_response, 'Unable to upload test config.')

  def _LogLogcat(self, level=logging.CRITICAL):
    """Prints out logcat downloaded from remote service.
    Args:
      level: logging level to print at.

    Raises:
      KeyError: If appurify_results/logcat.txt file cannot be found in
                downloaded zip.
    """
    zip_file = self._DownloadTestResults(None)
    with zipfile.ZipFile(zip_file) as z:
      try:
        logcat = z.read('appurify_results/logcat.txt')
        printable_logcat = ''.join(c for c in logcat if c in string.printable)
        for line in printable_logcat.splitlines():
          logging.log(level, line)
      except KeyError:
        logging.error('No logcat found.')

  def _LogAdbTraceLog(self):
    zip_file = self._DownloadTestResults(None)
    with zipfile.ZipFile(zip_file) as z:
      adb_trace_log = z.read('adb_trace.log')
      for line in adb_trace_log.splitlines():
        logging.critical(line)

  def _DidDeviceGoOffline(self):
    zip_file = self._DownloadTestResults(None)
    with zipfile.ZipFile(zip_file) as z:
      adb_trace_log = z.read('adb_trace.log')
      if any(_DEVICE_OFFLINE_RE.search(l) for l in adb_trace_log.splitlines()):
        return True
    return False

  def _DetectPlatformErrors(self, results):
    if not self._results['results']['pass']:
      if any(_SHORT_MSG_RE.search(l)
          for l in self._results['results']['output'].splitlines()):
        self._LogLogcat()
        for line in self._results['results']['output'].splitlines():
          if _LONG_MSG_RE.search(line):
            results.AddResult(base_test_result.BaseTestResult(
                line.split('=')[1], base_test_result.ResultType.CRASH))
            break
        else:
          results.AddResult(base_test_result.BaseTestResult(
              'Unknown platform error detected.',
              base_test_result.ResultType.UNKNOWN))
      elif self._DidDeviceGoOffline():
        self._LogLogcat()
        self._LogAdbTraceLog()
        raise remote_device_helper.RemoteDeviceError(
            'Remote service unable to reach device.', is_infra_error=True)
      else:
        results.AddResult(base_test_result.BaseTestResult(
            'Remote Service detected error.',
            base_test_result.ResultType.UNKNOWN))
<commit_msg>[Android] Improve error messaging for amp failures.<commit_after># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run specific test on specific environment."""

import json
import logging
import os
import re
import shutil
import string
import tempfile
import time
import zipfile

from devil.utils import zip_utils
from pylib.base import base_test_result
from pylib.base import test_run
from pylib.remote.device import appurify_constants
from pylib.remote.device import appurify_sanitized
from pylib.remote.device import remote_device_helper

_DEVICE_OFFLINE_RE = re.compile('error: device not found')
_LONG_MSG_RE = re.compile('longMsg=(.*)$')
_SHORT_MSG_RE = re.compile('shortMsg=(.*)$')

class RemoteDeviceTestRun(test_run.TestRun):
  """Run tests on a remote device."""

  _TEST_RUN_KEY = 'test_run'
  _TEST_RUN_ID_KEY = 'test_run_id'

  WAIT_TIME = 5
  COMPLETE = 'complete'
  HEARTBEAT_INTERVAL = 300

  def __init__(self, env, test_instance):
    """Constructor.

    Args:
      env: Environment the tests will run in.
      test_instance: The test that will be run.
    """
    super(RemoteDeviceTestRun, self).__init__(env, test_instance)
    self._env = env
    self._test_instance = test_instance
    self._app_id = ''
    self._test_id = ''
    self._results = ''
    self._test_run_id = ''
    self._results_temp_dir = None

  #override
  def SetUp(self):
    """Set up a test run."""
    if self._env.trigger:
      self._TriggerSetUp()
    elif self._env.collect:
      assert isinstance(self._env.collect, basestring), (
                        'File for storing test_run_id must be a string.')
      with open(self._env.collect, 'r') as persisted_data_file:
        persisted_data = json.loads(persisted_data_file.read())
        self._env.LoadFrom(persisted_data)
        self.LoadFrom(persisted_data)

  def _TriggerSetUp(self):
    """Set up the triggering of a test run."""
    raise NotImplementedError

  #override
  def RunTests(self):
    """Run the test."""
    if self._env.trigger:
      with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                              logging.WARNING):
        test_start_res = appurify_sanitized.api.tests_run(
            self._env.token, self._env.device_type_id, self._app_id,
            self._test_id)
      remote_device_helper.TestHttpResponse(
        test_start_res, 'Unable to run test.')
      self._test_run_id = test_start_res.json()['response']['test_run_id']
      logging.info('Test run id: %s', self._test_run_id)

    if self._env.collect:
      current_status = ''
      timeout_counter = 0
      heartbeat_counter = 0
      while self._GetTestStatus(self._test_run_id) != self.COMPLETE:
        if self._results['detailed_status'] != current_status:
          logging.info('Test status: %s', self._results['detailed_status'])
          current_status = self._results['detailed_status']
          timeout_counter = 0
          heartbeat_counter = 0
        if heartbeat_counter > self.HEARTBEAT_INTERVAL:
          logging.info('Test status: %s', self._results['detailed_status'])
          heartbeat_counter = 0

        timeout = self._env.timeouts.get(
            current_status, self._env.timeouts['unknown'])
        if timeout_counter > timeout:
          raise remote_device_helper.RemoteDeviceError(
              'Timeout while in %s state for %s seconds'
              % (current_status, timeout),
              is_infra_error=True)
        time.sleep(self.WAIT_TIME)
        timeout_counter += self.WAIT_TIME
        heartbeat_counter += self.WAIT_TIME
      self._DownloadTestResults(self._env.results_path)

      if self._results['results']['exception']:
        raise remote_device_helper.RemoteDeviceError(
            self._results['results']['exception'], is_infra_error=True)

      return self._ParseTestResults()

  #override
  def TearDown(self):
    """Tear down the test run."""
    if self._env.collect:
      self._CollectTearDown()
    elif self._env.trigger:
      assert isinstance(self._env.trigger, basestring), (
                        'File for storing test_run_id must be a string.')
      with open(self._env.trigger, 'w') as persisted_data_file:
        persisted_data = {}
        self.DumpTo(persisted_data)
        self._env.DumpTo(persisted_data)
        persisted_data_file.write(json.dumps(persisted_data))

  def _CollectTearDown(self):
    if self._GetTestStatus(self._test_run_id) != self.COMPLETE:
      with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                              logging.WARNING):
        test_abort_res = appurify_sanitized.api.tests_abort(
            self._env.token, self._test_run_id, reason='Test runner exiting.')
      remote_device_helper.TestHttpResponse(test_abort_res,
                                            'Unable to abort test.')
    if self._results_temp_dir:
      shutil.rmtree(self._results_temp_dir)

  def __enter__(self):
    """Set up the test run when used as a context manager."""
    self.SetUp()
    return self

  def __exit__(self, exc_type, exc_val, exc_tb):
    """Tear down the test run when used as a context manager."""
    self.TearDown()

  def DumpTo(self, persisted_data):
    test_run_data = {
      self._TEST_RUN_ID_KEY: self._test_run_id,
    }
    persisted_data[self._TEST_RUN_KEY] = test_run_data

  def LoadFrom(self, persisted_data):
    test_run_data = persisted_data[self._TEST_RUN_KEY]
    self._test_run_id = test_run_data[self._TEST_RUN_ID_KEY]

  def _ParseTestResults(self):
    raise NotImplementedError

  def _GetTestByName(self, test_name):
    """Gets test_id for specific test.

    Args:
      test_name: Test to find the ID of.
    """
    with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                            logging.WARNING):
      test_list_res = appurify_sanitized.api.tests_list(self._env.token)
    remote_device_helper.TestHttpResponse(test_list_res,
                                          'Unable to get tests list.')
    for test in test_list_res.json()['response']:
      if test['test_type'] == test_name:
        return test['test_id']
    raise remote_device_helper.RemoteDeviceError(
        'No test found with name %s' % (test_name))

  def _DownloadTestResults(self, results_path):
    """Download the test results from remote device service.

    Downloads results in temporary location, and then copys results
    to results_path if results_path is not set to None.

    Args:
      results_path: Path to download appurify results zipfile.

    Returns:
      Path to downloaded file.
    """

    if self._results_temp_dir is None:
      self._results_temp_dir = tempfile.mkdtemp()
      logging.info('Downloading results to %s.', self._results_temp_dir)
      with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                              logging.WARNING):
        appurify_sanitized.utils.wget(self._results['results']['url'],
                                      self._results_temp_dir + '/results')
    if results_path:
      logging.info('Copying results to %s', results_path)
      if not os.path.exists(os.path.dirname(results_path)):
        os.makedirs(os.path.dirname(results_path))
      shutil.copy(self._results_temp_dir + '/results', results_path)
    return self._results_temp_dir + '/results'

  def _GetTestStatus(self, test_run_id):
    """Checks the state of the test, and sets self._results

    Args:
      test_run_id: Id of test on on remote service.
    """

    with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                            logging.WARNING):
      test_check_res = appurify_sanitized.api.tests_check_result(
          self._env.token, test_run_id)
    remote_device_helper.TestHttpResponse(test_check_res,
                                          'Unable to get test status.')
    self._results = test_check_res.json()['response']
    return self._results['status']

  def _AmInstrumentTestSetup(self, app_path, test_path, runner_package,
                             environment_variables, extra_apks=None):
    config = {'runner': runner_package}
    if environment_variables:
      config['environment_vars'] = ','.join(
          '%s=%s' % (k, v) for k, v in environment_variables.iteritems())

    self._app_id = self._UploadAppToDevice(app_path)

    data_deps = self._test_instance.GetDataDependencies()
    if data_deps:
      with tempfile.NamedTemporaryFile(suffix='.zip') as test_with_deps:
        sdcard_files = []
        additional_apks = []
        host_test = os.path.basename(test_path)
        with zipfile.ZipFile(test_with_deps.name, 'w') as zip_file:
          zip_file.write(test_path, host_test, zipfile.ZIP_DEFLATED)
          for h, _ in data_deps:
            if os.path.isdir(h):
              zip_utils.WriteToZipFile(zip_file, h, '.')
              sdcard_files.extend(os.listdir(h))
            else:
              zip_utils.WriteToZipFile(zip_file, h, os.path.basename(h))
              sdcard_files.append(os.path.basename(h))
          for a in extra_apks or ():
            zip_utils.WriteToZipFile(zip_file, a, os.path.basename(a))
            additional_apks.append(os.path.basename(a))

        config['sdcard_files'] = ','.join(sdcard_files)
        config['host_test'] = host_test
        if additional_apks:
          config['additional_apks'] = ','.join(additional_apks)
        self._test_id = self._UploadTestToDevice(
            'robotium', test_with_deps.name, app_id=self._app_id)
    else:
      self._test_id = self._UploadTestToDevice('robotium', test_path)

    logging.info('Setting config: %s', config)
    appurify_configs = {}
    if self._env.network_config:
      appurify_configs['network'] = self._env.network_config
    self._SetTestConfig('robotium', config, **appurify_configs)

  def _UploadAppToDevice(self, app_path):
    """Upload app to device."""
    logging.info('Uploading %s to remote service as %s.', app_path,
                 self._test_instance.suite)
    with open(app_path, 'rb') as apk_src:
      with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                              logging.WARNING):
        upload_results = appurify_sanitized.api.apps_upload(
            self._env.token, apk_src, 'raw', name=self._test_instance.suite)
      remote_device_helper.TestHttpResponse(
          upload_results, 'Unable to upload %s.' % app_path)
      return upload_results.json()['response']['app_id']

  def _UploadTestToDevice(self, test_type, test_path, app_id=None):
    """Upload test to device
    Args:
      test_type: Type of test that is being uploaded. Ex. uirobot, gtest..
    """
    logging.info('Uploading %s to remote service.', test_path)
    with open(test_path, 'rb') as test_src:
      with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                              logging.WARNING):
        upload_results = appurify_sanitized.api.tests_upload(
            self._env.token, test_src, 'raw', test_type, app_id=app_id)
      remote_device_helper.TestHttpResponse(upload_results,
          'Unable to upload %s.' % test_path)
      return upload_results.json()['response']['test_id']

  def _SetTestConfig(self, runner_type, runner_configs,
                     network=appurify_constants.NETWORK.WIFI_1_BAR,
                     pcap=0, profiler=0, videocapture=0):
    """Generates and uploads config file for test.
    Args:
      runner_configs: Configs specific to the runner you are using.
      network: Config to specify the network environment the devices running
          the tests will be in.
      pcap: Option to set the recording the of network traffic from the device.
      profiler: Option to set the recording of CPU, memory, and network
          transfer usage in the tests.
      videocapture: Option to set video capture during the tests.

    """
    logging.info('Generating config file for test.')
    with tempfile.TemporaryFile() as config:
      config_data = [
          '[appurify]',
          'network=%s' % network,
          'pcap=%s' % pcap,
          'profiler=%s' % profiler,
          'videocapture=%s' % videocapture,
          '[%s]' % runner_type
      ]
      config_data.extend(
          '%s=%s' % (k, v) for k, v in runner_configs.iteritems())
      config.write(''.join('%s\n' % l for l in config_data))
      config.flush()
      config.seek(0)
      with appurify_sanitized.SanitizeLogging(self._env.verbose_count,
                                              logging.WARNING):
        config_response = appurify_sanitized.api.config_upload(
            self._env.token, config, self._test_id)
      remote_device_helper.TestHttpResponse(
          config_response, 'Unable to upload test config.')

  def _LogLogcat(self, level=logging.CRITICAL):
    """Prints out logcat downloaded from remote service.
    Args:
      level: logging level to print at.

    Raises:
      KeyError: If appurify_results/logcat.txt file cannot be found in
                downloaded zip.
    """
    zip_file = self._DownloadTestResults(None)
    with zipfile.ZipFile(zip_file) as z:
      try:
        logcat = z.read('appurify_results/logcat.txt')
        printable_logcat = ''.join(c for c in logcat if c in string.printable)
        for line in printable_logcat.splitlines():
          logging.log(level, line)
      except KeyError:
        logging.error('No logcat found.')

  def _LogAdbTraceLog(self):
    zip_file = self._DownloadTestResults(None)
    with zipfile.ZipFile(zip_file) as z:
      adb_trace_log = z.read('adb_trace.log')
      for line in adb_trace_log.splitlines():
        logging.critical(line)

  def _DidDeviceGoOffline(self):
    zip_file = self._DownloadTestResults(None)
    with zipfile.ZipFile(zip_file) as z:
      adb_trace_log = z.read('adb_trace.log')
      if any(_DEVICE_OFFLINE_RE.search(l) for l in adb_trace_log.splitlines()):
        return True
    return False

  def _DetectPlatformErrors(self, results):
    if not self._results['results']['pass']:
      crash_msg = None
      for line in self._results['results']['output'].splitlines():
        m = _LONG_MSG_RE.search(line)
        if m:
          crash_msg = m.group(1)
          break
        m = _SHORT_MSG_RE.search(line)
        if m:
          crash_msg = m.group(1)
      if crash_msg:
        self._LogLogcat()
        results.AddResult(base_test_result.BaseTestResult(
            crash_msg, base_test_result.ResultType.CRASH))
      elif self._DidDeviceGoOffline():
        self._LogLogcat()
        self._LogAdbTraceLog()
        raise remote_device_helper.RemoteDeviceError(
            'Remote service unable to reach device.', is_infra_error=True)
      else:
        # Remote service is reporting a failure, but no failure in results obj.
        if results.DidRunPass():
          results.AddResult(base_test_result.BaseTestResult(
              'Remote Service detected error.',
              base_test_result.ResultType.UNKNOWN))
<|endoftext|>