<commit_before>#!/usr/bin/env python3
# Copyright 2019 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""This script flashes and runs unit tests on stm32f429i-disc1 boards."""

import argparse
import logging
import os
import subprocess
import sys
import threading
from typing import List

import coloredlogs
import serial
from stm32f429i_disc1_utils import stm32f429i_detector

# Path used to access non-python resources in this python module.
_DIR = os.path.dirname(__file__)

# Path to default openocd configuration file.
_OPENOCD_CONFIG = os.path.join(_DIR, 'openocd_stm32f4xx.cfg')

# Path to scripts provided by openocd.
_OPENOCD_SCRIPTS_DIR = os.path.join(
    os.getenv('PW_PIGWEED_CIPD_INSTALL_DIR', ''), 'share', 'openocd',
    'scripts')

_LOG = logging.getLogger('unit_test_runner')

# Verification of test pass/failure depends on these strings. If the formatting
# or output of the simple_printing_event_handler changes, this may need to be
# updated.
_TESTS_STARTING_STRING = b'[==========] Running all tests.'
_TESTS_DONE_STRING = b'[==========] Done running all tests.'
_TEST_FAILURE_STRING = b'[  FAILED  ]'

# How long to wait for the first byte of a test to be emitted. This is longer
# than the user-configurable timeout as there's a delay while the device is
# flashed.
_FLASH_TIMEOUT = 5.0

class TestingFailure(Exception):
    """A simple exception to be raised when a testing step fails."""

class DeviceNotFound(Exception):
    """A simple exception to be raised when unable to connect to a device."""

def parse_args():
    """Parses command-line arguments."""

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('binary', help='The target test binary to run')
    parser.add_argument('--openocd-config',
                        default=_OPENOCD_CONFIG,
                        help='Path to openocd configuration file')
    parser.add_argument('--stlink-serial',
                        default=None,
                        help='The serial number of the stlink to use when '
                        'flashing the target device')
    parser.add_argument('--port',
                        default=None,
                        help='The name of the serial port to connect to when '
                        'running tests')
    parser.add_argument('--baud',
                        type=int,
                        default=115200,
                        help='Target baud rate to use for serial communication'
                        ' with target device')
    parser.add_argument('--test-timeout',
                        type=float,
                        default=2.0,
                        help='Maximum communication delay in seconds before a '
                        'test is considered unresponsive and aborted')
    parser.add_argument('--verbose',
                        '-v',
                        dest='verbose',
                        action="store_true",
                        help='Output additional logs as the script runs')

    return parser.parse_args()

def log_subprocess_output(level, output):
    """Logs subprocess output line-by-line."""

    lines = output.decode('utf-8', errors='replace').splitlines()
    for line in lines:
        _LOG.log(level, line)

def reset_device(openocd_config, stlink_serial):
    """Uses openocd to reset the attached device."""

    # Name/path of openocd.
    default_flasher = 'openocd'
    flash_tool = os.getenv('OPENOCD_PATH', default_flasher)

    cmd = [
        flash_tool, '-s', _OPENOCD_SCRIPTS_DIR, '-f', openocd_config, '-c',
        'init', '-c', 'reset run', '-c', 'exit'
    ]
    _LOG.debug('Resetting device')

    env = os.environ.copy()
    if stlink_serial:
        env['PW_STLINK_SERIAL'] = stlink_serial

    # Disable GDB port to support multi-device testing.
    env['PW_GDB_PORT'] = 'disabled'
    process = subprocess.run(cmd,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT,
                             env=env)
    if process.returncode:
        log_subprocess_output(logging.ERROR, process.stdout)
        raise TestingFailure('Failed to reset target device')

    log_subprocess_output(logging.DEBUG, process.stdout)

    _LOG.debug('Successfully reset device')

def read_serial(port, baud_rate, test_timeout) -> bytes:
    """Reads lines from a serial port until a line read times out.

    Returns bytes object containing the read serial data.
    """

    serial_data = bytearray()
    device = serial.Serial(baudrate=baud_rate,
                           port=port,
                           timeout=_FLASH_TIMEOUT)
    if not device.is_open:
        raise TestingFailure('Failed to open device')

    # Flush input buffer and reset the device to begin the test.
    device.reset_input_buffer()

    # Block and wait for the first byte.
    serial_data += device.read()
    if not serial_data:
        raise TestingFailure('Device not producing output')

    device.timeout = test_timeout

    # Read with a reasonable timeout until we stop getting characters.
    while True:
        bytes_read = device.readline()
        if not bytes_read:
            break
        serial_data += bytes_read
        if serial_data.rfind(_TESTS_DONE_STRING) != -1:
            # Set to much more aggressive timeout since the last one or two
            # lines should print out immediately. (one line if all fails or all
            # passes, two lines if mixed.)
            device.timeout = 0.01

    # Remove carriage returns.
    serial_data = serial_data.replace(b'\r', b'')

    # Try to trim captured results to only contain most recent test run.
    test_start_index = serial_data.rfind(_TESTS_STARTING_STRING)
    return serial_data if test_start_index == -1 else serial_data[
        test_start_index:]

def flash_device(binary, openocd_config, stlink_serial):
    """Flash binary to a connected device using the provided configuration."""

    # Name/path of openocd.
    default_flasher = 'openocd'
    flash_tool = os.getenv('OPENOCD_PATH', default_flasher)

    openocd_command = ' '.join(['program', binary, 'reset', 'exit'])
    cmd = [
        flash_tool, '-s', _OPENOCD_SCRIPTS_DIR, '-f', openocd_config, '-c',
        openocd_command
    ]
    _LOG.info('Flashing firmware to device')

    env = os.environ.copy()
    if stlink_serial:
        env['PW_STLINK_SERIAL'] = stlink_serial

    # Disable GDB port to support multi-device testing.
    env['PW_GDB_PORT'] = 'disabled'
    process = subprocess.run(cmd,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT,
                             env=env)
    if process.returncode:
        log_subprocess_output(logging.ERROR, process.stdout)
        raise TestingFailure('Failed to flash target device')

    log_subprocess_output(logging.DEBUG, process.stdout)

    _LOG.debug('Successfully flashed firmware to device')

def handle_test_results(test_output):
    """Parses test output to determine whether tests passed or failed."""

    if test_output.find(_TESTS_STARTING_STRING) == -1:
        raise TestingFailure('Failed to find test start')

    if test_output.rfind(_TESTS_DONE_STRING) == -1:
        log_subprocess_output(logging.INFO, test_output)
        raise TestingFailure('Tests did not complete')

    if test_output.rfind(_TEST_FAILURE_STRING) != -1:
        log_subprocess_output(logging.INFO, test_output)
        raise TestingFailure('Test suite had one or more failures')

    log_subprocess_output(logging.DEBUG, test_output)

    _LOG.info('Test passed!')

def _threaded_test_reader(dest, port, baud_rate, test_timeout):
    """Parses test output to the mutable "dest" passed to this function."""
    dest.append(read_serial(port, baud_rate, test_timeout))

def run_device_test(binary,
                    test_timeout,
                    openocd_config,
                    baud,
                    stlink_serial=None,
                    port=None) -> bool:
    """Flashes, runs, and checks an on-device test binary.

    Returns true on test pass.
    """

    if stlink_serial is None and port is None:
        _LOG.debug('Attempting to automatically detect dev board')
        boards = stm32f429i_detector.detect_boards()
        if not boards:
            error = 'Could not find an attached device'
            _LOG.error(error)
            raise DeviceNotFound(error)
        stlink_serial = boards[0].serial_number
        port = boards[0].dev_name

    _LOG.debug('Launching test binary %s', binary)
    try:
        # Begin capturing test output via another thread BEFORE flashing the
        # device since the test will automatically run after the image is
        # flashed. This reduces flake since there isn't a need to time a reset
        # correctly relative to the start of capturing device output.
        result: List[bytes] = []
        threaded_reader_args = (result, port, baud, test_timeout)
        read_thread = threading.Thread(target=_threaded_test_reader,
                                       args=threaded_reader_args)
        read_thread.start()
        _LOG.info('Running test')
        flash_device(binary, openocd_config, stlink_serial)
        read_thread.join()
        if result:
            handle_test_results(result[0])
    except TestingFailure as err:
        _LOG.error(err)
        return False

    return True

def main():
    """Set up runner, and then flash/run device test."""
    args = parse_args()

    # Try to use pw_cli logs, else default to something reasonable.
    try:
        import pw_cli.log  # pylint: disable=import-outside-toplevel
        log_level = logging.DEBUG if args.verbose else logging.INFO
        pw_cli.log.install(level=log_level)
    except ImportError:
        coloredlogs.install(level='DEBUG' if args.verbose else 'INFO',
                            level_styles={
                                'debug': {
                                    'color': 244
                                },
                                'error': {
                                    'color': 'red'
                                }
                            },
                            fmt='%(asctime)s %(levelname)s | %(message)s')

    if run_device_test(args.binary, args.test_timeout, args.openocd_config,
                       args.baud, args.stlink_serial, args.port):
        sys.exit(0)
    else:
        sys.exit(1)

if __name__ == '__main__':
    main()
<commit_msg>stm32f429i-disc1: Increase test runner timeout<commit_after>#!/usr/bin/env python3
# Copyright 2019 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""This script flashes and runs unit tests on stm32f429i-disc1 boards."""

import argparse
import logging
import os
import subprocess
import sys
import threading
from typing import List

import coloredlogs
import serial
from stm32f429i_disc1_utils import stm32f429i_detector

# Path used to access non-python resources in this python module.
_DIR = os.path.dirname(__file__)

# Path to default openocd configuration file.
_OPENOCD_CONFIG = os.path.join(_DIR, 'openocd_stm32f4xx.cfg')

# Path to scripts provided by openocd.
_OPENOCD_SCRIPTS_DIR = os.path.join(
    os.getenv('PW_PIGWEED_CIPD_INSTALL_DIR', ''), 'share', 'openocd',
    'scripts')

_LOG = logging.getLogger('unit_test_runner')

# Verification of test pass/failure depends on these strings. If the formatting
# or output of the simple_printing_event_handler changes, this may need to be
# updated.
_TESTS_STARTING_STRING = b'[==========] Running all tests.'
_TESTS_DONE_STRING = b'[==========] Done running all tests.'
_TEST_FAILURE_STRING = b'[  FAILED  ]'

# How long to wait for the first byte of a test to be emitted. This is longer
# than the user-configurable timeout as there's a delay while the device is
# flashed.
_FLASH_TIMEOUT = 5.0

class TestingFailure(Exception):
    """A simple exception to be raised when a testing step fails."""

class DeviceNotFound(Exception):
    """A simple exception to be raised when unable to connect to a device."""

def parse_args():
    """Parses command-line arguments."""

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('binary', help='The target test binary to run')
    parser.add_argument('--openocd-config',
                        default=_OPENOCD_CONFIG,
                        help='Path to openocd configuration file')
    parser.add_argument('--stlink-serial',
                        default=None,
                        help='The serial number of the stlink to use when '
                        'flashing the target device')
    parser.add_argument('--port',
                        default=None,
                        help='The name of the serial port to connect to when '
                        'running tests')
    parser.add_argument('--baud',
                        type=int,
                        default=115200,
                        help='Target baud rate to use for serial communication'
                        ' with target device')
    parser.add_argument('--test-timeout',
                        type=float,
                        default=5.0,
                        help='Maximum communication delay in seconds before a '
                        'test is considered unresponsive and aborted')
    parser.add_argument('--verbose',
                        '-v',
                        dest='verbose',
                        action="store_true",
                        help='Output additional logs as the script runs')

    return parser.parse_args()

def log_subprocess_output(level, output):
    """Logs subprocess output line-by-line."""

    lines = output.decode('utf-8', errors='replace').splitlines()
    for line in lines:
        _LOG.log(level, line)

def reset_device(openocd_config, stlink_serial):
    """Uses openocd to reset the attached device."""

    # Name/path of openocd.
    default_flasher = 'openocd'
    flash_tool = os.getenv('OPENOCD_PATH', default_flasher)

    cmd = [
        flash_tool, '-s', _OPENOCD_SCRIPTS_DIR, '-f', openocd_config, '-c',
        'init', '-c', 'reset run', '-c', 'exit'
    ]
    _LOG.debug('Resetting device')

    env = os.environ.copy()
    if stlink_serial:
        env['PW_STLINK_SERIAL'] = stlink_serial

    # Disable GDB port to support multi-device testing.
    env['PW_GDB_PORT'] = 'disabled'
    process = subprocess.run(cmd,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT,
                             env=env)
    if process.returncode:
        log_subprocess_output(logging.ERROR, process.stdout)
        raise TestingFailure('Failed to reset target device')

    log_subprocess_output(logging.DEBUG, process.stdout)

    _LOG.debug('Successfully reset device')

def read_serial(port, baud_rate, test_timeout) -> bytes:
    """Reads lines from a serial port until a line read times out.

    Returns bytes object containing the read serial data.
    """

    serial_data = bytearray()
    device = serial.Serial(baudrate=baud_rate,
                           port=port,
                           timeout=_FLASH_TIMEOUT)
    if not device.is_open:
        raise TestingFailure('Failed to open device')

    # Flush input buffer and reset the device to begin the test.
    device.reset_input_buffer()

    # Block and wait for the first byte.
    serial_data += device.read()
    if not serial_data:
        raise TestingFailure('Device not producing output')

    device.timeout = test_timeout

    # Read with a reasonable timeout until we stop getting characters.
    while True:
        bytes_read = device.readline()
        if not bytes_read:
            break
        serial_data += bytes_read
        if serial_data.rfind(_TESTS_DONE_STRING) != -1:
            # Set to much more aggressive timeout since the last one or two
            # lines should print out immediately. (one line if all fails or all
            # passes, two lines if mixed.)
            device.timeout = 0.01

    # Remove carriage returns.
    serial_data = serial_data.replace(b'\r', b'')

    # Try to trim captured results to only contain most recent test run.
    test_start_index = serial_data.rfind(_TESTS_STARTING_STRING)
    return serial_data if test_start_index == -1 else serial_data[
        test_start_index:]

def flash_device(binary, openocd_config, stlink_serial):
    """Flash binary to a connected device using the provided configuration."""

    # Name/path of openocd.
    default_flasher = 'openocd'
    flash_tool = os.getenv('OPENOCD_PATH', default_flasher)

    openocd_command = ' '.join(['program', binary, 'reset', 'exit'])
    cmd = [
        flash_tool, '-s', _OPENOCD_SCRIPTS_DIR, '-f', openocd_config, '-c',
        openocd_command
    ]
    _LOG.info('Flashing firmware to device')

    env = os.environ.copy()
    if stlink_serial:
        env['PW_STLINK_SERIAL'] = stlink_serial

    # Disable GDB port to support multi-device testing.
    env['PW_GDB_PORT'] = 'disabled'
    process = subprocess.run(cmd,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT,
                             env=env)
    if process.returncode:
        log_subprocess_output(logging.ERROR, process.stdout)
        raise TestingFailure('Failed to flash target device')

    log_subprocess_output(logging.DEBUG, process.stdout)

    _LOG.debug('Successfully flashed firmware to device')

def handle_test_results(test_output):
    """Parses test output to determine whether tests passed or failed."""

    if test_output.find(_TESTS_STARTING_STRING) == -1:
        raise TestingFailure('Failed to find test start')

    if test_output.rfind(_TESTS_DONE_STRING) == -1:
        log_subprocess_output(logging.INFO, test_output)
        raise TestingFailure('Tests did not complete')

    if test_output.rfind(_TEST_FAILURE_STRING) != -1:
        log_subprocess_output(logging.INFO, test_output)
        raise TestingFailure('Test suite had one or more failures')

    log_subprocess_output(logging.DEBUG, test_output)

    _LOG.info('Test passed!')

def _threaded_test_reader(dest, port, baud_rate, test_timeout):
    """Parses test output to the mutable "dest" passed to this function."""
    dest.append(read_serial(port, baud_rate, test_timeout))

def run_device_test(binary,
                    test_timeout,
                    openocd_config,
                    baud,
                    stlink_serial=None,
                    port=None) -> bool:
    """Flashes, runs, and checks an on-device test binary.

    Returns true on test pass.
    """

    if stlink_serial is None and port is None:
        _LOG.debug('Attempting to automatically detect dev board')
        boards = stm32f429i_detector.detect_boards()
        if not boards:
            error = 'Could not find an attached device'
            _LOG.error(error)
            raise DeviceNotFound(error)
        stlink_serial = boards[0].serial_number
        port = boards[0].dev_name

    _LOG.debug('Launching test binary %s', binary)
    try:
        # Begin capturing test output via another thread BEFORE flashing the
        # device since the test will automatically run after the image is
        # flashed. This reduces flake since there isn't a need to time a reset
        # correctly relative to the start of capturing device output.
        result: List[bytes] = []
        threaded_reader_args = (result, port, baud, test_timeout)
        read_thread = threading.Thread(target=_threaded_test_reader,
                                       args=threaded_reader_args)
        read_thread.start()
        _LOG.info('Running test')
        flash_device(binary, openocd_config, stlink_serial)
        read_thread.join()
        if result:
            handle_test_results(result[0])
    except TestingFailure as err:
        _LOG.error(err)
        return False

    return True

def main():
    """Set up runner, and then flash/run device test."""
    args = parse_args()

    # Try to use pw_cli logs, else default to something reasonable.
    try:
        import pw_cli.log  # pylint: disable=import-outside-toplevel
        log_level = logging.DEBUG if args.verbose else logging.INFO
        pw_cli.log.install(level=log_level)
    except ImportError:
        coloredlogs.install(level='DEBUG' if args.verbose else 'INFO',
                            level_styles={
                                'debug': {
                                    'color': 244
                                },
                                'error': {
                                    'color': 'red'
                                }
                            },
                            fmt='%(asctime)s %(levelname)s | %(message)s')

    if run_device_test(args.binary, args.test_timeout, args.openocd_config,
                       args.baud, args.stlink_serial, args.port):
        sys.exit(0)
    else:
        sys.exit(1)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import re

from livestreamer.compat import urlparse
from livestreamer.plugin import Plugin
from livestreamer.plugin.api import http, validate
from livestreamer.stream import HLSStream, RTMPStream

CHINFO_URL = "http://www.filmon.com/ajax/getChannelInfo"
SWF_URL = "http://www.filmon.com/tv/modules/FilmOnTV/files/flashapp/filmon/FilmonPlayer.swf"
VODINFO_URL = "http://www.filmon.com/vod/info/{0}"

AJAX_HEADERS = {
    "Referer": "http://www.filmon.com",
    "X-Requested-With": "XMLHttpRequest",
    "User-Agent": "Mozilla/5.0"
}
QUALITY_WEIGHTS = {
    "high": 720,
    "low": 480
}
STREAM_TYPES = ("hls", "rtmp")

_url_re = re.compile("http(s)?://(\w+\.)?filmon.com/(channel|tv|vod)/")
_channel_id_re = re.compile("/channels/(\d+)/extra_big_logo.png")
_vod_id_re = re.compile("movie_id=(\d+)")

_channel_schema = validate.Schema({
    "streams": [{
        "name": validate.text,
        "quality": validate.text,
        "url": validate.url(scheme=validate.any("http", "rtmp"))
    }]
})
_vod_schema = validate.Schema(
    {
        "data": {
            "streams": {
                validate.text: {
                    "name": validate.text,
                    "url": validate.url(scheme=validate.any("http", "rtmp"))
                }
            }
        }
    },
    validate.get("data")
)

def ajax(*args, **kwargs):
    kwargs["headers"] = AJAX_HEADERS
    return http.post(*args, **kwargs)

class Filmon(Plugin):
    @classmethod
    def can_handle_url(cls, url):
        return _url_re.match(url)

    @classmethod
    def stream_weight(cls, key):
        weight = QUALITY_WEIGHTS.get(key)
        if weight:
            return weight, "filmon"

        return Plugin.stream_weight(key)

    def _create_rtmp_stream(self, stream, live=True):
        rtmp = stream["url"]
        playpath = stream["name"]
        parsed = urlparse(rtmp)
        if parsed.query:
            app = "{0}?{1}".format(parsed.path[1:], parsed.query)
        else:
            app = parsed.path[1:]

        if playpath.endswith(".mp4"):
            playpath = "mp4:" + playpath

        params = {
            "rtmp": rtmp,
            "pageUrl": self.url,
            "swfUrl": SWF_URL,
            "playpath": playpath,
            "app": app,
        }
        if live:
            params["live"] = True

        return RTMPStream(self.session, params)

    def _get_live_streams(self, channel_id):
        params = {"channel_id": channel_id}
        for stream_type in STREAM_TYPES:
            cookies = {"flash-player-type": stream_type}
            res = ajax(CHINFO_URL, cookies=cookies, data=params)
            channel = http.json(res, schema=_channel_schema)

            # TODO: Replace with "yield from" when dropping Python 2.
            for stream in self._parse_live_streams(channel):
                yield stream

    def _parse_live_streams(self, channel):
        for stream in channel["streams"]:
            name = stream["quality"]
            scheme = urlparse(stream["url"]).scheme

            if scheme == "http":
                try:
                    streams = HLSStream.parse_variant_playlist(self.session, stream["url"])
                    for __, stream in streams.items():
                        yield name, stream

                except IOError as err:
                    self.logger.error("Failed to extract HLS stream '{0}': {1}", name, err)

            elif scheme == "rtmp":
                yield name, self._create_rtmp_stream(stream)

    def _get_vod_streams(self, movie_id):
        for stream_type in STREAM_TYPES:
            cookies = {"flash-player-type": stream_type}
            res = ajax(VODINFO_URL.format(movie_id), cookies=cookies)
            vod = http.json(res, schema=_vod_schema)

            # TODO: Replace with "yield from" when dropping Python 2.
            for stream in self._parse_vod_streams(vod):
                yield stream

    def _parse_vod_streams(self, vod):
        for name, stream in vod["streams"].items():
            scheme = urlparse(stream["url"]).scheme

            if scheme == "http":
                yield name, HLSStream(self.session, stream["url"])
            elif scheme == "rtmp":
                yield name, self._create_rtmp_stream(stream, live=False)

    def _get_streams(self):
        res = http.get(self.url)

        match = _vod_id_re.search(res.text)
        if match:
            return self._get_vod_streams(match.group(1))

        match = _channel_id_re.search(res.text)
        if match:
            return self._get_live_streams(match.group(1))

__plugin__ = Filmon
<commit_msg>plugins.filmon: Update for API change<commit_after>import re

from livestreamer.compat import urlparse
from livestreamer.plugin import Plugin
from livestreamer.plugin.api import http, validate
from livestreamer.stream import HLSStream

CHINFO_URL = "http://www.filmon.com/ajax/getChannelInfo"
SWF_URL = "http://www.filmon.com/tv/modules/FilmOnTV/files/flashapp/filmon/FilmonPlayer.swf"
VODINFO_URL = "http://www.filmon.com/api/vod/movie?id={0}"

AJAX_HEADERS = {
    "Referer": "http://www.filmon.com",
    "X-Requested-With": "XMLHttpRequest",
    "User-Agent": "Mozilla/5.0"
}
QUALITY_WEIGHTS = {
    "high": 720,
    "low": 480
}

_url_re = re.compile("http(s)?://(\w+\.)?filmon.com/(channel|tv|vod)/")
_channel_id_re = re.compile("/channels/(\d+)/extra_big_logo.png")
_vod_id_re = re.compile("movie_id=(\d+)")

_channel_schema = validate.Schema({
    "streams": [{
        "name": validate.text,
        "quality": validate.text,
        "url": validate.url(scheme="http")
    }]
})
_vod_schema = validate.Schema(
    {
        "response": {
            validate.optional("streams"): {
                validate.text: {
                    "name": validate.text,
                    "url": validate.url(scheme="http")
                }
            }
        }
    },
    validate.get("response")
)

def ajax(*args, **kwargs):
    kwargs["headers"] = AJAX_HEADERS
    return http.post(*args, **kwargs)

class Filmon(Plugin):
    @classmethod
    def can_handle_url(cls, url):
        return _url_re.match(url)

    @classmethod
    def stream_weight(cls, key):
        weight = QUALITY_WEIGHTS.get(key)
        if weight:
            return weight, "filmon"

        return Plugin.stream_weight(key)

    def _create_rtmp_stream(self, stream, live=True):
        rtmp = stream["url"]
        playpath = stream["name"]
        parsed = urlparse(rtmp)
        if parsed.query:
            app = "{0}?{1}".format(parsed.path[1:], parsed.query)
        else:
            app = parsed.path[1:]

        if playpath.endswith(".mp4"):
            playpath = "mp4:" + playpath

        params = {
            "rtmp": rtmp,
            "pageUrl": self.url,
            "swfUrl": SWF_URL,
            "playpath": playpath,
            "app": app,
        }
        if live:
            params["live"] = True

        return RTMPStream(self.session, params)

    def _get_live_streams(self, channel_id):
        params = {"channel_id": channel_id}
        res = ajax(CHINFO_URL, data=params)
        channel = http.json(res, schema=_channel_schema)

        # TODO: Replace with "yield from" when dropping Python 2.
        for stream in self._parse_live_streams(channel):
            yield stream

    def _parse_live_streams(self, channel):
        for stream in channel["streams"]:
            name = stream["quality"]
            scheme = urlparse(stream["url"]).scheme

            yield name, HLSStream(self.session, stream["url"])

    def _get_vod_streams(self, movie_id):
        res = ajax(VODINFO_URL.format(movie_id))
        vod = http.json(res, schema=_vod_schema)

        # TODO: Replace with "yield from" when dropping Python 2.
        for stream in self._parse_vod_streams(vod):
            yield stream

    def _parse_vod_streams(self, vod):
        for name, stream in vod["streams"].items():
            scheme = urlparse(stream["url"]).scheme

            yield name, HLSStream(self.session, stream["url"])

    def _get_streams(self):
        res = http.get(self.url)

        match = _vod_id_re.search(res.text)
        if match:
            return self._get_vod_streams(match.group(1))

        match = _channel_id_re.search(res.text)
        if match:
            return self._get_live_streams(match.group(1))

__plugin__ = Filmon
<|endoftext|>


<commit_before>from __future__ import absolute_import

from django.core.management.base import BaseCommand

def pluralize(desc, value):
    if value > 1:
        return desc + 's'
    return desc

class Command(BaseCommand):
    requires_model_validation = True

    def handle(self, *args, **options):
        from ...models import Message

        count = Message.objects.filter(visible=False).count()

        print("Removing %s invisible %s... " % (
                count, pluralize("message", count))
        Message.objects.cleanup()

<commit_msg>Fixes syntax error in django transport<commit_after>from __future__ import absolute_import

from django.core.management.base import BaseCommand

def pluralize(desc, value):
    if value > 1:
        return desc + 's'
    return desc

class Command(BaseCommand):
    requires_model_validation = True

    def handle(self, *args, **options):
        from ...models import Message

        count = Message.objects.filter(visible=False).count()

        print("Removing %s invisible %s... " % (
                count, pluralize("message", count)))
        Message.objects.cleanup()

<|endoftext|>


<commit_before>import json
import time

from django.core.cache import cache
from django.http import JsonResponse, Http404, HttpResponse, StreamingHttpResponse
from django.views.decorators.csrf import csrf_exempt

from datasets.models import Dataset
from participants.models import Participant
from submissions.models import Submission
from workers.tasks import run

def _yield_submission_output(submission_id):
    # yield something immediately for the connection to start on the cli for nicer printing
    yield json.dumps({"keepalive": "keep the dream alive"}) + "\r"

    last_message_time = time.time()

    while True:
        stdout = cache.get("submission-%s-stdout" % submission_id)
        stderr = cache.get("submission-%s-stderr" % submission_id)
        # After reading let's clear the cache and that lets the runner know to fill the buffer again
        cache.delete_many(
            ("submission-%s-stdout" % submission_id, "submission-%s-stderr" % submission_id)
        )

        # Don't want someone accidentally breaking our format where each message ends with \r, so get rid of them
        # The \r signifies the end of a message (so messages aren't merged together as they're streamed/chunked)
        if stdout:
            stdout = stdout.replace("\r", "")
        if stderr:
            stderr = stderr.replace("\r", "")

        if stdout or stderr:
            last_message_time = time.time()
            yield json.dumps({
                "stdout": stdout,
                "stderr": stderr,
            }) + "\r"  # Note we add carriage return at the very end to signify the end of a message, useful
                       # because we're chunking data and sending it in parts, need to know when it ends!

        # Heroku needs keepalive every 30s, so let's do it before that
        if time.time() - last_message_time > 20:  # 25 is too high
            print "yielding timeout message"
            last_message_time = time.time()
            yield json.dumps({"keepalive": "keep the dream alive"}) + "\r"

        if stdout is not None and "-%-%-%-%-END BRAIN SEQUENCE-%-%-%-%-" in stdout:
            # We don't check for a timeout here... I think that's OK since it's terminated when user view
            # yielding from this generator???
            return
        time.sleep(0.1)

@csrf_exempt
def create(request):
    """Post data like

    name - Name of submitter
    description -
    languages -
    dataset - Name of dataset to use
    submission - Zip file containing submission contents
    wait -
    """
    if request.method == "POST":
        # Validate
        required_fields = ("name", "description", "languages", "datasets", "wait",)
        if any(field not in request.POST for field in required_fields):
            return JsonResponse({
                "error": "missing one of the required fields: %s" % (required_fields,)
            }, status=400)
        if len(request.FILES) == 0:
            return JsonResponse({
                "error": "no file was uploaded?"
            }, status=400)

        # Get dataset
        datasets = []
        if request.POST["datasets"]:
            dataset_names = request.POST.getlist("datasets")
            for name in dataset_names:
                try:
                    datasets.append(Dataset.objects.get(name=name))
                except Dataset.DoesNotExist:
                    return JsonResponse({
                        "error": 'could not find dataset named "%s"' % name
                    }, status=400)

        # Get participant
        participant, _ = Participant.objects.get_or_create(name=request.POST["name"])

        # Save it
        submission = Submission.objects.create(
            participant=participant,
            zip_file=request.FILES["zip_file"],
        )
        for dataset in datasets:
            submission.datasets.add(dataset)

        # Run it (and then return immediately or stream data back)
        run.delay(submission.pk)
        if request.POST["wait"]:
            return StreamingHttpResponse(_yield_submission_output(submission.pk))
        else:
            return HttpResponse()
    return HttpResponse(status=403)

def submission_list(request):
    submissions = Submission.objects.filter()
    if len(submissions) == 0:
        raise Http404()
    return [submission.json_short for submission in submissions]

# @d("/submissions/detail/<int:pk>")
# def submission_detail(request, pk):
#     submission = get_object_or_404(Submission, pk=pk)
#     return submission.json
<commit_msg>much larger buffer flusher...<commit_after>import json
import time

from django.core.cache import cache
from django.http import JsonResponse, Http404, HttpResponse, StreamingHttpResponse
from django.views.decorators.csrf import csrf_exempt

from datasets.models import Dataset
from participants.models import Participant
from submissions.models import Submission
from workers.tasks import run

def _yield_submission_output(submission_id):
    # yield something immediately for the connection to start on the cli for nicer printing
    yield json.dumps({
        "keepalive": "keep the dream alive" * 100  # send a bunch of data to flush buffers...?
    }) + "\r"

    last_message_time = time.time()

    while True:
        stdout = cache.get("submission-%s-stdout" % submission_id)
        stderr = cache.get("submission-%s-stderr" % submission_id)
        # After reading let's clear the cache and that lets the runner know to fill the buffer again
        cache.delete_many(
            ("submission-%s-stdout" % submission_id, "submission-%s-stderr" % submission_id)
        )

        # Don't want someone accidentally breaking our format where each message ends with \r, so get rid of them
        # The \r signifies the end of a message (so messages aren't merged together as they're streamed/chunked)
        if stdout:
            stdout = stdout.replace("\r", "")
        if stderr:
            stderr = stderr.replace("\r", "")

        if stdout or stderr:
            last_message_time = time.time()
            yield json.dumps({
                "stdout": stdout,
                "stderr": stderr,
            }) + "\r"  # Note we add carriage return at the very end to signify the end of a message, useful
                       # because we're chunking data and sending it in parts, need to know when it ends!

        # Heroku needs keepalive every 30s, so let's do it before that
        if time.time() - last_message_time > 20:  # 25 is too high
            print "yielding timeout message"
            last_message_time = time.time()
            yield json.dumps({"keepalive": "keep the dream alive"}) + "\r"

        if stdout is not None and "-%-%-%-%-END BRAIN SEQUENCE-%-%-%-%-" in stdout:
            # We don't check for a timeout here... I think that's OK since it's terminated when user view
            # yielding from this generator???
            return
        time.sleep(0.1)

@csrf_exempt
def create(request):
    """Post data like

    name - Name of submitter
    description -
    languages -
    dataset - Name of dataset to use
    submission - Zip file containing submission contents
    wait -
    """
    if request.method == "POST":
        # Validate
        required_fields = ("name", "description", "languages", "datasets", "wait",)
        if any(field not in request.POST for field in required_fields):
            return JsonResponse({
                "error": "missing one of the required fields: %s" % (required_fields,)
            }, status=400)
        if len(request.FILES) == 0:
            return JsonResponse({
                "error": "no file was uploaded?"
            }, status=400)

        # Get dataset
        datasets = []
        if request.POST["datasets"]:
            dataset_names = request.POST.getlist("datasets")
            for name in dataset_names:
                try:
                    datasets.append(Dataset.objects.get(name=name))
                except Dataset.DoesNotExist:
                    return JsonResponse({
                        "error": 'could not find dataset named "%s"' % name
                    }, status=400)

        # Get participant
        participant, _ = Participant.objects.get_or_create(name=request.POST["name"])

        # Save it
        submission = Submission.objects.create(
            participant=participant,
            zip_file=request.FILES["zip_file"],
        )
        for dataset in datasets:
            submission.datasets.add(dataset)

        # Run it (and then return immediately or stream data back)
        run.delay(submission.pk)
        if request.POST["wait"]:
            return StreamingHttpResponse(_yield_submission_output(submission.pk))
        else:
            return HttpResponse()
    return HttpResponse(status=403)

def submission_list(request):
    submissions = Submission.objects.filter()
    if len(submissions) == 0:
        raise Http404()
    return [submission.json_short for submission in submissions]

# @d("/submissions/detail/<int:pk>")
# def submission_detail(request, pk):
#     submission = get_object_or_404(Submission, pk=pk)
#     return submission.json
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import, division, print_function

import ctypes.util

from enum import Enum
from six import iteritems, PY3

from ._ffi import ffi
from .exceptions import InvalidHash

lib = ffi.dlopen(ctypes.util.find_library("argon2"))

class Type(Enum):
    D = lib.Argon2_d
    I = lib.Argon2_i

NoneType = type(None)

def check_types(**kw):
    """
    Check each ``name: (value, types)`` in *kw*.

    Returns a human-readable string of all violations or `None``.
    """
    errors = []
    for name, (value, types) in iteritems(kw):
        if not isinstance(value, types):
            if isinstance(types, tuple):
                types = ", or ".join(t.__name__ for t in types)
            else:
                types = types.__name__
            errors.append("'{name}' must be a {type} (got {actual})".format(
                name=name,
                type=types,
                actual=type(value).__name__,
            ))

    if errors != []:
        return ", ".join(errors) + "."

def error_to_str(error):
    """
    Convert an Argon2 error code into a native string.
    """
    msg = ffi.string(lib.error_message(error))
    if PY3:
        msg = msg.decode("ascii")
    return msg

def encoded_str_len(l):
    """
    Compute how long a byte string of length *l* becomes if encoded to hex.
    """
    return (l << 2) / 3 + 2

def get_encoded_len(hash_len, salt_len):
    """
    Compute the size of the required buffer for an encoded hash with *hash_len*
    and *salt_len*.
    """
    # From https://github.com/P-H-C/phc-winner-argon2/blob/master/src/run.c:
    #
    # Sample encode: $argon2i$m=65536,t=2,p=4$c29tZXNhbHQAAAAAAAAAAA$QWLzI4TY9H
    # kL2ZTLc8g6SinwdhZewYrzz9zxCo0bkGY
    # Maximumum lengths are defined as:
    # strlen $argon2i$ = 9
    # m=65536 with strlen (uint32_t)-1 = 10, so this total is 12
    # ,t=2,p=4 If we consider each number to potentially reach four digits
    # in future, this = 14
    # $c29tZXNhbHQAAAAAAAAAAA Formula for this is
    # (SALT_LEN * 4 + 3) / 3 + 1 = 23
    # $QWLzI4TY9HkL2ZTLc8g6SinwdhZewYrzz9zxCo0bkGY per above formula, = 44
    # + NULL byte
    # 9 + 12 + 14 + 23 + 44 + 1 = 103
    # Rounded to 4 byte boundary: 104
    return (
        39 + int(encoded_str_len(hash_len) + encoded_str_len(salt_len))
    ) & ~3

def guess_type(s):
    """
    Guesses what type of encoded Argon2 hash *s* is or raises InvalidHash.
    """
    prefix = s[:8]
    if prefix == b"$argon2i":
        return Type.I
    elif prefix == b"$argon2d":
        return Type.D
    else:
        raise InvalidHash(s)
<commit_msg>Save some function calls<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import, division, print_function

import ctypes.util

from enum import Enum
from six import iteritems, PY3

from ._ffi import ffi
from .exceptions import InvalidHash

lib = ffi.dlopen(ctypes.util.find_library("argon2"))

class Type(Enum):
    D = lib.Argon2_d
    I = lib.Argon2_i

NoneType = type(None)

def check_types(**kw):
    """
    Check each ``name: (value, types)`` in *kw*.

    Returns a human-readable string of all violations or `None``.
    """
    errors = []
    for name, (value, types) in iteritems(kw):
        if not isinstance(value, types):
            if isinstance(types, tuple):
                types = ", or ".join(t.__name__ for t in types)
            else:
                types = types.__name__
            errors.append("'{name}' must be a {type} (got {actual})".format(
                name=name,
                type=types,
                actual=type(value).__name__,
            ))

    if errors != []:
        return ", ".join(errors) + "."

def error_to_str(error):
    """
    Convert an Argon2 error code into a native string.
    """
    msg = ffi.string(lib.error_message(error))
    if PY3:
        msg = msg.decode("ascii")
    return msg

def encoded_str_len(l):
    """
    Compute how long a byte string of length *l* becomes if encoded to hex.
    """
    return (l << 2) / 3 + 2

def get_encoded_len(hash_len, salt_len):
    """
    Compute the size of the required buffer for an encoded hash with *hash_len*
    and *salt_len*.
    """
    # From https://github.com/P-H-C/phc-winner-argon2/blob/master/src/run.c:
    #
    # Sample encode: $argon2i$m=65536,t=2,p=4$c29tZXNhbHQAAAAAAAAAAA$QWLzI4TY9H
    # kL2ZTLc8g6SinwdhZewYrzz9zxCo0bkGY
    # Maximumum lengths are defined as:
    # strlen $argon2i$ = 9
    # m=65536 with strlen (uint32_t)-1 = 10, so this total is 12
    # ,t=2,p=4 If we consider each number to potentially reach four digits
    # in future, this = 14
    # $c29tZXNhbHQAAAAAAAAAAA Formula for this is
    # (SALT_LEN * 4 + 3) / 3 + 1 = 23
    # $QWLzI4TY9HkL2ZTLc8g6SinwdhZewYrzz9zxCo0bkGY per above formula, = 44
    # + NULL byte
    # 9 + 12 + 14 + 23 + 44 + 1 = 103
    # Rounded to 4 byte boundary: 104
    return (
        39 + int((hash_len << 2) / 3 + 2 + (salt_len << 2) / 3 + 2)
    ) & ~3

def guess_type(s):
    """
    Guesses what type of encoded Argon2 hash *s* is or raises InvalidHash.
    """
    prefix = s[:8]
    if prefix == b"$argon2i":
        return Type.I
    elif prefix == b"$argon2d":
        return Type.D
    else:
        raise InvalidHash(s)
<|endoftext|>


<commit_before><commit_msg>Removed unused imports<commit_after><|endoftext|>


<commit_before>"""
Test lldb-vscode setBreakpoints request
"""

from __future__ import print_function

import unittest2
import vscode
from lldbsuite.test.decorators import *
from lldbsuite.test.lldbtest import *
from lldbsuite.test import lldbutil
import lldbvscode_testcase
import os
import shutil
import subprocess
import tempfile
import threading
import time

def spawn_and_wait(program, delay):
    if delay:
        time.sleep(delay)
    process = subprocess.Popen([program],
                               stdin=subprocess.PIPE,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    process.wait()

class TestVSCode_attach(lldbvscode_testcase.VSCodeTestCaseBase):

    mydir = TestBase.compute_mydir(__file__)

    def set_and_hit_breakpoint(self, continueToExit=True):
        source = 'main.c'
        breakpoint1_line = line_number(source, '// breakpoint 1')
        lines = [breakpoint1_line]
        # Set breakoint in the thread function so we can step the threads
        breakpoint_ids = self.set_source_breakpoints(source, lines)
        self.assertEqual(len(breakpoint_ids), len(lines),
                         "expect correct number of breakpoints")
        self.continue_to_breakpoints(breakpoint_ids)
        if continueToExit:
            self.continue_to_exit()

    @skipIfWindows
    @no_debug_info_test
    def test_by_pid(self):
        '''
            Tests attaching to a process by process ID.
        '''
        self.build_and_create_debug_adaptor()
        program = self.getBuildArtifact("a.out")
        self.process = subprocess.Popen([program],
                                        stdin=subprocess.PIPE,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE)
        self.attach(pid=self.process.pid)
        self.set_and_hit_breakpoint(continueToExit=True)

    @skipIfWindows
    @no_debug_info_test
    def test_by_name(self):
        '''
            Tests attaching to a process by process name.
        '''
        self.build_and_create_debug_adaptor()
        orig_program = self.getBuildArtifact("a.out")
        # Since we are going to attach by process name, we need a unique
        # process name that has minimal chance to match a process that is
        # already running. To do this we use tempfile.mktemp() to give us a
        # full path to a location where we can copy our executable. We then
        # run this copy to ensure we don't get the error "more that one
        # process matches 'a.out'".
        program = tempfile.mktemp()
        shutil.copyfile(orig_program, program)
        shutil.copymode(orig_program, program)

        def cleanup():
            if os.path.exists(program):
                os.unlink(program)
        # Execute the cleanup function during test case tear down.
        self.addTearDownHook(cleanup)

        self.process = subprocess.Popen([program],
                                        stdin=subprocess.PIPE,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE)
        # Wait for a bit to ensure the process is launched
        time.sleep(1)
        self.attach(program=program)
        self.set_and_hit_breakpoint(continueToExit=True)

    @skipUnlessDarwin
    @no_debug_info_test
    def test_by_name_waitFor(self):
        '''
            Tests attaching to a process by process name and waiting for the
            next instance of a process to be launched, ingoring all current
            ones.
        '''
        self.build_and_create_debug_adaptor()
        program = self.getBuildArtifact("a.out")
        self.spawn_thread = threading.Thread(target=spawn_and_wait,
                                             args=(program, 1.0,))
        self.spawn_thread.start()
        self.attach(program=program, waitFor=True)
        self.set_and_hit_breakpoint(continueToExit=True)

    @skipIfWindows
    @no_debug_info_test
    def test_commands(self):
        '''
            Tests the "initCommands", "preRunCommands", "stopCommands",
            "exitCommands", and "attachCommands" that can be passed during
            attach.

            "initCommands" are a list of LLDB commands that get executed
            before the targt is created.
            "preRunCommands" are a list of LLDB commands that get executed
            after the target has been created and before the launch.
            "stopCommands" are a list of LLDB commands that get executed each
            time the program stops.
            "exitCommands" are a list of LLDB commands that get executed when
            the process exits
            "attachCommands" are a list of LLDB commands that get executed and
            must have a valid process in the selected target in LLDB after
            they are done executing. This allows custom commands to create any
            kind of debug session.
        '''
        self.build_and_create_debug_adaptor()
        program = self.getBuildArtifact("a.out")
        # Here we just create a target and launch the process as a way to test
        # if we are able to use attach commands to create any kind of a target
        # and use it for debugging
        attachCommands = [
            'target create -d "%s"' % (program),
            'process launch -- arg1'
        ]
        initCommands = ['target list', 'platform list']
        preRunCommands = ['image list a.out', 'image dump sections a.out']
        stopCommands = ['frame variable', 'bt']
        exitCommands = ['expr 2+3', 'expr 3+4']
        self.attach(program=program,
                    attachCommands=attachCommands,
                    initCommands=initCommands,
                    preRunCommands=preRunCommands,
                    stopCommands=stopCommands,
                    exitCommands=exitCommands)

        # Get output from the console. This should contain both the
        # "initCommands" and the "preRunCommands".
        output = self.get_console()
        # Verify all "initCommands" were found in console output
        self.verify_commands('initCommands', output, initCommands)
        # Verify all "preRunCommands" were found in console output
        self.verify_commands('preRunCommands', output, preRunCommands)

        functions = ['main']
        breakpoint_ids = self.set_function_breakpoints(functions)
        self.assertTrue(len(breakpoint_ids) == len(functions),
                        "expect one breakpoint")
        self.continue_to_breakpoints(breakpoint_ids)
        output = self.get_console(timeout=1.0)
        self.verify_commands('stopCommands', output, stopCommands)

        # Continue after launch and hit the "pause()" call and stop the target.
        # Get output from the console. This should contain both the
        # "stopCommands" that were run after we stop.
        self.vscode.request_continue()
        time.sleep(0.5)
        self.vscode.request_pause()
        self.vscode.wait_for_stopped()
        output = self.get_console(timeout=1.0)
        self.verify_commands('stopCommands', output, stopCommands)

        # Continue until the program exits
        self.continue_to_exit()
        # Get output from the console. This should contain both the
        # "exitCommands" that were run after the second breakpoint was hit
        output = self.get_console(timeout=1.0)
        self.verify_commands('exitCommands', output, exitCommands)
<commit_msg>Skip the lldb-vscode attach with waitFor test for now, some build bots are having trouble.<commit_after>"""
Test lldb-vscode setBreakpoints request
"""

from __future__ import print_function

import unittest2
import vscode
from lldbsuite.test.decorators import *
from lldbsuite.test.lldbtest import *
from lldbsuite.test import lldbutil
import lldbvscode_testcase
import os
import shutil
import subprocess
import tempfile
import threading
import time

def spawn_and_wait(program, delay):
    if delay:
        time.sleep(delay)
    process = subprocess.Popen([program],
                               stdin=subprocess.PIPE,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    process.wait()

class TestVSCode_attach(lldbvscode_testcase.VSCodeTestCaseBase):

    mydir = TestBase.compute_mydir(__file__)

    def set_and_hit_breakpoint(self, continueToExit=True):
        source = 'main.c'
        breakpoint1_line = line_number(source, '// breakpoint 1')
        lines = [breakpoint1_line]
        # Set breakoint in the thread function so we can step the threads
        breakpoint_ids = self.set_source_breakpoints(source, lines)
        self.assertEqual(len(breakpoint_ids), len(lines),
                         "expect correct number of breakpoints")
        self.continue_to_breakpoints(breakpoint_ids)
        if continueToExit:
            self.continue_to_exit()

    @skipIfWindows
    @no_debug_info_test
    def test_by_pid(self):
        '''
            Tests attaching to a process by process ID.
        '''
        self.build_and_create_debug_adaptor()
        program = self.getBuildArtifact("a.out")
        self.process = subprocess.Popen([program],
                                        stdin=subprocess.PIPE,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE)
        self.attach(pid=self.process.pid)
        self.set_and_hit_breakpoint(continueToExit=True)

    @skipIfWindows
    @no_debug_info_test
    def test_by_name(self):
        '''
            Tests attaching to a process by process name.
        '''
        self.build_and_create_debug_adaptor()
        orig_program = self.getBuildArtifact("a.out")
        # Since we are going to attach by process name, we need a unique
        # process name that has minimal chance to match a process that is
        # already running. To do this we use tempfile.mktemp() to give us a
        # full path to a location where we can copy our executable. We then
        # run this copy to ensure we don't get the error "more that one
        # process matches 'a.out'".
        program = tempfile.mktemp()
        shutil.copyfile(orig_program, program)
        shutil.copymode(orig_program, program)

        def cleanup():
            if os.path.exists(program):
                os.unlink(program)
        # Execute the cleanup function during test case tear down.
        self.addTearDownHook(cleanup)

        self.process = subprocess.Popen([program],
                                        stdin=subprocess.PIPE,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE)
        # Wait for a bit to ensure the process is launched
        time.sleep(1)
        self.attach(program=program)
        self.set_and_hit_breakpoint(continueToExit=True)

    @skipUnlessDarwin
    @skipIfDarwin # Skip this test for now until we can figure out why tings aren't working on build bots
    @no_debug_info_test
    def test_by_name_waitFor(self):
        '''
            Tests attaching to a process by process name and waiting for the
            next instance of a process to be launched, ingoring all current
            ones.
        '''
        self.build_and_create_debug_adaptor()
        program = self.getBuildArtifact("a.out")
        self.spawn_thread = threading.Thread(target=spawn_and_wait,
                                             args=(program, 1.0,))
        self.spawn_thread.start()
        self.attach(program=program, waitFor=True)
        self.set_and_hit_breakpoint(continueToExit=True)

    @skipIfWindows
    @no_debug_info_test
    def test_commands(self):
        '''
            Tests the "initCommands", "preRunCommands", "stopCommands",
            "exitCommands", and "attachCommands" that can be passed during
            attach.

            "initCommands" are a list of LLDB commands that get executed
            before the targt is created.
            "preRunCommands" are a list of LLDB commands that get executed
            after the target has been created and before the launch.
            "stopCommands" are a list of LLDB commands that get executed each
            time the program stops.
            "exitCommands" are a list of LLDB commands that get executed when
            the process exits
            "attachCommands" are a list of LLDB commands that get executed and
            must have a valid process in the selected target in LLDB after
            they are done executing. This allows custom commands to create any
            kind of debug session.
        '''
        self.build_and_create_debug_adaptor()
        program = self.getBuildArtifact("a.out")
        # Here we just create a target and launch the process as a way to test
        # if we are able to use attach commands to create any kind of a target
        # and use it for debugging
        attachCommands = [
            'target create -d "%s"' % (program),
            'process launch -- arg1'
        ]
        initCommands = ['target list', 'platform list']
        preRunCommands = ['image list a.out', 'image dump sections a.out']
        stopCommands = ['frame variable', 'bt']
        exitCommands = ['expr 2+3', 'expr 3+4']
        self.attach(program=program,
                    attachCommands=attachCommands,
                    initCommands=initCommands,
                    preRunCommands=preRunCommands,
                    stopCommands=stopCommands,
                    exitCommands=exitCommands)

        # Get output from the console. This should contain both the
        # "initCommands" and the "preRunCommands".
        output = self.get_console()
        # Verify all "initCommands" were found in console output
        self.verify_commands('initCommands', output, initCommands)
        # Verify all "preRunCommands" were found in console output
        self.verify_commands('preRunCommands', output, preRunCommands)

        functions = ['main']
        breakpoint_ids = self.set_function_breakpoints(functions)
        self.assertTrue(len(breakpoint_ids) == len(functions),
                        "expect one breakpoint")
        self.continue_to_breakpoints(breakpoint_ids)
        output = self.get_console(timeout=1.0)
        self.verify_commands('stopCommands', output, stopCommands)

        # Continue after launch and hit the "pause()" call and stop the target.
        # Get output from the console. This should contain both the
        # "stopCommands" that were run after we stop.
        self.vscode.request_continue()
        time.sleep(0.5)
        self.vscode.request_pause()
        self.vscode.wait_for_stopped()
        output = self.get_console(timeout=1.0)
        self.verify_commands('stopCommands', output, stopCommands)

        # Continue until the program exits
        self.continue_to_exit()
        # Get output from the console. This should contain both the
        # "exitCommands" that were run after the second breakpoint was hit
        output = self.get_console(timeout=1.0)
        self.verify_commands('exitCommands', output, exitCommands)
<|endoftext|>


<commit_before>import feedparser
import re

class Feed(object):

    _PARAMS = frozenset([
        'author', 'author_detail', 'authors', 'generator', 'generator_detail',
        'image', 'itunes_explicit', 'language', 'link', 'links', 'publisher',
        'publisher_detail', 'rawvoice_frequency', 'rawvoice_rating', 'rights',
        'rights_detail', 'subtitle', 'subtitle_detail', 'summary',
        'summary_detail', 'sy_updatefrequency', 'sy_updateperiod', 'tags',
        'title', 'title_detail', 'updated', 'updated_parsed'
    ])

    def __init__(self, url=None, **kwargs):
        self.url = url
        if url is not None:
            d = feedparser.parse(url)
            feed_obj = d.feed
            self._entries = d.entries
            for key in self._PARAMS:
                setattr(self, key, feed_obj.feed.pop(key))
                setattr(self, key, kwargs.get(key))
            self._custom_keys = feed_obj.feed

    @property
    def entries(self):
        """
        Iterates through self._entries and yields an Entry object for each
        entry.

        :return: Set of all the Entry objects in this Feed object.
        :rtype:  ``frozenset``
        """
        return frozenset(Entry(**entry) for entry in self._entries)

    @property
    def lookup(self):
        """
        :return: Dictionary lookup containing `{id: entry}`.
        :rtype: ``dict``
        """
        return {entry.entry_id: entry for entry in self.entries}

class Entry(object):

    _PARAMS = frozenset([
        'author', 'author_detail', 'authors', 'comments', 'content',
        'guidislink', 'id', 'itunes_duration', 'itunes_explicit', 'link',
        'links', 'published', 'published_parsed', 'slash_comments', 'subtitle',
        'subtitle_detail', 'summary', 'summary_detail', 'tags', 'title',
        'title_detail', 'wfw_commentrss'
    ])

    def __init__(self, **kwargs):
        for key in self._PARAMS:
            setattr(self, key, kwargs.get(key))

    @property
    def entry_id(self, link_regex=r'\d+$'):
        m = re.search(link_regex, self.id)
        return int(m.group())

    @property
    def audio(self):
        """
        :return href: URL of audio file for podcast.
        :rtype  href: ``str``
        """
        return filter(
            lambda x: x.get('rel') == 'enclosure', self.links).next().get(
                'href')
<commit_msg>add parse_feed()<commit_after>from . import mc
import feedparser
import hashlib
import re

class Feed(object):

    _PARAMS = frozenset([
        'author', 'author_detail', 'authors', 'generator', 'generator_detail',
        'image', 'itunes_explicit', 'language', 'link', 'links', 'publisher',
        'publisher_detail', 'rawvoice_frequency', 'rawvoice_rating', 'rights',
        'rights_detail', 'subtitle', 'subtitle_detail', 'summary',
        'summary_detail', 'sy_updatefrequency', 'sy_updateperiod', 'tags',
        'title', 'title_detail', 'updated', 'updated_parsed'
    ])

    def __init__(self, url=None, **kwargs):
        self.url = url
        if url is not None:
            self.parse_feed()

    @property
    def lookup(self):
        """
        :return: Dictionary lookup containing `{id: entry}`.
        :rtype: ``dict``
        """
        return {entry.entry_id: entry for entry in self.entries}

    def parse_feed(self):
        url_hash = hashlib.md5(self.url).hexdigest()

        d = mc.get(url_hash)
        if not d:
            d = feedparser.parse(self.url)
            mc.set(url_hash, d)

        feed_obj = d.feed
        for key in self._PARAMS:
            setattr(self, key, feed_obj.feed.pop(key))
            setattr(self, key, kwargs.get(key))
        self._custom_keys = feed_obj.feed

        self.entries = frozenset(Entry(**entry) for entry in d.entries)

class Entry(object):

    _PARAMS = frozenset([
        'author', 'author_detail', 'authors', 'comments', 'content',
        'guidislink', 'id', 'itunes_duration', 'itunes_explicit', 'link',
        'links', 'published', 'published_parsed', 'slash_comments', 'subtitle',
        'subtitle_detail', 'summary', 'summary_detail', 'tags', 'title',
        'title_detail', 'wfw_commentrss'
    ])

    def __init__(self, **kwargs):
        for key in self._PARAMS:
            setattr(self, key, kwargs.get(key))

    @property
    def entry_id(self, link_regex=r'\d+$'):
        m = re.search(link_regex, self.id)
        return int(m.group())

    @property
    def audio(self):
        """
        :return href: URL of audio file for podcast.
        :rtype  href: ``str``
        """
        return filter(
            lambda x: x.get('rel') == 'enclosure', self.links).next().get(
                'href')
<|endoftext|>


<commit_before>from __future__ import print_function

import argparse
import sys
import smtplib
import re
import os.path
import email
import email.iterators
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText

import subprocess
import six

import markdown
import pynliner

from . import config
from . import __version__

__name__ = 'muttdown'

def convert_one(part, config):
    try:
        text = part.get_payload(decode=True)
        if not isinstance(text, six.text_type):
            # no, I don't know why decode=True sometimes fails to decode.
            text = text.decode('utf-8')
        if not text.startswith('!m'):
            return None
        text = re.sub('\s*!m\s*', '', text, re.M)
        if '\n-- \n' in text:
            pre_signature, signature = text.split('\n-- \n')
            md = markdown.markdown(pre_signature, output_format="html5")
            md += '\n

--
'
            md += '
'.join(signature.split('\n'))
            md += '

'
        else:
            md = markdown.markdown(text)
        if config.css:
            md = '' + md
            md = pynliner.fromString(md)
        message = MIMEText(md, 'html', _charset="UTF-8")
        return message
    except Exception:
        raise
        return None

def _move_headers(source, dest):
    for k, v in source.items():
        # mutt sometimes sticks in a fake bcc header
        if k.lower() == 'bcc':
            del source[k]
        elif not (k.startswith('Content-') or k.startswith('MIME')):
            dest.add_header(k, v)
            del source[k]

def convert_tree(message, config, indent=0):
    """Recursively convert a potentially-multipart tree.

    Returns a tuple of (the converted tree, whether any markdown was found)
    """
    ct = message.get_content_type()
    cs = message.get_content_subtype()
    if not message.is_multipart():
        # we're on a leaf
        converted = None
        disposition = message.get('Content-Disposition', 'inline')
        if disposition == 'inline' and ct in ('text/plain', 'text/markdown'):
            converted = convert_one(message, config)
        if converted is not None:
            new_tree = MIMEMultipart('alternative')
            _move_headers(message, new_tree)
            new_tree.attach(message)
            new_tree.attach(converted)
            return new_tree, True
        return message, False
    else:
        if ct == 'multipart/signed':
            # if this is a multipart/signed message, then let's just
            # recurse into the non-signature part
            for part in message.get_payload():
                if part.get_content_type() != 'application/pgp-signature':
                    return convert_tree(part, config, indent=indent + 1)
        else:
            did_conversion = False
            new_root = MIMEMultipart(cs, message.get_charset())
            if message.preamble:
                new_root.preamble = message.preamble
            _move_headers(message, new_root)
            for part in message.get_payload():
                part, did_this_conversion = convert_tree(part, config, indent=indent + 1)
                did_conversion |= did_this_conversion
                new_root.attach(part)
            return new_root, did_conversion

def process_message(mail, config):
    converted, did_any_markdown = convert_tree(mail, config)
    if 'Bcc' in converted:
        del converted['Bcc']
    return converted

def smtp_connection(c):
    """Create an SMTP connection from a Config object"""
    if c.smtp_ssl:
        klass = smtplib.SMTP_SSL
    else:
        klass = smtplib.SMTP
    conn = klass(c.smtp_host, c.smtp_port, timeout=c.smtp_timeout)
    if not c.smtp_ssl:
        conn.ehlo()
        conn.starttls()
    if c.smtp_username:
        conn.login(c.smtp_username, c.smtp_password)
    return conn

def main():
    parser = argparse.ArgumentParser(version='%s %s' % (__name__, __version__))
    parser.add_argument(
        '-c', '--config_file', default=os.path.expanduser('~/.muttdown.yaml'),
        type=argparse.FileType('r'), required=False,
        help='Path to YAML config file (default %(default)s)'
    )
    parser.add_argument(
        '-p', '--print-message', action='store_true',
        help='Print the translated message to stdout instead of sending it'
    )
    parser.add_argument('-f', '--envelope-from', required=True)
    parser.add_argument(
        '-s', '--sendmail-passthru', action='store_true',
        help='Pass mail through to sendmail for delivery'
    )
    parser.add_argument('addresses', nargs='+')
    args = parser.parse_args()

    c = config.Config()
    try:
        c.load(args.config_file)
    except config.ConfigError as e:
        sys.stderr.write('Error(s) in configuration %s:\n' % args.config_file.name)
        sys.stderr.write(' - %s\n' % e.message)
        sys.stderr.flush()
        return 1

    message = sys.stdin.read()

    mail = email.message_from_string(message)

    rebuilt = process_message(mail, c)
    rebuilt.set_unixfrom(args.envelope_from)

    if args.print_message:
        print(rebuilt.as_string())
    elif args.sendmail_passthru:
        cmd = [c.sendmail, '-f', args.envelope_from] + args.addresses

        proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, shell=False)
        proc.communicate(rebuilt.as_string())
        return proc.returncode
    else:
        conn = smtp_connection(c)
        conn.sendmail(args.envelope_from, args.addresses, rebuilt.as_string())
        conn.quit()
    return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Split on whitespace in sendmail comman<commit_after>from __future__ import print_function

import argparse
import sys
import smtplib
import re
import os.path
import email
import email.iterators
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText

import subprocess
import six

import markdown
import pynliner

from . import config
from . import __version__

__name__ = 'muttdown'

def convert_one(part, config):
    try:
        text = part.get_payload(decode=True)
        if not isinstance(text, six.text_type):
            # no, I don't know why decode=True sometimes fails to decode.
            text = text.decode('utf-8')
        if not text.startswith('!m'):
            return None
        text = re.sub('\s*!m\s*', '', text, re.M)
        if '\n-- \n' in text:
            pre_signature, signature = text.split('\n-- \n')
            md = markdown.markdown(pre_signature, output_format="html5")
            md += '\n

--
'
            md += '
'.join(signature.split('\n'))
            md += '

'
        else:
            md = markdown.markdown(text)
        if config.css:
            md = '' + md
            md = pynliner.fromString(md)
        message = MIMEText(md, 'html', _charset="UTF-8")
        return message
    except Exception:
        raise
        return None

def _move_headers(source, dest):
    for k, v in source.items():
        # mutt sometimes sticks in a fake bcc header
        if k.lower() == 'bcc':
            del source[k]
        elif not (k.startswith('Content-') or k.startswith('MIME')):
            dest.add_header(k, v)
            del source[k]

def convert_tree(message, config, indent=0):
    """Recursively convert a potentially-multipart tree.

    Returns a tuple of (the converted tree, whether any markdown was found)
    """
    ct = message.get_content_type()
    cs = message.get_content_subtype()
    if not message.is_multipart():
        # we're on a leaf
        converted = None
        disposition = message.get('Content-Disposition', 'inline')
        if disposition == 'inline' and ct in ('text/plain', 'text/markdown'):
            converted = convert_one(message, config)
        if converted is not None:
            new_tree = MIMEMultipart('alternative')
            _move_headers(message, new_tree)
            new_tree.attach(message)
            new_tree.attach(converted)
            return new_tree, True
        return message, False
    else:
        if ct == 'multipart/signed':
            # if this is a multipart/signed message, then let's just
            # recurse into the non-signature part
            for part in message.get_payload():
                if part.get_content_type() != 'application/pgp-signature':
                    return convert_tree(part, config, indent=indent + 1)
        else:
            did_conversion = False
            new_root = MIMEMultipart(cs, message.get_charset())
            if message.preamble:
                new_root.preamble = message.preamble
            _move_headers(message, new_root)
            for part in message.get_payload():
                part, did_this_conversion = convert_tree(part, config, indent=indent + 1)
                did_conversion |= did_this_conversion
                new_root.attach(part)
            return new_root, did_conversion

def process_message(mail, config):
    converted, did_any_markdown = convert_tree(mail, config)
    if 'Bcc' in converted:
        del converted['Bcc']
    return converted

def smtp_connection(c):
    """Create an SMTP connection from a Config object"""
    if c.smtp_ssl:
        klass = smtplib.SMTP_SSL
    else:
        klass = smtplib.SMTP
    conn = klass(c.smtp_host, c.smtp_port, timeout=c.smtp_timeout)
    if not c.smtp_ssl:
        conn.ehlo()
        conn.starttls()
    if c.smtp_username:
        conn.login(c.smtp_username, c.smtp_password)
    return conn

def main():
    parser = argparse.ArgumentParser(version='%s %s' % (__name__, __version__))
    parser.add_argument(
        '-c', '--config_file', default=os.path.expanduser('~/.muttdown.yaml'),
        type=argparse.FileType('r'), required=False,
        help='Path to YAML config file (default %(default)s)'
    )
    parser.add_argument(
        '-p', '--print-message', action='store_true',
        help='Print the translated message to stdout instead of sending it'
    )
    parser.add_argument('-f', '--envelope-from', required=True)
    parser.add_argument(
        '-s', '--sendmail-passthru', action='store_true',
        help='Pass mail through to sendmail for delivery'
    )
    parser.add_argument('addresses', nargs='+')
    args = parser.parse_args()

    c = config.Config()
    try:
        c.load(args.config_file)
    except config.ConfigError as e:
        sys.stderr.write('Error(s) in configuration %s:\n' % args.config_file.name)
        sys.stderr.write(' - %s\n' % e.message)
        sys.stderr.flush()
        return 1

    message = sys.stdin.read()

    mail = email.message_from_string(message)

    rebuilt = process_message(mail, c)
    rebuilt.set_unixfrom(args.envelope_from)

    if args.print_message:
        print(rebuilt.as_string())
    elif args.sendmail_passthru:
        cmd = c.sendmail.split() + ['-f', args.envelope_from] + args.addresses

        proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, shell=False)
        proc.communicate(rebuilt.as_string())
        return proc.returncode
    else:
        conn = smtp_connection(c)
        conn.sendmail(args.envelope_from, args.addresses, rebuilt.as_string())
        conn.quit()
    return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>import os

import numpy as np

def softmax(logits):
    """Transforms predictions into probability values.

    Parameters
    ----------
    logits : array_like
        The logits predicted by the model.

    Returns
    -------
    `numpy.ndarray`
        Probability values corresponding to the logits.
    """

    assert logits.ndim == 1

    # for numerical reasons we subtract the max logit
    # (mathematically it doesn't matter!)
    # otherwise exp(logits) might become too large or too small
    logits = logits - np.max(logits)
    e = np.exp(logits)
    return e / np.sum(e)

def crossentropy(label, logits):
    """Calculates the cross-entropy.

    Parameters
    ----------
    logits : array_like
        The logits predicted by the model.
    label : int
        The label describing the target distribution.

    Returns
    -------
    float
        The cross-entropy between softmax(logits) and onehot(label).

    """

    assert logits.ndim == 1

    # for numerical reasons we subtract the max logit
    # (mathematically it doesn't matter!)
    # otherwise exp(logits) might become too large or too small
    logits = logits - np.max(logits)
    e = np.exp(logits)
    s = np.sum(e)
    ce = np.log(s) - logits[label]
    return ce

def batch_crossentropy(label, logits):
    """Calculates the cross-entropy for a batch of logits.

    Parameters
    ----------
    logits : array_like
        The logits predicted by the model for a batch of inputs.
    label : int
        The label describing the target distribution.

    Returns
    -------
    np.ndarray
        The cross-entropy between softmax(logits[i]) and onehot(label)
        for all i.

    """

    assert logits.ndim == 2

    # for numerical reasons we subtract the max logit
    # (mathematically it doesn't matter!)
    # otherwise exp(logits) might become too large or too small
    logits = logits - np.max(logits, axis=1, keepdims=True)
    e = np.exp(logits)
    s = np.sum(e, axis=1)
    ces = np.log(s) - logits[:, label]
    return ces

def binarize(x, values, threshold=None, included_in="upper"):
    """Binarizes the values of x.

    Parameters
    ----------
    values : tuple of two floats
        The lower and upper value to which the inputs are mapped.
    threshold : float
        The threshold; defaults to (values[0] + values[1]) / 2 if None.
    included_in : str
        Whether the threshold value itself belongs to the lower or
        upper interval.

    """
    lower, upper = values

    if threshold is None:
        threshold = (lower + upper) / 2.0

    x = x.copy()
    if included_in == "lower":
        x[x <= threshold] = lower
        x[x > threshold] = upper
    elif included_in == "upper":
        x[x < threshold] = lower
        x[x >= threshold] = upper
    else:
        raise ValueError('included_in must be "lower" or "upper"')
    return x

def imagenet_example(shape=(224, 224), data_format="channels_last", bounds=(0, 255)):
    """ Returns an example image and its imagenet class label.

    Parameters
    ----------
    shape : list of integers
        The shape of the returned image.
    data_format : str
        "channels_first" or "channels_last"
    bounds : tuple
        smallest and largest allowed pixel value

    Returns
    -------
    image : array_like
        The example image.

    label : int
        The imagenet label associated with the image.

    NOTE: This function is deprecated and will be removed in the future.
    """
    assert len(shape) == 2
    assert data_format in ["channels_first", "channels_last"]

    from PIL import Image

    path = os.path.join(os.path.dirname(__file__), "example.png")
    image = Image.open(path)
    image = image.resize(shape)
    image = np.asarray(image, dtype=np.float32)
    image = image[:, :, :3]
    assert image.shape == shape + (3,)
    if data_format == "channels_first":
        image = np.transpose(image, (2, 0, 1))
    if bounds != (0, 255):
        image = image / 255 * (bounds[1] - bounds[0]) + bounds[0]
    return image, 282

def samples(
    dataset="imagenet",
    index=0,
    batchsize=1,
    shape=(224, 224),
    data_format="channels_last",
    bounds=(0, 255),
):
    """ Returns a batch of example images and the corresponding labels

    Parameters
    ----------
    dataset : string
        The data set to load (options: imagenet, mnist, cifar10,
        cifar100, fashionMNIST)
    index : int
        For each data set 20 example images exist. The returned batch
        contains the images with index [index, index + 1, index + 2, ...]
    batchsize : int
        Size of batch.
    shape : list of integers
        The shape of the returned image (only relevant for Imagenet).
    data_format : str
        "channels_first" or "channels_last"
    bounds : tuple
        smallest and largest allowed pixel value

    Returns
    -------
    images : array_like
        The batch of example images

    labels : array of int
        The labels associated with the images.

    """
    from PIL import Image

    images, labels = [], []
    basepath = os.path.dirname(__file__)
    samplepath = os.path.join(basepath, "data")
    files = os.listdir(samplepath)

    for idx in range(index, index + batchsize):
        i = idx % 20

        # get filename and label
        file = [n for n in files if "{}_{:02d}_".format(dataset, i) in n][0]
        label = int(file.split(".")[0].split("_")[-1])

        # open file
        path = os.path.join(samplepath, file)
        image = Image.open(path)

        if dataset == "imagenet":
            image = image.resize(shape)

        image = np.asarray(image, dtype=np.float32)

        if dataset != "mnist" and data_format == "channels_first":
            image = np.transpose(image, (2, 0, 1))

        images.append(image)
        labels.append(label)

    images = np.stack(images)
    labels = np.array(labels)

    if bounds != (0, 255):
        images = images / 255 * (bounds[1] - bounds[0]) + bounds[0]
    return images, labels

def onehot_like(a, index, value=1):
    """Creates an array like a, with all values
    set to 0 except one.

    Parameters
    ----------
    a : array_like
        The returned one-hot array will have the same shape
        and dtype as this array
    index : int
        The index that should be set to `value`
    value : single value compatible with a.dtype
        The value to set at the given index

    Returns
    -------
    `numpy.ndarray`
        One-hot array with the given value at the given
        location and zeros everywhere else.

    """

    x = np.zeros_like(a)
    x[index] = value
    return x
<commit_msg>fixes #279<commit_after>import os
import warnings

import numpy as np

def softmax(logits):
    """Transforms predictions into probability values.

    Parameters
    ----------
    logits : array_like
        The logits predicted by the model.

    Returns
    -------
    `numpy.ndarray`
        Probability values corresponding to the logits.
    """

    assert logits.ndim == 1

    # for numerical reasons we subtract the max logit
    # (mathematically it doesn't matter!)
    # otherwise exp(logits) might become too large or too small
    logits = logits - np.max(logits)
    e = np.exp(logits)
    return e / np.sum(e)

def crossentropy(label, logits):
    """Calculates the cross-entropy.

    Parameters
    ----------
    logits : array_like
        The logits predicted by the model.
    label : int
        The label describing the target distribution.

    Returns
    -------
    float
        The cross-entropy between softmax(logits) and onehot(label).

    """

    assert logits.ndim == 1

    # for numerical reasons we subtract the max logit
    # (mathematically it doesn't matter!)
    # otherwise exp(logits) might become too large or too small
    logits = logits - np.max(logits)
    e = np.exp(logits)
    s = np.sum(e)
    ce = np.log(s) - logits[label]
    return ce

def batch_crossentropy(label, logits):
    """Calculates the cross-entropy for a batch of logits.

    Parameters
    ----------
    logits : array_like
        The logits predicted by the model for a batch of inputs.
    label : int
        The label describing the target distribution.

    Returns
    -------
    np.ndarray
        The cross-entropy between softmax(logits[i]) and onehot(label)
        for all i.

    """

    assert logits.ndim == 2

    # for numerical reasons we subtract the max logit
    # (mathematically it doesn't matter!)
    # otherwise exp(logits) might become too large or too small
    logits = logits - np.max(logits, axis=1, keepdims=True)
    e = np.exp(logits)
    s = np.sum(e, axis=1)
    ces = np.log(s) - logits[:, label]
    return ces

def binarize(x, values, threshold=None, included_in="upper"):
    """Binarizes the values of x.

    Parameters
    ----------
    values : tuple of two floats
        The lower and upper value to which the inputs are mapped.
    threshold : float
        The threshold; defaults to (values[0] + values[1]) / 2 if None.
    included_in : str
        Whether the threshold value itself belongs to the lower or
        upper interval.

    """
    lower, upper = values

    if threshold is None:
        threshold = (lower + upper) / 2.0

    x = x.copy()
    if included_in == "lower":
        x[x <= threshold] = lower
        x[x > threshold] = upper
    elif included_in == "upper":
        x[x < threshold] = lower
        x[x >= threshold] = upper
    else:
        raise ValueError('included_in must be "lower" or "upper"')
    return x

def imagenet_example(shape=(224, 224), data_format="channels_last", bounds=(0, 255)):
    """ Returns an example image and its imagenet class label.

    Parameters
    ----------
    shape : list of integers
        The shape of the returned image.
    data_format : str
        "channels_first" or "channels_last"
    bounds : tuple
        smallest and largest allowed pixel value

    Returns
    -------
    image : array_like
        The example image.

    label : int
        The imagenet label associated with the image.

    NOTE: This function is deprecated and will be removed in the future.
    """
    assert len(shape) == 2
    assert data_format in ["channels_first", "channels_last"]

    from PIL import Image

    path = os.path.join(os.path.dirname(__file__), "example.png")
    image = Image.open(path)
    image = image.resize(shape)
    image = np.asarray(image, dtype=np.float32)
    image = image[:, :, :3]
    assert image.shape == shape + (3,)
    if data_format == "channels_first":
        image = np.transpose(image, (2, 0, 1))
    if bounds != (0, 255):
        image = image / 255 * (bounds[1] - bounds[0]) + bounds[0]
    return image, 282

def samples(
    dataset="imagenet",
    index=0,
    batchsize=1,
    shape=(224, 224),
    data_format="channels_last",
    bounds=(0, 255),
):
    """ Returns a batch of example images and the corresponding labels

    Parameters
    ----------
    dataset : string
        The data set to load (options: imagenet, mnist, cifar10,
        cifar100, fashionMNIST)
    index : int
        For each data set 20 example images exist. The returned batch
        contains the images with index [index, index + 1, index + 2, ...]
    batchsize : int
        Size of batch.
    shape : list of integers
        The shape of the returned image (only relevant for Imagenet).
    data_format : str
        "channels_first" or "channels_last"
    bounds : tuple
        smallest and largest allowed pixel value

    Returns
    -------
    images : array_like
        The batch of example images

    labels : array of int
        The labels associated with the images.

    """
    from PIL import Image

    images, labels = [], []
    basepath = os.path.dirname(__file__)
    samplepath = os.path.join(basepath, "data")
    files = os.listdir(samplepath)

    if batchsize > 20:
        warnings.warn(
            "foolbox.utils.samples() has only 20 samples and repeats itself if batchsize > 20"
        )

    for idx in range(index, index + batchsize):
        i = idx % 20

        # get filename and label
        file = [n for n in files if "{}_{:02d}_".format(dataset, i) in n][0]
        label = int(file.split(".")[0].split("_")[-1])

        # open file
        path = os.path.join(samplepath, file)
        image = Image.open(path)

        if dataset == "imagenet":
            image = image.resize(shape)

        image = np.asarray(image, dtype=np.float32)

        if dataset != "mnist" and data_format == "channels_first":
            image = np.transpose(image, (2, 0, 1))

        images.append(image)
        labels.append(label)

    images = np.stack(images)
    labels = np.array(labels)

    if bounds != (0, 255):
        images = images / 255 * (bounds[1] - bounds[0]) + bounds[0]
    return images, labels

def onehot_like(a, index, value=1):
    """Creates an array like a, with all values
    set to 0 except one.

    Parameters
    ----------
    a : array_like
        The returned one-hot array will have the same shape
        and dtype as this array
    index : int
        The index that should be set to `value`
    value : single value compatible with a.dtype
        The value to set at the given index

    Returns
    -------
    `numpy.ndarray`
        One-hot array with the given value at the given
        location and zeros everywhere else.

    """

    x = np.zeros_like(a)
    x[index] = value
    return x
<|endoftext|>


<commit_before>
#! /usr/bin/env python
"""Interface to the Dakota iterative systems analysis toolkit."""

import subprocess

class Dakota(object):
    """Describe and run a Dakota experiment."""

    def __init__(self):
        """Create a new Dakota experiment with default parameters."""
        self.model = None
        self.input_file = 'dakota.in'
        self.output_file = 'dakota.out'
        self.data_file = 'dakota.dat'

        self.method = None

        self.variable_type = 'continuous_design'
        self.n_variables = 0
        self.variable_descriptors = []

        self.analysis_driver = 'run_model.py'
        self.parameters_file = 'params.in'
        self.results_file = 'results.out'

        self.n_responses = 0
        self.response_descriptors = []
        self.response_files = []
        self.response_statistics = []

    def run(self):
        """Run the specified Dakota experiment."""
        print(self.method)
        print(self.analysis_driver)

    def write(self):
        """Write a Dakota input file."""
        pass

def get_labels(params_file):
    """Extract labels from a Dakota parameters file."""
    labels = []
    try:
        with open(params_file, 'r') as fp:
            for line in fp:
                if re.search('ASV_', line):
                    labels.append(''.join(re.findall(':(\S+)', line)))
    except IOError:
        return None
    else:
        return(labels)

def write_results(results_file, array, labels):
    """Write a Dakota results file from an input numpy array."""
    try:
        with open(results_file, 'w') as fp:
            for i in range(len(array)):
                fp.write('{0s}\t{1}\n'.format(array[i], labels[i]))
    except IOError:
        raise
<commit_msg>Add get_analysis_components() to the dakota module<commit_after>
#! /usr/bin/env python
"""Interface to the Dakota iterative systems analysis toolkit."""

import subprocess

class Dakota(object):
    """Describe and run a Dakota experiment."""

    def __init__(self):
        """Create a new Dakota experiment with default parameters."""
        self.model = None
        self.input_file = 'dakota.in'
        self.output_file = 'dakota.out'
        self.data_file = 'dakota.dat'

        self.method = None

        self.variable_type = 'continuous_design'
        self.n_variables = 0
        self.variable_descriptors = []

        self.analysis_driver = 'run_model.py'
        self.parameters_file = 'params.in'
        self.results_file = 'results.out'

        self.n_responses = 0
        self.response_descriptors = []
        self.response_files = []
        self.response_statistics = []

    def run(self):
        """Run the specified Dakota experiment."""
        print(self.method)
        print(self.analysis_driver)

    def write(self):
        """Write a Dakota input file."""
        pass

def get_labels(params_file):
    """Extract labels from a Dakota parameters file."""
    labels = []
    try:
        with open(params_file, 'r') as fp:
            for line in fp:
                if re.search('ASV_', line):
                    labels.append(''.join(re.findall(':(\S+)', line)))
    except IOError:
        return None
    else:
        return(labels)

def get_analysis_components(params_file):
    """Extract the analysis components from a Dakota parameters file.

    The analysis components are returned as a list. First is the name
    of the model being run by Dakota, followed by dicts containing an
    output file to analyze and the statistic to apply to the file.

    Parameters
    ----------
    params_file : str
      The path to a Dakota parameters file.

    Returns
    -------
    list
      A list of analysis components for the Dakota experiment.

    Examples
    --------
    Extract the analysis components from a Dakota parameters file:

    >>> ac = get_analysis_components(params_file)
    >>> ac.pop(0)
    'hydrotrend'
    >>> ac.pop(0)
    {'file': 'HYDROASCII.QS', 'statistic': 'median'}

    Notes
    -----
    The syntax expected by this function is defined in the Dakota
    input file; e.g., for the example cited above, the 'interface'
    section of the input file contains the line:

      analysis_components = 'hydrotrend' 'HYDROASCII.QS:median'

    """
    ac = []
    try:
        with open(params_file, 'r') as fp:
            for line in fp:
                if re.search('AC_1', line):
                    ac.append(line.split('AC_1')[0].strip())
                elif re.search('AC_', line):
                    parts = re.split(':', re.split('AC_', line)[0])
                    ac.append({'file':parts[0].strip(),
                               'statistic':parts[1].strip()})
    except IOError:
        return None
    else:
        return(ac)

def write_results(results_file, array, labels):
    """Write a Dakota results file from an input numpy array."""
    try:
        with open(results_file, 'w') as fp:
            for i in range(len(array)):
                fp.write('{0s}\t{1}\n'.format(array[i], labels[i]))
    except IOError:
        raise
<|endoftext|>