<commit_before>from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
import argparse
import sys

import chainer
from chainer import optimizers
from chainer import cuda
import gym
import numpy as np
from gym import spaces

from chainerrl import policy
from chainerrl import q_function
from chainerrl import replay_buffer
from chainerrl.agents.ddpg import DDPG, DDPGModel
from chainerrl.experiments.evaluator import eval_performance
from chainerrl.experiments.prepare_output_dir import prepare_output_dir
from chainerrl.experiments.train_agent import train_agent_with_evaluation
from chainerrl.explorers.additive_gaussian import AdditiveGaussian
from chainerrl.explorers.epsilon_greedy import LinearDecayEpsilonGreedy
from chainerrl.misc import env_modifiers
from chainerrl.misc import random_seed
from chainerrl.misc.init_like_torch import init_like_torch

def main():
    import logging
    logging.basicConfig(level=logging.DEBUG)

    parser = argparse.ArgumentParser()
    parser.add_argument('--outdir', type=str, default='out')
    parser.add_argument('--env', type=str, default='Humanoid-v1')
    parser.add_argument('--seed', type=int, default=None)
    parser.add_argument('--gpu', type=int, default=0)
    parser.add_argument('--final-exploration-steps',
                        type=int, default=10 ** 6)
    parser.add_argument('--start-epsilon', type=float, default=0.4)
    parser.add_argument('--end-epsilon', type=float, default=0.3)
    parser.add_argument('--load', type=str, default='')
    parser.add_argument('--steps', type=int, default=10 ** 7)
    parser.add_argument('--replay-start-size', type=int, default=5000)
    parser.add_argument('--target-update-frequency',
                        type=int, default=5000)
    parser.add_argument('--update-frequency', type=int, default=1000)
    parser.add_argument('--eval-n-runs', type=int, default=100)
    parser.add_argument('--eval-frequency', type=int, default=10 ** 5)
    #parser.add_argument('--n-x-layers', type=int, default=10)
    #parser.add_argument('--n-hidden-channels', type=int, default=4)
    parser.add_argument('--gamma', type=float, default=0.995)
    parser.add_argument('--minibatch-size', type=int, default=200)
    parser.add_argument('--render', action='store_true')
    parser.add_argument('--demo', action='store_true')
    parser.add_argument('--reward-scale-factor', type=float, default=1e-2)
    parser.set_defaults(window_visible=True)
    args = parser.parse_args()

    if args.seed is not None:
        random_seed.set_random_seed(args.seed)

    def clip_action_filter(a):
        if isinstance(a, cuda.cupy.ndarray):
            a = cuda.to_cpu(a)
        return np.clip(a, action_space.low, action_space.high)

    def reward_filter(r):
        return r * args.reward_scale_factor

    def make_env():
        env = gym.make(args.env)
        #env.monitor.start('/tmp', force=True, seed=0)
        timestep_limit = env.spec.timestep_limit
        env_modifiers.make_timestep_limited(env, timestep_limit)
        if isinstance(env.action_space, spaces.Box):
            env_modifiers.make_action_filtered(env, clip_action_filter)
        env_modifiers.make_reward_filtered(env, reward_filter)
        if args.render:
            env_modifiers.make_rendered(env)

        def __exit__(self, *args):
            pass
        env.__exit__ = __exit__
        return env

    env = make_env()
    # timestep_limit = sample_env.spec.timestep_limit
    obs_size = np.asarray(env.observation_space.shape).prod()
    action_space = env.action_space

    args.outdir = prepare_output_dir(args, args.outdir, argv=sys.argv)
    print('Output files are saved in {}'.format(args.outdir))

    action_size = np.asarray(action_space.shape).prod()
    model = DDPGModel(
        q_func=q_function.FCBNSAQFunction(
            obs_size, action_size, n_hidden_channels=300, n_hidden_layers=3),
        policy=policy.FCBNDeterministicPolicy(
            obs_size, action_size=action_size,
            n_hidden_layers=2, n_hidden_channels=300,
            min_action=-1, max_action=1, bound_action=True)
    )
    init_like_torch(model['q_function'])
    init_like_torch(model['policy'])
    opt_a = optimizers.Adam(alpha=0.001)
    opt_c = optimizers.Adam(alpha=0.0005)
    opt_a.setup(model['policy'])
    opt_c.setup(model['q_function'])
    opt_a.add_hook(chainer.optimizer.GradientClipping(1.0), 'hook_a')
    opt_c.add_hook(chainer.optimizer.GradientClipping(1.0), 'hook_c')
    opt_c.add_hook(chainer.optimizer.WeightDecay(1e-4))

    rbuf = replay_buffer.ReplayBuffer(5 * 10 ** 5)

    def phi(obs):
        return obs.astype(np.float32)

    def random_action():
        a = action_space.sample()
        if isinstance(a, np.ndarray):
            a = a.astype(np.float32)
        return a

    explorer = AdditiveGaussian(scale=0.4)
    agent = DDPG(model, opt_a, opt_c, rbuf, gamma=args.gamma,
                 explorer=explorer, replay_start_size=args.replay_start_size,
                 target_update_frequency=args.target_update_frequency,
                 update_frequency=args.update_frequency,
                 n_times_update=10,
                 phi=phi, gpu=args.gpu, minibatch_size=args.minibatch_size)
    agent.logger.setLevel(logging.DEBUG)

    if len(args.load) > 0:
        agent.load(args.load)

    if args.demo:
        mean, median, stdev = eval_performance(
            env=env,
            agent=agent,
            n_runs=args.eval_n_runs)
        print('n_runs: {} mean: {} median: {} stdev'.format(
            args.eval_n_runs, mean, median, stdev))
    else:
        train_agent_with_evaluation(
            agent=agent, env=env, steps=args.steps,
            eval_n_runs=args.eval_n_runs, eval_frequency=args.eval_frequency,
            outdir=args.outdir)

if __name__ == '__main__':
    main()
<commit_msg>Use action_space instead of specifying by hand<commit_after>from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
import argparse
import sys

import chainer
from chainer import optimizers
from chainer import cuda
import gym
import numpy as np
from gym import spaces

from chainerrl import policy
from chainerrl import q_function
from chainerrl import replay_buffer
from chainerrl.agents.ddpg import DDPG, DDPGModel
from chainerrl.experiments.evaluator import eval_performance
from chainerrl.experiments.prepare_output_dir import prepare_output_dir
from chainerrl.experiments.train_agent import train_agent_with_evaluation
from chainerrl.explorers.additive_gaussian import AdditiveGaussian
from chainerrl.explorers.epsilon_greedy import LinearDecayEpsilonGreedy
from chainerrl.misc import env_modifiers
from chainerrl.misc import random_seed
from chainerrl.misc.init_like_torch import init_like_torch

def main():
    import logging
    logging.basicConfig(level=logging.DEBUG)

    parser = argparse.ArgumentParser()
    parser.add_argument('--outdir', type=str, default='out')
    parser.add_argument('--env', type=str, default='Humanoid-v1')
    parser.add_argument('--seed', type=int, default=None)
    parser.add_argument('--gpu', type=int, default=0)
    parser.add_argument('--final-exploration-steps',
                        type=int, default=10 ** 6)
    parser.add_argument('--start-epsilon', type=float, default=0.4)
    parser.add_argument('--end-epsilon', type=float, default=0.3)
    parser.add_argument('--load', type=str, default='')
    parser.add_argument('--steps', type=int, default=10 ** 7)
    parser.add_argument('--replay-start-size', type=int, default=5000)
    parser.add_argument('--target-update-frequency',
                        type=int, default=5000)
    parser.add_argument('--update-frequency', type=int, default=1000)
    parser.add_argument('--eval-n-runs', type=int, default=100)
    parser.add_argument('--eval-frequency', type=int, default=10 ** 5)
    #parser.add_argument('--n-x-layers', type=int, default=10)
    #parser.add_argument('--n-hidden-channels', type=int, default=4)
    parser.add_argument('--gamma', type=float, default=0.995)
    parser.add_argument('--minibatch-size', type=int, default=200)
    parser.add_argument('--render', action='store_true')
    parser.add_argument('--demo', action='store_true')
    parser.add_argument('--reward-scale-factor', type=float, default=1e-2)
    parser.set_defaults(window_visible=True)
    args = parser.parse_args()

    if args.seed is not None:
        random_seed.set_random_seed(args.seed)

    def clip_action_filter(a):
        if isinstance(a, cuda.cupy.ndarray):
            a = cuda.to_cpu(a)
        return np.clip(a, action_space.low, action_space.high)

    def reward_filter(r):
        return r * args.reward_scale_factor

    def make_env():
        env = gym.make(args.env)
        #env.monitor.start('/tmp', force=True, seed=0)
        timestep_limit = env.spec.timestep_limit
        env_modifiers.make_timestep_limited(env, timestep_limit)
        if isinstance(env.action_space, spaces.Box):
            env_modifiers.make_action_filtered(env, clip_action_filter)
        env_modifiers.make_reward_filtered(env, reward_filter)
        if args.render:
            env_modifiers.make_rendered(env)

        def __exit__(self, *args):
            pass
        env.__exit__ = __exit__
        return env

    env = make_env()
    # timestep_limit = sample_env.spec.timestep_limit
    obs_size = np.asarray(env.observation_space.shape).prod()
    action_space = env.action_space

    args.outdir = prepare_output_dir(args, args.outdir, argv=sys.argv)
    print('Output files are saved in {}'.format(args.outdir))

    action_size = np.asarray(action_space.shape).prod()
    model = DDPGModel(
        q_func=q_function.FCBNSAQFunction(
            obs_size, action_size, n_hidden_channels=300, n_hidden_layers=3),
        policy=policy.FCBNDeterministicPolicy(
            obs_size, action_size=action_size,
            n_hidden_layers=2, n_hidden_channels=300,
            min_action=action_space.low, max_action=action_space.high,
            bound_action=True)
    )
    init_like_torch(model['q_function'])
    init_like_torch(model['policy'])
    opt_a = optimizers.Adam(alpha=0.001)
    opt_c = optimizers.Adam(alpha=0.0005)
    opt_a.setup(model['policy'])
    opt_c.setup(model['q_function'])
    opt_a.add_hook(chainer.optimizer.GradientClipping(1.0), 'hook_a')
    opt_c.add_hook(chainer.optimizer.GradientClipping(1.0), 'hook_c')
    opt_c.add_hook(chainer.optimizer.WeightDecay(1e-4))

    rbuf = replay_buffer.ReplayBuffer(5 * 10 ** 5)

    def phi(obs):
        return obs.astype(np.float32)

    def random_action():
        a = action_space.sample()
        if isinstance(a, np.ndarray):
            a = a.astype(np.float32)
        return a

    explorer = AdditiveGaussian(scale=0.4)
    agent = DDPG(model, opt_a, opt_c, rbuf, gamma=args.gamma,
                 explorer=explorer, replay_start_size=args.replay_start_size,
                 target_update_frequency=args.target_update_frequency,
                 update_frequency=args.update_frequency,
                 n_times_update=10,
                 phi=phi, gpu=args.gpu, minibatch_size=args.minibatch_size)
    agent.logger.setLevel(logging.DEBUG)

    if len(args.load) > 0:
        agent.load(args.load)

    if args.demo:
        mean, median, stdev = eval_performance(
            env=env,
            agent=agent,
            n_runs=args.eval_n_runs)
        print('n_runs: {} mean: {} median: {} stdev'.format(
            args.eval_n_runs, mean, median, stdev))
    else:
        train_agent_with_evaluation(
            agent=agent, env=env, steps=args.steps,
            eval_n_runs=args.eval_n_runs, eval_frequency=args.eval_frequency,
            outdir=args.outdir)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
mbed SDK
Copyright (c) 2011-2015 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
"""

import re
import sys
from time import time
from Queue import Queue, Empty
from threading import Thread
from subprocess import call, Popen, PIPE

from mbed_greentea_log import gt_log
from mbed_greentea_log import gt_log_tab
from mbed_greentea_log import gt_log_err
from mbed_greentea_log import gt_bright

# Return codes for test script
TEST_RESULT_OK = "OK"
TEST_RESULT_FAIL = "FAIL"
TEST_RESULT_ERROR = "ERROR"
TEST_RESULT_UNDEF = "UNDEF"
TEST_RESULT_IOERR_COPY = "IOERR_COPY"
TEST_RESULT_IOERR_DISK = "IOERR_DISK"
TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
TEST_RESULT_TIMEOUT = "TIMEOUT"
TEST_RESULT_NO_IMAGE = "NO_IMAGE"
TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"

TEST_RESULTS = [TEST_RESULT_OK,
                TEST_RESULT_FAIL,
                TEST_RESULT_ERROR,
                TEST_RESULT_UNDEF,
                TEST_RESULT_IOERR_COPY,
                TEST_RESULT_IOERR_DISK,
                TEST_RESULT_IOERR_SERIAL,
                TEST_RESULT_TIMEOUT,
                TEST_RESULT_NO_IMAGE,
                TEST_RESULT_MBED_ASSERT,
                TEST_RESULT_BUILD_FAILED
                ]

TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
                       "failure" : TEST_RESULT_FAIL,
                       "error" : TEST_RESULT_ERROR,
                       "end" : TEST_RESULT_UNDEF,
                       "ioerr_copy" : TEST_RESULT_IOERR_COPY,
                       "ioerr_disk" : TEST_RESULT_IOERR_DISK,
                       "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
                       "timeout" : TEST_RESULT_TIMEOUT,
                       "no_image" : TEST_RESULT_NO_IMAGE,
                       "mbed_assert" : TEST_RESULT_MBED_ASSERT,
                       "build_failed" : TEST_RESULT_BUILD_FAILED
                       }

RE_DETECT_TESTCASE_RESULT = re.compile("\\{(" + "|".join(TEST_RESULT_MAPPING.keys()) + ")\\}")

def run_host_test(image_path,
                  disk,
                  port,
                  duration=10,
                  micro=None,
                  reset=None,
                  reset_tout=None,
                  verbose=False,
                  copy_method=None,
                  program_cycle_s=None,
                  digest_source=None,
                  json_test_cfg=None,
                  run_app=None):
    """! This function runs host test supervisor (executes mbedhtrun) and checks output from host test process.
    @return Tuple with test results, test output and test duration times
    @param image_path Path to binary file for flashing
    @param disk Currently mounted mbed-enabled devices disk (mount point)
    @param port Currently mounted mbed-enabled devices serial port (console)
    @param duration Test case timeout
    @param micro Mbed-nebaled device name
    @param reset Reset type
    @param reset_tout Reset timeout (sec)
    @param verbose Verbose mode flag
    @param copy_method Copy method type (name)
    @param program_cycle_s Wait after flashing delay (sec)
    @param json_test_cfg Additional test configuration file path passed to host tests in JSON format
    @param run_app Run application mode flag (we run application and grab serial port data)
    @param digest_source if None mbedhtrun will be executed. If 'stdin',
                           stdin will be used via StdInObserver or file (if
                           file name was given as switch option)
    """

    class StdInObserver(Thread):
        """ Process used to read stdin only as console input from MUT
        """
        def __init__(self):
            Thread.__init__(self)
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            while self.active:
                c = sys.stdin.read(1)
                self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception:
                pass

    class FileObserver(Thread):
        """ process used to read file content as console input from MUT
        """
        def __init__(self, filename):
            Thread.__init__(self)
            self.filename = filename
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            with open(self.filename) as f:
                while self.active:
                    c = f.read(1)
                    self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception:
                pass

    class ProcessObserver(Thread):
        """ Default process used to observe stdout of another process as console input from MUT
        """
        def __init__(self, proc):
            Thread.__init__(self)
            self.proc = proc
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            while self.active:
                c = self.proc.stdout.read(1)
                self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception:
                pass

    def get_char_from_queue(obs):
        """ Get character from queue safe way
        """
        try:
            c = obs.queue.get(block=True, timeout=0.5)
            # signals to queue job is done
            obs.queue.task_done()
        except Empty:
            c = None
        except:
            raise
        return c

    def filter_queue_char(c):
        """ Filters out non ASCII characters from serial port
        """
        if ord(c) not in range(128):
            c = ' '
        return c

    def get_test_result(output):
        """! Parse test 'output' data
        @details If test result not found returns by default TEST_RESULT_TIMEOUT value
        @return Returns found test result
        """
        result = TEST_RESULT_TIMEOUT
        for line in "".join(output).splitlines():
            search_result = RE_DETECT_TESTCASE_RESULT.search(line)
            if search_result and len(search_result.groups()):
                result = TEST_RESULT_MAPPING[search_result.groups(0)[0]]
                break
        return result

    def get_auto_property_value(property_name, line):
        """! Scans auto detection line from MUT and returns scanned parameter 'property_name'
        @details Host test case has to print additional properties for test to be set up
        @return Returns string or None if property search failed
        """
        result = None
        if re.search("HOST: Property '%s'"% property_name, line) is not None:
            property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
            if property is not None and len(property.groups()) == 1:
                result = property.groups()[0]
        return result

    # Detect from where input should be taken, if no --digest switch is specified
    # normal test execution can be performed

    if verbose:
        gt_log("selecting test case observer...")
        if digest_source:
            gt_log_tab("selected digest source: %s"% digest_source)

    # Select who will digest test case serial port data
    if digest_source == 'stdin':
        # When we want to scan stdin for test results
        obs = StdInObserver()
    elif digest_source is not None:
        # When we want to open file to scan for test results
        obs = FileObserver(digest_source)
    else:
        # Command executing CLI for host test supervisor (in detect-mode)
        cmd = ["mbedhtrun",
                '-d', disk,
                '-p', port,
                '-f', '"%s"'% image_path,
                ]

        # Add extra parameters to host_test
        if program_cycle_s is not None:
            cmd += ["-C", str(program_cycle_s)]
        if copy_method is not None:
            cmd += ["-c", copy_method]
        if micro is not None:
            cmd += ["-m", micro]
        if reset is not None:
            cmd += ["-r", reset]
        if reset_tout is not None:
            cmd += ["-R", str(reset_tout)]
        if json_test_cfg is not None:
            cmd += ["--test-cfg", '"%s"' % str(json_test_cfg)]
        if run_app is not None:
            cmd += ["--run"]    # -f stores binary name!

        if verbose:
            gt_log_tab("calling mbedhtrun: %s"% " ".join(cmd))
            gt_log("mbed-host-test-runner: started")
        proc = Popen(cmd, stdout=PIPE)
        obs = ProcessObserver(proc)

    update_once_flag = {}   # Stores flags checking if some auto-parameter was already set
    line = ''
    output = []
    start_time = time()
    while (time() - start_time) < (2 * duration):
        try:
            c = get_char_from_queue(obs)
        except:
            break
        if c:
            if verbose:
                sys.stdout.write(c)
            c = filter_queue_char(c)
            output.append(c)
            # Give the mbed under test a way to communicate the end of the test
            if c in ['\n', '\r']:

                # Checking for auto-detection information from the test about MUT reset moment
                if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
                    # We will update this marker only once to prevent multiple time resets
                    update_once_flag['reset_target'] = True
                    start_time = time()

                # Checking for auto-detection information from the test about timeout
                auto_timeout_val = get_auto_property_value('timeout', line)
                if 'timeout' not in update_once_flag and auto_timeout_val is not None:
                    # We will update this marker only once to prevent multiple time resets
                    update_once_flag['timeout'] = True
                    duration = int(auto_timeout_val)

                # Detect mbed assert:
                if 'mbed assertation failed: ' in line:
                    output.append('{{mbed_assert}}')
                    break

                # Check for test end
                if '{end}' in line:
                    break
                line = ''
            else:
                line += c
    end_time = time()
    testcase_duration = end_time - start_time   # Test case duration from reset to {end}

    c = get_char_from_queue(obs)

    if c:
        if verbose:
            sys.stdout.write(c)
        c = filter_queue_char(c)
        output.append(c)

    # Stop test process
    obs.stop()
    if verbose:
        gt_log("mbed-host-test-runner: stopped")
    result = get_test_result(output)
    return (result, "".join(output), testcase_duration, duration)

def run_cli_command(cmd, shell=True, verbose=False):
    """! Runs command from command line
    @param shell Shell command (e.g. ls, ps)
    @param verbose Verbose mode flag
    @return Returns (True, 0) if command was executed successfully else return (False, error code)
    """
    result = True
    ret = 0
    try:
        ret = call(cmd, shell=shell)
        if ret:
            result = False
            if verbose:
                print "mbedgt: [ret=%d] Command: %s"% (int(ret), cmd)
    except Exception as e:
        result = False
        if verbose:
            print "mbedgt: [ret=%d] Command: %s"% (int(ret), cmd)
            print str(e)
    return (result, ret)

def run_cli_process(cmd):
    """! Runs command as a process and return stdout, stderr and ret code
    @param cmd Command to execute
    @return Tuple of (stdout, stderr, returncode)
    """
    p = Popen(cmd, stdout=PIPE, stderr=PIPE)
    _stdout, _stderr = p.communicate()
    return _stdout, _stderr, p.returncode
<commit_msg>Process observer timeout changes to 'duration' sec (from 2 * 'duration')<commit_after>"""
mbed SDK
Copyright (c) 2011-2015 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
"""

import re
import sys
from time import time
from Queue import Queue, Empty
from threading import Thread
from subprocess import call, Popen, PIPE

from mbed_greentea_log import gt_log
from mbed_greentea_log import gt_log_tab
from mbed_greentea_log import gt_log_err
from mbed_greentea_log import gt_bright

# Return codes for test script
TEST_RESULT_OK = "OK"
TEST_RESULT_FAIL = "FAIL"
TEST_RESULT_ERROR = "ERROR"
TEST_RESULT_UNDEF = "UNDEF"
TEST_RESULT_IOERR_COPY = "IOERR_COPY"
TEST_RESULT_IOERR_DISK = "IOERR_DISK"
TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
TEST_RESULT_TIMEOUT = "TIMEOUT"
TEST_RESULT_NO_IMAGE = "NO_IMAGE"
TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"

TEST_RESULTS = [TEST_RESULT_OK,
                TEST_RESULT_FAIL,
                TEST_RESULT_ERROR,
                TEST_RESULT_UNDEF,
                TEST_RESULT_IOERR_COPY,
                TEST_RESULT_IOERR_DISK,
                TEST_RESULT_IOERR_SERIAL,
                TEST_RESULT_TIMEOUT,
                TEST_RESULT_NO_IMAGE,
                TEST_RESULT_MBED_ASSERT,
                TEST_RESULT_BUILD_FAILED
                ]

TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
                       "failure" : TEST_RESULT_FAIL,
                       "error" : TEST_RESULT_ERROR,
                       "end" : TEST_RESULT_UNDEF,
                       "ioerr_copy" : TEST_RESULT_IOERR_COPY,
                       "ioerr_disk" : TEST_RESULT_IOERR_DISK,
                       "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
                       "timeout" : TEST_RESULT_TIMEOUT,
                       "no_image" : TEST_RESULT_NO_IMAGE,
                       "mbed_assert" : TEST_RESULT_MBED_ASSERT,
                       "build_failed" : TEST_RESULT_BUILD_FAILED
                       }

RE_DETECT_TESTCASE_RESULT = re.compile("\\{(" + "|".join(TEST_RESULT_MAPPING.keys()) + ")\\}")

def run_host_test(image_path,
                  disk,
                  port,
                  duration=10,
                  micro=None,
                  reset=None,
                  reset_tout=None,
                  verbose=False,
                  copy_method=None,
                  program_cycle_s=None,
                  digest_source=None,
                  json_test_cfg=None,
                  max_failed_properties=5,
                  run_app=None):
    """! This function runs host test supervisor (executes mbedhtrun) and checks output from host test process.
    @return Tuple with test results, test output and test duration times
    @param image_path Path to binary file for flashing
    @param disk Currently mounted mbed-enabled devices disk (mount point)
    @param port Currently mounted mbed-enabled devices serial port (console)
    @param duration Test case timeout
    @param micro Mbed-nebaled device name
    @param reset Reset type
    @param reset_tout Reset timeout (sec)
    @param verbose Verbose mode flag
    @param copy_method Copy method type (name)
    @param program_cycle_s Wait after flashing delay (sec)
    @param json_test_cfg Additional test configuration file path passed to host tests in JSON format
    @param max_failed_properties After how many unknown properties we will assume test is not ported
    @param run_app Run application mode flag (we run application and grab serial port data)
    @param digest_source if None mbedhtrun will be executed. If 'stdin',
                           stdin will be used via StdInObserver or file (if
                           file name was given as switch option)
    """

    class StdInObserver(Thread):
        """ Process used to read stdin only as console input from MUT
        """
        def __init__(self):
            Thread.__init__(self)
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            while self.active:
                c = sys.stdin.read(1)
                self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception:
                pass

    class FileObserver(Thread):
        """ process used to read file content as console input from MUT
        """
        def __init__(self, filename):
            Thread.__init__(self)
            self.filename = filename
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            with open(self.filename) as f:
                while self.active:
                    c = f.read(1)
                    self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception:
                pass

    class ProcessObserver(Thread):
        """ Default process used to observe stdout of another process as console input from MUT
        """
        def __init__(self, proc):
            Thread.__init__(self)
            self.proc = proc
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            while self.active:
                c = self.proc.stdout.read(1)
                self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception:
                pass

    def get_char_from_queue(obs):
        """ Get character from queue safe way
        """
        try:
            c = obs.queue.get(block=True, timeout=0.5)
            # signals to queue job is done
            obs.queue.task_done()
        except Empty:
            c = None
        except:
            raise
        return c

    def filter_queue_char(c):
        """ Filters out non ASCII characters from serial port
        """
        if ord(c) not in range(128):
            c = ' '
        return c

    def get_test_result(output):
        """! Parse test 'output' data
        @details If test result not found returns by default TEST_RESULT_TIMEOUT value
        @return Returns found test result
        """
        result = TEST_RESULT_TIMEOUT
        for line in "".join(output).splitlines():
            search_result = RE_DETECT_TESTCASE_RESULT.search(line)
            if search_result and len(search_result.groups()):
                result = TEST_RESULT_MAPPING[search_result.groups(0)[0]]
                break
        return result

    def get_auto_property_value(property_name, line):
        """! Scans auto detection line from MUT and returns scanned parameter 'property_name'
        @details Host test case has to print additional properties for test to be set up
        @return Returns string or None if property search failed
        """
        result = None
        if re.search("HOST: Property '%s'"% property_name, line) is not None:
            property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
            if property is not None and len(property.groups()) == 1:
                result = property.groups()[0]
        return result

    # Detect from where input should be taken, if no --digest switch is specified
    # normal test execution can be performed

    if verbose:
        gt_log("selecting test case observer...")
        if digest_source:
            gt_log_tab("selected digest source: %s"% digest_source)

    # Select who will digest test case serial port data
    if digest_source == 'stdin':
        # When we want to scan stdin for test results
        obs = StdInObserver()
    elif digest_source is not None:
        # When we want to open file to scan for test results
        obs = FileObserver(digest_source)
    else:
        # Command executing CLI for host test supervisor (in detect-mode)
        cmd = ["mbedhtrun",
                '-d', disk,
                '-p', port,
                '-f', '"%s"'% image_path,
                ]

        # Add extra parameters to host_test
        if program_cycle_s is not None:
            cmd += ["-C", str(program_cycle_s)]
        if copy_method is not None:
            cmd += ["-c", copy_method]
        if micro is not None:
            cmd += ["-m", micro]
        if reset is not None:
            cmd += ["-r", reset]
        if reset_tout is not None:
            cmd += ["-R", str(reset_tout)]
        if json_test_cfg is not None:
            cmd += ["--test-cfg", '"%s"' % str(json_test_cfg)]
        if run_app is not None:
            cmd += ["--run"]    # -f stores binary name!

        if verbose:
            gt_log_tab("calling mbedhtrun: %s"% " ".join(cmd))
            gt_log("mbed-host-test-runner: started")
        proc = Popen(cmd, stdout=PIPE)
        obs = ProcessObserver(proc)

    update_once_flag = {}   # Stores flags checking if some auto-parameter was already set
    unknown_property_count = 0
    line = ''
    output = []
    start_time = time()
    while (time() - start_time) < (duration):
        try:
            c = get_char_from_queue(obs)
        except:
            break
        if c:
            if verbose:
                sys.stdout.write(c)
            c = filter_queue_char(c)
            output.append(c)
            # Give the mbed under test a way to communicate the end of the test
            if c in ['\n', '\r']:

                # Check for unknown property prints
                # If there are too many we will stop test exeuction and assume test is not ported
                if "HOST: Unknown property" in line:
                    unknown_property_count += 1
                    if unknown_property_count >= max_failed_properties:
                        output.append('{{error}}')
                        break

                # Checking for auto-detection information from the test about MUT reset moment
                if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
                    # We will update this marker only once to prevent multiple time resets
                    update_once_flag['reset_target'] = True
                    start_time = time()

                # Checking for auto-detection information from the test about timeout
                auto_timeout_val = get_auto_property_value('timeout', line)
                if 'timeout' not in update_once_flag and auto_timeout_val is not None:
                    # We will update this marker only once to prevent multiple time resets
                    update_once_flag['timeout'] = True
                    duration = int(auto_timeout_val)

                # Detect mbed assert:
                if 'mbed assertation failed: ' in line:
                    output.append('{{mbed_assert}}')
                    break

                # Check for test end
                if '{end}' in line:
                    break
                line = ''
            else:
                line += c

    # We want to let the test suite know we've finished after we return from the loop
    if '{end}' not in line:
        output.append('{{end}}')

    end_time = time()
    testcase_duration = end_time - start_time   # Test case duration from reset to {end}

    c = get_char_from_queue(obs)

    if c:
        if verbose:
            sys.stdout.write(c)
        c = filter_queue_char(c)
        output.append(c)

    # Stop test process
    obs.stop()
    if verbose:
        gt_log("mbed-host-test-runner: stopped")
    result = get_test_result(output)
    return (result, "".join(output), testcase_duration, duration)

def run_cli_command(cmd, shell=True, verbose=False):
    """! Runs command from command line
    @param shell Shell command (e.g. ls, ps)
    @param verbose Verbose mode flag
    @return Returns (True, 0) if command was executed successfully else return (False, error code)
    """
    result = True
    ret = 0
    try:
        ret = call(cmd, shell=shell)
        if ret:
            result = False
            if verbose:
                print "mbedgt: [ret=%d] Command: %s"% (int(ret), cmd)
    except Exception as e:
        result = False
        if verbose:
            print "mbedgt: [ret=%d] Command: %s"% (int(ret), cmd)
            print str(e)
    return (result, ret)

def run_cli_process(cmd):
    """! Runs command as a process and return stdout, stderr and ret code
    @param cmd Command to execute
    @return Tuple of (stdout, stderr, returncode)
    """
    p = Popen(cmd, stdout=PIPE, stderr=PIPE)
    _stdout, _stderr = p.communicate()
    return _stdout, _stderr, p.returncode
<|endoftext|>


<commit_before>###################################################################
# Copyright 2013-2015 All Rights Reserved
# Authors: The Paradrop Team
###################################################################

"""
Functions associated with deploying and cleaning up docker containers.
"""

from pdtools.lib.output import out
import docker
import json
import os
import subprocess

from paradrop.lib import settings
from paradrop.lib.container.downloader import downloader
from pdtools.lib import nexus

DOCKER_CONF = """
# Docker systemd configuration
#
# This configuration file was automatically generated by Paradrop.  Any changes
# will be overwritten on startup.

# Tell docker not to start containers automatically on startup.
DOCKER_OPTIONS="--restart=false"
"""

def writeDockerConfig():
    """
    Write options to Docker configuration.

    Mainly, we want to tell Docker not to start containers automatically on
    system boot.
    """
    # First we have to find the configuration file.  On Snappy, it should be in
    # "/var/lib/apps/docker/{version}/etc/docker.conf", but version could
    # change.
    path = "/var/lib/apps/docker"
    if not os.path.exists(path):
        out.warn('No directory "{}" found'.format(path))
        return False

    written = False
    for d in os.listdir(path):
        finalPath = os.path.join(path, d, "etc/docker.conf")
        if not os.path.exists(finalPath):
            continue

        try:
            with open(finalPath, "w") as output:
                output.write(DOCKER_CONF)
            written = True
        except Exception as e:
            out.warn('Error writing to {}: {}'.format(finalPath, str(e)))

    if not written:
        out.warn('Could not write docker configuration.')
    return written

def buildImage(update, client, **buildArgs):
    """
    Build the Docker image and monitor progress.

    Returns True on success, False on failure.
    """
    output = client.build(**buildArgs)

    buildSuccess = True
    for line in output:
        #if we encountered an error make note of it
        if 'errorDetail' in line:
            buildSuccess = False

        for key, value in json.loads(line).iteritems():
            if isinstance(value, dict):
                continue
            else:
                msg = str(value).rstrip()
                update.progress(msg)

    return buildSuccess

def startChute(update):
    """
    Build and deploy a docker container based on the passed in update.

    :param update: The update object containing information about the chute.
    :type update: obj
    :returns: None
    """
    out.info('Attempting to start new Chute %s \n' % (update.name))

    repo = update.name + ":latest"
    name = update.name

    c = docker.Client(base_url="unix://var/run/docker.sock", version='auto')

    host_config = build_host_config(update, c)

    #Get Id's of current images for comparison upon failure
    validImages = c.images(quiet=True, all=False)
    validContainers = c.containers(quiet=True, all=True)

    if hasattr(update, 'dockerfile'):
        buildSuccess = buildImage(update, c, rm=True, tag=repo,
                fileobj=update.dockerfile)
    elif hasattr(update, 'download'):
        # download field should be an object with at least 'url' but may also
        # contain 'user' and 'secret' for authentication.
        download_args = update.download
        with downloader(**download_args) as dl:
            workDir, meta = dl.fetch()
            buildSuccess = buildImage(update, c, rm=True, tag=repo,
                    path=workDir)
    else:
        raise Exception("No Dockerfile or download location supplied.")

    #If we failed to build skip creating and starting clean up and fail
    if not buildSuccess:
        cleanUpDocker(validImages, validContainers)
        raise Exception("Building docker image failed; check your Dockerfile for errors.")

    # Set environment variables for the new container.
    # PARADROP_ROUTER_ID can be used to change application behavior based on
    # what router it is running on.
    environment = prepare_environment(update)

    try:
        container = c.create_container(
            image=repo, name=name, host_config=host_config,
            environment=environment
        )
        c.start(container.get('Id'))
        out.info("Successfully started chute with Id: %s\n" % (str(container.get('Id'))))
    except Exception as e:
        cleanUpDocker(validImages, validContainers)
        raise e

    setup_net_interfaces(update)

def removeChute(update):
    """
    Remove a docker container and the image it was built on based on the passed in update.

    :param update: The update object containing information about the chute.
    :type update: obj
    :returns: None
    """
    out.info('Attempting to remove chute %s\n' % (update.name))
    c = docker.Client(base_url='unix://var/run/docker.sock', version='auto')
    repo = update.name + ":latest"
    name = update.name
    try:
        c.remove_container(container=name, force=True)
        c.remove_image(image=repo)
    except Exception as e:
        #TODO: Might want to notify ourselves we could have removed container but failed to remove image for a number of reasons
        update.complete(success=False, message=e.message)
        raise e

def stopChute(update):
    """
    Stop a docker container based on the passed in update.

    :param update: The update object containing information about the chute.
    :type update: obj
    :returns: None
    """
    out.info('Attempting to stop chute %s\n' % (update.name))
    c = docker.Client(base_url='unix://var/run/docker.sock', version='auto')
    c.stop(container=update.name)

def restartChute(update):
    """
    Start a docker container based on the passed in update.

    :param update: The update object containing information about the chute.
    :type update: obj
    :returns: None
    """
    out.info('Attempting to restart chute %s\n' % (update.name))
    c = docker.Client(base_url='unix://var/run/docker.sock', version='auto')
    c.start(container=update.name)

    setup_net_interfaces(update)

def cleanUpDocker(validImages, validContainers):
    """
    Clean up any intermediate containers that may have resulted from a failure.

    :param validImages: A list of dicts containing the Id's of all the images that should exist on the system.
    :type validImages: list
    :param validContainers: A list of the Id's of all the containers that should exist on the system.
    :type validContainers: list
    :returns: None
    """
    c = docker.Client(base_url="unix://var/run/docker.sock", version='auto')

    #Clean up containers from failed build/start
    currContainers = c.containers(quiet=True, all=True)
    for cntr in currContainers:
        if not cntr in validContainers:
            out.info('Removing Invalid container with id: %s' % str(cntr.get('Id')))
            c.remove_container(container=cntr.get('Id'))

    #Clean up images from failed build
    currImages = c.images(quiet=True, all=False)
    for img in currImages:
        if not img in validImages:
            out.info('Removing Invalid image with id: %s' % str(img))
            c.remove_image(image=img)

def build_host_config(update, client=None):
    """
    Build the host_config dict for a docker container based on the passed in update.

    :param update: The update object containing information about the chute.
    :type update: obj
    :param client: Docker client object.
    :returns: (dict) The host_config dict which docker needs in order to create the container.
    """
    if client is None:
        client = docker.Client(base_url="unix://var/run/docker.sock", version='auto')

    if not hasattr(update.new, 'host_config') or update.new.host_config == None:
        config = dict()
    else:
        config = update.new.host_config

    host_conf = client.create_host_config(
        #TO support
        port_bindings=config.get('port_bindings'),
        dns=config.get('dns'),
        #not supported/managed by us
        #network_mode=update.host_config.get('network_mode'),
        network_mode='bridge',
        #extra_hosts=update.host_config.get('extra_hosts'),
        #binds=config.get('binds'),
        #links=config.get('links'),
        restart_policy={'MaximumRetryCount': 5, 'Name': 'on-failure'},
        devices=[],
        lxc_conf={},
        publish_all_ports=False,
        privileged=False,
        dns_search=[],
        volumes_from=None,
        cap_add=['NET_ADMIN'],
        cap_drop=[]
    )
    return host_conf

def setup_net_interfaces(update):
    """
    Link interfaces in the host to the internal interface in the docker container using pipework.

    :param update: The update object containing information about the chute.
    :type update: obj
    :returns: None
    """
    interfaces = update.new.getCache('networkInterfaces')
    for iface in interfaces:
        if iface.get('netType') == 'wifi':
            IP = iface.get('ipaddrWithPrefix')
            internalIntf = iface.get('internalIntf')
            externalIntf = iface.get('externalIntf')
        else: # pragma: no cover
            continue

        # Construct environment for pipework call.  It only seems to require
        # the PATH variable to include the directory containing the docker
        # client.  On Snappy this was not happening by default, which is why
        # this code is here.
        env = {"PATH": os.environ.get("PATH", "")}
        if settings.DOCKER_BIN_DIR not in env['PATH']:
            env['PATH'] += ":" + settings.DOCKER_BIN_DIR

        cmd = ['/apps/paradrop/current/bin/pipework', externalIntf, '-i',
               internalIntf, update.name,  IP]
        out.info("Calling: {}\n".format(" ".join(cmd)))
        try:
            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE, env=env)
            for line in proc.stdout:
                out.info("pipework: {}\n".format(line.strip()))
            for line in proc.stderr:
                out.warn("pipework: {}\n".format(line.strip()))
        except OSError as e:
            out.warn('Command "{}" failed\n'.format(" ".join(cmd)))
            out.exception(e, True)
            raise e

def prepare_environment(update):
    """
    Prepare environment variables for a chute container.
    """
    env = {
        'PARADROP_CHUTE_NAME': update.name,
        'PARADROP_ROUTER_ID': nexus.core.info.pdid
    }
    return env
<commit_msg>Suppress noisy progress messages from Docker build.<commit_after>###################################################################
# Copyright 2013-2015 All Rights Reserved
# Authors: The Paradrop Team
###################################################################

"""
Functions associated with deploying and cleaning up docker containers.
"""

from pdtools.lib.output import out
import docker
import json
import os
import re
import subprocess

from paradrop.lib import settings
from paradrop.lib.container.downloader import downloader
from pdtools.lib import nexus

DOCKER_CONF = """
# Docker systemd configuration
#
# This configuration file was automatically generated by Paradrop.  Any changes
# will be overwritten on startup.

# Tell docker not to start containers automatically on startup.
DOCKER_OPTIONS="--restart=false"
"""

# Used to match and suppress noisy progress messages from Docker output.
#
# Example:
# Extracting
# 862a3e9af0ae
# [================================================>  ] 64.06 MB/65.7 MB
suppress_re = re.compile("^(Downloading|Extracting|[a-z0-9]+|\[=*>?\s*\].*)$")

def writeDockerConfig():
    """
    Write options to Docker configuration.

    Mainly, we want to tell Docker not to start containers automatically on
    system boot.
    """
    # First we have to find the configuration file.  On Snappy, it should be in
    # "/var/lib/apps/docker/{version}/etc/docker.conf", but version could
    # change.
    path = "/var/lib/apps/docker"
    if not os.path.exists(path):
        out.warn('No directory "{}" found'.format(path))
        return False

    written = False
    for d in os.listdir(path):
        finalPath = os.path.join(path, d, "etc/docker.conf")
        if not os.path.exists(finalPath):
            continue

        try:
            with open(finalPath, "w") as output:
                output.write(DOCKER_CONF)
            written = True
        except Exception as e:
            out.warn('Error writing to {}: {}'.format(finalPath, str(e)))

    if not written:
        out.warn('Could not write docker configuration.')
    return written

def buildImage(update, client, **buildArgs):
    """
    Build the Docker image and monitor progress.

    Returns True on success, False on failure.
    """
    output = client.build(**buildArgs)

    buildSuccess = True
    for line in output:
        #if we encountered an error make note of it
        if 'errorDetail' in line:
            buildSuccess = False

        for key, value in json.loads(line).iteritems():
            if isinstance(value, dict):
                continue
            else:
                msg = str(value).rstrip()
                if len(msg) > 0 and suppress_re.match(msg) is None:
                    update.progress(msg)

    return buildSuccess

def startChute(update):
    """
    Build and deploy a docker container based on the passed in update.

    :param update: The update object containing information about the chute.
    :type update: obj
    :returns: None
    """
    out.info('Attempting to start new Chute %s \n' % (update.name))

    repo = update.name + ":latest"
    name = update.name

    c = docker.Client(base_url="unix://var/run/docker.sock", version='auto')

    host_config = build_host_config(update, c)

    #Get Id's of current images for comparison upon failure
    validImages = c.images(quiet=True, all=False)
    validContainers = c.containers(quiet=True, all=True)

    if hasattr(update, 'dockerfile'):
        buildSuccess = buildImage(update, c, rm=True, tag=repo,
                fileobj=update.dockerfile)
    elif hasattr(update, 'download'):
        # download field should be an object with at least 'url' but may also
        # contain 'user' and 'secret' for authentication.
        download_args = update.download
        with downloader(**download_args) as dl:
            workDir, meta = dl.fetch()
            buildSuccess = buildImage(update, c, rm=True, tag=repo,
                    path=workDir)
    else:
        raise Exception("No Dockerfile or download location supplied.")

    #If we failed to build skip creating and starting clean up and fail
    if not buildSuccess:
        cleanUpDocker(validImages, validContainers)
        raise Exception("Building docker image failed; check your Dockerfile for errors.")

    # Set environment variables for the new container.
    # PARADROP_ROUTER_ID can be used to change application behavior based on
    # what router it is running on.
    environment = prepare_environment(update)

    try:
        container = c.create_container(
            image=repo, name=name, host_config=host_config,
            environment=environment
        )
        c.start(container.get('Id'))
        out.info("Successfully started chute with Id: %s\n" % (str(container.get('Id'))))
    except Exception as e:
        cleanUpDocker(validImages, validContainers)
        raise e

    setup_net_interfaces(update)

def removeChute(update):
    """
    Remove a docker container and the image it was built on based on the passed in update.

    :param update: The update object containing information about the chute.
    :type update: obj
    :returns: None
    """
    out.info('Attempting to remove chute %s\n' % (update.name))
    c = docker.Client(base_url='unix://var/run/docker.sock', version='auto')
    repo = update.name + ":latest"
    name = update.name
    try:
        c.remove_container(container=name, force=True)
        c.remove_image(image=repo)
    except Exception as e:
        #TODO: Might want to notify ourselves we could have removed container but failed to remove image for a number of reasons
        update.complete(success=False, message=e.message)
        raise e

def stopChute(update):
    """
    Stop a docker container based on the passed in update.

    :param update: The update object containing information about the chute.
    :type update: obj
    :returns: None
    """
    out.info('Attempting to stop chute %s\n' % (update.name))
    c = docker.Client(base_url='unix://var/run/docker.sock', version='auto')
    c.stop(container=update.name)

def restartChute(update):
    """
    Start a docker container based on the passed in update.

    :param update: The update object containing information about the chute.
    :type update: obj
    :returns: None
    """
    out.info('Attempting to restart chute %s\n' % (update.name))
    c = docker.Client(base_url='unix://var/run/docker.sock', version='auto')
    c.start(container=update.name)

    setup_net_interfaces(update)

def cleanUpDocker(validImages, validContainers):
    """
    Clean up any intermediate containers that may have resulted from a failure.

    :param validImages: A list of dicts containing the Id's of all the images that should exist on the system.
    :type validImages: list
    :param validContainers: A list of the Id's of all the containers that should exist on the system.
    :type validContainers: list
    :returns: None
    """
    c = docker.Client(base_url="unix://var/run/docker.sock", version='auto')

    #Clean up containers from failed build/start
    currContainers = c.containers(quiet=True, all=True)
    for cntr in currContainers:
        if not cntr in validContainers:
            out.info('Removing Invalid container with id: %s' % str(cntr.get('Id')))
            c.remove_container(container=cntr.get('Id'))

    #Clean up images from failed build
    currImages = c.images(quiet=True, all=False)
    for img in currImages:
        if not img in validImages:
            out.info('Removing Invalid image with id: %s' % str(img))
            c.remove_image(image=img)

def build_host_config(update, client=None):
    """
    Build the host_config dict for a docker container based on the passed in update.

    :param update: The update object containing information about the chute.
    :type update: obj
    :param client: Docker client object.
    :returns: (dict) The host_config dict which docker needs in order to create the container.
    """
    if client is None:
        client = docker.Client(base_url="unix://var/run/docker.sock", version='auto')

    if not hasattr(update.new, 'host_config') or update.new.host_config == None:
        config = dict()
    else:
        config = update.new.host_config

    host_conf = client.create_host_config(
        #TO support
        port_bindings=config.get('port_bindings'),
        dns=config.get('dns'),
        #not supported/managed by us
        #network_mode=update.host_config.get('network_mode'),
        network_mode='bridge',
        #extra_hosts=update.host_config.get('extra_hosts'),
        #binds=config.get('binds'),
        #links=config.get('links'),
        restart_policy={'MaximumRetryCount': 5, 'Name': 'on-failure'},
        devices=[],
        lxc_conf={},
        publish_all_ports=False,
        privileged=False,
        dns_search=[],
        volumes_from=None,
        cap_add=['NET_ADMIN'],
        cap_drop=[]
    )
    return host_conf

def setup_net_interfaces(update):
    """
    Link interfaces in the host to the internal interface in the docker container using pipework.

    :param update: The update object containing information about the chute.
    :type update: obj
    :returns: None
    """
    interfaces = update.new.getCache('networkInterfaces')
    for iface in interfaces:
        if iface.get('netType') == 'wifi':
            IP = iface.get('ipaddrWithPrefix')
            internalIntf = iface.get('internalIntf')
            externalIntf = iface.get('externalIntf')
        else: # pragma: no cover
            continue

        # Construct environment for pipework call.  It only seems to require
        # the PATH variable to include the directory containing the docker
        # client.  On Snappy this was not happening by default, which is why
        # this code is here.
        env = {"PATH": os.environ.get("PATH", "")}
        if settings.DOCKER_BIN_DIR not in env['PATH']:
            env['PATH'] += ":" + settings.DOCKER_BIN_DIR

        cmd = ['/apps/paradrop/current/bin/pipework', externalIntf, '-i',
               internalIntf, update.name,  IP]
        out.info("Calling: {}\n".format(" ".join(cmd)))
        try:
            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE, env=env)
            for line in proc.stdout:
                out.info("pipework: {}\n".format(line.strip()))
            for line in proc.stderr:
                out.warn("pipework: {}\n".format(line.strip()))
        except OSError as e:
            out.warn('Command "{}" failed\n'.format(" ".join(cmd)))
            out.exception(e, True)
            raise e

def prepare_environment(update):
    """
    Prepare environment variables for a chute container.
    """
    env = {
        'PARADROP_CHUTE_NAME': update.name,
        'PARADROP_ROUTER_ID': nexus.core.info.pdid
    }
    return env
<|endoftext|>


<commit_before>from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.utils.safestring import mark_safe
from filer.fields.image import FilerImageField

ASSET_TYPES = (
  (u'article', u'article'),
  (u'image', u'image'),
  (u'map', u'map'),
)

class Asset(models.Model):
    caption = models.TextField(blank=True)
    title = models.CharField(max_length=200)
    type = models.CharField(max_length=10, choices=ASSET_TYPES)
    user = models.ForeignKey(User, related_name="assets")

    def __unicode__(self):
        return self.title

    def subclass(self):
        for attr in ('externalasset', 'htmlasset'):
            try:
                return getattr(self, attr)
            except ObjectDoesNotExist:
                pass

        return self

    def render(self, format='html'):
        try:
            return getattr(self, "render_" + format).__call__()
        except AttributeError:
            return self.__unicode__()

class ExternalAsset(Asset):
    url = models.URLField()

    def render_html(self):
        output = []
        if self.type == 'image':
            output.append('
')
            output.append('' % (self.url, self.title))
            if self.caption:
                output.append('
')
                output.append(self.caption)
                output.append('
')
            output.append('
')
        else:
            output.append("%s" % (self.url, self.title))

        return mark_safe(u'\n'.join(output))

class HtmlAsset(Asset):
    body = models.TextField(blank=True)

    def render_html(self):
        output = []
        if self.type == 'map':
            output.append('
')
            output.append(self.body)
            if self.caption:
                output.append('
')
                output.append(self.caption)
                output.append('
')
            output.append('
')
        else:
            output.append(self.body)

        return mark_safe(u'\n'.join(output))

class FilerImageAsset(Asset):
    image = FilerImageField()
<commit_msg>Basic rendering of local image assets in a story.<commit_after>from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.utils.safestring import mark_safe
from filer.fields.image import FilerImageField

ASSET_TYPES = (
  (u'article', u'article'),
  (u'image', u'image'),
  (u'map', u'map'),
)

class Asset(models.Model):
    caption = models.TextField(blank=True)
    title = models.CharField(max_length=200)
    type = models.CharField(max_length=10, choices=ASSET_TYPES)
    user = models.ForeignKey(User, related_name="assets")

    def __unicode__(self):
        return self.title

    def subclass(self):
        for attr in ('externalasset', 'htmlasset', 'filerimageasset'):
            try:
                return getattr(self, attr)
            except ObjectDoesNotExist:
                pass

        return self

    def render(self, format='html'):
        try:
            return getattr(self, "render_" + format).__call__()
        except AttributeError:
            return self.__unicode__()

class ExternalAsset(Asset):
    url = models.URLField()

    def render_html(self):
        output = []
        if self.type == 'image':
            output.append('
')
            output.append('' % (self.url, self.title))
            if self.caption:
                output.append('
')
                output.append(self.caption)
                output.append('
')
            output.append('
')
        else:
            output.append("%s" % (self.url, self.title))

        return mark_safe(u'\n'.join(output))

class HtmlAsset(Asset):
    body = models.TextField(blank=True)

    def render_html(self):
        output = []
        if self.type == 'map':
            output.append('
')
            output.append(self.body)
            if self.caption:
                output.append('
')
                output.append(self.caption)
                output.append('
')
            output.append('
')
        else:
            output.append(self.body)

        return mark_safe(u'\n'.join(output))

class FilerImageAsset(Asset):
    image = FilerImageField()

    def render_html(self):
        output = []
        output.append('
')
        output.append('' % (self.image.url, self.title))
        if self.caption:
            output.append('
')
            output.append(self.caption)
            output.append('
')
        output.append('
')

        return mark_safe(u'\n'.join(output))
<|endoftext|>


<commit_before># Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import unittest
import numpy as np
from op_test import OpTest
import paddle
import paddle.fluid as fluid
import paddle.fluid.core as core
from paddle.fluid import Program, program_guard

class BaseTestCase(OpTest):
    def initTestCase(self):
        self.op_type = 'arg_min'
        self.dims = (3, 4, 5)
        self.dtype = 'float32'
        self.axis = 0

    def setUp(self):
        self.initTestCase()
        self.x = (1000 * np.random.random(self.dims)).astype(self.dtype)
        self.inputs = {'X': self.x}
        self.attrs = {'axis': self.axis}
        if self.op_type == "arg_min":
            self.outputs = {'Out': np.argmin(self.x, axis=self.axis)}
        else:
            self.outputs = {'Out': np.argmax(self.x, axis=self.axis)}

    def test_check_output(self):
        self.check_output()

class TestCase0(BaseTestCase):
    def initTestCase(self):
        self.op_type = 'arg_max'
        self.dims = (3, 4, 5)
        self.dtype = 'float32'
        self.axis = 0

class TestCase1(BaseTestCase):
    def initTestCase(self):
        self.op_type = 'arg_min'
        self.dims = (3, 4)
        self.dtype = 'float64'
        self.axis = 1

class TestCase2(BaseTestCase):
    def initTestCase(self):
        self.op_type = 'arg_max'
        self.dims = (3, 4)
        self.dtype = 'int64'
        self.axis = 0

class TestCase2_1(BaseTestCase):
    def initTestCase(self):
        self.op_type = 'arg_max'
        self.dims = (3, 4)
        self.dtype = 'int64'
        self.axis = -1

class TestCase3(BaseTestCase):
    def initTestCase(self):
        self.op_type = 'arg_max'
        self.dims = (3, )
        self.dtype = 'int64'
        self.axis = 0

class TestCase4(BaseTestCase):
    def initTestCase(self):
        self.op_type = 'arg_min'
        self.dims = (1, )
        self.dtype = 'int32'
        self.axis = 0

class TestCase3(BaseTestCase):
    def initTestCase(self):
        self.op_type = 'arg_max'
        self.dims = (3, )
        self.axis = 0

class BaseTestComplex1_1(OpTest):
    def initTestCase(self):
        self.op_type = 'arg_max'
        self.dims = (4, 5, 6)
        self.dtype = 'int32'
        self.axis = 2

    def setUp(self):
        self.initTestCase()
        self.x = (np.random.random(self.dims)).astype(self.dtype)
        self.inputs = {'X': self.x}
        self.attrs = {'axis': self.axis}
        self.attrs = {'dtype': int(core.VarDesc.VarType.INT32)}
        if self.op_type == "arg_min":
            self.outputs = {
                'Out': np.argmin(
                    self.x, axis=self.axis).asdtype("int32")
            }
        else:
            self.outputs = {
                'Out': np.argmax(
                    self.x, axis=self.axis).asdtype("int32")
            }

class BaseTestComplex1_2(OpTest):
    def initTestCase(self):
        self.op_type = 'arg_min'
        self.dims = (4, 5, 6)
        self.dtype = 'int32'
        self.axis = 2

    def setUp(self):
        self.initTestCase()
        self.x = (np.random.random(self.dims)).astype(self.dtype)
        self.inputs = {'X': self.x}
        self.attrs = {'axis': self.axis}
        self.attrs = {'dtype': int(core.VarDesc.VarType.INT32)}
        if self.op_type == "arg_min":
            self.outputs = {
                'Out': np.argmin(
                    self.x, axis=self.axis).asdtype("int32")
            }
        else:
            self.outputs = {
                'Out': np.argmax(
                    self.x, axis=self.axis).asdtype("int32")
            }

class BaseTestComplex2_1(OpTest):
    def initTestCase(self):
        self.op_type = 'arg_max'
        self.dims = (4, 5, 6)
        self.dtype = 'int32'
        self.axis = 2

    def setUp(self):
        self.initTestCase()
        self.x = (np.random.random(self.dims)).astype(self.dtype)
        self.inputs = {'X': self.x}
        self.attrs = {'axis': self.axis}
        self.attrs = {'dtype': int(core.VarDesc.VarType.INT32)}
        self.attrs = {'keep_dims': True}
        if self.op_type == "arg_min":
            self.outputs = {
                'Out': np.argmin(
                    self.x, axis=self.axis).asdtype("int32").reshape(4, 5, 1)
            }
        else:
            self.outputs = {
                'Out': np.argmax(
                    self.x, axis=self.axis).asdtype("int32").reshape(4, 5, 1)
            }

class BaseTestComplex2_2(OpTest):
    def initTestCase(self):
        self.op_type = 'arg_min'
        self.dims = (4, 5, 6)
        self.dtype = 'int32'
        self.axis = 2

    def setUp(self):
        self.initTestCase()
        self.x = (np.random.random(self.dims)).astype(self.dtype)
        self.inputs = {'X': self.x}
        self.attrs = {'axis': self.axis}
        self.attrs = {'dtype': int(core.VarDesc.VarType.INT32)}
        self.attrs = {'keep_dims': True}
        if self.op_type == "arg_min":
            self.outputs = {
                'Out': np.argmin(
                    self.x, axis=self.axis).asdtype("int32").reshape(4, 5, 1)
            }
        else:
            self.outputs = {
                'Out': np.argmax(
                    self.x, axis=self.axis).asdtype("int32").reshape(4, 5, 1)
            }

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fix function redefine (#34185)<commit_after># Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import unittest
import numpy as np
from op_test import OpTest
import paddle
import paddle.fluid as fluid
import paddle.fluid.core as core
from paddle.fluid import Program, program_guard

class BaseTestCase(OpTest):
    def initTestCase(self):
        self.op_type = 'arg_min'
        self.dims = (3, 4, 5)
        self.dtype = 'float32'
        self.axis = 0

    def setUp(self):
        self.initTestCase()
        self.x = (1000 * np.random.random(self.dims)).astype(self.dtype)
        self.inputs = {'X': self.x}
        self.attrs = {'axis': self.axis}
        if self.op_type == "arg_min":
            self.outputs = {'Out': np.argmin(self.x, axis=self.axis)}
        else:
            self.outputs = {'Out': np.argmax(self.x, axis=self.axis)}

    def test_check_output(self):
        self.check_output()

class TestCase0(BaseTestCase):
    def initTestCase(self):
        self.op_type = 'arg_max'
        self.dims = (3, 4, 5)
        self.dtype = 'float32'
        self.axis = 0

class TestCase1(BaseTestCase):
    def initTestCase(self):
        self.op_type = 'arg_min'
        self.dims = (3, 4)
        self.dtype = 'float64'
        self.axis = 1

class TestCase2(BaseTestCase):
    def initTestCase(self):
        self.op_type = 'arg_max'
        self.dims = (3, 4)
        self.dtype = 'int64'
        self.axis = 0

class TestCase2_1(BaseTestCase):
    def initTestCase(self):
        self.op_type = 'arg_max'
        self.dims = (3, 4)
        self.dtype = 'int64'
        self.axis = -1

class TestCase3(BaseTestCase):
    def initTestCase(self):
        self.op_type = 'arg_max'
        self.dims = (3, )
        self.dtype = 'int64'
        self.axis = 0

class TestCase4(BaseTestCase):
    def initTestCase(self):
        self.op_type = 'arg_min'
        self.dims = (1, )
        self.dtype = 'int32'
        self.axis = 0

class TestCase3_(BaseTestCase):
    def initTestCase(self):
        self.op_type = 'arg_max'
        self.dims = (3, )
        self.axis = 0

class BaseTestComplex1_1(OpTest):
    def initTestCase(self):
        self.op_type = 'arg_max'
        self.dims = (4, 5, 6)
        self.dtype = 'int32'
        self.axis = 2

    def setUp(self):
        self.initTestCase()
        self.x = (np.random.random(self.dims)).astype(self.dtype)
        self.inputs = {'X': self.x}
        self.attrs = {'axis': self.axis}
        self.attrs = {'dtype': int(core.VarDesc.VarType.INT32)}
        if self.op_type == "arg_min":
            self.outputs = {
                'Out': np.argmin(
                    self.x, axis=self.axis).asdtype("int32")
            }
        else:
            self.outputs = {
                'Out': np.argmax(
                    self.x, axis=self.axis).asdtype("int32")
            }

class BaseTestComplex1_2(OpTest):
    def initTestCase(self):
        self.op_type = 'arg_min'
        self.dims = (4, 5, 6)
        self.dtype = 'int32'
        self.axis = 2

    def setUp(self):
        self.initTestCase()
        self.x = (np.random.random(self.dims)).astype(self.dtype)
        self.inputs = {'X': self.x}
        self.attrs = {'axis': self.axis}
        self.attrs = {'dtype': int(core.VarDesc.VarType.INT32)}
        if self.op_type == "arg_min":
            self.outputs = {
                'Out': np.argmin(
                    self.x, axis=self.axis).asdtype("int32")
            }
        else:
            self.outputs = {
                'Out': np.argmax(
                    self.x, axis=self.axis).asdtype("int32")
            }

class BaseTestComplex2_1(OpTest):
    def initTestCase(self):
        self.op_type = 'arg_max'
        self.dims = (4, 5, 6)
        self.dtype = 'int32'
        self.axis = 2

    def setUp(self):
        self.initTestCase()
        self.x = (np.random.random(self.dims)).astype(self.dtype)
        self.inputs = {'X': self.x}
        self.attrs = {'axis': self.axis}
        self.attrs = {'dtype': int(core.VarDesc.VarType.INT32)}
        self.attrs = {'keep_dims': True}
        if self.op_type == "arg_min":
            self.outputs = {
                'Out': np.argmin(
                    self.x, axis=self.axis).asdtype("int32").reshape(4, 5, 1)
            }
        else:
            self.outputs = {
                'Out': np.argmax(
                    self.x, axis=self.axis).asdtype("int32").reshape(4, 5, 1)
            }

class BaseTestComplex2_2(OpTest):
    def initTestCase(self):
        self.op_type = 'arg_min'
        self.dims = (4, 5, 6)
        self.dtype = 'int32'
        self.axis = 2

    def setUp(self):
        self.initTestCase()
        self.x = (np.random.random(self.dims)).astype(self.dtype)
        self.inputs = {'X': self.x}
        self.attrs = {'axis': self.axis}
        self.attrs = {'dtype': int(core.VarDesc.VarType.INT32)}
        self.attrs = {'keep_dims': True}
        if self.op_type == "arg_min":
            self.outputs = {
                'Out': np.argmin(
                    self.x, axis=self.axis).asdtype("int32").reshape(4, 5, 1)
            }
        else:
            self.outputs = {
                'Out': np.argmax(
                    self.x, axis=self.axis).asdtype("int32").reshape(4, 5, 1)
            }

if __name__ == '__main__':
    unittest.main()
<|endoftext|>