<commit_before>import interp
import functools
import func

class String():
    def __init__(self, val):
        self.val = val

class Env():
    def __init__(self):
        self.env = dict()
        init_env(self)

    def get(self, scope, name):
        while True:
            try:
                # print(scope)
                # print("get:", (name, scope), "\n  ->")
                tmp = self.env[(name, scope)]
                # print("get:", (name, scope), "\n  ->", tmp)
                return tmp
            except KeyError:
                if scope is None:
                    raise KeyError()
                else:
                    scope = scope[1]

    def set(self, scope, name, val):
        # print("set:", (name, scope), "\n  ->", val)
        if (name, scope) in self.env:
            # print((name, scope), "is already defined")
            assert (name, scope) not in self.env
        self.env[(name, scope)] = val

def _add(args, env, scope):
    args[0] = interp.interp0(args[0], env, scope)[0]
    return functools.reduce(lambda x, y: x + interp.interp0(y, env, scope)[0], args)

def _sub(args, env, scope):
    args[0] = interp.interp0(args[0], env, scope)[0]
    return functools.reduce(lambda x, y: x - interp.interp0(y, env, scope)[0], args)

def _mul(args, env, scope):
    args[0] = interp.interp0(args[0], env, scope)[0]
    return functools.reduce(lambda x, y: x * interp.interp0(y, env, scope)[0], args)

def _div(args, env, scope):
    args[0] = interp.interp0(args[0], env, scope)[0]
    return functools.reduce(lambda x, y: x / interp.interp0(y, env, scope)[0], args)

def _mod(args, env, scope):
    args[0] = interp.interp0(args[0], env, scope)[0]
    return functools.reduce(lambda x, y: x % interp.interp0(y, env, scope)[0], args)

def _do(args, env, scope):
    # (do ...)
    # print("do:", scope)
    res = None
    for i in args:
        # print(":", args)
        res = interp.interp0(i, env, scope)
    return res[0]

def _def(args, env, scope):
    # (def <name> <val>)
    # (def (<name> <args>) <body>) => (def <name> (lambda (<args>) <body>))
    if isinstance(args[0], list):
        env.set(scope[1], args[0][0], _fn((args[0][1:], args[1]), env, scope[1]))
    else:
        env.set(scope[1], str(args[0]), interp.interp0(args[1], env, scope[1])[0])

def _fn(args, env, scope):
    # (fn (<fun args>) <fun body>)
    return func.Func(args[0], args[1], scope)

def _print(args, env, scope):
    # (print ...)
    def _tostr(s):
        v = interp.interp0(s, env, scope)[0]
        if interp.is_string(s):
            return v.val
        else:
            return str(v)
    res = map(_tostr, args)
    print(" ".join(res))

def _eq(args, env, scope):
    return interp.interp0(args[0], env, scope)[0] == interp.interp0(args[1], env, scope)[0]

def _if(args, env, scope):
    # (if <b> <t> <f>)
    if interp.interp0(args[0], env, scope)[0]:
        return interp.interp0(args[1], env, scope)[0]
    else:
        return interp.interp0(args[2], env, scope)[0]

def _let(args, env, scope):
    # (let ((<name> <value>) ... ) <body>)
    for i in args[0]:
        env.set(
            scope, # scope
            i[0], # var name
            interp.interp0(i[1], env, scope)[0]) # value
    return interp.interp0(args[1], env, scope)[0]

def init_env(env):
    env.set(None, "do", func.PyFunc(_do))
    env.set(None, "print", func.PyFunc(_print))
    env.set(None, "def", func.PyFunc(_def))
    env.set(None, "let", func.PyFunc(_let))
    env.set(None, "fn", func.PyFunc(_fn))
    env.set(None, "if", func.PyFunc(_if))
    env.set(None, "+", func.PyFunc(_add))
    env.set(None, "-", func.PyFunc(_sub))
    env.set(None, "*", func.PyFunc(_mul))
    env.set(None, "/", func.PyFunc(_div))
    env.set(None, "%", func.PyFunc(_mod))
    env.set(None, "=", func.PyFunc(_eq))
    env.set(None, "#t", True)
    env.set(None, "#f", False)
<commit_msg>總感覺前幾個commit在繞圈子DATE:~0,10TIME:~0,8 UTC+8<commit_after>import interp
import functools
import func

class String():
    def __init__(self, val):
        self.val = val

class Env():
    def __init__(self):
        self.env = dict()
        init_env(self)

    def get(self, scope, name):
        while True:
            try:
                # print(scope)
                # print("get:", (name, scope), "\n  ->")
                tmp = self.env[(name, scope)]
                # print("get:", (name, scope), "\n  ->", tmp)
                return tmp
            except KeyError:
                if scope is None:
                    raise KeyError()
                else:
                    scope = scope[1]

    def set(self, scope, name, val):
        # print("set:", (name, scope), "\n  ->", val)
        if (name, scope) in self.env:
            # print((name, scope), "is already defined")
            assert (name, scope) not in self.env
        self.env[(name, scope)] = val

def _add(args, env, scope):
    args[0] = interp.interp0(args[0], env, scope)[0]
    return functools.reduce(lambda x, y: x + interp.interp0(y, env, scope)[0], args)

def _sub(args, env, scope):
    args[0] = interp.interp0(args[0], env, scope)[0]
    return functools.reduce(lambda x, y: x - interp.interp0(y, env, scope)[0], args)

def _mul(args, env, scope):
    args[0] = interp.interp0(args[0], env, scope)[0]
    return functools.reduce(lambda x, y: x * interp.interp0(y, env, scope)[0], args)

def _div(args, env, scope):
    args[0] = interp.interp0(args[0], env, scope)[0]
    return functools.reduce(lambda x, y: x / interp.interp0(y, env, scope)[0], args)

def _mod(args, env, scope):
    args[0] = interp.interp0(args[0], env, scope)[0]
    return functools.reduce(lambda x, y: x % interp.interp0(y, env, scope)[0], args)

def _do(args, env, scope):
    # (do ...)
    # print("do:", scope)
    res = None
    for i in args:
        # print(":", args)
        res = interp.interp0(i, env, scope)
    return res[0]

def _def(args, env, scope):
    # (def <name> <val>)
    # (def (<name> <args>) <body>) => (def <name> (lambda (<args>) <body>))
    if isinstance(args[0], list):
        env.set(scope[1], args[0][0], _fn((args[0][1:], args[1]), env, scope[1]))
    else:
        env.set(scope[1], str(args[0]), interp.interp0(args[1], env, scope[1])[0])

def _fn(args, env, scope):
    # (fn (<fun args>) <fun body>)
    return func.Func(args[0], args[1], scope)

def _print(args, env, scope):
    # (print ...)
    def _tostr(s):
        v = interp.interp0(s, env, scope)[0]
        if interp.is_string(s):
            return v.val
        else:
            return str(v)
    res = map(_tostr, args)
    print(" ".join(res))

def _eq(args, env, scope):
    return interp.interp0(args[0], env, scope)[0] == interp.interp0(args[1], env, scope)[0]

def _if(args, env, scope):
    # (if <b> <t> <f>)
    if interp.interp0(args[0], env, scope)[0]:
        return interp.interp0(args[1], env, scope)[0]
    else:
        return interp.interp0(args[2], env, scope)[0]

def _let(args, env, scope):
    # (let ((<name> <value>) ... ) <body>)
    for i in args[0]:
        env.set(
            scope, # scope
            i[0], # var name
            interp.interp0(i[1], env, scope)[0]) # value
    return interp.interp0(args[1], env, scope)[0]

def init_env(env):
    env.set(None, "do", func.PyFunc(_do))
    env.set(None, "print", func.PyFunc(_print))
    env.set(None, "def", func.PyFunc(_def))
    env.set(None, "let", func.PyFunc(_let))
    env.set(None, "fn", func.PyFunc(_fn))
    env.set(None, "if", func.PyFunc(_if))
    env.set(None, "+", func.PyFunc(_add))
    env.set(None, "-", func.PyFunc(_sub))
    env.set(None, "*", func.PyFunc(_mul))
    env.set(None, "/", func.PyFunc(_div))
    env.set(None, "%", func.PyFunc(_mod))
    env.set(None, "=", func.PyFunc(_eq))
    env.set(None, "#t", True)
    env.set(None, "#f", False)<|endoftext|>


<commit_before>from __future__ import absolute_import

import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "api.api.settings")

import logging

from scrapi import events
from scrapi.processing.base import BaseProcessor

from api.webview.models import Document

logger = logging.getLogger(__name__)

class PostgresProcessor(BaseProcessor):
    NAME = 'postgres'

    @events.logged(events.PROCESSING, 'raw.postgres')
    def process_raw(self, raw_doc):
        source, docID = raw_doc['source'], raw_doc['docID']
        document = self._get_by_source_id(Document, source, docID) or Document(source=source, docID=docID)

        raw_doc.attributes.pop('versions')  # TODO Versions not saved to postgres right now
        document.raw = raw_doc.attributes

        document.save()

    @events.logged(events.PROCESSING, 'normalized.postgres')
    def process_normalized(self, raw_doc, normalized):
        source, docID = raw_doc['source'], raw_doc['docID']
        document = self._get_by_source_id(Document, source, docID) or Document(source=source, docID=docID)

        document.normalized = normalized.attributes
        document.providerUpdatedDateTime = normalized['providerUpdatedDateTime']

        document.save()

    def _get_by_source_id(self, model, source, docID):
        try:
            return Document.objects.filter(source=source, docID=docID)[0]
        except IndexError:
            return None
<commit_msg>Postgres processor update to save versions<commit_after>from __future__ import absolute_import

import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "api.api.settings")

import copy
import logging

from scrapi import events
from scrapi.processing.base import BaseProcessor

from api.webview.models import Document

logger = logging.getLogger(__name__)

class PostgresProcessor(BaseProcessor):
    NAME = 'postgres'

    @events.logged(events.PROCESSING, 'raw.postgres')
    def process_raw(self, raw_doc):
        source, docID = raw_doc['source'], raw_doc['docID']
        document = self._get_by_source_id(Document, source, docID) or Document(source=source, docID=docID)

        modified_doc = copy.deepcopy(raw_doc.attributes)
        if modified_doc.get('versions'):
            modified_doc['versions'] = str(modified_doc['versions'])

        document.raw = modified_doc

        document.save()

    @events.logged(events.PROCESSING, 'normalized.postgres')
    def process_normalized(self, raw_doc, normalized):
        source, docID = raw_doc['source'], raw_doc['docID']
        document = self._get_by_source_id(Document, source, docID) or Document(source=source, docID=docID)

        document.normalized = normalized.attributes
        document.providerUpdatedDateTime = normalized['providerUpdatedDateTime']

        document.save()

    def _get_by_source_id(self, model, source, docID):
        try:
            return Document.objects.filter(source=source, docID=docID)[0]
        except IndexError:
            return None
<|endoftext|>


<commit_before># encoding: utf-8

import os

NAME = 'kcorrect'

KCORRECT_DIR = os.path.dirname(__file__)

__version__ = '5.0.0'
<commit_msg>version 5.0.0b<commit_after># encoding: utf-8

import os

NAME = 'kcorrect'

KCORRECT_DIR = os.path.dirname(__file__)

__version__ = '5.0.0b'
<|endoftext|>


<commit_before># TmServer - TissueMAPS server application.
# Copyright (C) 2016  Markus D. Herrmann, University of Zurich and Robin Hafen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""API view functions for workflow management."""
import json
import os
from cStringIO import StringIO
import logging
import numpy as np
from flask import jsonify, send_file, current_app, request
from flask_jwt import jwt_required
from flask_jwt import current_identity

import tmlib.models as tm
from tmlib.workflow.description import WorkflowDescription
from tmlib.workflow.submission import SubmissionManager
from tmlib.workflow.workflow import Workflow
from tmlib.log import LEVELS_TO_VERBOSITY
from tmlib import cfg as lib_cfg

from tmserver.util import decode_query_ids, decode_form_ids
from tmserver.util import assert_query_params, assert_form_params
from tmserver.model import decode_pk
from tmserver.model import encode_pk
from tmserver.extensions import gc3pie
from tmserver.api import api
from tmserver.error import (
    MalformedRequestError,
    MissingGETParameterError,
    MissingPOSTParameterError,
    ResourceNotFoundError,
    NotAuthorizedError
)
from tmserver import cfg as server_cfg

logger = logging.getLogger(__name__)

@api.route('/experiments/<experiment_id>/workflow/submit', methods=['POST'])
@jwt_required()
@assert_form_params('description')
@decode_query_ids('write')
def submit_workflow(experiment_id):
    """
    .. http:post:: /api/experiments/(string:experiment_id)/workflow/submit

        Submit a workflow based on a ``WorkflowDescription``.
        Please refer to the respective class documention for more details on how to
        structure such a description object.

        **Example request**:

        .. sourcecode:: http

            Content-Type: application/json

            {
                "description": {...}
            }

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "message": "ok",
                "submission_id": 1
            }

        :statuscode 200: no error

    """
    logger.info('submit workflow for experiment %d', experiment_id)
    data = request.get_json()
    # data = json.loads(request.data)
    workflow_description = WorkflowDescription(**data['description'])
    with tm.utils.MainSession() as session:
        experiment = session.query(tm.ExperimentReference).get(experiment_id)
        experiment.persist_workflow_description(workflow_description)
    submission_manager = SubmissionManager(experiment_id, 'workflow')
    submission_id, user_name = submission_manager.register_submission()
    verbosity = LEVELS_TO_VERBOSITY[server_cfg.log_level]
    workflow = Workflow(
        experiment_id=experiment_id,
        verbosity=verbosity,
        submission_id=submission_id,
        user_name=user_name,
        description=workflow_description
    )
    gc3pie.store_jobs(workflow)
    gc3pie.submit_jobs(workflow)

    return jsonify({
        'message': 'ok',
        'submission_id': workflow.submission_id
    })

@api.route('/experiments/<experiment_id>/workflow/resubmit', methods=['POST'])
@jwt_required()
@assert_form_params('description')
@decode_query_ids('write')
def resubmit_workflow(experiment_id):
    """
    .. http:post:: /api/experiments/(string:experiment_id)/workflow/resubmit

        Resubmit a workflow for an experiment providing a new ``WorkflowDescription``.
        Please refer to the respective class documention for more details on how to
        structure such a description object.

        **Example request**:

        .. sourcecode:: http

            Content-Type: application/json

            {
                "description": {...}
            }

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "message": "ok",
                "submission_id": 1
            }

        :statuscode 200: no error

    """
    logger.info('resubmit workflow for experiment %d', experiment_id)
    data = json.loads(request.data)
    index = data.get('index', 0)
    workflow_description = WorkflowDescription(**data['description'])
    workflow = gc3pie.retrieve_jobs(experiment_id, 'workflow')
    workflow.update_description(workflow_description)
    workflow.update_stage(index)
    gc3pie.resubmit_jobs(workflow, index)
    return jsonify({
        'message': 'ok',
        'submission_id': workflow.submission_id
    })

@api.route(
    '/experiments/<experiment_id>/workflow/status', methods=['GET']
)
@jwt_required()
@decode_query_ids('read')
def get_workflow_status(experiment_id):
    """
    .. http:get:: /api/experiments/(string:experiment_id)/workflow/status

        Query the status for the currently running workflow for the specified experiment.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": status # TODO
            }

        :statuscode 200: no error

    """
    logger.info('get workflow status for experiment %d', experiment_id)
    workflow = gc3pie.retrieve_jobs(experiment_id, 'workflow')
    status = gc3pie.get_status_of_submitted_jobs(workflow, 2)
    return jsonify({
        'data': status
    })

@api.route(
    '/experiments/<experiment_id>/workflow/status/jobs', methods=['GET']
)
@jwt_required()
@assert_query_params('step_name')
@decode_query_ids('read')
def get_jobs_status(experiment_id):
    """
    .. http:get:: /api/experiments/(string:experiment_id)/workflow/jobs

        Query the status of n jobs currently associated with step.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": [
                    {
                        "name": "Job X1",
                        "state": "RUNNING",
                        "exitcode": null,
                        "memory": 1024,
                        "time": "1:21:33"
                    },
                    ...
                ]
            }

        :query step_name: the name of the step (required)
        :query index: the index of the first job queried (optional)
        :query batch_size: the amount of job stati to return starting from ``index``.
        :statuscode 200: no error

    """
    step_name = request.args.get('step_name')
    index = request.args.get('index', 0, type=int)
    batch_size = request.args.get('batch_size', type=int)

    logger.info(
        'get status of jobs for workflow step "%s" for experiment %d',
        step_name, experiment_id
    )

    # If the index is negative don't send `batch_size` jobs.
    # For example, if the index is -5 and the batch_size 50,
    # send the first 45 jobs back.
    if index < 0 and batch_size is not None:
        batch_size = batch_size + index
        index = 0
        if batch_size <= 0:
            return jsonify({
                'data': []
            })

    submission_id = gc3pie.get_id_of_last_submission(experiment_id, 'workflow')
    # TODO: Upon reload, the submission_id of tasks doesn't get updated.
    # While this makes sense to track tasks belonging to the same collection
    # it doesn't allow the differentiation of submissions (as the name implies).
    with tm.utils.MainSession() as session:
        step_task_id = session.query(tm.Task.id).\
            filter(
                tm.Task.submission_id == submission_id,
                tm.Task.name == step_name,
                tm.Task.is_collection
            ).\
            one_or_none()
        if step_task_id is None:
            status = []
        else:
            step = gc3pie.retrieve_single_job(step_task_id)
            if len(step.tasks) == 0:
                status = []
            else:
                task_ids = []
                for phase in step.tasks:
                    if hasattr(phase, 'tasks'):
                        if len(phase.tasks) == 0:
                            continue
                        if hasattr(phase.tasks[0], 'tasks'):
                            for subphase in phase.tasks:
                                task_ids.extend([
                                    t.persistent_id for t in subphase.tasks
                                ])
                        else:
                            task_ids.extend([
                                t.persistent_id for t in phase.tasks
                            ])
                    else:
                        task_ids.append(phase.persistent_id)
                tasks = session.query(tm.Task).\
                    filter(
                        tm.Task.id.in_(task_ids),
                        ~tm.Task.is_collection
                    )
                if batch_size is not None:
                    logger.debug('query status of %d jobs', batch_size)
                    tasks = tasks.limit(batch_size)
                tasks = tasks.\
                    offset(index).\
                    all()
                # order_by(tm.Task.id).\
                tasks = sorted(tasks, key=lambda k: k.name)
                status = [t.status for t in tasks]

    return jsonify(data=status)

@api.route('/experiments/<experiment_id>/workflow/description', methods=['GET'])
@jwt_required()
@decode_query_ids('read')
def get_workflow_description(experiment_id):
    """
    .. http:get:: /api/experiments/(string:experiment_id)/workflow/description

        Get the workflow description for the experiment with id ``experiment_id``.
        Please refer to the respective documentation to see how such description objects
        are structured.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": {...}
            }

        :statuscode 200: no error

    """
    logger.info('get workflow description for experiment %d', experiment_id)
    with tm.utils.MainSession() as session:
        experiment = session.query(tm.ExperimentReference).get(experiment_id)
        description = experiment.workflow_description
    return jsonify({
        'data': description.to_dict()
    })

@api.route('/experiments/<experiment_id>/workflow/description', methods=['POST'])
@jwt_required()
@assert_form_params('description')
@decode_query_ids('write')
def save_workflow_description(experiment_id):
    """
    .. http:post:: /api/experiments/(string:experiment_id)/workflow/description

        Save a new workflow description for the specified experiment.

        **Example request**:

        .. sourcecode:: http

            Content-Type: application/json

            {
                "description": {...}
            }

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "message": "ok"
            }

        :statuscode 200: no error

    """
    logger.info('save workflow description for experiment %d', experiment_id)
    data = request.get_json()
    workflow_description = WorkflowDescription(**data['description'])
    with tm.utils.MainSession() as session:
        experiment = session.query(tm.ExperimentReference).get(experiment_id)
        experiment.persist_workflow_description(workflow_description)
    return jsonify({
        'message': 'ok'
    })

@api.route('/experiments/<experiment_id>/workflow/kill', methods=['POST'])
@jwt_required()
@decode_query_ids('write')
def kill_workflow(experiment_id):
    """
    .. http:post:: /api/experiments/(string:experiment_id)/workflow/kill

        Kill all jobs of the currently running workflow.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "message": "ok"
            }

        :statuscode 200: no error

    """
    logger.info('kill workflow for experiment %d', experiment_id)
    workflow = gc3pie.retrieve_jobs(experiment_id, 'workflow')
    gc3pie.kill_jobs(workflow)
    return jsonify({
        'message': 'ok'
    })

@api.route('/experiments/<experiment_id>/workflow/log', methods=['POST'])
@jwt_required()
@assert_form_params('job_id')
@decode_query_ids('read')
def get_job_log_output(experiment_id):
    """
    .. http:post:: /api/experiments/(string:experiment_id)/workflow/log

        Get the log file for a specific job.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "message": "ok",
                "stdout": string,
                "stderr": string
            }

        :statuscode 200: no error

    """
    data = request.get_json()
    job_id = data['job_id']
    logger.info(
        'get job log output for experiment %d and job %d',
        experiment_id, job_id
    )
    # NOTE: This is the persistent task ID of the job
    job = gc3pie.retrieve_single_job(job_id)
    stdout_file = os.path.join(job.output_dir, job.stdout)
    with open(stdout_file, 'r') as f:
        out = f.read()
    stderr_file = os.path.join(job.output_dir, job.stderr)
    with open(stderr_file, 'r') as f:
        err = f.read()
    return jsonify({
        'message': 'ok',
        'stdout': out,
        'stderr': err
    })

<commit_msg>Fix bug related to setting logging level<commit_after># TmServer - TissueMAPS server application.
# Copyright (C) 2016  Markus D. Herrmann, University of Zurich and Robin Hafen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""API view functions for workflow management."""
import json
import os
from cStringIO import StringIO
import logging
import numpy as np
from flask import jsonify, send_file, current_app, request
from flask_jwt import jwt_required
from flask_jwt import current_identity

import tmlib.models as tm
from tmlib.workflow.description import WorkflowDescription
from tmlib.workflow.submission import SubmissionManager
from tmlib.workflow.workflow import Workflow
from tmlib import cfg as lib_cfg

from tmserver.util import decode_query_ids, decode_form_ids
from tmserver.util import assert_query_params, assert_form_params
from tmserver.model import decode_pk
from tmserver.model import encode_pk
from tmserver.extensions import gc3pie
from tmserver.api import api
from tmserver.error import (
    MalformedRequestError,
    MissingGETParameterError,
    MissingPOSTParameterError,
    ResourceNotFoundError,
    NotAuthorizedError
)
from tmserver import cfg as server_cfg

logger = logging.getLogger(__name__)

@api.route('/experiments/<experiment_id>/workflow/submit', methods=['POST'])
@jwt_required()
@assert_form_params('description')
@decode_query_ids('write')
def submit_workflow(experiment_id):
    """
    .. http:post:: /api/experiments/(string:experiment_id)/workflow/submit

        Submit a workflow based on a ``WorkflowDescription``.
        Please refer to the respective class documention for more details on how to
        structure such a description object.

        **Example request**:

        .. sourcecode:: http

            Content-Type: application/json

            {
                "description": {...}
            }

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "message": "ok",
                "submission_id": 1
            }

        :statuscode 200: no error

    """
    logger.info('submit workflow for experiment %d', experiment_id)
    data = request.get_json()
    # data = json.loads(request.data)
    workflow_description = WorkflowDescription(**data['description'])
    with tm.utils.MainSession() as session:
        experiment = session.query(tm.ExperimentReference).get(experiment_id)
        experiment.persist_workflow_description(workflow_description)
    submission_manager = SubmissionManager(experiment_id, 'workflow')
    submission_id, user_name = submission_manager.register_submission()
    workflow = Workflow(
        experiment_id=experiment_id,
        verbosity=server_cfg.logging_verbosity,
        submission_id=submission_id,
        user_name=user_name,
        description=workflow_description
    )
    gc3pie.store_jobs(workflow)
    gc3pie.submit_jobs(workflow)

    return jsonify({
        'message': 'ok',
        'submission_id': workflow.submission_id
    })

@api.route('/experiments/<experiment_id>/workflow/resubmit', methods=['POST'])
@jwt_required()
@assert_form_params('description')
@decode_query_ids('write')
def resubmit_workflow(experiment_id):
    """
    .. http:post:: /api/experiments/(string:experiment_id)/workflow/resubmit

        Resubmit a workflow for an experiment providing a new ``WorkflowDescription``.
        Please refer to the respective class documention for more details on how to
        structure such a description object.

        **Example request**:

        .. sourcecode:: http

            Content-Type: application/json

            {
                "description": {...}
            }

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "message": "ok",
                "submission_id": 1
            }

        :statuscode 200: no error

    """
    logger.info('resubmit workflow for experiment %d', experiment_id)
    data = json.loads(request.data)
    index = data.get('index', 0)
    workflow_description = WorkflowDescription(**data['description'])
    workflow = gc3pie.retrieve_jobs(experiment_id, 'workflow')
    workflow.update_description(workflow_description)
    workflow.update_stage(index)
    gc3pie.resubmit_jobs(workflow, index)
    return jsonify({
        'message': 'ok',
        'submission_id': workflow.submission_id
    })

@api.route(
    '/experiments/<experiment_id>/workflow/status', methods=['GET']
)
@jwt_required()
@decode_query_ids('read')
def get_workflow_status(experiment_id):
    """
    .. http:get:: /api/experiments/(string:experiment_id)/workflow/status

        Query the status for the currently running workflow for the specified experiment.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": status # TODO
            }

        :statuscode 200: no error

    """
    logger.info('get workflow status for experiment %d', experiment_id)
    workflow = gc3pie.retrieve_jobs(experiment_id, 'workflow')
    status = gc3pie.get_status_of_submitted_jobs(workflow, 2)
    return jsonify({
        'data': status
    })

@api.route(
    '/experiments/<experiment_id>/workflow/status/jobs', methods=['GET']
)
@jwt_required()
@assert_query_params('step_name')
@decode_query_ids('read')
def get_jobs_status(experiment_id):
    """
    .. http:get:: /api/experiments/(string:experiment_id)/workflow/jobs

        Query the status of n jobs currently associated with step.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": [
                    {
                        "name": "Job X1",
                        "state": "RUNNING",
                        "exitcode": null,
                        "memory": 1024,
                        "time": "1:21:33"
                    },
                    ...
                ]
            }

        :query step_name: the name of the step (required)
        :query index: the index of the first job queried (optional)
        :query batch_size: the amount of job stati to return starting from ``index``.
        :statuscode 200: no error

    """
    step_name = request.args.get('step_name')
    index = request.args.get('index', 0, type=int)
    batch_size = request.args.get('batch_size', type=int)

    logger.info(
        'get status of jobs for workflow step "%s" for experiment %d',
        step_name, experiment_id
    )

    # If the index is negative don't send `batch_size` jobs.
    # For example, if the index is -5 and the batch_size 50,
    # send the first 45 jobs back.
    if index < 0 and batch_size is not None:
        batch_size = batch_size + index
        index = 0
        if batch_size <= 0:
            return jsonify({
                'data': []
            })

    submission_id = gc3pie.get_id_of_last_submission(experiment_id, 'workflow')
    # TODO: Upon reload, the submission_id of tasks doesn't get updated.
    # While this makes sense to track tasks belonging to the same collection
    # it doesn't allow the differentiation of submissions (as the name implies).
    with tm.utils.MainSession() as session:
        step_task_id = session.query(tm.Task.id).\
            filter(
                tm.Task.submission_id == submission_id,
                tm.Task.name == step_name,
                tm.Task.is_collection
            ).\
            one_or_none()
        if step_task_id is None:
            status = []
        else:
            step = gc3pie.retrieve_single_job(step_task_id)
            if len(step.tasks) == 0:
                status = []
            else:
                task_ids = []
                for phase in step.tasks:
                    if hasattr(phase, 'tasks'):
                        if len(phase.tasks) == 0:
                            continue
                        if hasattr(phase.tasks[0], 'tasks'):
                            for subphase in phase.tasks:
                                task_ids.extend([
                                    t.persistent_id for t in subphase.tasks
                                ])
                        else:
                            task_ids.extend([
                                t.persistent_id for t in phase.tasks
                            ])
                    else:
                        task_ids.append(phase.persistent_id)
                tasks = session.query(tm.Task).\
                    filter(
                        tm.Task.id.in_(task_ids),
                        ~tm.Task.is_collection
                    )
                if batch_size is not None:
                    logger.debug('query status of %d jobs', batch_size)
                    tasks = tasks.limit(batch_size)
                tasks = tasks.\
                    offset(index).\
                    all()
                # order_by(tm.Task.id).\
                tasks = sorted(tasks, key=lambda k: k.name)
                status = [t.status for t in tasks]

    return jsonify(data=status)

@api.route('/experiments/<experiment_id>/workflow/description', methods=['GET'])
@jwt_required()
@decode_query_ids('read')
def get_workflow_description(experiment_id):
    """
    .. http:get:: /api/experiments/(string:experiment_id)/workflow/description

        Get the workflow description for the experiment with id ``experiment_id``.
        Please refer to the respective documentation to see how such description objects
        are structured.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "data": {...}
            }

        :statuscode 200: no error

    """
    logger.info('get workflow description for experiment %d', experiment_id)
    with tm.utils.MainSession() as session:
        experiment = session.query(tm.ExperimentReference).get(experiment_id)
        description = experiment.workflow_description
    return jsonify({
        'data': description.to_dict()
    })

@api.route('/experiments/<experiment_id>/workflow/description', methods=['POST'])
@jwt_required()
@assert_form_params('description')
@decode_query_ids('write')
def save_workflow_description(experiment_id):
    """
    .. http:post:: /api/experiments/(string:experiment_id)/workflow/description

        Save a new workflow description for the specified experiment.

        **Example request**:

        .. sourcecode:: http

            Content-Type: application/json

            {
                "description": {...}
            }

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "message": "ok"
            }

        :statuscode 200: no error

    """
    logger.info('save workflow description for experiment %d', experiment_id)
    data = request.get_json()
    workflow_description = WorkflowDescription(**data['description'])
    with tm.utils.MainSession() as session:
        experiment = session.query(tm.ExperimentReference).get(experiment_id)
        experiment.persist_workflow_description(workflow_description)
    return jsonify({
        'message': 'ok'
    })

@api.route('/experiments/<experiment_id>/workflow/kill', methods=['POST'])
@jwt_required()
@decode_query_ids('write')
def kill_workflow(experiment_id):
    """
    .. http:post:: /api/experiments/(string:experiment_id)/workflow/kill

        Kill all jobs of the currently running workflow.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "message": "ok"
            }

        :statuscode 200: no error

    """
    logger.info('kill workflow for experiment %d', experiment_id)
    workflow = gc3pie.retrieve_jobs(experiment_id, 'workflow')
    gc3pie.kill_jobs(workflow)
    return jsonify({
        'message': 'ok'
    })

@api.route('/experiments/<experiment_id>/workflow/log', methods=['POST'])
@jwt_required()
@assert_form_params('job_id')
@decode_query_ids('read')
def get_job_log_output(experiment_id):
    """
    .. http:post:: /api/experiments/(string:experiment_id)/workflow/log

        Get the log file for a specific job.

        **Example response**:

        .. sourcecode:: http

            HTTP/1.1 200 OK
            Content-Type: application/json

            {
                "message": "ok",
                "stdout": string,
                "stderr": string
            }

        :statuscode 200: no error

    """
    data = request.get_json()
    job_id = data['job_id']
    logger.info(
        'get job log output for experiment %d and job %d',
        experiment_id, job_id
    )
    # NOTE: This is the persistent task ID of the job
    job = gc3pie.retrieve_single_job(job_id)
    stdout_file = os.path.join(job.output_dir, job.stdout)
    with open(stdout_file, 'r') as f:
        out = f.read()
    stderr_file = os.path.join(job.output_dir, job.stderr)
    with open(stderr_file, 'r') as f:
        err = f.read()
    return jsonify({
        'message': 'ok',
        'stdout': out,
        'stderr': err
    })

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Based on AboutNil in the Ruby Koans
#

from runner.koan import *

class AboutNone(Koan):

    def test_none_is_an_object(self):
        "Unlike NULL in a lot of languages"
        self.assertEqual(__, isinstance(None, object))

    def test_none_is_universal(self):
        "There is only one None"
        self.assertEqual(____, None is None)

    def test_what_exception_do_you_get_when_calling_nonexistent_methods(self):
        """
        What is the Exception that is thrown when you call a method that does
        not exist?

        Hint: launch python command console and try the code in the block below.

        Don't worry about what 'try' and 'except' do, we'll talk about this later
        """
        try:
            None.some_method_none_does_not_know_about()
        except Exception as ex:
            ex2 = ex

        # What exception has been caught?
        #
        # Need a recap on how to evaluate __class__ attributes?
        #
        #     http://bit.ly/__class__

        self.assertEqual(__, ex2.__class__)

        # What message was attached to the exception?
        # (HINT: replace __ with part of the error message.)
        self.assertRegex(ex2.args[0], __)

    def test_none_is_distinct(self):
        """
        None is distinct from other things which are False.
        """
        self.assertEqual(__, None is not 0)
        self.assertEqual(__, None is not False)
<commit_msg>Completed about_none<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Based on AboutNil in the Ruby Koans
#

from runner.koan import *

class AboutNone(Koan):

    def test_none_is_an_object(self):
        "Unlike NULL in a lot of languages"
        self.assertEqual(True, isinstance(None, object))

    def test_none_is_universal(self):
        "There is only one None"
        self.assertEqual(True, None is None)

    def test_what_exception_do_you_get_when_calling_nonexistent_methods(self):
        """
        What is the Exception that is thrown when you call a method that does
        not exist?

        Hint: launch python command console and try the code in the block below.

        Don't worry about what 'try' and 'except' do, we'll talk about this later
        """
        try:
            None.some_method_none_does_not_know_about()
        except Exception as ex:
            ex2 = ex

        # What exception has been caught?
        #
        # Need a recap on how to evaluate __class__ attributes?
        #
        #     http://bit.ly/__class__

        self.assertEqual(AttributeError, ex2.__class__)

        # What message was attached to the exception?
        # (HINT: replace __ with part of the error message.)
        self.assertRegex(ex2.args[0], "'NoneType' object has no attribute 'some_method_none_does_not_know_about'"
)

    def test_none_is_distinct(self):
        """
        None is distinct from other things which are False.
        """
        self.assertEqual(True, None is not 0)
        self.assertEqual(True, None is not False)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: iso-8859-1 -*-
# vi:ts=4:et
# $Id$

import sys, threading, Queue
import pycurl

# We should ignore SIGPIPE when using pycurl.NOSIGNAL - see the libcurl
# documentation `libcurl-the-guide' for more info.
try:
    import signal
    from signal import SIGPIPE, SIG_IGN
    signal.signal(signal.SIGPIPE, signal.SIG_IGN)
except ImportError:
    pass

class WorkerThread(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue

    def run(self):
        while 1:
            try:
                url, filename = self.queue.get_nowait()
            except Queue.Empty:
                raise SystemExit
            f = open(filename, "wb")
            curl = pycurl.Curl()
            curl.setopt(pycurl.HTTPHEADER, ["User-Agent: PycURL"])
            curl.setopt(pycurl.FOLLOWLOCATION, 1)
            curl.setopt(pycurl.MAXREDIRS, 5)
            curl.setopt(pycurl.URL, url)
            curl.setopt(pycurl.WRITEDATA, f)
            curl.setopt(pycurl.NOSIGNAL, 1)
            curl.setopt(pycurl.CONNECTTIMEOUT, 30)
            try:
                curl.perform()
            except:
                import traceback
                traceback.print_exc(file=sys.stderr)
                sys.stderr.flush()
            curl.close()
            f.close()
            sys.stdout.write(".")
            sys.stdout.flush()

# Read list of URLs from file specified on commandline
try:
    urls = open(sys.argv[1]).readlines()
    num_workers = int(sys.argv[2])
except:
    # File or number of workers was not specified, show usage string
    print "Usage: %s <file with URLs to fetch> <number of worker threads>" % sys.argv[0]
    raise SystemExit

# Initialize thread array and the file number used to store documents
threads = []
fileno = 0
queue = Queue.Queue()

# Fill the work input queue with URLs
for url in urls:
    fileno = fileno + 1
    filename = "data_%d" % (fileno,)
    queue.put((url, filename))

# Start a bunch of threads
for num_threads in range(num_workers):
    t = WorkerThread(queue)
    t.start()
    threads.append(t)

# Wait for all threads to finish
for thread in threads:
    thread.join()
<commit_msg>set default download time to 300 secs<commit_after>#! /usr/bin/env python
# -*- coding: iso-8859-1 -*-
# vi:ts=4:et
# $Id$

import sys, threading, Queue
import pycurl

# We should ignore SIGPIPE when using pycurl.NOSIGNAL - see the libcurl
# documentation `libcurl-the-guide' for more info.
try:
    import signal
    from signal import SIGPIPE, SIG_IGN
    signal.signal(signal.SIGPIPE, signal.SIG_IGN)
except ImportError:
    pass

class WorkerThread(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue

    def run(self):
        while 1:
            try:
                url, filename = self.queue.get_nowait()
            except Queue.Empty:
                raise SystemExit
            f = open(filename, "wb")
            curl = pycurl.Curl()
            curl.setopt(pycurl.HTTPHEADER, ["User-Agent: PycURL"])
            curl.setopt(pycurl.FOLLOWLOCATION, 1)
            curl.setopt(pycurl.MAXREDIRS, 5)
            curl.setopt(pycurl.URL, url)
            curl.setopt(pycurl.WRITEDATA, f)
            curl.setopt(pycurl.NOSIGNAL, 1)
            curl.setopt(pycurl.CONNECTTIMEOUT, 30)
            curl.setopt(pycurl.TIMEOUT, 300)
            try:
                curl.perform()
            except:
                import traceback
                traceback.print_exc(file=sys.stderr)
                sys.stderr.flush()
            curl.close()
            f.close()
            sys.stdout.write(".")
            sys.stdout.flush()

# Read list of URLs from file specified on commandline
try:
    urls = open(sys.argv[1]).readlines()
    num_workers = int(sys.argv[2])
except:
    # File or number of workers was not specified, show usage string
    print "Usage: %s <file with URLs to fetch> <number of worker threads>" % sys.argv[0]
    raise SystemExit

# Initialize thread array and the file number used to store documents
threads = []
fileno = 0
queue = Queue.Queue()

# Fill the work input queue with URLs
for url in urls:
    fileno = fileno + 1
    filename = "data_%d" % (fileno,)
    queue.put((url, filename))

# Start a bunch of threads
for num_threads in range(num_workers):
    t = WorkerThread(queue)
    t.start()
    threads.append(t)

# Wait for all threads to finish
for thread in threads:
    thread.join()
<|endoftext|>


<commit_before>import argparse
import copy
import numpy as np

import chainer
from chainer.datasets import TransformDataset
from chainer.optimizer import WeightDecay
from chainer import serializers
from chainer import training
from chainer.training import extensions
from chainer.training import triggers

from chainercv.datasets import voc_detection_label_names
from chainercv.datasets import VOCDetectionDataset
from chainercv.extensions import DetectionVOCEvaluator
from chainercv.links.model.ssd import ConcatenatedDataset
from chainercv.links.model.ssd import GradientScaling
from chainercv.links.model.ssd import multibox_loss
from chainercv.links import SSD300
from chainercv import transforms

from chainercv.links.model.ssd import random_crop_with_bbox_constraints
from chainercv.links.model.ssd import random_distort
from chainercv.links.model.ssd import resize_with_random_interpolation

class MultiboxTrainChain(chainer.Chain):

    def __init__(self, model, alpha=1, k=3):
        super(MultiboxTrainChain, self).__init__(model=model)
        self.alpha = alpha
        self.k = k

    def __call__(self, imgs, gt_mb_locs, gt_mb_labels):
        mb_locs, mb_confs = self.model(imgs)
        loc_loss, conf_loss = multibox_loss(
            mb_locs, mb_confs, gt_mb_locs, gt_mb_labels, self.k)
        loss = loc_loss * self.alpha + conf_loss

        chainer.reporter.report(
            {'loss': loss, 'loss/loc': loc_loss, 'loss/conf': conf_loss},
            self)

        return loss

class Transform(object):

    def __init__(self, coder, size, mean):
        # to send cpu, make a copy
        self.coder = copy.copy(coder)
        self.coder.to_cpu()

        self.size = size
        self.mean = mean

    def __call__(self, in_data):
        # There are five data augmentation steps
        # 1. Color augmentation
        # 2. Random expansion
        # 3. Random cropping
        # 4. Resizing with random interpolation
        # 5. Random horizontal flipping

        img, bbox, label = in_data

        # 1. Color augmentation
        img = random_distort(img)

        # 2. Random expansion
        if np.random.randint(2):
            img, param = transforms.random_expand(
                img, fill=self.mean, return_param=True)
            bbox = transforms.translate_bbox(
                bbox, y_offset=param['y_offset'], x_offset=param['x_offset'])

        # 3. Random cropping
        img, param = random_crop_with_bbox_constraints(
            img, bbox, return_param=True)
        bbox, param = transforms.crop_bbox(
            bbox, y_slice=param['y_slice'], x_slice=param['x_slice'],
            allow_outside_center=False, return_param=True)
        label = label[param['index']]

        # 4. Resizing with random interpolatation
        _, H, W = img.shape
        img = resize_with_random_interpolation(img, (self.size, self.size))
        bbox = transforms.resize_bbox(bbox, (H, W), (self.size, self.size))

        # 5. Random horizontal flipping
        img, params = transforms.random_flip(
            img, x_random=True, return_param=True)
        bbox = transforms.flip_bbox(
            bbox, (self.size, self.size), x_flip=params['x_flip'])

        # Preparation for SSD network
        img -= self.mean
        mb_loc, mb_label = self.coder.encode(bbox, label)

        return img, mb_loc, mb_label

def main():
    # Fix seed for stable training
    np.random.seed(0)

    parser = argparse.ArgumentParser()
    parser.add_argument('--batchsize', type=int, default=32)
    parser.add_argument('--gpu', type=int, default=-1)
    parser.add_argument('--out', default='result')
    parser.add_argument('--resume')
    args = parser.parse_args()

    model = SSD300(
        n_fg_class=len(voc_detection_label_names),
        pretrained_model='imagenet')
    model.use_preset('evaluate')
    train_chain = MultiboxTrainChain(model)
    if args.gpu >= 0:
        chainer.cuda.get_device(args.gpu).use()
        model.to_gpu()

    train = TransformDataset(
        ConcatenatedDataset(
            VOCDetectionDataset(year='2007', split='trainval'),
            VOCDetectionDataset(year='2012', split='trainval')
        ),
        Transform(model.coder, model.insize, model.mean))
    train_iter = chainer.iterators.MultiprocessIterator(train, args.batchsize)

    test = VOCDetectionDataset(
        year='2007', split='test',
        use_difficult=True, return_difficult=True)
    test_iter = chainer.iterators.SerialIterator(
        test, args.batchsize, repeat=False, shuffle=False)

    # initial lr is set to 1e-3 by ExponentialShift
    optimizer = chainer.optimizers.MomentumSGD()
    optimizer.setup(train_chain)
    for param in train_chain.params():
        if param.name == 'b':
            param.update_rule.add_hook(GradientScaling(2))
        else:
            param.update_rule.add_hook(WeightDecay(0.0005))

    updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu)
    trainer = training.Trainer(updater, (120000, 'iteration'), args.out)
    trainer.extend(
        extensions.ExponentialShift('lr', 0.1, init=1e-3),
        trigger=triggers.ManualScheduleTrigger([80000, 100000], 'iteration'))

    trainer.extend(
        DetectionVOCEvaluator(
            test_iter, model, use_07_metric=True,
            label_names=voc_detection_label_names),
        trigger=(10000, 'iteration'))

    log_interval = 10, 'iteration'
    trainer.extend(extensions.LogReport(trigger=log_interval))
    trainer.extend(extensions.observe_lr(), trigger=log_interval)
    trainer.extend(extensions.PrintReport(
        ['epoch', 'iteration', 'lr',
         'main/loss', 'main/loss/loc', 'main/loss/conf',
         'validation/main/map']),
        trigger=log_interval)
    trainer.extend(extensions.ProgressBar(update_interval=10))

    trainer.extend(extensions.snapshot(), trigger=(10000, 'iteration'))
    trainer.extend(
        extensions.snapshot_object(model, 'model_iter_{.updater.iteration}'),
        trigger=(120000, 'iteration'))

    if args.resume:
        serializers.load_npz(args.resume, trainer)

    trainer.run()

if __name__ == '__main__':
    main()
<commit_msg>change seed<commit_after>import argparse
import copy
import numpy as np

import chainer
from chainer.datasets import TransformDataset
from chainer.optimizer import WeightDecay
from chainer import serializers
from chainer import training
from chainer.training import extensions
from chainer.training import triggers

from chainercv.datasets import voc_detection_label_names
from chainercv.datasets import VOCDetectionDataset
from chainercv.extensions import DetectionVOCEvaluator
from chainercv.links.model.ssd import ConcatenatedDataset
from chainercv.links.model.ssd import GradientScaling
from chainercv.links.model.ssd import multibox_loss
from chainercv.links import SSD300
from chainercv import transforms

from chainercv.links.model.ssd import random_crop_with_bbox_constraints
from chainercv.links.model.ssd import random_distort
from chainercv.links.model.ssd import resize_with_random_interpolation

class MultiboxTrainChain(chainer.Chain):

    def __init__(self, model, alpha=1, k=3):
        super(MultiboxTrainChain, self).__init__(model=model)
        self.alpha = alpha
        self.k = k

    def __call__(self, imgs, gt_mb_locs, gt_mb_labels):
        mb_locs, mb_confs = self.model(imgs)
        loc_loss, conf_loss = multibox_loss(
            mb_locs, mb_confs, gt_mb_locs, gt_mb_labels, self.k)
        loss = loc_loss * self.alpha + conf_loss

        chainer.reporter.report(
            {'loss': loss, 'loss/loc': loc_loss, 'loss/conf': conf_loss},
            self)

        return loss

class Transform(object):

    def __init__(self, coder, size, mean):
        # to send cpu, make a copy
        self.coder = copy.copy(coder)
        self.coder.to_cpu()

        self.size = size
        self.mean = mean

    def __call__(self, in_data):
        # There are five data augmentation steps
        # 1. Color augmentation
        # 2. Random expansion
        # 3. Random cropping
        # 4. Resizing with random interpolation
        # 5. Random horizontal flipping

        img, bbox, label = in_data

        # 1. Color augmentation
        img = random_distort(img)

        # 2. Random expansion
        if np.random.randint(2):
            img, param = transforms.random_expand(
                img, fill=self.mean, return_param=True)
            bbox = transforms.translate_bbox(
                bbox, y_offset=param['y_offset'], x_offset=param['x_offset'])

        # 3. Random cropping
        img, param = random_crop_with_bbox_constraints(
            img, bbox, return_param=True)
        bbox, param = transforms.crop_bbox(
            bbox, y_slice=param['y_slice'], x_slice=param['x_slice'],
            allow_outside_center=False, return_param=True)
        label = label[param['index']]

        # 4. Resizing with random interpolatation
        _, H, W = img.shape
        img = resize_with_random_interpolation(img, (self.size, self.size))
        bbox = transforms.resize_bbox(bbox, (H, W), (self.size, self.size))

        # 5. Random horizontal flipping
        img, params = transforms.random_flip(
            img, x_random=True, return_param=True)
        bbox = transforms.flip_bbox(
            bbox, (self.size, self.size), x_flip=params['x_flip'])

        # Preparation for SSD network
        img -= self.mean
        mb_loc, mb_label = self.coder.encode(bbox, label)

        return img, mb_loc, mb_label

def main():
    # Fix seed for stable training
    np.random.seed(2017)

    parser = argparse.ArgumentParser()
    parser.add_argument('--batchsize', type=int, default=32)
    parser.add_argument('--gpu', type=int, default=-1)
    parser.add_argument('--out', default='result')
    parser.add_argument('--resume')
    args = parser.parse_args()

    model = SSD300(
        n_fg_class=len(voc_detection_label_names),
        pretrained_model='imagenet')
    model.use_preset('evaluate')
    train_chain = MultiboxTrainChain(model)
    if args.gpu >= 0:
        chainer.cuda.get_device(args.gpu).use()
        model.to_gpu()

    train = TransformDataset(
        ConcatenatedDataset(
            VOCDetectionDataset(year='2007', split='trainval'),
            VOCDetectionDataset(year='2012', split='trainval')
        ),
        Transform(model.coder, model.insize, model.mean))
    train_iter = chainer.iterators.MultiprocessIterator(train, args.batchsize)

    test = VOCDetectionDataset(
        year='2007', split='test',
        use_difficult=True, return_difficult=True)
    test_iter = chainer.iterators.SerialIterator(
        test, args.batchsize, repeat=False, shuffle=False)

    # initial lr is set to 1e-3 by ExponentialShift
    optimizer = chainer.optimizers.MomentumSGD()
    optimizer.setup(train_chain)
    for param in train_chain.params():
        if param.name == 'b':
            param.update_rule.add_hook(GradientScaling(2))
        else:
            param.update_rule.add_hook(WeightDecay(0.0005))

    updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu)
    trainer = training.Trainer(updater, (120000, 'iteration'), args.out)
    trainer.extend(
        extensions.ExponentialShift('lr', 0.1, init=1e-3),
        trigger=triggers.ManualScheduleTrigger([80000, 100000], 'iteration'))

    trainer.extend(
        DetectionVOCEvaluator(
            test_iter, model, use_07_metric=True,
            label_names=voc_detection_label_names),
        trigger=(10000, 'iteration'))

    log_interval = 10, 'iteration'
    trainer.extend(extensions.LogReport(trigger=log_interval))
    trainer.extend(extensions.observe_lr(), trigger=log_interval)
    trainer.extend(extensions.PrintReport(
        ['epoch', 'iteration', 'lr',
         'main/loss', 'main/loss/loc', 'main/loss/conf',
         'validation/main/map']),
        trigger=log_interval)
    trainer.extend(extensions.ProgressBar(update_interval=10))

    trainer.extend(extensions.snapshot(), trigger=(10000, 'iteration'))
    trainer.extend(
        extensions.snapshot_object(model, 'model_iter_{.updater.iteration}'),
        trigger=(120000, 'iteration'))

    if args.resume:
        serializers.load_npz(args.resume, trainer)

    trainer.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>
import test.test_support, unittest
import sys
import popen2

class CmdLineTest(unittest.TestCase):
    def start_python(self, cmd_line):
        outfp, infp = popen2.popen4('%s %s' % (sys.executable, cmd_line))
        infp.close()
        data = outfp.read()
        outfp.close()
        return data

    def test_directories(self):
        self.assertTrue('is a directory' in self.start_python('.'))
        self.assertTrue('is a directory' in self.start_python('< .'))

    def verify_valid_flag(self, cmd_line):
        data = self.start_python(cmd_line)
        self.assertTrue(data == '' or data.endswith('\n'))
        self.assertTrue('Traceback' not in data)

    def test_environment(self):
        self.verify_valid_flag('-E')

    def test_optimize(self):
        self.verify_valid_flag('-O')
        self.verify_valid_flag('-OO')

    def test_q(self):
        self.verify_valid_flag('-Qold')
        self.verify_valid_flag('-Qnew')
        self.verify_valid_flag('-Qwarn')
        self.verify_valid_flag('-Qwarnall')

    def test_site_flag(self):
        self.verify_valid_flag('-S')

    def test_usage(self):
        self.assertTrue('usage' in self.start_python('-h'))

    def test_version(self):
        version = 'Python %d.%d' % sys.version_info[:2]
        self.assertTrue(self.start_python('-V').startswith(version))

def test_main():
    test.test_support.run_unittest(CmdLineTest)

if __name__ == "__main__":
    test_main()
<commit_msg>test_directories():  This test had no chance of passing on Windows.  Hacked it to pass, but not sure it's worth the bother.<commit_after>
import test.test_support, unittest
import sys
import popen2

class CmdLineTest(unittest.TestCase):
    def start_python(self, cmd_line):
        outfp, infp = popen2.popen4('%s %s' % (sys.executable, cmd_line))
        infp.close()
        data = outfp.read()
        outfp.close()
        return data

    def test_directories(self):
        # Does this test make sense?  The message for "< ."  may depend on
        # the command shell, and the message for "." depends on the OS.
        if sys.platform.startswith("win"):
            # On WinXP w/ cmd.exe,
            #    "< ." gives "Access is denied.\n"
            #    "."   gives "C:\\Code\\python\\PCbuild\\python.exe: " +
            #                "can't open file '.':" +
            #                "[Errno 13] Permission denied\n"
            lookfor = " denied"  # common to both cases
        else:
            # This is what the test looked for originally, on all platforms.
            lookfor = "is a directory"
        self.assertTrue(lookfor in self.start_python('.'))
        self.assertTrue(lookfor in self.start_python('< .'))

    def verify_valid_flag(self, cmd_line):
        data = self.start_python(cmd_line)
        self.assertTrue(data == '' or data.endswith('\n'))
        self.assertTrue('Traceback' not in data)

    def test_environment(self):
        self.verify_valid_flag('-E')

    def test_optimize(self):
        self.verify_valid_flag('-O')
        self.verify_valid_flag('-OO')

    def test_q(self):
        self.verify_valid_flag('-Qold')
        self.verify_valid_flag('-Qnew')
        self.verify_valid_flag('-Qwarn')
        self.verify_valid_flag('-Qwarnall')

    def test_site_flag(self):
        self.verify_valid_flag('-S')

    def test_usage(self):
        self.assertTrue('usage' in self.start_python('-h'))

    def test_version(self):
        version = 'Python %d.%d' % sys.version_info[:2]
        self.assertTrue(self.start_python('-V').startswith(version))

def test_main():
    test.test_support.run_unittest(CmdLineTest)

if __name__ == "__main__":
    test_main()
<|endoftext|>


<commit_before>from .activity import Activity

class ReplyToActivity(Activity):
    def __init__(self, **kwargs):
        super(ReplyToActivity, self).__init__(reply_to_activity=True, **kwargs)

    def send(self):
        response_url = self.urljoin(self.serviceUrl,
                                    "/v3/conversations/{}/activities/{}".format(
                                    self.conversation['id'],
                                    self.activityId))

        response = self._request(response_url, 'post', self.to_dict())

        self.save_response('ReplyToActivity',
                           self.conversation['id'],
                           self.to_dict(),
                           {'conversationId': self.conversation['id'], 'activityId': self.activityId},
                           response.json())
        return response

class SendToConversation(Activity):
    def __init__(self, **kwargs):
        super(SendToConversation, self).__init__(**kwargs)

    def send(self):
        response_url = self.urljoin(self.serviceUrl,
                                    "/v3/conversations/{}/activities".format(
                                        self.conversation['id']))

        response = self._request(response_url, 'post', self.to_dict())

        self.save_response('SendToConversation',
                           self.conversation['id'],
                           self.to_dict(),
                           {'conversationId': self.conversation['id']},
                           response.json())
        return response

class DeleteActivity(Activity):
    def __init__(self, **kwargs):
        super(DeleteActivity, self).__init__(**kwargs)

    def send(self):
        response_url = self.urljoin(self.serviceUrl,
                                    "/v3/conversations/{}/activities/{}".format(
                                    self.conversation['id'],
                                    self.activityId))

        raw_response = self._request(response_url, 'delete')

        if raw_response.text != '':
            response = raw_response.json()
        else:
            response = None

        self.save_response('DeleteActivity',
                           self.conversation['id'],
                           None,
                           {'conversationId': self.conversation['id'], 'activityId': self.activityId},
                           response)
        return response

class CreateConversation(Activity):
    def __init__(self, **kwargs):
        super(CreateConversation, self).__init__(**kwargs)

    def send(self):
        # make sure that we remove and team or channel data from the request when working in teams.
        if self.channelData is not None and 'tenant' in self.channelData:
            self.channelData = {"tenant": {"id": self.channelData["tenant"]["id"]}}
        self.channelId = None
        self.conversation = None

        request_json = {
            'bot': self.fromAccount if self.bot is None else self.bot,
            'isGroup': False if self.isGroup is None else self.isGroup,
            'members': [self.recipient] if self.members is None else self.members,
            'channelData': self.channelData,
            'activity': self.to_dict(),
        }

        if len(request_json['members']) > 1:
            request_json['isGroup'] = True

        if self.topicName is not None:
            request_json['topicName'] = self.topicName

        response_url = self.urljoin(self.serviceUrl, "/v3/conversations")

        response = self._request(response_url, 'post', request_json)

        # Skype for Business returns the key 'Id', not 'id', fix the response
        # by changing 'Id' to 'id'
        response_json = response.json()
        if 'Id' in response_json and 'id' not in response_json:
            response._content = response._content.replace(b"\"Id\":",
                                                          b"\"id\":")
        response_json = response.json()
        self.save_response('CreateConversation',
                           response_json['id'],
                           request_json,
                           {},
                           response_json)
        return response

class GetConversationMembers(Activity):
    def __init__(self, **kwargs):
        super(GetConversationMembers, self).__init__(**kwargs)

    def send(self):
        response_url = self.urljoin(self.serviceUrl,
                                    "/v3/conversations/{}/members".format(
                                    self.conversation['id']))

        response = self._request(response_url, 'get')

        self.save_response('GetConversationMembers',
                           self.conversation['id'],
                           None,
                           {'conversationId': self.conversation['id']},
                           response.json())
        return response

class GetActivityMembers(Activity):
    def __init__(self, **kwargs):
        super(GetActivityMembers, self).__init__(**kwargs)

    def send(self):
        response_url = self.urljoin(self.serviceUrl,
                                    "/v3/conversations/{}/activities/{}/members".format(
                                    self.conversation['id'],
                                    self.activityId))

        response = self._request(response_url, 'get')

        self.save_response('GetActivityMembers',
                           self.conversation['id'],
                           None,
                           {'conversationId': self.conversation['id'], 'activityId': self.activityId},
                           response.json())
        return response
<commit_msg>Adds code to allow createconversation to fail without a key error<commit_after>from .activity import Activity

class ReplyToActivity(Activity):
    def __init__(self, **kwargs):
        super(ReplyToActivity, self).__init__(reply_to_activity=True, **kwargs)

    def send(self):
        response_url = self.urljoin(self.serviceUrl,
                                    "/v3/conversations/{}/activities/{}".format(
                                    self.conversation['id'],
                                    self.activityId))

        response = self._request(response_url, 'post', self.to_dict())

        self.save_response('ReplyToActivity',
                           self.conversation['id'],
                           self.to_dict(),
                           {'conversationId': self.conversation['id'], 'activityId': self.activityId},
                           response.json())
        return response

class SendToConversation(Activity):
    def __init__(self, **kwargs):
        super(SendToConversation, self).__init__(**kwargs)

    def send(self):
        response_url = self.urljoin(self.serviceUrl,
                                    "/v3/conversations/{}/activities".format(
                                        self.conversation['id']))

        response = self._request(response_url, 'post', self.to_dict())

        self.save_response('SendToConversation',
                           self.conversation['id'],
                           self.to_dict(),
                           {'conversationId': self.conversation['id']},
                           response.json())
        return response

class DeleteActivity(Activity):
    def __init__(self, **kwargs):
        super(DeleteActivity, self).__init__(**kwargs)

    def send(self):
        response_url = self.urljoin(self.serviceUrl,
                                    "/v3/conversations/{}/activities/{}".format(
                                    self.conversation['id'],
                                    self.activityId))

        raw_response = self._request(response_url, 'delete')

        if raw_response.text != '':
            response = raw_response.json()
        else:
            response = None

        self.save_response('DeleteActivity',
                           self.conversation['id'],
                           None,
                           {'conversationId': self.conversation['id'], 'activityId': self.activityId},
                           response)
        return response

class CreateConversation(Activity):
    def __init__(self, **kwargs):
        super(CreateConversation, self).__init__(**kwargs)

    def send(self):
        # make sure that we remove and team or channel data from the request when working in teams.
        if self.channelData is not None and 'tenant' in self.channelData:
            self.channelData = {"tenant": {"id": self.channelData["tenant"]["id"]}}
        self.channelId = None
        self.conversation = None

        request_json = {
            'bot': self.fromAccount if self.bot is None else self.bot,
            'isGroup': False if self.isGroup is None else self.isGroup,
            'members': [self.recipient] if self.members is None else self.members,
            'channelData': self.channelData,
            'activity': self.to_dict(),
        }

        if len(request_json['members']) > 1:
            request_json['isGroup'] = True

        if self.topicName is not None:
            request_json['topicName'] = self.topicName

        response_url = self.urljoin(self.serviceUrl, "/v3/conversations")

        response = self._request(response_url, 'post', request_json)

        # Skype for Business returns the key 'Id', not 'id', fix the response
        # by changing 'Id' to 'id'
        response_json = response.json()
        if 'Id' in response_json and 'id' not in response_json:
            response._content = response._content.replace(b"\"Id\":",
                                                          b"\"id\":")
        response_json = response.json()
        # save the response if it worked, http error codes won't save respones
        if 'id' in response_json:
            self.save_response('CreateConversation',
                               response_json['id'],
                               request_json,
                               {},
                               response_json)
        return response

class GetConversationMembers(Activity):
    def __init__(self, **kwargs):
        super(GetConversationMembers, self).__init__(**kwargs)

    def send(self):
        response_url = self.urljoin(self.serviceUrl,
                                    "/v3/conversations/{}/members".format(
                                    self.conversation['id']))

        response = self._request(response_url, 'get')

        self.save_response('GetConversationMembers',
                           self.conversation['id'],
                           None,
                           {'conversationId': self.conversation['id']},
                           response.json())
        return response

class GetActivityMembers(Activity):
    def __init__(self, **kwargs):
        super(GetActivityMembers, self).__init__(**kwargs)

    def send(self):
        response_url = self.urljoin(self.serviceUrl,
                                    "/v3/conversations/{}/activities/{}/members".format(
                                    self.conversation['id'],
                                    self.activityId))

        response = self._request(response_url, 'get')

        self.save_response('GetActivityMembers',
                           self.conversation['id'],
                           None,
                           {'conversationId': self.conversation['id'], 'activityId': self.activityId},
                           response.json())
        return response
<|endoftext|>


<commit_before># Ridiculously simple test of the winsound module for Windows.

import unittest
from test import test_support
import winsound, time
import os
import subprocess

class BeepTest(unittest.TestCase):

    def test_errors(self):
        self.assertRaises(TypeError, winsound.Beep)
        self.assertRaises(ValueError, winsound.Beep, 36, 75)
        self.assertRaises(ValueError, winsound.Beep, 32768, 75)

    def test_extremes(self):
        winsound.Beep(37, 75)
        winsound.Beep(32767, 75)

    def test_increasingfrequency(self):
        for i in xrange(100, 2000, 100):
            winsound.Beep(i, 75)

class MessageBeepTest(unittest.TestCase):

    def tearDown(self):
        time.sleep(0.5)

    def test_default(self):
        self.assertRaises(TypeError, winsound.MessageBeep, "bad")
        self.assertRaises(TypeError, winsound.MessageBeep, 42, 42)
        winsound.MessageBeep()

    def test_ok(self):
        winsound.MessageBeep(winsound.MB_OK)

    def test_asterisk(self):
        winsound.MessageBeep(winsound.MB_ICONASTERISK)

    def test_exclamation(self):
        winsound.MessageBeep(winsound.MB_ICONEXCLAMATION)

    def test_hand(self):
        winsound.MessageBeep(winsound.MB_ICONHAND)

    def test_question(self):
        winsound.MessageBeep(winsound.MB_ICONQUESTION)

class PlaySoundTest(unittest.TestCase):

    def test_errors(self):
        self.assertRaises(TypeError, winsound.PlaySound)
        self.assertRaises(TypeError, winsound.PlaySound, "bad", "bad")
        self.assertRaises(
            RuntimeError,
            winsound.PlaySound,
            "none", winsound.SND_ASYNC | winsound.SND_MEMORY
        )

    def test_alias_asterisk(self):
        if _have_soundcard():
            winsound.PlaySound('SystemAsterisk', winsound.SND_ALIAS)
        else:
            self.assertRaises(
                RuntimeError,
                winsound.PlaySound,
                'SystemAsterisk', winsound.SND_ALIAS
            )

    def test_alias_exclamation(self):
        if _have_soundcard():
            winsound.PlaySound('SystemExclamation', winsound.SND_ALIAS)
        else:
            self.assertRaises(
                RuntimeError,
                winsound.PlaySound,
                'SystemExclamation', winsound.SND_ALIAS
            )

    def test_alias_exit(self):
        if _have_soundcard():
            winsound.PlaySound('SystemExit', winsound.SND_ALIAS)
        else:
            self.assertRaises(
                RuntimeError,
                winsound.PlaySound,
                'SystemExit', winsound.SND_ALIAS
            )

    def test_alias_hand(self):
        if _have_soundcard():
            winsound.PlaySound('SystemHand', winsound.SND_ALIAS)
        else:
            self.assertRaises(
                RuntimeError,
                winsound.PlaySound,
                'SystemHand', winsound.SND_ALIAS
            )

    def test_alias_question(self):
        if _have_soundcard():
            winsound.PlaySound('SystemQuestion', winsound.SND_ALIAS)
        else:
            self.assertRaises(
                RuntimeError,
                winsound.PlaySound,
                'SystemQuestion', winsound.SND_ALIAS
            )

    def test_alias_fallback(self):
        # This test can't be expected to work on all systems.  The MS
        # PlaySound() docs say:
        #
        #     If it cannot find the specified sound, PlaySound uses the
        #     default system event sound entry instead.  If the function
        #     can find neither the system default entry nor the default
        #     sound, it makes no sound and returns FALSE.
        #
        # It's known to return FALSE on some real systems.

        # winsound.PlaySound('!"$%&/(#+*', winsound.SND_ALIAS)
        return

    def test_alias_nofallback(self):
        if _have_soundcard():
            # Note that this is not the same as asserting RuntimeError
            # will get raised:  you cannot convert this to
            # self.assertRaises(...) form.  The attempt may or may not
            # raise RuntimeError, but it shouldn't raise anything other
            # than RuntimeError, and that's all we're trying to test
            # here.  The MS docs aren't clear about whether the SDK
            # PlaySound() with SND_ALIAS and SND_NODEFAULT will return
            # True or False when the alias is unknown.  On Tim's WinXP
            # box today, it returns True (no exception is raised).  What
            # we'd really like to test is that no sound is played, but
            # that requires first wiring an eardrum class into unittest
            # <wink>.
            try:
                winsound.PlaySound(
                    '!"$%&/(#+*',
                    winsound.SND_ALIAS | winsound.SND_NODEFAULT
                )
            except RuntimeError:
                pass
        else:
            self.assertRaises(
                RuntimeError,
                winsound.PlaySound,
                '!"$%&/(#+*', winsound.SND_ALIAS | winsound.SND_NODEFAULT
            )

    def test_stopasync(self):
        if _have_soundcard():
            winsound.PlaySound(
                'SystemQuestion',
                winsound.SND_ALIAS | winsound.SND_ASYNC | winsound.SND_LOOP
            )
            time.sleep(0.5)
            try:
                winsound.PlaySound(
                    'SystemQuestion',
                    winsound.SND_ALIAS | winsound.SND_NOSTOP
                )
            except RuntimeError:
                pass
            else: # the first sound might already be finished
                pass
            winsound.PlaySound(None, winsound.SND_PURGE)
        else:
            self.assertRaises(
                RuntimeError,
                winsound.PlaySound,
                None, winsound.SND_PURGE
            )

def _get_cscript_path():
    """Return the full path to cscript.exe or None."""
    for dir in os.environ.get("PATH", "").split(os.pathsep):
        cscript_path = os.path.join(dir, "cscript.exe")
        if os.path.exists(cscript_path):
            return cscript_path

__have_soundcard_cache = None
def _have_soundcard():
    """Return True iff this computer has a soundcard."""
    global __have_soundcard_cache
    if __have_soundcard_cache is None:
        cscript_path = _get_cscript_path()
        if cscript_path is None:
            # Could not find cscript.exe to run our VBScript helper. Default
            # to True: most computers these days *do* have a soundcard.
            return True

        check_script = os.path.join(os.path.dirname(__file__),
                                    "check_soundcard.vbs")
        p = subprocess.Popen([cscript_path, check_script],
                             stdout=subprocess.PIPE)
        __have_soundcard_cache = not p.wait()
    return __have_soundcard_cache

def test_main():
    test_support.run_unittest(BeepTest, MessageBeepTest, PlaySoundTest)

if __name__=="__main__":
    test_main()
<commit_msg>Ensure this test passes even if there are no soundcards in the system.  Backport from trunk r61242.<commit_after># Ridiculously simple test of the winsound module for Windows.

import unittest
from test import test_support
import winsound, time
import os
import subprocess

class BeepTest(unittest.TestCase):
    # As with PlaySoundTest, incorporate the _have_soundcard() check
    # into our test methods.  If there's no audio device present,
    # winsound.Beep returns 0 and GetLastError() returns 127, which
    # is: ERROR_PROC_NOT_FOUND ("The specified procedure could not
    # be found").  (FWIW, virtual/Hyper-V systems fall under this
    # scenario as they have no sound devices whatsoever  (not even
    # a legacy Beep device).)

    def test_errors(self):
        self.assertRaises(TypeError, winsound.Beep)
        self.assertRaises(ValueError, winsound.Beep, 36, 75)
        self.assertRaises(ValueError, winsound.Beep, 32768, 75)

    def test_extremes(self):
        if _have_soundcard():
            winsound.Beep(37, 75)
            winsound.Beep(32767, 75)
        else:
            self.assertRaises(RuntimeError, winsound.Beep, 37, 75)
            self.assertRaises(RuntimeError, winsound.Beep, 32767, 75)

    def test_increasingfrequency(self):
        if _have_soundcard():
            for i in xrange(100, 2000, 100):
                winsound.Beep(i, 75)

class MessageBeepTest(unittest.TestCase):

    def tearDown(self):
        time.sleep(0.5)

    def test_default(self):
        self.assertRaises(TypeError, winsound.MessageBeep, "bad")
        self.assertRaises(TypeError, winsound.MessageBeep, 42, 42)
        winsound.MessageBeep()

    def test_ok(self):
        winsound.MessageBeep(winsound.MB_OK)

    def test_asterisk(self):
        winsound.MessageBeep(winsound.MB_ICONASTERISK)

    def test_exclamation(self):
        winsound.MessageBeep(winsound.MB_ICONEXCLAMATION)

    def test_hand(self):
        winsound.MessageBeep(winsound.MB_ICONHAND)

    def test_question(self):
        winsound.MessageBeep(winsound.MB_ICONQUESTION)

class PlaySoundTest(unittest.TestCase):

    def test_errors(self):
        self.assertRaises(TypeError, winsound.PlaySound)
        self.assertRaises(TypeError, winsound.PlaySound, "bad", "bad")
        self.assertRaises(
            RuntimeError,
            winsound.PlaySound,
            "none", winsound.SND_ASYNC | winsound.SND_MEMORY
        )

    def test_alias_asterisk(self):
        if _have_soundcard():
            winsound.PlaySound('SystemAsterisk', winsound.SND_ALIAS)
        else:
            self.assertRaises(
                RuntimeError,
                winsound.PlaySound,
                'SystemAsterisk', winsound.SND_ALIAS
            )

    def test_alias_exclamation(self):
        if _have_soundcard():
            winsound.PlaySound('SystemExclamation', winsound.SND_ALIAS)
        else:
            self.assertRaises(
                RuntimeError,
                winsound.PlaySound,
                'SystemExclamation', winsound.SND_ALIAS
            )

    def test_alias_exit(self):
        if _have_soundcard():
            winsound.PlaySound('SystemExit', winsound.SND_ALIAS)
        else:
            self.assertRaises(
                RuntimeError,
                winsound.PlaySound,
                'SystemExit', winsound.SND_ALIAS
            )

    def test_alias_hand(self):
        if _have_soundcard():
            winsound.PlaySound('SystemHand', winsound.SND_ALIAS)
        else:
            self.assertRaises(
                RuntimeError,
                winsound.PlaySound,
                'SystemHand', winsound.SND_ALIAS
            )

    def test_alias_question(self):
        if _have_soundcard():
            winsound.PlaySound('SystemQuestion', winsound.SND_ALIAS)
        else:
            self.assertRaises(
                RuntimeError,
                winsound.PlaySound,
                'SystemQuestion', winsound.SND_ALIAS
            )

    def test_alias_fallback(self):
        # This test can't be expected to work on all systems.  The MS
        # PlaySound() docs say:
        #
        #     If it cannot find the specified sound, PlaySound uses the
        #     default system event sound entry instead.  If the function
        #     can find neither the system default entry nor the default
        #     sound, it makes no sound and returns FALSE.
        #
        # It's known to return FALSE on some real systems.

        # winsound.PlaySound('!"$%&/(#+*', winsound.SND_ALIAS)
        return

    def test_alias_nofallback(self):
        if _have_soundcard():
            # Note that this is not the same as asserting RuntimeError
            # will get raised:  you cannot convert this to
            # self.assertRaises(...) form.  The attempt may or may not
            # raise RuntimeError, but it shouldn't raise anything other
            # than RuntimeError, and that's all we're trying to test
            # here.  The MS docs aren't clear about whether the SDK
            # PlaySound() with SND_ALIAS and SND_NODEFAULT will return
            # True or False when the alias is unknown.  On Tim's WinXP
            # box today, it returns True (no exception is raised).  What
            # we'd really like to test is that no sound is played, but
            # that requires first wiring an eardrum class into unittest
            # <wink>.
            try:
                winsound.PlaySound(
                    '!"$%&/(#+*',
                    winsound.SND_ALIAS | winsound.SND_NODEFAULT
                )
            except RuntimeError:
                pass
        else:
            self.assertRaises(
                RuntimeError,
                winsound.PlaySound,
                '!"$%&/(#+*', winsound.SND_ALIAS | winsound.SND_NODEFAULT
            )

    def test_stopasync(self):
        if _have_soundcard():
            winsound.PlaySound(
                'SystemQuestion',
                winsound.SND_ALIAS | winsound.SND_ASYNC | winsound.SND_LOOP
            )
            time.sleep(0.5)
            try:
                winsound.PlaySound(
                    'SystemQuestion',
                    winsound.SND_ALIAS | winsound.SND_NOSTOP
                )
            except RuntimeError:
                pass
            else: # the first sound might already be finished
                pass
            winsound.PlaySound(None, winsound.SND_PURGE)
        else:
            self.assertRaises(
                RuntimeError,
                winsound.PlaySound,
                None, winsound.SND_PURGE
            )

def _get_cscript_path():
    """Return the full path to cscript.exe or None."""
    for dir in os.environ.get("PATH", "").split(os.pathsep):
        cscript_path = os.path.join(dir, "cscript.exe")
        if os.path.exists(cscript_path):
            return cscript_path

__have_soundcard_cache = None
def _have_soundcard():
    """Return True iff this computer has a soundcard."""
    global __have_soundcard_cache
    if __have_soundcard_cache is None:
        cscript_path = _get_cscript_path()
        if cscript_path is None:
            # Could not find cscript.exe to run our VBScript helper. Default
            # to True: most computers these days *do* have a soundcard.
            return True

        check_script = os.path.join(os.path.dirname(__file__),
                                    "check_soundcard.vbs")
        p = subprocess.Popen([cscript_path, check_script],
                             stdout=subprocess.PIPE)
        __have_soundcard_cache = not p.wait()
    return __have_soundcard_cache

def test_main():
    test_support.run_unittest(BeepTest, MessageBeepTest, PlaySoundTest)

if __name__=="__main__":
    test_main()
<|endoftext|>