<commit_before>
import os
import logging
import json
import platform
import psutil
import multiprocessing
import socket
import emoji

from flask import request, current_app as app
from flask_restful import reqparse, abort, Resource

from kubernetes.client.rest import ApiException
import docker

from .openshiftmgr import OpenShiftManager
from .swarmmgr import SwarmManager

logger = logging.getLogger(__name__)

parser = reqparse.RequestParser(bundle_errors=True)
parser.add_argument('json',type = str, dest='json', required=False, default='')
parser.add_argument('jid', dest='jid', required=False)
parser.add_argument('cmd_args', dest='cmd_args', required=False)
parser.add_argument('cmd_path_flags', dest='cmd_path_flags', required=False)
parser.add_argument('auid', dest='auid', required=False)
parser.add_argument('number_of_workers', dest='number_of_workers', required=False)
parser.add_argument('cpu_limit', dest='cpu_limit', required=False)
parser.add_argument('memory_limit', dest='memory_limit', required=False)
parser.add_argument('gpu_limit', dest='gpu_limit', required=False)
parser.add_argument('image', dest='image', required=False)
parser.add_argument('selfexec', dest='selfexec', required=False)
parser.add_argument('selfpath', dest='selfpath', required=False)
parser.add_argument('execshell', dest='execshell', required=False)
parser.add_argument('type', dest='type', choices=('ds', 'fs', 'ts'), required=False)

class JobList(Resource):
    """
    Resource representing the list of jobs running on the compute.
    """

    def __init__(self):
        super(JobList, self).__init__()

        # mounting points for the input and outputdir in the app's container!
        self.str_app_container_inputdir = '/share/incoming'
        self.str_app_container_outputdir = '/share/outgoing'

        self.container_env = app.config.get('CONTAINER_ENV')
        self.openshiftmgr       = None

    def get(self):
        return {
            'server_version': app.config.get('SERVER_VERSION'),
        }

    def get_openshift_manager(self):
        self.openshiftmgr = OpenShiftManager()
        return self.openshiftmgr

    def post(self):
        args = parser.parse_args()
        str_image = ''
        number_of_workers=0
        cpu_limit=0
        memory_limit=0
        gpu_limit=0

        # Check if a json is passed, then parse each of the json fields
        # Add condition for additional json fields

        if len(args.json) > 0 :
            # json decoding
            json_payload = json.loads(args.json)

            if 'jid' in json_payload:
                job_id = json_payload['jid']

            if 'cmd_args' in json_payload:
                cmd_args = json_payload['cmd_args']

            if 'cmd_path_flags' in json_payload:
                cmd_path_flags = json_payload['cmd_path_flags']

            if 'number_of_workers' in json_payload:
                number_of_workers = json_payload['number_of_workers']

            if 'auid' in json_payload:
                auid = json_payload['auid']

            if 'cpu_limit' in json_payload:
                cpu_limit = json_payload['cpu_limit']

            if 'memory_limit' in json_payload:
                memory_limit = json_payload['memory_limit']

            if 'gpu_limit' in json_payload:
                gpu_limit = json_payload['gpu_limit']

            if 'image' in json_payload:
                image = json_payload['image']
                str_image = image

            if 'selfexec' in json_payload:
                selfexec = json_payload['selfexec']

            if 'selfpath' in json_payload:
                selfpath = json_payload['selfpath']

            if 'execshell' in json_payload:
                execshell = json_payload['execshell']

            if 'type' in json_payload:
                type = json_payload['type']

            outputdir = self.str_app_container_outputdir
            exec = os.path.join(selfpath, selfexec)
            cmd = f'{execshell} {exec}'
            if type == 'ds':
                inputdir = self.str_app_container_inputdir
                cmd = cmd + f' {cmd_args} {inputdir} {outputdir}'
            elif type in ('fs', 'ts'):
                cmd = cmd + f' {cmd_args} {outputdir}'

        else :
            # Parse the arguments manually from the parameters

            job_id = args.jid.lstrip('/')
            compute_data = {
                'cmd_args': args.cmd_args,
                'cmd_path_flags': args.cmd_path_flags,
                'auid': args.auid,
                'number_of_workers': args.number_of_workers,
                'cpu_limit': args.cpu_limit,
                'memory_limit': args.memory_limit,
                'gpu_limit': args.gpu_limit,
                'image': args.image,
                'selfexec': args.selfexec,
                'selfpath': args.selfpath,
                'execshell': args.execshell,
                'type': args.type,
            }
            cmd = self.build_app_cmd(compute_data)
            str_image = compute_data['image']

        job_logs = ''
        job_info = {'id': '', 'image': '', 'cmd': '', 'timestamp': '', 'message': '',
                    'status': 'undefined', 'containerid': '', 'exitcode': '', 'pid': ''}

        if self.container_env == 'swarm':
            storebase = app.config.get('STOREBASE')
            share_dir = os.path.join(storebase, 'key-' + job_id)

            swarm_mgr = SwarmManager()
            logger.info(f'Scheduling job {job_id} on the Swarm cluster')
            try:
                service = swarm_mgr.schedule(str_image, cmd, job_id, 'none',
                                             share_dir)
            except docker.errors.APIError as e:
                logger.error(f'Error from Swarm while scheduling job {job_id}, detail: '
                             f'{str(e)}')
                status_code = e.response.status_code
                status_code = 503 if status_code == 500 else status_code
                abort(status_code, message=str(e))
            job_info = swarm_mgr.get_service_task_info(service)
            logger.info(f'Successful job {job_id} schedule response from Swarm: '
                        f'{job_info}')
            job_logs = swarm_mgr.get_service_logs(service)

        else :
            # If the container env is Openshift
            logger.info(f'Scheduling job {job_id} on the Openshift cluster')
            # Create the Persistent Volume Claim
            if os.environ.get('STORAGE_TYPE') == 'swift':
                self.get_openshift_manager().create_pvc(job_id)

            # Set some variables
            incoming_dir = self.str_app_container_inputdir
            outgoing_dir = self.str_app_container_outputdir

            # Schedule the job
            job = self.get_openshift_manager().schedule(str_image, cmd, job_id, \
                                         number_of_workers or compute_data['number_of_workers'],\
                                         cpu_limit or compute_data['cpu_limit'],\
                                         memory_limit or compute_data['memory_limit'], \
                                         gpu_limit or compute_data['gpu_limit'], \
                                         incoming_dir, outgoing_dir)
            return {'job_details': str(job)}

        return {
            'jid': job_id,
            'image': job_info['image'],
            'cmd': job_info['cmd'],
            'status': job_info['status'],
            'message': job_info['message'],
            'timestamp': job_info['timestamp'],
            'containerid': job_info['containerid'],
            'exitcode': job_info['exitcode'],
            'pid': job_info['pid'],
            'logs': job_logs
        }

    def build_app_cmd(self, compute_data):
        """
        Build and return the app's cmd string.
        """
        cmd_args = compute_data['cmd_args']
        cmd_path_flags = compute_data['cmd_path_flags']
        if cmd_path_flags:
            # process the argument of any cmd flag that is a 'path'
            path_flags = cmd_path_flags.split(',')
            args = cmd_args.split()
            for i in range(len(args) - 1):
                if args[i] in path_flags:
                    # each flag value is a string of one or more paths separated by comma
                    # paths = args[i+1].split(',')
                    # base_inputdir = self.str_app_container_inputdir
                    # paths = [os.path.join(base_inputdir, p.lstrip('/')) for p in paths]
                    # args[i+1] = ','.join(paths)

                    # the next is tmp until CUBE's assumptions about inputdir and path
                    # parameters are removed
                    args[i+1] = self.str_app_container_inputdir
            cmd_args = ' '.join(args)
        selfpath = compute_data['selfpath']
        selfexec = compute_data['selfexec']
        execshell = compute_data['execshell']
        type = compute_data['type']
        outputdir = self.str_app_container_outputdir
        exec = os.path.join(selfpath, selfexec)
        cmd = f'{execshell} {exec}'
        if type == 'ds':
            inputdir = self.str_app_container_inputdir
            cmd = cmd + f' {cmd_args} {inputdir} {outputdir}'
        elif type in ('fs', 'ts'):
            cmd = cmd + f' {cmd_args} {outputdir}'
        return cmd

class Job(Resource):
    """
    Resource representing a single job running on the compute.
    """

    # Initiate an openshiftmgr instance
    def get_openshift_manager(self):
        self.openshiftmgr = OpenShiftManager()
        return self.openshiftmgr

    # Get the status of a running jon on Openshift
    def t_status_process_openshift(self, jid):
        """
        Determine the status of a job scheduled using the openshift manager.
        PRECONDITIONS:
        o   Only call this method if a container structure exists
            in the relevant job tree!
        POSTCONDITIONS:
        o   If the job is completed, then shutdown the container cluster
            service.
        """

        str_logs = ''
        # Get job-id from request
        #jid = self.jid

        # Query OpenShift API to get job state
        d_json  = self.get_openshift_manager().state(jid)

        print (d_json)
        print (jid)

        if d_json['Status']['Message'] == 'finished':
            pod_names = self.get_openshift_manager().get_pod_names_in_job(jid)
            for _, pod_name in enumerate(pod_names):
                str_logs += self.get_openshift_manager().get_job_pod_logs(pod_name, jid)
        else:
            str_logs = d_json['Status']['Message']

        status  = d_json['Status']
        currentState =  d_json['Status']['Message']

        return {
            'status':           status,
            'logs':             str_logs,
            'currentState':     [currentState]
        }

    def get(self, job_id):

        container_env = app.config.get('CONTAINER_ENV')

        job_logs = ''
        job_info = {'id': '', 'image': '', 'cmd': '', 'timestamp': '', 'message': '',
                    'status': 'undefined', 'containerid': '', 'exitcode': '', 'pid': ''}

        if container_env == 'swarm':
            swarm_mgr = SwarmManager()
            logger.info(f'Getting job {job_id} status from the Swarm cluster')
            try:
                service = swarm_mgr.get_service(job_id)
            except docker.errors.NotFound as e:
                abort(404, message=str(e))
            except docker.errors.APIError as e:
                status_code = e.response.status_code
                status_code = 503 if status_code == 500 else status_code
                abort(status_code, message=str(e))
            except docker.errors.InvalidVersion as e:
                abort(400, message=str(e))
            job_info = swarm_mgr.get_service_task_info(service)
            logger.info(f'Successful job {job_id} status response from Swarm: '
                        f'{job_info}')
            job_logs = swarm_mgr.get_service_logs(service)

            if job_info['status'] in ('undefined', 'finishedWithError',
                                      'finishedSuccessfully'):
                service.remove()  # remove job from swarm cluster
        else:
            logger.info(f'Getting job {job_id} status from the Openshift cluster')
            try:
                d_containerStatus       =   self.t_status_process_openshift(job_id)
                status                  =   d_containerStatus['status']
                logs                    =   d_containerStatus['logs']
                currentState            =   d_containerStatus['currentState']
            except Exception as e:
                if isinstance(e, ApiException) and e.reason == 'Not Found':
                    status = logs = currentState = e.reason
                else:
                    raise e

            d_ret = {
                'description':   str(status),
                'l_logs':     str(logs),
                'l_status': currentState
            }
            return {
                    "d_ret":    d_ret,
                    "status":   str(currentState)
            }

        return {
            'jid': job_id,
            'image': job_info['image'],
            'cmd': job_info['cmd'],
            'status': job_info['status'],
            'message': job_info['message'],
            'timestamp': job_info['timestamp'],
            'containerid': job_info['containerid'],
            'exitcode': job_info['exitcode'],
            'pid': job_info['pid'],
            'logs': job_logs
        }

class Hello(Resource):

     # Respond to simple 'hello' requests from the server
    def get(self):

            container_env = app.config.get('CONTAINER_ENV')

            smiling_face = emoji.emojize(":grinning_face_with_big_eyes:")
            logger.info(f'pman says hello from {container_env} {smiling_face}')
            b_status            = False
            d_ret               = {}
            d_ret['message']                = (f'pman says hello from {container_env} {smiling_face}')
            d_ret['sysinfo']                = {}
            d_ret['sysinfo']['system']      = platform.system()
            d_ret['sysinfo']['machine']     = platform.machine()
            d_ret['sysinfo']['platform']    = platform.platform()
            d_ret['sysinfo']['uname']       = platform.uname()
            d_ret['sysinfo']['version']     = platform.version()
            d_ret['sysinfo']['memory']      = psutil.virtual_memory()
            d_ret['sysinfo']['cpucount']    = multiprocessing.cpu_count()
            d_ret['sysinfo']['loadavg']     = os.getloadavg()
            d_ret['sysinfo']['cpu_percent'] = psutil.cpu_percent()
            d_ret['sysinfo']['hostname']    = socket.gethostname()
            d_ret['sysinfo']['inet']        = [l for l in ([ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] if not ip.startswith("127.")][:1], [[(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close()) for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]][0][1]]) if l][0][0]
            b_status                        = True

            return { 'd_ret':   d_ret,
                 'status':  b_status}

<commit_msg>Delete a pod after job finishes<commit_after>
import os
import logging
import json
import platform
import psutil
import multiprocessing
import socket
import emoji

from flask import request, current_app as app
from flask_restful import reqparse, abort, Resource

from kubernetes.client.rest import ApiException
import docker

from .openshiftmgr import OpenShiftManager
from .swarmmgr import SwarmManager

logger = logging.getLogger(__name__)

parser = reqparse.RequestParser(bundle_errors=True)
parser.add_argument('json',type = str, dest='json', required=False, default='')
parser.add_argument('jid', dest='jid', required=False)
parser.add_argument('cmd_args', dest='cmd_args', required=False)
parser.add_argument('cmd_path_flags', dest='cmd_path_flags', required=False)
parser.add_argument('auid', dest='auid', required=False)
parser.add_argument('number_of_workers', dest='number_of_workers', required=False)
parser.add_argument('cpu_limit', dest='cpu_limit', required=False)
parser.add_argument('memory_limit', dest='memory_limit', required=False)
parser.add_argument('gpu_limit', dest='gpu_limit', required=False)
parser.add_argument('image', dest='image', required=False)
parser.add_argument('selfexec', dest='selfexec', required=False)
parser.add_argument('selfpath', dest='selfpath', required=False)
parser.add_argument('execshell', dest='execshell', required=False)
parser.add_argument('type', dest='type', choices=('ds', 'fs', 'ts'), required=False)

class JobList(Resource):
    """
    Resource representing the list of jobs running on the compute.
    """

    def __init__(self):
        super(JobList, self).__init__()

        # mounting points for the input and outputdir in the app's container!
        self.str_app_container_inputdir = '/share/incoming'
        self.str_app_container_outputdir = '/share/outgoing'

        self.container_env = app.config.get('CONTAINER_ENV')
        self.openshiftmgr       = None

    def get(self):
        return {
            'server_version': app.config.get('SERVER_VERSION'),
        }

    def get_openshift_manager(self):
        self.openshiftmgr = OpenShiftManager()
        return self.openshiftmgr

    def post(self):
        args = parser.parse_args()
        str_image = ''
        number_of_workers=0
        cpu_limit=''
        memory_limit=''
        gpu_limit=0

        # Check if a json is passed, then parse each of the json fields
        # Add condition for additional json fields

        if len(args.json) > 0 :
            # json decoding
            json_payload = json.loads(args.json)

            if 'jid' in json_payload:
                job_id = json_payload['jid']

            if 'cmd_args' in json_payload:
                cmd_args = json_payload['cmd_args']

            if 'cmd_path_flags' in json_payload:
                cmd_path_flags = json_payload['cmd_path_flags']

            if 'number_of_workers' in json_payload:
                number_of_workers = json_payload['number_of_workers']

            if 'auid' in json_payload:
                auid = json_payload['auid']

            if 'cpu_limit' in json_payload:
                cpu_limit = json_payload['cpu_limit']

            if 'memory_limit' in json_payload:
                memory_limit = json_payload['memory_limit']

            if 'gpu_limit' in json_payload:
                gpu_limit = json_payload['gpu_limit']

            if 'image' in json_payload:
                image = json_payload['image']
                str_image = image

            if 'selfexec' in json_payload:
                selfexec = json_payload['selfexec']

            if 'selfpath' in json_payload:
                selfpath = json_payload['selfpath']

            if 'execshell' in json_payload:
                execshell = json_payload['execshell']

            if 'type' in json_payload:
                type = json_payload['type']

            outputdir = self.str_app_container_outputdir
            exec = os.path.join(selfpath, selfexec)
            cmd = f'{execshell} {exec}'
            if type == 'ds':
                inputdir = self.str_app_container_inputdir
                cmd = cmd + f' {cmd_args} {inputdir} {outputdir}'
            elif type in ('fs', 'ts'):
                cmd = cmd + f' {cmd_args} {outputdir}'

        else :
            # Parse the arguments manually from the parameters

            job_id = args.jid.lstrip('/')
            compute_data = {
                'cmd_args': args.cmd_args,
                'cmd_path_flags': args.cmd_path_flags,
                'auid': args.auid,
                'number_of_workers': args.number_of_workers,
                'cpu_limit': args.cpu_limit,
                'memory_limit': args.memory_limit,
                'gpu_limit': args.gpu_limit,
                'image': args.image,
                'selfexec': args.selfexec,
                'selfpath': args.selfpath,
                'execshell': args.execshell,
                'type': args.type,
            }
            cmd = self.build_app_cmd(compute_data)
            str_image = compute_data['image']

        job_logs = ''
        job_info = {'id': '', 'image': '', 'cmd': '', 'timestamp': '', 'message': '',
                    'status': 'undefined', 'containerid': '', 'exitcode': '', 'pid': ''}

        if self.container_env == 'swarm':
            storebase = app.config.get('STOREBASE')
            share_dir = os.path.join(storebase, 'key-' + job_id)

            swarm_mgr = SwarmManager()
            logger.info(f'Scheduling job {job_id} on the Swarm cluster')
            try:
                service = swarm_mgr.schedule(str_image, cmd, job_id, 'none',
                                             share_dir)
            except docker.errors.APIError as e:
                logger.error(f'Error from Swarm while scheduling job {job_id}, detail: '
                             f'{str(e)}')
                status_code = e.response.status_code
                status_code = 503 if status_code == 500 else status_code
                abort(status_code, message=str(e))
            job_info = swarm_mgr.get_service_task_info(service)
            logger.info(f'Successful job {job_id} schedule response from Swarm: '
                        f'{job_info}')
            job_logs = swarm_mgr.get_service_logs(service)

        if self.container_env == 'openshift' :
            # If the container env is Openshift
            logger.info(f'Scheduling job {job_id} on the Openshift cluster')
            # Create the Persistent Volume Claim
            if os.environ.get('STORAGE_TYPE') == 'swift':
                self.get_openshift_manager().create_pvc(job_id)

            # Set some variables
            incoming_dir = self.str_app_container_inputdir
            outgoing_dir = self.str_app_container_outputdir

            # Ensure that the limits are specified before scheduling a job
            if cpu_limit or compute_data['cpu_limit'] :
                cpu_limit = compute_data['cpu_limit'] + 'm'
            else :
                cpu_limit = '4000m'

            if memory_limit or compute_data['memory_limit'] :
                memory_limit = compute_data['memory_limit'] + 'Mi'
            else :
                memory_limit = '4000Mi'

            # Schedule the job
            job = self.get_openshift_manager().schedule(str_image, cmd, job_id, \
                                         number_of_workers or compute_data['number_of_workers'],\
                                         cpu_limit,\
                                         memory_limit, \
                                         gpu_limit or compute_data['gpu_limit'], \
                                         incoming_dir, outgoing_dir)
            return {'job_details': str(job)}

        return {
            'jid': job_id,
            'image': job_info['image'],
            'cmd': job_info['cmd'],
            'status': job_info['status'],
            'message': job_info['message'],
            'timestamp': job_info['timestamp'],
            'containerid': job_info['containerid'],
            'exitcode': job_info['exitcode'],
            'pid': job_info['pid'],
            'logs': job_logs
        }

    def build_app_cmd(self, compute_data):
        """
        Build and return the app's cmd string.
        """
        cmd_args = compute_data['cmd_args']
        cmd_path_flags = compute_data['cmd_path_flags']
        if cmd_path_flags:
            # process the argument of any cmd flag that is a 'path'
            path_flags = cmd_path_flags.split(',')
            args = cmd_args.split()
            for i in range(len(args) - 1):
                if args[i] in path_flags:
                    # each flag value is a string of one or more paths separated by comma
                    # paths = args[i+1].split(',')
                    # base_inputdir = self.str_app_container_inputdir
                    # paths = [os.path.join(base_inputdir, p.lstrip('/')) for p in paths]
                    # args[i+1] = ','.join(paths)

                    # the next is tmp until CUBE's assumptions about inputdir and path
                    # parameters are removed
                    args[i+1] = self.str_app_container_inputdir
            cmd_args = ' '.join(args)
        selfpath = compute_data['selfpath']
        selfexec = compute_data['selfexec']
        execshell = compute_data['execshell']
        type = compute_data['type']
        outputdir = self.str_app_container_outputdir
        exec = os.path.join(selfpath, selfexec)
        cmd = f'{execshell} {exec}'
        if type == 'ds':
            inputdir = self.str_app_container_inputdir
            cmd = cmd + f' {cmd_args} {inputdir} {outputdir}'
        elif type in ('fs', 'ts'):
            cmd = cmd + f' {cmd_args} {outputdir}'
        return cmd

class Job(Resource):
    """
    Resource representing a single job running on the compute.
    """

    # Initiate an openshiftmgr instance
    def get_openshift_manager(self):
        self.openshiftmgr = OpenShiftManager()
        return self.openshiftmgr

    # Get the status of a running jon on Openshift
    def t_status_process_openshift(self, jid):
        """
        Determine the status of a job scheduled using the openshift manager.
        PRECONDITIONS:
        o   Only call this method if a container structure exists
            in the relevant job tree!
        POSTCONDITIONS:
        o   If the job is completed, then shutdown the container cluster
            service.
        """

        str_logs = ''
        # Get job-id from request
        #jid = self.jid

        # Query OpenShift API to get job state
        d_json  = self.get_openshift_manager().state(jid)

        print (d_json)
        print (jid)

        if d_json['Status']['Message'] == 'finished':
            pod_names = self.get_openshift_manager().get_pod_names_in_job(jid)
            for _, pod_name in enumerate(pod_names):
                str_logs += self.get_openshift_manager().get_job_pod_logs(pod_name, jid)
        else:
            str_logs = d_json['Status']['Message']

        status  = d_json['Status']
        currentState =  d_json['Status']['Message']

        return {
            'status':           status,
            'logs':             str_logs,
            'currentState':     [currentState]
        }

    def get(self, job_id):

        container_env = app.config.get('CONTAINER_ENV')

        job_logs = ''
        job_info = {'id': '', 'image': '', 'cmd': '', 'timestamp': '', 'message': '',
                    'status': 'undefined', 'containerid': '', 'exitcode': '', 'pid': ''}

        if container_env == 'swarm':
            swarm_mgr = SwarmManager()
            logger.info(f'Getting job {job_id} status from the Swarm cluster')
            try:
                service = swarm_mgr.get_service(job_id)
            except docker.errors.NotFound as e:
                abort(404, message=str(e))
            except docker.errors.APIError as e:
                status_code = e.response.status_code
                status_code = 503 if status_code == 500 else status_code
                abort(status_code, message=str(e))
            except docker.errors.InvalidVersion as e:
                abort(400, message=str(e))
            job_info = swarm_mgr.get_service_task_info(service)
            logger.info(f'Successful job {job_id} status response from Swarm: '
                        f'{job_info}')
            job_logs = swarm_mgr.get_service_logs(service)

            if job_info['status'] in ('undefined', 'finishedWithError',
                                      'finishedSuccessfully'):
                service.remove()  # remove job from swarm cluster

        if container_env == 'openshift':
            logger.info(f'Getting job {job_id} status from the Openshift cluster')
            try:
                d_containerStatus       =   self.t_status_process_openshift(job_id)
                status                  =   d_containerStatus['status']
                logs                    =   d_containerStatus['logs']
                currentState            =   d_containerStatus['currentState']
            except Exception as e:
                if isinstance(e, ApiException) and e.reason == 'Not Found':
                    status = logs = currentState = e.reason
                else:
                    raise e

            d_ret = {
                'description':   str(status),
                'l_logs':     str(logs),
                'l_status': currentState
            }
            if 'finished' in str(currentState) :
                job_status = 'finishedSuccessfully'

                # Also delete the job pod and related pvc
                try:
                    job = self.get_openshift_manager().get_job(job_id)
                    self.get_openshift_manager().remove_pvc(job_id)
                    self.get_openshift_manager().remove_job(job_id)
                except Exception as err:
                    logger.info(f'Error deleting pvc/job: {err}')
            else :
                job_status = str(currentState)

            return {
                    'd_ret':    d_ret,
                    'status':   job_status
            }

        return {
            'jid': job_id,
            'image': job_info['image'],
            'cmd': job_info['cmd'],
            'status': job_info['status'],
            'message': job_info['message'],
            'timestamp': job_info['timestamp'],
            'containerid': job_info['containerid'],
            'exitcode': job_info['exitcode'],
            'pid': job_info['pid'],
            'logs': job_logs
        }

class Hello(Resource):

     # Respond to simple 'hello' requests from the server
    def get(self):

            container_env = app.config.get('CONTAINER_ENV')

            smiling_face = emoji.emojize(":grinning_face_with_big_eyes:")
            logger.info(f'pman says hello from {container_env} {smiling_face}')
            b_status            = False
            d_ret               = {}
            d_ret['message']                = (f'pman says hello from {container_env} {smiling_face}')
            d_ret['sysinfo']                = {}
            d_ret['sysinfo']['system']      = platform.system()
            d_ret['sysinfo']['machine']     = platform.machine()
            d_ret['sysinfo']['platform']    = platform.platform()
            d_ret['sysinfo']['uname']       = platform.uname()
            d_ret['sysinfo']['version']     = platform.version()
            d_ret['sysinfo']['memory']      = psutil.virtual_memory()
            d_ret['sysinfo']['cpucount']    = multiprocessing.cpu_count()
            d_ret['sysinfo']['loadavg']     = os.getloadavg()
            d_ret['sysinfo']['cpu_percent'] = psutil.cpu_percent()
            d_ret['sysinfo']['hostname']    = socket.gethostname()
            d_ret['sysinfo']['inet']        = [l for l in ([ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] if not ip.startswith("127.")][:1], [[(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close()) for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]][0][1]]) if l][0][0]
            b_status                        = True

            return { 'd_ret':   d_ret,
                 'status':  b_status}

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import re
import os
import sys
import shutil

import ipodio

from manager import Manager

manager = Manager()

def first(collection):
    for element in collection:
        return element

def error(message):
    print('Error: ' + message)
    sys.exit(1)

def _sorted_tracks(tracks, key=None):
    def by(field):
        def accessor(element):
            return (key(element) if key else element).internal[field]
        return accessor

    tracks.sort(key=by('track_nr'))
    tracks.sort(key=by('album'))
    tracks.sort(key=by('artist'))

    return tracks

def _line(data):
    title = data['title'] or ''
    album = data['album'] or ''
    artist = data['artist'] or ''

    return "{title:30}  {album:30}  {artist:18}".format(
        title=title[:30], album=album[:30], artist=artist[:18])

def _compile_regular_expression(expression):
    try:
        return re.compile(expression, flags=re.IGNORECASE)
    except (ValueError, TypeError, re.error):
        error('Invalid expression "{}"'.format(expression))

def _filter_by_regular_expression(regexp, tracks):
    return [track for track in tracks if regexp.search(_line(track.internal))]

@manager.command
def list(mountpoint, expression):
    """List ipod contents"""
    database = ipodio.Database.create(mountpoint)
    database.update_index()

    regexp = _compile_regular_expression(expression)
    tracks = _filter_by_regular_expression(regexp, database.tracks)

    if tracks:
        print(_line(dict(title='Title', album='Album', artist='Artist')))
        print('-' * 80)

    for track in _sorted_tracks(tracks):
        print(_line(track.internal))

    if database.updated:
        database.save()

@manager.command
def duplicates(mountpoint):
    """List ipod contents grouping duplicated tracks"""
    database = ipodio.Database.create(mountpoint)
    database.update_index()

    track_groups = _sorted_tracks(database.duplicates, key=lambda g: first(g))

    if track_groups:
        print(_line(dict(title='Title', album='Album', artist='Artist')))
        print('-' * 80)

    for group in track_groups:
        for track in group:
            print(_line(track.internal))

    if database.updated:
        database.save()

@manager.command
def push(mountpoint, filename, force=False):
    """List ipod contents grouping duplicated tracks"""
    database = ipodio.Database.create(mountpoint)
    database.update_index()

    track = ipodio.database.Track.create(filename)
    track.update_hash()

    if not force and database.get(track):
        return '{} is already in the ipod'.format(repr(track.internal))

    database.add(track)
    database.copy_files()
    database.save()

    return repr(track.internal)

@manager.command
def pull(mountpoint, expression):
    """List ipod contents grouping duplicated tracks"""
    database = ipodio.Database.create(mountpoint)
    database.update_index()

    destination = '.'

    regexp = _compile_regular_expression(expression)
    tracks = _filter_by_regular_expression(regexp, database.tracks)

    for track in tracks:
        track_name = u'{track_nr}_{title}_{album}_{artist}'.format(
            track_nr=track.internal['track_nr'],
            title=track.internal['title'],
            album=track.internal['album'],
            artist=track.internal['artist']
        )

        print(track.internal, track_name)
        shutil.copy(track.filename, os.path.join(destination, track_name))

@manager.command
def rm(mountpoint, expression):
    database = ipodio.Database.create(mountpoint)
    database.update_index()

    print(_line(dict(title='Title', album='Album', artist='Artist')))
    print('-' * 80)

    regexp = _compile_regular_expression(expression)
    tracks = _filter_by_regular_expression(regexp, database.tracks)

    for track in tracks:
        print(_line(track.internal))
        database.remove(track)

    if database.updated:
        database.save()

@manager.command
def rename(mountpoint, expression, replacement):
    database = ipodio.Database.create(mountpoint)
    database.update_index()

    regexp = _compile_regular_expression(expression)
    tracks = _filter_by_regular_expression(regexp, database.tracks)

    for track in tracks:
        track.internal['artist'] = regexp.sub(replacement, track.internal['artist'])
        track.internal['album'] = regexp.sub(replacement, track.internal['album'])
        track.internal['title'] = regexp.sub(replacement, track.internal['title'])

        print(_line(track.internal))

    if tracks:
        database.save()

def main():
    manager.main()

if __name__ == '__main__':
    main()
<commit_msg>Adds extension to filenames when pulling tracks<commit_after># -*- coding: utf-8 -*-

import re
import os
import sys
import shutil

import ipodio

from manager import Manager

manager = Manager()

def first(collection):
    for element in collection:
        return element

def error(message):
    print('Error: ' + message)
    sys.exit(1)

def _sorted_tracks(tracks, key=None):
    def by(field):
        def accessor(element):
            return (key(element) if key else element).internal[field]
        return accessor

    tracks.sort(key=by('track_nr'))
    tracks.sort(key=by('album'))
    tracks.sort(key=by('artist'))

    return tracks

def _line(data):
    title = data['title'] or ''
    album = data['album'] or ''
    artist = data['artist'] or ''

    return "{title:30}  {album:30}  {artist:18}".format(
        title=title[:30], album=album[:30], artist=artist[:18])

def _compile_regular_expression(expression):
    try:
        return re.compile(expression, flags=re.IGNORECASE)
    except (ValueError, TypeError, re.error):
        error('Invalid expression "{}"'.format(expression))

def _filter_by_regular_expression(regexp, tracks):
    return [track for track in tracks if regexp.search(_line(track.internal))]

@manager.command
def list(mountpoint, expression):
    """List ipod contents"""
    database = ipodio.Database.create(mountpoint)
    database.update_index()

    regexp = _compile_regular_expression(expression)
    tracks = _filter_by_regular_expression(regexp, database.tracks)

    if tracks:
        print(_line(dict(title='Title', album='Album', artist='Artist')))
        print('-' * 80)

    for track in _sorted_tracks(tracks):
        print(_line(track.internal))

    if database.updated:
        database.save()

@manager.command
def duplicates(mountpoint):
    """List ipod contents grouping duplicated tracks"""
    database = ipodio.Database.create(mountpoint)
    database.update_index()

    track_groups = _sorted_tracks(database.duplicates, key=lambda g: first(g))

    if track_groups:
        print(_line(dict(title='Title', album='Album', artist='Artist')))
        print('-' * 80)

    for group in track_groups:
        for track in group:
            print(_line(track.internal))

    if database.updated:
        database.save()

@manager.command
def push(mountpoint, filename, force=False):
    """List ipod contents grouping duplicated tracks"""
    database = ipodio.Database.create(mountpoint)
    database.update_index()

    track = ipodio.database.Track.create(filename)
    track.update_hash()

    if not force and database.get(track):
        return '{} is already in the ipod'.format(repr(track.internal))

    database.add(track)
    database.copy_files()
    database.save()

    return repr(track.internal)

@manager.command
def pull(mountpoint, expression):
    """List ipod contents grouping duplicated tracks"""
    database = ipodio.Database.create(mountpoint)
    database.update_index()

    destination = '.'

    regexp = _compile_regular_expression(expression)
    tracks = _filter_by_regular_expression(regexp, database.tracks)

    for track in tracks:
        track_name = u'{track_nr}_{title}_{album}_{artist}.{extension}'.format(
            track_nr=track.internal['track_nr'],
            title=track.internal['title'],
            album=track.internal['album'],
            artist=track.internal['artist'],
            extension=track.filename.split('.')[-1]
        )

        print(track.internal, track_name)
        shutil.copy(track.filename, os.path.join(destination, track_name))

@manager.command
def rm(mountpoint, expression):
    database = ipodio.Database.create(mountpoint)
    database.update_index()

    print(_line(dict(title='Title', album='Album', artist='Artist')))
    print('-' * 80)

    regexp = _compile_regular_expression(expression)
    tracks = _filter_by_regular_expression(regexp, database.tracks)

    for track in tracks:
        print(_line(track.internal))
        database.remove(track)

    if database.updated:
        database.save()

@manager.command
def rename(mountpoint, expression, replacement):
    database = ipodio.Database.create(mountpoint)
    database.update_index()

    regexp = _compile_regular_expression(expression)
    tracks = _filter_by_regular_expression(regexp, database.tracks)

    for track in tracks:
        track.internal['artist'] = regexp.sub(replacement, track.internal['artist'])
        track.internal['album'] = regexp.sub(replacement, track.internal['album'])
        track.internal['title'] = regexp.sub(replacement, track.internal['title'])

        print(_line(track.internal))

    if tracks:
        database.save()

def main():
    manager.main()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
mbed SDK
Copyright (c) 2011-2015 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
"""

import re
import sys
from time import time
from Queue import Queue, Empty
from threading import Thread
from subprocess import call, Popen, PIPE

# Return codes for test script
TEST_RESULT_OK = "OK"
TEST_RESULT_FAIL = "FAIL"
TEST_RESULT_ERROR = "ERROR"
TEST_RESULT_UNDEF = "UNDEF"
TEST_RESULT_IOERR_COPY = "IOERR_COPY"
TEST_RESULT_IOERR_DISK = "IOERR_DISK"
TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
TEST_RESULT_TIMEOUT = "TIMEOUT"
TEST_RESULT_NO_IMAGE = "NO_IMAGE"
TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"

TEST_RESULTS = [TEST_RESULT_OK,
                TEST_RESULT_FAIL,
                TEST_RESULT_ERROR,
                TEST_RESULT_UNDEF,
                TEST_RESULT_IOERR_COPY,
                TEST_RESULT_IOERR_DISK,
                TEST_RESULT_IOERR_SERIAL,
                TEST_RESULT_TIMEOUT,
                TEST_RESULT_NO_IMAGE,
                TEST_RESULT_MBED_ASSERT,
                TEST_RESULT_BUILD_FAILED
                ]

TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
                       "failure" : TEST_RESULT_FAIL,
                       "error" : TEST_RESULT_ERROR,
                       "end" : TEST_RESULT_UNDEF,
                       "ioerr_copy" : TEST_RESULT_IOERR_COPY,
                       "ioerr_disk" : TEST_RESULT_IOERR_DISK,
                       "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
                       "timeout" : TEST_RESULT_TIMEOUT,
                       "no_image" : TEST_RESULT_NO_IMAGE,
                       "mbed_assert" : TEST_RESULT_MBED_ASSERT,
                       "build_failed" : TEST_RESULT_BUILD_FAILED
                       }

RE_DETECT_TESTCASE_RESULT = re.compile("\\{(" + "|".join(TEST_RESULT_MAPPING.keys()) + ")\\}")

def run_host_test(image_path, disk, port, duration=10,
                  micro=None, reset=None, reset_tout=None,
                  verbose=False, copy_method=None, program_cycle_s=None,
                  digest_source=None, json_test_cfg=None, run_cmd=None):
    """ This function runs host test supervisor (executes mbedhtrun) and checks
        output from host test process.

        @param digest_source - if None mbedhtrun will be executed. If 'stdin',
                               stdin will be used via StdInObserver or file (if
                               file name was given as switch option)
    """

    class StdInObserver(Thread):
        """ process used to read stdin only as console
            input from MUT
        """
        def __init__(self):
            Thread.__init__(self)
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            while self.active:
                c = sys.stdin.read(1)
                self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception, _:
                pass

    class FileObserver(Thread):
        """ process used to read file content as console
            input from MUT
        """
        def __init__(self, filename):
            Thread.__init__(self)
            self.filename = filename
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            with open(self.filename) as f:
                while self.active:
                    c = f.read(1)
                    self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception, _:
                pass

    class ProcessObserver(Thread):
        """ Default process used to observe stdout of
            another process as console input from MUT
        """
        def __init__(self, proc):
            Thread.__init__(self)
            self.proc = proc
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            while self.active:
                c = self.proc.stdout.read(1)
                self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception, _:
                pass

    def get_char_from_queue(obs):
        """ Get character from queue safe way
        """
        try:
            c = obs.queue.get(block=True, timeout=0.5)
        except Empty, _:
            c = None
        return c

    def filter_queue_char(c):
        """ Filters out non ASCII characters from serial port
        """
        if ord(c) not in range(128):
            c = ' '
        return c

    def get_test_result(output):
        """ Parse test 'output' data
        """
        result = TEST_RESULT_TIMEOUT
        for line in "".join(output).splitlines():
            search_result = RE_DETECT_TESTCASE_RESULT.search(line)
            if search_result and len(search_result.groups()):
                result = TEST_RESULT_MAPPING[search_result.groups(0)[0]]
                break
        return result

    def get_auto_property_value(property_name, line):
        """ Scans auto detection line from MUT and returns scanned parameter 'property_name'
            Returns string or None if property search failed
        """
        result = None
        if re.search("HOST: Property '%s'"% property_name, line) is not None:
            property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
            if property is not None and len(property.groups()) == 1:
                result = property.groups()[0]
        return result

    # Detect from where input should be taken, if no --digest switch is specified
    # normal test execution can be performed
    if digest_source == 'stdin':
        # When we want to scan stdin for test results
        obs = StdInObserver()
    elif digest_source is not None:
        # When we want to open file to scan for test results
        obs = FileObserver(digest_source)
    else:
        # Command executing CLI for host test supervisor (in detect-mode)
        cmd = ["mbedhtrun",
                '-d', disk,
                '-f', '"%s"' % image_path,
                '-p', port,
                '-C', str(program_cycle_s)]

        # Add extra parameters to host_test
        if copy_method is not None:
            cmd += ["-c", copy_method]
        if micro is not None:
            cmd += ["-m", micro]
        if reset is not None:
            cmd += ["-r", reset]
        if reset_tout is not None:
            cmd += ["-R", str(reset_tout)]
        if json_test_cfg is not None:
            cmd += ["--test-cfg", '"%s"' % str(json_test_cfg)]
        if run_cmd is not None:
            cmd += ["--run", str(run_cmd)]

        if verbose:
            print "mbed-host-test-runner: %s" % (" ".join(cmd))

        proc = Popen(cmd, stdout=PIPE)
        obs = ProcessObserver(proc)

    update_once_flag = {}   # Stores flags checking if some auto-parameter was already set
    line = ''
    output = []
    start_time = time()
    while (time() - start_time) < (2 * duration):
        c = get_char_from_queue(obs)
        if c:
            if verbose:
                sys.stdout.write(c)
            c = filter_queue_char(c)
            output.append(c)
            # Give the mbed under test a way to communicate the end of the test
            if c in ['\n', '\r']:

                # Checking for auto-detection information from the test about MUT reset moment
                if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
                    # We will update this marker only once to prevent multiple time resets
                    update_once_flag['reset_target'] = True
                    start_time = time()

                # Checking for auto-detection information from the test about timeout
                auto_timeout_val = get_auto_property_value('timeout', line)
                if 'timeout' not in update_once_flag and auto_timeout_val is not None:
                    # We will update this marker only once to prevent multiple time resets
                    update_once_flag['timeout'] = True
                    duration = int(auto_timeout_val)

                # Detect mbed assert:
                if 'mbed assertation failed: ' in line:
                    output.append('{{mbed_assert}}')
                    break

                # Check for test end
                if '{end}' in line:
                    break
                line = ''
            else:
                line += c
    end_time = time()
    testcase_duration = end_time - start_time   # Test case duration from reset to {end}

    c = get_char_from_queue(obs)

    if c:
        if verbose:
            sys.stdout.write(c)
        c = filter_queue_char(c)
        output.append(c)

    # Stop test process
    obs.stop()
    result = get_test_result(output)
    return (result, "".join(output), testcase_duration, duration)

def run_cli_command(cmd, shell=True, verbose=False):
    """ Runs command from command line.
        Return True if command was executed successfully.
    """
    result = True
    try:
        ret = call(cmd, shell=shell)
        if ret:
            result = False
            if verbose:
                print "mbedgt: [ret=%d] Command: %s"% (int(ret), cmd)
    except Exception as e:
        result = False
        if verbose:
            print "mbedgt: [ret=%d] Command: %s"% (int(ret), cmd)
            print str(e)
    return result

def run_cli_process(cmd):
    """ Runs command as a process and return stdout, stderr and ret code
    """
    p = Popen(cmd, stdout=PIPE, stderr=PIPE)
    _stdout, _stderr = p.communicate()
    return _stdout, _stderr, p.returncode
<commit_msg>BugFix: mbedgt hangs forever waiting for dump_serial thread to finish<commit_after>"""
mbed SDK
Copyright (c) 2011-2015 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
"""

import re
import sys
from time import time
from Queue import Queue, Empty
from threading import Thread
from subprocess import call, Popen, PIPE

# Return codes for test script
TEST_RESULT_OK = "OK"
TEST_RESULT_FAIL = "FAIL"
TEST_RESULT_ERROR = "ERROR"
TEST_RESULT_UNDEF = "UNDEF"
TEST_RESULT_IOERR_COPY = "IOERR_COPY"
TEST_RESULT_IOERR_DISK = "IOERR_DISK"
TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
TEST_RESULT_TIMEOUT = "TIMEOUT"
TEST_RESULT_NO_IMAGE = "NO_IMAGE"
TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"

TEST_RESULTS = [TEST_RESULT_OK,
                TEST_RESULT_FAIL,
                TEST_RESULT_ERROR,
                TEST_RESULT_UNDEF,
                TEST_RESULT_IOERR_COPY,
                TEST_RESULT_IOERR_DISK,
                TEST_RESULT_IOERR_SERIAL,
                TEST_RESULT_TIMEOUT,
                TEST_RESULT_NO_IMAGE,
                TEST_RESULT_MBED_ASSERT,
                TEST_RESULT_BUILD_FAILED
                ]

TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
                       "failure" : TEST_RESULT_FAIL,
                       "error" : TEST_RESULT_ERROR,
                       "end" : TEST_RESULT_UNDEF,
                       "ioerr_copy" : TEST_RESULT_IOERR_COPY,
                       "ioerr_disk" : TEST_RESULT_IOERR_DISK,
                       "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
                       "timeout" : TEST_RESULT_TIMEOUT,
                       "no_image" : TEST_RESULT_NO_IMAGE,
                       "mbed_assert" : TEST_RESULT_MBED_ASSERT,
                       "build_failed" : TEST_RESULT_BUILD_FAILED
                       }

RE_DETECT_TESTCASE_RESULT = re.compile("\\{(" + "|".join(TEST_RESULT_MAPPING.keys()) + ")\\}")

def run_host_test(image_path, disk, port, duration=10,
                  micro=None, reset=None, reset_tout=None,
                  verbose=False, copy_method=None, program_cycle_s=None,
                  digest_source=None, json_test_cfg=None, run_cmd=None):
    """ This function runs host test supervisor (executes mbedhtrun) and checks
        output from host test process.

        @param digest_source - if None mbedhtrun will be executed. If 'stdin',
                               stdin will be used via StdInObserver or file (if
                               file name was given as switch option)
    """

    class StdInObserver(Thread):
        """ process used to read stdin only as console
            input from MUT
        """
        def __init__(self):
            Thread.__init__(self)
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            while self.active:
                c = sys.stdin.read(1)
                self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception, _:
                pass

    class FileObserver(Thread):
        """ process used to read file content as console
            input from MUT
        """
        def __init__(self, filename):
            Thread.__init__(self)
            self.filename = filename
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            with open(self.filename) as f:
                while self.active:
                    c = f.read(1)
                    self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception, _:
                pass

    class ProcessObserver(Thread):
        """ Default process used to observe stdout of
            another process as console input from MUT
        """
        def __init__(self, proc):
            Thread.__init__(self)
            self.proc = proc
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            while self.active:
                c = self.proc.stdout.read(1)
                self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception, _:
                pass

    def get_char_from_queue(obs):
        """ Get character from queue safe way
        """
        try:
            c = obs.queue.get(block=True, timeout=0.5)
            # signals to queue job is done
            obs.queue.task_done()
        except Empty:
            c = None
        except:
            raise
        return c

    def filter_queue_char(c):
        """ Filters out non ASCII characters from serial port
        """
        if ord(c) not in range(128):
            c = ' '
        return c

    def get_test_result(output):
        """ Parse test 'output' data
        """
        result = TEST_RESULT_TIMEOUT
        for line in "".join(output).splitlines():
            search_result = RE_DETECT_TESTCASE_RESULT.search(line)
            if search_result and len(search_result.groups()):
                result = TEST_RESULT_MAPPING[search_result.groups(0)[0]]
                break
        return result

    def get_auto_property_value(property_name, line):
        """ Scans auto detection line from MUT and returns scanned parameter 'property_name'
            Returns string or None if property search failed
        """
        result = None
        if re.search("HOST: Property '%s'"% property_name, line) is not None:
            property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
            if property is not None and len(property.groups()) == 1:
                result = property.groups()[0]
        return result

    # Detect from where input should be taken, if no --digest switch is specified
    # normal test execution can be performed
    if digest_source == 'stdin':
        # When we want to scan stdin for test results
        obs = StdInObserver()
    elif digest_source is not None:
        # When we want to open file to scan for test results
        obs = FileObserver(digest_source)
    else:
        # Command executing CLI for host test supervisor (in detect-mode)
        cmd = ["mbedhtrun",
                '-d', disk,
                '-f', '"%s"' % image_path,
                '-p', port,
                '-C', str(program_cycle_s)]

        # Add extra parameters to host_test
        if copy_method is not None:
            cmd += ["-c", copy_method]
        if micro is not None:
            cmd += ["-m", micro]
        if reset is not None:
            cmd += ["-r", reset]
        if reset_tout is not None:
            cmd += ["-R", str(reset_tout)]
        if json_test_cfg is not None:
            cmd += ["--test-cfg", '"%s"' % str(json_test_cfg)]
        if run_cmd is not None:
            cmd += ["--run", str(run_cmd)]

        if verbose:
            print "mbed-host-test-runner: %s" % (" ".join(cmd))

        proc = Popen(cmd, stdout=PIPE)
        obs = ProcessObserver(proc)

    update_once_flag = {}   # Stores flags checking if some auto-parameter was already set
    line = ''
    output = []
    start_time = time()
    while (time() - start_time) < (2 * duration):
        try:
            c = get_char_from_queue(obs)
        except:
            break
        if c:
            if verbose:
                sys.stdout.write(c)
            c = filter_queue_char(c)
            output.append(c)
            # Give the mbed under test a way to communicate the end of the test
            if c in ['\n', '\r']:

                # Checking for auto-detection information from the test about MUT reset moment
                if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
                    # We will update this marker only once to prevent multiple time resets
                    update_once_flag['reset_target'] = True
                    start_time = time()

                # Checking for auto-detection information from the test about timeout
                auto_timeout_val = get_auto_property_value('timeout', line)
                if 'timeout' not in update_once_flag and auto_timeout_val is not None:
                    # We will update this marker only once to prevent multiple time resets
                    update_once_flag['timeout'] = True
                    duration = int(auto_timeout_val)

                # Detect mbed assert:
                if 'mbed assertation failed: ' in line:
                    output.append('{{mbed_assert}}')
                    break

                # Check for test end
                if '{end}' in line:
                    break
                line = ''
            else:
                line += c
    end_time = time()
    testcase_duration = end_time - start_time   # Test case duration from reset to {end}

    c = get_char_from_queue(obs)

    if c:
        if verbose:
            sys.stdout.write(c)
        c = filter_queue_char(c)
        output.append(c)

    # Stop test process
    obs.stop()
    if verbose:
        print "mbed-host-test-runner: Stopped"
    result = get_test_result(output)
    return (result, "".join(output), testcase_duration, duration)

def run_cli_command(cmd, shell=True, verbose=False):
    """ Runs command from command line.
        Return True if command was executed successfully.
    """
    result = True
    try:
        ret = call(cmd, shell=shell)
        if ret:
            result = False
            if verbose:
                print "mbedgt: [ret=%d] Command: %s"% (int(ret), cmd)
    except Exception as e:
        result = False
        if verbose:
            print "mbedgt: [ret=%d] Command: %s"% (int(ret), cmd)
            print str(e)
    return result

def run_cli_process(cmd):
    """ Runs command as a process and return stdout, stderr and ret code
    """
    p = Popen(cmd, stdout=PIPE, stderr=PIPE)
    _stdout, _stderr = p.communicate()
    return _stdout, _stderr, p.returncode
<|endoftext|>


<commit_before>def extract_status_file_path(config_path):
    """ Extracts the path to the openvpn status file from a openvpn config.

        :raises EnvironmentError: if the config does not contain a status file
            entry
        :return: the absolute path to the status file"""
    import re
    status_search = re.compile(r'^\s*status\s+(?P<value>[^#]+)(#.*)?$')

    status_file = None

    with open(config_path, 'r') as conf_file:
        for conf_line in conf_file:
            match = status_search.match(conf_line)
            if match:
                status_file = match.group('value').strip()

    if not status_file:
        raise EnvironmentError('You must specify a status entry!')

    import os
    return os.path.abspath(os.path.join(config_path, '..', status_file))

def extract_zones_from_status_file(status_path):
    """ Parses a openvpn status file and extracts the list of connected clients
        and there ip address """
    with open(status_path, 'r') as status_file:
        mode = None
        mode_changes = {
            'OpenVPN CLIENT LIST': 'clients',
            'ROUTING TABLE': 'routes',
            'GLOBAL STATS': None
        }
        clients = {}
        skip_next_lines = 0
        for status_line in status_file:
            if skip_next_lines > 0:
                skip_next_lines -= 1
                continue
            status_line = status_line.strip()
            if status_line in mode_changes:
                mode = mode_changes[status_line]
                skip_next_lines = 1
                if mode == 'clients':
                    skip_next_lines += 1
                continue
            if mode == 'clients':
                clients[status_line.split(',')[0]] = None
            if mode == 'routes':
                address, client = status_line.split(',')[0:2]
                if '/' in address:
                    continue
                if client not in clients:
                    raise ValueError('Error in status file')
                clients[client] = address
        return clients

def build_zone_from_clients(clients, name, **zone_opts):
    """ Basic zone generation (uses only the client list),
        additional data like SOA information must be passed
        as keyword option """
    from twisted.names import dns
    name = name or '.'.join(clients.keys()[0].split('.')[1:])
    zone = []
    zone.append((name, dns.Record_SOA(**zone_opts)))
    for client, address in clients.items():
        zone.append((client, dns.Record_A(address)))
    return zone

status_file = extract_status_file_path('server.conf')

zone = []

clients = extract_zones_from_status_file(status_file)

zone = build_zone_from_clients(clients, name=None,
                               # This nameserver's name
                               mname="ns1.example-domain.com",
                               # Mailbox of individual who handles this
                               rname="root.example-domain.com",
                               # Unique serial identifying this SOA data
                               serial=2003010601,
                               # Time interval before zone should be refreshed
                               refresh="1H",
                               # Interval before failed refresh should be retried
                               retry="1H",
                               # Upper limit on time interval before expiry
                               expire="1H",
                               # Minimum TTL
                               minimum="1H"
                               )
<commit_msg>convert openvpnzone into  authority and watch status file<commit_after>import os.path
from datetime import datetime
import signal

from twisted.names import dns
from twisted.names.authority import FileAuthority

from twisted.internet import inotify
from twisted.python import filepath

def extract_status_file_path(config_path):
    """ Extracts the path to the openvpn status file from a openvpn config.

        :raises EnvironmentError: if the config does not contain a status file
            entry
        :return: the absolute path to the status file"""
    import re
    status_search = re.compile(r'^\s*status\s+(?P<value>[^#]+)(#.*)?$')

    status_file = None

    with open(config_path, 'r') as conf_file:
        for conf_line in conf_file:
            match = status_search.match(conf_line)
            if match:
                status_file = match.group('value').strip()

    if not status_file:
        raise EnvironmentError('You must specify a status entry!')

    import os
    return os.path.abspath(os.path.join(config_path, '..', status_file))

def extract_zones_from_status_file(status_path):
    """ Parses a openvpn status file and extracts the list of connected clients
        and there ip address """
    with open(status_path, 'r') as status_file:
        mode = None
        mode_changes = {
            'OpenVPN CLIENT LIST': 'clients',
            'ROUTING TABLE': 'routes',
            'GLOBAL STATS': None
        }
        clients = {}
        skip_next_lines = 0
        for status_line in status_file:
            if skip_next_lines > 0:
                skip_next_lines -= 1
                continue
            status_line = status_line.strip()
            if status_line in mode_changes:
                mode = mode_changes[status_line]
                skip_next_lines = 1
                if mode == 'clients':
                    skip_next_lines += 1
                continue
            if mode == 'clients':
                clients[status_line.split(',')[0]] = None
            if mode == 'routes':
                address, client = status_line.split(',')[0:2]
                if '/' in address:
                    continue
                if client not in clients:
                    raise ValueError('Error in status file')
                clients[client] = address
        return clients

class OpenVpnStatusAuthority(FileAuthority):
    def __init__(self, status_file,  # This nameserver's name
                 mname="ns1.example-domain.com",
                 # Mailbox of individual who handles this
                 rname="root.example-domain.com",
                 # Unique serial identifying this SOA data
                 serial=2003010601,
                 # Time interval before zone should be refreshed
                 refresh="1H",
                 # Interval before failed refresh should be retried
                 retry="1H",
                 # Upper limit on time interval before expiry
                 expire="1H",
                 # Minimum TTL
                 minimum="1H"
                 ):
        self.status_file = status_file
        self.mname = mname
        self.rname = rname
        self.refresh = refresh
        self.retry = retry
        self.expire = expire
        self.minimum = minimum
        FileAuthority.__init__(self, status_file)

        # watch for file changes:
        signal.signal(signal.SIGUSR1, self.handle_signal)
        notifier = inotify.INotify()
        notifier.startReading()
        notifier.watch(filepath.FilePath(status_file), callbacks=[self.notify])

    def loadFile(self, status_file):
        clients = extract_zones_from_status_file(status_file)
        self.build_zone_from_clients(clients, status_file)

    def build_zone_from_clients(self, clients, status_file):
        """ Basic zone generation (uses only the client list),
            additional data like SOA information must be passed
            as keyword option """
        name = '.'.join(clients.keys()[0].split('.')[1:])
        self.records = {}
        self.soa = (name, dns.Record_SOA(
            mname=self.mname,
            rname=self.rname,
            serial=int(os.path.getmtime(status_file)),
            refresh=self.refresh,
            retry=self.retry,
            expire=self.expire,
            minimum=self.minimum,
        ))
        for client, address in clients.items():
            self.records.setdefault(client.lower(), []).append(dns.Record_A(address))

    def handle_signal(self, a, b):
        self.loadFile(self.status_file)

    def notify(self, ignored, filepath, mask):
        print('{} changed ({}), rereading zone data'.format(filepath,
              ','.join(inotify.humanReadableMask(mask))))
        self.loadFile(self.status_file)
<|endoftext|>


<commit_before>from django.contrib import admin
from django.contrib import auth
from django.utils.translation import ugettext_lazy as _

from . import models

class UserAdmin(auth.admin.UserAdmin):
    fieldsets = (
        (None, {'fields': ('username', 'email', 'password')}),
        (_('Permissions'), {'fields': ('is_staff', 'is_superuser')}),
        (_('Important dates'), {'fields': ('last_login', 'date_joined')}),
    )
    add_fieldsets = (
        (None, {
            'classes': ('wide',),
            'fields': ('username', 'email', 'password1', 'password2'),
        }),
    )
    readonly_fields = ('date_joined', 'last_login')
    list_display = (
        'username', 'email', 'date_joined', 'last_login', 'is_staff',
        'is_superuser'
    )
    list_filter = ('is_staff', 'is_superuser', 'last_login')
    search_fields = ('username', 'email')

admin.site.register(models.User, UserAdmin)
<commit_msg>display id of user in django admin<commit_after>from django.contrib import admin
from django.contrib import auth
from django.utils.translation import ugettext_lazy as _

from . import models

class UserAdmin(auth.admin.UserAdmin):
    fieldsets = (
        (None, {'fields': ('username', 'email', 'password')}),
        (_('Permissions'), {'fields': ('is_staff', 'is_superuser')}),
        (_('Important dates'), {'fields': ('last_login', 'date_joined')}),
    )
    add_fieldsets = (
        (None, {
            'classes': ('wide',),
            'fields': ('username', 'email', 'password1', 'password2'),
        }),
    )
    readonly_fields = ('date_joined', 'last_login')
    list_display = (
        'id', 'username', 'email', 'date_joined', 'last_login', 'is_staff',
        'is_superuser'
    )
    list_filter = ('is_staff', 'is_superuser', 'last_login')
    search_fields = ('username', 'email')

admin.site.register(models.User, UserAdmin)
<|endoftext|>


<commit_before>import os, cherrypy
from opmuse.transcoder import Transcoder

class Playlist:
    @cherrypy.expose
    @cherrypy.tools.jinja(filename='playlist.html')
    def list(self):
        playlist = cherrypy.session.get('playlist', [])

        return {'playlist': playlist}

    @cherrypy.expose
    def add(self, slug):
        library = cherrypy.engine.library.library
        track = library.get_track_by_slug(slug)

        cherrypy.session.acquire_lock()
        playlist = cherrypy.session.get('playlist', [])
        playlist.append(track)

        cherrypy.session['playlist'] = playlist
        cherrypy.session.release_lock()

class Styles(object):
    @cherrypy.expose
    def default(self, file):
        cherrypy.response.headers['Content-Type'] = 'text/css'

        path = os.path.join(os.path.abspath("."), "public", "styles")

        csspath = os.path.join(path, file)

        if os.path.exists(csspath):
            return cherrypy.lib.static.serve_file(csspath)

        ext = os.path.splitext(file)
        lesspath = os.path.join(path, "%s%s" % (ext[0], ".less"))

        if os.path.exists(lesspath):
            from lesscpy.lessc import parser
            p = parser.LessParser()
            p.parse(
                filename=lesspath,
                debuglevel=0
            )

            items = {
                'nl': '\n',
                'tab': '\t',
                'ws': ' ',
                'eb': '\n'
            }

            return ''.join([u.fmt(items) for u in p.result if u]).strip()

class Root(object):
    styles = Styles()
    playlist = Playlist()

    @cherrypy.expose
    @cherrypy.tools.jinja(filename='index.html')
    def index(self):
        return { }

    @cherrypy.expose
    @cherrypy.tools.jinja(filename='library.html')
    def library(self):
        library = cherrypy.engine.library.library
        return {'artists': library.get_artists()}

    @cherrypy.expose
    @cherrypy.config(**{'response.stream': True})
    def stream(self, **kwargs):

        playlist = cherrypy.session.get('playlist', [])

        if len(playlist) == 0:
            raise cherrypy.HTTPError(409)

        return Transcoder().transcode([track.filename for track in playlist])

<commit_msg>set proper content-type header for /stream<commit_after>import os, cherrypy
from opmuse.transcoder import Transcoder

class Playlist:
    @cherrypy.expose
    @cherrypy.tools.jinja(filename='playlist.html')
    def list(self):
        playlist = cherrypy.session.get('playlist', [])

        return {'playlist': playlist}

    @cherrypy.expose
    def add(self, slug):
        library = cherrypy.engine.library.library
        track = library.get_track_by_slug(slug)

        cherrypy.session.acquire_lock()
        playlist = cherrypy.session.get('playlist', [])
        playlist.append(track)

        cherrypy.session['playlist'] = playlist
        cherrypy.session.release_lock()

class Styles(object):
    @cherrypy.expose
    def default(self, file):
        cherrypy.response.headers['Content-Type'] = 'text/css'

        path = os.path.join(os.path.abspath("."), "public", "styles")

        csspath = os.path.join(path, file)

        if os.path.exists(csspath):
            return cherrypy.lib.static.serve_file(csspath)

        ext = os.path.splitext(file)
        lesspath = os.path.join(path, "%s%s" % (ext[0], ".less"))

        if os.path.exists(lesspath):
            from lesscpy.lessc import parser
            p = parser.LessParser()
            p.parse(
                filename=lesspath,
                debuglevel=0
            )

            items = {
                'nl': '\n',
                'tab': '\t',
                'ws': ' ',
                'eb': '\n'
            }

            return ''.join([u.fmt(items) for u in p.result if u]).strip()

class Root(object):
    styles = Styles()
    playlist = Playlist()

    @cherrypy.expose
    @cherrypy.tools.jinja(filename='index.html')
    def index(self):
        return { }

    @cherrypy.expose
    @cherrypy.tools.jinja(filename='library.html')
    def library(self):
        library = cherrypy.engine.library.library
        return {'artists': library.get_artists()}

    @cherrypy.expose
    @cherrypy.config(**{'response.stream': True})
    # TODO reimplement Accept header support
    def stream(self, **kwargs):

        playlist = cherrypy.session.get('playlist', [])

        if len(playlist) == 0:
            raise cherrypy.HTTPError(409)

        cherrypy.response.headers['Content-Type'] = 'audio/ogg'

        return Transcoder().transcode([track.filename for track in playlist])

<|endoftext|>


<commit_before>import numpy as np
from menpo.groupalign.base import MultipleAlignment
from menpo.shape import PointCloud
from menpo.transform import AlignmentSimilarity, UniformScale, Translation

class GeneralizedProcrustesAnalysis(MultipleAlignment):
    r"""
    TODO: Write documentation
    """
    def __init__(self, sources, **kwargs):
        super(GeneralizedProcrustesAnalysis, self).__init__(sources, **kwargs)
        self.transforms = [AlignmentSimilarity(source, self.target)
                           for source in self.sources]
        self.initial_target_scale = self.target.norm()
        self.n_iterations = 1
        self.max_iterations = 100
        self.converged = self._recursive_procrustes()
        print self

    def _recursive_procrustes(self):
        """
        Recursively calculates a Procrustes alignment
        """
        if self.n_iterations > self.max_iterations:
            return False
        av_aligned_source = sum(
            t.aligned_source.points for t in self.transforms) / self.n_sources
        new_target = PointCloud(av_aligned_source)
        # rescale the new_target to be the same size as the original about
        # it's centre
        rescale = UniformScale(
            self.initial_target_scale / new_target.norm(), self.n_dims)
        centre = Translation(-new_target.centre)
        rescale_about_centre = centre.compose_before(rescale).compose_before(
            centre.pseudoinverse)
        rescale_about_centre.apply_inplace(new_target)
        # check to see if  we have converged yet
        delta_target = np.linalg.norm(self.target.points - new_target.points)
        if delta_target < 1e-6:
            return True
        else:
            self.n_iterations += 1
            for t in self.transforms:
                t.target = new_target
            self.target = new_target
            return self._recursive_procrustes()

    @property
    def mean_aligned_shape(self):
        return PointCloud(np.mean([t.target.points for t in self.transforms],
                                  axis=0))

    @property
    def av_alignment_error(self):
        return sum([t.alignment_error for t in self.transforms])/self.n_sources

    def __str__(self):
        if self.converged:
            return ('Converged after %d iterations with av. error %f'
                    % (self.n_iterations, self.av_alignment_error))
        else:
            return ('Failed to converge after %d iterations with av. error '
                    '%f' % (self.n_iterations, self.av_alignment_error))
<commit_msg>target setter fix in procrustes<commit_after>import numpy as np
from menpo.groupalign.base import MultipleAlignment
from menpo.shape import PointCloud
from menpo.transform import AlignmentSimilarity, UniformScale, Translation

class GeneralizedProcrustesAnalysis(MultipleAlignment):
    r"""
    TODO: Write documentation
    """
    def __init__(self, sources, **kwargs):
        super(GeneralizedProcrustesAnalysis, self).__init__(sources, **kwargs)
        self.transforms = [AlignmentSimilarity(source, self.target)
                           for source in self.sources]
        self.initial_target_scale = self.target.norm()
        self.n_iterations = 1
        self.max_iterations = 100
        self.converged = self._recursive_procrustes()
        print self

    def _recursive_procrustes(self):
        """
        Recursively calculates a Procrustes alignment
        """
        if self.n_iterations > self.max_iterations:
            return False
        av_aligned_source = sum(
            t.aligned_source.points for t in self.transforms) / self.n_sources
        new_target = PointCloud(av_aligned_source)
        # rescale the new_target to be the same size as the original about
        # it's centre
        rescale = UniformScale(
            self.initial_target_scale / new_target.norm(), self.n_dims)
        centre = Translation(-new_target.centre)
        rescale_about_centre = centre.compose_before(rescale).compose_before(
            centre.pseudoinverse)
        rescale_about_centre.apply_inplace(new_target)
        # check to see if  we have converged yet
        delta_target = np.linalg.norm(self.target.points - new_target.points)
        if delta_target < 1e-6:
            return True
        else:
            self.n_iterations += 1
            for t in self.transforms:
                t.set_target(new_target)
            self.target = new_target
            return self._recursive_procrustes()

    @property
    def mean_aligned_shape(self):
        return PointCloud(np.mean([t.target.points for t in self.transforms],
                                  axis=0))

    @property
    def av_alignment_error(self):
        return sum([t.alignment_error for t in self.transforms])/self.n_sources

    def __str__(self):
        if self.converged:
            return ('Converged after %d iterations with av. error %f'
                    % (self.n_iterations, self.av_alignment_error))
        else:
            return ('Failed to converge after %d iterations with av. error '
                    '%f' % (self.n_iterations, self.av_alignment_error))
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7

from __future__ import print_function
import argparse
import functools
import re
import subprocess

CMD_REGEX = re.compile(r'(?!#)((?P<index>\d+):){0,1}(?P<command>.*)')

def parse_args():
    """Parse the command line arguments."""
    parser = argparse.ArgumentParser(description='Run an Oraide script.')
    parser.add_argument('script', metavar='SCRIPT',
        type=argparse.FileType('r'), help='path to an Oraide script')
    parser.add_argument('screen', metavar='SCREEN',
        help='name of a screen session')
    args = parser.parse_args()
    return args.script, args.screen

def send_command(session, window, command):
    """docstring for send_command"""
    args = ['screen',
            '-S', session,
            '-p', str(window),
            '-X', 'stuff', "%s" % command]
    proc = subprocess.Popen(args)
    proc.communicate()

def parse_script_line(line):
    """Parse a script line and return a tuple.

    For commands, the first element is a screen window and the second is the
    command string. For notes, the first element is None and the second element
    is the note string. For blank lines, both elements are None.
    """
    if line.strip() == '':
        return None, None
    elif line.startswith('#'):
        if line.startswith('# '):
            return None, line[2:].rstrip()
        else:
            return None, line[1:].rstrip()
    else:
        result = CMD_REGEX.search(line).groupdict()
        index, command = result['index'], result['command'].strip()
        if result['index'] is not None:
            return int(result['index']), command
        else:
            return 0, command

def parse_script(screen, lines):
    """Yields a callable for each command line of the script."""
    for line in lines:
        window, cmd = parse_script_line(line)
        if (window, cmd) == (None, None):   # blank line: noop
            pass
        elif window == None:    # note: print
            print(cmd)
        else:
            def _send():    # command: send to screen
                print("SENDING: %s ==> window %s" % (repr(cmd), window))
                send_command(screen, window, cmd)
                raw_input()
                send_command(screen, window, '\n')
                raw_input()
            yield _send

def main():
    """Run Oraide."""
    try:
        script, screen = parse_args()

        print('Starting %s. Press ENTER to advance script.\n' % script.name)
        for call in parse_script(screen, script.readlines()):
            call()
    except KeyboardInterrupt:
        pass
<commit_msg>Remove: leftover shebang line from debugging<commit_after>from __future__ import print_function
import argparse
import functools
import re
import subprocess

CMD_REGEX = re.compile(r'(?!#)((?P<index>\d+):){0,1}(?P<command>.*)')

def parse_args():
    """Parse the command line arguments."""
    parser = argparse.ArgumentParser(description='Run an Oraide script.')
    parser.add_argument('script', metavar='SCRIPT',
        type=argparse.FileType('r'), help='path to an Oraide script')
    parser.add_argument('screen', metavar='SCREEN',
        help='name of a screen session')
    args = parser.parse_args()
    return args.script, args.screen

def send_command(session, window, command):
    """docstring for send_command"""
    args = ['screen',
            '-S', session,
            '-p', str(window),
            '-X', 'stuff', "%s" % command]
    proc = subprocess.Popen(args)
    proc.communicate()

def parse_script_line(line):
    """Parse a script line and return a tuple.

    For commands, the first element is a screen window and the second is the
    command string. For notes, the first element is None and the second element
    is the note string. For blank lines, both elements are None.
    """
    if line.strip() == '':
        return None, None
    elif line.startswith('#'):
        if line.startswith('# '):
            return None, line[2:].rstrip()
        else:
            return None, line[1:].rstrip()
    else:
        result = CMD_REGEX.search(line).groupdict()
        index, command = result['index'], result['command'].strip()
        if result['index'] is not None:
            return int(result['index']), command
        else:
            return 0, command

def parse_script(screen, lines):
    """Yields a callable for each command line of the script."""
    for line in lines:
        window, cmd = parse_script_line(line)
        if (window, cmd) == (None, None):   # blank line: noop
            pass
        elif window == None:    # note: print
            print(cmd)
        else:
            def _send():    # command: send to screen
                print("SENDING: %s ==> window %s" % (repr(cmd), window))
                send_command(screen, window, cmd)
                raw_input()
                send_command(screen, window, '\n')
                raw_input()
            yield _send

def main():
    """Run Oraide."""
    try:
        script, screen = parse_args()

        print('Starting %s. Press ENTER to advance script.\n' % script.name)
        for call in parse_script(screen, script.readlines()):
            call()
    except KeyboardInterrupt:
        pass
<|endoftext|>


<commit_before>from datetime import datetime
import math

from merc import errors
from merc import feature
from merc import message
from merc import emitter

class StatsFeature(feature.Feature):
  NAME = __name__

install = StatsFeature.install

class StatsLinkInfo(message.Reply):
  NAME = "211"

  def __init__(self, linkname, sendq, sent_messages, sent_bytes, recv_messages, recv_bytes, open_time):
    self.linkname = linkname
    self.sendq = sendq
    self.sent_messages = sent_messages
    self.sent_bytes = sent_bytes
    self.recv_messages = recv_messages
    self.recv_bytes = recv_bytes
    self.open_time = open_time

  def as_reply_params(self):
    return [self.linkname, str(self.sendq), str(self.sent_messages),
            str(self.sent_bytes), str(self.recv_messages), str(self.recv_bytes),
            str(round(self.open_time.total_seconds()))]

class StatsCommands(message.Reply):
  NAME = "212"

  def __init__(self, command, count):
    self.command = command
    self.count = count

  def as_reply_params(self):
    return [self.command, str(self.count)]

class StatsUptime(message.Reply):
  NAME = "242"

  def __init__(self, uptime):
    total = uptime.total_seconds()
    self.days, total = divmod(total, 84600)
    self.hours, total = divmod(total, 3600)
    self.minutes, total = divmod(total, 60)

    self.days = math.floor(self.days)
    self.hours = math.floor(self.hours)
    self.minutes = math.floor(self.minutes)
    self.seconds = round(total)

  def as_reply_params(self):
    return ["Server Up {days} days {hours}:{minutes:02}:{seconds:02}".format(
        days=self.days, hours=self.hours, minutes=self.minutes, seconds=self.seconds)]

class EndOfStats(message.Reply):
  NAME = "219"

  def __init__(self, type):
    self.type = type

  def as_reply_params(self):
    return [self.type, "End of /STATS report"]

class StatsCLine(StatsLine):
  NAME = "213"

  def __init__(self, host, name, port, type):
   self.host = host
   self.name = name
   self.port = port
   self.type = type

  def as_reply_params(self):
    return ["C", self.host, "*", self.name, str(self.port), self.type]

class StatsHLine(message.Reply):
  NAME = "244"

  def __init__(self, hostmask, server):
    self.hostmask = hostmask
    self.server = server_name

  def as_reply_params(self):
    return ["H", self.hostmask, "*", self.server]

@StatsFeature.register_user_command
class Stats(message.Command):
  NAME = "STATS"
  MIN_ARITY = 1

  def __init__(self, type, server=None, *args):
    self.type = type
    self.server = server

  @message.Command.requires_registration
  def handle_for(self, app, user, prefix):
    if self.server is not None and self.server != app.server_name:
      raise errors.NoSuchServer(self.server_name)

    app.run_hooks("server.stats", user, self.type)
    app.run_hooks("server.stats." + self.type, user)
    user.send_reply(EndOfStats(self.type))

@StatsFeature.hook("server.stats.c")
def send_allowed_links(app, user):
  try:
    user.check_is_irc_operator()
  except errors.BaseError as e:
    user.send_reply(e)
  else:
    for name, link in app.config["links"].items():
      type = "hub" if link["hub"] else "server"
      user.send_reply(StatsCLine("*@{}".format(link["host"]), name, link["port"], type))

@StatsFeature.hook("server.stats.h")
def send_hubs(app, user):
  try:
    user.check_is_irc_operator()
  except errors.BaseError as e:
    user.send_reply(e)
  else:
    for name, link in app.config["links"].items():
      if not link["hub"]:
        continue
      user.send_reply(StatsHLine("*@{}".format(link["host"]), name))

@StatsFeature.hook("server.stats.l")
def send_links(app, user):
  try:
    user.check_is_irc_operator()
  except errors.BaseError as e:
    user.send_reply(e)
  else:
    # TODO: keep track of bytes sent/received?

    for user in app.users.all():
      if not user.is_local:
        continue
      mask = emitter.emit_hostmask(None, user.username, user.host)
      user.send_reply(StatsLinkInfo("{}[{}]".format(user.username, mask),
          0, 0, 0, 0, 0, datetime.now() - user.creation_time))

    for server in app.network.neighborhood():
      mask = "*@{}".format(app.config["links"][server.name]["host"])
      user.send_reply(StatsLinkInfo("{}[{}]".format(server.name, mask),
          0, 0, 0, 0, 0, 0))

@StatsFeature.hook("server.stats.m")
def send_commands(app, user):
  for feature in app.features.values():
    for command in feature.USER_COMMANDS.values():
      user.send_reply(StatsCommands(command.NAME, 0))

@StatsFeature.hook("server.stats.u")
def send_uptime(app, user):
  user.send_reply(StatsUptime(datetime.now() - app.creation_time))
      <commit_msg>Fix typo in STATS feature.<commit_after>from datetime import datetime
import math

from merc import errors
from merc import feature
from merc import message
from merc import emitter

class StatsFeature(feature.Feature):
  NAME = __name__

install = StatsFeature.install

class StatsLinkInfo(message.Reply):
  NAME = "211"

  def __init__(self, linkname, sendq, sent_messages, sent_bytes, recv_messages, recv_bytes, open_time):
    self.linkname = linkname
    self.sendq = sendq
    self.sent_messages = sent_messages
    self.sent_bytes = sent_bytes
    self.recv_messages = recv_messages
    self.recv_bytes = recv_bytes
    self.open_time = open_time

  def as_reply_params(self):
    return [self.linkname, str(self.sendq), str(self.sent_messages),
            str(self.sent_bytes), str(self.recv_messages), str(self.recv_bytes),
            str(round(self.open_time.total_seconds()))]

class StatsCommands(message.Reply):
  NAME = "212"

  def __init__(self, command, count):
    self.command = command
    self.count = count

  def as_reply_params(self):
    return [self.command, str(self.count)]

class StatsUptime(message.Reply):
  NAME = "242"

  def __init__(self, uptime):
    total = uptime.total_seconds()
    self.days, total = divmod(total, 84600)
    self.hours, total = divmod(total, 3600)
    self.minutes, total = divmod(total, 60)

    self.days = math.floor(self.days)
    self.hours = math.floor(self.hours)
    self.minutes = math.floor(self.minutes)
    self.seconds = round(total)

  def as_reply_params(self):
    return ["Server Up {days} days {hours}:{minutes:02}:{seconds:02}".format(
        days=self.days, hours=self.hours, minutes=self.minutes, seconds=self.seconds)]

class EndOfStats(message.Reply):
  NAME = "219"

  def __init__(self, type):
    self.type = type

  def as_reply_params(self):
    return [self.type, "End of /STATS report"]

class StatsCLine(message.Reply):
  NAME = "213"

  def __init__(self, host, name, port, type):
   self.host = host
   self.name = name
   self.port = port
   self.type = type

  def as_reply_params(self):
    return ["C", self.host, "*", self.name, str(self.port), self.type]

class StatsHLine(message.Reply):
  NAME = "244"

  def __init__(self, hostmask, server):
    self.hostmask = hostmask
    self.server = server_name

  def as_reply_params(self):
    return ["H", self.hostmask, "*", self.server]

@StatsFeature.register_user_command
class Stats(message.Command):
  NAME = "STATS"
  MIN_ARITY = 1

  def __init__(self, type, server=None, *args):
    self.type = type
    self.server = server

  @message.Command.requires_registration
  def handle_for(self, app, user, prefix):
    if self.server is not None and self.server != app.server_name:
      raise errors.NoSuchServer(self.server_name)

    app.run_hooks("server.stats", user, self.type)
    app.run_hooks("server.stats." + self.type, user)
    user.send_reply(EndOfStats(self.type))

@StatsFeature.hook("server.stats.c")
def send_allowed_links(app, user):
  try:
    user.check_is_irc_operator()
  except errors.BaseError as e:
    user.send_reply(e)
  else:
    for name, link in app.config["links"].items():
      type = "hub" if link["hub"] else "server"
      user.send_reply(StatsCLine("*@{}".format(link["host"]), name, link["port"], type))

@StatsFeature.hook("server.stats.h")
def send_hubs(app, user):
  try:
    user.check_is_irc_operator()
  except errors.BaseError as e:
    user.send_reply(e)
  else:
    for name, link in app.config["links"].items():
      if not link["hub"]:
        continue
      user.send_reply(StatsHLine("*@{}".format(link["host"]), name))

@StatsFeature.hook("server.stats.l")
def send_links(app, user):
  try:
    user.check_is_irc_operator()
  except errors.BaseError as e:
    user.send_reply(e)
  else:
    # TODO: keep track of bytes sent/received?

    for user in app.users.all():
      if not user.is_local:
        continue
      mask = emitter.emit_hostmask(None, user.username, user.host)
      user.send_reply(StatsLinkInfo("{}[{}]".format(user.username, mask),
          0, 0, 0, 0, 0, datetime.now() - user.creation_time))

    for server in app.network.neighborhood():
      mask = "*@{}".format(app.config["links"][server.name]["host"])
      user.send_reply(StatsLinkInfo("{}[{}]".format(server.name, mask),
          0, 0, 0, 0, 0, 0))

@StatsFeature.hook("server.stats.m")
def send_commands(app, user):
  for feature in app.features.values():
    for command in feature.USER_COMMANDS.values():
      user.send_reply(StatsCommands(command.NAME, 0))

@StatsFeature.hook("server.stats.u")
def send_uptime(app, user):
  user.send_reply(StatsUptime(datetime.now() - app.creation_time))
      <|endoftext|>