<commit_before>import binascii
import json
import logging
import os
from datetime import timedelta
from functools import update_wrapper
from logging.handlers import TimedRotatingFileHandler
from multiprocessing import Process
from subprocess import STDOUT, CalledProcessError, check_output

import git
import pushover
import yaml
from flask import Flask, Response, current_app, jsonify, make_response, request
from utilkit import fileutil

import settings

app = Flask(__name__)
app.debug = settings.DEBUG

logger = logging.getLogger('webhaak')
logger.setLevel(logging.DEBUG)
#fh = logging.handlers.RotatingFileHandler('dcp_search.log', maxBytes=100000000, backupCount=10)
# Log will rotate daily with a max history of LOG_BACKUP_COUNT
fh = TimedRotatingFileHandler(
    settings.LOG_LOCATION,
    when='d',
    interval=1,
    backupCount=settings.LOG_BACKUP_COUNT
)
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
logger.addHandler(fh)

# Load the configuration of the various projects/hooks
with open(settings.PROJECTS_FILE, 'r') as pf:
    projects = fileutil.yaml_ordered_load(pf, yaml.SafeLoader)

def notify_user(result, config):
    """Send a PushOver message if configured, after git and command have run

    result is a dictionary with fields:
      command_result
      status: 'OK' | 'error'
      type: 'commanderror'
      message
    """
    try:
        projectname = config[0]
        triggerconfig = config[1]
        title = ''
        branch = 'master'
        if 'repo_branch' in triggerconfig:
            branch = triggerconfig['repo_branch']
        message = 'repo: {}\nbranch: {}\ncommand: {}'.format(triggerconfig['repo'], branch, triggerconfig['command'])
        if result['status'] == 'OK':
            title = "Hook for {} ran successfully".format(projectname)
        else:
            title = "Hook for {} failed: {}".format(projectname, result['type'])
            message = message + '\n\n{}'.format(result['message'])
        logging.debug(message)
        logging.info('Sending notification...')
        client = pushover.Client(settings.PUSHOVER_USERKEY, api_token=settings.PUSHOVER_APPTOKEN)
        client.send_message(message, title=title)
        logging.info('Notification sent')
    except AttributeError:
        logging.warning('Notification through PushOver failed because of missing configuration')

def gettriggersettings(appkey, triggerkey):
    """Look up the trigger and return the repo and command to be updated and fired"""
    for project in projects:
        if projects[project]['appkey'] == appkey:
            for trigger in projects[project]['triggers']:
                if projects[project]['triggers'][trigger]['triggerkey'] == triggerkey:
                    return (project, projects[project]['triggers'][trigger])
    return None

def get_repo_basename(repo_url):
    """Extract repository basename from its url, as that will be the name of directory it will be cloned into"""
    result = os.path.basename(repo_url)
    filename, file_extension = os.path.splitext(result)
    if file_extension == '.git':
        # Strip the .git from the name, as Git will do the same on non-bare checkouts
        result = filename
    return result

def fetchinfo_to_str(fetchinfo):
    """git.remote.FetchInfo to human readable representation"""
    result = fetchinfo[0].note
    return result

def update_repo(config):
    """Update (pull) the Git repo"""
    projectname = config[0]
    triggerconfig = config[1]

    repo_url = triggerconfig['repo']
    repo_parent = settings.REPOS_CACHE_DIR
    if 'repoparent' in triggerconfig and triggerconfig['repoparent']:
        repo_parent = triggerconfig['repoparent']

    logger.info('[' + projectname + '] Updating ' + repo_url)
    logger.info('[' + projectname + '] Repo parent ' + repo_parent)

    # Ensure cache dir for webhaak exists and is writable
    fileutil.ensure_dir_exists(repo_parent) # throws OSError if repo_parent is not writable

    # TODO: check whether dir exists with different repository
    repo_dir = os.path.join(repo_parent, get_repo_basename(repo_url))
    logger.info('[' + projectname + '] Repo dir ' + repo_dir)
    if os.path.isdir(repo_dir):
        # Repo already exists locally, do a pull
        logger.info('[' + projectname + '] Repo exists, pull')

        apprepo = git.Repo(repo_dir)
        origin = apprepo.remote('origin')
        result = fetchinfo_to_str(origin.fetch())  # assure we actually have data. fetch() returns useful information
        logger.info('Fetch result: %s', result)
        origin.pull()
        logger.info('[' + projectname + '] Done pulling, checkout()')
        #logger.debug(apprepo.git.branch())
        result += ' ' + str(apprepo.git.checkout())
    else:
        # Repo needs to be cloned
        logger.info('[' + projectname + '] Repo does not exist yet, clone')
        apprepo = git.Repo.init(repo_dir)
        origin = apprepo.create_remote('origin', repo_url)
        origin.fetch()                  # assure we actually have data. fetch() returns useful information
        # Setup a local tracking branch of a remote branch
        apprepo.create_head('master', origin.refs.master).set_tracking_branch(origin.refs.master)
        # push and pull behaves similarly to `git push|pull`
        result = origin.pull()
        logger.info('[' + projectname + '] Done pulling, checkout()')
        #logger.debug(apprepo.git.branch())
        result += ' ' + str(apprepo.git.checkout())
    return result

def run_command(config):
    """Run the command(s) defined for this trigger"""
    projectname = config[0]
    triggerconfig = config[1]
    if 'command' not in triggerconfig:
        # No command to execute, return
        logger.info('[%s] No command to execute', projectname)
        return None
    command = triggerconfig['command']
    # Replace some placeholders to be used in executing scripts from one of the repos
    command = command.replace('REPODIR', os.path.join(settings.REPOS_CACHE_DIR, projectname))
    command = command.replace('CACHEDIR', settings.REPOS_CACHE_DIR)
    logger.info('[' + projectname + '] Executing ' + command)

    command_parts = command.split(' ')
    logger.info(str(command_parts))
    command_parameters = ' '.join(command_parts[1:])
    result = check_output(command_parameters, executable=command_parts[0], stderr=STDOUT, shell=True)
    return result

def do_pull_andor_command(config):
    """Asynchronous task, performing the git pulling and the specified scripting inside a Process"""
    result = {'application': config[0]}
    result['trigger'] = config[1]
    if 'repo' in config[1]:
        try:
            result['repo_result'] = update_repo(config)
            logger.info('result repo: %s', str(result['repo_result']))
        except git.GitCommandError as e:
            result = {'status': 'error', 'type': 'giterror', 'message': str(e)}
            logger.error('giterror: %s', str(e))
            notify_user(result, config)
        except (OSError, KeyError) as e:
            result = {'status': 'error', 'type': 'oserror', 'message': str(e)}
            logger.error('oserror: %s', str(e))
            notify_user(result, config)

    try:
        result['command_result'] = run_command(config)
        logger.info('result command: %s', str(result['command_result']))
        result['status'] = 'OK'
    except (OSError, CalledProcessError) as e:
        result['status'] = 'error'
        result['type'] = 'commanderror'
        result['message'] = str(e)
        logger.error('commanderror: %s', str(e))

    notify_user(result, config)

# == API request support functions/mixins ======

def crossdomain(origin=None, methods=None, headers=None,
                max_age=21600, attach_to_all=True,
                automatic_options=True):
    """Decorator to send the correct cross-domain headers
    src: https://blog.skyred.fi/articles/better-crossdomain-snippet-for-flask.html
    """
    if methods is not None:
        methods = ', '.join(sorted(x.upper() for x in methods))
    if headers is not None and not isinstance(headers, str):
        headers = ', '.join(x.upper() for x in headers)
    if not isinstance(origin, str):
        origin = ', '.join(origin)
    if isinstance(max_age, timedelta):
        max_age = max_age.total_seconds()

    def get_methods():
        if methods is not None:
            return methods

        options_resp = current_app.make_default_options_response()
        return options_resp.headers['allow']

    def decorator(f):
        def wrapped_function(*args, **kwargs):
            if automatic_options and request.method == 'OPTIONS':
                resp = current_app.make_default_options_response()
            else:
                resp = make_response(f(*args, **kwargs))
            if not attach_to_all and request.method != 'OPTIONS':
                return resp

            h = resp.headers
            h['Access-Control-Allow-Origin'] = origin
            h['Access-Control-Allow-Methods'] = get_methods()
            h['Access-Control-Max-Age'] = str(max_age)
            h['Access-Control-Allow-Credentials'] = 'true'
            h['Access-Control-Allow-Headers'] = \
                "Origin, X-Requested-With, Content-Type, Accept, Authorization"
            if headers is not None:
                h['Access-Control-Allow-Headers'] = headers
            return resp

        f.provide_automatic_options = False
        return update_wrapper(wrapped_function, f)
    return decorator

class APIException(Exception):
    status_code = 400

    def __init__(self, message, status_code=None, payload=None):
        Exception.__init__(self)
        self.message = message
        if status_code is not None:
            self.status_code = status_code
        self.payload = payload

    def to_dict(self):
        rv = dict(self.payload or ())
        rv['message'] = self.message
        rv['status_code'] = self.status_code
        return rv

class InvalidAPIUsage(APIException):
    status_code = 400

@app.errorhandler(404)
def page_not_found(e):
    return jsonify(error=404, text=str(e)), 404

@app.errorhandler(InvalidAPIUsage)
def handle_invalid_usage(error):
    response = jsonify(error.to_dict())
    response.status_code = error.status_code
    response.mimetype = 'application/json'
    return response

# == Web app endpoints ======

@app.route('/')
def indexpage():
    logger.debug('Root page requested')
    return 'Welcome to Webhaak, see the documentation to how to setup and use webhooks.'

#@app.route('/app/<appkey>', methods=['GET', 'OPTIONS'])
#@crossdomain(origin='*', max_age=settings.MAX_CACHE_AGE)
#def approot(appkey):
#    """
#    List some generic info about the app
#    """
#    for app in projects:
#        print app
#        if projects[app]['appkey'] == appkey:
#            print 'found'
#            return app
#    return appkey

@app.route('/app/<appkey>/<triggerkey>', methods=['GET', 'OPTIONS', 'POST'])
#@crossdomain(origin='*', max_age=settings.MAX_CACHE_AGE)
def apptrigger(appkey, triggerkey):
    """Fire the trigger described by the configuration under `triggerkey`"""
    logger.info('%s on appkey: %s triggerkey: %s', request.method, appkey, triggerkey)
    if request.method == 'POST':
        # Likely some ping was sent, check if so
        if request.headers.get('X-GitHub-Event') == "ping":
            payload = request.get_json()
            logger.info('received GitHub ping for %s hook: %s ', payload['repository']['full_name'], payload['hook']['url'])
            return json.dumps({'msg': 'Hi!'})
        if request.headers.get('X-GitHub-Event') != "push":
            payload = request.get_json()
            logger.info('received wrong event type from GitHub for ' + payload['repository']['full_name'] + ' hook: ' + payload['hook']['url'])
            return json.dumps({'msg': "wrong event type"})
        else:
            payload = request.get_json()
            event_info = 'received push from GitHub for '
            if 'repository' in payload:
                event_info += payload['repository']['full_name']
            if 'pusher' in payload:
                event_info += ' by ' + payload['pusher']['name']
            if 'compare' in payload:
                event_info += ', compare: ' + payload['compare']
            logger.info(payload)
            logger.info(event_info)

    config = gettriggersettings(appkey, triggerkey)
    if config is None:
        #raise InvalidAPIUsage('Incorrect/incomplete parameter(s) provided', status_code=404)
        #raise NotFound('Incorrect app/trigger requested')
        logger.error('appkey/triggerkey combo not found')
        return Response(json.dumps({'status': 'Error'}), status=404, mimetype='application/json')
    p = Process(target=do_pull_andor_command, args=(config,))
    p.start()
    return Response(json.dumps({'status': 'OK'}), status=200, mimetype='application/json')

@app.route('/monitor')
#@app.route('/monitor/')
#@app.route('/monitor/monitor.html')
def monitor():
    """Monitoring ping"""
    result = 'OK'
    return result

# New in Flask 1.0, not released yet
#@app.cli.command()
#def getappkey():
#    """Generate new appkey"""
#    os.urandom(24)

@app.route('/getappkey')
def getappkey():
    """Generate new appkey"""
    return json.dumps({'key': binascii.hexlify(os.urandom(24)).decode('utf-8')})

if __name__ == '__main__':
    if settings.DEBUG == False:
        app.run(port=settings.PORT, debug=settings.DEBUG)
    else:
        app.run(host='0.0.0.0', port=settings.PORT, debug=settings.DEBUG)
<commit_msg>Notify the caller that the command is not done yet, just accepted<commit_after>import binascii
import json
import logging
import os
from datetime import timedelta
from functools import update_wrapper
from logging.handlers import TimedRotatingFileHandler
from multiprocessing import Process
from subprocess import STDOUT, CalledProcessError, check_output

import git
import pushover
import yaml
from flask import Flask, Response, current_app, jsonify, make_response, request
from utilkit import fileutil

import settings

app = Flask(__name__)
app.debug = settings.DEBUG

logger = logging.getLogger('webhaak')
logger.setLevel(logging.DEBUG)
#fh = logging.handlers.RotatingFileHandler('dcp_search.log', maxBytes=100000000, backupCount=10)
# Log will rotate daily with a max history of LOG_BACKUP_COUNT
fh = TimedRotatingFileHandler(
    settings.LOG_LOCATION,
    when='d',
    interval=1,
    backupCount=settings.LOG_BACKUP_COUNT
)
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
logger.addHandler(fh)

# Load the configuration of the various projects/hooks
with open(settings.PROJECTS_FILE, 'r') as pf:
    projects = fileutil.yaml_ordered_load(pf, yaml.SafeLoader)

def notify_user(result, config):
    """Send a PushOver message if configured, after git and command have run

    result is a dictionary with fields:
      command_result
      status: 'OK' | 'error'
      type: 'commanderror'
      message
    """
    try:
        projectname = config[0]
        triggerconfig = config[1]
        title = ''
        branch = 'master'
        if 'repo_branch' in triggerconfig:
            branch = triggerconfig['repo_branch']
        message = 'repo: {}\nbranch: {}\ncommand: {}'.format(triggerconfig['repo'], branch, triggerconfig['command'])
        if result['status'] == 'OK':
            title = "Hook for {} ran successfully".format(projectname)
        else:
            title = "Hook for {} failed: {}".format(projectname, result['type'])
            message = message + '\n\n{}'.format(result['message'])
        logging.debug(message)
        logging.info('Sending notification...')
        client = pushover.Client(settings.PUSHOVER_USERKEY, api_token=settings.PUSHOVER_APPTOKEN)
        client.send_message(message, title=title)
        logging.info('Notification sent')
    except AttributeError:
        logging.warning('Notification through PushOver failed because of missing configuration')

def gettriggersettings(appkey, triggerkey):
    """Look up the trigger and return the repo and command to be updated and fired"""
    for project in projects:
        if projects[project]['appkey'] == appkey:
            for trigger in projects[project]['triggers']:
                if projects[project]['triggers'][trigger]['triggerkey'] == triggerkey:
                    return (project, projects[project]['triggers'][trigger])
    return None

def get_repo_basename(repo_url):
    """Extract repository basename from its url, as that will be the name of directory it will be cloned into"""
    result = os.path.basename(repo_url)
    filename, file_extension = os.path.splitext(result)
    if file_extension == '.git':
        # Strip the .git from the name, as Git will do the same on non-bare checkouts
        result = filename
    return result

def fetchinfo_to_str(fetchinfo):
    """git.remote.FetchInfo to human readable representation"""
    result = fetchinfo[0].note
    return result

def update_repo(config):
    """Update (pull) the Git repo"""
    projectname = config[0]
    triggerconfig = config[1]

    repo_url = triggerconfig['repo']
    repo_parent = settings.REPOS_CACHE_DIR
    if 'repoparent' in triggerconfig and triggerconfig['repoparent']:
        repo_parent = triggerconfig['repoparent']

    logger.info('[' + projectname + '] Updating ' + repo_url)
    logger.info('[' + projectname + '] Repo parent ' + repo_parent)

    # Ensure cache dir for webhaak exists and is writable
    fileutil.ensure_dir_exists(repo_parent) # throws OSError if repo_parent is not writable

    # TODO: check whether dir exists with different repository
    repo_dir = os.path.join(repo_parent, get_repo_basename(repo_url))
    logger.info('[' + projectname + '] Repo dir ' + repo_dir)
    if os.path.isdir(repo_dir):
        # Repo already exists locally, do a pull
        logger.info('[' + projectname + '] Repo exists, pull')

        apprepo = git.Repo(repo_dir)
        origin = apprepo.remote('origin')
        result = fetchinfo_to_str(origin.fetch())  # assure we actually have data. fetch() returns useful information
        logger.info('Fetch result: %s', result)
        origin.pull()
        logger.info('[' + projectname + '] Done pulling, checkout()')
        #logger.debug(apprepo.git.branch())
        result += ' ' + str(apprepo.git.checkout())
    else:
        # Repo needs to be cloned
        logger.info('[' + projectname + '] Repo does not exist yet, clone')
        apprepo = git.Repo.init(repo_dir)
        origin = apprepo.create_remote('origin', repo_url)
        origin.fetch()                  # assure we actually have data. fetch() returns useful information
        # Setup a local tracking branch of a remote branch
        apprepo.create_head('master', origin.refs.master).set_tracking_branch(origin.refs.master)
        # push and pull behaves similarly to `git push|pull`
        result = origin.pull()
        logger.info('[' + projectname + '] Done pulling, checkout()')
        #logger.debug(apprepo.git.branch())
        result += ' ' + str(apprepo.git.checkout())
    return result

def run_command(config):
    """Run the command(s) defined for this trigger"""
    projectname = config[0]
    triggerconfig = config[1]
    if 'command' not in triggerconfig:
        # No command to execute, return
        logger.info('[%s] No command to execute', projectname)
        return None
    command = triggerconfig['command']
    # Replace some placeholders to be used in executing scripts from one of the repos
    command = command.replace('REPODIR', os.path.join(settings.REPOS_CACHE_DIR, projectname))
    command = command.replace('CACHEDIR', settings.REPOS_CACHE_DIR)
    logger.info('[' + projectname + '] Executing ' + command)

    command_parts = command.split(' ')
    logger.info(str(command_parts))
    command_parameters = ' '.join(command_parts[1:])
    result = check_output(command_parameters, executable=command_parts[0], stderr=STDOUT, shell=True)
    return result

def do_pull_andor_command(config):
    """Asynchronous task, performing the git pulling and the specified scripting inside a Process"""
    result = {'application': config[0]}
    result['trigger'] = config[1]
    if 'repo' in config[1]:
        try:
            result['repo_result'] = update_repo(config)
            logger.info('result repo: %s', str(result['repo_result']))
        except git.GitCommandError as e:
            result = {'status': 'error', 'type': 'giterror', 'message': str(e)}
            logger.error('giterror: %s', str(e))
            notify_user(result, config)
        except (OSError, KeyError) as e:
            result = {'status': 'error', 'type': 'oserror', 'message': str(e)}
            logger.error('oserror: %s', str(e))
            notify_user(result, config)

    try:
        result['command_result'] = run_command(config)
        logger.info('result command: %s', str(result['command_result']))
        result['status'] = 'OK'
    except (OSError, CalledProcessError) as e:
        result['status'] = 'error'
        result['type'] = 'commanderror'
        result['message'] = str(e)
        logger.error('commanderror: %s', str(e))

    notify_user(result, config)

# == API request support functions/mixins ======

def crossdomain(origin=None, methods=None, headers=None,
                max_age=21600, attach_to_all=True,
                automatic_options=True):
    """Decorator to send the correct cross-domain headers
    src: https://blog.skyred.fi/articles/better-crossdomain-snippet-for-flask.html
    """
    if methods is not None:
        methods = ', '.join(sorted(x.upper() for x in methods))
    if headers is not None and not isinstance(headers, str):
        headers = ', '.join(x.upper() for x in headers)
    if not isinstance(origin, str):
        origin = ', '.join(origin)
    if isinstance(max_age, timedelta):
        max_age = max_age.total_seconds()

    def get_methods():
        if methods is not None:
            return methods

        options_resp = current_app.make_default_options_response()
        return options_resp.headers['allow']

    def decorator(f):
        def wrapped_function(*args, **kwargs):
            if automatic_options and request.method == 'OPTIONS':
                resp = current_app.make_default_options_response()
            else:
                resp = make_response(f(*args, **kwargs))
            if not attach_to_all and request.method != 'OPTIONS':
                return resp

            h = resp.headers
            h['Access-Control-Allow-Origin'] = origin
            h['Access-Control-Allow-Methods'] = get_methods()
            h['Access-Control-Max-Age'] = str(max_age)
            h['Access-Control-Allow-Credentials'] = 'true'
            h['Access-Control-Allow-Headers'] = \
                "Origin, X-Requested-With, Content-Type, Accept, Authorization"
            if headers is not None:
                h['Access-Control-Allow-Headers'] = headers
            return resp

        f.provide_automatic_options = False
        return update_wrapper(wrapped_function, f)
    return decorator

class APIException(Exception):
    status_code = 400

    def __init__(self, message, status_code=None, payload=None):
        Exception.__init__(self)
        self.message = message
        if status_code is not None:
            self.status_code = status_code
        self.payload = payload

    def to_dict(self):
        rv = dict(self.payload or ())
        rv['message'] = self.message
        rv['status_code'] = self.status_code
        return rv

class InvalidAPIUsage(APIException):
    status_code = 400

@app.errorhandler(404)
def page_not_found(e):
    return jsonify(error=404, text=str(e)), 404

@app.errorhandler(InvalidAPIUsage)
def handle_invalid_usage(error):
    response = jsonify(error.to_dict())
    response.status_code = error.status_code
    response.mimetype = 'application/json'
    return response

# == Web app endpoints ======

@app.route('/')
def indexpage():
    logger.debug('Root page requested')
    return 'Welcome to Webhaak, see the documentation to how to setup and use webhooks.'

#@app.route('/app/<appkey>', methods=['GET', 'OPTIONS'])
#@crossdomain(origin='*', max_age=settings.MAX_CACHE_AGE)
#def approot(appkey):
#    """
#    List some generic info about the app
#    """
#    for app in projects:
#        print app
#        if projects[app]['appkey'] == appkey:
#            print 'found'
#            return app
#    return appkey

@app.route('/app/<appkey>/<triggerkey>', methods=['GET', 'OPTIONS', 'POST'])
#@crossdomain(origin='*', max_age=settings.MAX_CACHE_AGE)
def apptrigger(appkey, triggerkey):
    """Fire the trigger described by the configuration under `triggerkey`"""
    logger.info('%s on appkey: %s triggerkey: %s', request.method, appkey, triggerkey)
    if request.method == 'POST':
        # Likely some ping was sent, check if so
        if request.headers.get('X-GitHub-Event') == "ping":
            payload = request.get_json()
            logger.info('received GitHub ping for %s hook: %s ', payload['repository']['full_name'], payload['hook']['url'])
            return json.dumps({'msg': 'Hi!'})
        if request.headers.get('X-GitHub-Event') != "push":
            payload = request.get_json()
            logger.info('received wrong event type from GitHub for ' + payload['repository']['full_name'] + ' hook: ' + payload['hook']['url'])
            return json.dumps({'msg': "wrong event type"})
        else:
            payload = request.get_json()
            event_info = 'received push from GitHub for '
            if 'repository' in payload:
                event_info += payload['repository']['full_name']
            if 'pusher' in payload:
                event_info += ' by ' + payload['pusher']['name']
            if 'compare' in payload:
                event_info += ', compare: ' + payload['compare']
            logger.info(payload)
            logger.info(event_info)

    config = gettriggersettings(appkey, triggerkey)
    if config is None:
        #raise InvalidAPIUsage('Incorrect/incomplete parameter(s) provided', status_code=404)
        #raise NotFound('Incorrect app/trigger requested')
        logger.error('appkey/triggerkey combo not found')
        return Response(json.dumps({'status': 'Error'}), status=404, mimetype='application/json')
    p = Process(target=do_pull_andor_command, args=(config,))
    p.start()
    return Response(json.dumps({'status': 'OK', 'message': 'Command accepted and will be run in the background'}), status=200, mimetype='application/json')

@app.route('/monitor')
#@app.route('/monitor/')
#@app.route('/monitor/monitor.html')
def monitor():
    """Monitoring ping"""
    result = 'OK'
    return result

# New in Flask 1.0, not released yet
#@app.cli.command()
#def getappkey():
#    """Generate new appkey"""
#    os.urandom(24)

@app.route('/getappkey')
def getappkey():
    """Generate new appkey"""
    return json.dumps({'key': binascii.hexlify(os.urandom(24)).decode('utf-8')})

if __name__ == '__main__':
    if settings.DEBUG == False:
        app.run(port=settings.PORT, debug=settings.DEBUG)
    else:
        app.run(host='0.0.0.0', port=settings.PORT, debug=settings.DEBUG)
<|endoftext|>


<commit_before>"""
This module contains workspace definitions for building and using libedgetpu.
"""

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")

# TF release 2.5.0 as of 05/17/2021.
TENSORFLOW_COMMIT = "a4dfb8d1a71385bd6d122e4f27f86dcebb96712d"
TENSORFLOW_SHA256 = "cb99f136dc5c89143669888a44bfdd134c086e1e2d9e36278c1eb0f03fe62d76"

CORAL_CROSSTOOL_COMMIT = "6bcc2261d9fc60dff386b557428d98917f0af491"
CORAL_CROSSTOOL_SHA256 = "38cb4da13009d07ebc2fed4a9d055b0f914191b344dd2d1ca5803096343958b4"

def libedgetpu_dependencies(
        tensorflow_commit = TENSORFLOW_COMMIT,
        tensorflow_sha256 = TENSORFLOW_SHA256,
        coral_crosstool_commit = CORAL_CROSSTOOL_COMMIT,
        coral_crosstool_sha256 = CORAL_CROSSTOOL_SHA256):
    """Sets up libedgetpu dependencies.

    Args:
      tensorflow_commit: https://github.com/tensorflow/tensorflow commit ID
      tensorflow_sha256: corresponding sha256 of the source archive
      coral_crosstool_commit: https://github.com/google-coral/crosstool commit ID
      coral_crosstool_sha256: corresponding sha256 of the source archive
    """
    maybe(
        http_archive,
        name = "org_tensorflow",
        urls = [
            "https://github.com/tensorflow/tensorflow/archive/" + tensorflow_commit + ".tar.gz",
        ],
        sha256 = tensorflow_sha256,
        strip_prefix = "tensorflow-" + tensorflow_commit,
    )

    maybe(
        http_archive,
        name = "coral_crosstool",
        urls = [
            "https://github.com/google-coral/crosstool/archive/" + coral_crosstool_commit + ".tar.gz",
        ],
        sha256 = coral_crosstool_sha256,
        strip_prefix = "crosstool-" + coral_crosstool_commit,
    )

    maybe(
        libusb_repository,
        name = "libusb",
    )

    # Use bazel to query values defined here, e.g.:
    #     bazel query "@libedgetpu_properties//..." | grep tensorflow_commit | cut -d\# -f2
    _properties_repository(
        name = "libedgetpu_properties",
        properties = {
            "tensorflow_commit": tensorflow_commit,
            "coral_crosstool_commit": coral_crosstool_commit,
        },
    )

def _brew_libusb_path(ctx):
    res = ctx.execute(["/bin/bash", "-c", "command -v brew"])
    if res.return_code != 0:
        return None
    brew = ctx.path(res.stdout.strip())

    res = ctx.execute([brew, "ls", "--versions", "libusb"])
    if res.return_code != 0:
        return None

    res = ctx.execute([brew, "--prefix", "libusb"])
    if res.return_code != 0:
        return None

    return res.stdout.strip()

def _port_libusb_path(ctx):
    res = ctx.execute(["/bin/bash", "-c", "command -v port"])
    if res.return_code != 0:
        return None
    port = ctx.path(res.stdout.strip())

    res = ctx.execute([port, "info", "libusb"])
    if res.return_code != 0:
        return None

    return port.dirname.dirname

def _libusb_impl(ctx):
    lower_name = ctx.os.name.lower()
    if lower_name.startswith("linux"):
        path = "/usr/include"
        build_file_content = """
cc_library(
  name = "headers",
  includes = ["root"],
  hdrs = ["root/libusb-1.0/libusb.h"],
  linkopts = ["-l:libusb-1.0.so"],
  visibility = ["//visibility:public"],
)
"""
    elif lower_name.startswith("windows"):
        path = str(ctx.path(Label("@//:WORKSPACE"))) + "/../../libusb"
        build_file_content = """
cc_library(
  name = "headers",
  includes = ["root/include"],
  hdrs = ["root/include/libusb-1.0/libusb.h"],
  visibility = ["//visibility:public"],
)
cc_import(
  name = "shared",
  interface_library = "root/VS2019/MS64/dll/libusb-1.0.lib",
  shared_library = "root/VS2019/MS64/dll/libusb-1.0.dll",
  visibility = ["//visibility:public"],
)
"""
    elif lower_name.startswith("mac os x"):
        path = _brew_libusb_path(ctx)
        if not path:
            path = _port_libusb_path(ctx)
            if not path:
                fail("Install libusb using MacPorts or Homebrew.")

        build_file_content = """
cc_library(
  name = "headers",
  includes = ["root/include"],
  hdrs = ["root/include/libusb-1.0/libusb.h"],
  srcs = ["root/lib/libusb-1.0.dylib"],
  visibility = ["//visibility:public"],
)
"""
    else:
        fail("Unsupported operating system.")

    ctx.symlink(path, "root")
    ctx.file(
        "BUILD",
        content = build_file_content,
        executable = False,
    )

libusb_repository = repository_rule(
    implementation = _libusb_impl,
)

def _properties_impl(ctx):
    content = ""
    for name, value in ctx.attr.properties.items():
        content += "filegroup(name='" + name + "#" + value + "')\n"
    ctx.file("BUILD", content = content, executable = False)

_properties_repository = repository_rule(
    implementation = _properties_impl,
    attrs = {
        "properties": attr.string_dict(),
    },
)
<commit_msg>Avoid aliasing /usr/include in Bazel<commit_after>"""
This module contains workspace definitions for building and using libedgetpu.
"""

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")

# TF release 2.5.0 as of 05/17/2021.
TENSORFLOW_COMMIT = "a4dfb8d1a71385bd6d122e4f27f86dcebb96712d"
TENSORFLOW_SHA256 = "cb99f136dc5c89143669888a44bfdd134c086e1e2d9e36278c1eb0f03fe62d76"

CORAL_CROSSTOOL_COMMIT = "6bcc2261d9fc60dff386b557428d98917f0af491"
CORAL_CROSSTOOL_SHA256 = "38cb4da13009d07ebc2fed4a9d055b0f914191b344dd2d1ca5803096343958b4"

def libedgetpu_dependencies(
        tensorflow_commit = TENSORFLOW_COMMIT,
        tensorflow_sha256 = TENSORFLOW_SHA256,
        coral_crosstool_commit = CORAL_CROSSTOOL_COMMIT,
        coral_crosstool_sha256 = CORAL_CROSSTOOL_SHA256):
    """Sets up libedgetpu dependencies.

    Args:
      tensorflow_commit: https://github.com/tensorflow/tensorflow commit ID
      tensorflow_sha256: corresponding sha256 of the source archive
      coral_crosstool_commit: https://github.com/google-coral/crosstool commit ID
      coral_crosstool_sha256: corresponding sha256 of the source archive
    """
    maybe(
        http_archive,
        name = "org_tensorflow",
        urls = [
            "https://github.com/tensorflow/tensorflow/archive/" + tensorflow_commit + ".tar.gz",
        ],
        sha256 = tensorflow_sha256,
        strip_prefix = "tensorflow-" + tensorflow_commit,
    )

    maybe(
        http_archive,
        name = "coral_crosstool",
        urls = [
            "https://github.com/google-coral/crosstool/archive/" + coral_crosstool_commit + ".tar.gz",
        ],
        sha256 = coral_crosstool_sha256,
        strip_prefix = "crosstool-" + coral_crosstool_commit,
    )

    maybe(
        libusb_repository,
        name = "libusb",
    )

    # Use bazel to query values defined here, e.g.:
    #     bazel query "@libedgetpu_properties//..." | grep tensorflow_commit | cut -d\# -f2
    _properties_repository(
        name = "libedgetpu_properties",
        properties = {
            "tensorflow_commit": tensorflow_commit,
            "coral_crosstool_commit": coral_crosstool_commit,
        },
    )

def _brew_libusb_path(ctx):
    res = ctx.execute(["/bin/bash", "-c", "command -v brew"])
    if res.return_code != 0:
        return None
    brew = ctx.path(res.stdout.strip())

    res = ctx.execute([brew, "ls", "--versions", "libusb"])
    if res.return_code != 0:
        return None

    res = ctx.execute([brew, "--prefix", "libusb"])
    if res.return_code != 0:
        return None

    return res.stdout.strip()

def _port_libusb_path(ctx):
    res = ctx.execute(["/bin/bash", "-c", "command -v port"])
    if res.return_code != 0:
        return None
    port = ctx.path(res.stdout.strip())

    res = ctx.execute([port, "info", "libusb"])
    if res.return_code != 0:
        return None

    return port.dirname.dirname

def _libusb_impl(ctx):
    lower_name = ctx.os.name.lower()
    if lower_name.startswith("linux"):
        path = None
        build_file_content = """
cc_library(
  name = "headers",
  linkopts = ["-l:libusb-1.0.so"],
  visibility = ["//visibility:public"],
)
"""
    elif lower_name.startswith("windows"):
        path = str(ctx.path(Label("@//:WORKSPACE"))) + "/../../libusb"
        build_file_content = """
cc_library(
  name = "headers",
  includes = ["root/include"],
  hdrs = ["root/include/libusb-1.0/libusb.h"],
  visibility = ["//visibility:public"],
)
cc_import(
  name = "shared",
  interface_library = "root/VS2019/MS64/dll/libusb-1.0.lib",
  shared_library = "root/VS2019/MS64/dll/libusb-1.0.dll",
  visibility = ["//visibility:public"],
)
"""
    elif lower_name.startswith("mac os x"):
        path = _brew_libusb_path(ctx)
        if not path:
            path = _port_libusb_path(ctx)
            if not path:
                fail("Install libusb using MacPorts or Homebrew.")

        build_file_content = """
cc_library(
  name = "headers",
  includes = ["root/include"],
  hdrs = ["root/include/libusb-1.0/libusb.h"],
  srcs = ["root/lib/libusb-1.0.dylib"],
  visibility = ["//visibility:public"],
)
"""
    else:
        fail("Unsupported operating system.")

    if path:
        ctx.symlink(path, "root")
    ctx.file(
        "BUILD",
        content = build_file_content,
        executable = False,
    )

libusb_repository = repository_rule(
    implementation = _libusb_impl,
)

def _properties_impl(ctx):
    content = ""
    for name, value in ctx.attr.properties.items():
        content += "filegroup(name='" + name + "#" + value + "')\n"
    ctx.file("BUILD", content = content, executable = False)

_properties_repository = repository_rule(
    implementation = _properties_impl,
    attrs = {
        "properties": attr.string_dict(),
    },
)
<|endoftext|>


<commit_before>from __future__ import print_function

from metadatastore.api import (insert_run_start, insert_beamline_config,
                               insert_event, insert_event_descriptor,
                               insert_run_stop)
from metadatastore.api import find_last, find_event, fetch_events
import time
import numpy as np

b_config = insert_beamline_config(config_params={'my_beamline': 'my_value'},
                                  time=time.time())

data_keys = {'linear_motor': {'source': 'PV:pv1',
                              'shape': None,
                              'dtype': 'number'},
             'scalar_detector': {'source': 'PV:pv2',
                                 'shape': None,
                                 'dtype': 'number'},
             'Tsam': {'source': 'PV:pv3',
                      'dtype': 'number',
                      'shape': None}
             }

try:
    last_hdr = find_last()[0]
    scan_id = int(last_hdr.scan_id)+1
except (IndexError, TypeError):
    scan_id = 1

# cast to string
scan_id = str(scan_id)

custom = {'plotx': 'linear_motor', 'ploty': 'scalar_detector'}
# Create a BeginRunEvent that serves as entry point for a run
rs = insert_run_start(scan_id=scan_id, beamline_id='csx', time=time.time(),
                       beamline_config=b_config, custom=custom)

# Create an EventDescriptor that indicates the data
# keys and serves as header for set of Event(s)
e_desc = insert_event_descriptor(data_keys=data_keys, time=time.time(),
                                 run_start=rs)

func = np.cos
num = 1000
start = 0
stop = 10
sleep_time = .1

for idx, i in enumerate(np.linspace(start, stop, num)):
    data = {'linear_motor': [i, time.time()],
            'Tsam': [i + 5, time.time()],
            'scalar_detector': [func(i) + np.random.randn() / 100, time.time()]}
    e = insert_event(event_descriptor=e_desc, seq_num=idx,
                     time=time.time(),
                     data=data)

# re = insert_run_stop(rs, time.time())

    # time.sleep(sleep_time)
last_run = find_last()[0]

try:
    if last_run.id != rs.id:
        print("Either Arman or Eric broke find_last().")
except AttributeError as ae:
    print(ae)
res_2 = find_event(run_start=rs)
if not res_2:
    print("Either Arman or Eric broke find_event().")
else:
    for event in res_2:
        for idx, i in enumerate(np.linspace(start, stop, num)):
            print(event[idx].data, event[idx].seq_num)

if not fetch_events(descriptor=e_desc):
    print("Either Arman or Eric broke find_event().")
<commit_msg>MNT:Updated synthetic_cosine sample code<commit_after>from __future__ import print_function

from metadatastore.api import (insert_run_start, insert_beamline_config,
                               insert_event, insert_event_descriptor,
                               insert_run_stop)
from metadatastore.api import find_last, find_event
import time
import numpy as np

b_config = insert_beamline_config(config_params={'my_beamline': 'my_value'},
                                  time=time.time())

data_keys = {'linear_motor': {'source': 'PV:pv1',
                              'shape': None,
                              'dtype': 'number'},
             'scalar_detector': {'source': 'PV:pv2',
                                 'shape': None,
                                 'dtype': 'number'},
             'Tsam': {'source': 'PV:pv3',
                      'dtype': 'number',
                      'shape': None}
             }

try:
    last_hdr = find_last()[0]
    scan_id = int(last_hdr.scan_id)+1
except (IndexError, TypeError):
    scan_id = 1

# cast to string
scan_id = str(scan_id)

custom = {'plotx': 'linear_motor', 'ploty': 'scalar_detector'}
# Create a BeginRunEvent that serves as entry point for a run
rs = insert_run_start(scan_id=scan_id, beamline_id='csx', time=time.time(),
                      beamline_config=b_config, custom=custom)

# Create an EventDescriptor that indicates the data
# keys and serves as header for set of Event(s)
e_desc = insert_event_descriptor(data_keys=data_keys, time=time.time(),
                                 run_start=rs)
func = np.cos
num = 1000
start = 0
stop = 10
sleep_time = .1
for idx, i in enumerate(np.linspace(start, stop, num)):
    data = {'linear_motor': [i, time.time()],
            'Tsam': [i + 5, time.time()],
            'scalar_detector': [func(i) + np.random.randn() / 100,
                                time.time()]}
    e = insert_event(event_descriptor=e_desc, seq_num=idx,
                     time=time.time(),
                     data=data)
last_run = find_last()[0]
try:
    if str(last_run.id) != str(rs.id):
        print("find_last() is broken")
except AttributeError as ae:
    print(ae)
res_2 = find_event(run_start=rs)
if not res_2:
    print("find_event() is broken")
<|endoftext|>


<commit_before>import asyncio
import json
import logging
import os

import pkgpanda
from ssh.ssh_runner import Node
import ssh.utils

from .utils import REMOTE_TEMP_DIR, CLUSTER_PACKAGES_FILE, get_async_runner, add_post_action, add_pre_action

log = logging.getLogger(__name__)

class ExecuteException(Exception):
    """Raised when execution fails"""

def nodes_count_by_type(config):
    total_agents_count = len(config.get('agent_list', [])) + len(config.get('public_agent_list', []))
    return {
        'total_masters': len(config['master_list']),
        'total_agents': total_agents_count
    }

def get_full_nodes_list(config):
    def add_nodes(nodes, tag):
        return [Node(node, tag) for node in nodes]

    node_role_map = {
        'master_list': 'master',
        'agent_list': 'agent',
        'public_agent_list': 'public_agent'
    }
    full_target_list = []
    for config_field, role in node_role_map.items():
        if config_field in config:
            full_target_list += add_nodes(config[config_field], {'role': role})
    log.debug("full_target_list: {}".format(full_target_list))
    return full_target_list

@asyncio.coroutine
def run_preflight(config, pf_script_path='/genconf/serve/dcos_install.sh', block=False, state_json_dir=None,
                  async_delegate=None, retry=False, options=None):
    '''
    Copies preflight.sh to target hosts and executes the script. Gathers
    stdout, sterr and return codes and logs them to disk via SSH library.
    :param config: Dict, loaded config file from /genconf/config.yaml
    :param pf_script_path: preflight.sh script location on a local host
    :param preflight_remote_path: destination location
    '''
    if not os.path.isfile(pf_script_path):
        log.error("genconf/serve/dcos_install.sh does not exist. Please run --genconf before executing preflight.")
        raise FileNotFoundError('genconf/serve/dcos_install.sh does not exist')
    targets = get_full_nodes_list(config)

    pf = get_async_runner(config, targets, async_delegate=async_delegate)
    chains = []

    preflight_chain = ssh.utils.CommandChain('preflight')
    # In web mode run if no --offline flag used.
    if options.web:
        if options.offline:
            log.debug('Offline mode used. Do not install prerequisites on CentOS7, RHEL7 in web mode')
        else:
            _add_prereqs_script(preflight_chain)

    add_pre_action(preflight_chain, pf.ssh_user)
    preflight_chain.add_copy(pf_script_path, REMOTE_TEMP_DIR, stage='Copying preflight script')

    preflight_chain.add_execute(
        'sudo bash {} --preflight-only master'.format(
            os.path.join(REMOTE_TEMP_DIR, os.path.basename(pf_script_path))).split(),
        stage='Executing preflight check')
    chains.append(preflight_chain)

    # Setup the cleanup chain
    cleanup_chain = ssh.utils.CommandChain('preflight_cleanup')
    add_post_action(cleanup_chain)
    chains.append(cleanup_chain)
    result = yield from pf.run_commands_chain_async(chains, block=block, state_json_dir=state_json_dir,
                                                    delegate_extra_params=nodes_count_by_type(config))
    return result

def _add_copy_dcos_install(chain, local_install_path='/genconf/serve'):
    dcos_install_script = 'dcos_install.sh'
    local_install_path = os.path.join(local_install_path, dcos_install_script)
    remote_install_path = os.path.join(REMOTE_TEMP_DIR, dcos_install_script)
    chain.add_copy(local_install_path, remote_install_path, stage='Copying dcos_install.sh')

def _add_copy_packages(chain, local_pkg_base_path='/genconf/serve'):
    if not os.path.isfile(CLUSTER_PACKAGES_FILE):
        err_msg = '{} not found'.format(CLUSTER_PACKAGES_FILE)
        log.error(err_msg)
        raise ExecuteException(err_msg)

    cluster_packages = pkgpanda.load_json(CLUSTER_PACKAGES_FILE)
    for package, params in cluster_packages.items():
        destination_package_dir = os.path.join(REMOTE_TEMP_DIR, 'packages', package)
        local_pkg_path = os.path.join(local_pkg_base_path, params['filename'])

        chain.add_execute(['mkdir', '-p', destination_package_dir], stage='Creating package directory')
        chain.add_copy(local_pkg_path, destination_package_dir,
                       stage='Copying packages')

def _add_copy_bootstap(chain, local_bs_path):
    remote_bs_path = REMOTE_TEMP_DIR + '/bootstrap'
    chain.add_execute(['mkdir', '-p', remote_bs_path], stage='Creating directory')
    chain.add_copy(local_bs_path, remote_bs_path,
                   stage='Copying bootstrap')

def _get_bootstrap_tarball(tarball_base_dir='/genconf/serve/bootstrap'):
    '''
    Get a bootstrap tarball from a local filesystem
    :return: String, location of a tarball
    '''
    if 'BOOTSTRAP_ID' not in os.environ:
        err_msg = 'BOOTSTRAP_ID must be set'
        log.error(err_msg)
        raise ExecuteException(err_msg)

    tarball = os.path.join(tarball_base_dir, '{}.bootstrap.tar.xz'.format(os.environ['BOOTSTRAP_ID']))
    if not os.path.isfile(tarball):
        log.error('Ensure environment variable BOOTSTRAP_ID is set correctly')
        log.error('Ensure that the bootstrap tarball exists in '
                  '/genconf/serve/bootstrap/[BOOTSTRAP_ID].bootstrap.tar.xz')
        log.error('You must run genconf.py before attempting Deploy.')
        raise ExecuteException('bootstrap tarball not found /genconf/serve/bootstrap')
    return tarball

def _read_state_file(state_file):
    if not os.path.isfile(state_file):
        return {}

    with open(state_file) as fh:
        return json.load(fh)

def _remove_host(state_file, host):

    json_state = _read_state_file(state_file)

    if 'hosts' not in json_state or host not in json_state['hosts']:
        return False

    log.debug('removing host {} from {}'.format(host, state_file))
    try:
        del json_state['hosts'][host]
    except KeyError:
        return False

    with open(state_file, 'w') as fh:
        json.dump(json_state, fh)

    return True

@asyncio.coroutine
def install_dcos(config, block=False, state_json_dir=None, hosts=None, async_delegate=None, try_remove_stale_dcos=False,
                 **kwargs):
    if hosts is None:
        hosts = []
    assert isinstance(hosts, list)

    # Role specific parameters
    role_params = {
        'master': {
            'tags': {'role': 'master', 'dcos_install_param': 'master'},
            'hosts': config['master_list']
        },
        'agent': {
            'tags': {'role': 'agent', 'dcos_install_param': 'slave'},
            'hosts': config.get('agent_list', [])
        },
        'public_agent': {
            'tags': {'role': 'public_agent', 'dcos_install_param': 'slave_public'},
            'hosts': config.get('public_agent_list', [])
        }
    }

    bootstrap_tarball = _get_bootstrap_tarball()
    log.debug("Local bootstrap found: %s", bootstrap_tarball)

    targets = []
    if hosts:
        targets = hosts
    else:
        for role, params in role_params.items():
            targets += [Node(node, params['tags']) for node in params['hosts']]

    runner = get_async_runner(config, targets, async_delegate=async_delegate)
    chains = []
    if try_remove_stale_dcos:
        pkgpanda_uninstall_chain = ssh.utils.CommandChain('remove_stale_dcos')
        pkgpanda_uninstall_chain.add_execute(['sudo', '-i', '/opt/mesosphere/bin/pkgpanda', 'uninstall'],
                                             stage='Trying pkgpanda uninstall')
        chains.append(pkgpanda_uninstall_chain)

        remove_dcos_chain = ssh.utils.CommandChain('remove_stale_dcos')
        remove_dcos_chain.add_execute(['rm', '-rf', '/opt/mesosphere', '/etc/mesosphere'],
                                      stage="Removing DC/OS files")
        chains.append(remove_dcos_chain)

    chain = ssh.utils.CommandChain('deploy')
    chains.append(chain)

    add_pre_action(chain, runner.ssh_user)
    _add_copy_dcos_install(chain)
    _add_copy_packages(chain)
    _add_copy_bootstap(chain, bootstrap_tarball)

    chain.add_execute(
        lambda node: (
            'sudo bash {}/dcos_install.sh {}'.format(REMOTE_TEMP_DIR, node.tags['dcos_install_param'])).split(),
        stage=lambda node: 'Installing DC/OS'
    )

    # UI expects total_masters, total_agents to be top level keys in deploy.json
    delegate_extra_params = nodes_count_by_type(config)
    if kwargs.get('retry') and state_json_dir:
        state_file_path = os.path.join(state_json_dir, 'deploy.json')
        log.debug('retry executed for a state file deploy.json')
        for _host in hosts:
            _remove_host(state_file_path, '{}:{}'.format(_host.ip, _host.port))

        # We also need to update total number of hosts
        json_state = _read_state_file(state_file_path)
        delegate_extra_params['total_hosts'] = json_state['total_hosts']

    # Setup the cleanup chain
    cleanup_chain = ssh.utils.CommandChain('deploy_cleanup')
    add_post_action(cleanup_chain)
    chains.append(cleanup_chain)

    result = yield from runner.run_commands_chain_async(chains, block=block, state_json_dir=state_json_dir,
                                                        delegate_extra_params=delegate_extra_params)
    return result

@asyncio.coroutine
def run_postflight(config, dcos_diag=None, block=False, state_json_dir=None, async_delegate=None, retry=False,
                   options=None):
    targets = get_full_nodes_list(config)
    pf = get_async_runner(config, targets, async_delegate=async_delegate)
    postflight_chain = ssh.utils.CommandChain('postflight')
    add_pre_action(postflight_chain, pf.ssh_user)

    if dcos_diag is None:
        dcos_diag = """
#!/usr/bin/env bash
# Run the DC/OS diagnostic script for up to 15 minutes (900 seconds) to ensure
# we do not return ERROR on a cluster that hasn't fully achieved quorum.
T=900
until OUT=$(/opt/mesosphere/bin/./3dt -diag) || [[ T -eq 0 ]]; do
    sleep 1
    let T=T-1
done
RETCODE=$?
for value in $OUT; do
    echo $value
done
exit $RETCODE"""

    postflight_chain.add_execute([dcos_diag], stage='Executing post-flight check')
    add_post_action(postflight_chain)

    # Setup the cleanup chain
    cleanup_chain = ssh.utils.CommandChain('postflight_cleanup')
    add_post_action(cleanup_chain)
    result = yield from pf.run_commands_chain_async([postflight_chain, cleanup_chain], block=block,
                                                    state_json_dir=state_json_dir,
                                                    delegate_extra_params=nodes_count_by_type(config))
    return result

@asyncio.coroutine
def uninstall_dcos(config, block=False, state_json_dir=None, async_delegate=None, options=None):
    targets = get_full_nodes_list(config)

    # clean the file to all targets
    runner = get_async_runner(config, targets, async_delegate=async_delegate)
    uninstall_chain = ssh.utils.CommandChain('uninstall')

    uninstall_chain.add_execute([
        'sudo',
        '-i',
        '/opt/mesosphere/bin/pkgpanda',
        'uninstall',
        '&&',
        'sudo',
        'rm',
        '-rf',
        '/opt/mesosphere/'], stage='Uninstalling DC/OS')
    result = yield from runner.run_commands_chain_async([uninstall_chain], block=block, state_json_dir=state_json_dir)

    return result

def _add_prereqs_script(chain):
    inline_script = """
#/bin/sh
# setenforce is in this path
PATH=$PATH:/sbin

dist=$(cat /etc/*-release | sed -n 's@^ID="\(.*\)"$@\\1@p')

if ([ x$dist != 'xrhel' ] && [ x$dist != 'xcentos' ]); then
  echo "$dist is not supported. Only RHEL and CentOS are supported" >&2
  exit 0
fi

version=$(cat /etc/*-release | sed -n 's@^VERSION_ID="\(.*\)"$@\\1@p')
if [ $version -lt 7 ]; then
  echo "$version is not supported. Only >= 7 version is supported" >&2
  exit 0
fi

sudo setenforce 0 && \
sudo sed -i 's/^SELINUX=.*/SELINUX=disabled/g' /etc/sysconfig/selinux

sudo tee /etc/yum.repos.d/docker.repo <<-'EOF'
[dockerrepo]
name=Docker Repository
baseurl=https://yum.dockerproject.org/repo/main/centos/7
enabled=1
gpgcheck=1
gpgkey=https://yum.dockerproject.org/gpg
EOF

sudo yum -y update

sudo mkdir -p /etc/systemd/system/docker.service.d
sudo tee /etc/systemd/system/docker.service.d/override.conf <<- EOF
[Service]
ExecStart=
ExecStart=/usr/bin/docker daemon --storage-driver=overlay -H fd://
EOF

sudo yum install -y docker-engine
sudo systemctl start docker
sudo systemctl enable docker

sudo yum install -y wget
sudo yum install -y git
sudo yum install -y unzip
sudo yum install -y curl
sudo yum install -y xz
sudo yum install -y ipset

sudo getent group nogroup || sudo groupadd nogroup
"""
    # Run a first command to get json file generated.
    chain.add_execute(['echo', 'INSTALL', 'PREREQUISITES'], stage="Prepare install prerequisites")
    chain.add_execute([inline_script], stage='Installing preflight prerequisites')

@asyncio.coroutine
def install_prereqs(config, block=False, state_json_dir=None, async_delegate=None, options=None):
    targets = get_full_nodes_list(config)
    runner = get_async_runner(config, targets, async_delegate=async_delegate)
    prereqs_chain = ssh.utils.CommandChain('install_prereqs')
    _add_prereqs_script(prereqs_chain)
    result = yield from runner.run_commands_chain_async([prereqs_chain], block=block, state_json_dir=state_json_dir)
    return result
<commit_msg>run 3dt -diag as root<commit_after>import asyncio
import json
import logging
import os

import pkgpanda
from ssh.ssh_runner import Node
import ssh.utils

from .utils import REMOTE_TEMP_DIR, CLUSTER_PACKAGES_FILE, get_async_runner, add_post_action, add_pre_action

log = logging.getLogger(__name__)

class ExecuteException(Exception):
    """Raised when execution fails"""

def nodes_count_by_type(config):
    total_agents_count = len(config.get('agent_list', [])) + len(config.get('public_agent_list', []))
    return {
        'total_masters': len(config['master_list']),
        'total_agents': total_agents_count
    }

def get_full_nodes_list(config):
    def add_nodes(nodes, tag):
        return [Node(node, tag) for node in nodes]

    node_role_map = {
        'master_list': 'master',
        'agent_list': 'agent',
        'public_agent_list': 'public_agent'
    }
    full_target_list = []
    for config_field, role in node_role_map.items():
        if config_field in config:
            full_target_list += add_nodes(config[config_field], {'role': role})
    log.debug("full_target_list: {}".format(full_target_list))
    return full_target_list

@asyncio.coroutine
def run_preflight(config, pf_script_path='/genconf/serve/dcos_install.sh', block=False, state_json_dir=None,
                  async_delegate=None, retry=False, options=None):
    '''
    Copies preflight.sh to target hosts and executes the script. Gathers
    stdout, sterr and return codes and logs them to disk via SSH library.
    :param config: Dict, loaded config file from /genconf/config.yaml
    :param pf_script_path: preflight.sh script location on a local host
    :param preflight_remote_path: destination location
    '''
    if not os.path.isfile(pf_script_path):
        log.error("genconf/serve/dcos_install.sh does not exist. Please run --genconf before executing preflight.")
        raise FileNotFoundError('genconf/serve/dcos_install.sh does not exist')
    targets = get_full_nodes_list(config)

    pf = get_async_runner(config, targets, async_delegate=async_delegate)
    chains = []

    preflight_chain = ssh.utils.CommandChain('preflight')
    # In web mode run if no --offline flag used.
    if options.web:
        if options.offline:
            log.debug('Offline mode used. Do not install prerequisites on CentOS7, RHEL7 in web mode')
        else:
            _add_prereqs_script(preflight_chain)

    add_pre_action(preflight_chain, pf.ssh_user)
    preflight_chain.add_copy(pf_script_path, REMOTE_TEMP_DIR, stage='Copying preflight script')

    preflight_chain.add_execute(
        'sudo bash {} --preflight-only master'.format(
            os.path.join(REMOTE_TEMP_DIR, os.path.basename(pf_script_path))).split(),
        stage='Executing preflight check')
    chains.append(preflight_chain)

    # Setup the cleanup chain
    cleanup_chain = ssh.utils.CommandChain('preflight_cleanup')
    add_post_action(cleanup_chain)
    chains.append(cleanup_chain)
    result = yield from pf.run_commands_chain_async(chains, block=block, state_json_dir=state_json_dir,
                                                    delegate_extra_params=nodes_count_by_type(config))
    return result

def _add_copy_dcos_install(chain, local_install_path='/genconf/serve'):
    dcos_install_script = 'dcos_install.sh'
    local_install_path = os.path.join(local_install_path, dcos_install_script)
    remote_install_path = os.path.join(REMOTE_TEMP_DIR, dcos_install_script)
    chain.add_copy(local_install_path, remote_install_path, stage='Copying dcos_install.sh')

def _add_copy_packages(chain, local_pkg_base_path='/genconf/serve'):
    if not os.path.isfile(CLUSTER_PACKAGES_FILE):
        err_msg = '{} not found'.format(CLUSTER_PACKAGES_FILE)
        log.error(err_msg)
        raise ExecuteException(err_msg)

    cluster_packages = pkgpanda.load_json(CLUSTER_PACKAGES_FILE)
    for package, params in cluster_packages.items():
        destination_package_dir = os.path.join(REMOTE_TEMP_DIR, 'packages', package)
        local_pkg_path = os.path.join(local_pkg_base_path, params['filename'])

        chain.add_execute(['mkdir', '-p', destination_package_dir], stage='Creating package directory')
        chain.add_copy(local_pkg_path, destination_package_dir,
                       stage='Copying packages')

def _add_copy_bootstap(chain, local_bs_path):
    remote_bs_path = REMOTE_TEMP_DIR + '/bootstrap'
    chain.add_execute(['mkdir', '-p', remote_bs_path], stage='Creating directory')
    chain.add_copy(local_bs_path, remote_bs_path,
                   stage='Copying bootstrap')

def _get_bootstrap_tarball(tarball_base_dir='/genconf/serve/bootstrap'):
    '''
    Get a bootstrap tarball from a local filesystem
    :return: String, location of a tarball
    '''
    if 'BOOTSTRAP_ID' not in os.environ:
        err_msg = 'BOOTSTRAP_ID must be set'
        log.error(err_msg)
        raise ExecuteException(err_msg)

    tarball = os.path.join(tarball_base_dir, '{}.bootstrap.tar.xz'.format(os.environ['BOOTSTRAP_ID']))
    if not os.path.isfile(tarball):
        log.error('Ensure environment variable BOOTSTRAP_ID is set correctly')
        log.error('Ensure that the bootstrap tarball exists in '
                  '/genconf/serve/bootstrap/[BOOTSTRAP_ID].bootstrap.tar.xz')
        log.error('You must run genconf.py before attempting Deploy.')
        raise ExecuteException('bootstrap tarball not found /genconf/serve/bootstrap')
    return tarball

def _read_state_file(state_file):
    if not os.path.isfile(state_file):
        return {}

    with open(state_file) as fh:
        return json.load(fh)

def _remove_host(state_file, host):

    json_state = _read_state_file(state_file)

    if 'hosts' not in json_state or host not in json_state['hosts']:
        return False

    log.debug('removing host {} from {}'.format(host, state_file))
    try:
        del json_state['hosts'][host]
    except KeyError:
        return False

    with open(state_file, 'w') as fh:
        json.dump(json_state, fh)

    return True

@asyncio.coroutine
def install_dcos(config, block=False, state_json_dir=None, hosts=None, async_delegate=None, try_remove_stale_dcos=False,
                 **kwargs):
    if hosts is None:
        hosts = []
    assert isinstance(hosts, list)

    # Role specific parameters
    role_params = {
        'master': {
            'tags': {'role': 'master', 'dcos_install_param': 'master'},
            'hosts': config['master_list']
        },
        'agent': {
            'tags': {'role': 'agent', 'dcos_install_param': 'slave'},
            'hosts': config.get('agent_list', [])
        },
        'public_agent': {
            'tags': {'role': 'public_agent', 'dcos_install_param': 'slave_public'},
            'hosts': config.get('public_agent_list', [])
        }
    }

    bootstrap_tarball = _get_bootstrap_tarball()
    log.debug("Local bootstrap found: %s", bootstrap_tarball)

    targets = []
    if hosts:
        targets = hosts
    else:
        for role, params in role_params.items():
            targets += [Node(node, params['tags']) for node in params['hosts']]

    runner = get_async_runner(config, targets, async_delegate=async_delegate)
    chains = []
    if try_remove_stale_dcos:
        pkgpanda_uninstall_chain = ssh.utils.CommandChain('remove_stale_dcos')
        pkgpanda_uninstall_chain.add_execute(['sudo', '-i', '/opt/mesosphere/bin/pkgpanda', 'uninstall'],
                                             stage='Trying pkgpanda uninstall')
        chains.append(pkgpanda_uninstall_chain)

        remove_dcos_chain = ssh.utils.CommandChain('remove_stale_dcos')
        remove_dcos_chain.add_execute(['rm', '-rf', '/opt/mesosphere', '/etc/mesosphere'],
                                      stage="Removing DC/OS files")
        chains.append(remove_dcos_chain)

    chain = ssh.utils.CommandChain('deploy')
    chains.append(chain)

    add_pre_action(chain, runner.ssh_user)
    _add_copy_dcos_install(chain)
    _add_copy_packages(chain)
    _add_copy_bootstap(chain, bootstrap_tarball)

    chain.add_execute(
        lambda node: (
            'sudo bash {}/dcos_install.sh {}'.format(REMOTE_TEMP_DIR, node.tags['dcos_install_param'])).split(),
        stage=lambda node: 'Installing DC/OS'
    )

    # UI expects total_masters, total_agents to be top level keys in deploy.json
    delegate_extra_params = nodes_count_by_type(config)
    if kwargs.get('retry') and state_json_dir:
        state_file_path = os.path.join(state_json_dir, 'deploy.json')
        log.debug('retry executed for a state file deploy.json')
        for _host in hosts:
            _remove_host(state_file_path, '{}:{}'.format(_host.ip, _host.port))

        # We also need to update total number of hosts
        json_state = _read_state_file(state_file_path)
        delegate_extra_params['total_hosts'] = json_state['total_hosts']

    # Setup the cleanup chain
    cleanup_chain = ssh.utils.CommandChain('deploy_cleanup')
    add_post_action(cleanup_chain)
    chains.append(cleanup_chain)

    result = yield from runner.run_commands_chain_async(chains, block=block, state_json_dir=state_json_dir,
                                                        delegate_extra_params=delegate_extra_params)
    return result

@asyncio.coroutine
def run_postflight(config, dcos_diag=None, block=False, state_json_dir=None, async_delegate=None, retry=False,
                   options=None):
    targets = get_full_nodes_list(config)
    pf = get_async_runner(config, targets, async_delegate=async_delegate)
    postflight_chain = ssh.utils.CommandChain('postflight')
    add_pre_action(postflight_chain, pf.ssh_user)

    if dcos_diag is None:
        dcos_diag = """
#!/usr/bin/env bash
# Run the DC/OS diagnostic script for up to 15 minutes (900 seconds) to ensure
# we do not return ERROR on a cluster that hasn't fully achieved quorum.
T=900
until OUT=$(sudo /opt/mesosphere/bin/./3dt -diag) || [[ T -eq 0 ]]; do
    sleep 1
    let T=T-1
done
RETCODE=$?
for value in $OUT; do
    echo $value
done
exit $RETCODE"""

    postflight_chain.add_execute([dcos_diag], stage='Executing post-flight check')
    add_post_action(postflight_chain)

    # Setup the cleanup chain
    cleanup_chain = ssh.utils.CommandChain('postflight_cleanup')
    add_post_action(cleanup_chain)
    result = yield from pf.run_commands_chain_async([postflight_chain, cleanup_chain], block=block,
                                                    state_json_dir=state_json_dir,
                                                    delegate_extra_params=nodes_count_by_type(config))
    return result

@asyncio.coroutine
def uninstall_dcos(config, block=False, state_json_dir=None, async_delegate=None, options=None):
    targets = get_full_nodes_list(config)

    # clean the file to all targets
    runner = get_async_runner(config, targets, async_delegate=async_delegate)
    uninstall_chain = ssh.utils.CommandChain('uninstall')

    uninstall_chain.add_execute([
        'sudo',
        '-i',
        '/opt/mesosphere/bin/pkgpanda',
        'uninstall',
        '&&',
        'sudo',
        'rm',
        '-rf',
        '/opt/mesosphere/'], stage='Uninstalling DC/OS')
    result = yield from runner.run_commands_chain_async([uninstall_chain], block=block, state_json_dir=state_json_dir)

    return result

def _add_prereqs_script(chain):
    inline_script = """
#/bin/sh
# setenforce is in this path
PATH=$PATH:/sbin

dist=$(cat /etc/*-release | sed -n 's@^ID="\(.*\)"$@\\1@p')

if ([ x$dist != 'xrhel' ] && [ x$dist != 'xcentos' ]); then
  echo "$dist is not supported. Only RHEL and CentOS are supported" >&2
  exit 0
fi

version=$(cat /etc/*-release | sed -n 's@^VERSION_ID="\(.*\)"$@\\1@p')
if [ $version -lt 7 ]; then
  echo "$version is not supported. Only >= 7 version is supported" >&2
  exit 0
fi

sudo setenforce 0 && \
sudo sed -i 's/^SELINUX=.*/SELINUX=disabled/g' /etc/sysconfig/selinux

sudo tee /etc/yum.repos.d/docker.repo <<-'EOF'
[dockerrepo]
name=Docker Repository
baseurl=https://yum.dockerproject.org/repo/main/centos/7
enabled=1
gpgcheck=1
gpgkey=https://yum.dockerproject.org/gpg
EOF

sudo yum -y update

sudo mkdir -p /etc/systemd/system/docker.service.d
sudo tee /etc/systemd/system/docker.service.d/override.conf <<- EOF
[Service]
ExecStart=
ExecStart=/usr/bin/docker daemon --storage-driver=overlay -H fd://
EOF

sudo yum install -y docker-engine
sudo systemctl start docker
sudo systemctl enable docker

sudo yum install -y wget
sudo yum install -y git
sudo yum install -y unzip
sudo yum install -y curl
sudo yum install -y xz
sudo yum install -y ipset

sudo getent group nogroup || sudo groupadd nogroup
"""
    # Run a first command to get json file generated.
    chain.add_execute(['echo', 'INSTALL', 'PREREQUISITES'], stage="Prepare install prerequisites")
    chain.add_execute([inline_script], stage='Installing preflight prerequisites')

@asyncio.coroutine
def install_prereqs(config, block=False, state_json_dir=None, async_delegate=None, options=None):
    targets = get_full_nodes_list(config)
    runner = get_async_runner(config, targets, async_delegate=async_delegate)
    prereqs_chain = ssh.utils.CommandChain('install_prereqs')
    _add_prereqs_script(prereqs_chain)
    result = yield from runner.run_commands_chain_async([prereqs_chain], block=block, state_json_dir=state_json_dir)
    return result
<|endoftext|>


<commit_before>from django.conf.urls.defaults import url, patterns

urlpatterns = patterns("symposion.schedule.views",
    url(r"^$", "schedule_conference", name="schedule_conference"),
    url(r"^edit/$", "schedule_edit", name="schedule_edit"),
    url(r"^list/$", "schedule_list", name="schedule_list"),
    url(r"^presentations.csv$", "schedule_list_csv", name="schedule_list_csv"),
    url(r"^presentation/(\d+)/$", "schedule_presentation_detail", name="schedule_presentation_detail"),
    url(r"^(\w+)/$", "schedule_detail", name="schedule_detail"),
    url(r"^(\w+)/edit/$", "schedule_edit", name="schedule_edit"),
    url(r"^(\w+)/list/$", "schedule_list", name="schedule_list"),
    url(r"^(\w+)/presentations.csv$", "schedule_list_csv", name="schedule_list_csv"),
    url(r"^(\w+)/edit/slot/(\d+)/", "schedule_slot_edit", name="schedule_slot_edit"),
)
<commit_msg>allow hyphens in section names<commit_after>from django.conf.urls.defaults import url, patterns

urlpatterns = patterns("symposion.schedule.views",
    url(r"^$", "schedule_conference", name="schedule_conference"),
    url(r"^edit/$", "schedule_edit", name="schedule_edit"),
    url(r"^list/$", "schedule_list", name="schedule_list"),
    url(r"^presentations.csv$", "schedule_list_csv", name="schedule_list_csv"),
    url(r"^presentation/(\d+)/$", "schedule_presentation_detail", name="schedule_presentation_detail"),
    url(r"^([\w\-]+)/$", "schedule_detail", name="schedule_detail"),
    url(r"^([\w\-]+)/edit/$", "schedule_edit", name="schedule_edit"),
    url(r"^([\w\-]+)/list/$", "schedule_list", name="schedule_list"),
    url(r"^([\w\-]+)/presentations.csv$", "schedule_list_csv", name="schedule_list_csv"),
    url(r"^([\w\-]+)/edit/slot/(\d+)/", "schedule_slot_edit", name="schedule_slot_edit"),
)
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme
# pylint: disable=superfluous-parens, no-member, invalid-name
from __future__ import print_function
import sys, random
sys.path.insert(0, "../../python")
import numpy as np
import mxnet as mx

from lstm import lstm_unroll

from io import BytesIO
from captcha.image import ImageCaptcha
import cv2, random

class SimpleBatch(object):
    def __init__(self, data_names, data, label_names, label):
        self.data = data
        self.label = label
        self.data_names = data_names
        self.label_names = label_names

        self.pad = 0
        self.index = None # TODO: what is index?

    @property
    def provide_data(self):
        return [(n, x.shape) for n, x in zip(self.data_names, self.data)]

    @property
    def provide_label(self):
        return [(n, x.shape) for n, x in zip(self.label_names, self.label)]

def gen_rand():
    buf = ""
    max_len = random.randint(3,4)
    for i in range(max_len):
        buf += str(random.randint(0,9))
    return buf

def get_label(buf):
    ret = np.zeros(4)
    for i in range(len(buf)):
        ret[i] = 1 + int(buf[i])
    if len(buf) == 3:
        ret[3] = 0
    return ret

class OCRIter(mx.io.DataIter):
    def __init__(self, count, batch_size, num_label, init_states):
        super(OCRIter, self).__init__()
        # you can get this font from http://font.ubuntu.com/
        self.captcha = ImageCaptcha(fonts=['./font/Ubuntu-M.ttf'])
        self.batch_size = batch_size
        self.count = count
        self.num_label = num_label
        self.init_states = init_states
        self.init_state_arrays = [mx.nd.zeros(x[1]) for x in init_states]
        self.provide_data = [('data', (batch_size, 2400))] + init_states
        self.provide_label = [('label', (self.batch_size, 4))]

    def __iter__(self):
        print('iter')
        init_state_names = [x[0] for x in self.init_states]
        for k in range(self.count):
            data = []
            label = []
            for i in range(self.batch_size):
                num = gen_rand()
                img = self.captcha.generate(num)
                img = np.fromstring(img.getvalue(), dtype='uint8')
                img = cv2.imdecode(img, cv2.IMREAD_GRAYSCALE)
                img = cv2.resize(img, (80, 30))
                img = img.transpose(1, 0)
                img = img.reshape((80 * 30))
                img = np.multiply(img, 1/255.0)
                data.append(img)
                label.append(get_label(num))

            data_all = [mx.nd.array(data)] + self.init_state_arrays
            label_all = [mx.nd.array(label)]
            data_names = ['data'] + init_state_names
            label_names = ['label']

            data_batch = SimpleBatch(data_names, data_all, label_names, label_all)
            yield data_batch

    def reset(self):
        pass

BATCH_SIZE = 32
SEQ_LENGTH = 80

def ctc_label(p):
    ret = []
    p1 = [0] + p
    for i in range(len(p)):
        c1 = p1[i]
        c2 = p1[i+1]
        if c2 == 0 or c2 == c1:
            continue
        ret.append(c2)
    return ret

def remove_blank(l):
    ret = []
    for i in range(len(l)):
        if l[i] == 0:
            break
        ret.append(l[i])
    return ret

def Accuracy(label, pred):
    global BATCH_SIZE
    global SEQ_LENGTH
    hit = 0.
    total = 0.
    for i in range(BATCH_SIZE):
        l = remove_blank(label[i])
        p = []
        for k in range(SEQ_LENGTH):
            p.append(np.argmax(pred[k * BATCH_SIZE + i]))
        p = ctc_label(p)
        if len(p) == len(l):
            match = True
            for k in range(len(p)):
                if p[k] != int(l[k]):
                    match = False
                    break
            if match:
                hit += 1.0
        total += 1.0
    return hit / total

def LCS(p,l):
    # Dynamic Programming Finding LCS
    if len(p) == 0:
        return 0
    P = np.array(list(p)).reshape((1, len(p)))
    L = np.array(list(l)).reshape((len(l), 1))
    M = np.int32(P == L)
    for i in range(M.shape[0]):
        for j in range(M.shape[1]):
            up = 0 if i == 0 else M[i-1,j]
            left = 0 if j == 0 else M[i,j-1]
            M[i,j] = max(up, left, M[i,j] if (i == 0 or j == 0) else M[i,j] + M[i-1,j-1])
    return M.max()

def Accuracy_LCS(label, pred):
    global BATCH_SIZE
    global SEQ_LENGTH
    hit = 0.
    total = 0.
    for i in range(BATCH_SIZE):
        l = remove_blank(label[i])
        p = []
        for k in range(SEQ_LENGTH):
            p.append(np.argmax(pred[k * BATCH_SIZE + i]))
        p = ctc_label(p)
        hit += LCS(p,l) * 1.0 / len(l)
        total += 1.0
    return hit / total

if __name__ == '__main__':
    num_hidden = 100
    num_lstm_layer = 2

    num_epoch = 10
    learning_rate = 0.001
    momentum = 0.9
    num_label = 4

    contexts = [mx.context.gpu(0)]

    def sym_gen(seq_len):
        return lstm_unroll(num_lstm_layer, seq_len,
                           num_hidden=num_hidden,
                           num_label = num_label)

    init_c = [('l%d_init_c'%l, (BATCH_SIZE, num_hidden)) for l in range(num_lstm_layer)]
    init_h = [('l%d_init_h'%l, (BATCH_SIZE, num_hidden)) for l in range(num_lstm_layer)]
    init_states = init_c + init_h

    data_train = OCRIter(10000, BATCH_SIZE, num_label, init_states)
    data_val = OCRIter(1000, BATCH_SIZE, num_label, init_states)

    symbol = sym_gen(SEQ_LENGTH)

    model = mx.model.FeedForward(ctx=contexts,
                                 symbol=symbol,
                                 num_epoch=num_epoch,
                                 learning_rate=learning_rate,
                                 momentum=momentum,
                                 wd=0.00001,
                                 initializer=mx.init.Xavier(factor_type="in", magnitude=2.34))

    import logging
    head = '%(asctime)-15s %(message)s'
    logging.basicConfig(level=logging.DEBUG, format=head)

    print('begin fit')

    prefix = 'ocr'
    model.fit(X=data_train, eval_data=data_val,
              eval_metric = mx.metric.np(Accuracy),
              # Use the following eval_metric if your num_label >= 10, or varies in a wide range
              # eval_metric = mx.metric.np(Accuracy_LCS),
              batch_end_callback=mx.callback.Speedometer(BATCH_SIZE, 50),
              epoch_end_callback = mx.callback.do_checkpoint(prefix, 1))

    model.save(prefix)
<commit_msg>Fix Error (#7724)<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme
# pylint: disable=superfluous-parens, no-member, invalid-name
from __future__ import print_function
import sys, random
sys.path.insert(0, "../../python")
import numpy as np
import mxnet as mx

from lstm import lstm_unroll

from io import BytesIO
from captcha.image import ImageCaptcha
import cv2, random

class SimpleBatch(object):
    def __init__(self, data_names, data, label_names, label):
        self.data = data
        self.label = label
        self.data_names = data_names
        self.label_names = label_names

        self.pad = 0
        self.index = None # TODO: what is index?

    @property
    def provide_data(self):
        return [(n, x.shape) for n, x in zip(self.data_names, self.data)]

    @property
    def provide_label(self):
        return [(n, x.shape) for n, x in zip(self.label_names, self.label)]

def gen_rand():
    buf = ""
    max_len = random.randint(3,4)
    for i in range(max_len):
        buf += str(random.randint(0,9))
    return buf

def get_label(buf):
    ret = np.zeros(4)
    for i in range(len(buf)):
        ret[i] = 1 + int(buf[i])
    if len(buf) == 3:
        ret[3] = 0
    return ret

class OCRIter(mx.io.DataIter):
    def __init__(self, count, batch_size, num_label, init_states):
        super(OCRIter, self).__init__()
        # you can get this font from http://font.ubuntu.com/
        self.captcha = ImageCaptcha(fonts=['./font/Ubuntu-M.ttf'])
        self.batch_size = batch_size
        self.count = count
        self.num_label = num_label
        self.init_states = init_states
        self.init_state_arrays = [mx.nd.zeros(x[1]) for x in init_states]
        self.provide_data = [('data', (batch_size, 80, 30))] + init_states
        self.provide_label = [('label', (self.batch_size, 4))]

    def __iter__(self):
        print('iter')
        init_state_names = [x[0] for x in self.init_states]
        for k in range(self.count):
            data = []
            label = []
            for i in range(self.batch_size):
                num = gen_rand()
                img = self.captcha.generate(num)
                img = np.fromstring(img.getvalue(), dtype='uint8')
                img = cv2.imdecode(img, cv2.IMREAD_GRAYSCALE)
                img = cv2.resize(img, (80, 30))
                img = img.transpose(1, 0)
                img = img.reshape((80, 30))
                img = np.multiply(img, 1/255.0)
                data.append(img)
                label.append(get_label(num))

            data_all = [mx.nd.array(data)] + self.init_state_arrays
            label_all = [mx.nd.array(label)]
            data_names = ['data'] + init_state_names
            label_names = ['label']

            data_batch = SimpleBatch(data_names, data_all, label_names, label_all)
            yield data_batch

    def reset(self):
        pass

BATCH_SIZE = 32
SEQ_LENGTH = 80

def ctc_label(p):
    ret = []
    p1 = [0] + p
    for i in range(len(p)):
        c1 = p1[i]
        c2 = p1[i+1]
        if c2 == 0 or c2 == c1:
            continue
        ret.append(c2)
    return ret

def remove_blank(l):
    ret = []
    for i in range(len(l)):
        if l[i] == 0:
            break
        ret.append(l[i])
    return ret

def Accuracy(label, pred):
    global BATCH_SIZE
    global SEQ_LENGTH
    hit = 0.
    total = 0.
    for i in range(BATCH_SIZE):
        l = remove_blank(label[i])
        p = []
        for k in range(SEQ_LENGTH):
            p.append(np.argmax(pred[k * BATCH_SIZE + i]))
        p = ctc_label(p)
        if len(p) == len(l):
            match = True
            for k in range(len(p)):
                if p[k] != int(l[k]):
                    match = False
                    break
            if match:
                hit += 1.0
        total += 1.0
    return hit / total

def LCS(p,l):
    # Dynamic Programming Finding LCS
    if len(p) == 0:
        return 0
    P = np.array(list(p)).reshape((1, len(p)))
    L = np.array(list(l)).reshape((len(l), 1))
    M = np.int32(P == L)
    for i in range(M.shape[0]):
        for j in range(M.shape[1]):
            up = 0 if i == 0 else M[i-1,j]
            left = 0 if j == 0 else M[i,j-1]
            M[i,j] = max(up, left, M[i,j] if (i == 0 or j == 0) else M[i,j] + M[i-1,j-1])
    return M.max()

def Accuracy_LCS(label, pred):
    global BATCH_SIZE
    global SEQ_LENGTH
    hit = 0.
    total = 0.
    for i in range(BATCH_SIZE):
        l = remove_blank(label[i])
        p = []
        for k in range(SEQ_LENGTH):
            p.append(np.argmax(pred[k * BATCH_SIZE + i]))
        p = ctc_label(p)
        hit += LCS(p,l) * 1.0 / len(l)
        total += 1.0
    return hit / total

if __name__ == '__main__':
    num_hidden = 100
    num_lstm_layer = 2

    num_epoch = 10
    learning_rate = 0.001
    momentum = 0.9
    num_label = 4

    contexts = [mx.context.gpu(0)]

    def sym_gen(seq_len):
        return lstm_unroll(num_lstm_layer, seq_len,
                           num_hidden=num_hidden,
                           num_label = num_label)

    init_c = [('l%d_init_c'%l, (BATCH_SIZE, num_hidden)) for l in range(num_lstm_layer)]
    init_h = [('l%d_init_h'%l, (BATCH_SIZE, num_hidden)) for l in range(num_lstm_layer)]
    init_states = init_c + init_h

    data_train = OCRIter(10000, BATCH_SIZE, num_label, init_states)
    data_val = OCRIter(1000, BATCH_SIZE, num_label, init_states)

    symbol = sym_gen(SEQ_LENGTH)

    model = mx.model.FeedForward(ctx=contexts,
                                 symbol=symbol,
                                 num_epoch=num_epoch,
                                 learning_rate=learning_rate,
                                 momentum=momentum,
                                 wd=0.00001,
                                 initializer=mx.init.Xavier(factor_type="in", magnitude=2.34))

    import logging
    head = '%(asctime)-15s %(message)s'
    logging.basicConfig(level=logging.DEBUG, format=head)

    print('begin fit')

    prefix = 'ocr'
    model.fit(X=data_train, eval_data=data_val,
              eval_metric = mx.metric.np(Accuracy),
              # Use the following eval_metric if your num_label >= 10, or varies in a wide range
              # eval_metric = mx.metric.np(Accuracy_LCS),
              batch_end_callback=mx.callback.Speedometer(BATCH_SIZE, 50),
              epoch_end_callback = mx.callback.do_checkpoint(prefix, 1))

    model.save(prefix)
<|endoftext|>


<commit_before>"""Tools for constructing domains for expressions. """

from __future__ import print_function, division

from sympy.polys.polyutils import parallel_dict_from_basic
from sympy.polys.polyoptions import build_options
from sympy.polys.polyerrors import GeneratorsNeeded
from sympy.polys.domains import ZZ, QQ, RR, EX
from sympy.assumptions import ask, Q
from sympy.utilities import public
from sympy.core import sympify, Symbol

def _construct_simple(coeffs, opt):
    """Handle simple domains, e.g.: ZZ, QQ, RR and algebraic domains. """
    result, rationals, reals, algebraics = {}, False, False, False

    if opt.extension is True:
        is_algebraic = lambda coeff: ask(Q.algebraic(coeff))
    else:
        is_algebraic = lambda coeff: False

    # XXX: add support for a + b*I coefficients
    for coeff in coeffs:
        if coeff.is_Rational:
            if not coeff.is_Integer:
                rationals = True
        elif coeff.is_Float:
            if not algebraics:
                reals = True
            else:
                # there are both reals and algebraics -> EX
                return False
        elif is_algebraic(coeff):
            if not reals:
                algebraics = True
            else:
                # there are both algebraics and reals -> EX
                return False
        else:
            # this is a composite domain, e.g. ZZ[X], EX
            return None

    if algebraics:
        domain, result = _construct_algebraic(coeffs, opt)
    else:
        if reals:
            domain = RR
        else:
            if opt.field or rationals:
                domain = QQ
            else:
                domain = ZZ

        result = []

        for coeff in coeffs:
            result.append(domain.from_sympy(coeff))

    return domain, result

def _construct_algebraic(coeffs, opt):
    """We know that coefficients are algebraic so construct the extension. """
    from sympy.polys.numberfields import primitive_element

    result, exts = [], set([])

    for coeff in coeffs:
        if coeff.is_Rational:
            coeff = (None, 0, QQ.from_sympy(coeff))
        else:
            a = coeff.as_coeff_add()[0]
            coeff -= a

            b = coeff.as_coeff_mul()[0]
            coeff /= b

            exts.add(coeff)

            a = QQ.from_sympy(a)
            b = QQ.from_sympy(b)

            coeff = (coeff, b, a)

        result.append(coeff)

    exts = list(exts)

    g, span, H = primitive_element(exts, ex=True, polys=True)
    root = sum([ s*ext for s, ext in zip(span, exts) ])

    domain, g = QQ.algebraic_field((g, root)), g.rep.rep

    for i, (coeff, a, b) in enumerate(result):
        if coeff is not None:
            coeff = a*domain.dtype.from_list(H[exts.index(coeff)], g, QQ) + b
        else:
            coeff = domain.dtype.from_list([b], g, QQ)

        result[i] = coeff

    return domain, result

def _construct_composite(coeffs, opt):
    """Handle composite domains, e.g.: ZZ[X], QQ[X], ZZ(X), QQ(X). """
    numers, denoms = [], []

    for coeff in coeffs:
        numer, denom = coeff.as_numer_denom()

        numers.append(numer)
        denoms.append(denom)

    try:
        polys, gens = parallel_dict_from_basic(numers + denoms)  # XXX: sorting
    except GeneratorsNeeded:
        return None

    if opt.composite is None:
        if any(gen.is_number for gen in gens):
            return None # generators are number-like so lets better use EX

        all_symbols = set([])

        for gen in gens:
            symbols = gen.free_symbols

            if all_symbols & symbols:
                return None # there could be algebraic relations between generators
            else:
                all_symbols |= symbols

    n = len(gens)
    k = len(polys)//2

    numers = polys[:k]
    denoms = polys[k:]

    if opt.field:
        fractions = True
    else:
        fractions, zeros = False, (0,)*n

        for denom in denoms:
            if len(denom) > 1 or zeros not in denom:
                fractions = True
                break

    coeffs = set([])

    if not fractions:
        for numer, denom in zip(numers, denoms):
            denom = denom[zeros]

            for monom, coeff in numer.items():
                coeff /= denom
                coeffs.add(coeff)
                numer[monom] = coeff
    else:
        for numer, denom in zip(numers, denoms):
            coeffs.update(list(numer.values()))
            coeffs.update(list(denom.values()))

    rationals, reals = False, False

    for coeff in coeffs:
        if coeff.is_Rational:
            if not coeff.is_Integer:
                rationals = True
        elif coeff.is_Float:
            reals = True
            break

    if reals:
        ground = RR
    elif rationals:
        ground = QQ
    else:
        ground = ZZ

    result = []

    if not fractions:
        domain = ground.poly_ring(*gens)

        for numer in numers:
            for monom, coeff in numer.items():
                numer[monom] = ground.from_sympy(coeff)

            result.append(domain(numer))
    else:
        domain = ground.frac_field(*gens)

        for numer, denom in zip(numers, denoms):
            for monom, coeff in numer.items():
                numer[monom] = ground.from_sympy(coeff)

            for monom, coeff in denom.items():
                denom[monom] = ground.from_sympy(coeff)

            result.append(domain((numer, denom)))

    return domain, result

def _construct_expression(coeffs, opt):
    """The last resort case, i.e. use the expression domain. """
    domain, result = EX, []

    for coeff in coeffs:
        result.append(domain.from_sympy(coeff))

    return domain, result

@public
def construct_domain(obj, **args):
    """Construct a minimal domain for the list of coefficients. """
    opt = build_options(args)

    if hasattr(obj, '__iter__'):
        if isinstance(obj, dict):
            if not obj:
                monoms, coeffs = [], []
            else:
                monoms, coeffs = list(zip(*list(obj.items())))
        else:
            coeffs = obj
    else:
        coeffs = [obj]

    from sympy import Integral, Sum
    for coeff in coeffs:
        if isinstance(coeff, Integral) or isinstance(coeff, Sum):
            co = coeff.doit()
            loc = coeffs.index(coeff)
            coeffs[loc] = co

    coeffs = list(map(sympify, coeffs))
    result = _construct_simple(coeffs, opt)

    if result is not None:
        if result is not False:
            domain, coeffs = result
        else:
            domain, coeffs = _construct_expression(coeffs, opt)
    else:
        if opt.composite is False:
            result = None
        else:
            result = _construct_composite(coeffs, opt)

        if result is not None:
            domain, coeffs = result
        else:
            domain, coeffs = _construct_expression(coeffs, opt)

    if hasattr(obj, '__iter__'):
        if isinstance(obj, dict):
            return domain, dict(list(zip(monoms, coeffs)))
        else:
            return domain, coeffs
    else:
        return domain, coeffs[0]
<commit_msg>reverted back the constructor.py by removing simplification codes<commit_after>"""Tools for constructing domains for expressions. """

from __future__ import print_function, division

from sympy.polys.polyutils import parallel_dict_from_basic
from sympy.polys.polyoptions import build_options
from sympy.polys.polyerrors import GeneratorsNeeded
from sympy.polys.domains import ZZ, QQ, RR, EX
from sympy.assumptions import ask, Q
from sympy.utilities import public
from sympy.core import sympify, Symbol

def _construct_simple(coeffs, opt):
    """Handle simple domains, e.g.: ZZ, QQ, RR and algebraic domains. """
    result, rationals, reals, algebraics = {}, False, False, False

    if opt.extension is True:
        is_algebraic = lambda coeff: ask(Q.algebraic(coeff))
    else:
        is_algebraic = lambda coeff: False

    # XXX: add support for a + b*I coefficients
    for coeff in coeffs:
        if coeff.is_Rational:
            if not coeff.is_Integer:
                rationals = True
        elif coeff.is_Float:
            if not algebraics:
                reals = True
            else:
                # there are both reals and algebraics -> EX
                return False
        elif is_algebraic(coeff):
            if not reals:
                algebraics = True
            else:
                # there are both algebraics and reals -> EX
                return False
        else:
            # this is a composite domain, e.g. ZZ[X], EX
            return None

    if algebraics:
        domain, result = _construct_algebraic(coeffs, opt)
    else:
        if reals:
            domain = RR
        else:
            if opt.field or rationals:
                domain = QQ
            else:
                domain = ZZ

        result = []

        for coeff in coeffs:
            result.append(domain.from_sympy(coeff))

    return domain, result

def _construct_algebraic(coeffs, opt):
    """We know that coefficients are algebraic so construct the extension. """
    from sympy.polys.numberfields import primitive_element

    result, exts = [], set([])

    for coeff in coeffs:
        if coeff.is_Rational:
            coeff = (None, 0, QQ.from_sympy(coeff))
        else:
            a = coeff.as_coeff_add()[0]
            coeff -= a

            b = coeff.as_coeff_mul()[0]
            coeff /= b

            exts.add(coeff)

            a = QQ.from_sympy(a)
            b = QQ.from_sympy(b)

            coeff = (coeff, b, a)

        result.append(coeff)

    exts = list(exts)

    g, span, H = primitive_element(exts, ex=True, polys=True)
    root = sum([ s*ext for s, ext in zip(span, exts) ])

    domain, g = QQ.algebraic_field((g, root)), g.rep.rep

    for i, (coeff, a, b) in enumerate(result):
        if coeff is not None:
            coeff = a*domain.dtype.from_list(H[exts.index(coeff)], g, QQ) + b
        else:
            coeff = domain.dtype.from_list([b], g, QQ)

        result[i] = coeff

    return domain, result

def _construct_composite(coeffs, opt):
    """Handle composite domains, e.g.: ZZ[X], QQ[X], ZZ(X), QQ(X). """
    numers, denoms = [], []

    for coeff in coeffs:
        numer, denom = coeff.as_numer_denom()

        numers.append(numer)
        denoms.append(denom)

    try:
        polys, gens = parallel_dict_from_basic(numers + denoms)  # XXX: sorting
    except GeneratorsNeeded:
        return None

    if opt.composite is None:
        if any(gen.is_number for gen in gens):
            return None # generators are number-like so lets better use EX

        all_symbols = set([])

        for gen in gens:
            symbols = gen.free_symbols

            if all_symbols & symbols:
                return None # there could be algebraic relations between generators
            else:
                all_symbols |= symbols

    n = len(gens)
    k = len(polys)//2

    numers = polys[:k]
    denoms = polys[k:]

    if opt.field:
        fractions = True
    else:
        fractions, zeros = False, (0,)*n

        for denom in denoms:
            if len(denom) > 1 or zeros not in denom:
                fractions = True
                break

    coeffs = set([])

    if not fractions:
        for numer, denom in zip(numers, denoms):
            denom = denom[zeros]

            for monom, coeff in numer.items():
                coeff /= denom
                coeffs.add(coeff)
                numer[monom] = coeff
    else:
        for numer, denom in zip(numers, denoms):
            coeffs.update(list(numer.values()))
            coeffs.update(list(denom.values()))

    rationals, reals = False, False

    for coeff in coeffs:
        if coeff.is_Rational:
            if not coeff.is_Integer:
                rationals = True
        elif coeff.is_Float:
            reals = True
            break

    if reals:
        ground = RR
    elif rationals:
        ground = QQ
    else:
        ground = ZZ

    result = []

    if not fractions:
        domain = ground.poly_ring(*gens)

        for numer in numers:
            for monom, coeff in numer.items():
                numer[monom] = ground.from_sympy(coeff)

            result.append(domain(numer))
    else:
        domain = ground.frac_field(*gens)

        for numer, denom in zip(numers, denoms):
            for monom, coeff in numer.items():
                numer[monom] = ground.from_sympy(coeff)

            for monom, coeff in denom.items():
                denom[monom] = ground.from_sympy(coeff)

            result.append(domain((numer, denom)))

    return domain, result

def _construct_expression(coeffs, opt):
    """The last resort case, i.e. use the expression domain. """
    domain, result = EX, []

    for coeff in coeffs:
        result.append(domain.from_sympy(coeff))

    return domain, result

@public
def construct_domain(obj, **args):
    """Construct a minimal domain for the list of coefficients. """
    opt = build_options(args)

    if hasattr(obj, '__iter__'):
        if isinstance(obj, dict):
            if not obj:
                monoms, coeffs = [], []
            else:
                monoms, coeffs = list(zip(*list(obj.items())))
        else:
            coeffs = obj
    else:
        coeffs = [obj]

    coeffs = list(map(sympify, coeffs))
    result = _construct_simple(coeffs, opt)

    if result is not None:
        if result is not False:
            domain, coeffs = result
        else:
            domain, coeffs = _construct_expression(coeffs, opt)
    else:
        if opt.composite is False:
            result = None
        else:
            result = _construct_composite(coeffs, opt)

        if result is not None:
            domain, coeffs = result
        else:
            domain, coeffs = _construct_expression(coeffs, opt)

    if hasattr(obj, '__iter__'):
        if isinstance(obj, dict):
            return domain, dict(list(zip(monoms, coeffs)))
        else:
            return domain, coeffs
    else:
        return domain, coeffs[0]
<|endoftext|>