<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2015 GRNET S.A.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from flask import (
    request,
    jsonify,
    Response,
    Blueprint,
    copy_current_request_context
)

from functools import wraps
from datetime import datetime
from base64 import b64encode
import ConfigParser
import StringIO
import logging
import threading

from kamaki.clients import cyclades, ClientError
from kamaki.clients.utils import https

import astakosclient

from icaas.models import Build, User, db
from icaas.error import Error
from icaas.utils import destroy_agent
from icaas import settings

https.patch_with_certs(settings.KAMAKI_SSL_LOCATION)

builds = Blueprint('builds', __name__)

AGENT_CONFIG = "/etc/icaas/manifest.cfg"
AGENT_INIT = "/.icaas"

logger = logging.getLogger(__name__)

def _build_to_links(build):
    url = "%s%s" % (settings.ENDPOINT, build.id)
    return [{"href": url, "rel": "self"}]

def _build_to_dict(build):
    d = {"id": build.id,
         "name:": build.name,
         "src": build.src,
         "status": build.status,
         "status_details": build.status_details,
         "image": build.image,
         "log": build.log,
         "created": build.created,
         "updated": build.updated,
         "links": _build_to_links(build)}

    return d

def _create_manifest(src, name, log, image, token, buildid, icaas_token):
    """Create manifest file to be injected to the ICaaS Agent VM"""
    config = ConfigParser.ConfigParser()
    config.add_section("service")
    config.add_section("synnefo")
    config.add_section("image")

    config.set("image", "src", src)
    config.set("image", "name", name)
    config.set("image", "object", image)

    config.set("service", "status", "%s%s" % (settings.ENDPOINT, buildid))
    config.set("service", "token", icaas_token)
    config.set("service", "insecure", settings.INSECURE)

    config.set("synnefo", "url", settings.AUTH_URL)
    config.set("synnefo", "token", token)
    config.set("synnefo", "log", log)
    logger.debug("icaas-agent manifest file: %r" % config._sections)

    manifest = StringIO.StringIO()
    config.write(manifest)
    return manifest.getvalue()

def login_required(f):
    @wraps(f)
    def decorated_function(*args, **kwargs):
        logger.debug('checking X-Auth-Token before function "%s"' % f.__name__)
        if "X-Auth-Token" not in request.headers:
            logger.debug('X-Auth-Token missing')
            raise Error("Token is missing", status=401)
        token = request.headers["X-Auth-Token"]
        astakos = astakosclient.AstakosClient(token, settings.AUTH_URL)

        try:
            astakos = astakos.authenticate()
            logger.debug('X-Auth-Token is valid')
        except astakosclient.errors.Unauthorized:
            logger.debug('X-Auth-Token not valid')
            raise Error("Invalid token", status=401)
        except Exception as e:
            logger.debug("astakosclient raised exception: '%s'" % str(e))
            raise Error("Internal server error", status=500)

        logger.debug('checking if user is present in the database')
        uuid = astakos['access']['user']['id']
        user = User.query.filter_by(uuid=uuid).first()
        if not user:
            user = User(astakos['access']['user']['id'])
            user.token = token
            db.session.add(user)
            db.session.commit()
            logger.debug('added new user %d' % user.id)
        elif user.token != token:
            user.token = token
            db.session.commit()
            logger.debug('update existing user %d' % user.id)
        else:
            logger.debug('user %d found' % user.id)

        return f(user, *args, **kwargs)
    return decorated_function

@builds.route('/icaas/builds/<int:buildid>', methods=['PUT'])
def update(buildid):
    """Update the build status"""
    logger.info("update build %d" % buildid)

    if "X-Icaas-Token" not in request.headers:
        raise Error("Missing ICaaS token", status=401)
    token = request.headers["X-Icaas-Token"]

    build = Build.query.filter_by(id=buildid, token=token,
                                  deleted=False).first()  # noqa
    if build is None:
        raise Error("Build not found", status=404)

    params = request.get_json()
    logger.debug("update build %d with params %s" % (buildid, params))
    if params:
        status = params.get("status", None)
        details = params.get("details", None)
        if not status:
            raise Error("Parameter: 'status' is missing", status=400)
        if status not in ["CREATING", "ERROR", "COMPLETED"]:
            raise Error("Invalid 'status' parameter", status=400)

        build.status = status
        if details:
            build.status_details = details
        db.session.commit()

        # Should we delete the agent VM?
        if status == "COMPLETED" or (status == "ERROR" and not settings.DEBUG):
            # Check create_agent() to see why we are doing this
            buildid = build.id

            @copy_current_request_context
            def destroy_agent_wrapper():
                """Thread that will kill the agent VM"""
                # Check create_agent() to see why we are doing this
                build = Build.query.filter_by(id=buildid).first()
                if build is None:
                    # Normally this cannot happen
                    logger.error('Build with id=%d not found!' % build.id)
                    return
                destroy_agent(build)

            thread = threading.Thread(target=destroy_agent_wrapper)
            thread.daemon = False
            thread.start()
        elif status == 'ERROR':
            logger.warning('not deleting the agent VM on errors in debug mode')

        return Response(status=202)

    raise Error("Parameters 'status' and 'status_details' are missing",
                status=400)

@builds.route('/icaas/builds/<int:buildid>', methods=['GET'])
@login_required
def view(user, buildid):
    """View a specific build entry"""
    logger.info("view build %d by user %s" % (buildid, user.id))

    build = Build.query.filter_by(id=buildid, user=user.id,
                                  deleted=False).first()  # noqa
    if not build:
        raise Error("Build not found", status=404)

    return jsonify({"build": _build_to_dict(build)})

@builds.route('/icaas/builds/<int:buildid>', methods=['DELETE'])
@login_required
def delete(user, buildid):
    """Delete an existing build entry"""
    logger.info("delete buildid %d by user %s" % (buildid, user.id))

    build = Build.query.filter_by(id=buildid, user=user.id,
                                  deleted=False).first()  # noqa
    if not build:
        raise Error("Build not found", status=404)
    build.deleted = True
    db.session.commit()

    buildid = build.id

    @copy_current_request_context
    def destroy_agent_wrapper():
        """Thread that will kill the agent VM"""
        # Check create_agent() to see why we are doing this
        build = Build.query.filter_by(id=buildid).first()
        if build is None:
            # Normally this cannot happen
            logger.error('Build with id=%d not found!' % build.id)
            return
        destroy_agent(build)

    if build.agent_alive:
        thread = threading.Thread(target=destroy_agent_wrapper)
        thread.daemon = False
        thread.start()

    return Response(status=200)

@builds.route('/icaas/builds', methods=['POST'])
@login_required
def create(user):
    """Create a new image with ICaaS"""
    logger.info("create build by user %s" % user.id)

    token = request.headers["X-Auth-Token"]

    params = request.get_json()
    logger.debug("create build with params %s" % params)
    if params:
        params = params.get("build", None)
    else:
        raise Error('Required field "build" is missing')

    if params:
        missing = "Parameter: '%s' is missing from namespace 'build' or empty"
        invalid = "%s parameter's value not in <container>/<path> format"

        # Image Registration Name
        name = params.get("name", None)
        if not name:
            raise Error(missing % 'name', status=400)
        # User provided Image URL
        src = params.get("src", None)
        if not src:
            raise Error(missing % 'src', status=400)

        # Pithos image object
        image = params.get("image", None)
        if not image:
            raise Error(missing % 'image', status=400)
        separator = image.find('/')
        if separator < 1 or separator == len(image) - 1:
            raise Error(invalid % 'image', status=400)
        # Pithos log object
        log = params.get("log", None)
        if not log:
            raise Error(missing % 'log', status=400)
        separator = log.find('/')
        if separator < 1 or separator == len(image) - 1:
            raise Error(invalid % 'log', status=400)
        # Project to assign the agent VM to
        project = params.get("project", None)
        # Networks of the agent VM
        networks = params.get("networks", None)
    else:
        fields = ['name', 'src', 'image', 'log']
        raise Error('Required fields: "%s" are missing from namespace "build"'
                    % '", "'.join(fields), status=400)

    build = Build(user.id, name, src, None, image, log)
    db.session.add(build)
    db.session.commit()
    logger.debug('created build %r' % build.id)

    # Check comment below to see why we are doing this
    buildid = build.id

    @copy_current_request_context
    def create_agent():
        """Create ICaaS agent VM"""

        # This is needed because the original build object is attached to the
        # session of the father thread and any changes made to it from this
        # thread won't be committed to the database using db.session.commit()
        build = Build.query.filter_by(id=buildid,
                                      deleted=False).first()  # noqa
        if build is None:
            # Normally this cannot happen
            logger.error('Build with id=%d not found!' % build.id)
            return

        manifest = _create_manifest(src, name, log, image, token, build.id,
                                    build.token)

        personality = [
            {'contents': b64encode(manifest), 'path': AGENT_CONFIG,
             'owner': 'root', 'group': 'root', 'mode': 0600},
            {'contents': b64encode("empty"), 'path': AGENT_INIT,
             'owner': 'root', 'group': 'root', 'mode': 0600}]

        compute = cyclades.CycladesComputeClient(settings.COMPUTE_URL, token)
        date = datetime.now().strftime('%Y%m%d%H%M%S%f')
        try:
            agent = compute.create_server("icaas-agent-%s-%s" %
                                          (build.id, date),
                                          settings.AGENT_IMAGE_FLAVOR_ID,
                                          settings.AGENT_IMAGE_ID,
                                          project_id=project,
                                          networks=networks,
                                          personality=personality)
        except ClientError as e:
            build.status = 'ERROR'
            build.status_details = "icaas agent creation failed: (%d, %s)" \
                % (e.status, e)
            db.session.commit()
            logger.error("icaas agent creation failed: (%d, %s)"
                         % (e.status, e))
            return
        except Exception as e:
            build.status = 'ERROR'
            build.status_details = 'icaas agent creation failed'
            with current_ap.app_context():
                db.session.commit()
            logger.error("icaas agent creation failed: %s" % e)
            return

        logger.debug("create new icaas agent vm: %s" % agent)
        build.agent = agent['id']
        build.agent_alive = True
        build.status_details = 'started icaas agent creation'
        db.session.commit()

    thread = threading.Thread(target=create_agent)
    thread.daemon = False

    response = jsonify({"build": _build_to_dict(build)})
    response.status_code = 202

    thread.start()
    return response

@builds.route('/icaas/builds', methods=['GET'])
@login_required
def list_builds(user):
    """List the builds owned by a user"""
    logger.info('list_builds by user %s' % user.id)

    builds = Build.query.filter_by(user=user.id, deleted=False).all()  # noqa
    result = [{"links": _build_to_links(i), "id": i.id, "name": i.name} for i
              in builds]

    return jsonify({"builds": result})

# vim: ai ts=4 sts=4 et sw=4 ft=python
<commit_msg>Fix a bug introduced in 847fefaf8bf814519a964d0633<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2015 GRNET S.A.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from flask import (
    request,
    jsonify,
    Response,
    Blueprint,
    copy_current_request_context
)

from functools import wraps
from datetime import datetime
from base64 import b64encode
import ConfigParser
import StringIO
import logging
import threading

from kamaki.clients import cyclades, ClientError
from kamaki.clients.utils import https

import astakosclient

from icaas.models import Build, User, db
from icaas.error import Error
from icaas.utils import destroy_agent
from icaas import settings

https.patch_with_certs(settings.KAMAKI_SSL_LOCATION)

builds = Blueprint('builds', __name__)

AGENT_CONFIG = "/etc/icaas/manifest.cfg"
AGENT_INIT = "/.icaas"

logger = logging.getLogger(__name__)

def _build_to_links(build):
    url = "%s%s" % (settings.ENDPOINT, build.id)
    return [{"href": url, "rel": "self"}]

def _build_to_dict(build):
    d = {"id": build.id,
         "name:": build.name,
         "src": build.src,
         "status": build.status,
         "status_details": build.status_details,
         "image": build.image,
         "log": build.log,
         "created": build.created,
         "updated": build.updated,
         "links": _build_to_links(build)}

    return d

def _create_manifest(src, name, log, image, token, buildid, icaas_token):
    """Create manifest file to be injected to the ICaaS Agent VM"""
    config = ConfigParser.ConfigParser()
    config.add_section("service")
    config.add_section("synnefo")
    config.add_section("image")

    config.set("image", "src", src)
    config.set("image", "name", name)
    config.set("image", "object", image)

    config.set("service", "status", "%s%s" % (settings.ENDPOINT, buildid))
    config.set("service", "token", icaas_token)
    config.set("service", "insecure", settings.INSECURE)

    config.set("synnefo", "url", settings.AUTH_URL)
    config.set("synnefo", "token", token)
    config.set("synnefo", "log", log)
    logger.debug("icaas-agent manifest file: %r" % config._sections)

    manifest = StringIO.StringIO()
    config.write(manifest)
    return manifest.getvalue()

def login_required(f):
    @wraps(f)
    def decorated_function(*args, **kwargs):
        logger.debug('checking X-Auth-Token before function "%s"' % f.__name__)
        if "X-Auth-Token" not in request.headers:
            logger.debug('X-Auth-Token missing')
            raise Error("Token is missing", status=401)
        token = request.headers["X-Auth-Token"]
        astakos = astakosclient.AstakosClient(token, settings.AUTH_URL)

        try:
            astakos = astakos.authenticate()
            logger.debug('X-Auth-Token is valid')
        except astakosclient.errors.Unauthorized:
            logger.debug('X-Auth-Token not valid')
            raise Error("Invalid token", status=401)
        except Exception as e:
            logger.debug("astakosclient raised exception: '%s'" % str(e))
            raise Error("Internal server error", status=500)

        logger.debug('checking if user is present in the database')
        uuid = astakos['access']['user']['id']
        user = User.query.filter_by(uuid=uuid).first()
        if not user:
            user = User(astakos['access']['user']['id'])
            user.token = token
            db.session.add(user)
            db.session.commit()
            logger.debug('added new user %d' % user.id)
        elif user.token != token:
            user.token = token
            db.session.commit()
            logger.debug('update existing user %d' % user.id)
        else:
            logger.debug('user %d found' % user.id)

        return f(user, *args, **kwargs)
    return decorated_function

@builds.route('/icaas/builds/<int:buildid>', methods=['PUT'])
def update(buildid):
    """Update the build status"""
    logger.info("update build %d" % buildid)

    if "X-Icaas-Token" not in request.headers:
        raise Error("Missing ICaaS token", status=401)
    token = request.headers["X-Icaas-Token"]

    build = Build.query.filter_by(id=buildid, token=token,
                                  deleted=False).first()  # noqa
    if build is None:
        raise Error("Build not found", status=404)

    params = request.get_json()
    logger.debug("update build %d with params %s" % (buildid, params))
    if params:
        status = params.get("status", None)
        details = params.get("details", None)
        if not status:
            raise Error("Parameter: 'status' is missing", status=400)
        if status not in ["CREATING", "ERROR", "COMPLETED"]:
            raise Error("Invalid 'status' parameter", status=400)

        build.status = status
        if details:
            build.status_details = details
        db.session.commit()

        # Should we delete the agent VM?
        if status == "COMPLETED" or (status == "ERROR" and not settings.DEBUG):
            # Check create_agent() to see why we are doing this
            buildid = build.id

            @copy_current_request_context
            def destroy_agent_wrapper():
                """Thread that will kill the agent VM"""
                # Check create_agent() to see why we are doing this
                build = Build.query.filter_by(id=buildid).first()
                if build is None:
                    # Normally this cannot happen
                    logger.error('Build with id=%d not found!' % build.id)
                    return
                destroy_agent(build)

            thread = threading.Thread(target=destroy_agent_wrapper)
            thread.daemon = False
            thread.start()
        elif status == 'ERROR':
            logger.warning('not deleting the agent VM on errors in debug mode')

        return Response(status=202)

    raise Error("Parameters 'status' and 'status_details' are missing",
                status=400)

@builds.route('/icaas/builds/<int:buildid>', methods=['GET'])
@login_required
def view(user, buildid):
    """View a specific build entry"""
    logger.info("view build %d by user %s" % (buildid, user.id))

    build = Build.query.filter_by(id=buildid, user=user.id,
                                  deleted=False).first()  # noqa
    if not build:
        raise Error("Build not found", status=404)

    return jsonify({"build": _build_to_dict(build)})

@builds.route('/icaas/builds/<int:buildid>', methods=['DELETE'])
@login_required
def delete(user, buildid):
    """Delete an existing build entry"""
    logger.info("delete buildid %d by user %s" % (buildid, user.id))

    build = Build.query.filter_by(id=buildid, user=user.id,
                                  deleted=False).first()  # noqa
    if not build:
        raise Error("Build not found", status=404)
    build.deleted = True
    db.session.commit()

    buildid = build.id

    @copy_current_request_context
    def destroy_agent_wrapper():
        """Thread that will kill the agent VM"""
        # Check create_agent() to see why we are doing this
        build = Build.query.filter_by(id=buildid).first()
        if build is None:
            # Normally this cannot happen
            logger.error('Build with id=%d not found!' % build.id)
            return
        destroy_agent(build)

    if build.agent_alive:
        thread = threading.Thread(target=destroy_agent_wrapper)
        thread.daemon = False
        thread.start()

    return Response(status=200)

@builds.route('/icaas/builds', methods=['POST'])
@login_required
def create(user):
    """Create a new image with ICaaS"""
    logger.info("create build by user %s" % user.id)

    token = request.headers["X-Auth-Token"]

    params = request.get_json()
    logger.debug("create build with params %s" % params)
    if params:
        params = params.get("build", None)
    else:
        raise Error('Required field "build" is missing')

    if params:
        missing = "Parameter: '%s' is missing from namespace 'build' or empty"
        invalid = "%s parameter's value not in <container>/<path> format"

        # Image Registration Name
        name = params.get("name", None)
        if not name:
            raise Error(missing % 'name', status=400)
        # User provided Image URL
        src = params.get("src", None)
        if not src:
            raise Error(missing % 'src', status=400)

        # Pithos image object
        image = params.get("image", None)
        if not image:
            raise Error(missing % 'image', status=400)
        separator = image.find('/')
        if separator < 1 or separator == len(image) - 1:
            raise Error(invalid % 'image', status=400)
        # Pithos log object
        log = params.get("log", None)
        if not log:
            raise Error(missing % 'log', status=400)
        separator = log.find('/')
        if separator < 1 or separator == len(image) - 1:
            raise Error(invalid % 'log', status=400)
        # Project to assign the agent VM to
        project = params.get("project", None)
        # Networks of the agent VM
        networks = params.get("networks", None)
    else:
        fields = ['name', 'src', 'image', 'log']
        raise Error('Required fields: "%s" are missing from namespace "build"'
                    % '", "'.join(fields), status=400)

    build = Build(user.id, name, src, None, image, log)
    db.session.add(build)
    db.session.commit()
    logger.debug('created build %r' % build.id)

    # Check comment below to see why we are doing this
    buildid = build.id

    @copy_current_request_context
    def create_agent():
        """Create ICaaS agent VM"""

        # This is needed because the original build object is attached to the
        # session of the father thread and any changes made to it from this
        # thread won't be committed to the database using db.session.commit()
        build = Build.query.filter_by(id=buildid,
                                      deleted=False).first()  # noqa
        if build is None:
            # Normally this cannot happen
            logger.error('Build with id=%d not found!' % build.id)
            return

        manifest = _create_manifest(src, name, log, image, token, build.id,
                                    build.token)

        personality = [
            {'contents': b64encode(manifest), 'path': AGENT_CONFIG,
             'owner': 'root', 'group': 'root', 'mode': 0600},
            {'contents': b64encode("empty"), 'path': AGENT_INIT,
             'owner': 'root', 'group': 'root', 'mode': 0600}]

        compute = cyclades.CycladesComputeClient(settings.COMPUTE_URL, token)
        date = datetime.now().strftime('%Y%m%d%H%M%S%f')
        try:
            agent = compute.create_server("icaas-agent-%s-%s" %
                                          (build.id, date),
                                          settings.AGENT_IMAGE_FLAVOR_ID,
                                          settings.AGENT_IMAGE_ID,
                                          project_id=project,
                                          networks=networks,
                                          personality=personality)
        except ClientError as e:
            build.status = 'ERROR'
            build.status_details = "icaas agent creation failed: (%d, %s)" \
                % (e.status, e)
            db.session.commit()
            logger.error("icaas agent creation failed: (%d, %s)"
                         % (e.status, e))
            return
        except Exception as e:
            build.status = 'ERROR'
            build.status_details = 'icaas agent creation failed'
            db.session.commit()
            logger.error("icaas agent creation failed: %s" % e)
            return

        logger.debug("create new icaas agent vm: %s" % agent)
        build.agent = agent['id']
        build.agent_alive = True
        build.status_details = 'started icaas agent creation'
        db.session.commit()

    thread = threading.Thread(target=create_agent)
    thread.daemon = False

    response = jsonify({"build": _build_to_dict(build)})
    response.status_code = 202

    thread.start()
    return response

@builds.route('/icaas/builds', methods=['GET'])
@login_required
def list_builds(user):
    """List the builds owned by a user"""
    logger.info('list_builds by user %s' % user.id)

    builds = Build.query.filter_by(user=user.id, deleted=False).all()  # noqa
    result = [{"links": _build_to_links(i), "id": i.id, "name": i.name} for i
              in builds]

    return jsonify({"builds": result})

# vim: ai ts=4 sts=4 et sw=4 ft=python
<|endoftext|>


<commit_before>import logging
import urllib

import sentry_sdk
from cloudpathlib import AnyPath
from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration

from .config import Config
from .main import get_all_unsubmitted_roots, process_file, setup_logging

config = Config()
setup_logging(config)

sentry_sdk.init(
    dsn=config.reward_root_submitter_sentry_dsn,
    integrations=[
        AwsLambdaIntegration(),
    ],
    environment=config.environment,
    traces_sample_rate=1.0,
)

def handler(event, _context):
    if event.get("source") == "aws.events":
        logging.info("Cron event triggered")
        unsubmitted_roots = get_all_unsubmitted_roots(config)
        for index, row in unsubmitted_roots.iterrows():
            process_file(row["file"], config)
    elif event["Records"][0]["eventSource"] == "aws.s3":
        bucket = event["Records"][0]["s3"]["bucket"]["name"]
        key = urllib.parse.unquote_plus(
            event["Records"][0]["s3"]["object"]["key"], encoding="utf-8"
        )
        process_file(AnyPath(f"s3://{bucket}/{key}"), Config())
<commit_msg>Fix event source for S3 put events<commit_after>import logging
import urllib

import sentry_sdk
from cloudpathlib import AnyPath
from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration

from .config import Config
from .main import get_all_unsubmitted_roots, process_file, setup_logging

config = Config()
setup_logging(config)

sentry_sdk.init(
    dsn=config.reward_root_submitter_sentry_dsn,
    integrations=[
        AwsLambdaIntegration(),
    ],
    environment=config.environment,
    traces_sample_rate=1.0,
)

def handler(event, _context):
    if event.get("source") == "aws.events":
        logging.info("Cron event triggered")
        unsubmitted_roots = get_all_unsubmitted_roots(config)
        for index, row in unsubmitted_roots.iterrows():
            process_file(row["file"], config)
    elif event["Records"][0]["eventSource"] == "aws:s3":
        bucket = event["Records"][0]["s3"]["bucket"]["name"]
        key = urllib.parse.unquote_plus(
            event["Records"][0]["s3"]["object"]["key"], encoding="utf-8"
        )
        process_file(AnyPath(f"s3://{bucket}/{key}"), Config())
<|endoftext|>


<commit_before>"""LibVirt slivers"""

import accounts
import logger
import subprocess
import os
import os.path
import libvirt
import sys
import shutil
import bwlimit
import cgroups
import pprint

from string import Template

STATES = {
    libvirt.VIR_DOMAIN_NOSTATE:  'no state',
    libvirt.VIR_DOMAIN_RUNNING:  'running',
    libvirt.VIR_DOMAIN_BLOCKED:  'blocked on resource',
    libvirt.VIR_DOMAIN_PAUSED:   'paused by user',
    libvirt.VIR_DOMAIN_SHUTDOWN: 'being shut down',
    libvirt.VIR_DOMAIN_SHUTOFF:  'shut off',
    libvirt.VIR_DOMAIN_CRASHED:  'crashed',
}

connections = dict()

# Helper methods

def getConnection(sliver_type):
    # TODO: error checking
    # vtype is of the form sliver.[LXC/QEMU] we need to lower case to lxc/qemu
    vtype = sliver_type.split('.')[1].lower()
    uri = vtype + '://'
    return connections.setdefault(uri, libvirt.open(uri))

def debuginfo(dom):
    ''' Helper method to get a "nice" output of the info struct for debug'''
    [state, maxmem, mem, ncpu, cputime] = dom.info()
    return '%s is %s, maxmem = %s, mem = %s, ncpu = %s, cputime = %s' % (dom.name(), STATES.get(state, state), maxmem, mem, ncpu, cputime)

# Common Libvirt code

class Sliver_Libvirt(accounts.Account):

    def __init__(self, rec):
        self.name = rec['name']
        logger.verbose ('sliver_libvirt: %s init'%(self.name))

        # Assume the directory with the image and config files
        # are in place

        self.keys = ''
        self.rspec = {}
        self.slice_id = rec['slice_id']
        self.enabled = True
        self.conn = getConnection(rec['type'])
        self.xid = bwlimit.get_xid(self.name)

        try:
            self.dom = self.conn.lookupByName(self.name)
        except:
            logger.verbose('sliver_libvirt: Domain %s does not exist ' \
                           'UNEXPECTED: %s'%(self.name, sys.exc_info()[1]))

    def start(self, delay=0):
        ''' Just start the sliver '''
        logger.verbose('sliver_libvirt: %s start'%(self.name))

        # Check if it's running to avoid throwing an exception if the
        # domain was already running, create actually means start
        if not self.is_running():
            self.dom.create()
        else:
            logger.verbose('sliver_libvirt: sliver %s already started'%(self.name))

        # After the VM is started... we can play with the virtual interface
        # Create the ebtables rule to mark the packets going out from the virtual
        # interface to the actual device so the filter canmatch against the mark
        bwlimit.ebtables("-A INPUT -i veth%d -j mark --set-mark %d" % \
            (self.xid, self.xid))

    def stop(self):
        logger.verbose('sliver_libvirt: %s stop'%(self.name))

        # Remove the ebtables rule before stopping
        bwlimit.ebtables("-D INPUT -i veth%d -j mark --set-mark %d" % \
            (self.xid, self.xid))

        try:
            self.dom.destroy()
        except:
            logger.verbose('sliver_libvirt: Domain %s not running ' \
                           'UNEXPECTED: %s'%(self.name, sys.exc_info()[1]))
            print 'sliver_libvirt: Domain %s not running ' \
                  'UNEXPECTED: %s'%(self.name, sys.exc_info()[1])

    def is_running(self):
        ''' Return True if the domain is running '''
        logger.verbose('sliver_libvirt: %s is_running'%self.name)
        try:
            [state, _, _, _, _] = self.dom.info()
            if state == libvirt.VIR_DOMAIN_RUNNING:
                logger.verbose('sliver_libvirt: %s is RUNNING'%self.name)
                return True
            else:
                info = debuginfo(self.dom)
                logger.verbose('sliver_libvirt: %s is ' \
                               'NOT RUNNING...\n%s'%(self.name, info))
                return False
        except:
            logger.verbose('sliver_libvirt: UNEXPECTED ERROR in ' \
                           '%s: %s'%(self.name, sys.exc_info()[1]))
            print 'sliver_libvirt: UNEXPECTED ERROR in ' \
                  '%s: %s'%(self.name, sys.exc_info()[1])
            return False

    def configure(self, rec):

        #sliver.[LXC/QEMU] tolower case
        #sliver_type = rec['type'].split('.')[1].lower()

        #BASE_DIR = '/cgroup/libvirt/%s/%s/'%(sliver_type, self.name)

        # Disk allocation
        # No way through cgroups... figure out how to do that with user/dir quotas.
        # There is no way to do quota per directory. Chown-ing would create
        # problems as username namespaces are not yet implemented (and thus, host
        # and containers share the same name ids

        # Btrfs support quota per volumes

        # It will depend on the FS selection
        if rec.has_key('disk_max'):
            disk_max = rec['disk_max']
            if disk_max == 0:
                # unlimited
                pass
            else:
                # limit to certain number
                pass

        # Memory allocation
        if rec.has_key('memlock_hard'):
            mem = rec['memlock_hard'] * 1024 # hard limit in bytes
            cgroups.write(self.name, 'memory.limit_in_bytes', mem)
        if rec.has_key('memlock_soft'):
            mem = rec['memlock_soft'] * 1024 # soft limit in bytes
            cgroups.write(self.name, 'memory.soft_limit_in_bytes', mem)

        # CPU allocation
        # Only cpu_shares until figure out how to provide limits and guarantees
        # (RT_SCHED?)
        if rec.has_key('cpu_share'):
            cpu_share = rec['cpu_share']
            cgroups.write(self.name, 'cpu.shares', cpu_share)

        # Call the upper configure method (ssh keys...)
        accounts.Account.configure(self, rec)

<commit_msg>increase gravity of error message<commit_after>"""LibVirt slivers"""

import accounts
import logger
import subprocess
import os
import os.path
import libvirt
import sys
import shutil
import bwlimit
import cgroups
import pprint

from string import Template

STATES = {
    libvirt.VIR_DOMAIN_NOSTATE:  'no state',
    libvirt.VIR_DOMAIN_RUNNING:  'running',
    libvirt.VIR_DOMAIN_BLOCKED:  'blocked on resource',
    libvirt.VIR_DOMAIN_PAUSED:   'paused by user',
    libvirt.VIR_DOMAIN_SHUTDOWN: 'being shut down',
    libvirt.VIR_DOMAIN_SHUTOFF:  'shut off',
    libvirt.VIR_DOMAIN_CRASHED:  'crashed',
}

connections = dict()

# Helper methods

def getConnection(sliver_type):
    # TODO: error checking
    # vtype is of the form sliver.[LXC/QEMU] we need to lower case to lxc/qemu
    vtype = sliver_type.split('.')[1].lower()
    uri = vtype + '://'
    return connections.setdefault(uri, libvirt.open(uri))

def debuginfo(dom):
    ''' Helper method to get a "nice" output of the info struct for debug'''
    [state, maxmem, mem, ncpu, cputime] = dom.info()
    return '%s is %s, maxmem = %s, mem = %s, ncpu = %s, cputime = %s' % (dom.name(), STATES.get(state, state), maxmem, mem, ncpu, cputime)

# Common Libvirt code

class Sliver_Libvirt(accounts.Account):

    def __init__(self, rec):
        self.name = rec['name']
        logger.verbose ('sliver_libvirt: %s init'%(self.name))

        # Assume the directory with the image and config files
        # are in place

        self.keys = ''
        self.rspec = {}
        self.slice_id = rec['slice_id']
        self.enabled = True
        self.conn = getConnection(rec['type'])
        self.xid = bwlimit.get_xid(self.name)

        try:
            self.dom = self.conn.lookupByName(self.name)
        except:
            logger.log('sliver_libvirt: Domain %s does not exist ' \
                       'UNEXPECTED: %s'%(self.name, sys.exc_info()[1]))

    def start(self, delay=0):
        ''' Just start the sliver '''
        logger.verbose('sliver_libvirt: %s start'%(self.name))

        # Check if it's running to avoid throwing an exception if the
        # domain was already running, create actually means start
        if not self.is_running():
            self.dom.create()
        else:
            logger.verbose('sliver_libvirt: sliver %s already started'%(self.name))

        # After the VM is started... we can play with the virtual interface
        # Create the ebtables rule to mark the packets going out from the virtual
        # interface to the actual device so the filter canmatch against the mark
        bwlimit.ebtables("-A INPUT -i veth%d -j mark --set-mark %d" % \
            (self.xid, self.xid))

    def stop(self):
        logger.verbose('sliver_libvirt: %s stop'%(self.name))

        # Remove the ebtables rule before stopping
        bwlimit.ebtables("-D INPUT -i veth%d -j mark --set-mark %d" % \
            (self.xid, self.xid))

        try:
            self.dom.destroy()
        except:
            logger.verbose('sliver_libvirt: Domain %s not running ' \
                           'UNEXPECTED: %s'%(self.name, sys.exc_info()[1]))
            print 'sliver_libvirt: Domain %s not running ' \
                  'UNEXPECTED: %s'%(self.name, sys.exc_info()[1])

    def is_running(self):
        ''' Return True if the domain is running '''
        logger.verbose('sliver_libvirt: %s is_running'%self.name)
        try:
            [state, _, _, _, _] = self.dom.info()
            if state == libvirt.VIR_DOMAIN_RUNNING:
                logger.verbose('sliver_libvirt: %s is RUNNING'%self.name)
                return True
            else:
                info = debuginfo(self.dom)
                logger.verbose('sliver_libvirt: %s is ' \
                               'NOT RUNNING...\n%s'%(self.name, info))
                return False
        except:
            logger.verbose('sliver_libvirt: UNEXPECTED ERROR in ' \
                           '%s: %s'%(self.name, sys.exc_info()[1]))
            print 'sliver_libvirt: UNEXPECTED ERROR in ' \
                  '%s: %s'%(self.name, sys.exc_info()[1])
            return False

    def configure(self, rec):

        #sliver.[LXC/QEMU] tolower case
        #sliver_type = rec['type'].split('.')[1].lower()

        #BASE_DIR = '/cgroup/libvirt/%s/%s/'%(sliver_type, self.name)

        # Disk allocation
        # No way through cgroups... figure out how to do that with user/dir quotas.
        # There is no way to do quota per directory. Chown-ing would create
        # problems as username namespaces are not yet implemented (and thus, host
        # and containers share the same name ids

        # Btrfs support quota per volumes

        # It will depend on the FS selection
        if rec.has_key('disk_max'):
            disk_max = rec['disk_max']
            if disk_max == 0:
                # unlimited
                pass
            else:
                # limit to certain number
                pass

        # Memory allocation
        if rec.has_key('memlock_hard'):
            mem = rec['memlock_hard'] * 1024 # hard limit in bytes
            cgroups.write(self.name, 'memory.limit_in_bytes', mem)
        if rec.has_key('memlock_soft'):
            mem = rec['memlock_soft'] * 1024 # soft limit in bytes
            cgroups.write(self.name, 'memory.soft_limit_in_bytes', mem)

        # CPU allocation
        # Only cpu_shares until figure out how to provide limits and guarantees
        # (RT_SCHED?)
        if rec.has_key('cpu_share'):
            cpu_share = rec['cpu_share']
            cgroups.write(self.name, 'cpu.shares', cpu_share)

        # Call the upper configure method (ssh keys...)
        accounts.Account.configure(self, rec)

<|endoftext|>


<commit_before>"""
sinecosine.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:

In this assignment you must use *list comprehensions* to generate sprites that show the behavior
of certain mathematical functions: sine and cosine.

The sine and cosine functions are provided in the Python math library. These functions are used
to relate *angles* to *rectangular* (x,y) coordinate systems and can be very useful in computer
game design.

Unlike the last assignment using ggame`, this one will not provide any "skeleton" code to fill
in. You should use your submission for the Picture assignment
(https://github.com/HHS-IntroProgramming/Picture) as a reference for starting this assignment.

See:
https://github.com/HHS-IntroProgramming/Sine-Cosine/blob/master/README.md
for a detailed list of requirements for this assignment.

https://github.com/HHS-IntroProgramming/Standards-and-Syllabus/wiki/Displaying-Graphics
for general information on how to use ggame.

https://github.com/HHS-IntroProgramming/Standards-and-Syllabus/wiki/Programmed-Graphics
for general information on using list comprehensions to generate graphics.

http://brythonserver.github.io/ggame/
for detailed information on ggame.
"""

from ggame import App, Color, LineStyle, Sprite, CircleAsset
from math import sin, cos, radians

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
purple = Color(0x8904B1, 1.0)

thinline = LineStyle(1, black)
mycircle = CircleAsset(5, thinline, blue)
xcoordinates = range(100, 600, 10)

# Generate a list of sprites that form a line!
sprites = [Sprite(mycircle, (x, x*0.5 + 100)) for x in xcoordinates]

myapp = App()
myapp.run()<commit_msg>Updated from Brython Server: 1/29/2016 2:39:54 PM<commit_after>"""
sinecosine.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:

In this assignment you must use *list comprehensions* to generate sprites that show the behavior
of certain mathematical functions: sine and cosine.

The sine and cosine functions are provided in the Python math library. These functions are used
to relate *angles* to *rectangular* (x,y) coordinate systems and can be very useful in computer
game design.

Unlike the last assignment using ggame`, this one will not provide any "skeleton" code to fill
in. You should use your submission for the Picture assignment
(https://github.com/HHS-IntroProgramming/Picture) as a reference for starting this assignment.

See:
https://github.com/HHS-IntroProgramming/Sine-Cosine/blob/master/README.md
for a detailed list of requirements for this assignment.

https://github.com/HHS-IntroProgramming/Standards-and-Syllabus/wiki/Displaying-Graphics
for general information on how to use ggame.

https://github.com/HHS-IntroProgramming/Standards-and-Syllabus/wiki/Programmed-Graphics
for general information on using list comprehensions to generate graphics.

http://brythonserver.github.io/ggame/
for detailed information on ggame.
"""

from ggame import App, Color, LineStyle, Sprite, CircleAsset
from math import sin, cos, radians

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
purple = Color(0x8904B1, 1.0)

thinline = LineStyle(1, black)
mycircle = CircleAsset(5, thinline, blue)
xcoordinates = range(0, 360, 10)

# Generate a list of sprites that form a line!
sprites = [Sprite(mycircle, (x, 100sin(radians(x)) + 100)) for x in xcoordinates]

myapp = App()
myapp.run()<|endoftext|>


<commit_before># -*- encoding: UTF-8 -*-
from __future__ import absolute_import, unicode_literals

from pymongo.errors import PyMongoError
from types import MethodType, FunctionType

__all__ = [
    'Document',
]

class Document(object):
    """
    Document Class
    It will know how to validate its fields
    Will use the meta of the Collection to do so.
    """

    def __init__(self, collection, fields=None, processed=False):
        """
        Initializes the Document Object with the given attributes
        Processes the fields if not processed
        Then validates the fields based on the Collection
        """
        super(Document, self).__init__()

        self._collection = collection
        self._fields = fields or {}
        self._errors = {}

        if not processed:
            self._process()

        self._validate()

    def __getattr__(self, attr):
        """
        Tries to get the attribute on the following order:
        First checks if the attribute is one of:
        (__new__, _collection, _fields, _errors, get)
        Second checks if the attribute is in _fields
        Third checks if the attribute is in _collection
        If none of them is found, tries to get the attribute from self
        """
        if attr in ('__new__', '_collection', '_fields', '_errors', 'get'):
            return object.__getattribute__(self, attr)

        elif attr in self._fields:
            return self._fields[attr]

        elif hasattr(self._collection, attr):
            attribute = getattr(self._collection, attr)

            if type(attribute) == FunctionType:
                return MethodType(attribute, self)

            return attribute

        else:
            return object.__getattribute__(self, attr)

    def __setattr__(self, attr, value):
        """
        Tries to set the value to attr in the following order:
        Checks if the attribute is one of (_collection, _fields, _errors)
        If it is not, it will set the value in _fields[attr]
        """
        if attr in ('_collection', '_fields', '_errors'):
            object.__setattr__(self, attr, value)
        else:
            self._fields[attr] = value

    def __repr__(self):
        """
        Returns the representation of the Object formated like:
        <{Collection Name}Document object at {hex location of the object}>
        """
        return '<Document<{0}> object at {1}>'.format(
            self._collection.__name__, hex(id(self)),
        )

    def _validate(self):
        """
        Validates if the required fields are present on the Document
        Validates if the required fields are of the correct types
        If one of these two validations fail, add an error to self._errors
        """
        fields = dict(
            self.meta.get('optional', {}), **self.meta.get('required', {})
        )

        for (field, type_or_tuple) in list(fields.items()):
            if field in self._fields:
                if not isinstance(self._fields[field], type_or_tuple):
                    if isinstance(type_or_tuple, (tuple, list)):
                        types = ' or '.join(t.__name__ for t in type_or_tuple)
                    else:
                        types = type_or_tuple.__name__

                    self._errors[field] = \
                        'Field \'{0}\' must be of type(s): {1}.'.format(
                            field, types
                        )
            elif field in self.meta.get('required', {}):
                self._errors[field] = 'Field \'{0}\' is required.'.format(
                    field
                )

    def _process(self):
        """
        Calls every collection method that starts with 'process'.
        Does this in order to process the values on the fields
        So they will be ready to be saved on the Database
        """
        for attr in dir(self._collection):
            if attr.lower().startswith('process'):
                self.__getattr__(attr)()

    @property
    def is_valid(self):
        """
        Returns True if no errors have been found, False otherwise.
        """
        return len(self._errors) == 0

    @property
    def fields(self):
        """
        Returns the document's fields
        """
        return self._fields

    @property
    def errors(self):
        """
        Returns the document's errors
        """
        return self._errors

    def save(self):
        """
        Saves the Document to the database if it is valid.
        Returns the error dict otherwise.
        If the Document does not contain an _id it will insert a new Document
        If the Document contains an _id it will be replaced instead of inserted
        """
        if self.is_valid:
            try:
                if '_id' in self._fields:
                    self.replace_one({'_id': self._id}, self._fields, upsert=True)
                else:
                    self._fields['_id'] = self.insert_one(self._fields)
            except PyMongoError as error:
                self._errors['save'] = error.details.get(
                    'errmsg', error.details.get('err', 'Error.')
                )
                return self._errors

            return self._id
        else:
            return self._errors
<commit_msg>fixing codacy line too long<commit_after># -*- encoding: UTF-8 -*-
from __future__ import absolute_import, unicode_literals

from pymongo.errors import PyMongoError
from types import MethodType, FunctionType

__all__ = [
    'Document',
]

class Document(object):
    """
    Document Class
    It will know how to validate its fields
    Will use the meta of the Collection to do so.
    """

    def __init__(self, collection, fields=None, processed=False):
        """
        Initializes the Document Object with the given attributes
        Processes the fields if not processed
        Then validates the fields based on the Collection
        """
        super(Document, self).__init__()

        self._collection = collection
        self._fields = fields or {}
        self._errors = {}

        if not processed:
            self._process()

        self._validate()

    def __getattr__(self, attr):
        """
        Tries to get the attribute on the following order:
        First checks if the attribute is one of:
        (__new__, _collection, _fields, _errors, get)
        Second checks if the attribute is in _fields
        Third checks if the attribute is in _collection
        If none of them is found, tries to get the attribute from self
        """
        if attr in ('__new__', '_collection', '_fields', '_errors', 'get'):
            return object.__getattribute__(self, attr)

        elif attr in self._fields:
            return self._fields[attr]

        elif hasattr(self._collection, attr):
            attribute = getattr(self._collection, attr)

            if type(attribute) == FunctionType:
                return MethodType(attribute, self)

            return attribute

        else:
            return object.__getattribute__(self, attr)

    def __setattr__(self, attr, value):
        """
        Tries to set the value to attr in the following order:
        Checks if the attribute is one of (_collection, _fields, _errors)
        If it is not, it will set the value in _fields[attr]
        """
        if attr in ('_collection', '_fields', '_errors'):
            object.__setattr__(self, attr, value)
        else:
            self._fields[attr] = value

    def __repr__(self):
        """
        Returns the representation of the Object formated like:
        <{Collection Name}Document object at {hex location of the object}>
        """
        return '<Document<{0}> object at {1}>'.format(
            self._collection.__name__, hex(id(self)),
        )

    def _validate(self):
        """
        Validates if the required fields are present on the Document
        Validates if the required fields are of the correct types
        If one of these two validations fail, add an error to self._errors
        """
        fields = dict(
            self.meta.get('optional', {}), **self.meta.get('required', {})
        )

        for (field, type_or_tuple) in list(fields.items()):
            if field in self._fields:
                if not isinstance(self._fields[field], type_or_tuple):
                    if isinstance(type_or_tuple, (tuple, list)):
                        types = ' or '.join(t.__name__ for t in type_or_tuple)
                    else:
                        types = type_or_tuple.__name__

                    self._errors[field] = \
                        'Field \'{0}\' must be of type(s): {1}.'.format(
                            field, types
                        )
            elif field in self.meta.get('required', {}):
                self._errors[field] = 'Field \'{0}\' is required.'.format(
                    field
                )

    def _process(self):
        """
        Calls every collection method that starts with 'process'.
        Does this in order to process the values on the fields
        So they will be ready to be saved on the Database
        """
        for attr in dir(self._collection):
            if attr.lower().startswith('process'):
                self.__getattr__(attr)()

    @property
    def is_valid(self):
        """
        Returns True if no errors have been found, False otherwise.
        """
        return len(self._errors) == 0

    @property
    def fields(self):
        """
        Returns the document's fields
        """
        return self._fields

    @property
    def errors(self):
        """
        Returns the document's errors
        """
        return self._errors

    def save(self):
        """
        Saves the Document to the database if it is valid.
        Returns the error dict otherwise.
        If the Document does not contain an _id it will insert a new Document
        If the Document contains an _id it will be replaced instead of inserted
        """
        if self.is_valid:
            try:
                if '_id' in self._fields:
                    self.replace_one(
                        {'_id': self._id}, self._fields, upsert=True
                    )
                else:
                    self._fields['_id'] = self.insert_one(self._fields)
            except PyMongoError as error:
                self._errors['save'] = error.details.get(
                    'errmsg', error.details.get('err', 'Error.')
                )
                return self._errors

            return self._id
        else:
            return self._errors
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
TomoPy example script to reconstruct a single data set.
"""

from __future__ import print_function

import os
import sys
import json
import argparse
import numpy as np
import collections

import h5py
import tomopy
import dxchange

def get_dx_dims(fname, dataset):
    """
    Read array size of a specific group of Data Exchange file.ls

    Parameters
    ----------
    fname : str
        String defining the path of file or file name.
    dataset : str
        Path to the dataset inside hdf5 file where data is located.

    Returns
    -------
    ndarray
        Data set size.
    """

    grp = '/'.join(['exchange', dataset])

    with h5py.File(fname, "r") as f:
        try:
            data = f[grp]
        except KeyError:
            return None

        shape = data.shape

    return shape

def restricted_float(x):

    x = float(x)
    if x < 0.0 or x >= 1.0:
        raise argparse.ArgumentTypeError("%r not in range [0.0, 1.0]"%(x,))
    return x

def reconstruct(h5fname, sino, nframes, frame, nproj, binning, tv, rot_center):

    # Read APS 32-BM raw data.
    print("Read data")
    proj, flat, dark, theta = dxchange.read_aps_32id(h5fname, sino=sino)
    print("Projection size:" + str(proj.shape))

    print("Processing")
    if int(frame-nframes)>0:
        proj = proj[(frame-nframes/2)*nproj:(frame+nframes/2)*nproj,:,:]

    # Flat-field correction of raw data.
    print("Flat-field correcting")
    data = tomopy.normalize(proj, flat, dark, cutoff=1.4)

    # remove stripes
    #print("Removing stripes")
    #data = tomopy.remove_stripe_fw(data,level=7,wname='sym16',sigma=1,pad=True)

    print("Raw data: ", h5fname)
    if (frame-nframes)>0:
        print("Frames for reconstruction:",(frame-nframes/2),"..",(frame+nframes/2))
    else:
        print("Frames for reconstruction:",(0),"..",(nframes))
    # Phase retrieval for tomobank id 00080
    # sample_detector_distance = 25
    # detector_pixel_size_x = 3.0e-4
    # monochromator_energy = 16

    # phase retrieval
    # data = tomopy.prep.phase.retrieve_phase(data,pixel_size=detector_pixel_size_x,dist=sample_detector_distance,energy=monochromator_energy,alpha=8e-03,pad=True)

    data = tomopy.minus_log(data)

    data = tomopy.remove_nan(data, val=0.0)
    data = tomopy.remove_neg(data, val=0.00)
    data[np.where(data == np.inf)] = 0.00

    # Binning
    data = tomopy.downsample(data, level=binning, axis=2)
    if data.shape[1]>1:
       data = tomopy.downsample(data, level=binning, axis=1)

    theta = np.linspace(0,np.pi*nframes,nproj*nframes,endpoint=False)

    if tv:
	import rectv
	# Reconstruct. Iterative TV.
	[Ntheta,Nz,N] = data.shape
	Nzp = 4 # number of slices to process simultaniously by gpus
	M = nframes # number of basis functions, must be a multiple of nframes
	lambda0 = pow(2,-9) # regularization parameter 1
	lambda1 = pow(2,2) # regularization parameter 2
	niters = 1024 # number of iterations
	ngpus = 1 # number of gpus

	data = np.ndarray.flatten(data.swapaxes(0,1)) # reorder input data for compatibility
	rec = np.zeros([N*N*Nz*M],dtype='float32') # memory for result

	# Make a class for tv
	cl = rectv.rectv(N,Ntheta,M,nframes,Nz,Nzp,ngpus,lambda0,lambda1)
	# Run iterations
	cl.itertvR_wrap(rec,data,niters)

	rec = np.rot90(np.reshape(rec,[Nz,M,N,N]).swapaxes(0,1),axes=(2,3))/Ntheta*nframes*2 # reorder result for compatibility
	rec = rec[::M/nframes]
    else:
   	# Reconstruct object. FBP.
    	rec = np.zeros((nframes,data.shape[1],data.shape[2],data.shape[2]),dtype='float32')
	for time_frame in range(0,nframes):
		rec0 = tomopy.recon(data[time_frame*nproj:(time_frame+1)*nproj], theta[time_frame*nproj:(time_frame+1)*nproj], center=rot_center-np.mod(time_frame,2), algorithm='gridrec')
		# Mask each reconstructed slice with a circle.
		rec[time_frame] = tomopy.circ_mask(rec0, axis=0, ratio=0.95)
    return rec

def rec_full(h5fname, nframes, frame, nproj, binning, tv, rot_center):

    data_size = get_dx_dims(h5fname, 'data')

    # Select sinogram range to reconstruct.
    sino_start = 0
    sino_end = data_size[1]

    chunks = data_size[1]/(8*pow(2,binning))         # number of sinogram chunks to reconstruct
                        # only one chunk at the time is reconstructed
                        # allowing for limited RAM machines to complete a full reconstruction

    nSino_per_chunk = (sino_end - sino_start)/chunks
    print("Reconstructing [%d] slices from slice [%d] to [%d] in [%d] chunks of [%d] slices each" % ((sino_end - sino_start), sino_start, sino_end, chunks, nSino_per_chunk))

    strt = 0
    for iChunk in range(0,chunks):
        print('\n  -- chunk # %i' % (iChunk+1))
        sino_chunk_start = sino_start + nSino_per_chunk*iChunk
        sino_chunk_end = sino_start + nSino_per_chunk*(iChunk+1)
        print('\n  --------> [%i, %i]' % (sino_chunk_start, sino_chunk_end))

        if sino_chunk_end > sino_end:
            break

        sino = (int(sino_chunk_start), int(sino_chunk_end))

        # Reconstruct.
        rec = reconstruct(h5fname, sino, nframes, frame, nproj, binning, tv, rot_center)

        # Write data as stack of TIFs.
	for time_frame in range(0,nframes):
	        fname = os.path.dirname(os.path.abspath(h5fname)) + '/' + os.path.splitext(os.path.basename(h5fname))[0]+ '_rec_full/' + 'recon' + str(frame-nframes/2+time_frame) + '_'
	        print("Reconstructions: ", fname)
	        dxchange.write_tiff_stack(rec[time_frame], fname=fname, start=strt)
        strt += (sino[1] - sino[0])/pow(2,binning)

def rec_subset(h5fname, nsino, nframes, frame, nproj, binning, tv, rot_center):

    data_size = get_dx_dims(h5fname, 'data')

    # Select sinogram range to reconstruct.
    ssino = int(data_size[1] * nsino)
    sino_start = ssino-4*pow(2,binning)
    sino_end = ssino+4*pow(2,binning)

    print("Reconstructing [%d] slices from slice [%d] to [%d]" % ((sino_end - sino_start), sino_start, sino_end))

    sino = (int(sino_start), int(sino_end))
    # Reconstruct.
    rec = reconstruct(h5fname, sino, nframes, frame, nproj, binning, tv, rot_center)

    # Write data as stack of TIFs.
    for time_frame in range(0,nframes):
        fname = os.path.dirname(os.path.abspath(h5fname)) + '/' + os.path.splitext(os.path.basename(h5fname))[0]+ '_rec_subset/' + 'recon' + str(frame-nframes/2+time_frame) + '_'
        print("Reconstructions: ", fname)
        dxchange.write_tiff_stack(rec[time_frame], fname=fname, start=sino_start)

def rec_slice(h5fname, nsino, nframes, frame, nproj, binning, tv, rot_center):

    data_size = get_dx_dims(h5fname, 'data')
    ssino = int(data_size[1] * nsino)

    # Select sinogram range to reconstruct
    sino = None

    start = ssino
    end = start + 1
    sino = (start, end)

    # Reconstruct
    rec = reconstruct(h5fname, sino, nframes, frame, nproj, binning, tv, rot_center)
    # Write data as stack of TIFs.

    for time_frame in range(0,nframes):
        if (frame-nframes)>0:
            fname = os.path.dirname(os.path.abspath(h5fname)) + '/' + os.path.splitext(os.path.basename(h5fname))[0]+ '_rec_slice/' + 'recon' + str(frame-nframes/2+time_frame) + '_'
        else:
	    fname = os.path.dirname(os.path.abspath(h5fname)) + '/' + os.path.splitext(os.path.basename(h5fname))[0]+ '_rec_slice/' + 'recon' + str(time_frame) + '_'
	dxchange.write_tiff_stack(rec[time_frame], fname=fname)
        print("Rec: ", fname)
    print("Slice: ", start)

def main(arg):

    parser = argparse.ArgumentParser()
    parser.add_argument("fname", help="file name of a tomographic dataset")
    parser.add_argument("--type", nargs='?', type=str, default="slice", help="reconstruction type: full (default slice)")
    parser.add_argument("--nsino", nargs='?', type=restricted_float, default=0.5, help="location of the sinogram used by slice reconstruction (0 top, 1 bottom): 0.5 (default 0.5)")
    parser.add_argument("--tv", nargs='?', type=bool, default=False, help="Use Total variation reconstruction method (Gridrec otherwise): False (default False)")
    parser.add_argument("--binning", nargs='?', type=str, default=0, help="binning projections: 0 (default 0)")
    parser.add_argument("--frame", nargs='?', type=str, default=0, help="time frame with motion: 92 (default 92)")
    parser.add_argument("--axis", nargs='?', type=str, default="702", help="rotation axis location: 1024.0 (default 1/2 image horizontal size)")
    parser.add_argument("--ntframes", nargs='?', type=str, default=60, help="total number of time frames (default 60)")
    parser.add_argument("--nproj", nargs='?', type=str, default=300, help="number of projections per time frame (default 301)")

    args = parser.parse_args()

    # Set path to the micro-CT data to reconstruct.
    fname = args.fname

    rot_center = float(args.axis)

    # Set default rotation axis location
    if rot_center == 0:
        data_size = get_dx_dims(fname, 'data')
        rot_center =  data_size[2]/2

    nsino = float(args.nsino)
    binning = int(args.binning)
    frame = int(args.frame)
    tv = args.tv

    print("Reconstructing with tv: " + str(tv))
    nframes = int(args.ntframes) #number of time frames for reconstruction
    nproj = int(args.nproj) #number of projections per 180 degrees interval

    slice = False
    subset = False

    if args.type == "slice":
        slice = True
    if args.type == "subset":
        subset = True

    if os.path.isfile(fname):
        if slice:
            rec_slice(fname, nsino, nframes, frame, nproj, binning, tv, rot_center)
	else:
	    if subset:
	            rec_subset(fname, nsino, nframes, frame, nproj, binning, tv, rot_center)
	    else:
		    rec_full(fname, nframes, frame, nproj, binning, tv, rot_center)
    else:
        print("File Name does not exist: ", fname)

if __name__ == "__main__":
    main(sys.argv[1:])

<commit_msg>Remove projection size print<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
TomoPy example script to reconstruct a single data set.
"""

from __future__ import print_function

import os
import sys
import json
import argparse
import numpy as np
import collections

import h5py
import tomopy
import dxchange

def get_dx_dims(fname, dataset):
    """
    Read array size of a specific group of Data Exchange file.ls

    Parameters
    ----------
    fname : str
        String defining the path of file or file name.
    dataset : str
        Path to the dataset inside hdf5 file where data is located.

    Returns
    -------
    ndarray
        Data set size.
    """

    grp = '/'.join(['exchange', dataset])

    with h5py.File(fname, "r") as f:
        try:
            data = f[grp]
        except KeyError:
            return None

        shape = data.shape

    return shape

def restricted_float(x):

    x = float(x)
    if x < 0.0 or x >= 1.0:
        raise argparse.ArgumentTypeError("%r not in range [0.0, 1.0]"%(x,))
    return x

def reconstruct(h5fname, sino, nframes, frame, nproj, binning, tv, rot_center):

    # Read APS 32-BM raw data.
    print("Read data")
    proj, flat, dark, theta = dxchange.read_aps_32id(h5fname, sino=sino)

    print("Processing")
    if int(frame-nframes)>0:
        proj = proj[(frame-nframes/2)*nproj:(frame+nframes/2)*nproj,:,:]

    # Flat-field correction of raw data.
    print("Flat-field correcting")
    data = tomopy.normalize(proj, flat, dark, cutoff=1.4)

    # remove stripes
    #print("Removing stripes")
    #data = tomopy.remove_stripe_fw(data,level=7,wname='sym16',sigma=1,pad=True)

    print("Raw data: ", h5fname)
    if (frame-nframes)>0:
        print("Frames for reconstruction:",(frame-nframes/2),"..",(frame+nframes/2))
    else:
        print("Frames for reconstruction:",(0),"..",(nframes))
    # Phase retrieval for tomobank id 00080
    # sample_detector_distance = 25
    # detector_pixel_size_x = 3.0e-4
    # monochromator_energy = 16

    # phase retrieval
    # data = tomopy.prep.phase.retrieve_phase(data,pixel_size=detector_pixel_size_x,dist=sample_detector_distance,energy=monochromator_energy,alpha=8e-03,pad=True)

    data = tomopy.minus_log(data)

    data = tomopy.remove_nan(data, val=0.0)
    data = tomopy.remove_neg(data, val=0.00)
    data[np.where(data == np.inf)] = 0.00

    # Binning
    data = tomopy.downsample(data, level=binning, axis=2)
    if data.shape[1]>1:
       data = tomopy.downsample(data, level=binning, axis=1)

    theta = np.linspace(0,np.pi*nframes,nproj*nframes,endpoint=False)

    if tv:
	import rectv
	# Reconstruct. Iterative TV.
	[Ntheta,Nz,N] = data.shape
	Nzp = 4 # number of slices to process simultaniously by gpus
	M = nframes # number of basis functions, must be a multiple of nframes
	lambda0 = pow(2,-9) # regularization parameter 1
	lambda1 = pow(2,2) # regularization parameter 2
	niters = 1024 # number of iterations
	ngpus = 1 # number of gpus

	data = np.ndarray.flatten(data.swapaxes(0,1)) # reorder input data for compatibility
	rec = np.zeros([N*N*Nz*M],dtype='float32') # memory for result

	# Make a class for tv
	cl = rectv.rectv(N,Ntheta,M,nframes,Nz,Nzp,ngpus,lambda0,lambda1)
	# Run iterations
	cl.itertvR_wrap(rec,data,niters)

	rec = np.rot90(np.reshape(rec,[Nz,M,N,N]).swapaxes(0,1),axes=(2,3))/Ntheta*nframes*2 # reorder result for compatibility
	rec = rec[::M/nframes]
    else:
   	# Reconstruct object. FBP.
    	rec = np.zeros((nframes,data.shape[1],data.shape[2],data.shape[2]),dtype='float32')
	for time_frame in range(0,nframes):
		rec0 = tomopy.recon(data[time_frame*nproj:(time_frame+1)*nproj], theta[time_frame*nproj:(time_frame+1)*nproj], center=rot_center-np.mod(time_frame,2), algorithm='gridrec')
		# Mask each reconstructed slice with a circle.
		rec[time_frame] = tomopy.circ_mask(rec0, axis=0, ratio=0.95)
    return rec

def rec_full(h5fname, nframes, frame, nproj, binning, tv, rot_center):

    data_size = get_dx_dims(h5fname, 'data')

    # Select sinogram range to reconstruct.
    sino_start = 0
    sino_end = data_size[1]

    chunks = data_size[1]/(8*pow(2,binning))         # number of sinogram chunks to reconstruct
                        # only one chunk at the time is reconstructed
                        # allowing for limited RAM machines to complete a full reconstruction

    nSino_per_chunk = (sino_end - sino_start)/chunks
    print("Reconstructing [%d] slices from slice [%d] to [%d] in [%d] chunks of [%d] slices each" % ((sino_end - sino_start), sino_start, sino_end, chunks, nSino_per_chunk))

    strt = 0
    for iChunk in range(0,chunks):
        print('\n  -- chunk # %i' % (iChunk+1))
        sino_chunk_start = sino_start + nSino_per_chunk*iChunk
        sino_chunk_end = sino_start + nSino_per_chunk*(iChunk+1)
        print('\n  --------> [%i, %i]' % (sino_chunk_start, sino_chunk_end))

        if sino_chunk_end > sino_end:
            break

        sino = (int(sino_chunk_start), int(sino_chunk_end))

        # Reconstruct.
        rec = reconstruct(h5fname, sino, nframes, frame, nproj, binning, tv, rot_center)

        # Write data as stack of TIFs.
	for time_frame in range(0,nframes):
	        fname = os.path.dirname(os.path.abspath(h5fname)) + '/' + os.path.splitext(os.path.basename(h5fname))[0]+ '_rec_full/' + 'recon' + str(frame-nframes/2+time_frame) + '_'
	        print("Reconstructions: ", fname)
	        dxchange.write_tiff_stack(rec[time_frame], fname=fname, start=strt)
        strt += (sino[1] - sino[0])/pow(2,binning)

def rec_subset(h5fname, nsino, nframes, frame, nproj, binning, tv, rot_center):

    data_size = get_dx_dims(h5fname, 'data')

    # Select sinogram range to reconstruct.
    ssino = int(data_size[1] * nsino)
    sino_start = ssino-4*pow(2,binning)
    sino_end = ssino+4*pow(2,binning)

    print("Reconstructing [%d] slices from slice [%d] to [%d]" % ((sino_end - sino_start), sino_start, sino_end))

    sino = (int(sino_start), int(sino_end))
    # Reconstruct.
    rec = reconstruct(h5fname, sino, nframes, frame, nproj, binning, tv, rot_center)

    # Write data as stack of TIFs.
    for time_frame in range(0,nframes):
        fname = os.path.dirname(os.path.abspath(h5fname)) + '/' + os.path.splitext(os.path.basename(h5fname))[0]+ '_rec_subset/' + 'recon' + str(frame-nframes/2+time_frame) + '_'
        print("Reconstructions: ", fname)
        dxchange.write_tiff_stack(rec[time_frame], fname=fname, start=sino_start)

def rec_slice(h5fname, nsino, nframes, frame, nproj, binning, tv, rot_center):

    data_size = get_dx_dims(h5fname, 'data')
    ssino = int(data_size[1] * nsino)

    # Select sinogram range to reconstruct
    sino = None

    start = ssino
    end = start + 1
    sino = (start, end)

    # Reconstruct
    rec = reconstruct(h5fname, sino, nframes, frame, nproj, binning, tv, rot_center)
    # Write data as stack of TIFs.

    for time_frame in range(0,nframes):
        if (frame-nframes)>0:
            fname = os.path.dirname(os.path.abspath(h5fname)) + '/' + os.path.splitext(os.path.basename(h5fname))[0]+ '_rec_slice/' + 'recon' + str(frame-nframes/2+time_frame) + '_'
        else:
	    fname = os.path.dirname(os.path.abspath(h5fname)) + '/' + os.path.splitext(os.path.basename(h5fname))[0]+ '_rec_slice/' + 'recon' + str(time_frame) + '_'
	dxchange.write_tiff_stack(rec[time_frame], fname=fname)
        print("Rec: ", fname)
    print("Slice: ", start)

def main(arg):

    parser = argparse.ArgumentParser()
    parser.add_argument("fname", help="file name of a tomographic dataset")
    parser.add_argument("--type", nargs='?', type=str, default="slice", help="reconstruction type: full (default slice)")
    parser.add_argument("--nsino", nargs='?', type=restricted_float, default=0.5, help="location of the sinogram used by slice reconstruction (0 top, 1 bottom): 0.5 (default 0.5)")
    parser.add_argument("--tv", nargs='?', type=bool, default=False, help="Use Total variation reconstruction method (Gridrec otherwise): False (default False)")
    parser.add_argument("--binning", nargs='?', type=str, default=0, help="binning projections: 0 (default 0)")
    parser.add_argument("--frame", nargs='?', type=str, default=0, help="time frame with motion: 92 (default 92)")
    parser.add_argument("--axis", nargs='?', type=str, default="702", help="rotation axis location: 1024.0 (default 1/2 image horizontal size)")
    parser.add_argument("--ntframes", nargs='?', type=str, default=60, help="total number of time frames (default 60)")
    parser.add_argument("--nproj", nargs='?', type=str, default=300, help="number of projections per time frame (default 300)")

    args = parser.parse_args()

    # Set path to the micro-CT data to reconstruct.
    fname = args.fname

    rot_center = float(args.axis)

    # Set default rotation axis location
    if rot_center == 0:
        data_size = get_dx_dims(fname, 'data')
        rot_center =  data_size[2]/2

    nsino = float(args.nsino)
    binning = int(args.binning)
    frame = int(args.frame)
    tv = args.tv

    print("Reconstructing with tv: " + str(tv))
    nframes = int(args.ntframes) #number of time frames for reconstruction
    nproj = int(args.nproj) #number of projections per 180 degrees interval

    slice = False
    subset = False

    if args.type == "slice":
        slice = True
    if args.type == "subset":
        subset = True

    if os.path.isfile(fname):
        if slice:
            rec_slice(fname, nsino, nframes, frame, nproj, binning, tv, rot_center)
	else:
	    if subset:
	            rec_subset(fname, nsino, nframes, frame, nproj, binning, tv, rot_center)
	    else:
		    rec_full(fname, nframes, frame, nproj, binning, tv, rot_center)
    else:
        print("File Name does not exist: ", fname)

if __name__ == "__main__":
    main(sys.argv[1:])

<|endoftext|>


<commit_before>#! /usr/bin/env python
from __future__ import print_function, unicode_literals
import sys
import subprocess
import io
import re

def read_input(input):
  print(input)

def read( parameters ):
  print(paramters)

#def write_outputfile():

def main():
  output = "echo "
  prog = 'cyan'
  db = 'cyclus.sqlite'
  cmd = prog + ' -db ' + db + ' trans'
  print(cmd)

  output = subprocess.check_output(cmd.split())
  buf = output.decode("utf-8")
  #  line = buf.readline()
  # print(line)
  print(buf.__class__)
  lines = buf.splitlines()
  matrix = []

  print(len(lines))
  for line in lines:
    cols = line.split()
    matrix.append(cols)

  read_input(sys.argv[1])
#  print(len(matrix))

#cols = line.split()
#print(len(cols))

# print(cols.__class__)

#matrix = []

#matrix.append(cols)

# print(matrix)
main()<commit_msg>add function//still empty one...<commit_after>#! /usr/bin/env python
from __future__ import print_function, unicode_literals
import sys
import subprocess
import io
import re

def read_input(input):
  f = open(input, 'r')
  matrix = []
  for line in f:
    print(line)
    matrix.append(line)

  return matrix

def read( parameters ):
  print(paramters)

def recover_info( cmd ):
  output = subprocess.check_output(cmd.split())
  buf = output.decode("utf-8")
  lines = buf.splitlines()
  matrix = []
  for line in lines:
    cols = line.split()
    matrix.append(cols)
  return matrix

def treat_the_matrix( matrix)

def build_cmd(inputline)

#def write_outputfile():

def main():

  if len(sys.argv) != 2 :
    print("missing argument !!")
  #   quit()

  prog = 'cyan'
  db = 'cyclus.sqlite'
  cmd = prog + ' -db ' + db + ' trans'
  matrix = recover_info(cmd)

  read_input(sys.argv[1])
#  print(len(matrix))

#cols = line.split()
#print(len(cols))

# print(cols.__class__)

#matrix = []

#matrix.append(cols)

# print(matrix)
main()<|endoftext|>


<commit_before>import logging
import json
import uuid
import io
from functools import partial
import asyncio
from concurrent.futures import ThreadPoolExecutor

import tornado.web
import pymongo
import motor

try:
    import boto3
    import botocore.client
except ImportError:
    boto3 = None

from iceprod.server.util import nowstr
from iceprod.server.rest import RESTHandler, RESTHandlerSetup, authorization, catch_error

logger = logging.getLogger('rest.logs')

class S3:
    """S3 wrapper for uploading and downloading objects"""
    def __init__(self, config):
        self.s3 = None
        self.bucket = 'iceprod2-logs'
        if (boto3 and 's3' in config and 'access_key' in config['s3'] and
            'secret_key' in config['s3']):
            try:
                self.s3 = boto3.client('s3','us-east-1',
                    aws_access_key_id=config['s3']['access_key'],
                    aws_secret_access_key=config['s3']['secret_key'],
                    config=botocore.client.Config(max_pool_connections=101))
            except Exception:
                logger.warning('failed to connect to s3: %r',
                            config['s3'], exc_info=True)
        self.executor = ThreadPoolExecutor(max_workers=20)

    async def get(self, key):
        """Download object from S3"""
        ret = ''
        with io.BytesIO() as f:
            loop = asyncio.get_event_loop()
            await loop.run_in_executor(self.executor,
                    partial(self.s3.download_fileobj, Bucket=self.bucket,
                            Key=key, Fileobj=f))
            ret = f.getvalue()
        return ret.decode('utf-8')

    async def put(self, key, data):
        """Upload object to S3"""
        with io.BytesIO(data.encode('utf-8')) as f:
            loop = asyncio.get_event_loop()
            await loop.run_in_executor(self.executor,
                    partial(self.s3.upload_fileobj, f, self.bucket, key))

def setup(config, *args, **kwargs):
    """
    Setup method for Logs REST API.

    Sets up any database connections or other prerequisites.

    Args:
        config (dict): an instance of :py:class:`iceprod.server.config`.

    Returns:
        list: Routes for logs, which can be passed to :py:class:`tornado.web.Application`.
    """
    cfg_rest = config.get('rest',{}).get('logs',{})
    db_cfg = cfg_rest.get('database',{})

    # add indexes
    db = pymongo.MongoClient(**db_cfg).logs
    if 'log_id_index' not in db.logs.index_information():
        db.logs.create_index('log_id', name='log_id_index', unique=True)
    if 'task_id_index' not in db.logs.index_information():
        db.logs.create_index('task_id', name='task_id_index', unique=False)
    if 'dataset_id_index' not in db.logs.index_information():
        db.logs.create_index('dataset_id', name='dataset_id_index', unique=False)

    # S3
    s3 = S3(config) if boto3 and 's3' in config else None

    handler_cfg = RESTHandlerSetup(config, *args, **kwargs)
    handler_cfg.update({
        'database': motor.motor_tornado.MotorClient(**db_cfg).logs,
        's3': s3,
    })

    return [
        (r'/logs', MultiLogsHandler, handler_cfg),
        (r'/logs/(?P<log_id>\w+)', LogsHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/logs', DatasetMultiLogsHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/logs/(?P<log_id>\w+)', DatasetLogsHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/tasks/(?P<task_id>\w+)/logs', DatasetTaskLogsHandler, handler_cfg),
    ]

class BaseHandler(RESTHandler):
    def initialize(self, database=None, s3=None, **kwargs):
        super(BaseHandler, self).initialize(**kwargs)
        self.db = database
        self.s3 = s3

class MultiLogsHandler(BaseHandler):
    """
    Handle logs requests.
    """
    @authorization(roles=['admin','client','pilot'])
    async def post(self):
        """
        Create a log entry.

        Body should contain the following fields:
            data: str

        Optional fields:
            dataset_id: str
            task_id: str
            name: str

        Returns:
            dict: {'result': <log_id>}
        """
        data = json.loads(self.request.body)
        if 'data' not in data:
            raise tornado.web.HTTPError(400, reason='data field not in body')
        if 'name' not in data:
            data['name'] = 'log'
        data['log_id'] = uuid.uuid1().hex
        data['timestamp'] = nowstr()
        if self.s3 and len(data['data']) > 1000000:
            await self.s3.put(data['log_id'], data['data'])
            del data['data']
        ret = await self.db.logs.insert_one(data)
        self.set_status(201)
        self.write({'result': data['log_id']})
        self.finish()

class LogsHandler(BaseHandler):
    """
    Handle logs requests.
    """
    @authorization(roles=['admin'])
    async def get(self, log_id):
        """
        Get a log entry.

        Args:
            log_id (str): the log id of the entry

        Returns:
            dict: all body fields
        """
        ret = await self.db.logs.find_one({'log_id':log_id},
                projection={'_id':False})
        if not ret:
            self.send_error(404, reason="Log not found")
        else:
            if 'data' not in ret:
                if self.s3:
                    ret['data'] = await self.s3.get(ret['log_id'])
                else:
                    raise tornado.web.HTTPError(500, reason='no data field and s3 disabled')
            self.write(ret)
            self.finish()

class DatasetMultiLogsHandler(BaseHandler):
    """
    Handle logs requests.
    """
    @authorization(roles=['admin'], attrs=['dataset_id:write'])
    async def post(self, dataset_id):
        """
        Create a log entry.

        Body should contain the following fields:
            data: str

        Optional fields:
            task_id: str
            name: str

        Args:
            dataset_id (str): the dataset id

        Returns:
            dict: {'result': <log_id>}
        """
        data = json.loads(self.request.body)
        if 'data' not in data:
            raise tornado.web.HTTPError(400, reason='data field not in body')
        data['log_id'] = uuid.uuid1().hex
        data['dataset_id'] = dataset_id
        data['timestamp'] = nowstr()
        if self.s3 and len(data['data']) > 1000000:
            await self.s3.put(data['log_id'], data['data'])
            del data['data']
        ret = await self.db.logs.insert_one(data)
        self.set_status(201)
        self.write({'result': data['log_id']})
        self.finish()

class DatasetLogsHandler(BaseHandler):
    """
    Handle logs requests.
    """
    @authorization(roles=['admin'], attrs=['dataset_id:read'])
    async def get(self, dataset_id, log_id):
        """
        Get a log.

        Args:
            dataset_id (str): the dataset id
            log_id (str): the log id of the entry

        Returns:
            dict: all body fields
        """
        ret = await self.db.logs.find_one({'dataset_id':dataset_id,'log_id':log_id},
                projection={'_id':False})
        if not ret:
            self.send_error(404, reason="Log not found")
        else:
            if 'data' not in ret:
                if self.s3:
                    ret['data'] = await self.s3.get(ret['log_id'])
                else:
                    raise tornado.web.HTTPError(500, reason='no data field and s3 disabled')
            self.write(ret)
            self.finish()

class DatasetTaskLogsHandler(BaseHandler):
    """
    Handle log requests for a task
    """
    @authorization(roles=['admin'], attrs=['dataset_id:read'])
    async def get(self, dataset_id, task_id):
        """
        Get logs for a dataset and task.

        Note: "num" and "group" are generally not used together.

        Params (optional):
            num (int): number of logs, or groups of logs, to return
            group {true, false}: group by log name
            order {asc, desc}: order by time
            keys: | separated list of keys to return for each log

        Args:
            dataset_id (str): the dataset id
            task_id (str): the task id

        Returns:
            dict: {'logs': [log entry dict, log entry dict]}
        """
        filters = {'dataset_id': dataset_id, 'task_id': task_id}

        num = self.get_argument('num', None)
        if num:
            try:
                num = int(num)
            except Exception:
                raise tornado.web.HTTPError(400, reason='bad num param. must be int')

        group = self.get_argument('group', 'false').lower() == 'true'
        order = self.get_argument('order', 'desc').lower()
        if order not in ('asc', 'desc'):
            raise tornado.web.HTTPError(400, reason='bad order param. should be "asc" or "desc".')

        projection = {'_id': False}
        keys = self.get_argument('keys', None)
        if keys:
            projection.update({x:True for x in keys.split('|') if x})

        steps = [
            {'$match': filters},
            {'$sort': {'timestamp': -1 if order == 'desc' else 1}},
        ]
        if group:
            if not keys:
                keys = 'log_id|name|task_id|dataset_id|data|timestamp'
            grouping = {x:{'$first':'$'+x} for x in keys.split('|') if x}
            grouping['_id'] = '$name'
            if 'timestamp' not in grouping:
                grouping['timestamp'] = {'$first': '$timestamp'}
            steps.extend([
                {'$group': grouping},
                {'$sort': {'timestamp': -1 if order == 'desc' else 1}},
            ])
        steps.append({'$project': projection})
        if num:
            steps.append({'$limit': num})
        logger.debug('steps: %r', steps)

        cur = self.db.logs.aggregate(steps, allowDiskUse=True)
        ret = []
        async for entry in cur:
            ret.append(entry)
        if not ret:
            self.send_error(404, reason="Log not found")
        else:
            for log in ret:
                if 'data' not in log:
                    if self.s3:
                        log['data'] = await self.s3.get(log['log_id'])
                    else:
                        raise Exception('no data field and s3 disabled')
            self.write({'logs':ret})
            self.finish()
<commit_msg>allow specifying log S3 host, so we can use ceph s3<commit_after>import logging
import json
import uuid
import io
from functools import partial
import asyncio
from concurrent.futures import ThreadPoolExecutor

import tornado.web
import pymongo
import motor

try:
    import boto3
    import botocore.client
except ImportError:
    boto3 = None

from iceprod.server.util import nowstr
from iceprod.server.rest import RESTHandler, RESTHandlerSetup, authorization, catch_error

logger = logging.getLogger('rest.logs')

class S3:
    """S3 wrapper for uploading and downloading objects"""
    def __init__(self, config):
        self.s3 = None
        self.bucket = 'iceprod2-logs'
        if (boto3 and 's3' in config and 'access_key' in config['s3'] and
            'secret_key' in config['s3']):
            try:
                self.s3 = boto3.client('s3','us-east-1',
                    endpoint_url=config['s3'].get('host', None),
                    aws_access_key_id=config['s3']['access_key'],
                    aws_secret_access_key=config['s3']['secret_key'],
                    config=botocore.client.Config(max_pool_connections=101))
            except Exception:
                logger.warning('failed to connect to s3: %r',
                            config['s3'], exc_info=True)
        self.executor = ThreadPoolExecutor(max_workers=20)

    async def get(self, key):
        """Download object from S3"""
        ret = ''
        with io.BytesIO() as f:
            loop = asyncio.get_event_loop()
            await loop.run_in_executor(self.executor,
                    partial(self.s3.download_fileobj, Bucket=self.bucket,
                            Key=key, Fileobj=f))
            ret = f.getvalue()
        return ret.decode('utf-8')

    async def put(self, key, data):
        """Upload object to S3"""
        with io.BytesIO(data.encode('utf-8')) as f:
            loop = asyncio.get_event_loop()
            await loop.run_in_executor(self.executor,
                    partial(self.s3.upload_fileobj, f, self.bucket, key))

def setup(config, *args, **kwargs):
    """
    Setup method for Logs REST API.

    Sets up any database connections or other prerequisites.

    Args:
        config (dict): an instance of :py:class:`iceprod.server.config`.

    Returns:
        list: Routes for logs, which can be passed to :py:class:`tornado.web.Application`.
    """
    cfg_rest = config.get('rest',{}).get('logs',{})
    db_cfg = cfg_rest.get('database',{})

    # add indexes
    db = pymongo.MongoClient(**db_cfg).logs
    if 'log_id_index' not in db.logs.index_information():
        db.logs.create_index('log_id', name='log_id_index', unique=True)
    if 'task_id_index' not in db.logs.index_information():
        db.logs.create_index('task_id', name='task_id_index', unique=False)
    if 'dataset_id_index' not in db.logs.index_information():
        db.logs.create_index('dataset_id', name='dataset_id_index', unique=False)

    # S3
    s3 = S3(config) if boto3 and 's3' in config else None

    handler_cfg = RESTHandlerSetup(config, *args, **kwargs)
    handler_cfg.update({
        'database': motor.motor_tornado.MotorClient(**db_cfg).logs,
        's3': s3,
    })

    return [
        (r'/logs', MultiLogsHandler, handler_cfg),
        (r'/logs/(?P<log_id>\w+)', LogsHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/logs', DatasetMultiLogsHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/logs/(?P<log_id>\w+)', DatasetLogsHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/tasks/(?P<task_id>\w+)/logs', DatasetTaskLogsHandler, handler_cfg),
    ]

class BaseHandler(RESTHandler):
    def initialize(self, database=None, s3=None, **kwargs):
        super(BaseHandler, self).initialize(**kwargs)
        self.db = database
        self.s3 = s3

class MultiLogsHandler(BaseHandler):
    """
    Handle logs requests.
    """
    @authorization(roles=['admin','client','pilot'])
    async def post(self):
        """
        Create a log entry.

        Body should contain the following fields:
            data: str

        Optional fields:
            dataset_id: str
            task_id: str
            name: str

        Returns:
            dict: {'result': <log_id>}
        """
        data = json.loads(self.request.body)
        if 'data' not in data:
            raise tornado.web.HTTPError(400, reason='data field not in body')
        if 'name' not in data:
            data['name'] = 'log'
        data['log_id'] = uuid.uuid1().hex
        data['timestamp'] = nowstr()
        if self.s3 and len(data['data']) > 1000000:
            await self.s3.put(data['log_id'], data['data'])
            del data['data']
        ret = await self.db.logs.insert_one(data)
        self.set_status(201)
        self.write({'result': data['log_id']})
        self.finish()

class LogsHandler(BaseHandler):
    """
    Handle logs requests.
    """
    @authorization(roles=['admin'])
    async def get(self, log_id):
        """
        Get a log entry.

        Args:
            log_id (str): the log id of the entry

        Returns:
            dict: all body fields
        """
        ret = await self.db.logs.find_one({'log_id':log_id},
                projection={'_id':False})
        if not ret:
            self.send_error(404, reason="Log not found")
        else:
            if 'data' not in ret:
                if self.s3:
                    ret['data'] = await self.s3.get(ret['log_id'])
                else:
                    raise tornado.web.HTTPError(500, reason='no data field and s3 disabled')
            self.write(ret)
            self.finish()

class DatasetMultiLogsHandler(BaseHandler):
    """
    Handle logs requests.
    """
    @authorization(roles=['admin'], attrs=['dataset_id:write'])
    async def post(self, dataset_id):
        """
        Create a log entry.

        Body should contain the following fields:
            data: str

        Optional fields:
            task_id: str
            name: str

        Args:
            dataset_id (str): the dataset id

        Returns:
            dict: {'result': <log_id>}
        """
        data = json.loads(self.request.body)
        if 'data' not in data:
            raise tornado.web.HTTPError(400, reason='data field not in body')
        data['log_id'] = uuid.uuid1().hex
        data['dataset_id'] = dataset_id
        data['timestamp'] = nowstr()
        if self.s3 and len(data['data']) > 1000000:
            await self.s3.put(data['log_id'], data['data'])
            del data['data']
        ret = await self.db.logs.insert_one(data)
        self.set_status(201)
        self.write({'result': data['log_id']})
        self.finish()

class DatasetLogsHandler(BaseHandler):
    """
    Handle logs requests.
    """
    @authorization(roles=['admin'], attrs=['dataset_id:read'])
    async def get(self, dataset_id, log_id):
        """
        Get a log.

        Args:
            dataset_id (str): the dataset id
            log_id (str): the log id of the entry

        Returns:
            dict: all body fields
        """
        ret = await self.db.logs.find_one({'dataset_id':dataset_id,'log_id':log_id},
                projection={'_id':False})
        if not ret:
            self.send_error(404, reason="Log not found")
        else:
            if 'data' not in ret:
                if self.s3:
                    ret['data'] = await self.s3.get(ret['log_id'])
                else:
                    raise tornado.web.HTTPError(500, reason='no data field and s3 disabled')
            self.write(ret)
            self.finish()

class DatasetTaskLogsHandler(BaseHandler):
    """
    Handle log requests for a task
    """
    @authorization(roles=['admin'], attrs=['dataset_id:read'])
    async def get(self, dataset_id, task_id):
        """
        Get logs for a dataset and task.

        Note: "num" and "group" are generally not used together.

        Params (optional):
            num (int): number of logs, or groups of logs, to return
            group {true, false}: group by log name
            order {asc, desc}: order by time
            keys: | separated list of keys to return for each log

        Args:
            dataset_id (str): the dataset id
            task_id (str): the task id

        Returns:
            dict: {'logs': [log entry dict, log entry dict]}
        """
        filters = {'dataset_id': dataset_id, 'task_id': task_id}

        num = self.get_argument('num', None)
        if num:
            try:
                num = int(num)
            except Exception:
                raise tornado.web.HTTPError(400, reason='bad num param. must be int')

        group = self.get_argument('group', 'false').lower() == 'true'
        order = self.get_argument('order', 'desc').lower()
        if order not in ('asc', 'desc'):
            raise tornado.web.HTTPError(400, reason='bad order param. should be "asc" or "desc".')

        projection = {'_id': False}
        keys = self.get_argument('keys', None)
        if keys:
            projection.update({x:True for x in keys.split('|') if x})

        steps = [
            {'$match': filters},
            {'$sort': {'timestamp': -1 if order == 'desc' else 1}},
        ]
        if group:
            if not keys:
                keys = 'log_id|name|task_id|dataset_id|data|timestamp'
            grouping = {x:{'$first':'$'+x} for x in keys.split('|') if x}
            grouping['_id'] = '$name'
            if 'timestamp' not in grouping:
                grouping['timestamp'] = {'$first': '$timestamp'}
            steps.extend([
                {'$group': grouping},
                {'$sort': {'timestamp': -1 if order == 'desc' else 1}},
            ])
        steps.append({'$project': projection})
        if num:
            steps.append({'$limit': num})
        logger.debug('steps: %r', steps)

        cur = self.db.logs.aggregate(steps, allowDiskUse=True)
        ret = []
        async for entry in cur:
            ret.append(entry)
        if not ret:
            self.send_error(404, reason="Log not found")
        else:
            for log in ret:
                if 'data' not in log:
                    if self.s3:
                        log['data'] = await self.s3.get(log['log_id'])
                    else:
                        raise Exception('no data field and s3 disabled')
            self.write({'logs':ret})
            self.finish()
<|endoftext|>