<commit_before># encoding: utf-8

import os
import httplib
import logging

import requests
from flask import request, make_response

from framework.auth import Auth
from framework.flask import redirect
from framework.exceptions import HTTPError
from framework.analytics import update_counter
from framework.auth.decorators import must_be_signed
from framework.transactions.handlers import no_auto_transaction

from website.models import User
from website.project.decorators import (
    must_be_contributor_or_public,
    must_not_be_registration, must_have_addon,
)
from website.util import rubeus

from website.addons.osfstorage import model
from website.addons.osfstorage import utils
from website.addons.osfstorage import errors
from website.addons.osfstorage import settings as osf_storage_settings

logger = logging.getLogger(__name__)

def make_error(code, message_short=None, message_long=None):
    data = {}
    if message_short:
        data['message_short'] = message_short
    if message_long:
        data['message_long'] = message_long
    return HTTPError(code, data=data)

def get_record_or_404(path, node_addon):
    record = model.OsfStorageFileRecord.find_by_path(path, node_addon)
    if record is not None:
        return record
    raise HTTPError(httplib.NOT_FOUND)

@must_be_signed
@must_have_addon('osfstorage', 'node')
def osf_storage_download_file_hook(node_addon, payload, **kwargs):
    try:
        path = payload['path']
    except KeyError:
        raise HTTPError(httplib.BAD_REQUEST)

    version_idx, version, record = get_version(path, node_addon, payload.get('version'))

    if payload.get('mode') != 'render':
        update_analytics(node_addon.owner, path, version_idx)

    return {
        'data': {
            'path': version.location_hash,
        },
        'settings': {
            osf_storage_settings.WATERBUTLER_RESOURCE: version.location[osf_storage_settings.WATERBUTLER_RESOURCE],
        },
    }

def osf_storage_crud_prepare(node_addon, payload):
    try:
        auth = payload['auth']
        settings = payload['settings']
        metadata = payload['metadata']
        hashes = payload['hashes']
        worker = payload['worker']
        path = payload['path'].strip('/')
    except KeyError:
        raise HTTPError(httplib.BAD_REQUEST)
    user = User.load(auth.get('id'))
    if user is None:
        raise HTTPError(httplib.BAD_REQUEST)
    location = settings
    location.update({
        'object': metadata['name'],
        'service': metadata['provider'],
    })
    # TODO: Migrate existing worker host and URL
    location.update(worker)
    metadata.update(hashes)
    return path, user, location, metadata

@must_be_signed
@no_auto_transaction
@must_have_addon('osfstorage', 'node')
def osf_storage_upload_file_hook(node_addon, payload, **kwargs):
    path, user, location, metadata = osf_storage_crud_prepare(node_addon, payload)
    record, created = model.OsfStorageFileRecord.get_or_create(path, node_addon)

    version = record.create_version(user, location, metadata)

    code = httplib.CREATED if created else httplib.OK

    return {
        'status': 'success',
        'version': version._id,
        'downloads': record.get_download_count(),
    }, code

@must_be_signed
@must_have_addon('osfstorage', 'node')
def osf_storage_update_metadata_hook(node_addon, payload, **kwargs):
    try:
        version_id = payload['version']
        metadata = payload['metadata']
    except KeyError:
        raise HTTPError(httplib.BAD_REQUEST)

    version = model.OsfStorageFileVersion.load(version_id)

    if version is None:
        raise HTTPError(httplib.NOT_FOUND)

    version.update_metadata(metadata)

    return {'status': 'success'}

@must_be_signed
@must_not_be_registration
@must_have_addon('osfstorage', 'node')
def osf_storage_crud_hook_delete(payload, node_addon, **kwargs):
    file_record = model.OsfStorageFileRecord.find_by_path(payload.get('path'), node_addon)

    if file_record is None:
        raise HTTPError(httplib.NOT_FOUND)

    try:
        auth = Auth(User.load(payload['auth'].get('id')))
        if not auth:
            raise HTTPError(httplib.BAD_REQUEST)

        file_record.delete(auth)
    except errors.DeleteError:
        raise HTTPError(httplib.NOT_FOUND)

    file_record.save()
    return {'status': 'success'}

def parse_version_specifier(version_str):
    """
    :raise: `InvalidVersionError` if version specifier cannot be parsed
    """
    try:
        version_idx = int(version_str)
    except (TypeError, ValueError):
        raise errors.InvalidVersionError
    if version_idx < 1:
        raise errors.InvalidVersionError
    return version_idx

def get_version_helper(file_record, version_str):
    """
    :return: Tuple of (version_index, file_version); note that index is one-based
    :raise: `HTTPError` if version specifier is invalid or version not found
    """
    if version_str is None:
        return (
            len(file_record.versions),
            file_record.versions[-1],
        )
    try:
        version_idx = parse_version_specifier(version_str)
    except errors.InvalidVersionError:
        raise make_error(httplib.BAD_REQUEST, 'Invalid version')
    try:
        return version_idx, file_record.versions[version_idx - 1]
    except IndexError:
        raise HTTPError(httplib.NOT_FOUND)

def get_version(path, node_settings, version_str, throw=True):
    """Resolve version from request arguments.

    :param str path: Path to file
    :param node_settings: Node settings record
    :param str version_str: Version from query string
    :param bool throw: Throw `HTTPError` if version is incomplete
    :return: Tuple of (<one-based version index>, <file version>, <file record>)
    """
    record = model.OsfStorageFileRecord.find_by_path(path, node_settings)

    if record is None:
        raise HTTPError(httplib.NOT_FOUND)

    if record.is_deleted:
        raise HTTPError(httplib.GONE)

    version_idx, file_version = get_version_helper(record, version_str)
    return version_idx, file_version, record

def download_file(path, node_addon, version_query, **query):
    idx, version, record = get_version(path, node_addon, version_query)
    url = utils.get_waterbutler_download_url(idx, version, record, **query)
    # Redirect the user directly to the backend service (CloudFiles or S3) rather than
    # routing through OSF; this saves a request and avoids potential CORS configuration
    # errors in WaterButler.
    resp = requests.get(url, allow_redirects=False)
    if resp.status_code in [301, 302]:
        return redirect(resp.headers['Location'])
    else:
        response = make_response(resp.content)
        filename = record.name.encode('utf-8')
        if version != record.versions[-1]:
            # add revision to filename
            # foo.mp3 -> foo-abc123.mp3
            filename = '-{}'.format(version.date_created.strftime('%Y-%m-%d')).join(os.path.splitext(filename))
        disposition = 'attachment; filename={}'.format(filename)
        response.headers['Content-Disposition'] = disposition
        response.headers['Content-Type'] = 'application/octet-stream'
        return response

@must_be_contributor_or_public
@must_have_addon('osfstorage', 'node')
def osf_storage_view_file(auth, path, node_addon, **kwargs):
    action = request.args.get('action', 'view')
    version_idx = request.args.get('version')
    if action == 'download':
        mode = request.args.get('mode')
        return download_file(path, node_addon, version_idx, mode=mode)
    # if action == 'view':
    #     return view_file(auth, path, node_addon, version_idx)
    raise HTTPError(httplib.BAD_REQUEST)

def update_analytics(node, path, version_idx):
    """
    :param Node node: Root node to update
    :param str path: Path to file
    :param int version_idx: One-based version index
    """
    update_counter(u'download:{0}:{1}'.format(node._id, path))
    update_counter(u'download:{0}:{1}:{2}'.format(node._id, path, version_idx))

@must_be_signed
@must_have_addon('osfstorage', 'node')
def osf_storage_get_metadata_hook(node_addon, payload, **kwargs):
    node = node_addon.owner
    path = payload.get('path', '')

    if path.endswith('/') or not path:
        file_tree = model.OsfStorageFileTree.find_by_path(path, node_addon)
        if file_tree is None:
            if path == '':
                return []
            raise HTTPError(httplib.NOT_FOUND)
        # TODO: Handle nested folders
        return [
            utils.serialize_metadata_hgrid(item, node)
            for item in list(file_tree.children)
            if not item.is_deleted
        ]
    else:
        file_record = model.OsfStorageFileRecord.find_by_path(path, node_addon)
        if not file_record:
            raise HTTPError(httplib.NOT_FOUND)

        if file_record.is_deleted:
            raise HTTPError(httplib.GONE)

        return utils.serialize_metadata_hgrid(file_record, node)

def osf_storage_root(node_settings, auth, **kwargs):
    """Build HGrid JSON for root node. Note: include node URLs for client-side
    URL creation for uploaded files.
    """
    node = node_settings.owner
    root = rubeus.build_addon_root(
        node_settings=node_settings,
        name='',
        permissions=auth,
        user=auth.user,
        nodeUrl=node.url,
        nodeApiUrl=node.api_url,
    )
    return [root]

@must_be_signed
@must_have_addon('osfstorage', 'node')
def osf_storage_get_revisions(payload, node_addon, **kwargs):
    node = node_addon.owner
    page = payload.get('page') or 0
    path = payload.get('path')

    if not path:
        raise HTTPError(httplib.BAD_REQUEST)

    try:
        page = int(page)
    except (TypeError, ValueError):
        raise HTTPError(httplib.BAD_REQUEST)

    record = model.OsfStorageFileRecord.find_by_path(path, node_addon)

    if record is None:
        raise HTTPError(httplib.NOT_FOUND)

    indices, versions, more = record.get_versions(
        page,
        size=osf_storage_settings.REVISIONS_PAGE_SIZE,
    )

    return {
        'revisions': [
            utils.serialize_revision(node, record, versions[idx], indices[idx])
            for idx in range(len(versions))
        ],
        'more': more,
    }
<commit_msg>pass is_anon to revision serializer to exclude users when need be<commit_after># encoding: utf-8

import os
import httplib
import logging

import requests
from flask import request, make_response

from framework.auth import Auth
from framework.flask import redirect
from framework.exceptions import HTTPError
from framework.analytics import update_counter
from framework.auth.decorators import must_be_signed
from framework.transactions.handlers import no_auto_transaction

from website.models import User
from website.project.decorators import (
    must_be_contributor_or_public,
    must_not_be_registration, must_have_addon,
)
from website.util import rubeus
from website.project.model import has_anonymous_link

from website.addons.osfstorage import model
from website.addons.osfstorage import utils
from website.addons.osfstorage import errors
from website.addons.osfstorage import settings as osf_storage_settings

logger = logging.getLogger(__name__)

def make_error(code, message_short=None, message_long=None):
    data = {}
    if message_short:
        data['message_short'] = message_short
    if message_long:
        data['message_long'] = message_long
    return HTTPError(code, data=data)

def get_record_or_404(path, node_addon):
    record = model.OsfStorageFileRecord.find_by_path(path, node_addon)
    if record is not None:
        return record
    raise HTTPError(httplib.NOT_FOUND)

@must_be_signed
@must_have_addon('osfstorage', 'node')
def osf_storage_download_file_hook(node_addon, payload, **kwargs):
    try:
        path = payload['path']
    except KeyError:
        raise HTTPError(httplib.BAD_REQUEST)

    version_idx, version, record = get_version(path, node_addon, payload.get('version'))

    if payload.get('mode') != 'render':
        update_analytics(node_addon.owner, path, version_idx)

    return {
        'data': {
            'path': version.location_hash,
        },
        'settings': {
            osf_storage_settings.WATERBUTLER_RESOURCE: version.location[osf_storage_settings.WATERBUTLER_RESOURCE],
        },
    }

def osf_storage_crud_prepare(node_addon, payload):
    try:
        auth = payload['auth']
        settings = payload['settings']
        metadata = payload['metadata']
        hashes = payload['hashes']
        worker = payload['worker']
        path = payload['path'].strip('/')
    except KeyError:
        raise HTTPError(httplib.BAD_REQUEST)
    user = User.load(auth.get('id'))
    if user is None:
        raise HTTPError(httplib.BAD_REQUEST)
    location = settings
    location.update({
        'object': metadata['name'],
        'service': metadata['provider'],
    })
    # TODO: Migrate existing worker host and URL
    location.update(worker)
    metadata.update(hashes)
    return path, user, location, metadata

@must_be_signed
@no_auto_transaction
@must_have_addon('osfstorage', 'node')
def osf_storage_upload_file_hook(node_addon, payload, **kwargs):
    path, user, location, metadata = osf_storage_crud_prepare(node_addon, payload)
    record, created = model.OsfStorageFileRecord.get_or_create(path, node_addon)

    version = record.create_version(user, location, metadata)

    code = httplib.CREATED if created else httplib.OK

    return {
        'status': 'success',
        'version': version._id,
        'downloads': record.get_download_count(),
    }, code

@must_be_signed
@must_have_addon('osfstorage', 'node')
def osf_storage_update_metadata_hook(node_addon, payload, **kwargs):
    try:
        version_id = payload['version']
        metadata = payload['metadata']
    except KeyError:
        raise HTTPError(httplib.BAD_REQUEST)

    version = model.OsfStorageFileVersion.load(version_id)

    if version is None:
        raise HTTPError(httplib.NOT_FOUND)

    version.update_metadata(metadata)

    return {'status': 'success'}

@must_be_signed
@must_not_be_registration
@must_have_addon('osfstorage', 'node')
def osf_storage_crud_hook_delete(payload, node_addon, **kwargs):
    file_record = model.OsfStorageFileRecord.find_by_path(payload.get('path'), node_addon)

    if file_record is None:
        raise HTTPError(httplib.NOT_FOUND)

    try:
        auth = Auth(User.load(payload['auth'].get('id')))
        if not auth:
            raise HTTPError(httplib.BAD_REQUEST)

        file_record.delete(auth)
    except errors.DeleteError:
        raise HTTPError(httplib.NOT_FOUND)

    file_record.save()
    return {'status': 'success'}

def parse_version_specifier(version_str):
    """
    :raise: `InvalidVersionError` if version specifier cannot be parsed
    """
    try:
        version_idx = int(version_str)
    except (TypeError, ValueError):
        raise errors.InvalidVersionError
    if version_idx < 1:
        raise errors.InvalidVersionError
    return version_idx

def get_version_helper(file_record, version_str):
    """
    :return: Tuple of (version_index, file_version); note that index is one-based
    :raise: `HTTPError` if version specifier is invalid or version not found
    """
    if version_str is None:
        return (
            len(file_record.versions),
            file_record.versions[-1],
        )
    try:
        version_idx = parse_version_specifier(version_str)
    except errors.InvalidVersionError:
        raise make_error(httplib.BAD_REQUEST, 'Invalid version')
    try:
        return version_idx, file_record.versions[version_idx - 1]
    except IndexError:
        raise HTTPError(httplib.NOT_FOUND)

def get_version(path, node_settings, version_str, throw=True):
    """Resolve version from request arguments.

    :param str path: Path to file
    :param node_settings: Node settings record
    :param str version_str: Version from query string
    :param bool throw: Throw `HTTPError` if version is incomplete
    :return: Tuple of (<one-based version index>, <file version>, <file record>)
    """
    record = model.OsfStorageFileRecord.find_by_path(path, node_settings)

    if record is None:
        raise HTTPError(httplib.NOT_FOUND)

    if record.is_deleted:
        raise HTTPError(httplib.GONE)

    version_idx, file_version = get_version_helper(record, version_str)
    return version_idx, file_version, record

def download_file(path, node_addon, version_query, **query):
    idx, version, record = get_version(path, node_addon, version_query)
    url = utils.get_waterbutler_download_url(idx, version, record, **query)
    # Redirect the user directly to the backend service (CloudFiles or S3) rather than
    # routing through OSF; this saves a request and avoids potential CORS configuration
    # errors in WaterButler.
    resp = requests.get(url, allow_redirects=False)
    if resp.status_code in [301, 302]:
        return redirect(resp.headers['Location'])
    else:
        response = make_response(resp.content)
        filename = record.name.encode('utf-8')
        if version != record.versions[-1]:
            # add revision to filename
            # foo.mp3 -> foo-abc123.mp3
            filename = '-{}'.format(version.date_created.strftime('%Y-%m-%d')).join(os.path.splitext(filename))
        disposition = 'attachment; filename={}'.format(filename)
        response.headers['Content-Disposition'] = disposition
        response.headers['Content-Type'] = 'application/octet-stream'
        return response

@must_be_contributor_or_public
@must_have_addon('osfstorage', 'node')
def osf_storage_view_file(auth, path, node_addon, **kwargs):
    action = request.args.get('action', 'view')
    version_idx = request.args.get('version')
    if action == 'download':
        mode = request.args.get('mode')
        return download_file(path, node_addon, version_idx, mode=mode)
    # if action == 'view':
    #     return view_file(auth, path, node_addon, version_idx)
    raise HTTPError(httplib.BAD_REQUEST)

def update_analytics(node, path, version_idx):
    """
    :param Node node: Root node to update
    :param str path: Path to file
    :param int version_idx: One-based version index
    """
    update_counter(u'download:{0}:{1}'.format(node._id, path))
    update_counter(u'download:{0}:{1}:{2}'.format(node._id, path, version_idx))

@must_be_signed
@must_have_addon('osfstorage', 'node')
def osf_storage_get_metadata_hook(node_addon, payload, **kwargs):
    node = node_addon.owner
    path = payload.get('path', '')

    if path.endswith('/') or not path:
        file_tree = model.OsfStorageFileTree.find_by_path(path, node_addon)
        if file_tree is None:
            if path == '':
                return []
            raise HTTPError(httplib.NOT_FOUND)
        # TODO: Handle nested folders
        return [
            utils.serialize_metadata_hgrid(item, node)
            for item in list(file_tree.children)
            if not item.is_deleted
        ]
    else:
        file_record = model.OsfStorageFileRecord.find_by_path(path, node_addon)
        if not file_record:
            raise HTTPError(httplib.NOT_FOUND)

        if file_record.is_deleted:
            raise HTTPError(httplib.GONE)

        return utils.serialize_metadata_hgrid(file_record, node)

def osf_storage_root(node_settings, auth, **kwargs):
    """Build HGrid JSON for root node. Note: include node URLs for client-side
    URL creation for uploaded files.
    """
    node = node_settings.owner
    root = rubeus.build_addon_root(
        node_settings=node_settings,
        name='',
        permissions=auth,
        user=auth.user,
        nodeUrl=node.url,
        nodeApiUrl=node.api_url,
    )
    return [root]

@must_be_signed
@must_have_addon('osfstorage', 'node')
def osf_storage_get_revisions(payload, node_addon, **kwargs):
    node = node_addon.owner
    page = payload.get('page') or 0
    path = payload.get('path')
    is_anon = has_anonymous_link(node, Auth(private_key=payload.get('view_only')))

    if not path:
        raise HTTPError(httplib.BAD_REQUEST)

    try:
        page = int(page)
    except (TypeError, ValueError):
        raise HTTPError(httplib.BAD_REQUEST)

    record = model.OsfStorageFileRecord.find_by_path(path, node_addon)

    if record is None:
        raise HTTPError(httplib.NOT_FOUND)

    indices, versions, more = record.get_versions(
        page,
        size=osf_storage_settings.REVISIONS_PAGE_SIZE,
    )

    return {
        'revisions': [
            utils.serialize_revision(node, record, versions[idx], indices[idx], anon=is_anon)
            for idx in range(len(versions))
        ],
        'more': more,
    }
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
import sys

sys.path.insert(0, "src")
sys.path.insert(0, "../src")

from attribute_wrapper import GenericWrapper

# Tests =======================================================================
class PrintWrapper(GenericWrapper):
    def download_handler(self, method, url, data):
        return "%s:%s:%s" % (method, url, str(data))

def test_wrapper():
    p = PrintWrapper("print")

    assert p.attribute.get() == "get:print/attribute:None"
<commit_msg>#1: PrintWrapper changed to ReturnWrapper. Added test_http_url().<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
import sys

sys.path.insert(0, "src")
sys.path.insert(0, "../src")

from attribute_wrapper import GenericWrapper

# Classes =====================================================================
class ReturnWrapper(GenericWrapper):
    def download_handler(self, method, url, data):
        return (method, url, data)

# Tests =======================================================================
def test_wrapper():
    p = ReturnWrapper("return")

    assert p.attribute.get() == ("get", "return/attribute", None)

def test_http_url():
    p = ReturnWrapper("http://kitakitsune.org")

    assert p.get() == ("get", "http://kitakitsune.org", None)
    assert p.attr.get() == ("get", "http://kitakitsune.org/attr", None)
<|endoftext|>


<commit_before>##
# copyright 2009, James William Pye
# http://python.projects.postgresql.org
##
# Copy I/O: To and From performance
##
import os, sys, random, time

if __name__ == '__main__':
	with open('/usr/share/dict/words', mode='brU') as wordfile:
		Words = wordfile.readlines()
else:
	Words = [b'/usr/share/dict/words', b'is', b'read', b'in', b'__main__']
wordcount = len(Words)
random.seed()

def getWord():
	"extract a random word from ``Words``"
	return Words[random.randrange(0, wordcount)].strip()

def testSpeed(tuples = 50000 * 3):
	execute("CREATE TEMP TABLE _copy "
	"(i int, t text, mt text, ts text, ty text, tx text);")
	try:
		Q = prepare("COPY _copy FROM STDIN")
		size = [0]
		def incsize(data):
			'count of bytes'
			size[0] += len(data)
			return data
		sys.stderr.write("preparing data(%d tuples)...\n" %(tuples,))

		# Use an LC to avoid the Python overhead involved with a GE
		data = [incsize(b'\t'.join((
			str(x).encode('ascii'), getWord(), getWord(),
			getWord(), getWord(), getWord()
		)))+b'\n' for x in range(tuples)]

		sys.stderr.write("starting copy...\n")
		start = time.time()
		copied_in = Q.load(data)
		duration = time.time() - start
		sys.stderr.write(
			"COPY FROM STDIN Summary,\n " \
			"copied tuples: %d\n " \
			"copied bytes: %d\n " \
			"duration: %f\n " \
			"average tuple size(bytes): %f\n " \
			"average KB per second: %f\n " \
			"average tuples per second: %f\n" %(
				tuples, size[0], duration,
				size[0] / tuples,
				size[0] / 1024 / duration,
				tuples / duration,
			)
		)
		Q = prepare("COPY _copy TO STDOUT")
		start = time.time()
		c = 0
		for rows in Q().chunks:
			c += len(rows)
		duration = time.time() - start
		sys.stderr.write(
			"COPY TO STDOUT Summary,\n " \
			"copied tuples: %d\n " \
			"duration: %f\n " \
			"average KB per second: %f\n " \
			"average tuples per second: %f\n " %(
				c, duration,
				size[0] / 1024 / duration,
				tuples / duration,
			)
		)
	finally:
		execute("DROP TABLE _copy")

if __name__ == '__main__':
	testSpeed()
<commit_msg>Use the new chunks() interface.<commit_after>##
# copyright 2009, James William Pye
# http://python.projects.postgresql.org
##
# Copy I/O: To and From performance
##
import os, sys, random, time

if __name__ == '__main__':
	with open('/usr/share/dict/words', mode='brU') as wordfile:
		Words = wordfile.readlines()
else:
	Words = [b'/usr/share/dict/words', b'is', b'read', b'in', b'__main__']
wordcount = len(Words)
random.seed()

def getWord():
	"extract a random word from ``Words``"
	return Words[random.randrange(0, wordcount)].strip()

def testSpeed(tuples = 50000 * 3):
	execute("CREATE TEMP TABLE _copy "
	"(i int, t text, mt text, ts text, ty text, tx text);")
	try:
		Q = prepare("COPY _copy FROM STDIN")
		size = [0]
		def incsize(data):
			'count of bytes'
			size[0] += len(data)
			return data
		sys.stderr.write("preparing data(%d tuples)...\n" %(tuples,))

		# Use an LC to avoid the Python overhead involved with a GE
		data = [incsize(b'\t'.join((
			str(x).encode('ascii'), getWord(), getWord(),
			getWord(), getWord(), getWord()
		)))+b'\n' for x in range(tuples)]

		sys.stderr.write("starting copy...\n")
		start = time.time()
		copied_in = Q.load(data)
		duration = time.time() - start
		sys.stderr.write(
			"COPY FROM STDIN Summary,\n " \
			"copied tuples: %d\n " \
			"copied bytes: %d\n " \
			"duration: %f\n " \
			"average tuple size(bytes): %f\n " \
			"average KB per second: %f\n " \
			"average tuples per second: %f\n" %(
				tuples, size[0], duration,
				size[0] / tuples,
				size[0] / 1024 / duration,
				tuples / duration,
			)
		)
		Q = prepare("COPY _copy TO STDOUT")
		start = time.time()
		c = 0
		for rows in Q.chunks():
			c += len(rows)
		duration = time.time() - start
		sys.stderr.write(
			"COPY TO STDOUT Summary,\n " \
			"copied tuples: %d\n " \
			"duration: %f\n " \
			"average KB per second: %f\n " \
			"average tuples per second: %f\n " %(
				c, duration,
				size[0] / 1024 / duration,
				tuples / duration,
			)
		)
	finally:
		execute("DROP TABLE _copy")

if __name__ == '__main__':
	testSpeed()
<|endoftext|>


<commit_before># Copyright (C) 2007-2012 Michael Foord & the mock team
# E-mail: fuzzyman AT voidspace DOT org DOT uk
# http://www.voidspace.org.uk/python/mock/

import unittest2 as unittest
from tests.support import is_instance, X, SomeClass

from mock import (
    Mock, MagicMock, NonCallableMagicMock,
    NonCallableMock, patch, create_autospec,
    CallableMixin
)

class TestCallable(unittest.TestCase):

    def assertNotCallable(self, mock):
        self.assertTrue(is_instance(mock, NonCallableMagicMock))
        self.assertFalse(is_instance(mock, CallableMixin))

    def test_non_callable(self):
        for mock in NonCallableMagicMock(), NonCallableMock():
            self.assertRaises(TypeError, mock)
            self.assertFalse(hasattr(mock, '__call__'))
            self.assertIn(mock.__class__.__name__, repr(mock))

    def test_heirarchy(self):
        self.assertTrue(issubclass(MagicMock, Mock))
        self.assertTrue(issubclass(NonCallableMagicMock, NonCallableMock))

    def test_attributes(self):
        one = NonCallableMock()
        self.assertTrue(issubclass(type(one.one), Mock))

        two = NonCallableMagicMock()
        self.assertTrue(issubclass(type(two.two), MagicMock))

    def test_subclasses(self):
        class MockSub(Mock):
            pass

        one = MockSub()
        self.assertTrue(issubclass(type(one.one), MockSub))

        class MagicSub(MagicMock):
            pass

        two = MagicSub()
        self.assertTrue(issubclass(type(two.two), MagicSub))

    def test_patch_spec(self):
        patcher = patch('%s.X' % __name__, spec=True)
        mock = patcher.start()
        self.addCleanup(patcher.stop)

        instance = mock()
        mock.assert_called_once_with()

        self.assertNotCallable(instance)
        self.assertRaises(TypeError, instance)

    def test_patch_spec_set(self):
        patcher = patch('%s.X' % __name__, spec_set=True)
        mock = patcher.start()
        self.addCleanup(patcher.stop)

        instance = mock()
        mock.assert_called_once_with()

        self.assertNotCallable(instance)
        self.assertRaises(TypeError, instance)

    def test_patch_spec_instance(self):
        patcher = patch('%s.X' % __name__, spec=X())
        mock = patcher.start()
        self.addCleanup(patcher.stop)

        self.assertNotCallable(mock)
        self.assertRaises(TypeError, mock)

    def test_patch_spec_set_instance(self):
        patcher = patch('%s.X' % __name__, spec_set=X())
        mock = patcher.start()
        self.addCleanup(patcher.stop)

        self.assertNotCallable(mock)
        self.assertRaises(TypeError, mock)

    def test_patch_spec_callable_class(self):
        class CallableX(X):
            def __call__(self):
                pass

        class Sub(CallableX):
            pass

        class Multi(SomeClass, Sub):
            pass

        class OldStyle:
            def __call__(self):
                pass

        class OldStyleSub(OldStyle):
            pass

        for arg in 'spec', 'spec_set':
            for Klass in CallableX, Sub, Multi, OldStyle, OldStyleSub:
                with patch('%s.X' % __name__, **{arg: Klass}) as mock:
                    instance = mock()
                    mock.assert_called_once_with()

                    self.assertTrue(is_instance(instance, MagicMock))
                    # inherited spec
                    self.assertRaises(AttributeError, getattr, instance,
                                      'foobarbaz')

                    result = instance()
                    # instance is callable, result has no spec
                    instance.assert_called_once_with()

                    result(3, 2, 1)
                    result.assert_called_once_with(3, 2, 1)
                    result.foo(3, 2, 1)
                    result.foo.assert_called_once_with(3, 2, 1)

    def test_create_autospec(self):
        mock = create_autospec(X)
        instance = mock()
        self.assertRaises(TypeError, instance)

        mock = create_autospec(X())
        self.assertRaises(TypeError, mock)

    def test_create_autospec_instance(self):
        mock = create_autospec(SomeClass, instance=True)

        self.assertRaises(TypeError, mock)
        mock.wibble()
        mock.wibble.assert_called_once_with()

        self.assertRaises(TypeError, mock.wibble, 'some',  'args')
<commit_msg>Issue #19013: add unittest.main() epilogs to unittest.mock's own test modules<commit_after># Copyright (C) 2007-2012 Michael Foord & the mock team
# E-mail: fuzzyman AT voidspace DOT org DOT uk
# http://www.voidspace.org.uk/python/mock/

import unittest2 as unittest
from tests.support import is_instance, X, SomeClass

from mock import (
    Mock, MagicMock, NonCallableMagicMock,
    NonCallableMock, patch, create_autospec,
    CallableMixin
)

class TestCallable(unittest.TestCase):

    def assertNotCallable(self, mock):
        self.assertTrue(is_instance(mock, NonCallableMagicMock))
        self.assertFalse(is_instance(mock, CallableMixin))

    def test_non_callable(self):
        for mock in NonCallableMagicMock(), NonCallableMock():
            self.assertRaises(TypeError, mock)
            self.assertFalse(hasattr(mock, '__call__'))
            self.assertIn(mock.__class__.__name__, repr(mock))

    def test_heirarchy(self):
        self.assertTrue(issubclass(MagicMock, Mock))
        self.assertTrue(issubclass(NonCallableMagicMock, NonCallableMock))

    def test_attributes(self):
        one = NonCallableMock()
        self.assertTrue(issubclass(type(one.one), Mock))

        two = NonCallableMagicMock()
        self.assertTrue(issubclass(type(two.two), MagicMock))

    def test_subclasses(self):
        class MockSub(Mock):
            pass

        one = MockSub()
        self.assertTrue(issubclass(type(one.one), MockSub))

        class MagicSub(MagicMock):
            pass

        two = MagicSub()
        self.assertTrue(issubclass(type(two.two), MagicSub))

    def test_patch_spec(self):
        patcher = patch('%s.X' % __name__, spec=True)
        mock = patcher.start()
        self.addCleanup(patcher.stop)

        instance = mock()
        mock.assert_called_once_with()

        self.assertNotCallable(instance)
        self.assertRaises(TypeError, instance)

    def test_patch_spec_set(self):
        patcher = patch('%s.X' % __name__, spec_set=True)
        mock = patcher.start()
        self.addCleanup(patcher.stop)

        instance = mock()
        mock.assert_called_once_with()

        self.assertNotCallable(instance)
        self.assertRaises(TypeError, instance)

    def test_patch_spec_instance(self):
        patcher = patch('%s.X' % __name__, spec=X())
        mock = patcher.start()
        self.addCleanup(patcher.stop)

        self.assertNotCallable(mock)
        self.assertRaises(TypeError, mock)

    def test_patch_spec_set_instance(self):
        patcher = patch('%s.X' % __name__, spec_set=X())
        mock = patcher.start()
        self.addCleanup(patcher.stop)

        self.assertNotCallable(mock)
        self.assertRaises(TypeError, mock)

    def test_patch_spec_callable_class(self):
        class CallableX(X):
            def __call__(self):
                pass

        class Sub(CallableX):
            pass

        class Multi(SomeClass, Sub):
            pass

        class OldStyle:
            def __call__(self):
                pass

        class OldStyleSub(OldStyle):
            pass

        for arg in 'spec', 'spec_set':
            for Klass in CallableX, Sub, Multi, OldStyle, OldStyleSub:
                with patch('%s.X' % __name__, **{arg: Klass}) as mock:
                    instance = mock()
                    mock.assert_called_once_with()

                    self.assertTrue(is_instance(instance, MagicMock))
                    # inherited spec
                    self.assertRaises(AttributeError, getattr, instance,
                                      'foobarbaz')

                    result = instance()
                    # instance is callable, result has no spec
                    instance.assert_called_once_with()

                    result(3, 2, 1)
                    result.assert_called_once_with(3, 2, 1)
                    result.foo(3, 2, 1)
                    result.foo.assert_called_once_with(3, 2, 1)

    def test_create_autospec(self):
        mock = create_autospec(X)
        instance = mock()
        self.assertRaises(TypeError, instance)

        mock = create_autospec(X())
        self.assertRaises(TypeError, mock)

    def test_create_autospec_instance(self):
        mock = create_autospec(SomeClass, instance=True)

        self.assertRaises(TypeError, mock)
        mock.wibble()
        mock.wibble.assert_called_once_with()

        self.assertRaises(TypeError, mock.wibble, 'some',  'args')

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>from math import pi
from numpy import arange, sin, zeros
from unit_timeside import *

from timeside.encoder.gstutils import get_loop_thread

import os.path

class TestEncoding(TestCase):
    "Test the low level streaming features"

    def setUp(self):
        self.channels = 1
        self.samplerate = 44100
        import tempfile
        self.tmpfile = tempfile.NamedTemporaryFile(delete=True)
        self.sink = self.tmpfile.name

    def testWav(self):
        "Test wav encoding"
        from timeside.encoder.wav import WavEncoder
        self.encoder = WavEncoder(self.sink)

    def testVorbis(self):
        "Test vorbis encoding"
        from timeside.encoder.ogg import VorbisEncoder
        self.encoder = VorbisEncoder(self.sink)

    def testMp3(self):
        "Test mp3 encoding"
        from timeside.encoder.mp3 import Mp3Encoder
        self.encoder = Mp3Encoder(self.sink)

    def testAac(self):
        "Test aac encoding"
        from timeside.encoder.m4a import AacEncoder
        self.encoder = AacEncoder(self.sink)

    def testFlac(self):
        "Test flac encoding"
        from timeside.encoder.flac import FlacEncoder
        self.encoder = FlacEncoder(self.sink)

    def testWebm(self):
        "Test webm encoding"
        from timeside.encoder.webm import WebMEncoder
        self.encoder = WebMEncoder(self.sink)

    def tearDown(self):

        self.encoder.setup(channels = self.channels, samplerate = self.samplerate)

        written_frames, eod = 0, False
        duration = 3. * self.samplerate
        block_size = 1024
        f0 = 800.
        omega = 2. * pi * f0 / float(self.samplerate)

        while True:
            remaining = duration - written_frames
            if remaining >= block_size:
              write_length = block_size
            else:
              write_length = remaining
              eod = True
            # build a sinusoid
            frames = .75 * sin( omega * (arange(write_length) + written_frames) )
            # process encoder, writing to file
            self.encoder.process(frames, eod)
            written_frames += frames.shape[0]
            if eod:
                break
            if self.encoder.eod :
                break

        self.tmpfile.close()

if __name__ == '__main__':
    unittest.main(testRunner=TestRunner())

<commit_msg>tests/testencoding.py: use total_frames, not duration, remove trailing spaces<commit_after>from math import pi
from numpy import arange, sin, zeros
from unit_timeside import *

from timeside.encoder.gstutils import get_loop_thread

import os.path

class TestEncoding(TestCase):
    "Test the low level streaming features"

    def setUp(self):
        self.channels = 1
        self.samplerate = 44100
        import tempfile
        self.tmpfile = tempfile.NamedTemporaryFile(delete=True)
        self.sink = self.tmpfile.name

    def testWav(self):
        "Test wav encoding"
        from timeside.encoder.wav import WavEncoder
        self.encoder = WavEncoder(self.sink)

    def testVorbis(self):
        "Test vorbis encoding"
        from timeside.encoder.ogg import VorbisEncoder
        self.encoder = VorbisEncoder(self.sink)

    def testMp3(self):
        "Test mp3 encoding"
        from timeside.encoder.mp3 import Mp3Encoder
        self.encoder = Mp3Encoder(self.sink)

    def testAac(self):
        "Test aac encoding"
        from timeside.encoder.m4a import AacEncoder
        self.encoder = AacEncoder(self.sink)

    def testFlac(self):
        "Test flac encoding"
        from timeside.encoder.flac import FlacEncoder
        self.encoder = FlacEncoder(self.sink)

    def testWebm(self):
        "Test webm encoding"
        from timeside.encoder.webm import WebMEncoder
        self.encoder = WebMEncoder(self.sink)

    def tearDown(self):

        self.encoder.setup(channels = self.channels, samplerate = self.samplerate)

        written_frames, eod = 0, False
        total_frames = 3. * self.samplerate
        block_size = 1024
        f0 = 800.
        omega = 2. * pi * f0 / float(self.samplerate)

        while True:
            remaining = total_frames - written_frames
            if remaining >= block_size:
              write_length = block_size
            else:
              write_length = remaining
              eod = True
            # build a sinusoid
            frames = .75 * sin( omega * (arange(write_length) + written_frames) )
            # process encoder, writing to file
            self.encoder.process(frames, eod)
            written_frames += frames.shape[0]
            if eod:
                self.assertEquals(self.encoder.eod, eod)
                break

        self.assertEquals(written_frames, total_frames)
        self.tmpfile.close()

if __name__ == '__main__':
    unittest.main(testRunner=TestRunner())

<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import sitenco

try:
    import wdb.ext
except ImportError:
    pass
else:
    wdb.ext.add_w_builtin()

if __name__ == '__main__':
    arg_parser = argparse.ArgumentParser()
    arg_parser.add_argument('project', nargs='?', help='project name')
    arg_parser.add_argument('--cache', action='store_true', help='use cache')
    arg_parser.add_argument('--port', action='store', type=int,
                            default=5000, help='server port')
    args = arg_parser.parse_args()
    sitenco.PROJECT_NAME = getattr(args, 'project')

    try:
        import wdb.ext
    except ImportError:
        pass
    else:
        sitenco.app.wsgi_app = wdb.ext.WdbMiddleware(
            sitenco.app.wsgi_app, start_disabled=True)
    if not args.cache:
        sitenco.cache.set_cache = lambda cache, key, value: None
    sitenco.app.run(host='0.0.0.0', debug=True, port=args.port)
<commit_msg>Fix sitenco.py<commit_after>#!/usr/bin/env python3

import argparse
import sitenco

try:
    import wdb.ext
except ImportError:
    pass
else:
    wdb.ext.add_w_builtin()

app = sitenco.app

if __name__ == '__main__':
    arg_parser = argparse.ArgumentParser()
    arg_parser.add_argument('project', nargs='?', help='project name')
    arg_parser.add_argument('--cache', action='store_true', help='use cache')
    arg_parser.add_argument('--port', action='store', type=int,
                            default=5000, help='server port')
    args = arg_parser.parse_args()
    sitenco.PROJECT_NAME = getattr(args, 'project')

    try:
        import wdb.ext
    except ImportError:
        pass
    else:
        app.wsgi_app = wdb.ext.WdbMiddleware(
            app.wsgi_app, start_disabled=True)
    if not args.cache:
        sitenco.cache.set_cache = lambda cache, key, value: None
    app.run(host='0.0.0.0', debug=True, port=args.port)
<|endoftext|>


<commit_before>"""
The Gentoo module for running tests on builds from the portage tree.

This will install a stage3 image of gentoo together with a recent snapshot
of the portage tree. For building / executing arbitrary projects successfully it
is necessary to keep the installed image as close to the host system as
possible.
In order to speed up your experience, you can replace the stage3 image that
we pull from the distfiles mirror with a new image that contains all necessary
dependencies for your experiments. Make sure you update the hash alongside
the gentoo image in pprof's source directory.

The following packages are required to run GentooGroup:
    * fakeroot
"""
from os import path
from plumbum.cmd import cp, tar, mv, fakeroot, rm  # pylint: disable=E0401
from plumbum.cmd import mkdir, curl, cut, tail  # pylint: disable=E0401
from plumbum import local
from pprof.project import Project
from pprof.utils.run import run, uchroot
from pprof.utils.downloader import Wget
from lazy import lazy

def latest_src_uri():
    """
    Get the latest src_uri for a stage 3 tarball.

    Returns (str):
        Latest src_uri from gentoo's distfiles mirror.
    """
    from plumbum import ProcessExecutionError
    from logging import error

    latest_txt = "http://distfiles.gentoo.org/releases/amd64/autobuilds/"\
            "latest-stage3-amd64.txt"
    try:
        src_uri = (curl[latest_txt] | tail["-n", "+3"]
                   | cut["-f1", "-d "])().strip()
    except ProcessExecutionError as proc_ex:
        src_uri = "NOT-FOUND"
        error("Could not determine latest stage3 src uri: {0}", str(proc_ex))
    return src_uri

class GentooGroup(Project):
    """
    Gentoo ProjectGroup is the base class for every portage build.
    """
    GROUP = 'gentoo'

    def __init__(self, exp):
        super(GentooGroup, self).__init__(exp, "gentoo")

    src_dir = "gentoo"
    src_file = src_dir + ".tar.bz2"

    @lazy
    def src_uri(self):  # pylint: disable=R0201
        """Fetches the lates src_uri from the gentoo mirrors."""
        return "http://distfiles.gentoo.org/releases/amd64/autobuilds/{0}" \
                .format(latest_src_uri())

    # download location for portage files
    src_uri_portage = "ftp://sunsite.informatik.rwth-aachen.de/pub/Linux/"\
                    "gentoo/snapshots/portage-latest.tar.bz2"
    src_file_portage = "portage_snap.tar.bz2"

    def build(self):
        pass

    def download(self):
        from pprof.settings import config
        with local.cwd(self.builddir):
            Wget(self.src_uri, self.src_file)

            cp(config["sourcedir"] + "/bin/uchroot", "uchroot")
            run(fakeroot["tar", "xfj", self.src_file])
            rm(self.src_file)
            with local.cwd(self.builddir + "/usr"):
                Wget(self.src_uri_portage, self.src_file_portage)
                run(tar["xfj", self.src_file_portage])
                rm(self.src_file_portage)

    def configure(self):
        with local.cwd(self.builddir):
            with open("etc/portage/make.conf", 'w') as makeconf:
                lines = '''
PATH="/llvm/bin:/pprof/bin:${PATH}"
LD_LIBRARY_PATH="/llvm/lib:/pprof/lib:${LD_LIBRARY_PATH}"
CFLAGS="-O2 -pipe"
CXXFLAGS="${CFLAGS}"
FEATURES="-sandbox -usersandbox -usersync -xattr"
CC="/llvm/bin/clang"
CXX="/llvm/bin/clang++"

PORTAGE_USERNAME = "root"
PORTAGE_GRPNAME = "root"
PORTAGE_INST_GID = 0
PORTAGE_INST_UID = 0

CHOST="x86_64-pc-linux-gnu"
USE="bindist mmx sse sse2"
PORTDIR="/usr/portage"
DISTDIR="${PORTDIR}/distfiles"
PKGDIR="${PORTDIR}/packages"
'''

                makeconf.write(lines)

            mkdir("-p", "etc/portage/metadata")
            with open("etc/portage/metadata/layout.conf", 'w') as layoutconf:
                lines = '''masters = gentoo'''
                layoutconf.write(lines)
            cp("/etc/resolv.conf", "etc/resolv.conf")

class PrepareStage3(GentooGroup):
    """
    A project that can be used for interactive stage3 generation.
    """
    NAME = "stage3"
    DOMAIN = "debug"

    def build(self):
        import sys
        # Don't do something when running non-interactive.
        if not sys.stdout.isatty():
            return

        from plumbum import FG
        from pprof.utils.downloader import update_hash
        from logging import info
        from pprof.settings import config

        root = config["tmpdir"]
        src_file = self.src_file + ".new"
        with local.cwd(self.builddir):
            bash_in_uchroot = uchroot()["/bin/bash"]
            print("Entering User-Chroot. Prepare your image and "
                  "type 'exit' when you are done.")
            bash_in_uchroot & FG  # pylint: disable=W0104
            tgt_path = path.join(root, self.src_file)
            tgt_path_new = path.join(root, src_file)
            print("Packing new stage3 image. "
                  "This will replace the original one at: ", tgt_path)
            tar("cjf", tgt_path_new, ".")
            update_hash(src_file, root)
            mv(path.join(root, src_file), tgt_path)

    def run_tests(self, experiment):
        pass

class AutoPrepareStage3(GentooGroup):
    """
    A project that can be used to install pprof in the stage3 archive.
    """
    NAME = "auto-stage3"
    DOMAIN = "debug"

    def build(self):
        import sys
        # Don't do something when running non-interactive.
        if not sys.stdout.isatty():
            return

        from plumbum import FG
        from pprof.utils.downloader import update_hash
        from logging import info
        from pprof.settings import config

        root = config["tmpdir"]
        src_file = self.src_file + ".new"
        with local.cwd(self.builddir):
            mkdir("-p", "pprof-src")
            w_pprof_src = uchroot("-m", "{}:pprof-src".format(config[
                "sourcedir"]))
            pip_in_uchroot = w_pprof_src["/usr/bin/pip3"]
            pip_in_uchroot["install", "--upgrade", "/pprof-src/"] & FG

            tgt_path = path.join(root, self.src_file)
            tgt_path_new = path.join(root, src_file)
            tar("cjf", tgt_path_new, ".")
            update_hash(src_file, root)
            mv(path.join(root, src_file), tgt_path)

    def run_tests(self, experiment):
        pass

class Eix(GentooGroup):
    """
    Represents the package eix from the portage tree.

    Building this class will create bare gentoo and compile eix.
    """
    NAME = 'eix'
    DOMAIN = 'debug'

    def build(self):
        with local.cwd(self.builddir):
            emerge_in_chroot = uchroot()["/usr/bin/emerge"]
            run(emerge_in_chroot["eix"])

    def run_tests(self, experiment):
        pass
<commit_msg>gentoo: use wrap_{cc,cxx}_in_uchroot for all gentoo projects<commit_after>"""
The Gentoo module for running tests on builds from the portage tree.

This will install a stage3 image of gentoo together with a recent snapshot
of the portage tree. For building / executing arbitrary projects successfully it
is necessary to keep the installed image as close to the host system as
possible.
In order to speed up your experience, you can replace the stage3 image that
we pull from the distfiles mirror with a new image that contains all necessary
dependencies for your experiments. Make sure you update the hash alongside
the gentoo image in pprof's source directory.

The following packages are required to run GentooGroup:
    * fakeroot
"""
from os import path
from plumbum.cmd import cp, tar, mv, fakeroot, rm  # pylint: disable=E0401
from plumbum.cmd import mkdir, curl, cut, tail  # pylint: disable=E0401
from plumbum import local
from pprof.project import Project
from pprof.utils.compiler import wrap_cc_in_uchroot, wrap_cxx_in_uchroot
from pprof.utils.run import run, uchroot
from pprof.utils.downloader import Wget
from lazy import lazy

def latest_src_uri():
    """
    Get the latest src_uri for a stage 3 tarball.

    Returns (str):
        Latest src_uri from gentoo's distfiles mirror.
    """
    from plumbum import ProcessExecutionError
    from logging import error

    latest_txt = "http://distfiles.gentoo.org/releases/amd64/autobuilds/"\
            "latest-stage3-amd64.txt"
    try:
        src_uri = (curl[latest_txt] | tail["-n", "+3"]
                   | cut["-f1", "-d "])().strip()
    except ProcessExecutionError as proc_ex:
        src_uri = "NOT-FOUND"
        error("Could not determine latest stage3 src uri: {0}", str(proc_ex))
    return src_uri

class GentooGroup(Project):
    """
    Gentoo ProjectGroup is the base class for every portage build.
    """
    GROUP = 'gentoo'

    def __init__(self, exp):
        super(GentooGroup, self).__init__(exp, "gentoo")

    src_dir = "gentoo"
    src_file = src_dir + ".tar.bz2"

    @lazy
    def src_uri(self):  # pylint: disable=R0201
        """Fetches the lates src_uri from the gentoo mirrors."""
        return "http://distfiles.gentoo.org/releases/amd64/autobuilds/{0}" \
                .format(latest_src_uri())

    # download location for portage files
    src_uri_portage = "ftp://sunsite.informatik.rwth-aachen.de/pub/Linux/"\
                    "gentoo/snapshots/portage-latest.tar.bz2"
    src_file_portage = "portage_snap.tar.bz2"

    def build(self):
        pass

    def download(self):
        from pprof.settings import config
        with local.cwd(self.builddir):
            Wget(self.src_uri, self.src_file)

            cp(config["sourcedir"] + "/bin/uchroot", "uchroot")
            run(fakeroot["tar", "xfj", self.src_file])
            rm(self.src_file)
            with local.cwd(self.builddir + "/usr"):
                Wget(self.src_uri_portage, self.src_file_portage)
                run(tar["xfj", self.src_file_portage])
                rm(self.src_file_portage)

    def configure(self):
        with local.cwd(self.builddir):
            with open("etc/portage/make.conf", 'w') as makeconf:
                lines = '''
PATH="/llvm/bin:/pprof/bin:${PATH}"
LD_LIBRARY_PATH="/llvm/lib:/pprof/lib:${LD_LIBRARY_PATH}"
CFLAGS="-O2 -pipe"
CXXFLAGS="${CFLAGS}"
FEATURES="-sandbox -usersandbox -usersync -xattr"
CC="/clang"
CXX="/clang++"

PORTAGE_USERNAME = "root"
PORTAGE_GRPNAME = "root"
PORTAGE_INST_GID = 0
PORTAGE_INST_UID = 0

CHOST="x86_64-pc-linux-gnu"
USE="bindist mmx sse sse2"
PORTDIR="/usr/portage"
DISTDIR="${PORTDIR}/distfiles"
PKGDIR="${PORTDIR}/packages"
'''

                makeconf.write(lines)

            mkdir("-p", "etc/portage/metadata")
            with open("etc/portage/metadata/layout.conf", 'w') as layoutconf:
                lines = '''masters = gentoo'''
                layoutconf.write(lines)
            cp("/etc/resolv.conf", "etc/resolv.conf")

            wrap_cc_in_uchroot(self.cflags, self.ldflags,
                               self.compiler_extension, "/llvm/bin")
            wrap_cxx_in_uchroot(self.cflags, self.ldflags,
                                self.compiler_extension, "/llvm/bin")

class PrepareStage3(GentooGroup):
    """
    A project that can be used for interactive stage3 generation.
    """
    NAME = "stage3"
    DOMAIN = "debug"

    def build(self):
        import sys
        # Don't do something when running non-interactive.
        if not sys.stdout.isatty():
            return

        from plumbum import FG
        from pprof.utils.downloader import update_hash
        from logging import info
        from pprof.settings import config

        root = config["tmpdir"]
        src_file = self.src_file + ".new"
        with local.cwd(self.builddir):
            bash_in_uchroot = uchroot()["/bin/bash"]
            print("Entering User-Chroot. Prepare your image and "
                  "type 'exit' when you are done.")
            bash_in_uchroot & FG  # pylint: disable=W0104
            tgt_path = path.join(root, self.src_file)
            tgt_path_new = path.join(root, src_file)
            print("Packing new stage3 image. "
                  "This will replace the original one at: ", tgt_path)
            tar("cjf", tgt_path_new, ".")
            update_hash(src_file, root)
            mv(path.join(root, src_file), tgt_path)

    def run_tests(self, experiment):
        pass

class AutoPrepareStage3(GentooGroup):
    """
    A project that can be used to install pprof in the stage3 archive.
    """
    NAME = "auto-stage3"
    DOMAIN = "debug"

    def build(self):
        import sys
        # Don't do something when running non-interactive.
        if not sys.stdout.isatty():
            return

        from plumbum import FG
        from pprof.utils.downloader import update_hash
        from logging import info
        from pprof.settings import config

        root = config["tmpdir"]
        src_file = self.src_file + ".new"
        with local.cwd(self.builddir):
            mkdir("-p", "pprof-src")
            w_pprof_src = uchroot("-m", "{}:pprof-src".format(config[
                "sourcedir"]))
            pip_in_uchroot = w_pprof_src["/usr/bin/pip3"]
            pip_in_uchroot["install", "--upgrade", "/pprof-src/"] & FG

            tgt_path = path.join(root, self.src_file)
            tgt_path_new = path.join(root, src_file)
            tar("cjf", tgt_path_new, ".")
            update_hash(src_file, root)
            mv(path.join(root, src_file), tgt_path)

    def run_tests(self, experiment):
        pass

class Eix(GentooGroup):
    """
    Represents the package eix from the portage tree.

    Building this class will create bare gentoo and compile eix.
    """
    NAME = 'eix'
    DOMAIN = 'debug'

    def build(self):
        with local.cwd(self.builddir):
            emerge_in_chroot = uchroot()["/usr/bin/emerge"]
            run(emerge_in_chroot["eix"])

    def run_tests(self, experiment):
        pass
<|endoftext|>


<commit_before>#!/usr/bin/evn python
# encoding: utf-8

from pprof.project import ProjectFactory, log
from pprof.settings import config
from group import PprofGroup

from os import path, getenv
from glob import glob
from plumbum import FG, local

class Minisat(PprofGroup):

    """ minisat benchmark """

    class Factory:

        def create(self, exp):
            return Minisat(exp, "minisat", "verification")
    ProjectFactory.addFactory("Minisat", Factory())

    def run_tests(self, experiment):
        from pprof.project import wrap

        minisat_dir = path.join(self.builddir, self.src_dir)

        exp = wrap(
            path.join(minisat_dir, "build", "dynamic", "bin", "minisat"))

        testfiles = glob(path.join(self.testdir, "*.cnf.gz"))
        minisat_dir = path.join(self.builddir, self.src_dir)
        minisat_lib_path = path.join(minisat_dir, "build", "dynamic", "lib")

        for f in testfiles:
            with local.env(LD_LIBRARY_PATH=minisat_lib_path + ":" + getenv("LD_LIBRARY_PATH", "")):
                (exp < f) & FG(retcode=None)

    src_dir = "minisat.git"
    src_uri = "https://github.com/niklasso/minisat"

    def download(self):
        from pprof.utils.downloader import Git
        from plumbum.cmd import git

        minisat_dir = path.join(self.builddir, self.src_dir)
        with local.cwd(self.builddir):
            Git(self.src_uri, self.src_dir)
            with local.cwd(minisat_dir):
                git("fetch", "origin", "pull/17/head:clang")
                git("checkout", "clang")

    def configure(self):
        from plumbum.cmd import make

        minisat_dir = path.join(self.builddir, self.src_dir)
        with local.cwd(minisat_dir):
            make("config")

    def build(self):
        from plumbum.cmd import make, ln
        from pprof.utils.compiler import lt_clang, lt_clang_cxx

        minisat_dir = path.join(self.builddir, self.src_dir)
        clang = lt_clang(self.cflags, self.ldflags)
        clang_cxx = lt_clang_cxx(self.cflags, self.ldflags)

        with local.cwd(minisat_dir):
            make("CC=" + str(clang),
                 "CXX=" + str(clang_cxx),
                 "clean", "lsh", "sh")
<commit_msg>minisat: missing argument to wrap<commit_after>#!/usr/bin/evn python
# encoding: utf-8

from pprof.project import ProjectFactory, log
from pprof.settings import config
from group import PprofGroup

from os import path, getenv
from glob import glob
from plumbum import FG, local

class Minisat(PprofGroup):

    """ minisat benchmark """

    class Factory:

        def create(self, exp):
            return Minisat(exp, "minisat", "verification")
    ProjectFactory.addFactory("Minisat", Factory())

    def run_tests(self, experiment):
        from pprof.project import wrap

        minisat_dir = path.join(self.builddir, self.src_dir)

        exp = wrap(
            path.join(minisat_dir, "build", "dynamic", "bin", "minisat"), experiment)

        testfiles = glob(path.join(self.testdir, "*.cnf.gz"))
        minisat_dir = path.join(self.builddir, self.src_dir)
        minisat_lib_path = path.join(minisat_dir, "build", "dynamic", "lib")

        for f in testfiles:
            with local.env(LD_LIBRARY_PATH=minisat_lib_path + ":" + getenv("LD_LIBRARY_PATH", "")):
                (exp < f) & FG(retcode=None)

    src_dir = "minisat.git"
    src_uri = "https://github.com/niklasso/minisat"

    def download(self):
        from pprof.utils.downloader import Git
        from plumbum.cmd import git

        minisat_dir = path.join(self.builddir, self.src_dir)
        with local.cwd(self.builddir):
            Git(self.src_uri, self.src_dir)
            with local.cwd(minisat_dir):
                git("fetch", "origin", "pull/17/head:clang")
                git("checkout", "clang")

    def configure(self):
        from plumbum.cmd import make

        minisat_dir = path.join(self.builddir, self.src_dir)
        with local.cwd(minisat_dir):
            make("config")

    def build(self):
        from plumbum.cmd import make, ln
        from pprof.utils.compiler import lt_clang, lt_clang_cxx

        minisat_dir = path.join(self.builddir, self.src_dir)
        clang = lt_clang(self.cflags, self.ldflags)
        clang_cxx = lt_clang_cxx(self.cflags, self.ldflags)

        with local.cwd(minisat_dir):
            make("CC=" + str(clang),
                 "CXX=" + str(clang_cxx),
                 "clean", "lsh", "sh")
<|endoftext|>


<commit_before># pylint: disable=missing-docstring
from django.shortcuts import render
from django.http import HttpResponseForbidden, HttpResponseRedirect
from django.contrib.syndication.views import Feed
from django.conf import settings
from django.views.generic import DetailView, TemplateView
from django.urls import reverse

from common.forms import UploadForm
from common.models import News
from games.models import Game

def home(request):
    """Homepage view"""
    new_games = Game.objects.with_installer().order_by('-created')[:6]
    updated_games = Game.objects.with_installer().order_by('installers__updated_at')[:6]

    return render(request, 'home.html', {
        "new_games": new_games,
        "updated_games": updated_games
    })

class Downloads(TemplateView):
    template_name = "common/downloads.html"

    def get_context_data(self, **kwargs):
        context = super(Downloads, self).get_context_data(**kwargs)
        context['version'] = settings.CLIENT_VERSION
        context['download_url'] = 'https://lutris.net/releases/'
        return context

def news_archives(request):
    news = News.objects.all()
    return render(request, 'news.html', {'news': news})

class NewsDetails(DetailView):
    queryset = News.objects.all()
    template_name = 'common/news_details.html'
    context_object_name = 'news'

    def get_object(self, queryset=None):
        return super(NewsDetails, self).get_object()

class NewsFeed(Feed):
    title = "Lutris news"
    link = '/news/'
    description = u"Latest news about Lutris"
    feed_size = 20

    def items(self):
        return News.objects.order_by("-publish_date")[:self.feed_size]

    def item_title(self, item):
        return item.title

    def item_description(self, item):
        return item.content.rendered

    def item_link(self, item):
        return item.get_absolute_url()

def upload_file(request):
    if not request.user.is_staff:
        return HttpResponseForbidden()
    form = UploadForm(request.POST or None, request.FILES or None)
    if request.method == 'POST' and form.is_valid():
        upload = form.save(commit=False)
        upload.uploaded_by = request.user
        upload.save()
        return HttpResponseRedirect(reverse('upload_file'))
    return render(request, 'common/upload.html', {'form': form})
<commit_msg>Fix updated games on  homepage<commit_after># pylint: disable=missing-docstring
from django.shortcuts import render
from django.http import HttpResponseForbidden, HttpResponseRedirect
from django.contrib.syndication.views import Feed
from django.conf import settings
from django.views.generic import DetailView, TemplateView
from django.urls import reverse

from common.forms import UploadForm
from common.models import News
from games.models import Game, Installer

def home(request):
    """Homepage view"""
    new_games = Game.objects.with_installer().order_by('-created')[:6]
    updated_games = [
        installer.game for installer in (
            Installer
                .objects
                .prefetch_related('game')
                .filter(published=True)
                .order_by('-updated_at')
        )[:6]
    ]
    return render(request, 'home.html', {
        "new_games": new_games,
        "updated_games": updated_games
    })

class Downloads(TemplateView):
    template_name = "common/downloads.html"

    def get_context_data(self, **kwargs):
        context = super(Downloads, self).get_context_data(**kwargs)
        context['version'] = settings.CLIENT_VERSION
        context['download_url'] = 'https://lutris.net/releases/'
        return context

def news_archives(request):
    news = News.objects.all()
    return render(request, 'news.html', {'news': news})

class NewsDetails(DetailView):
    queryset = News.objects.all()
    template_name = 'common/news_details.html'
    context_object_name = 'news'

    def get_object(self, queryset=None):
        return super(NewsDetails, self).get_object()

class NewsFeed(Feed):
    title = "Lutris news"
    link = '/news/'
    description = u"Latest news about Lutris"
    feed_size = 20

    def items(self):
        return News.objects.order_by("-publish_date")[:self.feed_size]

    def item_title(self, item):
        return item.title

    def item_description(self, item):
        return item.content.rendered

    def item_link(self, item):
        return item.get_absolute_url()

def upload_file(request):
    if not request.user.is_staff:
        return HttpResponseForbidden()
    form = UploadForm(request.POST or None, request.FILES or None)
    if request.method == 'POST' and form.is_valid():
        upload = form.save(commit=False)
        upload.uploaded_by = request.user
        upload.save()
        return HttpResponseRedirect(reverse('upload_file'))
    return render(request, 'common/upload.html', {'form': form})
<|endoftext|>


<commit_before>'''
pr0ntools
Copyright 2011 John McMaster <JohnDMcMaster@gmail.com>
Licensed under a 2 clause BSD license, see COPYING for details

This is a stitching strategy where a regular input grid is assumed
I get this using my CNC microscope because the pictures *are* taken as fairly precise intervals
This allows considerable optimization since we know where all the picture are
'''

import image_coordinate_map
from image_coordinate_map import ImageCoordinateMap
import os
import sys
from pr0ntools.stitch.pto.util import dbg
import threading
import Queue
import traceback
import common_stitch
#from pr0ntools.util import msg
import time

# FIXME: placeholder
def msg(s=''):
    #print '%s: %s' % (now(), s)
    print s

class Worker(threading.Thread):
    def __init__(self, i):
        threading.Thread.__init__(self)
        self.i = i
        self.qi = Queue.Queue()
        self.qo = Queue.Queue()
        self.running = threading.Event()
        self.generate_control_points_by_pair = None

    def run(self):
        self.running.set()
        while self.running.is_set():
            try:
                task = self.qi.get(True, 0.1)
            except Queue.Empty:
                continue

            try:
                (pair, pair_images) = task

                msg()
                msg()
                msg()
                msg()
                msg()
                msg('*' * 80)
                msg('w%d: task rx' % self.i)

                final_pair_project = self.generate_control_points_by_pair(pair, pair_images)

                if not final_pair_project:
                    msg('WARNING: bad project @ %s, %s' % (repr(pair), repr(pair_images)))
                else:
                    if False:
                        msg()
                        msg('Final pair project')
                        msg(final_pair_project.get_a_file_name())
                        msg()
                        msg()
                        msg(final_pair_project)
                        msg()
                        msg()
                        msg()
                        #sys.exit(1)
                    if len(final_pair_project.get_text().strip()) == 0:
                        raise Exception('Generated empty pair project')

                self.qo.put(('done', (task, final_pair_project)))
                msg('w%d: task done' % self.i)

            except Exception as e:
                traceback.print_exc()
                self.qo.put(('exception', (task, e)))

class GridStitch(common_stitch.CommonStitch):
    def __init__(self):
        common_stitch.CommonStitch.__init__(self)
        self.coordinate_map = None
        self.set_regular(True)
        self.canon2orig = dict()
        self.skip_missing = False
        self.threads = 1
        self.workers = []

    @staticmethod
    def from_tagged_file_names(image_file_names):
        engine = GridStitch()
        engine.image_file_names = image_file_names
        dbg('Orig file names: %s' % str(image_file_names))

        file_names_canonical = list()
        for file_name in image_file_names:
            new_fn = os.path.realpath(file_name)
            engine.canon2orig[new_fn] = file_name
            file_names_canonical.append(new_fn)

        engine.coordinate_map = ImageCoordinateMap.from_tagged_file_names(file_names_canonical)

        return engine

    def init_failures(self):
        open_list = set()
        for (file_name, _row, _col) in self.coordinate_map.images():
            open_list.add(file_name)
        self.failures = common_stitch.FailedImages(open_list)

    def generate_control_points(self):
        '''
        Generate control points
        Generate to all neighbors to start with
        '''
        #temp_projects = list()

        msg()
        n_pairs = len(list(self.coordinate_map.gen_pairs(1, 1)))
        msg('***Pairs: %d***' % n_pairs)
        msg()
        pair_submit = 0
        pair_complete = 0

        if self.skip_missing:
            msg('Not verifying image map')
        else:
            msg('Verifying image map')
            try:
                self.coordinate_map.is_complete()
            except image_coordinate_map.MissingImage as e:
                msg('Missing images.  Use --skip-missing to continue')
                raise e

        msg('Initializing %d workers' % self.threads)
        for ti in xrange(self.threads):
            w = Worker(ti)
            w.generate_control_points_by_pair = self.generate_control_points_by_pair
            self.workers.append(w)
            w.start()

        coord_pairs = self.coordinate_map.gen_pairs(1, 1)

        all_allocated = False

        # Seed project with all images in order
        # note we used the filename that will get used below
        # not the final output file name
        for can_fn in sorted(self.canon2orig.keys()):
            self.project.add_image(can_fn)
        self.project.save()

        while not (all_allocated and pair_complete == pair_submit):
            # Most efficient to merge things in batches as they complete
            final_pair_projects = []
            # Check for completed jobs
            for wi, worker in enumerate(self.workers):
                try:
                    out = worker.qo.get(False)
                except Queue.Empty:
                    continue
                pair_complete += 1
                what = out[0]

                if what == 'done':
                    (_task, final_pair_project) = out[1]
                    msg('W%d: done' % wi)
                    # May have failed
                    if final_pair_project:
                        final_pair_projects.append(final_pair_project)
                        if pair_complete % 10 == 0:
                            print 'Saving intermediate result to %s' % self.project.file_name
                            self.project.save()
                            print 'Saved'

                elif what == 'exception':
                    #(_task, e) = out[1]
                    msg('ERROR: W%d failed w/ exception' % wi)
                    raise Exception('Shutdown on worker failure')
                else:
                    msg('%s' % (out,))
                    raise Exception('Internal error: bad task type %s' % what)
            # Merge projects
            if len(final_pair_projects):
                print 'Merging %d projects' % len(final_pair_projects)
                self.project.merge_into(final_pair_projects)

            # Any workers need more work?
            for wi, worker in enumerate(self.workers):
                if all_allocated:
                    break
                if worker.qi.empty():
                    while True:
                        try:
                            pair = coord_pairs.next()
                        except  StopIteration:
                            msg('All tasks allocated')
                            all_allocated = True
                            break

                        pair_submit += 1

                        msg('*' * 80)
                        msg('W%d: submit %s (%d / %d)' % (wi, repr(pair), pair_submit, n_pairs))

                        # Image file names as list
                        pair_images = self.coordinate_map.get_images_from_pair(pair)
                        msg('pair images: ' + repr(pair_images))
                        if pair_images[0] is None or pair_images[1] is None:
                            msg('WARNING: skipping missing image')
                            continue

                        worker.qi.put((pair, pair_images))
                        break

            time.sleep(0.1)

        msg('pairs done')

        for worker in self.workers:
            worker.running.clear()

        msg('Reverting canonical file names to original input...')
        # Fixup the canonical hack
        for can_fn in self.canon2orig:
            # FIXME: if we have issues with images missing from the project due to bad stitch
            # we should add them (here?) instead of throwing an error
            orig = self.canon2orig[can_fn]
            il = self.project.get_image_by_fn(can_fn)
            if il:
                il.set_name(orig)
            else:
                msg('WARNING: adding image without feature match %s' % orig)
                self.project.add_image(orig)

        self.project.save()

        '''
        if 0:
            msg('Sub projects (full image):')
            for project in temp_projects:
                # prefix so I can grep it for debugging
                msg('\tSUB: ' + project.file_name)
        '''
        '''
        if 0:
            msg()
            msg()
            msg('Master project file: %s' % self.project.file_name)
            msg()
            msg()
            msg(self.project.text)
            msg()
            msg()
        '''

    def do_generate_control_points_by_pair(self, pair, image_fn_pair):
        ret = common_stitch.CommonStitch.do_generate_control_points_by_pair(self, pair, image_fn_pair)
        if ret is None and pair.adjacent():
            msg('WARNING: last ditch effort, increasing field of view')

        return ret

<commit_msg>grid stitch: better guarantee threads terminate<commit_after>'''
pr0ntools
Copyright 2011 John McMaster <JohnDMcMaster@gmail.com>
Licensed under a 2 clause BSD license, see COPYING for details

This is a stitching strategy where a regular input grid is assumed
I get this using my CNC microscope because the pictures *are* taken as fairly precise intervals
This allows considerable optimization since we know where all the picture are
'''

import image_coordinate_map
from image_coordinate_map import ImageCoordinateMap
import os
import sys
from pr0ntools.stitch.pto.util import dbg
import threading
import Queue
import traceback
import common_stitch
#from pr0ntools.util import msg
import time

# FIXME: placeholder
def msg(s=''):
    #print '%s: %s' % (now(), s)
    print s

class Worker(threading.Thread):
    def __init__(self, i):
        threading.Thread.__init__(self)
        self.i = i
        self.qi = Queue.Queue()
        self.qo = Queue.Queue()
        self.running = threading.Event()
        self.generate_control_points_by_pair = None

    def run(self):
        self.running.set()
        while self.running.is_set():
            try:
                task = self.qi.get(True, 0.1)
            except Queue.Empty:
                continue

            try:
                (pair, pair_images) = task

                msg()
                msg()
                msg()
                msg()
                msg()
                msg('*' * 80)
                msg('w%d: task rx' % self.i)

                final_pair_project = self.generate_control_points_by_pair(pair, pair_images)

                if not final_pair_project:
                    msg('WARNING: bad project @ %s, %s' % (repr(pair), repr(pair_images)))
                else:
                    if False:
                        msg()
                        msg('Final pair project')
                        msg(final_pair_project.get_a_file_name())
                        msg()
                        msg()
                        msg(final_pair_project)
                        msg()
                        msg()
                        msg()
                        #sys.exit(1)
                    if len(final_pair_project.get_text().strip()) == 0:
                        raise Exception('Generated empty pair project')

                self.qo.put(('done', (task, final_pair_project)))
                msg('w%d: task done' % self.i)

            except Exception as e:
                traceback.print_exc()
                self.qo.put(('exception', (task, e)))

class GridStitch(common_stitch.CommonStitch):
    def __init__(self):
        common_stitch.CommonStitch.__init__(self)
        self.coordinate_map = None
        self.set_regular(True)
        self.canon2orig = dict()
        self.skip_missing = False
        self.threads = 1
        self.workers = []

    @staticmethod
    def from_tagged_file_names(image_file_names):
        engine = GridStitch()
        engine.image_file_names = image_file_names
        dbg('Orig file names: %s' % str(image_file_names))

        file_names_canonical = list()
        for file_name in image_file_names:
            new_fn = os.path.realpath(file_name)
            engine.canon2orig[new_fn] = file_name
            file_names_canonical.append(new_fn)

        engine.coordinate_map = ImageCoordinateMap.from_tagged_file_names(file_names_canonical)

        return engine

    def init_failures(self):
        open_list = set()
        for (file_name, _row, _col) in self.coordinate_map.images():
            open_list.add(file_name)
        self.failures = common_stitch.FailedImages(open_list)

    def generate_control_points(self):
        '''
        Generate control points
        Generate to all neighbors to start with
        '''
        #temp_projects = list()

        msg()
        n_pairs = len(list(self.coordinate_map.gen_pairs(1, 1)))
        msg('***Pairs: %d***' % n_pairs)
        msg()
        pair_submit = 0
        pair_complete = 0

        if self.skip_missing:
            msg('Not verifying image map')
        else:
            msg('Verifying image map')
            try:
                self.coordinate_map.is_complete()
            except image_coordinate_map.MissingImage as e:
                msg('!' * 80)
                msg('Missing images.  Use --skip-missing to continue')
                msg('!' * 80)
                raise e

        msg('Initializing %d workers' % self.threads)
        for ti in xrange(self.threads):
            w = Worker(ti)
            w.generate_control_points_by_pair = self.generate_control_points_by_pair
            self.workers.append(w)
            w.start()

        try:
            coord_pairs = self.coordinate_map.gen_pairs(1, 1)

            all_allocated = False

            # Seed project with all images in order
            # note we used the filename that will get used below
            # not the final output file name
            for can_fn in sorted(self.canon2orig.keys()):
                self.project.add_image(can_fn)
            self.project.save()

            while not (all_allocated and pair_complete == pair_submit):
                # Most efficient to merge things in batches as they complete
                final_pair_projects = []
                # Check for completed jobs
                for wi, worker in enumerate(self.workers):
                    try:
                        out = worker.qo.get(False)
                    except Queue.Empty:
                        continue
                    pair_complete += 1
                    what = out[0]

                    if what == 'done':
                        (_task, final_pair_project) = out[1]
                        msg('W%d: done' % wi)
                        # May have failed
                        if final_pair_project:
                            final_pair_projects.append(final_pair_project)
                            if pair_complete % 10 == 0:
                                print 'Saving intermediate result to %s' % self.project.file_name
                                self.project.save()
                                print 'Saved'

                    elif what == 'exception':
                        for worker in self.workers:
                            worker.running.clear()
                        # let stdout clear up
                        time.sleep(1)

                        #(_task, e) = out[1]
                        msg('!' * 80)
                        msg('ERROR: W%d failed w/ exception' % wi)
                        msg('!' * 80)
                        raise Exception('Shutdown on worker failure')
                    else:
                        msg('%s' % (out,))
                        raise Exception('Internal error: bad task type %s' % what)
                # Merge projects
                if len(final_pair_projects):
                    print 'Merging %d projects' % len(final_pair_projects)
                    self.project.merge_into(final_pair_projects)

                # Any workers need more work?
                for wi, worker in enumerate(self.workers):
                    if all_allocated:
                        break
                    if worker.qi.empty():
                        while True:
                            try:
                                pair = coord_pairs.next()
                            except  StopIteration:
                                msg('All tasks allocated')
                                all_allocated = True
                                break

                            pair_submit += 1

                            msg('*' * 80)
                            msg('W%d: submit %s (%d / %d)' % (wi, repr(pair), pair_submit, n_pairs))

                            # Image file names as list
                            pair_images = self.coordinate_map.get_images_from_pair(pair)
                            msg('pair images: ' + repr(pair_images))
                            if pair_images[0] is None or pair_images[1] is None:
                                msg('WARNING: skipping missing image')
                                continue

                            worker.qi.put((pair, pair_images))
                            break

                time.sleep(0.1)

            msg('pairs done')

        finally:
            for worker in self.workers:
                worker.running.clear()

        msg('Reverting canonical file names to original input...')
        # Fixup the canonical hack
        for can_fn in self.canon2orig:
            # FIXME: if we have issues with images missing from the project due to bad stitch
            # we should add them (here?) instead of throwing an error
            orig = self.canon2orig[can_fn]
            il = self.project.get_image_by_fn(can_fn)
            if il:
                il.set_name(orig)
            else:
                msg('WARNING: adding image without feature match %s' % orig)
                self.project.add_image(orig)

        self.project.save()

        '''
        if 0:
            msg('Sub projects (full image):')
            for project in temp_projects:
                # prefix so I can grep it for debugging
                msg('\tSUB: ' + project.file_name)
        '''
        '''
        if 0:
            msg()
            msg()
            msg('Master project file: %s' % self.project.file_name)
            msg()
            msg()
            msg(self.project.text)
            msg()
            msg()
        '''

    def do_generate_control_points_by_pair(self, pair, image_fn_pair):
        ret = common_stitch.CommonStitch.do_generate_control_points_by_pair(self, pair, image_fn_pair)
        if ret is None and pair.adjacent():
            msg('WARNING: last ditch effort, increasing field of view')

        return ret

<|endoftext|>