<commit_before>import six
from hashlib import sha1
from collections import OrderedDict, Sequence
from six.moves.urllib.parse import urlparse

QUERY_STEP = 1000
row_type = OrderedDict

class DatasetException(Exception):
    pass

def convert_row(row_type, row):
    if row is None:
        return None
    return row_type(row.items())

def iter_result_proxy(rp, step=None):
    """Iterate over the ResultProxy."""
    while True:
        if step is None:
            chunk = rp.fetchall()
        else:
            chunk = rp.fetchmany(step)
        if not chunk:
            break
        for row in chunk:
            yield row

class ResultIter(object):
    """ SQLAlchemy ResultProxies are not iterable to get a
    list of dictionaries. This is to wrap them. """

    def __init__(self, result_proxy, row_type=row_type, step=None):
        self.row_type = row_type
        self.result_proxy = result_proxy
        self.keys = list(result_proxy.keys())
        self._iter = iter_result_proxy(result_proxy, step=step)

    def __next__(self):
        return convert_row(self.row_type, next(self._iter))

    next = __next__

    def __iter__(self):
        return self

    def close(self):
        self.result_proxy.close()

def normalize_column_name(name):
    """Check if a string is a reasonable thing to use as a column name."""
    if not isinstance(name, six.string_types):
        raise ValueError('%r is not a valid column name.' % name)

    # limit to 63 characters
    name = name.strip()[:63]
    # column names can be 63 *bytes* max in postgresql
    if isinstance(name, six.text_type):
        while len(name.encode('utf-8')) >= 64:
            name = name[:len(name) - 1]

    if not len(name) or '.' in name or '-' in name:
        raise ValueError('%r is not a valid column name.' % name)
    return name

def normalize_table_name(name):
    """Check if the table name is obviously invalid."""
    if not isinstance(name, six.string_types):
        raise ValueError("Invalid table name: %r" % name)
    name = name.strip()[:63]
    if not len(name):
        raise ValueError("Invalid table name: %r" % name)
    return name

def safe_url(url):
    """Remove password from printed connection URLs."""
    parsed = urlparse(url)
    if parsed.password is not None:
        pwd = ':%s@' % parsed.password
        url = url.replace(pwd, ':*****@')
    return url

def index_name(table, columns):
    """Generate an artificial index name."""
    sig = '||'.join(columns)
    key = sha1(sig.encode('utf-8')).hexdigest()[:16]
    return 'ix_%s_%s' % (table, key)

def ensure_tuple(obj):
    """Try and make the given argument into a tuple."""
    if obj is None:
        return tuple()
    if isinstance(obj, Sequence) and not isinstance(obj, six.string_types):
        return tuple(obj)
    return obj,

def pad_chunk_columns(chunk):
    """Given a set of items to be inserted, make sure they all have the
    same columns by padding columns with None if they are missing."""
    columns = set()
    for record in chunk:
        columns.update(record.keys())
    for record in chunk:
        for column in columns:
            record.setdefault(column, None)
    return chunk
<commit_msg>Update util.py - fix ensure_tuple instance check<commit_after>import six
from hashlib import sha1
from collections import OrderedDict, Iterable
from six.moves.urllib.parse import urlparse

QUERY_STEP = 1000
row_type = OrderedDict

class DatasetException(Exception):
    pass

def convert_row(row_type, row):
    if row is None:
        return None
    return row_type(row.items())

def iter_result_proxy(rp, step=None):
    """Iterate over the ResultProxy."""
    while True:
        if step is None:
            chunk = rp.fetchall()
        else:
            chunk = rp.fetchmany(step)
        if not chunk:
            break
        for row in chunk:
            yield row

class ResultIter(object):
    """ SQLAlchemy ResultProxies are not iterable to get a
    list of dictionaries. This is to wrap them. """

    def __init__(self, result_proxy, row_type=row_type, step=None):
        self.row_type = row_type
        self.result_proxy = result_proxy
        self.keys = list(result_proxy.keys())
        self._iter = iter_result_proxy(result_proxy, step=step)

    def __next__(self):
        return convert_row(self.row_type, next(self._iter))

    next = __next__

    def __iter__(self):
        return self

    def close(self):
        self.result_proxy.close()

def normalize_column_name(name):
    """Check if a string is a reasonable thing to use as a column name."""
    if not isinstance(name, six.string_types):
        raise ValueError('%r is not a valid column name.' % name)

    # limit to 63 characters
    name = name.strip()[:63]
    # column names can be 63 *bytes* max in postgresql
    if isinstance(name, six.text_type):
        while len(name.encode('utf-8')) >= 64:
            name = name[:len(name) - 1]

    if not len(name) or '.' in name or '-' in name:
        raise ValueError('%r is not a valid column name.' % name)
    return name

def normalize_table_name(name):
    """Check if the table name is obviously invalid."""
    if not isinstance(name, six.string_types):
        raise ValueError("Invalid table name: %r" % name)
    name = name.strip()[:63]
    if not len(name):
        raise ValueError("Invalid table name: %r" % name)
    return name

def safe_url(url):
    """Remove password from printed connection URLs."""
    parsed = urlparse(url)
    if parsed.password is not None:
        pwd = ':%s@' % parsed.password
        url = url.replace(pwd, ':*****@')
    return url

def index_name(table, columns):
    """Generate an artificial index name."""
    sig = '||'.join(columns)
    key = sha1(sig.encode('utf-8')).hexdigest()[:16]
    return 'ix_%s_%s' % (table, key)

def ensure_tuple(obj):
    """Try and make the given argument into a tuple."""
    if obj is None:
        return tuple()
    if isinstance(obj, Iterable) and not isinstance(obj, six.string_types):
        return tuple(obj)
    return obj,

def pad_chunk_columns(chunk):
    """Given a set of items to be inserted, make sure they all have the
    same columns by padding columns with None if they are missing."""
    columns = set()
    for record in chunk:
        columns.update(record.keys())
    for record in chunk:
        for column in columns:
            record.setdefault(column, None)
    return chunk
<|endoftext|>


<commit_before>'''
Created on Jan 30, 2012

@author: guillaume.aubert@gmail.com
'''

from cmdline_utils  import CmdLineParser
import argparse
import sys

if __name__ == '__main__':

    parser = argparse.ArgumentParser()

    subparsers = parser.add_subparsers(help='commands')

    # A list command
    list_parser = subparsers.add_parser('sync', help='synchronize with given gmail account')
    list_parser.add_argument('email', action='store', help='email to sync with')
    list_parser.add_argument('-t', action='store', help='type of synchronisation')

    # A create command
    create_parser = subparsers.add_parser('create', help='Create a directory')
    create_parser.add_argument('dirname', action='store', help='New directory to create')
    create_parser.add_argument('--read-only', default=False, action='store_true',
                               help='Set permissions to prevent writing to the directory',
                               )

    # A delete command
    delete_parser = subparsers.add_parser('delete', help='Remove a directory')
    delete_parser.add_argument('dirname', action='store', help='The directory to remove')
    delete_parser.add_argument('--recursive', '-r', default=False, action='store_true',
                               help='Remove the contents of the directory, too',
                               )

    #sys.argv = ['gmvault.py', 'sync', '-h']

    sys.argv = ['gmvault.py', 'sync', 'guillaume.aubert@gmail.com', '-t', 'full_sync']

    print(parser.parse_args())

    #print("options = %s\n" % (options))
    #print("args = %s\n" % (args))

    <commit_msg>start integrating new command parser<commit_after>'''
Created on Jan 30, 2012

@author: guillaume.aubert@gmail.com
'''

import argparse
import sys

class CmdLineParser(argparse.ArgumentParser): #pylint: disable-msg=R0904
    """ Added service on OptionParser """

    BOOL_TRUE  = ['yes', 'true', '1']
    BOOL_FALSE = ['no', 'false', '0']
    BOOL_VALS  = BOOL_TRUE + BOOL_FALSE

    def __init__(self, *args, **kwargs):
        """ constructor """
        argparse.ArgumentParser.__init__(self, *args, **kwargs) #pylint: disable-msg=W0142

        # I like my help option message better than the default...
        #self.remove_option('-h')
        #self.add_option('-h', '--help', action='help', help='Show this message and exit.')

        self.epilogue = None

    @classmethod
    def convert_to_boolean(cls, val):
        """
           Convert yes, True, true, YES to boolean True and
           no, False, false, NO to boolean NO
        """
        lower_val = val.lower()
        if lower_val in cls.BOOL_TRUE:
            return True
        elif lower_val in cls.BOOL_FALSE:
            return False
        else:
            raise Exception("val %s should be in %s to be convertible to a boolean." % (val, cls.BOOL_VALS))

    def print_help(self, out=sys.stderr):
        """
          Print the help message, followed by the epilogue (if set), to the
          specified output file. You can define an epilogue by setting the
          ``epilogue`` field.

          :param out: file desc where to write the usage message

        """
        argparse.ArgumentParse.print_help(self, out)
        if self.epilogue:
            #print >> out, '\n%s' % textwrap.fill(self.epilogue, 100, replace_whitespace = False)
            print >> out, '\n%s' % self.epilogue
            out.flush()

    def show_usage(self, msg=None):
        """
           Print usage message
        """
        self.die_with_usage(msg)

    def die_with_usage(self, msg=None, exit_code=2):
        """
          Display a usage message and exit.

          :Parameters:
              msg : str
                  If not set to ``None`` (the default), this message will be
                  displayed before the usage message

              exit_code : int
                  The process exit code. Defaults to 2.
        """
        if msg != None:
            print >> sys.stderr, msg

        self.print_help(sys.stderr)
        sys.exit(exit_code)

    def error(self, msg):
        """
          Overrides parent ``OptionParser`` class's ``error()`` method and
          forces the full usage message on error.
        """
        self.die_with_usage("%s: error: %s\n" % (self.get_prog_name(), msg))

    def message(self, msg):
        """
           Print a message
        """
        print("%s: %s\n" % (self.get_prog_name(), msg))

def test_command_parser():
    """
       Test the command parser
    """
    #parser = argparse.ArgumentParser()

    parser = CmdLineParser()

    subparsers = parser.add_subparsers(help='commands')

    # A sync command
    sync_parser = subparsers.add_parser('sync', help='synchronize with given gmail account')
    sync_parser.add_argument('email', action='store', help='email to sync with')
    sync_parser.add_argument('-t','--type', action='store', help='type of synchronisation')
    sync_parser.set_defaults(verb='sync')

    # A create command
    create_parser = subparsers.add_parser('create', help='Create a directory')
    create_parser.add_argument('dirname', action='store', help='New directory to create')
    create_parser.add_argument('--read-only', default=False, action='store_true',
                               help='Set permissions to prevent writing to the directory',
                               )

    # A delete command
    delete_parser = subparsers.add_parser('delete', help='Remove a directory')
    delete_parser.add_argument('dirname', action='store', help='The directory to remove')
    delete_parser.add_argument('--recursive', '-r', default=False, action='store_true',
                               help='Remove the contents of the directory, too',
                               )

    sys.argv = ['gmvault.py', 'sync', '-h']

    print(parser.parse_args())

    sys.argv = ['gmvault.py', 'sync', 'guillaume.aubert@gmail.com', '--type', 'full_sync']

    print(parser.parse_args())

    #print("options = %s\n" % (options))
    #print("args = %s\n" % (args))

if __name__ == '__main__':

    test_command_parser()

    <|endoftext|>


<commit_before>#!/bin/python

# Copyright (c) 2013, Arvid Norberg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the distribution.
#     * Neither the name of the author nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# this is meant to be run from the root of the repository
# the arguments are the boost-build toolsets to use.
# these will vary between testers and operating systems
# common ones are: clang, darwin, gcc, msvc, icc

import random
import os
import platform
import subprocess
import xml.etree.ElementTree as et
from datetime import datetime
import json
import sys
import yaml
from multiprocessing import Pool

# the .regression.yml configuration file format looks like this (it's yaml):

# test_dirs:
# - <path-to-test-folder>
# - ...
#
# features:
# - <list of boost-built features>
# - ...
#

def svn_info():
	# figure out which revision this is
	p = subprocess.Popen(['svn', 'info'], stdout=subprocess.PIPE)

	revision = -1
	author = ''

	for l in p.stdout:
		if 'Last Changed Rev' in l:
			revision = int(l.split(':')[1].strip())
		if 'Last Changed Author' in l:
			author = l.split(':')[1].strip()

	if revision == -1:
		print 'Failed to extract subversion revision'
		sys.exit(1)

	if author == '':
		print 'Failed to extract subversion author'
		sys.exit(1)

	return (revision, author)

def run_tests(toolset, tests, features, options, test_dir, time_limit, incremental):
	xml_file = 'bjam_build.%d.xml' % random.randint(0, 100000)
	try:

		results = {}
		toolset_found = False

		os.chdir(test_dir)

		if not incremental:
			p = subprocess.Popen(['bjam', '--abbreviate-paths', toolset, 'clean'] + options + features.split(' '), stdout=subprocess.PIPE)
			for l in p.stdout: pass
			p.wait()

		for t in tests:
			p = subprocess.Popen(['bjam', '--out-xml=%s' % xml_file, '-l%d' % time_limit, '-q', '--abbreviate-paths', toolset, t] + options + features.split(' '), stdout=subprocess.PIPE)
			output = ''
			for l in p.stdout:
				output += l.decode('latin-1')
			p.wait()

			# parse out the toolset version from the xml file
			compiler = ''
			compiler_version = ''
			command = ''

			# make this parse the actual test to pick up the time
			# spent runnin the test
			try:
				dom = et.parse(xml_file)

				command = dom.find('./command').text

				prop = dom.findall('./action/properties/property')
				for a in prop:
					name = a.attrib['name']
					if name == 'toolset':
						compiler = a.text
						if compiler_version != '': break
					if name.startswith('toolset-') and name.endswith(':version'):
						compiler_version = a.text
						if compiler != '': break

				if compiler != '' and compiler_version != '':
					toolset = compiler + '-' + compiler_version
			except: pass

			r = { 'status': p.returncode, 'output': output, 'command': command }
			results[t + '|' + features] = r

			fail_color = '\033[31;1m'
			pass_color = '\033[32;1m'
			end_seq = '\033[0m'

			if platform.system() == 'Windows':
				fail_color == ''
				pass_color == ''
				end_seq = ''

			if p.returncode == 0: sys.stdout.write('.')
			else: sys.stdout.write('X')
			sys.stdout.flush()

	except:
		# need this to make child processes exit
		sys.exit(1)
	finally:
		try: os.unlink(xml_file)
		except: pass

	return (toolset, results)

def print_usage():
		print '''usage: run_tests.py [options] bjam-toolset [bjam-toolset...]
options:
-j<n>     use n parallel processes
-h        prints this message and exits
-i        build incrementally (i.e. don't clean between checkouts)
'''

def main(argv):

	toolsets = []

	num_processes = 4
	incremental = False

	for arg in argv:
		if arg[0] == '-':
			if arg[1] == 'j':
				num_processes = int(arg[2:])
			elif arg[1] == 'h':
				print_usage()
				sys.exit(1)
			elif arg[1] == 'i':
				incremental = True
			else:
				print 'unknown option: %s' % arg
				print_usage()
				sys.exit(1)
		else:
			toolsets.append(arg)

	if toolsets == []:
		print_usage()
		sys.exit(1)

	try:
		cfg = open('.regression.yml', 'r')
	except:
		print '.regression.yml not found in current directory'
		sys.exit(1)

	cfg = yaml.load(cfg.read())

	test_dirs = []
	configs = []
	options = ['boost=source']
	time_limit = 1200
	if 'test_dirs' in cfg:
		for d in cfg['test_dirs']:
			test_dirs.append(d)
	else:
		print 'no test directory specified by .regression.yml'
		sys.exit(1)

	configs = []
	if 'features' in cfg:
		for d in cfg['features']:
			configs.append(d)
	else:
		configs = [['']]

	clean_files = []
	if 'clean' in cfg:
		clean_files = cfg['clean']

	branch_name = 'trunk'
	if 'branch' in cfg:
		branch_name = cfg['branch']

	if 'time_limit' in cfg:
		time_limit = int(cfg['time_limit'])

	architecture = platform.machine()
	build_platform = platform.system() + '-' + platform.release()

	revision, author = svn_info()

	timestamp = datetime.now()

	tester_pool = Pool(processes=num_processes)

	print '%s-%d - %s - %s' % (branch_name, revision, author, timestamp)

	print 'toolsets: %s' % ' '.join(toolsets)
#	print 'configs: %s' % '|'.join(configs)

	current_dir = os.getcwd()

	try:
		rev_dir = os.path.join(current_dir, 'regression_tests')
		try: os.mkdir(rev_dir)
		except: pass
		rev_dir = os.path.join(rev_dir, '%s-%d' % (branch_name, revision))
		try: os.mkdir(rev_dir)
		except: pass

		for test_dir in test_dirs:
			print 'running tests from "%s" in %s' % (test_dir, branch_name)
			os.chdir(test_dir)
			test_dir = os.getcwd()

			# figure out which tests are exported by this Jamfile
			p = subprocess.Popen(['bjam', '--dump-tests', 'non-existing-target'], stdout=subprocess.PIPE)

			tests = []

			for l in p.stdout:
				if not 'boost-test(RUN)' in l: continue
				test_name = os.path.split(l.split(' ')[1][1:-1])[1]
				tests.append(test_name)
			print 'found %d tests' % len(tests)

			for toolset in toolsets:
				results = {}
				toolset_found = False

				futures = []
				for features in configs:
					futures.append(tester_pool.apply_async(run_tests, [toolset, tests, features, options, test_dir, time_limit, incremental]))

				for future in futures:
					(toolset, r) = future.get()
					results.update(r)

#				for features in configs:
#					(toolset, r) = run_tests(toolset, tests, features, options, test_dir, time_limit, incremental)
#					results.update(r)

				print ''

				# each file contains a full set of tests for one speific toolset and platform
				try:
					f = open(os.path.join(rev_dir, build_platform + '#' + toolset + '.json'), 'w+')
				except IOError:
					rev_dir = os.path.join(current_dir, 'regression_tests')
					try: os.mkdir(rev_dir)
					except: pass
					rev_dir = os.path.join(rev_dir, '%s-%d' % (branch_name, revision))
					try: os.mkdir(rev_dir)
					except: pass
					f = open(os.path.join(rev_dir, build_platform + '#' + toolset + '.json'), 'w+')

				print >>f, json.dumps(results)
				f.close()

				if len(clean_files) > 0:
					for filt in clean_files:
						for f in glob.glob(filt):
							# a precautio to make sure a malicious repo
							# won't clean things outside of the test directory
							if not os.path.abspath(f).startswith(test_dir): continue
							print 'deleting %s' %f
							os.rmdirs(f)
	finally:
		# always restore current directory
		os.chdir(current_dir)

if __name__ == "__main__":
    main(sys.argv[1:])

<commit_msg>fix import issue<commit_after>#!/bin/python

# Copyright (c) 2013, Arvid Norberg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the distribution.
#     * Neither the name of the author nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# this is meant to be run from the root of the repository
# the arguments are the boost-build toolsets to use.
# these will vary between testers and operating systems
# common ones are: clang, darwin, gcc, msvc, icc

import random
import os
import platform
import subprocess
import xml.etree.ElementTree as et
from datetime import datetime
import json
import sys
import yaml
from multiprocessing import Pool
import glob
import shutil

# the .regression.yml configuration file format looks like this (it's yaml):

# test_dirs:
# - <path-to-test-folder>
# - ...
#
# features:
# - <list of boost-built features>
# - ...
#

def svn_info():
	# figure out which revision this is
	p = subprocess.Popen(['svn', 'info'], stdout=subprocess.PIPE)

	revision = -1
	author = ''

	for l in p.stdout:
		if 'Last Changed Rev' in l:
			revision = int(l.split(':')[1].strip())
		if 'Last Changed Author' in l:
			author = l.split(':')[1].strip()

	if revision == -1:
		print 'Failed to extract subversion revision'
		sys.exit(1)

	if author == '':
		print 'Failed to extract subversion author'
		sys.exit(1)

	return (revision, author)

def run_tests(toolset, tests, features, options, test_dir, time_limit, incremental):
	xml_file = 'bjam_build.%d.xml' % random.randint(0, 100000)
	try:

		results = {}
		toolset_found = False

		os.chdir(test_dir)

		if not incremental:
			p = subprocess.Popen(['bjam', '--abbreviate-paths', toolset, 'clean'] + options + features.split(' '), stdout=subprocess.PIPE)
			for l in p.stdout: pass
			p.wait()

		for t in tests:
			p = subprocess.Popen(['bjam', '--out-xml=%s' % xml_file, '-l%d' % time_limit, '-q', '--abbreviate-paths', toolset, t] + options + features.split(' '), stdout=subprocess.PIPE)
			output = ''
			for l in p.stdout:
				output += l.decode('latin-1')
			p.wait()

			# parse out the toolset version from the xml file
			compiler = ''
			compiler_version = ''
			command = ''

			# make this parse the actual test to pick up the time
			# spent runnin the test
			try:
				dom = et.parse(xml_file)

				command = dom.find('./command').text

				prop = dom.findall('./action/properties/property')
				for a in prop:
					name = a.attrib['name']
					if name == 'toolset':
						compiler = a.text
						if compiler_version != '': break
					if name.startswith('toolset-') and name.endswith(':version'):
						compiler_version = a.text
						if compiler != '': break

				if compiler != '' and compiler_version != '':
					toolset = compiler + '-' + compiler_version
			except: pass

			r = { 'status': p.returncode, 'output': output, 'command': command }
			results[t + '|' + features] = r

			fail_color = '\033[31;1m'
			pass_color = '\033[32;1m'
			end_seq = '\033[0m'

			if platform.system() == 'Windows':
				fail_color == ''
				pass_color == ''
				end_seq = ''

			if p.returncode == 0: sys.stdout.write('.')
			else: sys.stdout.write('X')
			sys.stdout.flush()

	except:
		# need this to make child processes exit
		sys.exit(1)
	finally:
		try: os.unlink(xml_file)
		except: pass

	return (toolset, results)

def print_usage():
		print '''usage: run_tests.py [options] bjam-toolset [bjam-toolset...]
options:
-j<n>     use n parallel processes
-h        prints this message and exits
-i        build incrementally (i.e. don't clean between checkouts)
'''

def main(argv):

	toolsets = []

	num_processes = 4
	incremental = False

	for arg in argv:
		if arg[0] == '-':
			if arg[1] == 'j':
				num_processes = int(arg[2:])
			elif arg[1] == 'h':
				print_usage()
				sys.exit(1)
			elif arg[1] == 'i':
				incremental = True
			else:
				print 'unknown option: %s' % arg
				print_usage()
				sys.exit(1)
		else:
			toolsets.append(arg)

	if toolsets == []:
		print_usage()
		sys.exit(1)

	try:
		cfg = open('.regression.yml', 'r')
	except:
		print '.regression.yml not found in current directory'
		sys.exit(1)

	cfg = yaml.load(cfg.read())

	test_dirs = []
	configs = []
	options = ['boost=source']
	time_limit = 1200
	if 'test_dirs' in cfg:
		for d in cfg['test_dirs']:
			test_dirs.append(d)
	else:
		print 'no test directory specified by .regression.yml'
		sys.exit(1)

	configs = []
	if 'features' in cfg:
		for d in cfg['features']:
			configs.append(d)
	else:
		configs = [['']]

	clean_files = []
	if 'clean' in cfg:
		clean_files = cfg['clean']

	branch_name = 'trunk'
	if 'branch' in cfg:
		branch_name = cfg['branch']

	if 'time_limit' in cfg:
		time_limit = int(cfg['time_limit'])

	architecture = platform.machine()
	build_platform = platform.system() + '-' + platform.release()

	revision, author = svn_info()

	timestamp = datetime.now()

	tester_pool = Pool(processes=num_processes)

	print '%s-%d - %s - %s' % (branch_name, revision, author, timestamp)

	print 'toolsets: %s' % ' '.join(toolsets)
#	print 'configs: %s' % '|'.join(configs)

	current_dir = os.getcwd()

	try:
		rev_dir = os.path.join(current_dir, 'regression_tests')
		try: os.mkdir(rev_dir)
		except: pass
		rev_dir = os.path.join(rev_dir, '%s-%d' % (branch_name, revision))
		try: os.mkdir(rev_dir)
		except: pass

		for test_dir in test_dirs:
			print 'running tests from "%s" in %s' % (test_dir, branch_name)
			os.chdir(test_dir)
			test_dir = os.getcwd()

			# figure out which tests are exported by this Jamfile
			p = subprocess.Popen(['bjam', '--dump-tests', 'non-existing-target'], stdout=subprocess.PIPE)

			tests = []

			for l in p.stdout:
				if not 'boost-test(RUN)' in l: continue
				test_name = os.path.split(l.split(' ')[1][1:-1])[1]
				tests.append(test_name)
			print 'found %d tests' % len(tests)

			for toolset in toolsets:
				results = {}
				toolset_found = False

				futures = []
				for features in configs:
					futures.append(tester_pool.apply_async(run_tests, [toolset, tests, features, options, test_dir, time_limit, incremental]))

				for future in futures:
					(toolset, r) = future.get()
					results.update(r)

#				for features in configs:
#					(toolset, r) = run_tests(toolset, tests, features, options, test_dir, time_limit, incremental)
#					results.update(r)

				print ''

				# each file contains a full set of tests for one speific toolset and platform
				try:
					f = open(os.path.join(rev_dir, build_platform + '#' + toolset + '.json'), 'w+')
				except IOError:
					rev_dir = os.path.join(current_dir, 'regression_tests')
					try: os.mkdir(rev_dir)
					except: pass
					rev_dir = os.path.join(rev_dir, '%s-%d' % (branch_name, revision))
					try: os.mkdir(rev_dir)
					except: pass
					f = open(os.path.join(rev_dir, build_platform + '#' + toolset + '.json'), 'w+')

				print >>f, json.dumps(results)
				f.close()

				if len(clean_files) > 0:
					for filt in clean_files:
						for f in glob.glob(filt):
							# a precautio to make sure a malicious repo
							# won't clean things outside of the test directory
							if not os.path.abspath(f).startswith(test_dir): continue
							print 'deleting %s' %f
							shutil.rmtree(f)
	finally:
		# always restore current directory
		os.chdir(current_dir)

if __name__ == "__main__":
    main(sys.argv[1:])

<|endoftext|>


<commit_before><commit_msg>Netflow aggregator: skip invalid record, closes https://github.com/opnsense/core/issues/4796<commit_after><|endoftext|>


<commit_before>#!/bin/python

# Copyright (c) 2013, Arvid Norberg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the distribution.
#     * Neither the name of the author nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# this is meant to be run from the root of the repository
# the arguments are the boost-build toolsets to use.
# these will vary between testers and operating systems
# common ones are: clang, darwin, gcc, msvc, icc

import random
import os
import platform
import subprocess
import xml.etree.ElementTree as et
from datetime import datetime
import json
import sys
import yaml
from multiprocessing import Pool
import glob
import shutil

# the .regression.yml configuration file format looks like this (it's yaml):

# test_dirs:
# - <path-to-test-folder>
# - ...
#
# features:
# - <list of boost-built features>
# - ...
#

def svn_info():
	# figure out which revision this is
	p = subprocess.Popen(['svn', 'info'], stdout=subprocess.PIPE)

	revision = -1
	author = ''

	for l in p.stdout:
		if 'Last Changed Rev' in l:
			revision = int(l.split(':')[1].strip())
		if 'Last Changed Author' in l:
			author = l.split(':')[1].strip()

	if revision == -1:
		print 'Failed to extract subversion revision'
		sys.exit(1)

	if author == '':
		print 'Failed to extract subversion author'
		sys.exit(1)

	return (revision, author)

def run_tests(toolset, tests, features, options, test_dir, time_limit, incremental):
	xml_file = 'bjam_build.%d.xml' % random.randint(0, 100000)
	try:

		results = {}
		toolset_found = False

		os.chdir(test_dir)

		if not incremental:
			p = subprocess.Popen(['bjam', '--abbreviate-paths', toolset, 'clean'] + options + features.split(' '), stdout=subprocess.PIPE)
			for l in p.stdout: pass
			p.wait()

		for t in tests:
			p = subprocess.Popen(['bjam', '--out-xml=%s' % xml_file, '-l%d' % time_limit, '-q', '--abbreviate-paths', toolset, t] + options + features.split(' '), stdout=subprocess.PIPE)
			output = ''
			for l in p.stdout:
				output += l.decode('latin-1')
			p.wait()

			# parse out the toolset version from the xml file
			compiler = ''
			compiler_version = ''
			command = ''

			# make this parse the actual test to pick up the time
			# spent runnin the test
			try:
				dom = et.parse(xml_file)

				command = dom.find('./command').text

				prop = dom.findall('./action/properties/property')
				for a in prop:
					name = a.attrib['name']
					if name == 'toolset':
						compiler = a.text
						if compiler_version != '': break
					if name.startswith('toolset-') and name.endswith(':version'):
						compiler_version = a.text
						if compiler != '': break

				if compiler != '' and compiler_version != '':
					toolset = compiler + '-' + compiler_version
			except: pass

			r = { 'status': p.returncode, 'output': output, 'command': command }
			results[t + '|' + features] = r

			fail_color = '\033[31;1m'
			pass_color = '\033[32;1m'
			end_seq = '\033[0m'

			if platform.system() == 'Windows':
				fail_color == ''
				pass_color == ''
				end_seq = ''

			if p.returncode == 0: sys.stdout.write('.')
			else: sys.stdout.write('X')
			sys.stdout.flush()

	except:
		# need this to make child processes exit
		sys.exit(1)
	finally:
		try: os.unlink(xml_file)
		except: pass

	return (toolset, results)

def print_usage():
		print '''usage: run_tests.py [options] bjam-toolset [bjam-toolset...]
options:
-j<n>     use n parallel processes
-h        prints this message and exits
-i        build incrementally (i.e. don't clean between checkouts)
'''

def main(argv):

	toolsets = []

	num_processes = 4
	incremental = False

	for arg in argv:
		if arg[0] == '-':
			if arg[1] == 'j':
				num_processes = int(arg[2:])
			elif arg[1] == 'h':
				print_usage()
				sys.exit(1)
			elif arg[1] == 'i':
				incremental = True
			else:
				print 'unknown option: %s' % arg
				print_usage()
				sys.exit(1)
		else:
			toolsets.append(arg)

	if toolsets == []:
		print_usage()
		sys.exit(1)

	try:
		cfg = open('.regression.yml', 'r')
	except:
		print '.regression.yml not found in current directory'
		sys.exit(1)

	cfg = yaml.load(cfg.read())

	test_dirs = []
	configs = []
	options = ['boost=source']
	time_limit = 1200
	if 'test_dirs' in cfg:
		for d in cfg['test_dirs']:
			test_dirs.append(d)
	else:
		print 'no test directory specified by .regression.yml'
		sys.exit(1)

	configs = []
	if 'features' in cfg:
		for d in cfg['features']:
			configs.append(d)
	else:
		configs = [['']]

	clean_files = []
	if 'clean' in cfg:
		clean_files = cfg['clean']

	branch_name = 'trunk'
	if 'branch' in cfg:
		branch_name = cfg['branch']

	if 'time_limit' in cfg:
		time_limit = int(cfg['time_limit'])

	architecture = platform.machine()
	build_platform = platform.system() + '-' + platform.release()

	revision, author = svn_info()

	timestamp = datetime.now()

	tester_pool = Pool(processes=num_processes)

	print '%s-%d - %s - %s' % (branch_name, revision, author, timestamp)

	print 'toolsets: %s' % ' '.join(toolsets)
#	print 'configs: %s' % '|'.join(configs)

	current_dir = os.getcwd()

	try:
		rev_dir = os.path.join(current_dir, 'regression_tests')
		try: os.mkdir(rev_dir)
		except: pass
		rev_dir = os.path.join(rev_dir, '%s-%d' % (branch_name, revision))
		try: os.mkdir(rev_dir)
		except: pass

		for test_dir in test_dirs:
			print 'running tests from "%s" in %s' % (test_dir, branch_name)
			os.chdir(test_dir)
			test_dir = os.getcwd()

			# figure out which tests are exported by this Jamfile
			p = subprocess.Popen(['bjam', '--dump-tests', 'non-existing-target'], stdout=subprocess.PIPE)

			tests = []

			for l in p.stdout:
				if not 'boost-test(RUN)' in l: continue
				test_name = os.path.split(l.split(' ')[1][1:-1])[1]
				tests.append(test_name)
			print 'found %d tests' % len(tests)

			for toolset in toolsets:
				results = {}
				toolset_found = False

				futures = []
				for features in configs:
					futures.append(tester_pool.apply_async(run_tests, [toolset, tests, features, options, test_dir, time_limit, incremental]))

				for future in futures:
					(toolset, r) = future.get()
					results.update(r)

#				for features in configs:
#					(toolset, r) = run_tests(toolset, tests, features, options, test_dir, time_limit, incremental)
#					results.update(r)

				print ''

				# each file contains a full set of tests for one speific toolset and platform
				try:
					f = open(os.path.join(rev_dir, build_platform + '#' + toolset + '.json'), 'w+')
				except IOError:
					rev_dir = os.path.join(current_dir, 'regression_tests')
					try: os.mkdir(rev_dir)
					except: pass
					rev_dir = os.path.join(rev_dir, '%s-%d' % (branch_name, revision))
					try: os.mkdir(rev_dir)
					except: pass
					f = open(os.path.join(rev_dir, build_platform + '#' + toolset + '.json'), 'w+')

				print >>f, json.dumps(results)
				f.close()

				if len(clean_files) > 0:
					for filt in clean_files:
						for f in glob.glob(filt):
							# a precautio to make sure a malicious repo
							# won't clean things outside of the test directory
							if not os.path.abspath(f).startswith(test_dir): continue
							print 'deleting %s' %f
							shutil.rmtree(f)
	finally:
		# always restore current directory
		os.chdir(current_dir)

if __name__ == "__main__":
    main(sys.argv[1:])

<commit_msg>add error reporting and fix bug in run_tests.py<commit_after>#!/bin/python

# Copyright (c) 2013, Arvid Norberg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the distribution.
#     * Neither the name of the author nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# this is meant to be run from the root of the repository
# the arguments are the boost-build toolsets to use.
# these will vary between testers and operating systems
# common ones are: clang, darwin, gcc, msvc, icc

import random
import os
import platform
import subprocess
import xml.etree.ElementTree as et
from datetime import datetime
import json
import sys
import yaml
from multiprocessing import Pool
import glob
import shutil
import traceback

# the .regression.yml configuration file format looks like this (it's yaml):

# test_dirs:
# - <path-to-test-folder>
# - ...
#
# features:
# - <list of boost-built features>
# - ...
#

def svn_info():
	# figure out which revision this is
	p = subprocess.Popen(['svn', 'info'], stdout=subprocess.PIPE)

	revision = -1
	author = ''

	for l in p.stdout:
		if 'Last Changed Rev' in l:
			revision = int(l.split(':')[1].strip())
		if 'Last Changed Author' in l:
			author = l.split(':')[1].strip()

	if revision == -1:
		print 'Failed to extract subversion revision'
		sys.exit(1)

	if author == '':
		print 'Failed to extract subversion author'
		sys.exit(1)

	return (revision, author)

def run_tests(toolset, tests, features, options, test_dir, time_limit, incremental):
	xml_file = 'bjam_build.%d.xml' % random.randint(0, 100000)
	try:

		results = {}
		toolset_found = False

		os.chdir(test_dir)

#		if not incremental:
#			p = subprocess.Popen(['bjam', '--abbreviate-paths', toolset, 'clean'] + options + features., stdout=subprocess.PIPE)
#			for l in p.stdout: pass
#			p.wait()

		for t in tests:
#			print 'calling bjam %s %s %s' % (toolset, t, ' '.join(features))
			p = subprocess.Popen(['bjam', '--out-xml=%s' % xml_file, '-l%d' % time_limit, '-q', '--abbreviate-paths', toolset, t] + options + features, stdout=subprocess.PIPE)
			output = ''
			for l in p.stdout:
				output += l.decode('latin-1')
			p.wait()

			# parse out the toolset version from the xml file
			compiler = ''
			compiler_version = ''
			command = ''

			# make this parse the actual test to pick up the time
			# spent runnin the test
			try:
				dom = et.parse(xml_file)

				command = dom.find('./command').text

				prop = dom.findall('./action/properties/property')
				for a in prop:
					name = a.attrib['name']
					if name == 'toolset':
						compiler = a.text
						if compiler_version != '': break
					if name.startswith('toolset-') and name.endswith(':version'):
						compiler_version = a.text
						if compiler != '': break

				if compiler != '' and compiler_version != '':
					toolset = compiler + '-' + compiler_version
			except: pass

			r = { 'status': p.returncode, 'output': output, 'command': command }
			results[t + '|' + features] = r

			if p.returncode == 0: sys.stdout.write('.')
			else: sys.stdout.write('X')
			sys.stdout.flush()

	except Exception, e:
		# need this to make child processes exit
		print 'exiting test process: ', traceback.format_exc()
		sys.exit(1)
	finally:
		print 'done runnint tests'
		try: os.unlink(xml_file)
		except: pass

	return (toolset, results)

def print_usage():
		print '''usage: run_tests.py [options] bjam-toolset [bjam-toolset...]
options:
-j<n>     use n parallel processes
-h        prints this message and exits
-i        build incrementally (i.e. don't clean between checkouts)
'''

def main(argv):

	toolsets = []

	num_processes = 4
	incremental = False

	for arg in argv:
		if arg[0] == '-':
			if arg[1] == 'j':
				num_processes = int(arg[2:])
			elif arg[1] == 'h':
				print_usage()
				sys.exit(1)
			elif arg[1] == 'i':
				incremental = True
			else:
				print 'unknown option: %s' % arg
				print_usage()
				sys.exit(1)
		else:
			toolsets.append(arg)

	if toolsets == []:
		print_usage()
		sys.exit(1)

	try:
		cfg = open('.regression.yml', 'r')
	except:
		print '.regression.yml not found in current directory'
		sys.exit(1)

	cfg = yaml.load(cfg.read())

	test_dirs = []
	configs = []
	options = ['boost=source']
	time_limit = 1200
	if 'test_dirs' in cfg:
		for d in cfg['test_dirs']:
			test_dirs.append(d)
	else:
		print 'no test directory specified by .regression.yml'
		sys.exit(1)

	configs = []
	if 'features' in cfg:
		for d in cfg['features']:
			configs.append(d)
	else:
		configs = [['']]

	clean_files = []
	if 'clean' in cfg:
		clean_files = cfg['clean']

	branch_name = 'trunk'
	if 'branch' in cfg:
		branch_name = cfg['branch']

	if 'time_limit' in cfg:
		time_limit = int(cfg['time_limit'])

	architecture = platform.machine()
	build_platform = platform.system() + '-' + platform.release()

	revision, author = svn_info()

	timestamp = datetime.now()

	tester_pool = Pool(processes=num_processes)

	print '%s-%d - %s - %s' % (branch_name, revision, author, timestamp)

	print 'toolsets: %s' % ' '.join(toolsets)
#	print 'configs: %s' % '|'.join(configs)

	current_dir = os.getcwd()

	try:
		rev_dir = os.path.join(current_dir, 'regression_tests')
		try: os.mkdir(rev_dir)
		except: pass
		rev_dir = os.path.join(rev_dir, '%s-%d' % (branch_name, revision))
		try: os.mkdir(rev_dir)
		except: pass

		for test_dir in test_dirs:
			print 'running tests from "%s" in %s' % (test_dir, branch_name)
			os.chdir(test_dir)
			test_dir = os.getcwd()

			# figure out which tests are exported by this Jamfile
			p = subprocess.Popen(['bjam', '--dump-tests', 'non-existing-target'], stdout=subprocess.PIPE)

			tests = []

			for l in p.stdout:
				if not 'boost-test(RUN)' in l: continue
				test_name = os.path.split(l.split(' ')[1][1:-1])[1]
				tests.append(test_name)
			print 'found %d tests' % len(tests)

			for toolset in toolsets:
				results = {}
				toolset_found = False

				futures = []
				for features in configs:
					futures.append(tester_pool.apply_async(run_tests, [toolset, tests, features, options, test_dir, time_limit, incremental]))

				for future in futures:
					(toolset, r) = future.get()
					results.update(r)

#				for features in configs:
#					(toolset, r) = run_tests(toolset, tests, features, options, test_dir, time_limit, incremental)
#					results.update(r)

				print ''

				# each file contains a full set of tests for one speific toolset and platform
				try:
					f = open(os.path.join(rev_dir, build_platform + '#' + toolset + '.json'), 'w+')
				except IOError:
					rev_dir = os.path.join(current_dir, 'regression_tests')
					try: os.mkdir(rev_dir)
					except: pass
					rev_dir = os.path.join(rev_dir, '%s-%d' % (branch_name, revision))
					try: os.mkdir(rev_dir)
					except: pass
					f = open(os.path.join(rev_dir, build_platform + '#' + toolset + '.json'), 'w+')

				print >>f, json.dumps(results)
				f.close()

				if len(clean_files) > 0:
					for filt in clean_files:
						for f in glob.glob(filt):
							# a precautio to make sure a malicious repo
							# won't clean things outside of the test directory
							if not os.path.abspath(f).startswith(test_dir): continue
							print 'deleting %s' %f
							shutil.rmtree(f)
	finally:
		# always restore current directory
		os.chdir(current_dir)

if __name__ == "__main__":
    main(sys.argv[1:])

<|endoftext|>


<commit_before>import functools
import subprocess
import os
import string

from optparse import OptionParser

OLD_CWD = os.getcwd()

parser = OptionParser()
parser.add_option("-r", "--root", dest="root", default="../ROOT",
        help="Root directory for the env", metavar="ROOT")
parser.add_option("-y", "--no-x", dest="no_x", default=False,
        action="store_true", help="Do not build anything depending on X")
parser.add_option("-c", "--client", dest="client", default=False,
        action="store_true", help="Include the client interfaces")
(options, args) = parser.parse_args()

OCAML='3.12.1'
ROOT = os.path.realpath(options.root)
PREFIX = "%s/%s" % (ROOT,'OCAML')

def sh(x, **kwargs):
    print x
    if subprocess.call(x,**kwargs):
        raise RuntimeError("Failed to run %s %s" % (x, kwargs))

def mr_proper():
    sh (['rm','-rf', ROOT])

print PREFIX
env = {'PATH': string.join([PREFIX + '/bin',
                            '/bin',
                            '/usr/bin',
                            ],':')}

extract_flags = {
    '.tar.gz':'-zxvf',
    '.tar.bz2':'-jxvf',
    '.tbz': '-jxvf',
    }

class Lib:
    def __init__(self, name, extension, url_t):
        self._name = name
        self._extension = extension
        self._archive = name + extension
        self._url = url_t % self._archive

    def download(self):
        fn = '%s/%s' % (ROOT, self._archive)
        sh(['wget', '-O', fn, self._url])

    def extract(self):
        flags = extract_flags[self._extension]
        sh(['tar', flags, self._archive], cwd = ROOT)

    def sh(self, cmd):
        d = '%s/%s' % (ROOT, self._name)
        sh(cmd, cwd = d, env = env)

def install_ocaml():
    lib = Lib('ocaml-%s' % OCAML,'.tar.bz2',
              'http://caml.inria.fr/pub/distrib/ocaml-3.12/%s')
    lib.download()
    lib.extract()
    lib.sh(['./configure', '--prefix',PREFIX])
    lib.sh(['make','world.opt'])
    lib.sh(['make','install'])

def install_ocamlfind():
    lib = Lib('findlib-1.2.7', '.tar.gz',
              'http://download.camlcity.org/download/%s')
    lib.download()
    lib.extract()
    lib.sh(['./configure'])
    lib.sh(['make','all','opt','install'])

def install_ounit():
    lib = Lib('ounit-1.1.0','.tar.gz',
              'http://forge.ocamlcore.org/frs/download.php/495/%s')
    lib.download()
    lib.extract()
    lib.sh(['ocaml', 'setup.ml', '-configure'])
    lib.sh(['ocaml', 'setup.ml', '-build'])
    lib.sh(['ocaml', 'setup.ml', '-install'])

def install_react():
    lib = Lib('react-0.9.2','.tbz',
              'http://erratique.ch/software/react/releases/%s')
    lib.download()
    lib.extract()
    lib.sh(['chmod','u+x','build'])
    lib.sh(['./build'])
    lib.sh(['./build','install'])
    sh(['mv', '%s/lib/ocaml/react' % PREFIX,
        '%s/lib/ocaml/site-lib/' % PREFIX])

def install_lwt():
    # Tell lwt where libev can be found
    env['LIBRARY_PATH'] = '%s/lib' % PREFIX
    env['C_INCLUDE_PATH'] = '%s/include' % PREFIX

    lib = Lib('lwt-2.3.0','.tar.gz',
              'http://ocsigen.org/download/%s')
    lib.download()
    lib.extract()
    lib.sh(['rm','-f',  'setup.data','setup.log'])
    lib.sh(['make','clean'])
    lib.sh(['ocaml', 'setup.ml', '-configure', '--prefix', PREFIX])
    lib.sh(['ocaml', 'setup.ml', '-build'])
    lib.sh(['ocaml', 'setup.ml', '-install'])

def install_camlbz2():
    lib = Lib('camlbz2-0.6.0','.tar.gz',
              'https://forge.ocamlcore.org/frs/download.php/72/%s')
    lib.download()
    lib.extract()
    lib.sh(['./configure'])
    lib.sh(['make', 'all'])
    lib.sh(['make', 'install'])

def install_lablgtk():
    lib = Lib('lablgtk-2.14.2',
              '.tar.gz',
              'http://wwwfun.kurims.kyoto-u.ac.jp/soft/olabl/dist/%s')
    lib.download()
    lib.extract()
    lib.sh(['./configure','--prefix=%s' % PREFIX])
    lib.sh(['make','world'])
    lib.sh(['make', 'install'])

def install_cairo_ocaml():
    lib = Lib('cairo-ocaml-1.2.0',
              '.tar.bz2',
              'http://cgit.freedesktop.org/cairo-ocaml/snapshot/%s')
    lib.download()
    lib.extract()
    lib.sh(['aclocal','-I','support'])
    lib.sh(['autoconf'])
    lib.sh(['./configure','--prefix=%s' % PREFIX])
    lib.sh(['make'])
    lib.sh(['make', 'install'])

def install_ocamlviz():
    lib = Lib('ocamlviz-1.01', '.tar.gz',
              'http://ocamlviz.forge.ocamlcore.org/%s')
    lib.download()
    lib.extract()
    # the rest is foefelare-trucare(TM)
    d = '%s/%s' % (ROOT, 'ocamlviz') ## differs from the rest
    sh(['autoconf'], cwd = d, env = env)
    sh(['./configure','--prefix=%s' % PREFIX ], cwd = d , env = env)
    sh(['make'], cwd = d, env = env)
    sh(['make','install'], cwd = d, env = env)
    LIBDIR = PREFIX + '/lib/ocaml/site-lib/ocamlviz'
    sh(['mkdir','-p',LIBDIR])
    files = ['src/ocamlviz.mli',
             'src/ocamlviz.cmi',
             'src/ocamlviz_threads.cmi']
    cmd = ['cp','-f']
    cmd.extend(files)
    cmd.append(LIBDIR)
    sh(cmd, cwd = d)

    files2 = [
        'libocamlviz.a',
        'libocamlviz.cma',
        'libocamlviz.cmxa',
        'camlp4/pa_ocamlviz.cmi',
        'camlp4/pa_ocamlviz.cmo',
        ]
    cmd = ['cp','-f']
    cmd.extend(files2)
    cmd.append(LIBDIR)
    sh(cmd,cwd = d)
    sh(['cp','-f','ascii.opt', PREFIX + '/bin/ocamlviz-ascii'], cwd =d)
    sh(['cp','-f','gui.opt', PREFIX + '/bin/ocamlviz'], cwd = d)
    # create a META file
    meta = """
version = "1.0.1"
description = "real time profiling"
requires = "unix"
archive(byte) = "libocamlviz.cma"
archive(native) = "libocamlviz.cmxa"
exists_if = "libocamlviz.cma"

package "syntax" (
  exists_if = "pa_ocamlviz.cmo"
  description = "auto instrumentation sugars for ocamlviz"
  requires = "camlp4"
  archive(syntax,preprocessor) = "pa_ocamlviz.cmo"

)
"""
    f = open('%s/META' % LIBDIR, 'w')
    f.write(meta)
    f.close()

def install_libev():
    lib = Lib('libev-4.04','.tar.gz',
              'http://dist.schmorp.de/libev/%s')
    lib.download()
    lib.extract()
    lib.sh(['./configure', '--prefix=%s' % PREFIX])
    lib.sh(['make'])
    lib.sh(['make', 'install'])

def install_client():
    env['LIBRARY_PATH'] = os.path.join(PREFIX, "lib")
    client_sh = functools.partial(sh, cwd=OLD_CWD, env=env)
    client_sh(["ocamlbuild", "-use-ocamlfind", "arakoon_client.cma", "arakoon_client.cmxa", "arakoon_client.a"])
    env['OCAML_LIBDIR'] = os.path.join(PREFIX, "lib", "ocaml", "site-lib")
    client_sh(["make", "uninstall_client"])
    client_sh(["make", "install_client"])

def do_it():
    mr_proper()
    sh(['mkdir', ROOT])
    install_ocaml()
    install_ocamlfind()
    install_ounit()
    install_react()
    install_libev()
    install_lwt()
    install_camlbz2()
    if not options.no_x:
        install_lablgtk()
        install_cairo_ocaml()
        install_ocamlviz()
    if options.client:
        install_client()
    #sudo cp lablgtk-2.14.2/examples/test.xpm /usr/share/pixmaps/ocaml.xpm
    print '\n\nnow prepend %s/bin to your PATH' % PREFIX

if __name__ == '__main__':
    do_it()
<commit_msg>No longer use Makefile for install of client<commit_after>import functools
import subprocess
import os
import string

from optparse import OptionParser

OLD_CWD = os.getcwd()

parser = OptionParser()
parser.add_option("-r", "--root", dest="root", default="../ROOT",
        help="Root directory for the env", metavar="ROOT")
parser.add_option("-y", "--no-x", dest="no_x", default=False,
        action="store_true", help="Do not build anything depending on X")
parser.add_option("-c", "--client", dest="client", default=False,
        action="store_true", help="Include the client interfaces")
(options, args) = parser.parse_args()

OCAML='3.12.1'
ROOT = os.path.realpath(options.root)
PREFIX = "%s/%s" % (ROOT,'OCAML')

def sh(x, **kwargs):
    print x
    if subprocess.call(x,**kwargs):
        raise RuntimeError("Failed to run %s %s" % (x, kwargs))

def mr_proper():
    sh (['rm','-rf', ROOT])

print PREFIX
env = {'PATH': string.join([PREFIX + '/bin',
                            '/bin',
                            '/usr/bin',
                            ],':')}

extract_flags = {
    '.tar.gz':'-zxvf',
    '.tar.bz2':'-jxvf',
    '.tbz': '-jxvf',
    }

class Lib:
    def __init__(self, name, extension, url_t):
        self._name = name
        self._extension = extension
        self._archive = name + extension
        self._url = url_t % self._archive

    def download(self):
        fn = '%s/%s' % (ROOT, self._archive)
        sh(['wget', '-O', fn, self._url])

    def extract(self):
        flags = extract_flags[self._extension]
        sh(['tar', flags, self._archive], cwd = ROOT)

    def sh(self, cmd):
        d = '%s/%s' % (ROOT, self._name)
        sh(cmd, cwd = d, env = env)

def install_ocaml():
    lib = Lib('ocaml-%s' % OCAML,'.tar.bz2',
              'http://caml.inria.fr/pub/distrib/ocaml-3.12/%s')
    lib.download()
    lib.extract()
    lib.sh(['./configure', '--prefix',PREFIX])
    lib.sh(['make','world.opt'])
    lib.sh(['make','install'])

def install_ocamlfind():
    lib = Lib('findlib-1.2.7', '.tar.gz',
              'http://download.camlcity.org/download/%s')
    lib.download()
    lib.extract()
    lib.sh(['./configure'])
    lib.sh(['make','all','opt','install'])

def install_ounit():
    lib = Lib('ounit-1.1.0','.tar.gz',
              'http://forge.ocamlcore.org/frs/download.php/495/%s')
    lib.download()
    lib.extract()
    lib.sh(['ocaml', 'setup.ml', '-configure'])
    lib.sh(['ocaml', 'setup.ml', '-build'])
    lib.sh(['ocaml', 'setup.ml', '-install'])

def install_react():
    lib = Lib('react-0.9.2','.tbz',
              'http://erratique.ch/software/react/releases/%s')
    lib.download()
    lib.extract()
    lib.sh(['chmod','u+x','build'])
    lib.sh(['./build'])
    lib.sh(['./build','install'])
    sh(['mv', '%s/lib/ocaml/react' % PREFIX,
        '%s/lib/ocaml/site-lib/' % PREFIX])

def install_lwt():
    # Tell lwt where libev can be found
    env['LIBRARY_PATH'] = '%s/lib' % PREFIX
    env['C_INCLUDE_PATH'] = '%s/include' % PREFIX

    lib = Lib('lwt-2.3.0','.tar.gz',
              'http://ocsigen.org/download/%s')
    lib.download()
    lib.extract()
    lib.sh(['rm','-f',  'setup.data','setup.log'])
    lib.sh(['make','clean'])
    lib.sh(['ocaml', 'setup.ml', '-configure', '--prefix', PREFIX])
    lib.sh(['ocaml', 'setup.ml', '-build'])
    lib.sh(['ocaml', 'setup.ml', '-install'])

def install_camlbz2():
    lib = Lib('camlbz2-0.6.0','.tar.gz',
              'https://forge.ocamlcore.org/frs/download.php/72/%s')
    lib.download()
    lib.extract()
    lib.sh(['./configure'])
    lib.sh(['make', 'all'])
    lib.sh(['make', 'install'])

def install_lablgtk():
    lib = Lib('lablgtk-2.14.2',
              '.tar.gz',
              'http://wwwfun.kurims.kyoto-u.ac.jp/soft/olabl/dist/%s')
    lib.download()
    lib.extract()
    lib.sh(['./configure','--prefix=%s' % PREFIX])
    lib.sh(['make','world'])
    lib.sh(['make', 'install'])

def install_cairo_ocaml():
    lib = Lib('cairo-ocaml-1.2.0',
              '.tar.bz2',
              'http://cgit.freedesktop.org/cairo-ocaml/snapshot/%s')
    lib.download()
    lib.extract()
    lib.sh(['aclocal','-I','support'])
    lib.sh(['autoconf'])
    lib.sh(['./configure','--prefix=%s' % PREFIX])
    lib.sh(['make'])
    lib.sh(['make', 'install'])

def install_ocamlviz():
    lib = Lib('ocamlviz-1.01', '.tar.gz',
              'http://ocamlviz.forge.ocamlcore.org/%s')
    lib.download()
    lib.extract()
    # the rest is foefelare-trucare(TM)
    d = '%s/%s' % (ROOT, 'ocamlviz') ## differs from the rest
    sh(['autoconf'], cwd = d, env = env)
    sh(['./configure','--prefix=%s' % PREFIX ], cwd = d , env = env)
    sh(['make'], cwd = d, env = env)
    sh(['make','install'], cwd = d, env = env)
    LIBDIR = PREFIX + '/lib/ocaml/site-lib/ocamlviz'
    sh(['mkdir','-p',LIBDIR])
    files = ['src/ocamlviz.mli',
             'src/ocamlviz.cmi',
             'src/ocamlviz_threads.cmi']
    cmd = ['cp','-f']
    cmd.extend(files)
    cmd.append(LIBDIR)
    sh(cmd, cwd = d)

    files2 = [
        'libocamlviz.a',
        'libocamlviz.cma',
        'libocamlviz.cmxa',
        'camlp4/pa_ocamlviz.cmi',
        'camlp4/pa_ocamlviz.cmo',
        ]
    cmd = ['cp','-f']
    cmd.extend(files2)
    cmd.append(LIBDIR)
    sh(cmd,cwd = d)
    sh(['cp','-f','ascii.opt', PREFIX + '/bin/ocamlviz-ascii'], cwd =d)
    sh(['cp','-f','gui.opt', PREFIX + '/bin/ocamlviz'], cwd = d)
    # create a META file
    meta = """
version = "1.0.1"
description = "real time profiling"
requires = "unix"
archive(byte) = "libocamlviz.cma"
archive(native) = "libocamlviz.cmxa"
exists_if = "libocamlviz.cma"

package "syntax" (
  exists_if = "pa_ocamlviz.cmo"
  description = "auto instrumentation sugars for ocamlviz"
  requires = "camlp4"
  archive(syntax,preprocessor) = "pa_ocamlviz.cmo"

)
"""
    f = open('%s/META' % LIBDIR, 'w')
    f.write(meta)
    f.close()

def install_libev():
    lib = Lib('libev-4.04','.tar.gz',
              'http://dist.schmorp.de/libev/%s')
    lib.download()
    lib.extract()
    lib.sh(['./configure', '--prefix=%s' % PREFIX])
    lib.sh(['make'])
    lib.sh(['make', 'install'])

def install_client():
    client_sh = functools.partial(sh, cwd=OLD_CWD, env=env)
    client_sh(["ocamlbuild", "-use-ocamlfind", "arakoon_client.cma", "arakoon_client.cmxa", "arakoon_client.a"])
    client_sh(["ocamlfind", "remove", "arakoon_client"])
    client_sh(["ocamlfind", "install", "arakoon_client",
        "META",
        "_build/src/arakoon_client.cma",
        "_build/src/arakoon_client.cmxa",
        "_build/src/client/arakoon_exc.mli",
        "_build/src/client/arakoon_exc.cmi",
        "_build/src/client/arakoon_client.mli",
        "_build/src/client/arakoon_client.cmi",
        "_build/src/client/arakoon_remote_client.mli",
        "_build/src/client/arakoon_remote_client.cmi",
        "_build/src/plugins/registry.mli",
        "_build/src/plugins/registry.cmi",
        "_build/src/tools/llio.mli",
        "_build/src/tools/llio.cmi",
        "_build/src/arakoon_client.a"
        ])

def do_it():
    mr_proper()
    sh(['mkdir', ROOT])
    install_ocaml()
    install_ocamlfind()
    install_ounit()
    install_react()
    install_libev()
    install_lwt()
    install_camlbz2()
    if not options.no_x:
        install_lablgtk()
        install_cairo_ocaml()
        install_ocamlviz()
    if options.client:
        install_client()
    #sudo cp lablgtk-2.14.2/examples/test.xpm /usr/share/pixmaps/ocaml.xpm
    print '\n\nnow prepend %s/bin to your PATH' % PREFIX

if __name__ == '__main__':
    do_it()
<|endoftext|>


<commit_before>from .util import prepare_for_display, window_manager, GuiLockError
import numpy as np
import sys

# We try to aquire the gui lock first or else the gui import might
# trample another GUI's PyOS_InputHook.
window_manager.acquire('qt')

try:
    from PyQt4.QtGui import (QApplication, QImage,
                             QLabel, QMainWindow, QPixmap, QWidget)
    from PyQt4 import QtCore, QtGui
    import sip
    import warnings

except ImportError:
    window_manager._release('qt')

    raise ImportError("""\
    PyQt4 libraries not installed. Please refer to

    http://www.riverbankcomputing.co.uk/software/pyqt/intro

    for more information.  PyQt4 is GPL licensed.  For an
    LGPL equivalent, see

    http://www.pyside.org
    """)

app = None

class ImageLabel(QLabel):
    def __init__(self, parent, arr):
        QLabel.__init__(self)

        # we need to hold a reference to
        # arr because QImage doesn't copy the data
        # and the buffer must be alive as long
        # as the image is alive.
        self.arr = arr

        # we also need to pass in the row-stride to
        # the constructor, because we can't guarantee
        # that every row of the numpy data is
        # 4-byte aligned. Which Qt would require
        # if we didnt pass the stride.
        self.img = QImage(arr.data, arr.shape[1], arr.shape[0],
                          arr.strides[0], QImage.Format_RGB888)
        self.pm = QPixmap.fromImage(self.img)
        self.setPixmap(self.pm)
        self.setAlignment(QtCore.Qt.AlignTop)
        self.setMinimumSize(100, 100)

    def resizeEvent(self, evt):
        width = self.width()
        pm = QPixmap.fromImage(self.img)
        self.pm = pm.scaledToWidth(width)
        self.setPixmap(self.pm)

class ImageWindow(QMainWindow):
    def __init__(self, arr, mgr):
        QMainWindow.__init__(self)
        self.setWindowTitle('skimage')
        self.mgr = mgr
        self.main_widget = QWidget()
        self.layout = QtGui.QGridLayout(self.main_widget)
        self.setCentralWidget(self.main_widget)

        self.label = ImageLabel(self, arr)
        self.layout.addWidget(self.label, 0, 0)
        self.layout.addLayout
        self.mgr.add_window(self)
        self.main_widget.show()

    def closeEvent(self, event):
        # Allow window to be destroyed by removing any
        # references to it
        self.mgr.remove_window(self)

def imread_qt(filename):
    """
    Read an image using QT's QImage.load
    """
    qtimg = QImage()
    if not qtimg.load(filename):
        # QImage.load() returns false on failure, so raise an exception
        raise IOError('Unable to load file %s' % filename)
    if qtimg.depth() == 1:
        raise IOError('1-bit images currently not supported')
    # TODO: Warn about other odd formats we don't currently handle properly,
    # such as the odd 16-bit packed formats QT supports
    arrayptr = qtimg.bits()
    # QT may pad the image, so we need to use bytesPerLine, not width for
    # the conversion to a numpy array
    bytesPerPixel = qtimg.depth() // 8
    pixelsPerLine = qtimg.bytesPerLine() // bytesPerPixel
    img_size = pixelsPerLine * qtimg.height() * bytesPerPixel
    arrayptr.setsize(img_size)
    img = np.array(arrayptr)
    # Reshape and trim down to correct dimensions
    if bytesPerPixel > 1:
        img = img.reshape((qtimg.height(), pixelsPerLine, bytesPerPixel))
        img = img[:, :qtimg.width(), :]
    else:
        img = img.reshape((qtimg.height(), pixelsPerLine))
        img = img[:, :qtimg.width()]
    # Strip qt's false alpha channel if needed
    # and reorder color axes as required
    if bytesPerPixel == 4 and not qtimg.hasAlphaChannel():
        img = img[:, :, 2::-1]
    elif bytesPerPixel == 4:
        img[:, :, 0:3] = img[:, :, 2::-1]
    return img

if sip.SIP_VERSION >= 0x040c00:
    # sip.voidptr only acquired a buffer view in 4.12.0, so our imread
    # doesn't work with earlier versions
    imread = imread_qt
else:
    warnings.warn(RuntimeWarning(
        "sip version too old. QT imread disabled"))

def imshow(arr, fancy=False):
    global app
    if not app:
        app = QApplication([])

    arr = prepare_for_display(arr)

    if not fancy:
        iw = ImageWindow(arr, window_manager)
    else:
        from .skivi import SkiviImageWindow
        iw = SkiviImageWindow(arr, window_manager)

    iw.show()

def _app_show():
    global app
    if app and window_manager.has_windows():
        app.exec_()
    else:
        print 'No images to show.  See `imshow`.'

def imsave(filename, img, format_str=None):
    # we can add support for other than 3D uint8 here...
    img = prepare_for_display(img)
    qimg = QImage(img.data, img.shape[1], img.shape[0],
                          img.strides[0], QImage.Format_RGB888)
    if _is_filelike(filename):
        byte_array = QtCore.QByteArray()
        qbuffer = QtCore.QBuffer()
        qbuffer.open(QtCore.QIODevice.ReadWrite)
        saved = qimg.save(qbuffer, format_str.upper())
        qbuffer.seek(0)
        filename.write(qbuffer.readAll())
        qbuffer.close()
    else:
        saved = qimg.save(filename)
    if not saved:
        from textwrap import dedent
        msg = dedent(
            '''The image was not saved. Allowable file formats
            for the QT imsave plugin are:
            BMP, JPG, JPEG, PNG, PPM, TIFF, XBM, XPM''')
        raise RuntimeError(msg)

def _is_filelike(possible_filelike):
    return callable(getattr(possible_filelike, 'write', None))
<commit_msg>open buffer on on byte_array<commit_after>from .util import prepare_for_display, window_manager, GuiLockError
import numpy as np
import sys

# We try to aquire the gui lock first or else the gui import might
# trample another GUI's PyOS_InputHook.
window_manager.acquire('qt')

try:
    from PyQt4.QtGui import (QApplication, QImage,
                             QLabel, QMainWindow, QPixmap, QWidget)
    from PyQt4 import QtCore, QtGui
    import sip
    import warnings

except ImportError:
    window_manager._release('qt')

    raise ImportError("""\
    PyQt4 libraries not installed. Please refer to

    http://www.riverbankcomputing.co.uk/software/pyqt/intro

    for more information.  PyQt4 is GPL licensed.  For an
    LGPL equivalent, see

    http://www.pyside.org
    """)

app = None

class ImageLabel(QLabel):
    def __init__(self, parent, arr):
        QLabel.__init__(self)

        # we need to hold a reference to
        # arr because QImage doesn't copy the data
        # and the buffer must be alive as long
        # as the image is alive.
        self.arr = arr

        # we also need to pass in the row-stride to
        # the constructor, because we can't guarantee
        # that every row of the numpy data is
        # 4-byte aligned. Which Qt would require
        # if we didnt pass the stride.
        self.img = QImage(arr.data, arr.shape[1], arr.shape[0],
                          arr.strides[0], QImage.Format_RGB888)
        self.pm = QPixmap.fromImage(self.img)
        self.setPixmap(self.pm)
        self.setAlignment(QtCore.Qt.AlignTop)
        self.setMinimumSize(100, 100)

    def resizeEvent(self, evt):
        width = self.width()
        pm = QPixmap.fromImage(self.img)
        self.pm = pm.scaledToWidth(width)
        self.setPixmap(self.pm)

class ImageWindow(QMainWindow):
    def __init__(self, arr, mgr):
        QMainWindow.__init__(self)
        self.setWindowTitle('skimage')
        self.mgr = mgr
        self.main_widget = QWidget()
        self.layout = QtGui.QGridLayout(self.main_widget)
        self.setCentralWidget(self.main_widget)

        self.label = ImageLabel(self, arr)
        self.layout.addWidget(self.label, 0, 0)
        self.layout.addLayout
        self.mgr.add_window(self)
        self.main_widget.show()

    def closeEvent(self, event):
        # Allow window to be destroyed by removing any
        # references to it
        self.mgr.remove_window(self)

def imread_qt(filename):
    """
    Read an image using QT's QImage.load
    """
    qtimg = QImage()
    if not qtimg.load(filename):
        # QImage.load() returns false on failure, so raise an exception
        raise IOError('Unable to load file %s' % filename)
    if qtimg.depth() == 1:
        raise IOError('1-bit images currently not supported')
    # TODO: Warn about other odd formats we don't currently handle properly,
    # such as the odd 16-bit packed formats QT supports
    arrayptr = qtimg.bits()
    # QT may pad the image, so we need to use bytesPerLine, not width for
    # the conversion to a numpy array
    bytesPerPixel = qtimg.depth() // 8
    pixelsPerLine = qtimg.bytesPerLine() // bytesPerPixel
    img_size = pixelsPerLine * qtimg.height() * bytesPerPixel
    arrayptr.setsize(img_size)
    img = np.array(arrayptr)
    # Reshape and trim down to correct dimensions
    if bytesPerPixel > 1:
        img = img.reshape((qtimg.height(), pixelsPerLine, bytesPerPixel))
        img = img[:, :qtimg.width(), :]
    else:
        img = img.reshape((qtimg.height(), pixelsPerLine))
        img = img[:, :qtimg.width()]
    # Strip qt's false alpha channel if needed
    # and reorder color axes as required
    if bytesPerPixel == 4 and not qtimg.hasAlphaChannel():
        img = img[:, :, 2::-1]
    elif bytesPerPixel == 4:
        img[:, :, 0:3] = img[:, :, 2::-1]
    return img

if sip.SIP_VERSION >= 0x040c00:
    # sip.voidptr only acquired a buffer view in 4.12.0, so our imread
    # doesn't work with earlier versions
    imread = imread_qt
else:
    warnings.warn(RuntimeWarning(
        "sip version too old. QT imread disabled"))

def imshow(arr, fancy=False):
    global app
    if not app:
        app = QApplication([])

    arr = prepare_for_display(arr)

    if not fancy:
        iw = ImageWindow(arr, window_manager)
    else:
        from .skivi import SkiviImageWindow
        iw = SkiviImageWindow(arr, window_manager)

    iw.show()

def _app_show():
    global app
    if app and window_manager.has_windows():
        app.exec_()
    else:
        print 'No images to show.  See `imshow`.'

def imsave(filename, img, format_str=None):
    # we can add support for other than 3D uint8 here...
    img = prepare_for_display(img)
    qimg = QImage(img.data, img.shape[1], img.shape[0],
                          img.strides[0], QImage.Format_RGB888)
    if _is_filelike(filename):
        byte_array = QtCore.QByteArray()
        qbuffer = QtCore.QBuffer(byte_array)
        qbuffer.open(QtCore.QIODevice.ReadWrite)
        saved = qimg.save(qbuffer, format_str.upper())
        qbuffer.seek(0)
        filename.write(qbuffer.readAll())
        qbuffer.close()
    else:
        saved = qimg.save(filename)
    if not saved:
        from textwrap import dedent
        msg = dedent(
            '''The image was not saved. Allowable file formats
            for the QT imsave plugin are:
            BMP, JPG, JPEG, PNG, PPM, TIFF, XBM, XPM''')
        raise RuntimeError(msg)

def _is_filelike(possible_filelike):
    return callable(getattr(possible_filelike, 'write', None))
<|endoftext|>


<commit_before>import itertools
import numpy
import unittest

import cupy
from cupy import testing

float_types = [numpy.float16, numpy.float32, numpy.float64]
complex_types = [numpy.complex, numpy.complex64, numpy.complex128]
signed_int_types = [numpy.int8, numpy.int16, numpy.int32, numpy.int64]
unsigned_int_types = [numpy.uint8, numpy.uint16, numpy.uint32, numpy.uint64]
int_types = signed_int_types + unsigned_int_types
all_types = [numpy.bool] + float_types + int_types + complex_types
negative_types = ([numpy.bool] + float_types + signed_int_types
                  + complex_types)
negative_no_complex_types = [numpy.bool] + float_types + signed_int_types
no_complex_types = [numpy.bool] + float_types + int_types

@testing.gpu
@testing.parameterize(*(
    testing.product({
        'nargs': [1],
        'name': ['reciprocal', 'conj', 'angle', 'real', 'imag'],
    }) + testing.product({
        'nargs': [2],
        'name': [
            'add', 'multiply', 'divide', 'power', 'subtract', 'true_divide',
            'floor_divide', 'fmod', 'remainder'],
    })
))
class TestArithmeticRaisesWithNumpyInput(unittest.TestCase):

    def test_raises_with_numpy_input(self):
        nargs = self.nargs
        name = self.name

        # Check TypeError is raised if numpy.ndarray is given as input
        func = getattr(cupy, name)
        for input_xp_list in itertools.product(*[[numpy, cupy]] * nargs):
            if all(xp is cupy for xp in input_xp_list):
                # We don't test all-cupy-array inputs here
                continue
            arys = [xp.array([2, -3]) for xp in input_xp_list]
            with self.assertRaises(TypeError):
                func(*arys)

@testing.gpu
@testing.parameterize(*(
    testing.product({
        'arg1': ([testing.shaped_arange((2, 3), numpy, dtype=d)
                  for d in all_types]
                 + [0, 0.0j, 0j, 2, 2.0, 2j, True, False]),
        'name': ['conj', 'angle', 'real', 'imag'],
    }) + testing.product({
        'arg1': ([numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
                  for d in negative_types]
                 + [0, 0.0j, 0j, 2, 2.0, 2j, -2, -2.0, -2j, True, False]),
        'name': ['angle'],
    }) + testing.product({
        'arg1': ([testing.shaped_arange((2, 3), numpy, dtype=d) + 1
                  for d in all_types]
                 + [2, 2.0, 2j, True]),
        'name': ['reciprocal'],
    })
))
class TestArithmeticUnary(unittest.TestCase):

    @testing.numpy_cupy_allclose(atol=1e-5)
    def test_unary(self, xp):
        arg1 = self.arg1
        if isinstance(arg1, numpy.ndarray):
            arg1 = xp.asarray(arg1)
        y = getattr(xp, self.name)(arg1)

        # From NumPy>=1.11, some functions return Python scalars for Python
        # scalar inputs.
        # We need to convert them to arrays to compare with CuPy outputs.
        if xp is numpy:

            # Note that Python `bool` is a subclass of `int`
            if (isinstance(arg1, int)
                    and self.name in ('real', 'imag')):
                y = xp.asarray(y)

            elif (isinstance(arg1, (float, complex))
                    and self.name in ('real', 'imag')):
                y = xp.asarray(y)

        # TODO(niboshi): Fix this
        # numpy.real and numpy.imag return Python int if the input is Python
        # bool. CuPy should return an array of dtype.int64 in such cases,
        # instead of an array of dtype.bool.
        if xp is cupy:
            if (self.name in ('real', 'imag') and isinstance(arg1, bool)):
                y = y.astype(numpy.int64)

        return y

@testing.gpu
@testing.parameterize(*(
    testing.product({
        # TODO(unno): boolean subtract causes DeprecationWarning in numpy>=1.13
        'arg1': ([testing.shaped_arange((2, 3), numpy, dtype=d)
                  for d in all_types]
                 + [0, 0.0, 0j, 2, 2.0, 2j, True, False]),
        'arg2': ([testing.shaped_reverse_arange((2, 3), numpy, dtype=d)
                  for d in all_types]
                 + [0, 0.0, 0j, 2, 2.0, 2j, True, False]),
        'name': ['add', 'multiply', 'power', 'subtract'],
    }) + testing.product({
        'arg1': ([numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
                  for d in negative_types]
                 + [0, 0.0, 0j, 2, 2.0, 2j, -2, -2.0, -2j, True, False]),
        'arg2': ([numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
                  for d in negative_types]
                 + [0, 0.0, 0j, 2, 2.0, 2j, -2, -2.0, -2j, True, False]),
        'name': ['divide', 'true_divide', 'subtract'],
    }) + testing.product({
        'arg1': [numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
                 for d in float_types] + [0.0, 2.0, -2.0],
        'arg2': [numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
                 for d in float_types] + [0.0, 2.0, -2.0],
        'name': ['power', 'true_divide', 'subtract'],
    }) + testing.product({
        'arg1': ([testing.shaped_arange((2, 3), numpy, dtype=d)
                  for d in no_complex_types]
                 + [0, 0.0, 2, 2.0, -2, -2.0, True, False]),
        'arg2': ([testing.shaped_reverse_arange((2, 3), numpy, dtype=d)
                  for d in no_complex_types]
                 + [0, 0.0, 2, 2.0, -2, -2.0, True, False]),
        'name': ['floor_divide', 'fmod', 'remainder'],
    }) + testing.product({
        'arg1': ([numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
                  for d in negative_no_complex_types]
                 + [0, 0.0, 2, 2.0, -2, -2.0, True, False]),
        'arg2': ([numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
                  for d in negative_no_complex_types]
                 + [0, 0.0, 2, 2.0, -2, -2.0, True, False]),
        'name': ['floor_divide', 'fmod', 'remainder'],
    })
))
class TestArithmeticBinary(unittest.TestCase):

    @testing.numpy_cupy_allclose(atol=1e-4)
    def test_binary(self, xp):
        arg1 = self.arg1
        arg2 = self.arg2

        # TODO(niboshi): Fix this: power(0, 1j)
        #     numpy => 1+0j
        #     cupy => 0j
        if (self.name == 'power'
                and numpy.asarray(arg1 == 0).any()
                and numpy.asarray(arg2).dtype in complex_types):
            return xp.array(True)

        # TODO(niboshi): Fix this: xp.power(0j, 0)
        #     numpy => 1+0j
        #     cupy => 0j
        if (self.name == 'power'
                and numpy.asarray(arg1).dtype in complex_types
                and numpy.asarray(arg1 == 0j).any()
                and numpy.asarray(arg2 == 0).any()):
            return xp.array(True)

        # TODO(niboshi): Fix this: xp.add(0j, xp.array([2.], 'f')).dtype
        #     numpy => complex64
        #     cupy => complex128
        if (isinstance(arg1, complex)
                and (numpy.asarray(arg2).dtype
                     in (numpy.float16, numpy.float32))):
            return xp.array(True)

        if isinstance(arg1, numpy.ndarray):
            arg1 = xp.asarray(arg1)
        if isinstance(arg2, numpy.ndarray):
            arg2 = xp.asarray(arg2)

        func = getattr(xp, self.name)
        with testing.NumpyError(divide='ignore'):
            with numpy.warnings.catch_warnings():
                numpy.warnings.filterwarnings('ignore')
                y = func(arg1, arg2)

        # TODO(niboshi): Fix this. If rhs is a Python complex,
        #    numpy returns complex64
        #    cupy returns complex128
        if xp is cupy:
            if (xp.asarray(arg1).dtype in (numpy.float16, numpy.float32)
                    and isinstance(arg2, complex)):
                y = y.astype(numpy.complex64)

        # NumPy returns different values (nan/inf) on division by zero
        # depending on the architecture.
        # As it is not possible for CuPy to replicate this behavior, we ignore
        # the difference here.
        if (self.name in ('floor_divide', 'remainder')
                and y.dtype in (float_types + complex_types)
                and (xp.asarray(self.arg2) == 0).any()):
            y = xp.asarray(y)
            y[y == numpy.inf] = numpy.nan
            y[y == -numpy.inf] = numpy.nan

        return y

class TestArithmeticModf(unittest.TestCase):

    @testing.for_float_dtypes()
    @testing.numpy_cupy_allclose()
    def test_modf(self, xp, dtype):
        a = xp.array([-2.5, -1.5, -0.5, 0, 0.5, 1.5, 2.5], dtype=dtype)
        b, c = xp.modf(a)
        d = xp.empty((2, 7), dtype=dtype)
        d[0] = b
        d[1] = c
        return d
<commit_msg>Arithmetic test: Avoid subtraction between booleans<commit_after>import itertools
import numpy
import pkg_resources
import unittest

import cupy
from cupy import testing

def numpy_satisfies(version_range):
    """Returns True if numpy version satisfies the specified criteria."""
    spec = 'numpy{}'.format(version_range)
    try:
        pkg_resources.require(spec)
    except pkg_resources.VersionConflict:
        return False
    return True

float_types = [numpy.float16, numpy.float32, numpy.float64]
complex_types = [numpy.complex, numpy.complex64, numpy.complex128]
signed_int_types = [numpy.int8, numpy.int16, numpy.int32, numpy.int64]
unsigned_int_types = [numpy.uint8, numpy.uint16, numpy.uint32, numpy.uint64]
int_types = signed_int_types + unsigned_int_types
all_types = [numpy.bool] + float_types + int_types + complex_types
negative_types = ([numpy.bool] + float_types + signed_int_types
                  + complex_types)
negative_no_complex_types = [numpy.bool] + float_types + signed_int_types
no_complex_types = [numpy.bool] + float_types + int_types

@testing.gpu
@testing.parameterize(*(
    testing.product({
        'nargs': [1],
        'name': ['reciprocal', 'conj', 'angle', 'real', 'imag'],
    }) + testing.product({
        'nargs': [2],
        'name': [
            'add', 'multiply', 'divide', 'power', 'subtract', 'true_divide',
            'floor_divide', 'fmod', 'remainder'],
    })
))
class TestArithmeticRaisesWithNumpyInput(unittest.TestCase):

    def test_raises_with_numpy_input(self):
        nargs = self.nargs
        name = self.name

        # Check TypeError is raised if numpy.ndarray is given as input
        func = getattr(cupy, name)
        for input_xp_list in itertools.product(*[[numpy, cupy]] * nargs):
            if all(xp is cupy for xp in input_xp_list):
                # We don't test all-cupy-array inputs here
                continue
            arys = [xp.array([2, -3]) for xp in input_xp_list]
            with self.assertRaises(TypeError):
                func(*arys)

@testing.gpu
@testing.parameterize(*(
    testing.product({
        'arg1': ([testing.shaped_arange((2, 3), numpy, dtype=d)
                  for d in all_types]
                 + [0, 0.0j, 0j, 2, 2.0, 2j, True, False]),
        'name': ['conj', 'angle', 'real', 'imag'],
    }) + testing.product({
        'arg1': ([numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
                  for d in negative_types]
                 + [0, 0.0j, 0j, 2, 2.0, 2j, -2, -2.0, -2j, True, False]),
        'name': ['angle'],
    }) + testing.product({
        'arg1': ([testing.shaped_arange((2, 3), numpy, dtype=d) + 1
                  for d in all_types]
                 + [2, 2.0, 2j, True]),
        'name': ['reciprocal'],
    })
))
class TestArithmeticUnary(unittest.TestCase):

    @testing.numpy_cupy_allclose(atol=1e-5)
    def test_unary(self, xp):
        arg1 = self.arg1
        if isinstance(arg1, numpy.ndarray):
            arg1 = xp.asarray(arg1)
        y = getattr(xp, self.name)(arg1)

        # From NumPy>=1.11, some functions return Python scalars for Python
        # scalar inputs.
        # We need to convert them to arrays to compare with CuPy outputs.
        if xp is numpy:

            # Note that Python `bool` is a subclass of `int`
            if (isinstance(arg1, int)
                    and self.name in ('real', 'imag')):
                y = xp.asarray(y)

            elif (isinstance(arg1, (float, complex))
                    and self.name in ('real', 'imag')):
                y = xp.asarray(y)

        # TODO(niboshi): Fix this
        # numpy.real and numpy.imag return Python int if the input is Python
        # bool. CuPy should return an array of dtype.int64 in such cases,
        # instead of an array of dtype.bool.
        if xp is cupy:
            if (self.name in ('real', 'imag') and isinstance(arg1, bool)):
                y = y.astype(numpy.int64)

        return y

@testing.gpu
@testing.parameterize(*(
    testing.product({
        # TODO(unno): boolean subtract causes DeprecationWarning in numpy>=1.13
        'arg1': ([testing.shaped_arange((2, 3), numpy, dtype=d)
                  for d in all_types]
                 + [0, 0.0, 0j, 2, 2.0, 2j, True, False]),
        'arg2': ([testing.shaped_reverse_arange((2, 3), numpy, dtype=d)
                  for d in all_types]
                 + [0, 0.0, 0j, 2, 2.0, 2j, True, False]),
        'name': ['add', 'multiply', 'power', 'subtract'],
    }) + testing.product({
        'arg1': ([numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
                  for d in negative_types]
                 + [0, 0.0, 0j, 2, 2.0, 2j, -2, -2.0, -2j, True, False]),
        'arg2': ([numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
                  for d in negative_types]
                 + [0, 0.0, 0j, 2, 2.0, 2j, -2, -2.0, -2j, True, False]),
        'name': ['divide', 'true_divide', 'subtract'],
    }) + testing.product({
        'arg1': [numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
                 for d in float_types] + [0.0, 2.0, -2.0],
        'arg2': [numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
                 for d in float_types] + [0.0, 2.0, -2.0],
        'name': ['power', 'true_divide', 'subtract'],
    }) + testing.product({
        'arg1': ([testing.shaped_arange((2, 3), numpy, dtype=d)
                  for d in no_complex_types]
                 + [0, 0.0, 2, 2.0, -2, -2.0, True, False]),
        'arg2': ([testing.shaped_reverse_arange((2, 3), numpy, dtype=d)
                  for d in no_complex_types]
                 + [0, 0.0, 2, 2.0, -2, -2.0, True, False]),
        'name': ['floor_divide', 'fmod', 'remainder'],
    }) + testing.product({
        'arg1': ([numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
                  for d in negative_no_complex_types]
                 + [0, 0.0, 2, 2.0, -2, -2.0, True, False]),
        'arg2': ([numpy.array([-3, -2, -1, 1, 2, 3], dtype=d)
                  for d in negative_no_complex_types]
                 + [0, 0.0, 2, 2.0, -2, -2.0, True, False]),
        'name': ['floor_divide', 'fmod', 'remainder'],
    })
))
class TestArithmeticBinary(unittest.TestCase):

    @testing.numpy_cupy_allclose(atol=1e-4)
    def test_binary(self, xp):
        arg1 = self.arg1
        arg2 = self.arg2

        # TODO(niboshi): Fix this: power(0, 1j)
        #     numpy => 1+0j
        #     cupy => 0j
        if (self.name == 'power'
                and numpy.asarray(arg1 == 0).any()
                and numpy.asarray(arg2).dtype in complex_types):
            return xp.array(True)

        # TODO(niboshi): Fix this: xp.power(0j, 0)
        #     numpy => 1+0j
        #     cupy => 0j
        if (self.name == 'power'
                and numpy.asarray(arg1).dtype in complex_types
                and numpy.asarray(arg1 == 0j).any()
                and numpy.asarray(arg2 == 0).any()):
            return xp.array(True)

        # TODO(niboshi): Fix this: xp.add(0j, xp.array([2.], 'f')).dtype
        #     numpy => complex64
        #     cupy => complex128
        if (isinstance(arg1, complex)
                and (numpy.asarray(arg2).dtype
                     in (numpy.float16, numpy.float32))):
            return xp.array(True)

        if isinstance(arg1, numpy.ndarray):
            arg1 = xp.asarray(arg1)
        if isinstance(arg2, numpy.ndarray):
            arg2 = xp.asarray(arg2)

        # NumPy>=1.13.0 does not support subtraction between booleans
        # TODO(niboshi): Write a separate test to check both NumPy and CuPy
        # raise TypeError.
        if (numpy_satisfies('>=1.13.0')
                and self.name == 'subtract'
                and xp.asarray(arg1).dtype == numpy.bool_
                and xp.asarray(arg2).dtype == numpy.bool_):
            return xp.array(True)

        func = getattr(xp, self.name)
        with testing.NumpyError(divide='ignore'):
            with numpy.warnings.catch_warnings():
                numpy.warnings.filterwarnings('ignore')
                y = func(arg1, arg2)

        # TODO(niboshi): Fix this. If rhs is a Python complex,
        #    numpy returns complex64
        #    cupy returns complex128
        if xp is cupy:
            if (xp.asarray(arg1).dtype in (numpy.float16, numpy.float32)
                    and isinstance(arg2, complex)):
                y = y.astype(numpy.complex64)

        # NumPy returns different values (nan/inf) on division by zero
        # depending on the architecture.
        # As it is not possible for CuPy to replicate this behavior, we ignore
        # the difference here.
        if (self.name in ('floor_divide', 'remainder')
                and y.dtype in (float_types + complex_types)
                and (xp.asarray(self.arg2) == 0).any()):
            y = xp.asarray(y)
            y[y == numpy.inf] = numpy.nan
            y[y == -numpy.inf] = numpy.nan

        return y

class TestArithmeticModf(unittest.TestCase):

    @testing.for_float_dtypes()
    @testing.numpy_cupy_allclose()
    def test_modf(self, xp, dtype):
        a = xp.array([-2.5, -1.5, -0.5, 0, 0.5, 1.5, 2.5], dtype=dtype)
        b, c = xp.modf(a)
        d = xp.empty((2, 7), dtype=dtype)
        d[0] = b
        d[1] = c
        return d
<|endoftext|>