<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2014, Google Inc. All rights reserved.
# Use of this source code is governed by a BSD-style license that can
# be found in the LICENSE file.

# This file contains integration tests for the go/vt/binlog package.
# It sets up filtered replication between two shards and checks how data flows
# through binlog streamer.

import logging
import unittest

import environment
import tablet
import utils
from mysql_flavor import mysql_flavor

from vtdb import keyrange_constants
from vtdb import update_stream_service

src_master = tablet.Tablet()
src_replica = tablet.Tablet()
src_rdonly1 = tablet.Tablet()
src_rdonly2 = tablet.Tablet()
dst_master = tablet.Tablet()
dst_replica = tablet.Tablet()

def setUpModule():
  try:
    environment.topo_server().setup()

    setup_procs = [
        src_master.init_mysql(),
        src_replica.init_mysql(),
        src_rdonly1.init_mysql(),
        src_rdonly2.init_mysql(),
        dst_master.init_mysql(),
        dst_replica.init_mysql(),
        ]
    utils.Vtctld().start()
    utils.wait_procs(setup_procs)

    # Set up binlog stream from shard 0 to shard 1.
    # Modeled after initial_sharding.py.
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
    utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', 'test_keyspace',
                     'keyspace_id', keyrange_constants.KIT_UINT64])

    src_master.init_tablet('master', 'test_keyspace', '0')
    src_replica.init_tablet('replica', 'test_keyspace', '0')
    src_rdonly1.init_tablet('rdonly', 'test_keyspace', '0')
    src_rdonly2.init_tablet('rdonly', 'test_keyspace', '0')

    utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
    utils.validate_topology()

    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)

    for t in [src_master, src_replica, src_rdonly1, src_rdonly2]:
      t.create_db('vt_test_keyspace')
      t.start_vttablet(wait_for_state=None)

    for t in [src_master, src_replica, src_rdonly1, src_rdonly2]:
      t.wait_for_vttablet_state('SERVING')

    utils.run_vtctl(['InitShardMaster', 'test_keyspace/0',
                     src_master.tablet_alias], auto_log=True)

    # Create schema
    logging.debug("Creating schema...")
    create_table = '''create table test_table(
        id bigint auto_increment,
        keyspace_id bigint(20) unsigned,
        msg varchar(64),
        primary key (id),
        index by_msg (msg)
        ) Engine=InnoDB'''

    utils.run_vtctl(['ApplySchema',
                     '-sql=' + create_table,
                     'test_keyspace'], auto_log=True)

    # Create destination shard.
    dst_master.init_tablet('master', 'test_keyspace', '1')
    dst_replica.init_tablet('replica', 'test_keyspace', '1')
    dst_master.start_vttablet(wait_for_state='NOT_SERVING')
    dst_replica.start_vttablet(wait_for_state='NOT_SERVING')

    utils.run_vtctl(['InitShardMaster', 'test_keyspace/1',
                     dst_master.tablet_alias], auto_log=True)
    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)

    # copy the schema
    utils.run_vtctl(['CopySchemaShard', src_replica.tablet_alias,
                     'test_keyspace/1'], auto_log=True)

    # run the clone worked (this is a degenerate case, source and destination
    # both have the full keyrange. Happens to work correctly).
    logging.debug("Running the clone worker to start binlog stream...")
    utils.run_vtworker(['--cell', 'test_nj',
                        'SplitClone',
                        '--strategy=-populate_blp_checkpoint',
                        '--source_reader_count', '10',
                        '--min_table_size_for_split', '1',
                        'test_keyspace/0'],
                        auto_log=True)
    dst_master.wait_for_binlog_player_count(1)

    # Wait for dst_replica to be ready.
    dst_replica.wait_for_binlog_server_state("Enabled")
  except:
    tearDownModule()
    raise

def tearDownModule():
  if utils.options.skip_teardown:
    return

  tablet.kill_tablets([src_master, src_replica, src_rdonly1, src_rdonly2,
                       dst_master, dst_replica])

  teardown_procs = [
      src_master.teardown_mysql(),
      src_replica.teardown_mysql(),
      src_rdonly1.teardown_mysql(),
      src_rdonly2.teardown_mysql(),
      dst_master.teardown_mysql(),
      dst_replica.teardown_mysql(),
      ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  src_master.remove_tree()
  src_replica.remove_tree()
  src_rdonly1.remove_tree()
  src_rdonly2.remove_tree()
  dst_master.remove_tree()
  dst_replica.remove_tree()

def _get_update_stream(tblt):
  return update_stream_service.UpdateStreamConnection('localhost:%u' %
                                                      tblt.port, 30)

class TestBinlog(unittest.TestCase):

  def test_charset(self):
    start_position = mysql_flavor().master_position(dst_replica)
    logging.debug('test_charset: starting @ %s', start_position)

    # Insert something that will replicate incorrectly if the charset is not
    # propagated through binlog streamer to the destination.
    #
    # Vitess tablets default to using utf8, so we insert something crazy and
    # pretend it's latin1. If the binlog player doesn't also pretend it's
    # latin1, it will be inserted as utf8, which will change its value.
    src_master.mquery("vt_test_keyspace",
        "INSERT INTO test_table (id, keyspace_id, msg) VALUES (41523, 1, 'Šṛ́rỏé') /* EMD keyspace_id:1 */",
        conn_params={'charset': 'latin1'}, write=True)

    # Wait for it to replicate.
    stream = _get_update_stream(dst_replica)
    stream.dial()
    data = stream.stream_start(start_position)
    while data:
      if data['Category'] == 'POS':
        break
      data = stream.stream_next()
    stream.close()

    # Check the value.
    data = dst_master.mquery("vt_test_keyspace",
        "SELECT id, keyspace_id, msg FROM test_table WHERE id=41523 LIMIT 1")
    self.assertEqual(len(data), 1, 'No data replicated.')
    self.assertEqual(len(data[0]), 3, 'Wrong number of columns.')
    self.assertEqual(data[0][2], 'Šṛ́rỏé', 'Data corrupted due to wrong charset.')

  def test_checksum_enabled(self):
    start_position = mysql_flavor().master_position(dst_replica)
    logging.debug('test_checksum_enabled: starting @ %s', start_position)

    # Enable binlog_checksum, which will also force a log rotation that should
    # cause binlog streamer to notice the new checksum setting.
    if not mysql_flavor().enable_binlog_checksum(dst_replica):
      logging.debug('skipping checksum test on flavor without binlog_checksum setting')
      return

    # Insert something and make sure it comes through intact.
    sql = "INSERT INTO test_table (id, keyspace_id, msg) VALUES (19283, 1, 'testing checksum enabled') /* EMD keyspace_id:1 */"
    src_master.mquery("vt_test_keyspace",
        sql,
        write=True)

    # Look for it using update stream to see if binlog streamer can talk to
    # dst_replica, which now has binlog_checksum enabled.
    stream = _get_update_stream(dst_replica)
    stream.dial()
    data = stream.stream_start(start_position)
    found = False
    while data:
      if data['Category'] == 'POS':
        break
      if data['Sql'] == sql:
        found = True
        break
      data = stream.stream_next()
    stream.close()
    self.assertEqual(found, True, 'expected query not found in update stream')

  def test_checksum_disabled(self):
    # Disable binlog_checksum to make sure we can also talk to a server without
    # checksums enabled, in case they are enabled by default.
    start_position = mysql_flavor().master_position(dst_replica)
    logging.debug('test_checksum_disabled: starting @ %s', start_position)

    # For flavors that don't support checksums, this is a no-op.
    mysql_flavor().disable_binlog_checksum(dst_replica)

    # Insert something and make sure it comes through intact.
    sql = "INSERT INTO test_table (id, keyspace_id, msg) VALUES (58812, 1, 'testing checksum disabled') /* EMD keyspace_id:1 */"
    src_master.mquery("vt_test_keyspace",
        sql,
        write=True)

    # Look for it using update stream to see if binlog streamer can talk to
    # dst_replica, which now has binlog_checksum disabled.
    stream = _get_update_stream(dst_replica)
    stream.dial()
    data = stream.stream_start(start_position)
    found = False
    while data:
      if data['Category'] == 'POS':
        break
      if data['Sql'] == sql:
        found = True
        break
      data = stream.stream_next()
    stream.close()
    self.assertEqual(found, True, 'expected query not found in update stream')

if __name__ == '__main__':
  utils.main()
<commit_msg>Fixing this test.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2014, Google Inc. All rights reserved.
# Use of this source code is governed by a BSD-style license that can
# be found in the LICENSE file.

# This file contains integration tests for the go/vt/binlog package.
# It sets up filtered replication between two shards and checks how data flows
# through binlog streamer.

import logging
import unittest

import environment
import tablet
import utils
from mysql_flavor import mysql_flavor

from vtdb import keyrange_constants
from vtdb import update_stream_service

src_master = tablet.Tablet()
src_replica = tablet.Tablet()
src_rdonly1 = tablet.Tablet()
src_rdonly2 = tablet.Tablet()
dst_master = tablet.Tablet()
dst_replica = tablet.Tablet()

def setUpModule():
  try:
    environment.topo_server().setup()

    setup_procs = [
        src_master.init_mysql(),
        src_replica.init_mysql(),
        src_rdonly1.init_mysql(),
        src_rdonly2.init_mysql(),
        dst_master.init_mysql(),
        dst_replica.init_mysql(),
        ]
    utils.Vtctld().start()
    utils.wait_procs(setup_procs)

    # Set up binlog stream from shard 0 to shard 1.
    # Modeled after initial_sharding.py.
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
    utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', 'test_keyspace',
                     'keyspace_id', keyrange_constants.KIT_UINT64])

    src_master.init_tablet('master', 'test_keyspace', '0')
    src_replica.init_tablet('replica', 'test_keyspace', '0')
    src_rdonly1.init_tablet('rdonly', 'test_keyspace', '0')
    src_rdonly2.init_tablet('rdonly', 'test_keyspace', '0')

    utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
    utils.validate_topology()

    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)

    for t in [src_master, src_replica, src_rdonly1, src_rdonly2]:
      t.create_db('vt_test_keyspace')
      t.start_vttablet(wait_for_state=None)

    for t in [src_master, src_replica, src_rdonly1, src_rdonly2]:
      t.wait_for_vttablet_state('SERVING')

    utils.run_vtctl(['InitShardMaster', 'test_keyspace/0',
                     src_master.tablet_alias], auto_log=True)

    # Create schema
    logging.debug("Creating schema...")
    create_table = '''create table test_table(
        id bigint auto_increment,
        keyspace_id bigint(20) unsigned,
        msg varchar(64),
        primary key (id),
        index by_msg (msg)
        ) Engine=InnoDB'''

    utils.run_vtctl(['ApplySchema',
                     '-sql=' + create_table,
                     'test_keyspace'], auto_log=True)

    # Create destination shard.
    dst_master.init_tablet('master', 'test_keyspace', '-')
    dst_replica.init_tablet('replica', 'test_keyspace', '-')
    dst_master.start_vttablet(wait_for_state='NOT_SERVING')
    dst_replica.start_vttablet(wait_for_state='NOT_SERVING')

    utils.run_vtctl(['InitShardMaster', 'test_keyspace/-',
                     dst_master.tablet_alias], auto_log=True)
    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)

    # copy the schema
    utils.run_vtctl(['CopySchemaShard', src_replica.tablet_alias,
                     'test_keyspace/-'], auto_log=True)

    # run the clone worked (this is a degenerate case, source and destination
    # both have the full keyrange. Happens to work correctly).
    logging.debug("Running the clone worker to start binlog stream...")
    utils.run_vtworker(['--cell', 'test_nj',
                        'SplitClone',
                        '--strategy=-populate_blp_checkpoint',
                        '--source_reader_count', '10',
                        '--min_table_size_for_split', '1',
                        'test_keyspace/0'],
                        auto_log=True)
    dst_master.wait_for_binlog_player_count(1)

    # Wait for dst_replica to be ready.
    dst_replica.wait_for_binlog_server_state("Enabled")
  except:
    tearDownModule()
    raise

def tearDownModule():
  if utils.options.skip_teardown:
    return

  tablet.kill_tablets([src_master, src_replica, src_rdonly1, src_rdonly2,
                       dst_master, dst_replica])

  teardown_procs = [
      src_master.teardown_mysql(),
      src_replica.teardown_mysql(),
      src_rdonly1.teardown_mysql(),
      src_rdonly2.teardown_mysql(),
      dst_master.teardown_mysql(),
      dst_replica.teardown_mysql(),
      ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  src_master.remove_tree()
  src_replica.remove_tree()
  src_rdonly1.remove_tree()
  src_rdonly2.remove_tree()
  dst_master.remove_tree()
  dst_replica.remove_tree()

def _get_update_stream(tblt):
  return update_stream_service.UpdateStreamConnection('localhost:%u' %
                                                      tblt.port, 30)

class TestBinlog(unittest.TestCase):

  def test_charset(self):
    start_position = mysql_flavor().master_position(dst_replica)
    logging.debug('test_charset: starting @ %s', start_position)

    # Insert something that will replicate incorrectly if the charset is not
    # propagated through binlog streamer to the destination.
    #
    # Vitess tablets default to using utf8, so we insert something crazy and
    # pretend it's latin1. If the binlog player doesn't also pretend it's
    # latin1, it will be inserted as utf8, which will change its value.
    src_master.mquery("vt_test_keyspace",
        "INSERT INTO test_table (id, keyspace_id, msg) VALUES (41523, 1, 'Šṛ́rỏé') /* EMD keyspace_id:1 */",
        conn_params={'charset': 'latin1'}, write=True)

    # Wait for it to replicate.
    stream = _get_update_stream(dst_replica)
    stream.dial()
    data = stream.stream_start(start_position)
    while data:
      if data['Category'] == 'POS':
        break
      data = stream.stream_next()
    stream.close()

    # Check the value.
    data = dst_master.mquery("vt_test_keyspace",
        "SELECT id, keyspace_id, msg FROM test_table WHERE id=41523 LIMIT 1")
    self.assertEqual(len(data), 1, 'No data replicated.')
    self.assertEqual(len(data[0]), 3, 'Wrong number of columns.')
    self.assertEqual(data[0][2], 'Šṛ́rỏé', 'Data corrupted due to wrong charset.')

  def test_checksum_enabled(self):
    start_position = mysql_flavor().master_position(dst_replica)
    logging.debug('test_checksum_enabled: starting @ %s', start_position)

    # Enable binlog_checksum, which will also force a log rotation that should
    # cause binlog streamer to notice the new checksum setting.
    if not mysql_flavor().enable_binlog_checksum(dst_replica):
      logging.debug('skipping checksum test on flavor without binlog_checksum setting')
      return

    # Insert something and make sure it comes through intact.
    sql = "INSERT INTO test_table (id, keyspace_id, msg) VALUES (19283, 1, 'testing checksum enabled') /* EMD keyspace_id:1 */"
    src_master.mquery("vt_test_keyspace",
        sql,
        write=True)

    # Look for it using update stream to see if binlog streamer can talk to
    # dst_replica, which now has binlog_checksum enabled.
    stream = _get_update_stream(dst_replica)
    stream.dial()
    data = stream.stream_start(start_position)
    found = False
    while data:
      if data['Category'] == 'POS':
        break
      if data['Sql'] == sql:
        found = True
        break
      data = stream.stream_next()
    stream.close()
    self.assertEqual(found, True, 'expected query not found in update stream')

  def test_checksum_disabled(self):
    # Disable binlog_checksum to make sure we can also talk to a server without
    # checksums enabled, in case they are enabled by default.
    start_position = mysql_flavor().master_position(dst_replica)
    logging.debug('test_checksum_disabled: starting @ %s', start_position)

    # For flavors that don't support checksums, this is a no-op.
    mysql_flavor().disable_binlog_checksum(dst_replica)

    # Insert something and make sure it comes through intact.
    sql = "INSERT INTO test_table (id, keyspace_id, msg) VALUES (58812, 1, 'testing checksum disabled') /* EMD keyspace_id:1 */"
    src_master.mquery("vt_test_keyspace",
        sql,
        write=True)

    # Look for it using update stream to see if binlog streamer can talk to
    # dst_replica, which now has binlog_checksum disabled.
    stream = _get_update_stream(dst_replica)
    stream.dial()
    data = stream.stream_start(start_position)
    found = False
    while data:
      if data['Category'] == 'POS':
        break
      if data['Sql'] == sql:
        found = True
        break
      data = stream.stream_next()
    stream.close()
    self.assertEqual(found, True, 'expected query not found in update stream')

if __name__ == '__main__':
  utils.main()
<|endoftext|>


<commit_before># -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4

import os
import commands

# use twisted if we can so we get nice trial goodness
try:
    from twisted.trial import unittest
except ImportError:
    import unittest

def canImport(module):
    '''
    Check if we can import the given module name.

    @type module: str

    @rtype: bool
    '''
    try:
        __import__(module)
        return True
    except ImportError:
        return False

# lifted from flumotion with permission
def _diff(old, new, desc):
    import difflib
    lines = difflib.unified_diff(old, new)
    lines = list(lines)
    if not lines:
        return
    output = ''
    for line in lines:
        output += '%s: %s\n' % (desc, line[:-1])

    raise AssertionError(
        ("\nError while comparing strings:\n"
         "%s") % (output,))

def diffStrings(orig, new, desc='input'):
    def tolines(s):
        return [line + '\n' for line in s.split('\n')]

    return _diff(tolines(orig), tolines(new), desc=desc)

# our base class for all tests
class TestCase(unittest.TestCase):
    def check(self, testname, args=''):
        """
        Run pychecker on the given test, located in input/
        Will compare to output of the same name in expected/
        """
        return self.checkMultiple(testname, [testname + '.py'], args)

    def checkMultiple(self, testname, checkables, args=''):
        """
        Run pychecker on the given test, located in input/
        Will compare to output of the same name in expected/

        @type args:       str
        @type checkables: list of str
        """
        abstestdir = os.path.dirname(__file__)
        abspycheckerdir = os.path.dirname(abstestdir)

        # make this relative to where we are, so paths shown are relative too
        #if abstestdir.startswith(os.getcwd()):
        #    abstestdir = abstestdir[len(os.getcwd()) + 1:]

        pycheckerpy = os.path.join(abspycheckerdir, 'pychecker', 'checker.py')
        testfiles = [os.path.join('input', c) for c in checkables]

        cmd = "cd %s; python -tt %s " \
            "--limit 0 --no-argsused " \
            "%s %s" % (abstestdir, pycheckerpy, args, " ".join(testfiles))
        # getoutput output never ends on a newline the way
        # pychecker ... > expected/... would
        output = commands.getoutput(cmd) + '\n'

        # here we can select a different file based on os/python version/arch
        relexpectedfile = os.path.join('expected', testname)
        if args:
            underscoredArgs = args.replace('-', '_').replace(' ', '_')
            relexpectedfile += underscoredArgs
        absexpectedfile = os.path.join(abstestdir, relexpectedfile)

        # FIXME: make generating an option
        # for now, do it every time we don't have the expected output
        # to help us
        if not os.path.exists(absexpectedfile):
            open(absexpectedfile, 'w').write(output)
            print 'Generated non-existing output file', absexpectedfile

        expected = open(absexpectedfile).read()

        diffStrings(expected, output, desc=relexpectedfile)

<commit_msg>remove trailing whitespace<commit_after># -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4

import os
import commands

# use twisted if we can so we get nice trial goodness
try:
    from twisted.trial import unittest
except ImportError:
    import unittest

def canImport(module):
    '''
    Check if we can import the given module name.

    @type module: str

    @rtype: bool
    '''
    try:
        __import__(module)
        return True
    except ImportError:
        return False

# lifted from flumotion with permission
def _diff(old, new, desc):
    import difflib
    lines = difflib.unified_diff(old, new)
    lines = list(lines)
    if not lines:
        return
    output = ''
    for line in lines:
        output += '%s: %s\n' % (desc, line[:-1])

    raise AssertionError(
        ("\nError while comparing strings:\n"
         "%s") % (output,))

def diffStrings(orig, new, desc='input'):
    def tolines(s):
        return [line + '\n' for line in s.split('\n')]

    return _diff(tolines(orig), tolines(new), desc=desc)

# our base class for all tests
class TestCase(unittest.TestCase):
    def check(self, testname, args=''):
        """
        Run pychecker on the given test, located in input/
        Will compare to output of the same name in expected/
        """
        return self.checkMultiple(testname, [testname + '.py'], args)

    def checkMultiple(self, testname, checkables, args=''):
        """
        Run pychecker on the given test, located in input/
        Will compare to output of the same name in expected/

        @type args:       str
        @type checkables: list of str
        """
        abstestdir = os.path.dirname(__file__)
        abspycheckerdir = os.path.dirname(abstestdir)

        # make this relative to where we are, so paths shown are relative too
        #if abstestdir.startswith(os.getcwd()):
        #    abstestdir = abstestdir[len(os.getcwd()) + 1:]

        pycheckerpy = os.path.join(abspycheckerdir, 'pychecker', 'checker.py')
        testfiles = [os.path.join('input', c) for c in checkables]

        cmd = "cd %s; python -tt %s " \
            "--limit 0 --no-argsused " \
            "%s %s" % (abstestdir, pycheckerpy, args, " ".join(testfiles))
        # getoutput output never ends on a newline the way
        # pychecker ... > expected/... would
        output = commands.getoutput(cmd) + '\n'

        # here we can select a different file based on os/python version/arch
        relexpectedfile = os.path.join('expected', testname)
        if args:
            underscoredArgs = args.replace('-', '_').replace(' ', '_')
            relexpectedfile += underscoredArgs
        absexpectedfile = os.path.join(abstestdir, relexpectedfile)

        # FIXME: make generating an option
        # for now, do it every time we don't have the expected output
        # to help us
        if not os.path.exists(absexpectedfile):
            open(absexpectedfile, 'w').write(output)
            print 'Generated non-existing output file', absexpectedfile

        expected = open(absexpectedfile).read()

        diffStrings(expected, output, desc=relexpectedfile)
<|endoftext|>


<commit_before>import subprocess
import tempfile
import struct
import socket
import time
import os

__all__ = ['Daemon', 'Datagram', 'DatagramIterator', 'MDConnection']

class Daemon(object):
    DAEMON_PATH = './openotpd'

    def __init__(self, config):
        self.config = config

        self.daemon = None
        self.config_file = None

    def start(self):
        cfg, self.config_file = tempfile.mkstemp()
        os.write(cfg, self.config)
        os.close(cfg)

        self.daemon = subprocess.Popen([self.DAEMON_PATH,
                                        '-config', self.config_file])

        time.sleep(1.0) # Allow some time for daemon to initialize...

    def stop(self):
        self.daemon.kill()
        os.remove(self.config_file)

DATATYPES = {
    'int8': '<b',
    'uint8': '<B',
    'int16': '<h',
    'uint16': '<H',
    'int32': '<i',
    'uint32': '<I',
    'int64': '<q',
    'uint64': '<Q',
}

CONSTANTS = {
    # Reserved Values
    'INVALID_CHANNEL': 0,
    'INVALID_DO_ID': 0,
    'INVALID_MSG_TYPE': 0,

    # Control Channels
    'CONTROL_CHANNEL': 4001,
    'CONTROL_ADD_CHANNEL': 2001,
    'CONTROL_REMOVE_CHANNEL': 2002,

    'CONTROL_ADD_RANGE': 2008,
    'CONTROL_REMOVE_RANGE': 2009,

    'CONTROL_ADD_POST_REMOVE': 2010,
    'CONTROL_CLEAR_POST_REMOVE': 2011,

    # State Server
    'STATESERVER_OBJECT_GENERATE_WITH_REQUIRED': 2001,
    'STATESERVER_OBJECT_GENERATE_WITH_REQUIRED_OTHER': 2003,
    'STATESERVER_OBJECT_UPDATE_FIELD': 2004,
    'STATESERVER_OBJECT_UPDATE_FIELD_MULTIPLE': 2005,
    'STATESERVER_OBJECT_DELETE_RAM': 2007,
    'STATESERVER_OBJECT_SET_ZONE': 2008,
    'STATESERVER_OBJECT_CHANGE_ZONE': 2009,
    'STATESERVER_OBJECT_LOCATE': 2022,
    'STATESERVER_OBJECT_LOCATE_RESP': 2023,
    'STATESERVER_OBJECT_SET_AI_CHANNEL': 2045,
    'STATESERVER_OBJECT_QUERY_ALL': 2020,
    'STATESERVER_OBJECT_QUERY_ALL_RESP': 2030,
    'STATESERVER_SHARD_RESET': 2061,
    'STATESERVER_OBJECT_ENTERZONE_WITH_REQUIRED': 2065,
    'STATESERVER_OBJECT_ENTERZONE_WITH_REQUIRED_OTHER': 2066,
    'STATESERVER_OBJECT_ENTER_AI_RECV': 2067,
    'STATESERVER_OBJECT_LEAVING_AI_INTEREST': 2033,
    'STATESERVER_OBJECT_QUERY_MANAGING_AI': 2083,
    'STATESERVER_OBJECT_NOTIFY_MANAGING_AI': 2047,

    # Database Server
    'DBSERVER_CREATE_STORED_OBJECT': 1003,
    'DBSERVER_CREATE_STORED_OBJECT_RESP': 1004,
    'DBSERVER_DELETE_STORED_OBJECT': 1008,
    'DBSERVER_DELETE_QUERY': 1010,
    'DBSERVER_SELECT_STORED_OBJECT': 1012,
    'DBSERVER_SELECT_STORED_OBJECT_RESP': 1013,
    'DBSERVER_SELECT_STORED_OBJECT_ALL': 1020,
    'DBSERVER_SELECT_STORED_OBJECT_ALL_RESP': 1021,
    'DBSERVER_SELECT_QUERY': 1016,
    'DBSERVER_SELECT_QUERY_RESP': 1017,
    'DBSERVER_UPDATE_STORED_OBJECT': 1014,
    'DBSERVER_UPDATE_STORED_OBJECT_IF_EQUALS': 1024,
    'DBSERVER_UPDATE_STORED_OBJECT_IF_EQUALS_RESP': 1025,
    'DBSERVER_UPDATE_QUERY': 1018,
}

locals().update(CONSTANTS)
__all__.extend(CONSTANTS.keys())

class Datagram(object):
    def __init__(self, data=b''):
        self._data = data

        def make_adder(v):
            def adder(data):
                self.add_raw(struct.pack(v, data))
            return adder

        for k,v in DATATYPES.items():
            adder = make_adder(v)
            setattr(self, 'add_' + k, adder)

    def add_raw(self, data):
        self._data += data

    def add_string(self, string):
        self.add_uint16(len(string))
        self.add_raw(string)

    def get_data(self):
        return self._data

    def get_payload(self):
        return self._data[8*ord(self._data[0])+1:]

    def get_channels(self):
        return set(struct.unpack('<x' + 'Q'*ord(self._data[0]),
                                 self._data[:8*ord(self._data[0])+1]))

    def is_subset_of(self, other):
        return self.get_payload() == other.get_payload() and \
               self.get_channels() <= other.get_channels()

    # Common datagram type helpers:
    @classmethod
    def create(cls, recipients, sender, msgtype):
        dg = cls()
        dg.add_uint8(len(recipients))
        for recipient in recipients: dg.add_uint64(recipient)
        dg.add_uint64(sender)
        dg.add_uint16(msgtype)
        return dg

    @classmethod
    def create_control(cls):
        dg = cls()
        dg.add_uint8(1)
        dg.add_uint64(CONTROL_CHANNEL)
        return dg

    @classmethod
    def create_add_channel(cls, channel):
        dg = cls.create_control()
        dg.add_uint16(CONTROL_ADD_CHANNEL)
        dg.add_uint64(channel)
        return dg

    @classmethod
    def create_remove_channel(cls, channel):
        dg = cls.create_control()
        dg.add_uint16(CONTROL_REMOVE_CHANNEL)
        dg.add_uint64(channel)
        return dg

    @classmethod
    def create_add_range(cls, upper, lower):
        dg = cls.create_control()
        dg.add_uint16(CONTROL_ADD_RANGE)
        dg.add_uint64(upper)
        dg.add_uint64(lower)
        return dg

    @classmethod
    def create_remove_range(cls, upper, lower):
        dg = cls.create_control()
        dg.add_uint16(CONTROL_REMOVE_RANGE)
        dg.add_uint64(upper)
        dg.add_uint64(lower)
        return dg

    @classmethod
    def create_add_post_remove(cls, datagram):
        dg = cls.create_control()
        dg.add_uint16(CONTROL_ADD_POST_REMOVE)
        dg.add_string(datagram.get_data())
        return dg

    @classmethod
    def create_clear_post_remove(cls):
        dg = cls.create_control()
        dg.add_uint16(CONTROL_CLEAR_POST_REMOVE)
        return dg

class DatagramIterator(object):
    def __init__(self, datagram, offset = 0):
        self._datagram = datagram
        self._data = datagram.get_data()
        self._offset = offset

    def matches_header(self, recipients, sender, msgtype, remaining=-1):
        self.seek(0)
        channels = [i for i, j in zip(self._datagram.get_channels(), recipients) if i == j]
        if len(channels) != len(recipients):
            print "Channels don't match"
            return False

        self.seek(8*ord(self._data[0])+1)
        if sender != self.read_uint64():
            print "Sender doesn't match"
            return False

        if msgtype != self.read_uint16():
            print "MsgType doesn't match"
            return False

        if remaining != -1 and remaining != len(self._data) - self._offset:
            print "Remaining doesn't match"
            return False

        return True

    def read_uint8(self):
        self._offset += 1
        if self._offset > len(self._data):
            raise EOFError('End of Datagram')

        return struct.unpack("<B", self._data[self._offset-1:self._offset])

    def read_uint16(self):
        self._offset += 2
        if self._offset > len(self._data):
            raise EOFError('End of Datagram')

        return struct.unpack("<H", self._data[self._offset-2:self._offset])

    def read_uint64(self):
        self._offset += 8
        if self._offset > len(self._data):
            raise EOFError('End of Datagram')

        return struct.unpack("<Q", self._data[self._offset-8:self._offset])

    def seek(self, offset):
        self._offset = offset

    def tell(self):
        return self._offset

class MDConnection(object):
    def __init__(self, sock):
        self.s = sock
        self.s.settimeout(0.1)

    def send(self, datagram):
        data = datagram.get_data()
        msg = struct.pack('<H', len(data)) + data
        self.s.send(msg)

    def recv(self):
        dg = self._read()
        if dg is None:
            raise EOFError('No message recieved')
        return Datagram(dg)

    def _read(self):
        try:
            length = 2
            result = ''
            while len(result) < length:
                data = self.s.recv(length - len(result))
                if data == '':
                    raise EOFError('Remote socket closed connection')
                result += data
            length = struct.unpack('<H', result)[0]
        except socket.error:
            return None

        result = ''
        while len(result) < length:
            data = self.s.recv(length - len(result))
            if data == '':
                    raise EOFError('Remote socket closed connection')
            result += data

        return result

    def close(self):
        self.s.close()

    def flush(self):
        while self._read(): pass

    def expect(self, datagram):
        return self.expect_multi([datagram], only=True)

    def expect_multi(self, datagrams, only=False):
        datagrams = list(datagrams) # We're going to be doing datagrams.remove()

        while datagrams:
            dg = self._read()
            if dg is None: return False # Augh, we didn't see all the dgs yet!
            dg = Datagram(dg)

            for datagram in datagrams:
                if datagram.is_subset_of(dg):
                    datagrams.remove(datagram)
                    break
            else:
                if only:
                    return False

        return True

    def expect_none(self):
        return self._read() == None
<commit_msg>DBServer: DGIterator read should return only first argument.<commit_after>import subprocess
import tempfile
import struct
import socket
import time
import os

__all__ = ['Daemon', 'Datagram', 'DatagramIterator', 'MDConnection']

class Daemon(object):
    DAEMON_PATH = './openotpd'

    def __init__(self, config):
        self.config = config

        self.daemon = None
        self.config_file = None

    def start(self):
        cfg, self.config_file = tempfile.mkstemp()
        os.write(cfg, self.config)
        os.close(cfg)

        self.daemon = subprocess.Popen([self.DAEMON_PATH,
                                        '-config', self.config_file])

        time.sleep(1.0) # Allow some time for daemon to initialize...

    def stop(self):
        self.daemon.kill()
        os.remove(self.config_file)

DATATYPES = {
    'int8': '<b',
    'uint8': '<B',
    'int16': '<h',
    'uint16': '<H',
    'int32': '<i',
    'uint32': '<I',
    'int64': '<q',
    'uint64': '<Q',
}

CONSTANTS = {
    # Reserved Values
    'INVALID_CHANNEL': 0,
    'INVALID_DO_ID': 0,
    'INVALID_MSG_TYPE': 0,

    # Control Channels
    'CONTROL_CHANNEL': 4001,
    'CONTROL_ADD_CHANNEL': 2001,
    'CONTROL_REMOVE_CHANNEL': 2002,

    'CONTROL_ADD_RANGE': 2008,
    'CONTROL_REMOVE_RANGE': 2009,

    'CONTROL_ADD_POST_REMOVE': 2010,
    'CONTROL_CLEAR_POST_REMOVE': 2011,

    # State Server
    'STATESERVER_OBJECT_GENERATE_WITH_REQUIRED': 2001,
    'STATESERVER_OBJECT_GENERATE_WITH_REQUIRED_OTHER': 2003,
    'STATESERVER_OBJECT_UPDATE_FIELD': 2004,
    'STATESERVER_OBJECT_UPDATE_FIELD_MULTIPLE': 2005,
    'STATESERVER_OBJECT_DELETE_RAM': 2007,
    'STATESERVER_OBJECT_SET_ZONE': 2008,
    'STATESERVER_OBJECT_CHANGE_ZONE': 2009,
    'STATESERVER_OBJECT_LOCATE': 2022,
    'STATESERVER_OBJECT_LOCATE_RESP': 2023,
    'STATESERVER_OBJECT_SET_AI_CHANNEL': 2045,
    'STATESERVER_OBJECT_QUERY_ALL': 2020,
    'STATESERVER_OBJECT_QUERY_ALL_RESP': 2030,
    'STATESERVER_SHARD_RESET': 2061,
    'STATESERVER_OBJECT_ENTERZONE_WITH_REQUIRED': 2065,
    'STATESERVER_OBJECT_ENTERZONE_WITH_REQUIRED_OTHER': 2066,
    'STATESERVER_OBJECT_ENTER_AI_RECV': 2067,
    'STATESERVER_OBJECT_LEAVING_AI_INTEREST': 2033,
    'STATESERVER_OBJECT_QUERY_MANAGING_AI': 2083,
    'STATESERVER_OBJECT_NOTIFY_MANAGING_AI': 2047,

    # Database Server
    'DBSERVER_CREATE_STORED_OBJECT': 1003,
    'DBSERVER_CREATE_STORED_OBJECT_RESP': 1004,
    'DBSERVER_DELETE_STORED_OBJECT': 1008,
    'DBSERVER_DELETE_QUERY': 1010,
    'DBSERVER_SELECT_STORED_OBJECT': 1012,
    'DBSERVER_SELECT_STORED_OBJECT_RESP': 1013,
    'DBSERVER_SELECT_STORED_OBJECT_ALL': 1020,
    'DBSERVER_SELECT_STORED_OBJECT_ALL_RESP': 1021,
    'DBSERVER_SELECT_QUERY': 1016,
    'DBSERVER_SELECT_QUERY_RESP': 1017,
    'DBSERVER_UPDATE_STORED_OBJECT': 1014,
    'DBSERVER_UPDATE_STORED_OBJECT_IF_EQUALS': 1024,
    'DBSERVER_UPDATE_STORED_OBJECT_IF_EQUALS_RESP': 1025,
    'DBSERVER_UPDATE_QUERY': 1018,
}

locals().update(CONSTANTS)
__all__.extend(CONSTANTS.keys())

class Datagram(object):
    def __init__(self, data=b''):
        self._data = data

        def make_adder(v):
            def adder(data):
                self.add_raw(struct.pack(v, data))
            return adder

        for k,v in DATATYPES.items():
            adder = make_adder(v)
            setattr(self, 'add_' + k, adder)

    def add_raw(self, data):
        self._data += data

    def add_string(self, string):
        self.add_uint16(len(string))
        self.add_raw(string)

    def get_data(self):
        return self._data

    def get_payload(self):
        return self._data[8*ord(self._data[0])+1:]

    def get_channels(self):
        return set(struct.unpack('<x' + 'Q'*ord(self._data[0]),
                                 self._data[:8*ord(self._data[0])+1]))

    def is_subset_of(self, other):
        return self.get_payload() == other.get_payload() and \
               self.get_channels() <= other.get_channels()

    # Common datagram type helpers:
    @classmethod
    def create(cls, recipients, sender, msgtype):
        dg = cls()
        dg.add_uint8(len(recipients))
        for recipient in recipients: dg.add_uint64(recipient)
        dg.add_uint64(sender)
        dg.add_uint16(msgtype)
        return dg

    @classmethod
    def create_control(cls):
        dg = cls()
        dg.add_uint8(1)
        dg.add_uint64(CONTROL_CHANNEL)
        return dg

    @classmethod
    def create_add_channel(cls, channel):
        dg = cls.create_control()
        dg.add_uint16(CONTROL_ADD_CHANNEL)
        dg.add_uint64(channel)
        return dg

    @classmethod
    def create_remove_channel(cls, channel):
        dg = cls.create_control()
        dg.add_uint16(CONTROL_REMOVE_CHANNEL)
        dg.add_uint64(channel)
        return dg

    @classmethod
    def create_add_range(cls, upper, lower):
        dg = cls.create_control()
        dg.add_uint16(CONTROL_ADD_RANGE)
        dg.add_uint64(upper)
        dg.add_uint64(lower)
        return dg

    @classmethod
    def create_remove_range(cls, upper, lower):
        dg = cls.create_control()
        dg.add_uint16(CONTROL_REMOVE_RANGE)
        dg.add_uint64(upper)
        dg.add_uint64(lower)
        return dg

    @classmethod
    def create_add_post_remove(cls, datagram):
        dg = cls.create_control()
        dg.add_uint16(CONTROL_ADD_POST_REMOVE)
        dg.add_string(datagram.get_data())
        return dg

    @classmethod
    def create_clear_post_remove(cls):
        dg = cls.create_control()
        dg.add_uint16(CONTROL_CLEAR_POST_REMOVE)
        return dg

class DatagramIterator(object):
    def __init__(self, datagram, offset = 0):
        self._datagram = datagram
        self._data = datagram.get_data()
        self._offset = offset

    def matches_header(self, recipients, sender, msgtype, remaining=-1):
        self.seek(0)
        channels = [i for i, j in zip(self._datagram.get_channels(), recipients) if i == j]
        if len(channels) != len(recipients):
            print "Channels don't match"
            return False

        self.seek(8*ord(self._data[0])+1)
        if sender != self.read_uint64():
            print "Sender doesn't match"
            return False

        if msgtype != self.read_uint16():
            print "MsgType doesn't match"
            return False

        if remaining != -1 and remaining != len(self._data) - self._offset:
            print "Remaining doesn't match"
            return False

        return True

    def read_uint8(self):
        self._offset += 1
        if self._offset > len(self._data):
            raise EOFError('End of Datagram')

        return struct.unpack("<B", self._data[self._offset-1:self._offset])[0]

    def read_uint16(self):
        self._offset += 2
        if self._offset > len(self._data):
            raise EOFError('End of Datagram')

        return struct.unpack("<H", self._data[self._offset-2:self._offset])[0]

    def read_uint64(self):
        self._offset += 8
        if self._offset > len(self._data):
            raise EOFError('End of Datagram')

        return struct.unpack("<Q", self._data[self._offset-8:self._offset])[0]

    def seek(self, offset):
        self._offset = offset

    def tell(self):
        return self._offset

class MDConnection(object):
    def __init__(self, sock):
        self.s = sock
        self.s.settimeout(0.1)

    def send(self, datagram):
        data = datagram.get_data()
        msg = struct.pack('<H', len(data)) + data
        self.s.send(msg)

    def recv(self):
        dg = self._read()
        if dg is None:
            raise EOFError('No message recieved')
        return Datagram(dg)

    def _read(self):
        try:
            length = 2
            result = ''
            while len(result) < length:
                data = self.s.recv(length - len(result))
                if data == '':
                    raise EOFError('Remote socket closed connection')
                result += data
            length = struct.unpack('<H', result)[0]
        except socket.error:
            return None

        result = ''
        while len(result) < length:
            data = self.s.recv(length - len(result))
            if data == '':
                    raise EOFError('Remote socket closed connection')
            result += data

        return result

    def close(self):
        self.s.close()

    def flush(self):
        while self._read(): pass

    def expect(self, datagram):
        return self.expect_multi([datagram], only=True)

    def expect_multi(self, datagrams, only=False):
        datagrams = list(datagrams) # We're going to be doing datagrams.remove()

        while datagrams:
            dg = self._read()
            if dg is None: return False # Augh, we didn't see all the dgs yet!
            dg = Datagram(dg)

            for datagram in datagrams:
                if datagram.is_subset_of(dg):
                    datagrams.remove(datagram)
                    break
            else:
                if only:
                    return False

        return True

    def expect_none(self):
        return self._read() == None
<|endoftext|>


<commit_before>#!/usr/bin/env python

__revision__ = "test/t0003.py __REVISION__ __DATE__ __DEVELOPER__"

import TestCmd

test = TestCmd.TestCmd(program = 'scons.py',
			workdir = '',
			interpreter = 'python')

test.write('SConstruct1', """
a ! int(2.0)
""")
test.run(chdir = '.', arguments='-f SConstruct1')
test.fail_test(test.stderr() != """  File "SConstruct1", line 2

    a ! int(2.0)

      ^

SyntaxError: invalid syntax

""")

test.write('SConstruct2', """
raise UserError, 'Depends() require both sources and targets.'
""")
test.run(chdir = '.', arguments='-f SConstruct2')
test.fail_test(test.stderr() != """
SCons error: Depends() require both sources and targets.
File "SConstruct2", line 2, in ?
""")

import os
import string
sconspath = os.path.join(os.getcwd(), 'scons.py')

# Since we're using regular expression matches below, escape any
# backslashes that ended up in the path name.  (Hello, Windows!)
sconspath = string.replace(sconspath, '\\', '\\\\')

test.write('SConstruct3', """
raise InternalError, 'error inside'
""")
test.run(chdir = '.', arguments='-f SConstruct3')
expect = r"""Traceback \((most recent call|innermost) last\):
  File "%s", line 163, in \?
    main\(\)
  File "%s", line 153, in main
    execfile\(file\)
  File "SConstruct3", line 2, in \?
    raise InternalError, 'error inside'
InternalError: error inside
""" % (sconspath, sconspath)
test.fail_test(not test.match_re(test.stderr(), expect))

test.pass_test()
<commit_msg>Generalize the hard-coded line numbers in test/errors.py.<commit_after>#!/usr/bin/env python

__revision__ = "test/t0003.py __REVISION__ __DATE__ __DEVELOPER__"

import TestCmd

test = TestCmd.TestCmd(program = 'scons.py',
			workdir = '',
			interpreter = 'python')

test.write('SConstruct1', """
a ! int(2.0)
""")
test.run(chdir = '.', arguments='-f SConstruct1')
test.fail_test(test.stderr() != """  File "SConstruct1", line 2

    a ! int(2.0)

      ^

SyntaxError: invalid syntax

""")

test.write('SConstruct2', """
raise UserError, 'Depends() require both sources and targets.'
""")
test.run(chdir = '.', arguments='-f SConstruct2')
test.fail_test(test.stderr() != """
SCons error: Depends() require both sources and targets.
File "SConstruct2", line 2, in ?
""")

import os
import string
sconspath = os.path.join(os.getcwd(), 'scons.py')

# Since we're using regular expression matches below, escape any
# backslashes that ended up in the path name.  (Hello, Windows!)
sconspath = string.replace(sconspath, '\\', '\\\\')

test.write('SConstruct3', """
raise InternalError, 'error inside'
""")
test.run(chdir = '.', arguments='-f SConstruct3')
expect = r"""Traceback \((most recent call|innermost) last\):
  File "%s", line \d+, in \?
    main\(\)
  File "%s", line \d+, in main
    execfile\(file\)
  File "SConstruct3", line \d+, in \?
    raise InternalError, 'error inside'
InternalError: error inside
""" % (sconspath, sconspath)
test.fail_test(not test.match_re(test.stderr(), expect))

test.pass_test()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
@author: Sebi

File: test_get_image6d.py
Date: 03.02.2016
Version. 1.0
"""

import numpy as np
import os
import bfimage as bf
import pytest

def test_metainfo():

    filename = r'testdata/Beads_63X_NA1.35_xy=0.042_z=0.1.czi'

    # get image meta-information
    MetaInfo = bf.get_relevant_metainfo_wrapper(filename)
    img6d = bf.get_image6d(filename, MetaInfo['Sizes'])

    # check Sizes
    assert MetaInfo['Sizes'] == [1, 1, 100, 1, 346, 580]
    # check Dimension Order
    assert MetaInfo['DimOrder BF'] == 'XYCZT'
    # check Dimesnion Order CZI Style
    assert MetaInfo['OrderCZI'] == 'BCZYX0'
    # check Image Dimensions
    assert MetaInfo['TotalSeries'] == 1
    assert MetaInfo['SizeT'] == 1
    assert MetaInfo['SizeZ'] == 100
    assert MetaInfo['SizeC'] == 1
    assert MetaInfo['SizeY'] == 346
    assert MetaInfo['SizeX'] == 580
    # check Scaling
    assert MetaInfo['XScale'] == 0.042
    assert MetaInfo['YScale'] == 0.042
    assert MetaInfo['ZScale'] == 0.1
    # check Objective Data
    assert MetaInfo['ObjMag'] == 63.0
    assert MetaInfo['NA'] == 1.4
    assert MetaInfo['Immersion'] == 'Oil'
    # check objective Name
    assert MetaInfo['ObjModel'] == 'na'
    # check Excitation and Emission Wavelengths
    assert MetaInfo['WLEx'] == 493
    assert MetaInfo['WLEm'] == 517
    # check Dye Names
    assert MetaInfo['Dyes'] == ['Dye1']
    # check Channels
    assert MetaInfo['Channels'] == ['Alexa Fluor 488']
    # check Channel Description
    assert MetaInfo['ChDesc'] == []
    # check Numpy Array Shape
    dims = [1, 1, 100, 1, 346, 580]
    for i in range(0, len(dims)):
        assert np.shape(img6d)[i] == dims[i]

if __name__ == '__main__':
    pytest.main()<commit_msg>Update test_bfread.py<commit_after># -*- coding: utf-8 -*-
"""
@author: Sebi

File: test_bfread.py
Date: 08.04.2016
Version. 0.5
"""

import numpy as np
import os
import bfimage as bf
import pytest

def test_metainfo():

    filename = r'testdata/Beads_63X_NA1.35_xy=0.042_z=0.1.czi'

    # get image meta-information
    MetaInfo = bf.get_relevant_metainfo_wrapper(filename)
    img6d = bf.get_image6d(filename, MetaInfo['Sizes'])

    # check Sizes
    assert MetaInfo['Sizes'] == [1, 1, 100, 1, 346, 580]
    # check Dimension Order
    assert MetaInfo['DimOrder BF'] == 'XYCZT'
    # check Dimesnion Order CZI Style
    assert MetaInfo['OrderCZI'] == 'BCZYX0'
    # check Image Dimensions
    assert MetaInfo['TotalSeries'] == 1
    assert MetaInfo['SizeT'] == 1
    assert MetaInfo['SizeZ'] == 100
    assert MetaInfo['SizeC'] == 1
    assert MetaInfo['SizeY'] == 346
    assert MetaInfo['SizeX'] == 580
    # check Scaling
    assert MetaInfo['XScale'] == 0.042
    assert MetaInfo['YScale'] == 0.042
    assert MetaInfo['ZScale'] == 0.1
    # check Objective Data
    assert MetaInfo['ObjMag'] == 63.0
    assert MetaInfo['NA'] == 1.4
    assert MetaInfo['Immersion'] == 'Oil'
    # check objective Name
    assert MetaInfo['ObjModel'] == 'na'
    # check Excitation and Emission Wavelengths
    assert MetaInfo['WLEx'] == 493
    assert MetaInfo['WLEm'] == 517
    # check Dye Names
    assert MetaInfo['Dyes'] == ['Dye1']
    # check Channels
    assert MetaInfo['Channels'] == ['Alexa Fluor 488']
    # check Channel Description
    assert MetaInfo['ChDesc'] == []
    # check Numpy Array Shape
    dims = [1, 1, 100, 1, 346, 580]
    for i in range(0, len(dims)):
        assert np.shape(img6d)[i] == dims[i]

def test_timeseries():

    filename = r'testdata/T=5_Z=3_CH=2_CZT_All_CH_per_Slice.czi'
    # get image meta-information
    MetaInfo = bf.bftools.get_relevant_metainfo_wrapper(filename)
    seriesID = 0
    timepoint = 2
    channel = 1
    zplane = 2
    dims = [5, 2, 640, 640]
    # get the actual time series from the data set
    tseries = bf.bftools.get_timeseries(filename, MetaInfo['Sizes'], seriesID, zplane)

    for i in range(0, len(dims)):
        assert np.shape(tseries)[i] == dims[i]

def test_zstack():

    filename = r'testdata/Beads_63X_NA1.35_xy=0.042_z=0.1.czi'
    # get image meta-information
    MetaInfo = bf.bftools.get_relevant_metainfo_wrapper(filename)
    seriesID = 0
    timepoint = 0
    channel = 0
    dims = [100, 1, 346, 580]
    # get the actual z-stack from the data set
    zstack = bf.bftools.get_zstack(filename, MetaInfo['Sizes'], seriesID, timepoint)

    # get plane with the brightest pixel
    zplane = (zstack == zstack.max()).nonzero()[0][0]
    # check found zplane
    assert zplane == 46

    for i in range(0, len(dims)):
        assert np.shape(zstack)[i] == dims[i]

def test_getdimonly():

    filename = r'testdata/2x2_SNAP_CH=2_Z=5_T=2.czi'

    sizes_czi = bf.czitools.read_dimensions_czi(filename)

    dims = [1, 1, 2, 2, 5, 1216, 1216, 1]
    dimorder = 'BSTCZYX0'

    for i in range(0, len(dims)):
        assert sizes_czi[0][i] == dims[i]

    assert sizes_czi[1] == dimorder

if __name__ == '__main__':
    pytest.main()
<|endoftext|>


<commit_before>from __future__ import (absolute_import, division, print_function)

from collections import defaultdict
from operator import add, iadd, mul, imul
from itertools import product, cycle, chain
import sys

import pytest

from cycler import cycler, Cycler, concat

if sys.version_info < (3,):
    from itertools import izip as zip
    range = xrange
    str = unicode

def _cycler_helper(c, length, keys, values):
    assert len(c) == length
    assert len(c) == len(list(c))
    assert c.keys == set(keys)
    for k, vals in zip(keys, values):
        for v, v_target in zip(c, vals):
            assert v[k] == v_target

def _cycles_equal(c1, c2):
    assert list(c1) == list(c2)
    assert c1 == c2

@pytest.mark.parametrize('c', [cycler(c='rgb'),
                               cycler(c=list('rgb')),
                               cycler(cycler(c='rgb'))],
                         ids=['from string',
                              'from list',
                              'from cycler'])
def test_creation(c):
    _cycler_helper(c, 3, ['c'], [['r', 'g', 'b']])

def test_add():
    c1 = cycler(c='rgb')
    c2 = cycler(lw=range(3))
    # addition
    _cycler_helper(c1+c2, 3, ['c', 'lw'], [list('rgb'), range(3)])
    _cycler_helper(c2+c1, 3, ['c', 'lw'], [list('rgb'), range(3)])
    _cycles_equal(c2+c1, c1+c2)

def test_add_len_mismatch():
    # miss-matched add lengths
    c1 = cycler(c='rgb')
    c3 = cycler(lw=range(15))
    with pytest.raises(ValueError):
        c1 + c3
    with pytest.raises(ValueError):
        c3 + c1

def test_prod():
    c1 = cycler(c='rgb')
    c2 = cycler(lw=range(3))
    c3 = cycler(lw=range(15))
    # multiplication
    target = zip(*product(list('rgb'), range(3)))
    _cycler_helper(c1 * c2, 9, ['c', 'lw'], target)

    target = zip(*product(range(3), list('rgb')))
    _cycler_helper(c2 * c1, 9, ['lw', 'c'], target)

    target = zip(*product(range(15), list('rgb')))
    _cycler_helper(c3 * c1, 45, ['lw', 'c'], target)

def test_inplace():
    c1 = cycler(c='rgb')
    c2 = cycler(lw=range(3))
    c2 += c1
    _cycler_helper(c2, 3, ['c', 'lw'], [list('rgb'), range(3)])

    c3 = cycler(c='rgb')
    c4 = cycler(lw=range(3))
    c3 *= c4
    target = zip(*product(list('rgb'), range(3)))
    _cycler_helper(c3, 9, ['c', 'lw'], target)

def test_constructor():
    c1 = cycler(c='rgb')
    c2 = cycler(ec=c1)
    _cycler_helper(c1+c2, 3, ['c', 'ec'], [['r', 'g', 'b']]*2)
    c3 = cycler(c=c1)
    _cycler_helper(c3+c2, 3, ['c', 'ec'], [['r', 'g', 'b']]*2)
    # Using a non-string hashable
    c4 = cycler(1, range(3))
    _cycler_helper(c4+c1, 3, [1, 'c'], [range(3), ['r', 'g', 'b']])

    # addition using cycler()
    _cycler_helper(cycler(c='rgb', lw=range(3)),
                   3, ['c', 'lw'], [list('rgb'), range(3)])
    _cycler_helper(cycler(lw=range(3), c='rgb'),
                   3, ['c', 'lw'], [list('rgb'), range(3)])
    # Purposely mixing them
    _cycler_helper(cycler(c=range(3), lw=c1),
                   3, ['c', 'lw'], [range(3), list('rgb')])

def test_failures():
    c1 = cycler(c='rgb')
    c2 = cycler(c=c1)
    pytest.raises(ValueError, add, c1, c2)
    pytest.raises(ValueError, iadd, c1, c2)
    pytest.raises(ValueError, mul, c1, c2)
    pytest.raises(ValueError, imul, c1, c2)
    pytest.raises(TypeError, iadd, c2, 'aardvark')
    pytest.raises(TypeError, imul, c2, 'aardvark')

    c3 = cycler(ec=c1)

    pytest.raises(ValueError, cycler, c=c2+c3)

def test_simplify():
    c1 = cycler(c='rgb')
    c2 = cycler(ec=c1)
    for c in [c1 * c2, c2 * c1, c1 + c2]:
        _cycles_equal(c, c.simplify())

def test_multiply():
    c1 = cycler(c='rgb')
    _cycler_helper(2*c1, 6, ['c'], ['rgb'*2])

    c2 = cycler(ec=c1)
    c3 = c1 * c2

    _cycles_equal(2*c3, c3*2)

def test_mul_fails():
    c1 = cycler(c='rgb')
    pytest.raises(TypeError, mul, c1, 2.0)
    pytest.raises(TypeError, mul, c1, 'a')
    pytest.raises(TypeError, mul, c1, [])

def test_getitem():
    c1 = cycler(3, range(15))
    widths = list(range(15))
    for slc in (slice(None, None, None),
                slice(None, None, -1),
                slice(1, 5, None),
                slice(0, 5, 2)):
        _cycles_equal(c1[slc], cycler(3, widths[slc]))

def test_fail_getime():
    c1 = cycler(lw=range(15))
    pytest.raises(ValueError, Cycler.__getitem__, c1, 0)
    pytest.raises(ValueError, Cycler.__getitem__, c1, [0, 1])

def _repr_tester_helper(rpr_func, cyc, target_repr):
    test_repr = getattr(cyc, rpr_func)()

    assert str(test_repr) == str(target_repr)

def test_repr():
    c = cycler(c='rgb')
    # Using an identifier that would be not valid as a kwarg
    c2 = cycler('3rd', range(3))

    c_sum_rpr = "(cycler('c', ['r', 'g', 'b']) + cycler('3rd', [0, 1, 2]))"
    c_prod_rpr = "(cycler('c', ['r', 'g', 'b']) * cycler('3rd', [0, 1, 2]))"

    _repr_tester_helper('__repr__', c + c2, c_sum_rpr)
    _repr_tester_helper('__repr__', c * c2, c_prod_rpr)

    sum_html = (
        "
"
        "'3rd'	'c'	"
        "
0	'r'
"
        "
1	'g'
"
        "
2	'b'
"
        "
")
    prod_html = (
        "
"
        "'3rd'	'c'	"
        "
0	'r'
"
        "
1	'r'
"
        "
2	'r'
"
        "
0	'g'
"
        "
1	'g'
"
        "
2	'g'
"
        "
0	'b'
"
        "
1	'b'
"
        "
2	'b'
"
        "
")

    _repr_tester_helper('_repr_html_', c + c2, sum_html)
    _repr_tester_helper('_repr_html_', c * c2, prod_html)

def test_call():
    c = cycler(c='rgb')
    c_cycle = c()
    assert isinstance(c_cycle, cycle)
    j = 0
    for a, b in zip(2*c, c_cycle):
        j += 1
        assert a == b

    assert j == len(c) * 2

def test_copying():
    # Just about everything results in copying the cycler and
    # its contents (shallow). This set of tests is intended to make sure
    # of that. Our iterables will be mutable for extra fun!
    i1 = [1, 2, 3]
    i2 = ['r', 'g', 'b']
    # For more mutation fun!
    i3 = [['y', 'g'], ['b', 'k']]

    c1 = cycler('c', i1)
    c2 = cycler('lw', i2)
    c3 = cycler('foo', i3)

    c_before = (c1 + c2) * c3

    i1.pop()
    i2.append('cyan')
    i3[0].append('blue')

    c_after = (c1 + c2) * c3

    assert c1 == cycler('c', [1, 2, 3])
    assert c2 == cycler('lw', ['r', 'g', 'b'])
    assert c3 == cycler('foo', [['y', 'g', 'blue'], ['b', 'k']])
    assert c_before == (cycler(c=[1, 2, 3], lw=['r', 'g', 'b']) *
                        cycler('foo', [['y', 'g', 'blue'], ['b', 'k']]))
    assert c_after == (cycler(c=[1, 2, 3], lw=['r', 'g', 'b']) *
                       cycler('foo', [['y', 'g', 'blue'], ['b', 'k']]))

    # Make sure that changing the key for a specific cycler
    # doesn't break things for a composed cycler
    c = (c1 + c2) * c3
    c4 = cycler('bar', c3)
    assert c == (cycler(c=[1, 2, 3], lw=['r', 'g', 'b']) *
                 cycler('foo', [['y', 'g', 'blue'], ['b', 'k']]))
    assert c3 == cycler('foo', [['y', 'g', 'blue'], ['b', 'k']])
    assert c4 == cycler('bar', [['y', 'g', 'blue'], ['b', 'k']])

def test_keychange():
    c1 = cycler('c', 'rgb')
    c2 = cycler('lw', [1, 2, 3])
    c3 = cycler('ec', 'yk')

    c3.change_key('ec', 'edgecolor')
    assert c3 == cycler('edgecolor', c3)

    c = c1 + c2
    c.change_key('lw', 'linewidth')
    # Changing a key in one cycler should have no
    # impact in the original cycler.
    assert c2 == cycler('lw', [1, 2, 3])
    assert c == c1 + cycler('linewidth', c2)

    c = (c1 + c2) * c3
    c.change_key('c', 'color')
    assert c1 == cycler('c', 'rgb')
    assert c == (cycler('color', c1) + c2) * c3

    # Perfectly fine, it is a no-op
    c.change_key('color', 'color')
    assert c == (cycler('color', c1) + c2) * c3

    # Can't change a key to one that is already in there
    pytest.raises(ValueError, Cycler.change_key, c, 'color', 'lw')
    # Can't change a key you don't have
    pytest.raises(KeyError, Cycler.change_key, c, 'c', 'foobar')

def _eq_test_helper(a, b, res):
    if res:
        assert a == b
    else:
        assert a != b

def test_eq():
    a = cycler(c='rgb')
    b = cycler(c='rgb')
    _eq_test_helper(a, b, True)
    _eq_test_helper(a, b[::-1], False)
    c = cycler(lw=range(3))
    _eq_test_helper(a+c, c+a, True)
    _eq_test_helper(a+c, c+b, True)
    _eq_test_helper(a*c, c*a, False)
    _eq_test_helper(a, c, False)
    d = cycler(c='ymk')
    _eq_test_helper(b, d, False)
    e = cycler(c='orange')
    _eq_test_helper(b, e, False)

def test_cycler_exceptions():
    pytest.raises(TypeError, cycler)
    pytest.raises(TypeError, cycler, 'c', 'rgb', lw=range(3))
    pytest.raises(TypeError, cycler, 'c')
    pytest.raises(TypeError, cycler, 'c', 'rgb', 'lw', range(3))

def test_starange_init():
    c = cycler('r', 'rgb')
    c2 = cycler('lw', range(3))
    cy = Cycler(list(c), list(c2), zip)
    assert cy == c + c2

def test_concat():
    a = cycler('a', range(3))
    b = cycler('a', 'abc')
    for con, chn in zip(a.concat(b), chain(a, b)):
        assert con == chn

    for con, chn in zip(concat(a, b), chain(a, b)):
        assert con == chn

def test_concat_fail():
    a = cycler('a', range(3))
    b = cycler('b', range(3))
    pytest.raises(ValueError, concat, a, b)
    pytest.raises(ValueError, a.concat, b)

def _by_key_helper(cy):
    res = cy.by_key()
    target = defaultdict(list)
    for sty in cy:
        for k, v in sty.items():
            target[k].append(v)

    assert res == target

def test_by_key_add():
    input_dict = dict(c=list('rgb'), lw=[1, 2, 3])
    cy = cycler(c=input_dict['c']) + cycler(lw=input_dict['lw'])
    res = cy.by_key()
    assert res == input_dict
    _by_key_helper(cy)

def test_by_key_mul():
    input_dict = dict(c=list('rg'), lw=[1, 2, 3])
    cy = cycler(c=input_dict['c']) * cycler(lw=input_dict['lw'])
    res = cy.by_key()
    assert input_dict['lw'] * len(input_dict['c']) == res['lw']
    _by_key_helper(cy)

def test_contains():
    a = cycler('a', range(3))
    b = cycler('b', range(3))

    assert 'a' in a
    assert 'b' in b
    assert 'a' not in b
    assert 'b' not in a

    ab = a+b

    assert 'a' in ab
    assert 'b' in ab
<commit_msg>improved some formatting<commit_after>from __future__ import (absolute_import, division, print_function)

from collections import defaultdict
from operator import add, iadd, mul, imul
from itertools import product, cycle, chain
import sys

import pytest

from cycler import cycler, Cycler, concat

if sys.version_info < (3,):
    from itertools import izip as zip
    range = xrange
    str = unicode

def _cycler_helper(c, length, keys, values):
    assert len(c) == length
    assert len(c) == len(list(c))
    assert c.keys == set(keys)
    for k, vals in zip(keys, values):
        for v, v_target in zip(c, vals):
            assert v[k] == v_target

def _cycles_equal(c1, c2):
    assert list(c1) == list(c2)
    assert c1 == c2

@pytest.mark.parametrize('c', [cycler(c='rgb'),
                               cycler(c=list('rgb')),
                               cycler(cycler(c='rgb'))],
                         ids=['from string',
                              'from list',
                              'from cycler'])
def test_creation(c):
    _cycler_helper(c, 3, ['c'], [['r', 'g', 'b']])

def test_add():
    c1 = cycler(c='rgb')
    c2 = cycler(lw=range(3))
    # addition
    _cycler_helper(c1 + c2, 3, ['c', 'lw'], [list('rgb'), range(3)])
    _cycler_helper(c2 + c1, 3, ['c', 'lw'], [list('rgb'), range(3)])
    _cycles_equal(c2 + c1, c1 + c2)

def test_add_len_mismatch():
    # miss-matched add lengths
    c1 = cycler(c='rgb')
    c3 = cycler(lw=range(15))
    with pytest.raises(ValueError):
        c1 + c3
    with pytest.raises(ValueError):
        c3 + c1

def test_prod():
    c1 = cycler(c='rgb')
    c2 = cycler(lw=range(3))
    c3 = cycler(lw=range(15))
    # multiplication
    target = zip(*product(list('rgb'), range(3)))
    _cycler_helper(c1 * c2, 9, ['c', 'lw'], target)

    target = zip(*product(range(3), list('rgb')))
    _cycler_helper(c2 * c1, 9, ['lw', 'c'], target)

    target = zip(*product(range(15), list('rgb')))
    _cycler_helper(c3 * c1, 45, ['lw', 'c'], target)

def test_inplace():
    c1 = cycler(c='rgb')
    c2 = cycler(lw=range(3))
    c2 += c1
    _cycler_helper(c2, 3, ['c', 'lw'], [list('rgb'), range(3)])

    c3 = cycler(c='rgb')
    c4 = cycler(lw=range(3))
    c3 *= c4
    target = zip(*product(list('rgb'), range(3)))
    _cycler_helper(c3, 9, ['c', 'lw'], target)

def test_constructor():
    c1 = cycler(c='rgb')
    c2 = cycler(ec=c1)
    _cycler_helper(c1 + c2, 3, ['c', 'ec'], [['r', 'g', 'b']] * 2)
    c3 = cycler(c=c1)
    _cycler_helper(c3 + c2, 3, ['c', 'ec'], [['r', 'g', 'b']] * 2)
    # Using a non-string hashable
    c4 = cycler(1, range(3))
    _cycler_helper(c4 + c1, 3, [1, 'c'], [range(3), ['r', 'g', 'b']])

    # addition using cycler()
    _cycler_helper(cycler(c='rgb', lw=range(3)),
                   3, ['c', 'lw'], [list('rgb'), range(3)])
    _cycler_helper(cycler(lw=range(3), c='rgb'),
                   3, ['c', 'lw'], [list('rgb'), range(3)])
    # Purposely mixing them
    _cycler_helper(cycler(c=range(3), lw=c1),
                   3, ['c', 'lw'], [range(3), list('rgb')])

def test_failures():
    c1 = cycler(c='rgb')
    c2 = cycler(c=c1)
    pytest.raises(ValueError, add, c1, c2)
    pytest.raises(ValueError, iadd, c1, c2)
    pytest.raises(ValueError, mul, c1, c2)
    pytest.raises(ValueError, imul, c1, c2)
    pytest.raises(TypeError, iadd, c2, 'aardvark')
    pytest.raises(TypeError, imul, c2, 'aardvark')

    c3 = cycler(ec=c1)

    pytest.raises(ValueError, cycler, c=c2 + c3)

def test_simplify():
    c1 = cycler(c='rgb')
    c2 = cycler(ec=c1)
    for c in [c1 * c2, c2 * c1, c1 + c2]:
        _cycles_equal(c, c.simplify())

def test_multiply():
    c1 = cycler(c='rgb')
    _cycler_helper(2 * c1, 6, ['c'], ['rgb' * 2])

    c2 = cycler(ec=c1)
    c3 = c1 * c2

    _cycles_equal(2 * c3, c3 * 2)

def test_mul_fails():
    c1 = cycler(c='rgb')
    pytest.raises(TypeError, mul, c1, 2.0)
    pytest.raises(TypeError, mul, c1, 'a')
    pytest.raises(TypeError, mul, c1, [])

def test_getitem():
    c1 = cycler(3, range(15))
    widths = list(range(15))
    for slc in (slice(None, None, None),
                slice(None, None, -1),
                slice(1, 5, None),
                slice(0, 5, 2)):
        _cycles_equal(c1[slc], cycler(3, widths[slc]))

def test_fail_getime():
    c1 = cycler(lw=range(15))
    pytest.raises(ValueError, Cycler.__getitem__, c1, 0)
    pytest.raises(ValueError, Cycler.__getitem__, c1, [0, 1])

def _repr_tester_helper(rpr_func, cyc, target_repr):
    test_repr = getattr(cyc, rpr_func)()

    assert str(test_repr) == str(target_repr)

def test_repr():
    c = cycler(c='rgb')
    # Using an identifier that would be not valid as a kwarg
    c2 = cycler('3rd', range(3))

    c_sum_rpr = "(cycler('c', ['r', 'g', 'b']) + cycler('3rd', [0, 1, 2]))"
    c_prod_rpr = "(cycler('c', ['r', 'g', 'b']) * cycler('3rd', [0, 1, 2]))"

    _repr_tester_helper('__repr__', c + c2, c_sum_rpr)
    _repr_tester_helper('__repr__', c * c2, c_prod_rpr)

    sum_html = (
        "
"
        "'3rd'	'c'	"
        "
0	'r'
"
        "
1	'g'
"
        "
2	'b'
"
        "
")
    prod_html = (
        "
"
        "'3rd'	'c'	"
        "
0	'r'
"
        "
1	'r'
"
        "
2	'r'
"
        "
0	'g'
"
        "
1	'g'
"
        "
2	'g'
"
        "
0	'b'
"
        "
1	'b'
"
        "
2	'b'
"
        "
")

    _repr_tester_helper('_repr_html_', c + c2, sum_html)
    _repr_tester_helper('_repr_html_', c * c2, prod_html)

def test_call():
    c = cycler(c='rgb')
    c_cycle = c()
    assert isinstance(c_cycle, cycle)
    j = 0
    for a, b in zip(2 * c, c_cycle):
        j += 1
        assert a == b

    assert j == len(c) * 2

def test_copying():
    # Just about everything results in copying the cycler and
    # its contents (shallow). This set of tests is intended to make sure
    # of that. Our iterables will be mutable for extra fun!
    i1 = [1, 2, 3]
    i2 = ['r', 'g', 'b']
    # For more mutation fun!
    i3 = [['y', 'g'], ['b', 'k']]

    c1 = cycler('c', i1)
    c2 = cycler('lw', i2)
    c3 = cycler('foo', i3)

    c_before = (c1 + c2) * c3

    i1.pop()
    i2.append('cyan')
    i3[0].append('blue')

    c_after = (c1 + c2) * c3

    assert c1 == cycler('c', [1, 2, 3])
    assert c2 == cycler('lw', ['r', 'g', 'b'])
    assert c3 == cycler('foo', [['y', 'g', 'blue'], ['b', 'k']])
    assert c_before == (cycler(c=[1, 2, 3], lw=['r', 'g', 'b']) *
                        cycler('foo', [['y', 'g', 'blue'], ['b', 'k']]))
    assert c_after == (cycler(c=[1, 2, 3], lw=['r', 'g', 'b']) *
                       cycler('foo', [['y', 'g', 'blue'], ['b', 'k']]))

    # Make sure that changing the key for a specific cycler
    # doesn't break things for a composed cycler
    c = (c1 + c2) * c3
    c4 = cycler('bar', c3)
    assert c == (cycler(c=[1, 2, 3], lw=['r', 'g', 'b']) *
                 cycler('foo', [['y', 'g', 'blue'], ['b', 'k']]))
    assert c3 == cycler('foo', [['y', 'g', 'blue'], ['b', 'k']])
    assert c4 == cycler('bar', [['y', 'g', 'blue'], ['b', 'k']])

def test_keychange():
    c1 = cycler('c', 'rgb')
    c2 = cycler('lw', [1, 2, 3])
    c3 = cycler('ec', 'yk')

    c3.change_key('ec', 'edgecolor')
    assert c3 == cycler('edgecolor', c3)

    c = c1 + c2
    c.change_key('lw', 'linewidth')
    # Changing a key in one cycler should have no
    # impact in the original cycler.
    assert c2 == cycler('lw', [1, 2, 3])
    assert c == c1 + cycler('linewidth', c2)

    c = (c1 + c2) * c3
    c.change_key('c', 'color')
    assert c1 == cycler('c', 'rgb')
    assert c == (cycler('color', c1) + c2) * c3

    # Perfectly fine, it is a no-op
    c.change_key('color', 'color')
    assert c == (cycler('color', c1) + c2) * c3

    # Can't change a key to one that is already in there
    pytest.raises(ValueError, Cycler.change_key, c, 'color', 'lw')
    # Can't change a key you don't have
    pytest.raises(KeyError, Cycler.change_key, c, 'c', 'foobar')

def _eq_test_helper(a, b, res):
    if res:
        assert a == b
    else:
        assert a != b

def test_eq():
    a = cycler(c='rgb')
    b = cycler(c='rgb')
    _eq_test_helper(a, b, True)
    _eq_test_helper(a, b[::-1], False)
    c = cycler(lw=range(3))
    _eq_test_helper(a+c, c+a, True)
    _eq_test_helper(a+c, c+b, True)
    _eq_test_helper(a*c, c*a, False)
    _eq_test_helper(a, c, False)
    d = cycler(c='ymk')
    _eq_test_helper(b, d, False)
    e = cycler(c='orange')
    _eq_test_helper(b, e, False)

def test_cycler_exceptions():
    pytest.raises(TypeError, cycler)
    pytest.raises(TypeError, cycler, 'c', 'rgb', lw=range(3))
    pytest.raises(TypeError, cycler, 'c')
    pytest.raises(TypeError, cycler, 'c', 'rgb', 'lw', range(3))

def test_starange_init():
    c = cycler('r', 'rgb')
    c2 = cycler('lw', range(3))
    cy = Cycler(list(c), list(c2), zip)
    assert cy == c + c2

def test_concat():
    a = cycler('a', range(3))
    b = cycler('a', 'abc')
    for con, chn in zip(a.concat(b), chain(a, b)):
        assert con == chn

    for con, chn in zip(concat(a, b), chain(a, b)):
        assert con == chn

def test_concat_fail():
    a = cycler('a', range(3))
    b = cycler('b', range(3))
    pytest.raises(ValueError, concat, a, b)
    pytest.raises(ValueError, a.concat, b)

def _by_key_helper(cy):
    res = cy.by_key()
    target = defaultdict(list)
    for sty in cy:
        for k, v in sty.items():
            target[k].append(v)

    assert res == target

def test_by_key_add():
    input_dict = dict(c=list('rgb'), lw=[1, 2, 3])
    cy = cycler(c=input_dict['c']) + cycler(lw=input_dict['lw'])
    res = cy.by_key()
    assert res == input_dict
    _by_key_helper(cy)

def test_by_key_mul():
    input_dict = dict(c=list('rg'), lw=[1, 2, 3])
    cy = cycler(c=input_dict['c']) * cycler(lw=input_dict['lw'])
    res = cy.by_key()
    assert input_dict['lw'] * len(input_dict['c']) == res['lw']
    _by_key_helper(cy)

def test_contains():
    a = cycler('a', range(3))
    b = cycler('b', range(3))

    assert 'a' in a
    assert 'b' in b
    assert 'a' not in b
    assert 'b' not in a

    ab = a + b

    assert 'a' in ab
    assert 'b' in ab
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (C) 2015 Jouni Roivas <jroivas@iki.fi>

import sshserver

class MyHandler(sshserver.SSHThread):
    def __init__(self, conn, key_handler):
        sshserver.SSHThread.__init__(self, conn, key_handler)

    def handler(self, transport, server, channel):
        channel.send('Hi %s, fingerprint %s\r\n' %
            (server.username,
             self.key_handler.key_str(server.key.get_fingerprint())))
        channel.send('Write "exit" to quit.\r\n\r\n')

        fd = channel.makefile('rU')
        channel.send('entry: ')
        data = ''

        while self.running:
            tmp = fd.read(1)
            if ord(tmp) == 127:  # Special backspace
                if data:
                    data = data[:-1]
                    channel.send('\b \b')
            elif tmp == '\n' or tmp == '\r':
                channel.send('\r\n')
                if data.strip() == 'exit':
                    self.running = False
                channel.send(data.strip() + '\r\n')
                channel.send('entry: ')
                data = ''
            else:
                data += tmp
                channel.send(tmp)

if __name__ == '__main__':
    key_handler = sshserver.SSHKeyHandler(auth_file='auth_keys', host_key='~/.ssh/id_rsa')
    server = sshserver.ThreadedSSHServer(MyHandler, key_handler, verbose=True)
    server.start()
    server.join()
<commit_msg>Fix old example<commit_after>#!/usr/bin/env python

# Copyright (C) 2015 Jouni Roivas <jroivas@iki.fi>

import sshserver

class MyHandler(sshserver.SSHThread):
    def __init__(self, conn, key_handler, master):
        sshserver.SSHThread.__init__(self, conn, key_handler, master)

    def handler(self):
        self._channel.send('Hi %s, fingerprint %s\r\n' %
            (self._server.username,
             self.key_handler.key_str(self._server.key.get_fingerprint())))
        self._channel.send('Write "exit" to quit.\r\n\r\n')

        fd = self._channel.makefile('rU')
        self._channel.send('entry: ')
        data = ''

        while self.running:
            tmp = fd.read(1)
            if ord(tmp) == 127:  # Special backspace
                if data:
                    data = data[:-1]
                    self._channel.send('\b \b')
            elif tmp == '\n' or tmp == '\r':
                self._channel.send('\r\n')
                if data.strip() == 'exit':
                    self.running = False
                self._channel.send(data.strip() + '\r\n')
                self._channel.send('entry: ')
                data = ''
            else:
                data += tmp
                self._channel.send(tmp)

if __name__ == '__main__':
    key_handler = sshserver.SSHKeyHandler(auth_file='auth_keys', host_key='~/.ssh/id_rsa')
    server = sshserver.ThreadedSSHServer(MyHandler, key_handler, verbose=True)
    server.start()
    server.join()
<|endoftext|>


<commit_before>import unittest
import pg8000
import datetime
from connection_settings import db_connect

dbapi = pg8000.DBAPI
db2 = dbapi.connect(**db_connect)

# DBAPI compatible interface tests
class Tests(unittest.TestCase):
    def setUp(self):
        c = db2.cursor()
        try:
            c.execute("DROP TABLE t1")
        except pg8000.DatabaseError, e:
            # the only acceptable error is:
            self.assert_(e.args[1] == '42P01', # table does not exist
                    "incorrect error for drop table")
        c.execute("CREATE TEMPORARY TABLE t1 (f1 int primary key, f2 int not null, f3 varchar(50) null)")
        c.execute("INSERT INTO t1 (f1, f2, f3) VALUES (%s, %s, %s)", (1, 1, None))
        c.execute("INSERT INTO t1 (f1, f2, f3) VALUES (%s, %s, %s)", (2, 10, None))
        c.execute("INSERT INTO t1 (f1, f2, f3) VALUES (%s, %s, %s)", (3, 100, None))
        c.execute("INSERT INTO t1 (f1, f2, f3) VALUES (%s, %s, %s)", (4, 1000, None))
        c.execute("INSERT INTO t1 (f1, f2, f3) VALUES (%s, %s, %s)", (5, 10000, None))

    def testParallelQueries(self):
        c1 = db2.cursor()
        c2 = db2.cursor()
        c1.execute("SELECT f1, f2, f3 FROM t1")
        while 1:
            row = c1.fetchone()
            if row == None:
                break
            f1, f2, f3 = row
            c2.execute("SELECT f1, f2, f3 FROM t1 WHERE f1 > %s", (f1,))
            while 1:
                row = c2.fetchone()
                if row == None:
                    break
                f1, f2, f3 = row

    def testQmark(self):
        orig_paramstyle = dbapi.paramstyle
        try:
            dbapi.paramstyle = "qmark"
            c1 = db2.cursor()
            c1.execute("SELECT f1, f2, f3 FROM t1 WHERE f1 > ?", (3,))
            while 1:
                row = c1.fetchone()
                if row == None:
                    break
                f1, f2, f3 = row
        finally:
            dbapi.paramstyle = orig_paramstyle

    def testNumeric(self):
        orig_paramstyle = dbapi.paramstyle
        try:
            dbapi.paramstyle = "numeric"
            c1 = db2.cursor()
            c1.execute("SELECT f1, f2, f3 FROM t1 WHERE f1 > :1", (3,))
            while 1:
                row = c1.fetchone()
                if row == None:
                    break
                f1, f2, f3 = row
        finally:
            dbapi.paramstyle = orig_paramstyle

    def testNamed(self):
        orig_paramstyle = dbapi.paramstyle
        try:
            dbapi.paramstyle = "named"
            c1 = db2.cursor()
            c1.execute("SELECT f1, f2, f3 FROM t1 WHERE f1 > :f1", {"f1": 3})
            while 1:
                row = c1.fetchone()
                if row == None:
                    break
                f1, f2, f3 = row
        finally:
            dbapi.paramstyle = orig_paramstyle

    def testFormat(self):
        orig_paramstyle = dbapi.paramstyle
        try:
            dbapi.paramstyle = "format"
            c1 = db2.cursor()
            c1.execute("SELECT f1, f2, f3 FROM t1 WHERE f1 > %s", (3,))
            while 1:
                row = c1.fetchone()
                if row == None:
                    break
                f1, f2, f3 = row
        finally:
            dbapi.paramstyle = orig_paramstyle

    def testPyformat(self):
        orig_paramstyle = dbapi.paramstyle
        try:
            dbapi.paramstyle = "pyformat"
            c1 = db2.cursor()
            c1.execute("SELECT f1, f2, f3 FROM t1 WHERE f1 > %(f1)s", {"f1": 3})
            while 1:
                row = c1.fetchone()
                if row == None:
                    break
                f1, f2, f3 = row
        finally:
            dbapi.paramstyle = orig_paramstyle

    def testArraysize(self):
        c1 = db2.cursor()
        c1.arraysize = 3
        c1.execute("SELECT * FROM t1")
        retval = c1.fetchmany()
        self.assert_(len(retval) == c1.arraysize,
                "fetchmany returned wrong number of rows")

    def testDate(self):
        val = dbapi.Date(2001, 2, 3)
        self.assert_(val == datetime.date(2001, 2, 3),
                "Date constructor value match failed")

    def testTime(self):
        val = dbapi.Time(4, 5, 6)
        self.assert_(val == datetime.time(4, 5, 6),
                "Time constructor value match failed")

    def testTimestamp(self):
        val = dbapi.Timestamp(2001, 2, 3, 4, 5, 6)
        self.assert_(val == datetime.datetime(2001, 2, 3, 4, 5, 6),
                "Timestamp constructor value match failed")

    def testDateFromTicks(self):
        val = dbapi.DateFromTicks(1173804319)
        self.assert_(val == datetime.date(2007, 3, 13),
                "DateFromTicks constructor value match failed")

    def testTimeFromTicks(self):
        val = dbapi.TimeFromTicks(1173804319)
        self.assert_(val == datetime.time(10, 45, 19),
                "TimeFromTicks constructor value match failed")

    def testTimestampFromTicks(self):
        val = dbapi.TimestampFromTicks(1173804319)
        self.assert_(val == datetime.datetime(2007, 3, 13, 10, 45, 19),
                "TimestampFromTicks constructor value match failed")

    def testBinary(self):
        v = dbapi.Binary("\x00\x01\x02\x03\x02\x01\x00")
        self.assert_(v == "\x00\x01\x02\x03\x02\x01\x00",
                "Binary value match failed")
        self.assert_(isinstance(v, pg8000.Bytea),
                "Binary type match failed")

    def testRowCount(self):
        c1 = db2.cursor()
        c1.execute("SELECT * FROM t1")
        self.assertEquals(5, c1.rowcount)

    def testFetchMany(self):
        cursor = db2.cursor()
        cursor.arraysize = 2
        cursor.execute("SELECT * FROM t1")
        self.assertEquals(2, len(cursor.fetchmany()))
        self.assertEquals(2, len(cursor.fetchmany()))
        self.assertEquals(1, len(cursor.fetchmany()))
        self.assertEquals(0, len(cursor.fetchmany()))

if __name__ == "__main__":
    unittest.main()

<commit_msg>add UPDATE/DELETE rowcount tests<commit_after>import unittest
import pg8000
import datetime
from connection_settings import db_connect

dbapi = pg8000.DBAPI
db2 = dbapi.connect(**db_connect)

# DBAPI compatible interface tests
class Tests(unittest.TestCase):
    def setUp(self):
        c = db2.cursor()
        try:
            c.execute("DROP TABLE t1")
        except pg8000.DatabaseError, e:
            # the only acceptable error is:
            self.assert_(e.args[1] == '42P01', # table does not exist
                    "incorrect error for drop table")
        c.execute("CREATE TEMPORARY TABLE t1 (f1 int primary key, f2 int not null, f3 varchar(50) null)")
        c.execute("INSERT INTO t1 (f1, f2, f3) VALUES (%s, %s, %s)", (1, 1, None))
        c.execute("INSERT INTO t1 (f1, f2, f3) VALUES (%s, %s, %s)", (2, 10, None))
        c.execute("INSERT INTO t1 (f1, f2, f3) VALUES (%s, %s, %s)", (3, 100, None))
        c.execute("INSERT INTO t1 (f1, f2, f3) VALUES (%s, %s, %s)", (4, 1000, None))
        c.execute("INSERT INTO t1 (f1, f2, f3) VALUES (%s, %s, %s)", (5, 10000, None))

    def testParallelQueries(self):
        c1 = db2.cursor()
        c2 = db2.cursor()
        c1.execute("SELECT f1, f2, f3 FROM t1")
        while 1:
            row = c1.fetchone()
            if row == None:
                break
            f1, f2, f3 = row
            c2.execute("SELECT f1, f2, f3 FROM t1 WHERE f1 > %s", (f1,))
            while 1:
                row = c2.fetchone()
                if row == None:
                    break
                f1, f2, f3 = row

    def testQmark(self):
        orig_paramstyle = dbapi.paramstyle
        try:
            dbapi.paramstyle = "qmark"
            c1 = db2.cursor()
            c1.execute("SELECT f1, f2, f3 FROM t1 WHERE f1 > ?", (3,))
            while 1:
                row = c1.fetchone()
                if row == None:
                    break
                f1, f2, f3 = row
        finally:
            dbapi.paramstyle = orig_paramstyle

    def testNumeric(self):
        orig_paramstyle = dbapi.paramstyle
        try:
            dbapi.paramstyle = "numeric"
            c1 = db2.cursor()
            c1.execute("SELECT f1, f2, f3 FROM t1 WHERE f1 > :1", (3,))
            while 1:
                row = c1.fetchone()
                if row == None:
                    break
                f1, f2, f3 = row
        finally:
            dbapi.paramstyle = orig_paramstyle

    def testNamed(self):
        orig_paramstyle = dbapi.paramstyle
        try:
            dbapi.paramstyle = "named"
            c1 = db2.cursor()
            c1.execute("SELECT f1, f2, f3 FROM t1 WHERE f1 > :f1", {"f1": 3})
            while 1:
                row = c1.fetchone()
                if row == None:
                    break
                f1, f2, f3 = row
        finally:
            dbapi.paramstyle = orig_paramstyle

    def testFormat(self):
        orig_paramstyle = dbapi.paramstyle
        try:
            dbapi.paramstyle = "format"
            c1 = db2.cursor()
            c1.execute("SELECT f1, f2, f3 FROM t1 WHERE f1 > %s", (3,))
            while 1:
                row = c1.fetchone()
                if row == None:
                    break
                f1, f2, f3 = row
        finally:
            dbapi.paramstyle = orig_paramstyle

    def testPyformat(self):
        orig_paramstyle = dbapi.paramstyle
        try:
            dbapi.paramstyle = "pyformat"
            c1 = db2.cursor()
            c1.execute("SELECT f1, f2, f3 FROM t1 WHERE f1 > %(f1)s", {"f1": 3})
            while 1:
                row = c1.fetchone()
                if row == None:
                    break
                f1, f2, f3 = row
        finally:
            dbapi.paramstyle = orig_paramstyle

    def testArraysize(self):
        c1 = db2.cursor()
        c1.arraysize = 3
        c1.execute("SELECT * FROM t1")
        retval = c1.fetchmany()
        self.assert_(len(retval) == c1.arraysize,
                "fetchmany returned wrong number of rows")

    def testDate(self):
        val = dbapi.Date(2001, 2, 3)
        self.assert_(val == datetime.date(2001, 2, 3),
                "Date constructor value match failed")

    def testTime(self):
        val = dbapi.Time(4, 5, 6)
        self.assert_(val == datetime.time(4, 5, 6),
                "Time constructor value match failed")

    def testTimestamp(self):
        val = dbapi.Timestamp(2001, 2, 3, 4, 5, 6)
        self.assert_(val == datetime.datetime(2001, 2, 3, 4, 5, 6),
                "Timestamp constructor value match failed")

    def testDateFromTicks(self):
        val = dbapi.DateFromTicks(1173804319)
        self.assert_(val == datetime.date(2007, 3, 13),
                "DateFromTicks constructor value match failed")

    def testTimeFromTicks(self):
        val = dbapi.TimeFromTicks(1173804319)
        self.assert_(val == datetime.time(10, 45, 19),
                "TimeFromTicks constructor value match failed")

    def testTimestampFromTicks(self):
        val = dbapi.TimestampFromTicks(1173804319)
        self.assert_(val == datetime.datetime(2007, 3, 13, 10, 45, 19),
                "TimestampFromTicks constructor value match failed")

    def testBinary(self):
        v = dbapi.Binary("\x00\x01\x02\x03\x02\x01\x00")
        self.assert_(v == "\x00\x01\x02\x03\x02\x01\x00",
                "Binary value match failed")
        self.assert_(isinstance(v, pg8000.Bytea),
                "Binary type match failed")

    def testRowCount(self):
        c1 = db2.cursor()
        c1.execute("SELECT * FROM t1")
        self.assertEquals(5, c1.rowcount)

        c1.execute("UPDATE t1 SET f3 = %s WHERE f2 > 101", ("Hello!",))
        self.assertEquals(2, c1.rowcount)

        c1.execute("DELETE FROM t1")
        self.assertEquals(5, c1.rowcount)

    def testFetchMany(self):
        cursor = db2.cursor()
        cursor.arraysize = 2
        cursor.execute("SELECT * FROM t1")
        self.assertEquals(2, len(cursor.fetchmany()))
        self.assertEquals(2, len(cursor.fetchmany()))
        self.assertEquals(1, len(cursor.fetchmany()))
        self.assertEquals(0, len(cursor.fetchmany()))

if __name__ == "__main__":
    unittest.main()

<|endoftext|>


<commit_before>from django import forms

from ..forms import UserKwargModelFormMixin
from .models import Article

class FormWithUserKwarg(UserKwargModelFormMixin, forms.Form):
    field1 = forms.CharField()

class ArticleForm(forms.ModelForm):
    class Meta:
        model = Article
<commit_msg>not sure why I did that.<commit_after>from django import forms

from braces.forms import UserKwargModelFormMixin
from .models import Article

class FormWithUserKwarg(UserKwargModelFormMixin, forms.Form):
    field1 = forms.CharField()

class ArticleForm(forms.ModelForm):
    class Meta:
        model = Article
<|endoftext|>


<commit_before>import CTK

# http://stackoverflow.com/questions/1446765/python-cgi-fieldstorage-too-slow
# http://tools.cherrypy.org/wiki/DirectToDiskFileUpload

class default:
    def __init__ (self):
        self.page  = CTK.Page ()
        self.page += CTK.Uploader()

    def __call__ (self):
        return self.page.Render()

tmp = """
import os
import cgi

class MyFieldStorage(cgi.FieldStorage):
    def make_file (self, binary=None):
        print "~~~~~> %s <~~~~~~" %(self.filename)
        return open(os.path.join('/tmp', self.filename), 'wb')

def upload():
    scgi = CTK.request.scgi

    print "Reading.."
    form = MyFieldStorage (fp=scgi.rfile, environ=scgi.env, keep_blank_values=1)

    return "ok"
"""

CTK.publish ('', default)
#CTK.publish ("/upload", upload)
CTK.run (port=8000)
<commit_msg>Clean up<commit_after>import CTK

class default:
    def __init__ (self):
        self.page  = CTK.Page ()
        self.page += CTK.Uploader()

    def __call__ (self):
        return self.page.Render()

CTK.publish ('', default)
CTK.run (port=8000)
<|endoftext|>