<commit_before>"""
Test the color-format conversion utilities.

"""
import unittest

import webcolors

class HexConversionTests(unittest.TestCase):
    """
    Test the functions which convert from hex color codes to other
    formats.

    """
    def test_hex_to_name(self):
        """
        Test conversion from hex to color name.
        """
        test_pairs = (
            (u'#ffffff', u'white'),
            (u'#fff', u'white'),
            (u'#000080', u'navy'),
            (u'#daa520', u'goldenrod')
        )

        for hex_value, name in test_pairs:
            assert name == webcolors.hex_to_name(hex_value)

    def test_hex_to_name_unnamed(self):
        """
        A hex code which does not correspond to a named color, or does
        not correspond to a named color in the given specification,
        raises ValueError.

        """
        # No name in any spec.
        self.assertRaises(
            ValueError,
            webcolors.hex_to_name,
            '#123456'
        )

        # This is 'goldenrod' in CSS 3 list, unnamed in HTML 4.
        self.assertRaises(
            ValueError,
            webcolors.hex_to_name,
            '#daa520',
            spec=u'html4'
        )

    def test_hex_to_name_specs(self):
        """
        Using one of the supported specifications succeeds; using an
        unsupported specification raises ValueError.

        """
        for supported_spec in webcolors.SUPPORTED_SPECIFICATIONS:
            result = webcolors.hex_to_name(u'#ffffff', spec=supported_spec)
            assert u'white' == result

        for unsupported_spec in (u'css1', u'css4', u'html5'):
            self.assertRaises(
                ValueError,
                webcolors.hex_to_name,
                '#ffffff',
                spec=unsupported_spec
            )

    def test_hex_to_rgb(self):
        """
        Test conversion from hex to integer RGB triplet.

        """
        test_pairs = (
            (u'#fff', (255, 255, 255)),
            (u'#ffffff', (255, 255, 255)),
            (u'#000080', (0, 0, 128))
        )

        for hex_value, triplet in test_pairs:
            result = webcolors.hex_to_rgb(hex_value)
            assert isinstance(result, webcolors.IntegerRGB)
            assert triplet == result

    def test_hex_to_rgb_percent(self):
        """
        Test conversion from hex to percent RGB triplet.

        """
        test_pairs = (
            (u'#fff', (u'100%', u'100%', u'100%')),
            (u'#ffffff', (u'100%', u'100%', u'100%')),
            (u'#000080', (u'0%', u'0%', u'50%'))
        )

        for hex_value, triplet in test_pairs:
            result = webcolors.hex_to_rgb_percent(hex_value)
            assert isinstance(result, webcolors.PercentRGB)
            assert triplet == result

class IntegerRGBConversionTests(unittest.TestCase):
    """
    Test the functions which convert from integer RGB triplets to
    other formats.

    """
    def test_rgb_to_name(self):
        """
        Test conversion from integer RGB triplet to color name.

        """
        test_pairs = (
            ((255, 255, 255), u'white'),
            ((0, 0, 128), u'navy'),
            ((218, 165, 32), u'goldenrod'),
            (webcolors.IntegerRGB(218, 165, 32), u'goldenrod')
        )

        for triplet, name in test_pairs:
            assert name == webcolors.rgb_to_name(triplet)

    def test_rgb_to_name_unnamed(self):
        """
        An integer RGB triplet which does not correspond to a named
        color, or does not correspond to a named color in the given
        specification, raises ValueError.

        """
        # No name in any spec.
        self.assertRaises(
            ValueError,
            webcolors.rgb_to_name,
            (18, 52, 86)
        )

        # This is 'goldenrod' in CSS 3 list, unnamed in HTML 4.
        self.assertRaises(
            ValueError,
            webcolors.rgb_to_name,
            (218, 165, 32),
            spec=u'html4'
        )

    def test_rgb_to_name_specs(self):
        """
        Using one of the supported specifications succeeds; an
        unsupported specification raises ValueError.

        """
        for supported_spec in webcolors.SUPPORTED_SPECIFICATIONS:
            result = webcolors.rgb_to_name(
                (255, 255, 255), spec=supported_spec
            )
            assert u'white' == result

        for unsupported_spec in (u'css1', u'css4', u'html5'):
            self.assertRaises(
                ValueError,
                webcolors.rgb_to_name,
                (255, 255, 255),
                spec=unsupported_spec
            )

    def test_rgb_to_hex(self):
        """
        Test conversion from integer RGB triplet to hex.

        """
        test_pairs = (
            ((255, 255, 255), u'#ffffff'),
            ((0, 0, 128), u'#000080'),
            ((218, 165, 32), u'#daa520')
        )

        for triplet, hex_value in test_pairs:
            assert hex_value == webcolors.rgb_to_hex(triplet)

    def test_rgb_to_rgb_percent(self):
        """
        Test conversion from integer RGB triplet to percent RGB
        triplet.

        """
        test_pairs = (
            ((255, 255, 255), (u'100%', u'100%', u'100%')),
            ((0, 0, 128), (u'0%', u'0%', u'50%')),
            ((218, 165, 32), (u'85.49%', u'64.71%', u'12.5%'))
        )

        for triplet, percent_triplet in test_pairs:
            result = webcolors.rgb_to_rgb_percent(triplet)
            assert isinstance(result, webcolors.PercentRGB)
            assert percent_triplet == result

class NameConversionTests(unittest.TestCase):
    """
    Test the functions which convert from color names to other
    formats.

    """
    def test_name_to_hex(self):
        """
        Test correct conversion of color names to hex.
        """
        test_pairs = (
            (u'white', u'#ffffff'),
            (u'navy', u'#000080'),
            (u'goldenrod', u'#daa520')
        )

        for name, hex_value in test_pairs:
            assert hex_value == webcolors.name_to_hex(name)

    def test_name_to_hex_bad_name(self):
        """
        A name which does not correspond to a color, or does not
        correspond to a color in the given specification, raises
        ValueError.

        """
        test_values = (
            {u'name': u'goldenrod',
             u'spec': u'html4'},
            {u'name': u'glue',
             u'spec': u'css21'},
            {u'name': u'breen',
             u'spec': u'css3'},
        )

        for kwarg_dict in test_values:
            self.assertRaises(
                ValueError,
                webcolors.name_to_hex,
                **kwarg_dict
            )

    def test_name_to_hex_specs(self):
        """
        Using one of the supported specifications succeeds; using an
        unsupported specification raises ValueError.

        """
        for supported_spec in webcolors.SUPPORTED_SPECIFICATIONS:
            result = webcolors.name_to_hex(u'white', spec=supported_spec)
            assert u'#ffffff' == result

        for unsupported_spec in (u'css1', u'css4', u'html5'):
            self.assertRaises(
                ValueError,
                webcolors.name_to_hex,
                'white', spec=unsupported_spec
            )

    def test_name_to_rgb(self):
        """
        Test conversion from color name to integer RGB triplet.

        """
        test_pairs = (
            (u'white', (255, 255, 255)),
            (u'navy', (0, 0, 128)),
            (u'goldenrod', (218, 165, 32))
        )

        for name, triplet in test_pairs:
            result = webcolors.name_to_rgb(name)
            assert isinstance(result, webcolors.IntegerRGB)
            assert triplet == result

    def test_name_to_rgb_percent(self):
        """
        Test conversion from color name to percent RGB triplet.

        """
        test_pairs = (
            (u'white', (u'100%', u'100%', u'100%')),
            (u'navy', (u'0%', u'0%', u'50%')),
            (u'goldenrod', (u'85.49%', u'64.71%', u'12.5%'))
        )

        for name, triplet in test_pairs:
            result = webcolors.name_to_rgb_percent(name)
            assert isinstance(result, webcolors.PercentRGB)
            assert triplet == result

class PercentRGBConversionTests(unittest.TestCase):
    """
    Test the functions which convert from percent RGB triplets to
    other formats.

    """
    def test_rgb_percent_to_name(self):
        """
        Test conversion from percent RGB triplet to color name.
        """
        test_pairs = (
            ((u'100%', u'100%', u'100%'), u'white'),
            ((u'0%', u'0%', u'50%'), u'navy'),
            ((u'85.49%', u'64.71%', u'12.5%'), u'goldenrod'),
            (webcolors.PercentRGB(u'85.49%', u'64.71%', u'12.5%'),
             u'goldenrod')
        )

        for triplet, name in test_pairs:
            assert name == webcolors.rgb_percent_to_name(triplet)

    def test_rgb_percent_to_name_unnamed(self):
        """
        A percent RGB triplet which does not correspond to a named
        color, or does not correspond to a named color in the given
        specification, raises ValueError.

        """
        # No name in any spec.
        self.assertRaises(
            ValueError,
            webcolors.rgb_percent_to_name,
            (u'7.06%', u'20.39%', u'33.73%')
        )

        # This is 'goldenrod' in CSS 3 list, unnamed in HTML 4.
        self.assertRaises(
            ValueError,
            webcolors.rgb_percent_to_name,
            (u'85.49%', u'64.71%', u'12.5%'),
            spec=u'html4'
        )

    def test_rgb_percent_to_name_specs(self):
        """
        Using one of the supported specifications succeeds; an
        unsupported specification raises ValueError.

        """
        for supported_spec in (u'html4', u'css2', u'css21', u'css3'):
            result = webcolors.rgb_percent_to_name(
                (u'100%', u'100%', u'100%'), spec=supported_spec
            )
            assert u'white' == result

        for unsupported_spec in (u'css1', u'css4', u'html5'):
            self.assertRaises(
                ValueError,
                webcolors.rgb_percent_to_name,
                (u'100%', u'100%', u'100%'),
                spec=unsupported_spec
            )

    def test_rgb_percent_to_hex(self):
        """
        Test conversion from percent RGB triplet to hex.

        """
        test_pairs = (
            ((u'100%', u'100%', u'0%'), u'#ffff00'),
            ((u'0%', u'0%', u'50%'), u'#000080'),
            ((u'85.49%', u'64.71%', u'12.5%'), u'#daa520')
        )

        for triplet, hex_value in test_pairs:
            assert hex_value == webcolors.rgb_percent_to_hex(triplet)

    def test_rgb_percent_to_rgb(self):
        """
        Test conversion from percent RGB triplet to integer RGB
        triplet.

        """
        test_pairs = (
            ((u'100%', u'100%', u'0%'), (255, 255, 0)),
            ((u'0%', u'0%', u'50%'), (0, 0, 128)),
            ((u'85.49%', u'64.71%', u'12.5%'), (218, 165, 32))
        )

        for triplet, int_triplet in test_pairs:
            result = webcolors.rgb_percent_to_rgb(triplet)
            assert isinstance(result, webcolors.IntegerRGB)
            assert int_triplet == result

class ConversionTests(unittest.TestCase):
    """
    Test other aspects of convevrsion not covered by format-specific
    test cases.

    """
    def test_spelling_variant(self):
        """
        When asked to name a color value that maps to either of 'gray' or
        'grey' in CSS3, webcolors always picks 'gray' as the spelling.

        """
        for converter, value in (
                (webcolors.hex_to_name, u'#808080'),
                (webcolors.rgb_to_name, (128, 128, 128)),
                (webcolors.rgb_percent_to_name, (u'50%', u'50%', u'50%'))
        ):
            assert u"gray" == converter(value, spec=webcolors.CSS3)
<commit_msg>And use the spec constants in tests.<commit_after>"""
Test the color-format conversion utilities.

"""
import unittest

import webcolors

class HexConversionTests(unittest.TestCase):
    """
    Test the functions which convert from hex color codes to other
    formats.

    """
    def test_hex_to_name(self):
        """
        Test conversion from hex to color name.
        """
        test_pairs = (
            (u'#ffffff', u'white'),
            (u'#fff', u'white'),
            (u'#000080', u'navy'),
            (u'#daa520', u'goldenrod')
        )

        for hex_value, name in test_pairs:
            assert name == webcolors.hex_to_name(hex_value)

    def test_hex_to_name_unnamed(self):
        """
        A hex code which does not correspond to a named color, or does
        not correspond to a named color in the given specification,
        raises ValueError.

        """
        # No name in any spec.
        self.assertRaises(
            ValueError,
            webcolors.hex_to_name,
            '#123456'
        )

        # This is 'goldenrod' in CSS 3 list, unnamed in HTML 4.
        self.assertRaises(
            ValueError,
            webcolors.hex_to_name,
            '#daa520',
            spec=webcolors.HTML4
        )

    def test_hex_to_name_specs(self):
        """
        Using one of the supported specifications succeeds; using an
        unsupported specification raises ValueError.

        """
        for supported_spec in webcolors.SUPPORTED_SPECIFICATIONS:
            result = webcolors.hex_to_name(u'#ffffff', spec=supported_spec)
            assert u'white' == result

        for unsupported_spec in (u'css1', u'css4', u'html5'):
            self.assertRaises(
                ValueError,
                webcolors.hex_to_name,
                '#ffffff',
                spec=unsupported_spec
            )

    def test_hex_to_rgb(self):
        """
        Test conversion from hex to integer RGB triplet.

        """
        test_pairs = (
            (u'#fff', (255, 255, 255)),
            (u'#ffffff', (255, 255, 255)),
            (u'#000080', (0, 0, 128))
        )

        for hex_value, triplet in test_pairs:
            result = webcolors.hex_to_rgb(hex_value)
            assert isinstance(result, webcolors.IntegerRGB)
            assert triplet == result

    def test_hex_to_rgb_percent(self):
        """
        Test conversion from hex to percent RGB triplet.

        """
        test_pairs = (
            (u'#fff', (u'100%', u'100%', u'100%')),
            (u'#ffffff', (u'100%', u'100%', u'100%')),
            (u'#000080', (u'0%', u'0%', u'50%'))
        )

        for hex_value, triplet in test_pairs:
            result = webcolors.hex_to_rgb_percent(hex_value)
            assert isinstance(result, webcolors.PercentRGB)
            assert triplet == result

class IntegerRGBConversionTests(unittest.TestCase):
    """
    Test the functions which convert from integer RGB triplets to
    other formats.

    """
    def test_rgb_to_name(self):
        """
        Test conversion from integer RGB triplet to color name.

        """
        test_pairs = (
            ((255, 255, 255), u'white'),
            ((0, 0, 128), u'navy'),
            ((218, 165, 32), u'goldenrod'),
            (webcolors.IntegerRGB(218, 165, 32), u'goldenrod')
        )

        for triplet, name in test_pairs:
            assert name == webcolors.rgb_to_name(triplet)

    def test_rgb_to_name_unnamed(self):
        """
        An integer RGB triplet which does not correspond to a named
        color, or does not correspond to a named color in the given
        specification, raises ValueError.

        """
        # No name in any spec.
        self.assertRaises(
            ValueError,
            webcolors.rgb_to_name,
            (18, 52, 86)
        )

        # This is 'goldenrod' in CSS 3 list, unnamed in HTML 4.
        self.assertRaises(
            ValueError,
            webcolors.rgb_to_name,
            (218, 165, 32),
            spec=webcolors.HTML4
        )

    def test_rgb_to_name_specs(self):
        """
        Using one of the supported specifications succeeds; an
        unsupported specification raises ValueError.

        """
        for supported_spec in webcolors.SUPPORTED_SPECIFICATIONS:
            result = webcolors.rgb_to_name(
                (255, 255, 255), spec=supported_spec
            )
            assert u'white' == result

        for unsupported_spec in (u'css1', u'css4', u'html5'):
            self.assertRaises(
                ValueError,
                webcolors.rgb_to_name,
                (255, 255, 255),
                spec=unsupported_spec
            )

    def test_rgb_to_hex(self):
        """
        Test conversion from integer RGB triplet to hex.

        """
        test_pairs = (
            ((255, 255, 255), u'#ffffff'),
            ((0, 0, 128), u'#000080'),
            ((218, 165, 32), u'#daa520')
        )

        for triplet, hex_value in test_pairs:
            assert hex_value == webcolors.rgb_to_hex(triplet)

    def test_rgb_to_rgb_percent(self):
        """
        Test conversion from integer RGB triplet to percent RGB
        triplet.

        """
        test_pairs = (
            ((255, 255, 255), (u'100%', u'100%', u'100%')),
            ((0, 0, 128), (u'0%', u'0%', u'50%')),
            ((218, 165, 32), (u'85.49%', u'64.71%', u'12.5%'))
        )

        for triplet, percent_triplet in test_pairs:
            result = webcolors.rgb_to_rgb_percent(triplet)
            assert isinstance(result, webcolors.PercentRGB)
            assert percent_triplet == result

class NameConversionTests(unittest.TestCase):
    """
    Test the functions which convert from color names to other
    formats.

    """
    def test_name_to_hex(self):
        """
        Test correct conversion of color names to hex.
        """
        test_pairs = (
            (u'white', u'#ffffff'),
            (u'navy', u'#000080'),
            (u'goldenrod', u'#daa520')
        )

        for name, hex_value in test_pairs:
            assert hex_value == webcolors.name_to_hex(name)

    def test_name_to_hex_bad_name(self):
        """
        A name which does not correspond to a color, or does not
        correspond to a color in the given specification, raises
        ValueError.

        """
        test_values = (
            {u'name': u'goldenrod',
             u'spec': u'html4'},
            {u'name': u'glue',
             u'spec': u'css21'},
            {u'name': u'breen',
             u'spec': u'css3'},
        )

        for kwarg_dict in test_values:
            self.assertRaises(
                ValueError,
                webcolors.name_to_hex,
                **kwarg_dict
            )

    def test_name_to_hex_specs(self):
        """
        Using one of the supported specifications succeeds; using an
        unsupported specification raises ValueError.

        """
        for supported_spec in webcolors.SUPPORTED_SPECIFICATIONS:
            result = webcolors.name_to_hex(u'white', spec=supported_spec)
            assert u'#ffffff' == result

        for unsupported_spec in (u'css1', u'css4', u'html5'):
            self.assertRaises(
                ValueError,
                webcolors.name_to_hex,
                'white', spec=unsupported_spec
            )

    def test_name_to_rgb(self):
        """
        Test conversion from color name to integer RGB triplet.

        """
        test_pairs = (
            (u'white', (255, 255, 255)),
            (u'navy', (0, 0, 128)),
            (u'goldenrod', (218, 165, 32))
        )

        for name, triplet in test_pairs:
            result = webcolors.name_to_rgb(name)
            assert isinstance(result, webcolors.IntegerRGB)
            assert triplet == result

    def test_name_to_rgb_percent(self):
        """
        Test conversion from color name to percent RGB triplet.

        """
        test_pairs = (
            (u'white', (u'100%', u'100%', u'100%')),
            (u'navy', (u'0%', u'0%', u'50%')),
            (u'goldenrod', (u'85.49%', u'64.71%', u'12.5%'))
        )

        for name, triplet in test_pairs:
            result = webcolors.name_to_rgb_percent(name)
            assert isinstance(result, webcolors.PercentRGB)
            assert triplet == result

class PercentRGBConversionTests(unittest.TestCase):
    """
    Test the functions which convert from percent RGB triplets to
    other formats.

    """
    def test_rgb_percent_to_name(self):
        """
        Test conversion from percent RGB triplet to color name.
        """
        test_pairs = (
            ((u'100%', u'100%', u'100%'), u'white'),
            ((u'0%', u'0%', u'50%'), u'navy'),
            ((u'85.49%', u'64.71%', u'12.5%'), u'goldenrod'),
            (webcolors.PercentRGB(u'85.49%', u'64.71%', u'12.5%'),
             u'goldenrod')
        )

        for triplet, name in test_pairs:
            assert name == webcolors.rgb_percent_to_name(triplet)

    def test_rgb_percent_to_name_unnamed(self):
        """
        A percent RGB triplet which does not correspond to a named
        color, or does not correspond to a named color in the given
        specification, raises ValueError.

        """
        # No name in any spec.
        self.assertRaises(
            ValueError,
            webcolors.rgb_percent_to_name,
            (u'7.06%', u'20.39%', u'33.73%')
        )

        # This is 'goldenrod' in CSS 3 list, unnamed in HTML 4.
        self.assertRaises(
            ValueError,
            webcolors.rgb_percent_to_name,
            (u'85.49%', u'64.71%', u'12.5%'),
            spec=webcolors.HTML4
        )

    def test_rgb_percent_to_name_specs(self):
        """
        Using one of the supported specifications succeeds; an
        unsupported specification raises ValueError.

        """
        for supported_spec in (u'html4', u'css2', u'css21', u'css3'):
            result = webcolors.rgb_percent_to_name(
                (u'100%', u'100%', u'100%'), spec=supported_spec
            )
            assert u'white' == result

        for unsupported_spec in (u'css1', u'css4', u'html5'):
            self.assertRaises(
                ValueError,
                webcolors.rgb_percent_to_name,
                (u'100%', u'100%', u'100%'),
                spec=unsupported_spec
            )

    def test_rgb_percent_to_hex(self):
        """
        Test conversion from percent RGB triplet to hex.

        """
        test_pairs = (
            ((u'100%', u'100%', u'0%'), u'#ffff00'),
            ((u'0%', u'0%', u'50%'), u'#000080'),
            ((u'85.49%', u'64.71%', u'12.5%'), u'#daa520')
        )

        for triplet, hex_value in test_pairs:
            assert hex_value == webcolors.rgb_percent_to_hex(triplet)

    def test_rgb_percent_to_rgb(self):
        """
        Test conversion from percent RGB triplet to integer RGB
        triplet.

        """
        test_pairs = (
            ((u'100%', u'100%', u'0%'), (255, 255, 0)),
            ((u'0%', u'0%', u'50%'), (0, 0, 128)),
            ((u'85.49%', u'64.71%', u'12.5%'), (218, 165, 32))
        )

        for triplet, int_triplet in test_pairs:
            result = webcolors.rgb_percent_to_rgb(triplet)
            assert isinstance(result, webcolors.IntegerRGB)
            assert int_triplet == result

class ConversionTests(unittest.TestCase):
    """
    Test other aspects of convevrsion not covered by format-specific
    test cases.

    """
    def test_spelling_variant(self):
        """
        When asked to name a color value that maps to either of 'gray' or
        'grey' in CSS3, webcolors always picks 'gray' as the spelling.

        """
        for converter, value in (
                (webcolors.hex_to_name, u'#808080'),
                (webcolors.rgb_to_name, (128, 128, 128)),
                (webcolors.rgb_percent_to_name, (u'50%', u'50%', u'50%'))
        ):
            assert u"gray" == converter(value, spec=webcolors.CSS3)
<|endoftext|>


<commit_before># Manifest of files to include in the simple HTTP server.
# Run this script to generate the httpdocs.cpp source file.

manifest = [
    ('/', 'index.html', 'text/html'),

    # 404 error document must be last.
    (None, '404.html', 'text/html'),
]

print """/*
 * HTTP Document data.
 * Automatically generated by manifest.py
 */

#include "netserver.h"

NetServer::HTTPDocument NetServer::httpDocumentList[] = {"""

import json

def quote(str):
    if str is None:
        return 'NULL'
    return json.dumps(str)

for path, filename, contentType in manifest:
    if contentType.startswith('text/'):
        mode = 'r'
    else:
        mode = 'rb'
    data = open(filename, mode).read()
    print "{ %s, %s, %s }," % (quote(path), quote(data), quote(contentType))

print "};"
<commit_msg>Python 3 compatibility<commit_after># Manifest of files to include in the simple HTTP server.
# Run this script to generate the httpdocs.cpp source file.

manifest = [
    ('/', 'index.html', 'text/html'),

    # 404 error document must be last.
    (None, '404.html', 'text/html'),
]

import json, sys

sys.stdout.write("""/*
 * HTTP Document data.
 * Automatically generated by manifest.py
 */

#include "netserver.h"

NetServer::HTTPDocument NetServer::httpDocumentList[] = {
""")

def quote(str):
    if str is None:
        return 'NULL'
    return json.dumps(str)

for path, filename, contentType in manifest:
    if contentType.startswith('text/'):
        mode = 'r'
    else:
        mode = 'rb'
    data = open(filename, mode).read()
    sys.stdout.write("{ %s, %s, %s },\n" % (quote(path), quote(data), quote(contentType)))

sys.stdout.write("};\n")<|endoftext|>


<commit_before>from os.path import *
from os import makedirs
from math import ceil

def cutter(filepath, over, lastprop, folder, size=0, number=0):
	overlap = int(over)

	chunkarraynames = []

	with open(filepath, 'r') as edit:
		text = edit.read().decode('utf-8')
		splittext = text.split()

		if number:
			chunksize = int(ceil(len(splittext)/float(number)))
			print chunksize
			lastprop = 0

		else:
			chunksize = int(size)

		chunkarray = [splittext[i:i+chunksize] for i in xrange(0, len(splittext), chunksize-overlap)]
		# chunkarraynames = [str(index) + "_" + str(i) + '-' + str(i+chunksize) for index, i in enumerate(range(0, len(splittext), chunksize-overlap))]
		chunkarraynames = [str(index+1) + "_" + str(i) for index, i in enumerate(range(0, len(splittext), chunksize-overlap))]
		print chunkarraynames

		lastsize = float(lastprop)/100.0 * chunksize

		if len(chunkarray) > 1 and len(chunkarray[-1]) < lastsize:
			last = chunkarray.pop()
			if overlap:
				chunkarray[-1] = chunkarray[-1][:-overlap] + last
			else:
				chunkarray[-1] = chunkarray[-1] + last
	try:
		makedirs(folder)
	except:
		pass

	chunkpreview = {}

	for index, chunk in enumerate(chunkarray):
		with open(folder + splitext(basename(filepath))[0] + str(index) + '.txt', 'a+') as chunkfile:
			chunkfile.write(' '.join(chunk).encode('utf-8'))
			chunkpreview[index] = ' '.join(chunk[:15])
			if len(chunk) > 15:
				chunkpreview[index] += u"\u2026"

	# generate_frequency(chunkarray, folder)
	transposed = generate_other(chunkarray, folder)
	# print chunkpreview, names, transposed
	dendrogram(transposed, chunkarraynames, folder)
	return chunkpreview

from collections import Counter, defaultdict
import csv

def generate_frequency(chunkarray, folder):
	chunkcounters = {}
	allwords = set()
	for index, chunk in enumerate(chunkarray):
		chunkcounters[index] = Counter(chunk)
		allwords.update(chunkcounters[index].keys())
	frequencymatrix = []
	chunknames = sorted(chunkcounters.keys())
	for word in sorted(allwords):
		frequencymatrix.append([word])
		for chunk in chunknames:
			frequencymatrix[-1].append(chunkcounters[chunk][word])
	chunknames.insert(0, "")
	with open(folder + "frequency_matrix.csv", 'w') as out:
		csvFile = csv.writer(out, quoting=csv.QUOTE_NONE)
		csvFile.writerow(chunknames)
		for line in frequencymatrix:
			csvFile.writerow(line)

def generate_other(chunkarray, folder):
	chunkcounters = {}
	allwords = set()
	for index, chunk in enumerate(chunkarray):
		# print index, chunk
		chunkcounters[index] = Counter(chunk)
		allwords.update(chunkcounters[index].keys())
	masterDict = defaultdict(lambda: [0]*len(chunkcounters))
	for index, chunk in chunkcounters.items():
		# print index#, chunk
		total = float(sum(chunk.values()))
		for key, value in chunk.items():
			masterDict[key][index] = value/total
	# print masterDict
	# transposed = zip(*sorted(masterDict.iterkeys(), key=lambda k: masterDict[k]))
	# print masterDict
	transposed = zip(*masterDict.values())
	# print masterDict.keys()
	# print transposed
	# with open(folder + "frequency_matrix.csv", 'w') as out:
	# 	csvFile = csv.writer(out, quoting=csv.QUOTE_NONE)
	# 	csvFile.writerow([""] + list(masterDict.keys()))
	# 	for index, line in enumerate(transposed):
	# 		csvFile.writerow([chunkcounters.keys()[index]] + list(line))
	# return masterDict.keys(), transposed
	return transposed

from scipy.cluster import hierarchy
from scipy.spatial.distance import pdist
from matplotlib import pyplot

def dendrogram(transposed, names, folder):
	Y = pdist(transposed)
	Z = hierarchy.linkage(Y, method='average', metric='euclidean')
	hierarchy.dendrogram(Z, labels=names, leaf_rotation=0, orientation='right')
	with open(folder + 'dendrogram.png', 'w') as denimg:
		pyplot.savefig(denimg, format='png')
<commit_msg>Fixed dendros<commit_after>from os.path import *
from os import makedirs
from math import ceil

def cutter(filepath, over, lastprop, folder, size=0, number=0):
	overlap = int(over)

	chunkarraynames = []

	with open(filepath, 'r') as edit:
		text = edit.read().decode('utf-8')
		splittext = text.split()

		if number:
			chunksize = int(ceil(len(splittext)/float(number)))
			print chunksize
			lastprop = 0

		else:
			chunksize = int(size)

		chunkarray = [splittext[i:i+chunksize] for i in xrange(0, len(splittext), chunksize-overlap)]
		# chunkarraynames = [str(index) + "_" + str(i) + '-' + str(i+chunksize) for index, i in enumerate(range(0, len(splittext), chunksize-overlap))]
		chunkarraynames = [str(index+1) + "_" + str(i+chunksize) for index, i in enumerate(range(0, len(splittext), chunksize-overlap))]
		print chunkarraynames

		lastsize = float(lastprop)/100.0 * chunksize

		if len(chunkarray) > 1 and len(chunkarray[-1]) < lastsize:
			last = chunkarray.pop()
			if overlap:
				chunkarray[-1] = chunkarray[-1][:-overlap] + last
			else:
				chunkarray[-1] = chunkarray[-1] + last
	try:
		makedirs(folder)
	except:
		pass

	chunkpreview = {}

	for index, chunk in enumerate(chunkarray):
		with open(folder + splitext(basename(filepath))[0] + str(index) + '.txt', 'a+') as chunkfile:
			chunkfile.write(' '.join(chunk).encode('utf-8'))
			chunkpreview[index] = ' '.join(chunk[:15])
			if len(chunk) > 15:
				chunkpreview[index] += u"\u2026"

	# generate_frequency(chunkarray, folder)
	transposed = generate_other(chunkarray, folder)
	# print chunkpreview, names, transposed
	dendrogram(transposed, chunkarraynames, folder)
	return chunkpreview

from collections import Counter, defaultdict
import csv

def generate_frequency(chunkarray, folder):
	chunkcounters = {}
	allwords = set()
	for index, chunk in enumerate(chunkarray):
		chunkcounters[index] = Counter(chunk)
		allwords.update(chunkcounters[index].keys())
	frequencymatrix = []
	chunknames = sorted(chunkcounters.keys())
	for word in sorted(allwords):
		frequencymatrix.append([word])
		for chunk in chunknames:
			frequencymatrix[-1].append(chunkcounters[chunk][word])
	chunknames.insert(0, "")
	with open(folder + "frequency_matrix.csv", 'w') as out:
		csvFile = csv.writer(out, quoting=csv.QUOTE_NONE)
		csvFile.writerow(chunknames)
		for line in frequencymatrix:
			csvFile.writerow(line)

from collections import OrderedDict

def generate_other(chunkarray, folder):
	chunkcounters = {}
	allwords = set()
	for index, chunk in enumerate(chunkarray):
		chunkcounters[index] = Counter(chunk)
		allwords.update(chunkcounters[index].keys())
	masterDict = defaultdict(lambda: [0]*len(chunkcounters))
	for index, chunk in chunkcounters.items():
		total = float(sum(chunk.values()))
		for key, value in chunk.items():
			masterDict[key.encode('utf-8')][index] = value/total
	sortedDict = OrderedDict(sorted(masterDict.items(), key=lambda k: k[0]))
	transposed = zip(*sortedDict.values())
	with open(folder + "frequency_matrix.csv", 'wb') as out:
		csvFile = csv.writer(out, quoting=csv.QUOTE_NONE)
		csvFile.writerow([" "] + list(sortedDict.keys()))
		for index, line in enumerate(transposed):
			csvFile.writerow([chunkcounters.keys()[index]] + list(line))
	return transposed

from scipy.cluster import hierarchy
from scipy.spatial.distance import pdist
from matplotlib import pyplot

def dendrogram(transposed, names, folder):
	Y = pdist(transposed)
	Z = hierarchy.linkage(Y, method='average', metric='euclidean')
	hierarchy.dendrogram(Z, labels=names, leaf_rotation=0, orientation='right')
	with open(folder + 'dendrogram.png', 'w') as denimg:
		pyplot.savefig(denimg, format='png')
<|endoftext|>


<commit_before>#----------------------------------------------------------------------
# ssd1351.py from https://github.com/boxysean/py-gaugette
# ported by boxysean
# modified to work with BBB and Adafruit_BBIO and *not* gaugette by AKA
#
# This library works with
#   Adafruit's 128x128 SPI RGB OLED   https://www.adafruit.com/products/1431
#
# The code is based heavily on Adafruit's Arduino library
#   https://github.com/adafruit/Adafruit-SSD1351-library
# written by Limor Fried/Ladyada for Adafruit Industries.
#----------------------------------------------------------------------
from __future__ import division
import Adafruit_BBIO.GPIO as GPIO
import Adafruit_BBIO.SPI as SPI
import font5x8
import time
import sys

class SSD1351:

    DELAYS_HWFILL = 3
    DELAYS_HWLINE = 1

    # SSD1351 Commands
    CMD_SETCOLUMN          = 0x15
    CMD_SETROW             = 0x75
    CMD_WRITERAM           = 0x5C
    CMD_READRAM            = 0x5D
    CMD_SETREMAP           = 0xA0
    CMD_STARTLINE          = 0xA1
    CMD_DISPLAYOFFSET      = 0xA2
    CMD_DISPLAYALLOFF      = 0xA4
    CMD_DISPLAYALLON       = 0xA5
    CMD_NORMALDISPLAY      = 0xA6
    CMD_INVERTDISPLAY      = 0xA7
    CMD_FUNCTIONSELECT     = 0xAB
    CMD_DISPLAYOFF         = 0xAE
    CMD_DISPLAYON          = 0xAF
    CMD_PRECHARGE          = 0xB1
    CMD_DISPLAYENHANCE     = 0xB2
    CMD_CLOCKDIV           = 0xB3
    CMD_SETVSL             = 0xB4
    CMD_SETGPIO            = 0xB5
    CMD_PRECHARGE2         = 0xB6
    CMD_SETGRAY            = 0xB8
    CMD_USELUT             = 0xB9
    CMD_PRECHARGELEVEL     = 0xBB
    CMD_VCOMH              = 0xBE
    CMD_CONTRASTABC        = 0xC1
    CMD_CONTRASTMASTER     = 0xC7
    CMD_MUXRATIO           = 0xCA
    CMD_COMMANDLOCK        = 0xFD
    CMD_HORIZSCROLL        = 0x96
    CMD_STOPSCROLL         = 0x9E
    CMD_STARTSCROLL        = 0x9F

    SSD1351WIDTH           = 128
    SSD1351HEIGHT          = 128

    # Device name will be /dev/spidev-{bus}.{device}
    # dc_pin is the data/commmand pin.  This line is HIGH for data, LOW for command.
    # We will keep d/c low and bump it high only for commands with data
    # reset is normally HIGH, and pulled LOW to reset the display

    def __init__(self, bus=0, device=0, dc_pin="P9_15", reset_pin="P9_13", buffer_rows=128, buffer_cols=128, rows=128, cols=128):
        self.cols = cols
        self.rows = rows
        self.buffer_rows = buffer_rows
        self.mem_bytes = self.buffer_rows * self.cols / 8 # total bytes in SSD1306 display ram
        self.dc_pin = dc_pin
        self.reset_pin = reset_pin
        self.spi = SPI.SPI(bus, device)
        self.spi.mode = 3 # necessary!
        self.gpio = GPIO
        self.gpio.setup(self.reset_pin, self.gpio.OUT)
        self.gpio.output(self.reset_pin, self.gpio.HIGH)
        self.gpio.setup(self.dc_pin, self.gpio.OUT)
        self.gpio.output(self.dc_pin, self.gpio.LOW)
        self.font = font5x8.Font5x8
        self.col_offset = 0
        self.bitmap = self.SimpleBitmap(buffer_cols, buffer_rows)
        self.flipped = False

    def reset(self):
        self.gpio.output(self.reset_pin, self.gpio.LOW)
        time.sleep(0.010) # 10ms
        self.gpio.output(self.reset_pin, self.gpio.HIGH)

    def command(self, cmd, cmddata=None):
        # already low
        #self.gpio.output(self.dc_pin, self.gpio.LOW)

        if type(cmd) == list:
            self.spi.writebytes(cmd)
        else:
            self.spi.writebytes([cmd])

        if cmddata != None:
            if type(cmddata) == list:
                self.data(cmddata)
            else:
                self.data([cmddata])

    def data(self, bytes):
        self.gpio.output(self.dc_pin, self.gpio.HIGH)
        #  chunk data to work around 255 byte limitation in adafruit implementation of writebytes
        # revisit - change to 1024 when Adafruit_BBIO is fixed.
        # max_xfer = 255 if gaugette.platform == 'beaglebone' else 1024
        # max_xfer = 1024 # whatever BBIO error existed seems fine now
        max_xfer = 255
        start = 0
        remaining = len(bytes)
        while remaining>0:
            count = remaining if remaining <= max_xfer else max_xfer
            remaining -= count
            self.spi.writebytes(bytes[start:start+count])
            start += count
        self.gpio.output(self.dc_pin, self.gpio.LOW)

    def begin(self):
        time.sleep(0.001) # 1ms
        self.reset()

        self.command(self.CMD_COMMANDLOCK, 0x12) # Unlock OLED driver IC MCU interface from entering command
        self.command(self.CMD_COMMANDLOCK, 0xB1) # Command A2,B1,B3,BB,BE,C1 accessible if in unlock state
        self.command(self.CMD_DISPLAYOFF)
        self.command([self.CMD_CLOCKDIV, 0xF1]) # 7:4 = Oscillator Frequency, 3:0 = CLK Div Ratio (A[3:0]+1 = 1..16)
        self.command(self.CMD_MUXRATIO, 127)
        self.command(self.CMD_SETREMAP, 0x74)
        self.command(self.CMD_SETCOLUMN, [0x00, 0x7F])
        self.command(self.CMD_SETROW, [0x00, 0x7F])
        self.command(self.CMD_STARTLINE, 0x00)
        self.command(self.CMD_DISPLAYOFFSET, 0x00)
        self.command(self.CMD_SETGPIO, 0x00)
        self.command(self.CMD_FUNCTIONSELECT, 0x01)
        self.command([self.CMD_PRECHARGE, 0x32])
        self.command([self.CMD_VCOMH, 0x05])
        self.command(self.CMD_NORMALDISPLAY)
        self.command(self.CMD_CONTRASTABC, [0xC8, 0x80, 0xC8])
        self.command(self.CMD_CONTRASTMASTER, 0x0F)
        self.command(self.CMD_SETVSL, [0xA0, 0xB5, 0x55])
        self.command(self.CMD_PRECHARGE2, 0x01)
        self.command(self.CMD_DISPLAYON)

    def clear_display(self):
        self.bitmap.clear()

    def invert_display(self):
        self.command(self.CMD_INVERTDISPLAY)

    def flip_display(self, flipped=True):
        self.flipped = flipped
        if flipped:
            self.command(self.COM_SCAN_INC)
            self.command(self.SEG_REMAP | 0x00)
        else:
            self.command(self.COM_SCAN_DEC)
            self.command(self.SET_COM_PINS, 0x02)

    def normal_display(self):
        self.command(self.CMD_NORMALDISPLAY)

    def set_contrast(self, contrast=0x7f):
        self.command(self.SET_CONTRAST, contrast)

    def goTo(self, x, y):
        if x >= self.SSD1351WIDTH or y >= self.SSD1351HEIGHT:
            return

        # set x and y coordinate
        self.command(self.CMD_SETCOLUMN, [x, self.SSD1351WIDTH-1])
        self.command(self.CMD_SETROW, [y, self.SSD1351HEIGHT-1])
        self.command(self.CMD_WRITERAM)

    def scale(self, x, inLow, inHigh, outLow, outHigh):
        return int(((x - inLow) / float(inHigh) * outHigh) + outLow)

    def encode_color(self, color):
        red = (color >> 16) & 0xFF
        green = (color >> 8) & 0xFF
        blue = color & 0xFF
        print 'r g b are ', red, green, blue

        redScaled = int(self.scale(red, 0, 0xFF, 0, 0x1F))
        greenScaled = int(self.scale(green, 0, 0xFF, 0, 0x3E))
        blueScaled = int(self.scale(blue, 0, 0xFF, 0, 0x1F))

        print color, redScaled, greenScaled, blueScaled

        return (((redScaled << 6) | greenScaled) << 5) | blueScaled

    def color565(self, r, g, b): # ints
        c = r >> 3
        c <<= 6
        c |= g >> 2
        c <<= 5
        c |= b >> 3
        return c

    def fillScreen(self, fillcolor): # int
        self.fillRect(0, 0, self.SSD1351WIDTH, self.SSD1351HEIGHT, fillcolor)

    def fillRect(self, x, y, w, h, fillcolor):
        # Bounds check
        if x >= self.SSD1351WIDTH or y >= self.SSD1351HEIGHT:
            return

        if y+h > self.SSD1351HEIGHT:
            h = self.SSD1351HEIGHT - y - 1

        if x+w > self.SSD1351WIDTH:
            w = self.SSD1351WIDTH - x - 1

        # set location
        self.command(self.CMD_SETCOLUMN, [x, x+w-1])
        self.command(self.CMD_SETROW, [y, y-h-1])
        # fill!
        self.command(self.CMD_WRITERAM)

        fillcolor = self.encode_color(fillcolor)

        self.data([fillcolor >> 8, fillcolor] * (w*h))

    def drawPixel(self, x, y, color):
        if x >= self.SSD1351WIDTH or y >= self.SSD1351HEIGHT:
            return

        if x < 0 or y < 0:
            return

        color = self.encode_color(color)

        # set location
        self.goTo(x, y)
        self.data([color >> 8, color])

    def drawBitmap(self, x, y, bitmap):
        h = len(bitmap)
        w = len(bitmap[0])

        self.command(self.CMD_SETCOLUMN, [x, w])
        self.command(self.CMD_SETROW, [y, h])
        self.command(self.CMD_WRITERAM)

        pixels = []

        for r in range(y, y+h):
            if len(pixels) + 4*w >= 1024:
                print "pixels!", pixels
                self.data(pixels)
                pixels = []

            for x in bitmap[r]:
                pixels = pixels + [(x >> 8) & 0xFF, x & 0xFF]

        print "pixels!", pixels
        self.data(pixels)

    # Diagnostic print of the memory buffer to stdout
    def dump_buffer(self):
        self.bitmap.dump()

    def draw_text(self, x, y, string, color=0xFFFFFF):
        # print 'text is %s' % string
        font_bytes = self.font.bytes
        font_rows = self.font.rows
        font_cols = self.font.cols

        for c in string:
            p = ord(c) * font_cols
            for col in range(font_cols):
                mask = font_bytes[p]
                p += 1
                for row in range(8):
                    if (mask & 1) != 0:
                        self.drawPixel(x, y+row, self.encode_color(color))
                        # self.bitmap.draw_pixel(x, y+row, self.encode_color(color))
                    else:
                        self.drawPixel(x, y+row, 0)
                        # self.bitmap.draw_pixel(x, y+row, 0)
                    mask >>= 1
                x += 1

    def draw_text2(self, x, y, string, color=0xFFFFFF, size=2, space=1):
        font_bytes = self.font.bytes
        font_rows = self.font.rows
        font_cols = self.font.cols
        for c in string:
            p = ord(c) * font_cols
            for col in range(0,font_cols):
                mask = font_bytes[p]
                p+=1
                py = y
                for row in range(0,8):
                    for sy in range(0,size):
                        px = x
                        for sx in range(0,size):
                            if mask & 1:
                                self.bitmap.draw_pixel(px, py, self.encode_color(color))
                            else:
                                self.bitmap.draw_pixel(px, py, 0)
                            px += 1
                        py += 1
                    mask >>= 1
                x += size
            x += space

    def clear_block(self, x0,y0,dx,dy):
        self.bitmap.clear_block(x0,y0,dx,dy)

    def draw_text3(self, x, y, string, font):
        return self.bitmap.draw_text(x,y,string,font)

    def text_width(self, string, font):
        return self.bitmap.text_width(string, font)

    class SimpleBitmap:
        def __init__(self, cols, rows):
            self.rows = rows
            self.cols = cols
            print rows, cols
            self.data = [([0] * self.cols) for i in range(self.rows)]

        def clear(self):
            for r in range(len(self.data)):
                for c in range(len(self.data[r])):
                    self.data[r][c] = 0

        # Diagnostic print of the memory buffer to stdout
        def dump(self):
            for row in self.data:
                for col in row:
                    sys.stdout.write('X' if col else '.')
                sys.stdout.write('\n')

        def draw_pixel(self, x, y, color):
            if (x<0 or x>=self.cols or y<0 or y>=self.rows):
                return

            self.data[y][x] = color

        def clear_block(self, x0,y0,dx,dy):
            for x in range(x0,x0+dx):
                for y in range(y0,y0+dy):
                    self.draw_pixel(x,y,0)

        def display(self, ssd1351):
            ssd1351.command(ssd1351.CMD_SETCOLUMN, [0, ssd1351.SSD1351WIDTH])
            ssd1351.command(ssd1351.CMD_SETROW, [0, ssd1351.SSD1351HEIGHT])
            ssd1351.command(ssd1351.CMD_WRITERAM)

            pixels = []

            ## something is wrong in here... not sure what!

            for r in range(ssd1351.SSD1351WIDTH):
                if len(pixels) + ssd1351.SSD1351HEIGHT >= 513: # dump it out!
                    print "pixels!", pixels
                    ssd1351.data(pixels)
                    pixels = []

                pixels = pixels + self.data[r]

            print "pixels!", pixels
            ssd1351.data(pixels)<commit_msg>color testing<commit_after>#----------------------------------------------------------------------
# ssd1351.py from https://github.com/boxysean/py-gaugette
# ported by boxysean
# modified to work with BBB and Adafruit_BBIO and *not* gaugette by AKA
#
# This library works with
#   Adafruit's 128x128 SPI RGB OLED   https://www.adafruit.com/products/1431
#
# The code is based heavily on Adafruit's Arduino library
#   https://github.com/adafruit/Adafruit-SSD1351-library
# written by Limor Fried/Ladyada for Adafruit Industries.
#----------------------------------------------------------------------
from __future__ import division
import Adafruit_BBIO.GPIO as GPIO
import Adafruit_BBIO.SPI as SPI
import font5x8
import time
import sys

class SSD1351:

    DELAYS_HWFILL = 3
    DELAYS_HWLINE = 1

    # SSD1351 Commands
    CMD_SETCOLUMN          = 0x15
    CMD_SETROW             = 0x75
    CMD_WRITERAM           = 0x5C
    CMD_READRAM            = 0x5D
    CMD_SETREMAP           = 0xA0
    CMD_STARTLINE          = 0xA1
    CMD_DISPLAYOFFSET      = 0xA2
    CMD_DISPLAYALLOFF      = 0xA4
    CMD_DISPLAYALLON       = 0xA5
    CMD_NORMALDISPLAY      = 0xA6
    CMD_INVERTDISPLAY      = 0xA7
    CMD_FUNCTIONSELECT     = 0xAB
    CMD_DISPLAYOFF         = 0xAE
    CMD_DISPLAYON          = 0xAF
    CMD_PRECHARGE          = 0xB1
    CMD_DISPLAYENHANCE     = 0xB2
    CMD_CLOCKDIV           = 0xB3
    CMD_SETVSL             = 0xB4
    CMD_SETGPIO            = 0xB5
    CMD_PRECHARGE2         = 0xB6
    CMD_SETGRAY            = 0xB8
    CMD_USELUT             = 0xB9
    CMD_PRECHARGELEVEL     = 0xBB
    CMD_VCOMH              = 0xBE
    CMD_CONTRASTABC        = 0xC1
    CMD_CONTRASTMASTER     = 0xC7
    CMD_MUXRATIO           = 0xCA
    CMD_COMMANDLOCK        = 0xFD
    CMD_HORIZSCROLL        = 0x96
    CMD_STOPSCROLL         = 0x9E
    CMD_STARTSCROLL        = 0x9F

    SSD1351WIDTH           = 128
    SSD1351HEIGHT          = 128

    # Device name will be /dev/spidev-{bus}.{device}
    # dc_pin is the data/commmand pin.  This line is HIGH for data, LOW for command.
    # We will keep d/c low and bump it high only for commands with data
    # reset is normally HIGH, and pulled LOW to reset the display

    def __init__(self, bus=0, device=0, dc_pin="P9_15", reset_pin="P9_13", buffer_rows=128, buffer_cols=128, rows=128, cols=128):
        self.cols = cols
        self.rows = rows
        self.buffer_rows = buffer_rows
        self.mem_bytes = self.buffer_rows * self.cols / 8 # total bytes in SSD1306 display ram
        self.dc_pin = dc_pin
        self.reset_pin = reset_pin
        self.spi = SPI.SPI(bus, device)
        self.spi.mode = 3 # necessary!
        self.gpio = GPIO
        self.gpio.setup(self.reset_pin, self.gpio.OUT)
        self.gpio.output(self.reset_pin, self.gpio.HIGH)
        self.gpio.setup(self.dc_pin, self.gpio.OUT)
        self.gpio.output(self.dc_pin, self.gpio.LOW)
        self.font = font5x8.Font5x8
        self.col_offset = 0
        self.bitmap = self.SimpleBitmap(buffer_cols, buffer_rows)
        self.flipped = False

    def reset(self):
        self.gpio.output(self.reset_pin, self.gpio.LOW)
        time.sleep(0.010) # 10ms
        self.gpio.output(self.reset_pin, self.gpio.HIGH)

    def command(self, cmd, cmddata=None):
        # already low
        #self.gpio.output(self.dc_pin, self.gpio.LOW)

        if type(cmd) == list:
            self.spi.writebytes(cmd)
        else:
            self.spi.writebytes([cmd])

        if cmddata != None:
            if type(cmddata) == list:
                self.data(cmddata)
            else:
                self.data([cmddata])

    def data(self, bytes):
        self.gpio.output(self.dc_pin, self.gpio.HIGH)
        #  chunk data to work around 255 byte limitation in adafruit implementation of writebytes
        # revisit - change to 1024 when Adafruit_BBIO is fixed.
        # max_xfer = 255 if gaugette.platform == 'beaglebone' else 1024
        # max_xfer = 1024 # whatever BBIO error existed seems fine now
        max_xfer = 255
        start = 0
        remaining = len(bytes)
        while remaining>0:
            count = remaining if remaining <= max_xfer else max_xfer
            remaining -= count
            self.spi.writebytes(bytes[start:start+count])
            start += count
        self.gpio.output(self.dc_pin, self.gpio.LOW)

    def begin(self):
        time.sleep(0.001) # 1ms
        self.reset()

        self.command(self.CMD_COMMANDLOCK, 0x12) # Unlock OLED driver IC MCU interface from entering command
        self.command(self.CMD_COMMANDLOCK, 0xB1) # Command A2,B1,B3,BB,BE,C1 accessible if in unlock state
        self.command(self.CMD_DISPLAYOFF)
        self.command([self.CMD_CLOCKDIV, 0xF1]) # 7:4 = Oscillator Frequency, 3:0 = CLK Div Ratio (A[3:0]+1 = 1..16)
        self.command(self.CMD_MUXRATIO, 127)
        self.command(self.CMD_SETREMAP, 0x74)
        self.command(self.CMD_SETCOLUMN, [0x00, 0x7F])
        self.command(self.CMD_SETROW, [0x00, 0x7F])
        self.command(self.CMD_STARTLINE, 0x00)
        self.command(self.CMD_DISPLAYOFFSET, 0x00)
        self.command(self.CMD_SETGPIO, 0x00)
        self.command(self.CMD_FUNCTIONSELECT, 0x01)
        self.command([self.CMD_PRECHARGE, 0x32])
        self.command([self.CMD_VCOMH, 0x05])
        self.command(self.CMD_NORMALDISPLAY)
        self.command(self.CMD_CONTRASTABC, [0xC8, 0x80, 0xC8])
        self.command(self.CMD_CONTRASTMASTER, 0x0F)
        self.command(self.CMD_SETVSL, [0xA0, 0xB5, 0x55])
        self.command(self.CMD_PRECHARGE2, 0x01)
        self.command(self.CMD_DISPLAYON)

    def clear_display(self):
        self.bitmap.clear()

    def invert_display(self):
        self.command(self.CMD_INVERTDISPLAY)

    def flip_display(self, flipped=True):
        self.flipped = flipped
        if flipped:
            self.command(self.COM_SCAN_INC)
            self.command(self.SEG_REMAP | 0x00)
        else:
            self.command(self.COM_SCAN_DEC)
            self.command(self.SET_COM_PINS, 0x02)

    def normal_display(self):
        self.command(self.CMD_NORMALDISPLAY)

    def set_contrast(self, contrast=0x7f):
        self.command(self.SET_CONTRAST, contrast)

    def goTo(self, x, y):
        if x >= self.SSD1351WIDTH or y >= self.SSD1351HEIGHT:
            return

        # set x and y coordinate
        self.command(self.CMD_SETCOLUMN, [x, self.SSD1351WIDTH-1])
        self.command(self.CMD_SETROW, [y, self.SSD1351HEIGHT-1])
        self.command(self.CMD_WRITERAM)

    def scale(self, x, inLow, inHigh, outLow, outHigh):
        return int(((x - inLow) / float(inHigh) * outHigh) + outLow)

    def encode_color(self, color):
        red = (color >> 16) & 0xFF
        green = (color >> 8) & 0xFF
        blue = color & 0xFF
        print 'r g b are ', red, green, blue

        redScaled = int(self.scale(red, 0, 0xFF, 0, 0x1F))
        greenScaled = int(self.scale(green, 0, 0xFF, 0, 0x3F))
        blueScaled = int(self.scale(blue, 0, 0xFF, 0, 0x1F))

        print color, redScaled, greenScaled, blueScaled

        return (((redScaled << 6) | greenScaled) << 5) | blueScaled

    def color565(self, r, g, b): # ints
        c = r >> 3
        c <<= 6
        c |= g >> 2
        c <<= 5
        c |= b >> 3
        return c

    def fillScreen(self, fillcolor): # int
        self.fillRect(0, 0, self.SSD1351WIDTH, self.SSD1351HEIGHT, fillcolor)

    def fillRect(self, x, y, w, h, fillcolor):
        # Bounds check
        if x >= self.SSD1351WIDTH or y >= self.SSD1351HEIGHT:
            return

        if y+h > self.SSD1351HEIGHT:
            h = self.SSD1351HEIGHT - y - 1

        if x+w > self.SSD1351WIDTH:
            w = self.SSD1351WIDTH - x - 1

        # set location
        self.command(self.CMD_SETCOLUMN, [x, x+w-1])
        self.command(self.CMD_SETROW, [y, y-h-1])
        # fill!
        self.command(self.CMD_WRITERAM)

        fillcolor = self.encode_color(fillcolor)

        self.data([fillcolor >> 8, fillcolor] * (w*h))

    def drawPixel(self, x, y, color):
        if x >= self.SSD1351WIDTH or y >= self.SSD1351HEIGHT:
            return

        if x < 0 or y < 0:
            return

        color = self.encode_color(color)

        # set location
        self.goTo(x, y)
        self.data([color >> 8, color])

    def drawBitmap(self, x, y, bitmap):
        h = len(bitmap)
        w = len(bitmap[0])

        self.command(self.CMD_SETCOLUMN, [x, w])
        self.command(self.CMD_SETROW, [y, h])
        self.command(self.CMD_WRITERAM)

        pixels = []

        for r in range(y, y+h):
            if len(pixels) + 4*w >= 1024:
                print "pixels!", pixels
                self.data(pixels)
                pixels = []

            for x in bitmap[r]:
                pixels = pixels + [(x >> 8) & 0xFF, x & 0xFF]

        print "pixels!", pixels
        self.data(pixels)

    # Diagnostic print of the memory buffer to stdout
    def dump_buffer(self):
        self.bitmap.dump()

    def draw_text(self, x, y, string, color=0xFFFFFF):
        # print 'text is %s' % string
        font_bytes = self.font.bytes
        font_rows = self.font.rows
        font_cols = self.font.cols

        for c in string:
            p = ord(c) * font_cols
            for col in range(font_cols):
                mask = font_bytes[p]
                p += 1
                for row in range(8):
                    if (mask & 1) != 0:
                        self.drawPixel(x, y+row, self.encode_color(color))
                        # self.bitmap.draw_pixel(x, y+row, self.encode_color(color))
                    else:
                        self.drawPixel(x, y+row, 0)
                        # self.bitmap.draw_pixel(x, y+row, 0)
                    mask >>= 1
                x += 1

    def draw_text2(self, x, y, string, color=0xFFFFFF, size=2, space=1):
        font_bytes = self.font.bytes
        font_rows = self.font.rows
        font_cols = self.font.cols
        for c in string:
            p = ord(c) * font_cols
            for col in range(0,font_cols):
                mask = font_bytes[p]
                p+=1
                py = y
                for row in range(0,8):
                    for sy in range(0,size):
                        px = x
                        for sx in range(0,size):
                            if mask & 1:
                                self.bitmap.draw_pixel(px, py, self.encode_color(color))
                            else:
                                self.bitmap.draw_pixel(px, py, 0)
                            px += 1
                        py += 1
                    mask >>= 1
                x += size
            x += space

    def clear_block(self, x0,y0,dx,dy):
        self.bitmap.clear_block(x0,y0,dx,dy)

    def draw_text3(self, x, y, string, font):
        return self.bitmap.draw_text(x,y,string,font)

    def text_width(self, string, font):
        return self.bitmap.text_width(string, font)

    class SimpleBitmap:
        def __init__(self, cols, rows):
            self.rows = rows
            self.cols = cols
            print rows, cols
            self.data = [([0] * self.cols) for i in range(self.rows)]

        def clear(self):
            for r in range(len(self.data)):
                for c in range(len(self.data[r])):
                    self.data[r][c] = 0

        # Diagnostic print of the memory buffer to stdout
        def dump(self):
            for row in self.data:
                for col in row:
                    sys.stdout.write('X' if col else '.')
                sys.stdout.write('\n')

        def draw_pixel(self, x, y, color):
            if (x<0 or x>=self.cols or y<0 or y>=self.rows):
                return

            self.data[y][x] = color

        def clear_block(self, x0,y0,dx,dy):
            for x in range(x0,x0+dx):
                for y in range(y0,y0+dy):
                    self.draw_pixel(x,y,0)

        def display(self, ssd1351):
            ssd1351.command(ssd1351.CMD_SETCOLUMN, [0, ssd1351.SSD1351WIDTH])
            ssd1351.command(ssd1351.CMD_SETROW, [0, ssd1351.SSD1351HEIGHT])
            ssd1351.command(ssd1351.CMD_WRITERAM)

            pixels = []

            ## something is wrong in here... not sure what!

            for r in range(ssd1351.SSD1351WIDTH):
                if len(pixels) + ssd1351.SSD1351HEIGHT >= 513: # dump it out!
                    print "pixels!", pixels
                    ssd1351.data(pixels)
                    pixels = []

                pixels = pixels + self.data[r]

            print "pixels!", pixels
            ssd1351.data(pixels)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import string
import unittest
import random
import os
import platform
from pyperclip import _executable_exists, HAS_DISPLAY
from pyperclip.clipboards import (init_gtk_clipboard, init_xsel_clipboard, init_xclip_clipboard, init_klipper_clipboard,
                                  init_qt_clipboard, init_osx_clipboard, init_no_clipboard)
from pyperclip.windows import init_windows_clipboard

class _TestClipboard(unittest.TestCase):
    clipboard = None
    supports_unicode = True

    @property
    def copy(self):
        return self.clipboard[0]

    @property
    def paste(self):
        return self.clipboard[1]

    def setUp(self):
        if not self.clipboard:
            self.skipTest("Clipboard not supported.")

    def test_copy_simple(self):
        self.copy("pyper\r\nclip")

    def test_copy_paste_simple(self):
        msg = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(1000))
        self.copy(msg)
        self.assertEqual(self.paste(), msg)

    def test_copy_paste_whitespace(self):
        msg = ''.join(random.choice(string.whitespace) for _ in range(1000))
        self.copy(msg)
        self.assertEqual(self.paste(), msg)

    def test_copy_blank(self):
        self.copy('TEST')
        self.copy('')
        self.assertEqual(self.paste(), '')

    def test_copy_unicode(self):
        if not self.supports_unicode:
            raise unittest.SkipTest()
        self.copy(u"ಠ_ಠ")

    def test_copy_paste_unicode(self):
        if not self.supports_unicode:
            raise unittest.SkipTest()
        msg = u"ಠ_ಠ"
        self.copy(msg)
        self.assertEqual(self.paste(), msg)

class TestCygwin(_TestClipboard):
    if 'cygwin' in platform.system().lower():
        clipboard = init_windows_clipboard(True)

class TestWindows(_TestClipboard):
    if os.name == 'nt' or platform.system() == 'Windows':
        clipboard = init_windows_clipboard()

class TestOSX(_TestClipboard):
    if os.name == 'mac' or platform.system() == 'Darwin':
        clipboard = init_osx_clipboard()

class TestGtk(_TestClipboard):
    if not HAS_DISPLAY:
        return

    try:
        import gtk
    except ImportError:
        pass
    else:
        clipboard = init_gtk_clipboard()

class TestQt(_TestClipboard):
    if not HAS_DISPLAY:
        return

    try:
        import PyQt4
    except ImportError:
        pass
    else:
        clipboard = init_qt_clipboard()

class TestXClip(_TestClipboard):
    if _executable_exists("xclip"):
        clipboard = init_xclip_clipboard()

class TestXSel(_TestClipboard):
    if _executable_exists("xsel"):
        clipboard = init_xsel_clipboard()

class TestKlipper(_TestClipboard):
    if _executable_exists("klipper") and _executable_exists("qdbus"):
        clipboard = init_klipper_clipboard()

class TestNoClipboard(unittest.TestCase):
    copy, paste = init_no_clipboard()

    def test_copy(self):
        with self.assertRaises(RuntimeError):
            self.copy("foo")

    def test_paste(self):
        with self.assertRaises(RuntimeError):
            self.paste()

if __name__ == '__main__':
    unittest.main()
<commit_msg>Undoing change which introduced syntax errors<commit_after># -*- coding: utf-8 -*-
import string
import unittest
import random
import os
import platform
from pyperclip import _executable_exists, HAS_DISPLAY
from pyperclip.clipboards import (init_gtk_clipboard, init_xsel_clipboard, init_xclip_clipboard, init_klipper_clipboard,
                                  init_qt_clipboard, init_osx_clipboard, init_no_clipboard)
from pyperclip.windows import init_windows_clipboard

class _TestClipboard(unittest.TestCase):
    clipboard = None
    supports_unicode = True

    @property
    def copy(self):
        return self.clipboard[0]

    @property
    def paste(self):
        return self.clipboard[1]

    def setUp(self):
        if not self.clipboard:
            self.skipTest("Clipboard not supported.")

    def test_copy_simple(self):
        self.copy("pyper\r\nclip")

    def test_copy_paste_simple(self):
        msg = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(1000))
        self.copy(msg)
        self.assertEqual(self.paste(), msg)

    def test_copy_paste_whitespace(self):
        msg = ''.join(random.choice(string.whitespace) for _ in range(1000))
        self.copy(msg)
        self.assertEqual(self.paste(), msg)

    def test_copy_blank(self):
        self.copy('TEST')
        self.copy('')
        self.assertEqual(self.paste(), '')

    def test_copy_unicode(self):
        if not self.supports_unicode:
            raise unittest.SkipTest()
        self.copy(u"ಠ_ಠ")

    def test_copy_paste_unicode(self):
        if not self.supports_unicode:
            raise unittest.SkipTest()
        msg = u"ಠ_ಠ"
        self.copy(msg)
        self.assertEqual(self.paste(), msg)

class TestCygwin(_TestClipboard):
    if 'cygwin' in platform.system().lower():
        clipboard = init_windows_clipboard(True)

class TestWindows(_TestClipboard):
    if os.name == 'nt' or platform.system() == 'Windows':
        clipboard = init_windows_clipboard()

class TestOSX(_TestClipboard):
    if os.name == 'mac' or platform.system() == 'Darwin':
        clipboard = init_osx_clipboard()

class TestGtk(_TestClipboard):
    if HAS_DISPLAY:
        try:
            import gtk
        except ImportError:
            pass
        else:
            clipboard = init_gtk_clipboard()

class TestQt(_TestClipboard):
    if HAS_DISPLAY:
        try:
            import PyQt4
        except ImportError:
            pass
        else:
            clipboard = init_qt_clipboard()

class TestXClip(_TestClipboard):
    if _executable_exists("xclip"):
        clipboard = init_xclip_clipboard()

class TestXSel(_TestClipboard):
    if _executable_exists("xsel"):
        clipboard = init_xsel_clipboard()

class TestKlipper(_TestClipboard):
    if _executable_exists("klipper") and _executable_exists("qdbus"):
        clipboard = init_klipper_clipboard()

class TestNoClipboard(unittest.TestCase):
    copy, paste = init_no_clipboard()

    def test_copy(self):
        with self.assertRaises(RuntimeError):
            self.copy("foo")

    def test_paste(self):
        with self.assertRaises(RuntimeError):
            self.paste()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import unittest
import os
import numpy as np
import pytest
import scipy

import deepchem as dc
from deepchem.data import NumpyDataset
from deepchem.models import GraphConvModel, DAGModel, WeaveModel, MPNNModel
from deepchem.molnet import load_bace_classification, load_delaney
from deepchem.feat import ConvMolFeaturizer

from flaky import flaky

def get_dataset(mode='classification',
                featurizer='GraphConv',
                num_tasks=2,
                data_points=20):
  if mode == 'classification':
    tasks, all_dataset, transformers = load_bace_classification(
        featurizer, reload=False)
  else:
    tasks, all_dataset, transformers = load_delaney(featurizer, reload=False)

  train, valid, test = all_dataset
  for i in range(1, num_tasks):
    tasks.append("random_task")
  w = np.ones(shape=(data_points, len(tasks)))

  if mode == 'classification':
    y = np.random.randint(0, 2, size=(data_points, len(tasks)))
    metric = dc.metrics.Metric(
        dc.metrics.roc_auc_score, np.mean, mode="classification")
  else:
    y = np.random.normal(size=(data_points, len(tasks)))
    metric = dc.metrics.Metric(
        dc.metrics.mean_absolute_error, mode="regression")

  ds = NumpyDataset(train.X[:data_points], y, w, train.ids[:data_points])

  return tasks, ds, transformers, metric

def test_compute_features_on_infinity_distance():
  """Test that WeaveModel correctly transforms WeaveMol objects into tensors with infinite max_pair_distance."""
  featurizer = dc.feat.WeaveFeaturizer(max_pair_distance=None)
  X = featurizer(["C", "CCC"])
  batch_size = 20
  model = WeaveModel(
      1,
      batch_size=batch_size,
      mode='classification',
      fully_connected_layer_sizes=[2000, 1000],
      batch_normalize=True,
      batch_normalize_kwargs={
          "fused": False,
          "trainable": True,
          "renorm": True
      },
      learning_rage=0.0005)
  atom_feat, pair_feat, pair_split, atom_split, atom_to_pair = model.compute_features_on_batch(
      X)

  # There are 4 atoms each of which have 75 atom features
  assert atom_feat.shape == (4, 75)
  # There are 10 pairs with infinity distance and 14 pair features
  assert pair_feat.shape == (10, 14)
  # 4 atoms in total
  assert atom_split.shape == (4,)
  assert np.all(atom_split == np.array([0, 1, 1, 1]))
  # 10 pairs in total
  assert pair_split.shape == (10,)
  assert np.all(pair_split == np.array([0, 1, 1, 1, 2, 2, 2, 3, 3, 3]))
  # 10 pairs in total each with start/finish
  assert atom_to_pair.shape == (10, 2)
  assert np.all(
      atom_to_pair == np.array([[0, 0], [1, 1], [1, 2], [1, 3], [2, 1], [2, 2],
                                [2, 3], [3, 1], [3, 2], [3, 3]]))

def test_compute_features_on_distance_1():
  """Test that WeaveModel correctly transforms WeaveMol objects into tensors with finite max_pair_distance."""
  featurizer = dc.feat.WeaveFeaturizer(max_pair_distance=1)
  X = featurizer(["C", "CCC"])
  batch_size = 20
  model = WeaveModel(
      1,
      batch_size=batch_size,
      mode='classification',
      fully_connected_layer_sizes=[2000, 1000],
      batch_normalize=True,
      batch_normalize_kwargs={
          "fused": False,
          "trainable": True,
          "renorm": True
      },
      learning_rage=0.0005)
  atom_feat, pair_feat, pair_split, atom_split, atom_to_pair = model.compute_features_on_batch(
      X)

  # There are 4 atoms each of which have 75 atom features
  assert atom_feat.shape == (4, 75)
  # There are 8 pairs with distance 1 and 14 pair features. (To see why 8,
  # there's the self pair for "C". For "CCC" there are 7 pairs including self
  # connections and accounting for symmetry.)
  assert pair_feat.shape == (8, 14)
  # 4 atoms in total
  assert atom_split.shape == (4,)
  assert np.all(atom_split == np.array([0, 1, 1, 1]))
  # 10 pairs in total
  assert pair_split.shape == (8,)
  # The center atom is self connected and to both neighbors so it appears
  # thrice. The canonical ranking used in MolecularFeaturizer means this
  # central atom is ranked last in ordering.
  assert np.all(pair_split == np.array([0, 1, 1, 2, 2, 3, 3, 3]))
  # 10 pairs in total each with start/finish
  assert atom_to_pair.shape == (8, 2)
  assert np.all(atom_to_pair == np.array([[0, 0], [1, 1], [1, 3], [2, 2],
                                          [3, 3], [3, 1], [3, 2], [3, 3]]))

@flaky
@pytest.mark.slow
def test_weave_model():
  tasks, dataset, transformers, metric = get_dataset(
      'classification', 'Weave', data_points=10)

  batch_size = 10
  model = WeaveModel(
      len(tasks),
      batch_size=batch_size,
      mode='classification',
      final_conv_activation_fn=None,
      dropouts=0,
      learning_rage=0.0003)
  model.fit(dataset, nb_epoch=100)
  scores = model.evaluate(dataset, [metric], transformers)
  assert scores['mean-roc_auc_score'] >= 0.9

@pytest.mark.slow
def test_weave_regression_model():
  import numpy as np
  import tensorflow as tf
  tf.random.set_seed(123)
  np.random.seed(123)
  tasks, dataset, transformers, metric = get_dataset(
      'regression', 'Weave', data_points=10)

  batch_size = 10
  model = WeaveModel(
      len(tasks),
      batch_size=batch_size,
      mode='regression',
      dropouts=0,
      learning_rate=0.00003)
  model.fit(dataset, nb_epoch=400)
  scores = model.evaluate(dataset, [metric], transformers)
  assert scores['mean_absolute_error'] < 0.1

def test_weave_fit_simple_infinity_distance():
  featurizer = dc.feat.WeaveFeaturizer(max_pair_distance=None)
  X = featurizer(["C", "CCC"])
  y = np.array([0, 1.])
  dataset = dc.data.NumpyDataset(X, y)

  batch_size = 20
  model = WeaveModel(
      1,
      batch_size=batch_size,
      mode='classification',
      fully_connected_layer_sizes=[2000, 1000],
      batch_normalize=True,
      batch_normalize_kwargs={
          "fused": False,
          "trainable": True,
          "renorm": True
      },
      learning_rage=0.0005)
  model.fit(dataset, nb_epoch=200)
  transformers = []
  metric = dc.metrics.Metric(
      dc.metrics.roc_auc_score, np.mean, mode="classification")
  scores = model.evaluate(dataset, [metric], transformers)
  assert scores['mean-roc_auc_score'] >= 0.9

def test_weave_fit_simple_distance_1():
  featurizer = dc.feat.WeaveFeaturizer(max_pair_distance=1)
  X = featurizer(["C", "CCC"])
  y = np.array([0, 1.])
  dataset = dc.data.NumpyDataset(X, y)

  batch_size = 20
  model = WeaveModel(
      1,
      batch_size=batch_size,
      mode='classification',
      fully_connected_layer_sizes=[2000, 1000],
      batch_normalize=True,
      batch_normalize_kwargs={
          "fused": False,
          "trainable": True,
          "renorm": True
      },
      learning_rage=0.0005)
  model.fit(dataset, nb_epoch=200)
  transformers = []
  metric = dc.metrics.Metric(
      dc.metrics.roc_auc_score, np.mean, mode="classification")
  scores = model.evaluate(dataset, [metric], transformers)
  assert scores['mean-roc_auc_score'] >= 0.9
<commit_msg>Fixing changed constant<commit_after>import unittest
import os
import numpy as np
import pytest
import scipy

import deepchem as dc
from deepchem.data import NumpyDataset
from deepchem.models import GraphConvModel, DAGModel, WeaveModel, MPNNModel
from deepchem.molnet import load_bace_classification, load_delaney
from deepchem.feat import ConvMolFeaturizer

from flaky import flaky

def get_dataset(mode='classification',
                featurizer='GraphConv',
                num_tasks=2,
                data_points=20):
  if mode == 'classification':
    tasks, all_dataset, transformers = load_bace_classification(
        featurizer, reload=False)
  else:
    tasks, all_dataset, transformers = load_delaney(featurizer, reload=False)

  train, valid, test = all_dataset
  for i in range(1, num_tasks):
    tasks.append("random_task")
  w = np.ones(shape=(data_points, len(tasks)))

  if mode == 'classification':
    y = np.random.randint(0, 2, size=(data_points, len(tasks)))
    metric = dc.metrics.Metric(
        dc.metrics.roc_auc_score, np.mean, mode="classification")
  else:
    y = np.random.normal(size=(data_points, len(tasks)))
    metric = dc.metrics.Metric(
        dc.metrics.mean_absolute_error, mode="regression")

  ds = NumpyDataset(train.X[:data_points], y, w, train.ids[:data_points])

  return tasks, ds, transformers, metric

def test_compute_features_on_infinity_distance():
  """Test that WeaveModel correctly transforms WeaveMol objects into tensors with infinite max_pair_distance."""
  featurizer = dc.feat.WeaveFeaturizer(max_pair_distance=None)
  X = featurizer(["C", "CCC"])
  batch_size = 20
  model = WeaveModel(
      1,
      batch_size=batch_size,
      mode='classification',
      fully_connected_layer_sizes=[2000, 1000],
      batch_normalize=True,
      batch_normalize_kwargs={
          "fused": False,
          "trainable": True,
          "renorm": True
      },
      learning_rage=0.0005)
  atom_feat, pair_feat, pair_split, atom_split, atom_to_pair = model.compute_features_on_batch(
      X)

  # There are 4 atoms each of which have 75 atom features
  assert atom_feat.shape == (4, 75)
  # There are 10 pairs with infinity distance and 14 pair features
  assert pair_feat.shape == (10, 14)
  # 4 atoms in total
  assert atom_split.shape == (4,)
  assert np.all(atom_split == np.array([0, 1, 1, 1]))
  # 10 pairs in total
  assert pair_split.shape == (10,)
  assert np.all(pair_split == np.array([0, 1, 1, 1, 2, 2, 2, 3, 3, 3]))
  # 10 pairs in total each with start/finish
  assert atom_to_pair.shape == (10, 2)
  assert np.all(
      atom_to_pair == np.array([[0, 0], [1, 1], [1, 2], [1, 3], [2, 1], [2, 2],
                                [2, 3], [3, 1], [3, 2], [3, 3]]))

def test_compute_features_on_distance_1():
  """Test that WeaveModel correctly transforms WeaveMol objects into tensors with finite max_pair_distance."""
  featurizer = dc.feat.WeaveFeaturizer(max_pair_distance=1)
  X = featurizer(["C", "CCC"])
  batch_size = 20
  model = WeaveModel(
      1,
      batch_size=batch_size,
      mode='classification',
      fully_connected_layer_sizes=[2000, 1000],
      batch_normalize=True,
      batch_normalize_kwargs={
          "fused": False,
          "trainable": True,
          "renorm": True
      },
      learning_rage=0.0005)
  atom_feat, pair_feat, pair_split, atom_split, atom_to_pair = model.compute_features_on_batch(
      X)

  # There are 4 atoms each of which have 75 atom features
  assert atom_feat.shape == (4, 75)
  # There are 8 pairs with distance 1 and 14 pair features. (To see why 8,
  # there's the self pair for "C". For "CCC" there are 7 pairs including self
  # connections and accounting for symmetry.)
  assert pair_feat.shape == (8, 14)
  # 4 atoms in total
  assert atom_split.shape == (4,)
  assert np.all(atom_split == np.array([0, 1, 1, 1]))
  # 10 pairs in total
  assert pair_split.shape == (8,)
  # The center atom is self connected and to both neighbors so it appears
  # thrice. The canonical ranking used in MolecularFeaturizer means this
  # central atom is ranked last in ordering.
  assert np.all(pair_split == np.array([0, 1, 1, 2, 2, 3, 3, 3]))
  # 10 pairs in total each with start/finish
  assert atom_to_pair.shape == (8, 2)
  assert np.all(atom_to_pair == np.array([[0, 0], [1, 1], [1, 3], [2, 2],
                                          [2, 3], [3, 1], [3, 2], [3, 3]]))

@flaky
@pytest.mark.slow
def test_weave_model():
  tasks, dataset, transformers, metric = get_dataset(
      'classification', 'Weave', data_points=10)

  batch_size = 10
  model = WeaveModel(
      len(tasks),
      batch_size=batch_size,
      mode='classification',
      final_conv_activation_fn=None,
      dropouts=0,
      learning_rage=0.0003)
  model.fit(dataset, nb_epoch=100)
  scores = model.evaluate(dataset, [metric], transformers)
  assert scores['mean-roc_auc_score'] >= 0.9

@pytest.mark.slow
def test_weave_regression_model():
  import numpy as np
  import tensorflow as tf
  tf.random.set_seed(123)
  np.random.seed(123)
  tasks, dataset, transformers, metric = get_dataset(
      'regression', 'Weave', data_points=10)

  batch_size = 10
  model = WeaveModel(
      len(tasks),
      batch_size=batch_size,
      mode='regression',
      dropouts=0,
      learning_rate=0.00003)
  model.fit(dataset, nb_epoch=400)
  scores = model.evaluate(dataset, [metric], transformers)
  assert scores['mean_absolute_error'] < 0.1

def test_weave_fit_simple_infinity_distance():
  featurizer = dc.feat.WeaveFeaturizer(max_pair_distance=None)
  X = featurizer(["C", "CCC"])
  y = np.array([0, 1.])
  dataset = dc.data.NumpyDataset(X, y)

  batch_size = 20
  model = WeaveModel(
      1,
      batch_size=batch_size,
      mode='classification',
      fully_connected_layer_sizes=[2000, 1000],
      batch_normalize=True,
      batch_normalize_kwargs={
          "fused": False,
          "trainable": True,
          "renorm": True
      },
      learning_rage=0.0005)
  model.fit(dataset, nb_epoch=200)
  transformers = []
  metric = dc.metrics.Metric(
      dc.metrics.roc_auc_score, np.mean, mode="classification")
  scores = model.evaluate(dataset, [metric], transformers)
  assert scores['mean-roc_auc_score'] >= 0.9

def test_weave_fit_simple_distance_1():
  featurizer = dc.feat.WeaveFeaturizer(max_pair_distance=1)
  X = featurizer(["C", "CCC"])
  y = np.array([0, 1.])
  dataset = dc.data.NumpyDataset(X, y)

  batch_size = 20
  model = WeaveModel(
      1,
      batch_size=batch_size,
      mode='classification',
      fully_connected_layer_sizes=[2000, 1000],
      batch_normalize=True,
      batch_normalize_kwargs={
          "fused": False,
          "trainable": True,
          "renorm": True
      },
      learning_rage=0.0005)
  model.fit(dataset, nb_epoch=200)
  transformers = []
  metric = dc.metrics.Metric(
      dc.metrics.roc_auc_score, np.mean, mode="classification")
  scores = model.evaluate(dataset, [metric], transformers)
  assert scores['mean-roc_auc_score'] >= 0.9
<|endoftext|>


<commit_before>import datetime
from flask import flash, redirect, url_for, request, render_template, send_from_directory
from flask.ext.login import login_required
from app import db, telomere
from app.forms.spreadsheet import SpreadsheetUpload
from app.services.spreadsheet import SpreadsheetService
from app.services.batch import BatchService
from app.model.spreadsheet import Spreadsheet
from app.model.user import User
from app.model.manifest import Manifest
from flask_login import current_user
from app.helpers.wrappers import manifest_required

@telomere.route('/spreadsheet/upload/', methods=['GET', 'POST'])
@login_required
@manifest_required
def speadsheet_upload():

    form = SpreadsheetUpload()

    users = User.query.order_by(User.code.asc()).all()
    form.batch.operatorUserId.choices = [(u.id, u.GetCodeAndName()) for u in users]
    form.batch.operatorUserId.data = form.batch.operatorUserId.data or current_user.id

    if form.validate_on_submit():

        batchService = BatchService()
        batch = batchService.SaveAndReturn(form.batch)

        if batchService.IsBatchDuplicate(batch):
            if batch.is_initial() or batch.is_replate():
                flash("The plate name and half plate have been used in a previous batch.", "error")
                db.session.rollback()
                return render_template('spreadsheet/upload.html', form=form)
        else:
            if batch.is_duplicate():
                flash("The plate name and half plate have not been used in a previous batch.", "error")
                db.session.rollback()
                return render_template('spreadsheet/upload.html', form=form)

        if (batch and batch.batchFailureReason):
            db.session.commit()
            return redirect(url_for('batch_index'))

        if (batch):
            spreadsheetService = SpreadsheetService()
            spreadsheet = spreadsheetService.SaveAndReturn(form.spreadsheet.data, batch)

            if not spreadsheetService.ValidateFormat(spreadsheet):
                flash("Spreadsheet does not have a valid format", "error")

                db.session.rollback()
                return render_template('spreadsheet/upload.html', form=form)

            spreadsheetLoadResult = spreadsheetService.Process(spreadsheet, not batch.is_replate())

            if spreadsheetLoadResult.abortUpload():
                if len(spreadsheetLoadResult.missingSampleCodes) > 0:
                    flash("The following samples are not in the manifest: %s" % ", ".join(str(x) for x in spreadsheetLoadResult.missingSampleCodes), "error")
                if len(spreadsheetLoadResult.incorrectPlateName) > 0:
                    flash("The following samples have a different plate name to the manifest: %s" % ", ".join(str(x) for x in spreadsheetLoadResult.incorrectPlateName), "error")

                flash("File '%s' has not been uploaded" % spreadsheet.filename, "error")

                db.session.rollback()
            else:
                flash("File '%s' Uploaded" % spreadsheet.filename)

                if spreadsheetLoadResult.hasOutstandingErrors:
                    flash("Batch was loaded with errors", "warning")

                db.session.commit()

                return redirect(url_for('batch_index'))

    return render_template('spreadsheet/upload.html', form=form)

@telomere.route("/spreadsheet/download/<int:id>")
@login_required
def speadsheet_download(id):
    spreadsheet = Spreadsheet.query.get(id)
    service = SpreadsheetService()

    return send_from_directory(telomere.config['SPREADSHEET_UPLOAD_DIRECTORY'], service.GetFilename(spreadsheet), as_attachment=True, attachment_filename=spreadsheet.filename)

<commit_msg>Debug missing spreadsheet<commit_after>import datetime
from flask import flash, redirect, url_for, request, render_template, send_from_directory
from flask.ext.login import login_required
from app import db, telomere
from app.forms.spreadsheet import SpreadsheetUpload
from app.services.spreadsheet import SpreadsheetService
from app.services.batch import BatchService
from app.model.spreadsheet import Spreadsheet
from app.model.user import User
from app.model.manifest import Manifest
from flask_login import current_user
from app.helpers.wrappers import manifest_required

@telomere.route('/spreadsheet/upload/', methods=['GET', 'POST'])
@login_required
@manifest_required
def speadsheet_upload():

    form = SpreadsheetUpload()

    users = User.query.order_by(User.code.asc()).all()
    form.batch.operatorUserId.choices = [(u.id, u.GetCodeAndName()) for u in users]
    form.batch.operatorUserId.data = form.batch.operatorUserId.data or current_user.id

    if form.validate_on_submit():

        batchService = BatchService()
        batch = batchService.SaveAndReturn(form.batch)

        if batchService.IsBatchDuplicate(batch):
            if batch.is_initial() or batch.is_replate():
                flash("The plate name and half plate have been used in a previous batch.", "error")
                db.session.rollback()
                return render_template('spreadsheet/upload.html', form=form)
        else:
            if batch.is_duplicate():
                flash("The plate name and half plate have not been used in a previous batch.", "error")
                db.session.rollback()
                return render_template('spreadsheet/upload.html', form=form)

        if (batch and batch.batchFailureReason):
            db.session.commit()
            return redirect(url_for('batch_index'))

        if (batch):
            spreadsheetService = SpreadsheetService()
            spreadsheet = spreadsheetService.SaveAndReturn(form.spreadsheet.data, batch)

            if not spreadsheetService.ValidateFormat(spreadsheet):
                flash("Spreadsheet does not have a valid format", "error")

                db.session.rollback()
                return render_template('spreadsheet/upload.html', form=form)

            spreadsheetLoadResult = spreadsheetService.Process(spreadsheet, not batch.is_replate())

            if spreadsheetLoadResult.abortUpload():
                if len(spreadsheetLoadResult.missingSampleCodes) > 0:
                    flash("The following samples are not in the manifest: %s" % ", ".join(str(x) for x in spreadsheetLoadResult.missingSampleCodes), "error")
                if len(spreadsheetLoadResult.incorrectPlateName) > 0:
                    flash("The following samples have a different plate name to the manifest: %s" % ", ".join(str(x) for x in spreadsheetLoadResult.incorrectPlateName), "error")

                flash("File '%s' has not been uploaded" % spreadsheet.filename, "error")

                db.session.rollback()
            else:
                flash("File '%s' Uploaded" % spreadsheet.filename)

                if spreadsheetLoadResult.hasOutstandingErrors:
                    flash("Batch was loaded with errors", "warning")

                db.session.commit()

                return redirect(url_for('batch_index'))

    return render_template('spreadsheet/upload.html', form=form)

@telomere.route("/spreadsheet/download/<int:id>")
@login_required
def speadsheet_download(id):
    spreadsheet = Spreadsheet.query.get(id)
    service = SpreadsheetService()

    print dir(spreadsheet)

    return send_from_directory(telomere.config['SPREADSHEET_UPLOAD_DIRECTORY'], service.GetFilename(spreadsheet), as_attachment=True, attachment_filename=spreadsheet.filename)

<|endoftext|>


<commit_before># This file is part of khmer, https://github.com/dib-lab/khmer/, and is
# Copyright (C) 2016, The Regents of the University of California.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
#     * Neither the name of the University of California nor the names
#       of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written
#       permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Contact: khmer-project@idyll.org
# pylint: disable=C0111,C0103,missing-docstring,no-member,protected-access

import khmer

import pytest
from . import khmer_tst_utils as utils

def test_get_kmer_hashes():
    s = "ATGGATATGGAGGACAAGTATATGGAGGACAAGTATATGGAGGACAAGTAT"
    a = khmer.Counttable(33, 1e6, 3)
    assert a.get_kmer_hashes(s[:33]) == [4743239192574154715]
    assert a.get_kmer_hashes(s[:34]) == [4743239192574154715,
                                         2122462908541313313]

    assert a.get_kmer_hashes(s[0:33]) == [4743239192574154715]
    assert a.get_kmer_hashes(s[1:34]) == [2122462908541313313]

@pytest.mark.parametrize('kmer', [
    ('GATTACA' * 3),
    ('ATG' * 7),
    ('AGGACAAGTATATGGAGGACA'),
])
def test_kmer_revcom_hash(kmer):
    a = khmer.Counttable(21, 1e4, 3)
    assert a.hash(kmer) == a.hash(khmer.reverse_complement(kmer))

@pytest.mark.parametrize('ksize,sketch_allocator', [
    (21, khmer.Nodetable),
    (21, khmer.Counttable),
    (21, khmer.SmallCounttable),
    (49, khmer.Nodetable),
    (49, khmer.Counttable),
    (49, khmer.SmallCounttable),
])
def test_reverse_hash(ksize, sketch_allocator):
    multiplier = int(ksize / len('GATTACA'))
    kmer = 'GATTACA' * multiplier

    sketch = sketch_allocator(ksize, 1e4, 4)
    kmer_hash = sketch.hash(kmer)
    with pytest.raises(ValueError) as ve:
        _ = sketch.reverse_hash(kmer_hash)
    assert 'not implemented' in str(ve)

def test_consume_with_mask():
    """
    Test bulk loading with a mask

    The top sequence is the mask, the bottom sequence is to be loaded. The
    bottom 3 k-mers are not present in the mask and therefore should be the
    only ones loaded into the counttable.

    TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAA        <--- mask
       ATCTGCTTGAAACAAGTGGATTTGAGAAAAAAGT     <--- sequence
                          |-----------|
                           |-----------|
                            |-----------|
    """
    maskfile = utils.get_test_data('seq-a.fa')
    mask = khmer.Counttable(13, 1e3, 4)
    mask.consume_seqfile(maskfile)

    infile = utils.get_test_data('seq-b.fa')
    ct = khmer.Counttable(13, 1e3, 4)
    nr, nk = ct.consume_seqfile_with_mask(infile, mask)

    assert nr == 1
    assert nk == 3
    assert ct.get('GATTTGAGAAAAA') == 0  # in the mask
    assert ct.get('ATTTGAGAAAAAA') == 1
    assert ct.get('TTTGAGAAAAAAG') == 1
    assert ct.get('TTGAGAAAAAAGT') == 1

def test_consume_banding_with_mask():
    """
    Test bulk loading with a mask *in k-mer banding mode*

    TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAA        <--- mask
       ATCTGCTTGAAACAAGTGGATTTGAGAAAAAAGT     <--- sequence
                          |-----------|
                           |-----------|
                            |-----------|     <--- only k-mer in band 1/4
    """
    maskfile = utils.get_test_data('seq-a.fa')
    mask = khmer.Counttable(13, 1e3, 4)
    mask.consume_seqfile(maskfile)

    infile = utils.get_test_data('seq-b.fa')
    ct = khmer.Counttable(13, 1e3, 4)
    nr, nk = ct.consume_seqfile_banding_with_mask(infile, 4, 1, mask)

    assert nr == 1
    assert nk == 1
    assert ct.get('GATTTGAGAAAAA') == 0  # in the mask
    assert ct.get('ATTTGAGAAAAAA') == 0  # out of band
    assert ct.get('TTTGAGAAAAAAG') == 0  # out of band
    assert ct.get('TTGAGAAAAAAGT') == 1

def test_consume_with_mask_threshold():
    """
    Test bulk loading with a mask and an abundance threshold

    The top sequence is the mask, the bottom sequence is to be loaded. The
    bottom 3 k-mers are not present in the mask and therefore should be the
    only ones loaded into the counttable.

    TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAA        |
    TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAA        |
    TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAA        | <--- mask input
    TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAAAGT     |
    TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAAAGT     |

       ATCTGCTTGAAACAAGTGGATTTGAGAAAAAAGT     <--- sequence
                          |-----------|
                           |-----------|      <--- only these k-mers are
                            |-----------|          abundance <= 3 in the mask
    """
    maskfile = utils.get_test_data('seq-a.fa')
    mask = khmer.Counttable(13, 1e3, 4)
    for _ in range(3):
        mask.consume('TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAA')
    for _ in range(2):
        mask.consume('TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAAAGT')

    infile = utils.get_test_data('seq-b.fa')
    ct = khmer.Counttable(13, 1e3, 4)
    nr, nk = ct.consume_seqfile_with_mask(infile, mask, 3)

    assert nr == 1
    assert nk == 3
    assert ct.get('GATTTGAGAAAAA') == 0  # in the mask
    assert ct.get('ATTTGAGAAAAAA') == 1
    assert ct.get('TTTGAGAAAAAAG') == 1
    assert ct.get('TTGAGAAAAAAGT') == 1
<commit_msg>Simple test of complement behavior<commit_after># This file is part of khmer, https://github.com/dib-lab/khmer/, and is
# Copyright (C) 2016, The Regents of the University of California.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
#     * Neither the name of the University of California nor the names
#       of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written
#       permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Contact: khmer-project@idyll.org
# pylint: disable=C0111,C0103,missing-docstring,no-member,protected-access

import khmer

import pytest
from . import khmer_tst_utils as utils

def test_get_kmer_hashes():
    s = "ATGGATATGGAGGACAAGTATATGGAGGACAAGTATATGGAGGACAAGTAT"
    a = khmer.Counttable(33, 1e6, 3)
    assert a.get_kmer_hashes(s[:33]) == [4743239192574154715]
    assert a.get_kmer_hashes(s[:34]) == [4743239192574154715,
                                         2122462908541313313]

    assert a.get_kmer_hashes(s[0:33]) == [4743239192574154715]
    assert a.get_kmer_hashes(s[1:34]) == [2122462908541313313]

@pytest.mark.parametrize('kmer', [
    ('GATTACA' * 3),
    ('ATG' * 7),
    ('AGGACAAGTATATGGAGGACA'),
])
def test_kmer_revcom_hash(kmer):
    a = khmer.Counttable(21, 1e4, 3)
    assert a.hash(kmer) == a.hash(khmer.reverse_complement(kmer))

@pytest.mark.parametrize('ksize,sketch_allocator', [
    (21, khmer.Nodetable),
    (21, khmer.Counttable),
    (21, khmer.SmallCounttable),
    (49, khmer.Nodetable),
    (49, khmer.Counttable),
    (49, khmer.SmallCounttable),
])
def test_reverse_hash(ksize, sketch_allocator):
    multiplier = int(ksize / len('GATTACA'))
    kmer = 'GATTACA' * multiplier

    sketch = sketch_allocator(ksize, 1e4, 4)
    kmer_hash = sketch.hash(kmer)
    with pytest.raises(ValueError) as ve:
        _ = sketch.reverse_hash(kmer_hash)
    assert 'not implemented' in str(ve)

def test_consume_with_mask():
    """
    Test bulk loading with a mask

    The top sequence is the mask, the bottom sequence is to be loaded. The
    bottom 3 k-mers are not present in the mask and therefore should be the
    only ones loaded into the counttable.

    TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAA        <--- mask
       ATCTGCTTGAAACAAGTGGATTTGAGAAAAAAGT     <--- sequence
                          |-----------|
                           |-----------|
                            |-----------|
    """
    maskfile = utils.get_test_data('seq-a.fa')
    mask = khmer.Counttable(13, 1e3, 4)
    mask.consume_seqfile(maskfile)

    infile = utils.get_test_data('seq-b.fa')
    ct = khmer.Counttable(13, 1e3, 4)
    nr, nk = ct.consume_seqfile_with_mask(infile, mask)

    assert nr == 1
    assert nk == 3
    assert ct.get('GATTTGAGAAAAA') == 0  # in the mask
    assert ct.get('ATTTGAGAAAAAA') == 1
    assert ct.get('TTTGAGAAAAAAG') == 1
    assert ct.get('TTGAGAAAAAAGT') == 1

def test_consume_banding_with_mask():
    """
    Test bulk loading with a mask *in k-mer banding mode*

    TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAA        <--- mask
       ATCTGCTTGAAACAAGTGGATTTGAGAAAAAAGT     <--- sequence
                          |-----------|
                           |-----------|
                            |-----------|     <--- only k-mer in band 1/4
    """
    maskfile = utils.get_test_data('seq-a.fa')
    mask = khmer.Counttable(13, 1e3, 4)
    mask.consume_seqfile(maskfile)

    infile = utils.get_test_data('seq-b.fa')
    ct = khmer.Counttable(13, 1e3, 4)
    nr, nk = ct.consume_seqfile_banding_with_mask(infile, 4, 1, mask)

    assert nr == 1
    assert nk == 1
    assert ct.get('GATTTGAGAAAAA') == 0  # in the mask
    assert ct.get('ATTTGAGAAAAAA') == 0  # out of band
    assert ct.get('TTTGAGAAAAAAG') == 0  # out of band
    assert ct.get('TTGAGAAAAAAGT') == 1

def test_consume_with_mask_threshold():
    """
    Test bulk loading with a mask and an abundance threshold

    The top sequence is the mask, the bottom sequence is to be loaded. The
    bottom 3 k-mers are not present in the mask and therefore should be the
    only ones loaded into the counttable.

    TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAA        |
    TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAA        |
    TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAA        | <--- mask input
    TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAAAGT     |
    TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAAAGT     |

       ATCTGCTTGAAACAAGTGGATTTGAGAAAAAAGT     <--- sequence
                          |-----------|
                           |-----------|      <--- only these k-mers are
                            |-----------|          abundance <= 3 in the mask
    """
    mask = khmer.Counttable(13, 1e3, 4)
    for _ in range(3):
        mask.consume('TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAA')
    for _ in range(2):
        mask.consume('TAGATCTGCTTGAAACAAGTGGATTTGAGAAAAAAGT')

    infile = utils.get_test_data('seq-b.fa')
    ct = khmer.Counttable(13, 1e3, 4)
    nr, nk = ct.consume_seqfile_with_mask(infile, mask, 3)

    assert nr == 1
    assert nk == 3
    assert ct.get('GATTTGAGAAAAA') == 0  # in the mask
    assert ct.get('ATTTGAGAAAAAA') == 1
    assert ct.get('TTTGAGAAAAAAG') == 1
    assert ct.get('TTGAGAAAAAAGT') == 1

def test_consume_with_mask_complement():
    mask = khmer.Nodetable(13, 1e3, 4)
    mask.consume('TGCTTGAAACAAGTG')

    infile = utils.get_test_data('seq-b.fa')
    ct = khmer.Counttable(13, 1e3, 4)
    nr, nk = ct.consume_seqfile_with_mask(infile, mask, threshold=1,
                                          complement=True)

    assert ct.get_kmer_counts('TGCTTGAAACAAGTG') == [1, 1, 1]
    assert ct.get_kmer_counts('GAAACAAGTGGATTT') == [0, 0, 0]
<|endoftext|>