<commit_before>no_init_paren = ('f', 'm', 'o', 'u', '.e', '.m', '.M', '.U')
end_statement = ('B', 'R', '.u')
variables = 'bdGHkNQTYzZ'

# Variables cheat sheet:
# b = "\n"
# d is for map, d=' '
# G is for reduce, G=string.ascii_lowercase (abc..xyz)
# H is for reduce, H = {}
# k = ''
# J - Autoinitializer - copies, no stringing.
# K - Autoinitializer - can be strung (KJw), no copy.
# N = None, second option variable for map,filter,reduce
# T is for filter, second variable option for reduce, T=10
# Y = []
# Z = 0

c_to_s = {
    'D': (('@memoized\ndef ', ':'), 1),
    'E': (('else:', ), 0),
    'F': (('for ', ' in num_to_range(', '):'), 2),
    'I': (('if ', ':'), 1),
    'W': (('while ', ':'), 1),
    '#': (('while True:\n try:', '\n except Exception:\n  break'), 0, 1),
    }

# Arbitrary format operators - use for assignment, infix, etc.
# All surrounding strings, arity
c_to_i = {
    '~':  (('', '+=', ''), 2),
    '=':  (('assign(\'', '\',', ')'), 2),
    '&':  (('(', ' and ', ')'), 2),
    '|':  (('(', ' or ', ')'), 2),
    ']':  (('[', ']'), 1),
    '}':  (('(', ' in ', ')'), 2),
    '?':  (('(', ' if ', ' else ', ')'), 3),
    ',':  (('(', ',', ')'), 2),
    'B':  (('break', ), 0),
    'J':  (('assign("J",', ')'), 1),
    'K':  (('assign("K",', ')'), 1),
    'R':  (('return ', ''), 1),
    '.*': (('*(', ')'), 1),
    '.u': (('*(', ')'), 1),
    '.)': (('', '.pop()'), 1),
    '.(': (('', '.pop(', ')'), 2),
    '.=': (('assign(\'', '\',', ')'), 2),
    }

# Simple functions only.
# Extensible is allowed, nothing else complicated is.
# -1 means extensible
# name,arity
c_to_f = {
    '`': ('repr', 1),
    '!': ('Pnot', 1),
    '@': ('lookup', 2),
    '%': ('mod', 2),
    '^': ('Ppow', 2),
    '*': ('times', 2),
    '(': ('Ptuple', -1),
    '-': ('minus', 2),
    '_': ('neg', 1),
    '+': ('plus', 2),
    '[': ('Plist', -1),
    '{': ('Pset', 1),
    "'": ('read_file', 1),
    ':': ('at_slice', 3),
    '<': ('lt', 2),
    '>': ('gt', 2),
    '/': ('div', 2),
    ' ': ('', 1),
    '\n': ('', 1),
    'a': ('append', 2),
    'C': ('Pchr', 1),
    'c': ('chop', 2),
    'e': ('end', 1),
    'f': ('Pfilter(lambda T:', 2),
    'g': ('gte', 2),
    'h': ('head', 1),
    'i': ('base_10', 2),
    'j': ('join', 2),
    'l': ('Plen', 1),
    'm': ('Pmap(lambda d:', 2),
    'n': ('ne', 2),
    'O': ('rchoice', 1),
    'o': ('order(lambda N:', 2),
    'P': ('primes_upper', 1),
    'p': ('Pprint', 2),
    'q': ('equal', 2),
    'r': ('Prange', 2),
    'S': ('Psorted', 1),
    's': ('Psum', 1),
    't': ('tail', 1),
    'U': ('urange', 1),
    'u': ('reduce(lambda G, H:', 3),
    'v': ('eval', 1),
    'w': ('input', 0),
    'X': ('assign_at', 3),
    'x': ('index', 2),
    'y': ('subsets', 1),
    '.A': ('all', 1),
    '.a': ('abs', 1),
    '.B': ('Pbin', 1),
    '.c': ('combinations', 2),
    '.C': ('combinations_with_replacement', 2),
    '.d': ('dict', 1),
    '.D': ('divmod', 2),
    '.E': ('any', 1),
    '.e': ('Penumerate(lambda k, Y:', 2),
    '.f': ('float', 1),
    '.F': ('Pformat', 2),
    '.H': ('Phex', 1),
    '.h': ('Phash', 1),
    '.i': ('interleave', 2),
    '.j': ('Pcomplex', 2),
    '.l': ('log', 2),
    '.m': ('minimal(lambda b:', 2),
    '.M': ('maximal(lambda Z:', 2),
    '.n': ('Pnumbers', 1),
    '.O': ('Poct', 1),
    '.p': ('permutations', 1),
    '.P': ('permutations2', 2),
    '.q': ('Pexit', 0),
    '.Q': ('eval_all_input', 0),
    '.r': ('rotate', 2),
    '.R': ('Pround', 2),
    '.S': ('shuffle', 1),
    '.s': ('stripchars', 2),
    '.t': ('trig', 2),
    '.U': ('reduce2(lambda b, Z:', 2),
    '.w': ('Pwrite', 2),
    '.x': ('product', 1),
    '.z': ('all_input', 0),
    '.^': ('pow', 3),
    '.&': ('bitand', 2),
    '.|': ('bitor', 2),
    '.<': ('leftshift', 2),
    '.>': ('rightshift', 2),
    '._': ('sign', 1),
    '.-': ('remove', 2),
    '.:': ('substrings', 2),
    '.{': ('unique', 1),
    }

replacements = {
    '\\': ('"{0}"', 1),
    'V': ('FNU', 0),
    'A': ('=,{0}{1}', 2),
    'L': ('DybR', 0),
    'M': ('DgGHR', 0),
    '.N': ('D:NTYR', 0),
    }

#Provides parsing functions and arity conditions for syntactical sugar
#<non-zero>=: Augmented assignment
#<binary>F: Fold operator

syntax_sugar = {
    '=': (lambda c, r: r[:2] + c+r[1:], lambda n: n),
    'F': (lambda c, r: ".U" + c + c_to_f['.U'][0][15:19:3] + r[1:],
          lambda n: n == 2)
}

# Gives next function header to use - for filter, map, reduce.
# map: d, k, b
# filter: T, Y, Z
# order: N, Z,
# reduce: (G,H), (N,T)
# enumerate: (k, Y), (b, Z)

next_c_to_f = {
    'f': [('Pfilter(lambda Y:', 2), ('Pfilter(lambda Z:', 2), ],
    'm': [('Pmap(lambda k:', 2), ('Pmap(lambda b:', 2), ],
    'o': [('order(lambda Z:', 2), ],
    'u': [('reduce(lambda N, T:', 3), ],
    '.e': [('Penumerate(lambda b, Z:', 2), ],
    '.U': [('reduce2(lambda k, Y:', 2), ],
    }

# For autoinitializers. One shot, not rotating.
next_c_to_i = {
    'J': (('J'), 0),
    'K': (('K'), 0),
    }

# Prependers.
prepend = {
    'Q': "=Qvw",
    'z': "=zw",
    }
<commit_msg>~ is now alias for +=.<commit_after>no_init_paren = ('f', 'm', 'o', 'u', '.e', '.m', '.M', '.U')
end_statement = ('B', 'R', '.u')
variables = 'bdGHkNQTYzZ'

# Variables cheat sheet:
# b = "\n"
# d is for map, d=' '
# G is for reduce, G=string.ascii_lowercase (abc..xyz)
# H is for reduce, H = {}
# k = ''
# J - Autoinitializer - copies, no stringing.
# K - Autoinitializer - can be strung (KJw), no copy.
# N = None, second option variable for map,filter,reduce
# T is for filter, second variable option for reduce, T=10
# Y = []
# Z = 0

c_to_s = {
    'D': (('@memoized\ndef ', ':'), 1),
    'E': (('else:', ), 0),
    'F': (('for ', ' in num_to_range(', '):'), 2),
    'I': (('if ', ':'), 1),
    'W': (('while ', ':'), 1),
    '#': (('while True:\n try:', '\n except Exception:\n  break'), 0, 1),
    }

# Arbitrary format operators - use for assignment, infix, etc.
# All surrounding strings, arity
c_to_i = {
    '=':  (('assign(\'', '\',', ')'), 2),
    '&':  (('(', ' and ', ')'), 2),
    '|':  (('(', ' or ', ')'), 2),
    ']':  (('[', ']'), 1),
    '}':  (('(', ' in ', ')'), 2),
    '?':  (('(', ' if ', ' else ', ')'), 3),
    ',':  (('(', ',', ')'), 2),
    'B':  (('break', ), 0),
    'J':  (('assign("J",', ')'), 1),
    'K':  (('assign("K",', ')'), 1),
    'R':  (('return ', ''), 1),
    '.*': (('*(', ')'), 1),
    '.u': (('*(', ')'), 1),
    '.)': (('', '.pop()'), 1),
    '.(': (('', '.pop(', ')'), 2),
    '.=': (('assign(\'', '\',', ')'), 2),
    }

# Simple functions only.
# Extensible is allowed, nothing else complicated is.
# -1 means extensible
# name,arity
c_to_f = {
    '`': ('repr', 1),
    '!': ('Pnot', 1),
    '@': ('lookup', 2),
    '%': ('mod', 2),
    '^': ('Ppow', 2),
    '*': ('times', 2),
    '(': ('Ptuple', -1),
    '-': ('minus', 2),
    '_': ('neg', 1),
    '+': ('plus', 2),
    '[': ('Plist', -1),
    '{': ('Pset', 1),
    "'": ('read_file', 1),
    ':': ('at_slice', 3),
    '<': ('lt', 2),
    '>': ('gt', 2),
    '/': ('div', 2),
    ' ': ('', 1),
    '\n': ('', 1),
    'a': ('append', 2),
    'C': ('Pchr', 1),
    'c': ('chop', 2),
    'e': ('end', 1),
    'f': ('Pfilter(lambda T:', 2),
    'g': ('gte', 2),
    'h': ('head', 1),
    'i': ('base_10', 2),
    'j': ('join', 2),
    'l': ('Plen', 1),
    'm': ('Pmap(lambda d:', 2),
    'n': ('ne', 2),
    'O': ('rchoice', 1),
    'o': ('order(lambda N:', 2),
    'P': ('primes_upper', 1),
    'p': ('Pprint', 2),
    'q': ('equal', 2),
    'r': ('Prange', 2),
    'S': ('Psorted', 1),
    's': ('Psum', 1),
    't': ('tail', 1),
    'U': ('urange', 1),
    'u': ('reduce(lambda G, H:', 3),
    'v': ('eval', 1),
    'w': ('input', 0),
    'X': ('assign_at', 3),
    'x': ('index', 2),
    'y': ('subsets', 1),
    '.A': ('all', 1),
    '.a': ('abs', 1),
    '.B': ('Pbin', 1),
    '.c': ('combinations', 2),
    '.C': ('combinations_with_replacement', 2),
    '.d': ('dict', 1),
    '.D': ('divmod', 2),
    '.E': ('any', 1),
    '.e': ('Penumerate(lambda k, Y:', 2),
    '.f': ('float', 1),
    '.F': ('Pformat', 2),
    '.H': ('Phex', 1),
    '.h': ('Phash', 1),
    '.i': ('interleave', 2),
    '.j': ('Pcomplex', 2),
    '.l': ('log', 2),
    '.m': ('minimal(lambda b:', 2),
    '.M': ('maximal(lambda Z:', 2),
    '.n': ('Pnumbers', 1),
    '.O': ('Poct', 1),
    '.p': ('permutations', 1),
    '.P': ('permutations2', 2),
    '.q': ('Pexit', 0),
    '.Q': ('eval_all_input', 0),
    '.r': ('rotate', 2),
    '.R': ('Pround', 2),
    '.S': ('shuffle', 1),
    '.s': ('stripchars', 2),
    '.t': ('trig', 2),
    '.U': ('reduce2(lambda b, Z:', 2),
    '.w': ('Pwrite', 2),
    '.x': ('product', 1),
    '.z': ('all_input', 0),
    '.^': ('pow', 3),
    '.&': ('bitand', 2),
    '.|': ('bitor', 2),
    '.<': ('leftshift', 2),
    '.>': ('rightshift', 2),
    '._': ('sign', 1),
    '.-': ('remove', 2),
    '.:': ('substrings', 2),
    '.{': ('unique', 1),
    }

replacements = {
    '\\': ('"{0}"', 1),
    'V': ('FNU', 0),
    'A': ('=,{0}{1}', 2),
    'L': ('DybR', 0),
    'M': ('DgGHR', 0),
    '.N': ('D:NTYR', 0),
    '~': ('+=', 0),
    }

#Provides parsing functions and arity conditions for syntactical sugar
#<non-zero>=: Augmented assignment
#<binary>F: Fold operator

syntax_sugar = {
    '=': (lambda c, r: r[:2] + c+r[1:], lambda n: n),
    'F': (lambda c, r: ".U" + c + c_to_f['.U'][0][15:19:3] + r[1:],
          lambda n: n == 2)
}

# Gives next function header to use - for filter, map, reduce.
# map: d, k, b
# filter: T, Y, Z
# order: N, Z,
# reduce: (G,H), (N,T)
# enumerate: (k, Y), (b, Z)

next_c_to_f = {
    'f': [('Pfilter(lambda Y:', 2), ('Pfilter(lambda Z:', 2), ],
    'm': [('Pmap(lambda k:', 2), ('Pmap(lambda b:', 2), ],
    'o': [('order(lambda Z:', 2), ],
    'u': [('reduce(lambda N, T:', 3), ],
    '.e': [('Penumerate(lambda b, Z:', 2), ],
    '.U': [('reduce2(lambda k, Y:', 2), ],
    }

# For autoinitializers. One shot, not rotating.
next_c_to_i = {
    'J': (('J'), 0),
    'K': (('K'), 0),
    }

# Prependers.
prepend = {
    'Q': "=Qvw",
    'z': "=zw",
    }
<|endoftext|>


<commit_before># test_filewriter.py
"""unittest cases for dicom.filewriter module"""
# Copyright (c) 2008-2012 Darcy Mason
# This file is part of pydicom, released under a modified MIT license.
#    See the file license.txt included with this distribution, also
#    available at http://pydicom.googlecode.com

import sys
import os.path
import os
import unittest
from dicom.filereader import read_file
from dicom.filewriter import write_data_element
from dicom.tag import Tag
from dicom.dataset import Dataset, FileDataset
from dicom.sequence import Sequence
from dicom.util.hexutil import hex2bytes, bytes2hex

# from io import BytesIO
from dicom.filebase import DicomBytesIO
from dicom.dataelem import DataElement
from dicom.util.hexutil import hex2bytes, bytes2hex

from pkg_resources import Requirement, resource_filename
test_dir = resource_filename(Requirement.parse("pydicom"), "dicom/testfiles")

rtplan_name = os.path.join(test_dir, "rtplan.dcm")
rtdose_name = os.path.join(test_dir, "rtdose.dcm")
ct_name = os.path.join(test_dir, "CT_small.dcm")
mr_name = os.path.join(test_dir, "MR_small.dcm")
jpeg_name = os.path.join(test_dir, "JPEG2000.dcm")

# Set up rtplan_out, rtdose_out etc. Filenames as above, with '2' appended
for inname in ['rtplan', 'rtdose', 'ct', 'mr', 'jpeg']:
    exec(inname + "_out = " + inname + "_name + '2'")

def files_identical(a, b):
    """Return a tuple (file a == file b, index of first difference)"""
    a_bytes = file(a, "rb").read()
    b_bytes = file(b, "rb").read()
    return bytes_identical(a_bytes, b_bytes)

def bytes_identical(a_bytes, b_bytes):
    """Return a tuple (bytes a == bytes b, index of first difference)"""
    if a_bytes == b_bytes:
        return True, 0     # True, dummy argument
    else:
        pos = 0
        while a_bytes[pos] == b_bytes[pos]:
            pos += 1
        return False, pos   # False if not identical, position of 1st diff

class WriteFileTests(unittest.TestCase):
    def compare(self, in_filename, out_filename):
        """Read file1, write file2, then compare.
        Return value as for files_identical.
        """
        dataset = read_file(in_filename)
        dataset.save_as(out_filename)
        same, pos = files_identical(in_filename, out_filename)
        self.assertTrue(same,
            "Files are not identical - first difference at 0x%x" % pos)
        if os.path.exists(out_filename):
            os.remove(out_filename)  # get rid of the file

    def testRTPlan(self):
        """Input file, write back and verify them identical (RT Plan file)"""
        self.compare(rtplan_name, rtplan_out)

    def testRTDose(self):
        """Input file, write back and verify them identical (RT Dose file)"""
        self.compare(rtdose_name, rtdose_out)

    def testCT(self):
        """Input file, write back and verify them identical (CT file)....."""
        self.compare(ct_name, ct_out)

    def testMR(self):
        """Input file, write back and verify them identical (MR file)....."""
        self.compare(mr_name, mr_out)

    def testJPEG2000(self):
        """Input file, write back and verify them identical (JPEG2K file)."""
        self.compare(jpeg_name, jpeg_out)

    def testListItemWriteBack(self):
        """Change item in a list and confirm it is written to file      .."""
        DS_expected = 0
        CS_expected = b"new"
        SS_expected = 999
        ds = read_file(ct_name)
        ds.ImagePositionPatient[2] = DS_expected
        ds.ImageType[1] = CS_expected
        ds[(0x0043, 0x1012)].value[0] = SS_expected
        ds.save_as(ct_out)
        # Now read it back in and check that the values were changed
        ds = read_file(ct_out)
        self.assertTrue(ds.ImageType[1] == CS_expected,
                "Item in a list not written correctly to file (VR=CS)")
        self.assertTrue(ds[0x00431012].value[0] == SS_expected,
                "Item in a list not written correctly to file (VR=SS)")
        self.assertTrue(ds.ImagePositionPatient[2] == DS_expected,
                "Item in a list not written correctly to file (VR=DS)")
        if os.path.exists(ct_out):
            os.remove(ct_out)

class WriteDataElementTests(unittest.TestCase):
    """Attempt to write data elements has the expected behaviour"""
    def setUp(self):
        # Create a dummy (in memory) file to write to
        self.f1 = DicomBytesIO()
        self.f1.is_little_endian = True
        self.f1.is_implicit_VR = True

    def test_empty_AT(self):
        """Write empty AT correctly.........."""
        # Was issue 74
        data_elem = DataElement(0x00280009, "AT", [])
        expected = hex2bytes((
            " 28 00 09 00"   # (0028,0009) Frame Increment Pointer
            " 00 00 00 00"   # length 0
            ))
        write_data_element(self.f1, data_elem)
        got = self.f1.parent.getvalue()
        msg = ("Did not write zero-length AT value correctly. "
            "Expected %r, got %r") % (bytes2hex(expected), bytes2hex(got))
        msg = "%r %r" % (type(expected), type(got))
        msg = "'%r' '%r'" % (expected, got)
        self.assertEqual(expected, got, msg)

class ScratchWriteTests(unittest.TestCase):
    """Simple dataset from scratch, written in all endian/VR combinations"""
    def setUp(self):
        # Create simple dataset for all tests
        ds = Dataset()
        ds.PatientName = "Name^Patient"

        # Set up a simple nested sequence
        # first, the innermost sequence
        subitem1 = Dataset()
        subitem1.ContourNumber = 1
        subitem1.ContourData = ['2', '4', '8', '16']
        subitem2 = Dataset()
        subitem2.ContourNumber = 2
        subitem2.ContourData = ['32', '64', '128', '196']

        sub_ds = Dataset()
        sub_ds.ContourSequence = Sequence((subitem1, subitem2))

        # Now the top-level sequence
        ds.ROIContourSequence = Sequence((sub_ds,))  # Comma to make one-tuple

        # Store so each test can use it
        self.ds = ds

    def compare_write(self, hex_std, file_ds):
        """Write file and compare with expected byte string

        :arg hex_std: the bytes which should be written, as space separated hex
        :arg file_ds: a FileDataset instance containing the dataset to write
        """
        out_filename = "scratch.dcm"
        file_ds.save_as(out_filename)
        std = hex2bytes(hex_std)
        bytes_written = open(out_filename, 'rb').read()
        # print "std    :", bytes2hex(std)
        # print "written:", bytes2hex(bytes_written)
        same, pos = bytes_identical(std, bytes_written)
        self.assertTrue(same,
            "Writing from scratch unexpected result - 1st diff at 0x%x" % pos)
        if os.path.exists(out_filename):
            os.remove(out_filename)  # get rid of the file

    def testImpl_LE_deflen_write(self):
        """Scratch Write for implicit VR little endian, defined length SQ's"""
        from dicom.test._write_stds import impl_LE_deflen_std_hex as std

        file_ds = FileDataset("test", self.ds)
        self.compare_write(std, file_ds)

if __name__ == "__main__":
    # This is called if run alone, but not if loaded through run_tests.py
    # If not run from the directory where the sample images are,
    #    then need to switch there
    dir_name = os.path.dirname(sys.argv[0])
    save_dir = os.getcwd()
    if dir_name:
        os.chdir(dir_name)
    os.chdir("../testfiles")
    unittest.main()
    os.chdir(save_dir)
<commit_msg>py3: filewriter unit tests now pass<commit_after># test_filewriter.py
"""unittest cases for dicom.filewriter module"""
# Copyright (c) 2008-2012 Darcy Mason
# This file is part of pydicom, released under a modified MIT license.
#    See the file license.txt included with this distribution, also
#    available at http://pydicom.googlecode.com

import sys
import os.path
import os
import unittest
from dicom.filereader import read_file
from dicom.filewriter import write_data_element
from dicom.tag import Tag
from dicom.dataset import Dataset, FileDataset
from dicom.sequence import Sequence
from dicom.util.hexutil import hex2bytes, bytes2hex

# from io import BytesIO
from dicom.filebase import DicomBytesIO
from dicom.dataelem import DataElement
from dicom.util.hexutil import hex2bytes, bytes2hex

from pkg_resources import Requirement, resource_filename
test_dir = resource_filename(Requirement.parse("pydicom"), "dicom/testfiles")

rtplan_name = os.path.join(test_dir, "rtplan.dcm")
rtdose_name = os.path.join(test_dir, "rtdose.dcm")
ct_name = os.path.join(test_dir, "CT_small.dcm")
mr_name = os.path.join(test_dir, "MR_small.dcm")
jpeg_name = os.path.join(test_dir, "JPEG2000.dcm")

# Set up rtplan_out, rtdose_out etc. Filenames as above, with '2' appended
for inname in ['rtplan', 'rtdose', 'ct', 'mr', 'jpeg']:
    exec(inname + "_out = " + inname + "_name + '2'")

def files_identical(a, b):
    """Return a tuple (file a == file b, index of first difference)"""
    a_bytes = open(a, "rb").read()
    b_bytes = open(b, "rb").read()
    return bytes_identical(a_bytes, b_bytes)

def bytes_identical(a_bytes, b_bytes):
    """Return a tuple (bytes a == bytes b, index of first difference)"""
    if a_bytes == b_bytes:
        return True, 0     # True, dummy argument
    else:
        pos = 0
        while a_bytes[pos] == b_bytes[pos]:
            pos += 1
        return False, pos   # False if not identical, position of 1st diff

class WriteFileTests(unittest.TestCase):
    def compare(self, in_filename, out_filename):
        """Read file1, write file2, then compare.
        Return value as for files_identical.
        """
        dataset = read_file(in_filename)
        dataset.save_as(out_filename)
        same, pos = files_identical(in_filename, out_filename)
        self.assertTrue(same,
            "Files are not identical - first difference at 0x%x" % pos)
        if os.path.exists(out_filename):
            os.remove(out_filename)  # get rid of the file

    def testRTPlan(self):
        """Input file, write back and verify them identical (RT Plan file)"""
        self.compare(rtplan_name, rtplan_out)

    def testRTDose(self):
        """Input file, write back and verify them identical (RT Dose file)"""
        self.compare(rtdose_name, rtdose_out)

    def testCT(self):
        """Input file, write back and verify them identical (CT file)....."""
        self.compare(ct_name, ct_out)

    def testMR(self):
        """Input file, write back and verify them identical (MR file)....."""
        self.compare(mr_name, mr_out)

    def testJPEG2000(self):
        """Input file, write back and verify them identical (JPEG2K file)."""
        self.compare(jpeg_name, jpeg_out)

    def testListItemWriteBack(self):
        """Change item in a list and confirm it is written to file      .."""
        DS_expected = 0
        CS_expected = "new"
        SS_expected = 999
        ds = read_file(ct_name)
        ds.ImagePositionPatient[2] = DS_expected
        ds.ImageType[1] = CS_expected
        ds[(0x0043, 0x1012)].value[0] = SS_expected
        ds.save_as(ct_out)
        # Now read it back in and check that the values were changed
        ds = read_file(ct_out)
        self.assertTrue(ds.ImageType[1] == CS_expected,
                "Item in a list not written correctly to file (VR=CS)")
        self.assertTrue(ds[0x00431012].value[0] == SS_expected,
                "Item in a list not written correctly to file (VR=SS)")
        self.assertTrue(ds.ImagePositionPatient[2] == DS_expected,
                "Item in a list not written correctly to file (VR=DS)")
        if os.path.exists(ct_out):
            os.remove(ct_out)

class WriteDataElementTests(unittest.TestCase):
    """Attempt to write data elements has the expected behaviour"""
    def setUp(self):
        # Create a dummy (in memory) file to write to
        self.f1 = DicomBytesIO()
        self.f1.is_little_endian = True
        self.f1.is_implicit_VR = True

    def test_empty_AT(self):
        """Write empty AT correctly.........."""
        # Was issue 74
        data_elem = DataElement(0x00280009, "AT", [])
        expected = hex2bytes((
            " 28 00 09 00"   # (0028,0009) Frame Increment Pointer
            " 00 00 00 00"   # length 0
            ))
        write_data_element(self.f1, data_elem)
        got = self.f1.parent.getvalue()
        msg = ("Did not write zero-length AT value correctly. "
            "Expected %r, got %r") % (bytes2hex(expected), bytes2hex(got))
        msg = "%r %r" % (type(expected), type(got))
        msg = "'%r' '%r'" % (expected, got)
        self.assertEqual(expected, got, msg)

class ScratchWriteTests(unittest.TestCase):
    """Simple dataset from scratch, written in all endian/VR combinations"""
    def setUp(self):
        # Create simple dataset for all tests
        ds = Dataset()
        ds.PatientName = "Name^Patient"

        # Set up a simple nested sequence
        # first, the innermost sequence
        subitem1 = Dataset()
        subitem1.ContourNumber = 1
        subitem1.ContourData = ['2', '4', '8', '16']
        subitem2 = Dataset()
        subitem2.ContourNumber = 2
        subitem2.ContourData = ['32', '64', '128', '196']

        sub_ds = Dataset()
        sub_ds.ContourSequence = Sequence((subitem1, subitem2))

        # Now the top-level sequence
        ds.ROIContourSequence = Sequence((sub_ds,))  # Comma to make one-tuple

        # Store so each test can use it
        self.ds = ds

    def compare_write(self, hex_std, file_ds):
        """Write file and compare with expected byte string

        :arg hex_std: the bytes which should be written, as space separated hex
        :arg file_ds: a FileDataset instance containing the dataset to write
        """
        out_filename = "scratch.dcm"
        file_ds.save_as(out_filename)
        std = hex2bytes(hex_std)
        bytes_written = open(out_filename, 'rb').read()
        # print "std    :", bytes2hex(std)
        # print "written:", bytes2hex(bytes_written)
        same, pos = bytes_identical(std, bytes_written)
        self.assertTrue(same,
            "Writing from scratch unexpected result - 1st diff at 0x%x" % pos)
        if os.path.exists(out_filename):
            os.remove(out_filename)  # get rid of the file

    def testImpl_LE_deflen_write(self):
        """Scratch Write for implicit VR little endian, defined length SQ's"""
        from dicom.test._write_stds import impl_LE_deflen_std_hex as std

        file_ds = FileDataset("test", self.ds)
        self.compare_write(std, file_ds)

if __name__ == "__main__":
    # This is called if run alone, but not if loaded through run_tests.py
    # If not run from the directory where the sample images are,
    #    then need to switch there
    dir_name = os.path.dirname(sys.argv[0])
    save_dir = os.getcwd()
    if dir_name:
        os.chdir(dir_name)
    os.chdir("../testfiles")
    unittest.main()
    os.chdir(save_dir)
<|endoftext|>


<commit_before>#-------------------------------------------------------------------------------
import shutil
import os
import os.path
import config

#-------------------------------------------------------------------------------

DATADIR = config.DATADIR

FT_UNKNOWN = 0
# Read file (fastq, fq)
FT_FAST = 1
# Compressed Read file (fastq.gz ...)
FT_TGZ_FAST = 2
# Nucleotide seqs database (.fas)
FT_SEQ_DB_NUC = 3
# Amino acids seqs database (.fas)
FT_SEQ_DB_AA = 4
# Assembled transcripts (.fas)
FT_ASSEM = 5
# Mapped transcripts (.bam )
FT_MAP_ASSEM = 6
# Protein hmm for hmmer (.hmm)
FT_HMM = 7

fileExtTable = {
    'fastq': FT_FAST,
    'fq': FT_FAST,
    'fastq.tar.gz': FT_TGZ_FAST,
    'fq.tar.gz': FT_TGZ_FAST,
    'bam': FT_MAP_ASSEM,
    'sam': FT_MAP_ASSEM,
    'hmm': FT_HMM
}

fileOptionTable = {
    'undef' : FT_UNKNOWN,
    'fast' : FT_FAST,
    'tgz_fast' : FT_TGZ_FAST,
    'seq_db_nuc' : FT_SEQ_DB_NUC,
    'seq_db_aa' : FT_SEQ_DB_AA,
    'assem' : FT_ASSEM,
    'map_assem' : FT_MAP_ASSEM,
    'hmm_profile' : FT_HMM
}

#         typo            extension
# 0       Unknown
# 1       fastq           .fastq or .fq
# 2       compressed fq   .fastq.tar.gz or .fq.tar.gz
# 3       seq databases nuc   .fas
# 4       seq db AA       .fas
# 5       assemblies      .fas
# 6       mapped assbl.   .bam or .sam
# 7       hmm profile     .hmm

#-------------------------------------------------------------------------------
def getUserFilename( username, filename ):
    # NOTE EK: NOT SURE THIS IS CORRECT TO CHANGE PATH FOR DATA LIKE THIS
    # BEFORE WAS:     return os.path.join( DATADIR, username, filename )
    return os.path.join( DATADIR, username, "data", filename )

#-------------------------------------------------------------------------------
def saveFile( filename, filedata ):
    dir = os.path.dirname( filename )
    if not os.path.isdir( dir ):
        os.makedirs( dir )

    with open( filename, 'w' ) as f:
        shutil.copyfileobj( filedata, f )

#-------------------------------------------------------------------------------
def linkDemoFile(username, demo_f):
    # Now only work for fastq files (f_type is set to 1)

    dir = os.path.join(config.DATADIR, username, "data")
    if not os.path.isdir( dir ):
        os.makedirs( dir )

    lpath = os.path.join( dir , demo_f )
    fpath = os.path.join( config.DEMO_DIR, demo_f )

    if not os.path.islink(lpath):
        os.symlink( fpath, lpath )

#-------------------------------------------------------------------------------
def clearFilePart( filename ):
    if os.path.isfile( filename + '.part' ):
        os.remove( filename + '.part' )

#-------------------------------------------------------------------------------
def saveFilePart( filename, filedata ):
    dir = os.path.dirname( filename )
    if not os.path.isdir( dir ):
        os.makedirs( dir )

    with open( filename + '.part', 'a' ) as f:
        shutil.copyfileobj( filedata, f )

#-------------------------------------------------------------------------------
def endFilePart( filename ):
    if os.path.isfile( filename + '.part' ):
        shutil.move( filename + '.part', filename )

#-------------------------------------------------------------------------------
def getFileType( filename, option ):
    if option == 'auto':
        lname = filename.lower()
        for k,v in fileExtTable.items():
            if lname.endswith( k ):
                return v
        return FT_UNKNOWN

    return fileOptionTable.get( option.lower(), FT_UNKNOWN )

#-------------------------------------------------------------------------------
<commit_msg>move to use constants<commit_after>#-------------------------------------------------------------------------------
import shutil
import os
import os.path
import config

#-------------------------------------------------------------------------------

DATADIR = config.DATADIR

FT_UNKNOWN = 0
# Read file (fastq, fq)
FT_FAST = 1
# Compressed Read file (fastq.gz ...)
FT_TGZ_FAST = 2
# Nucleotide seqs database (.fas)
FT_SEQ_DB_NUC = 3
# Amino acids seqs database (.fas)
FT_SEQ_DB_AA = 4
# Assembled transcripts (.fas)
FT_ASSEM = 5
# Mapped transcripts (.bam )
FT_MAP_ASSEM = 6
# Protein hmm for hmmer (.hmm)
FT_HMM = 7

fileExtTable = {
    'fastq': FT_FAST,
    'fq': FT_FAST,
    'fastq.tar.gz': FT_TGZ_FAST,
    'fq.tar.gz': FT_TGZ_FAST,
    'bam': FT_MAP_ASSEM,
    'sam': FT_MAP_ASSEM,
    'hmm': FT_HMM
}

fileOptionTable = {
    'undef' : FT_UNKNOWN,
    'fast' : FT_FAST,
    'tgz_fast' : FT_TGZ_FAST,
    'seq_db_nuc' : FT_SEQ_DB_NUC,
    'seq_db_aa' : FT_SEQ_DB_AA,
    'assem' : FT_ASSEM,
    'map_assem' : FT_MAP_ASSEM,
    'hmm_profile' : FT_HMM
}

#         typo            extension
# 0       Unknown
# 1       fastq           .fastq or .fq
# 2       compressed fq   .fastq.tar.gz or .fq.tar.gz
# 3       seq databases nuc   .fas
# 4       seq db AA       .fas
# 5       assemblies      .fas
# 6       mapped assbl.   .bam or .sam
# 7       hmm profile     .hmm

#-------------------------------------------------------------------------------
IO_FOLDER = "data"
PARTIAL_SUFFIX = ".part"

#-------------------------------------------------------------------------------
def getUserFilename( username, filename ):
    return os.path.join( DATADIR, username, IO_FOLDER, filename )

#-------------------------------------------------------------------------------
def saveFile( filename, filedata ):
    dir = os.path.dirname( filename )
    if not os.path.isdir( dir ):
        os.makedirs( dir )

    with open( filename, 'w' ) as f:
        shutil.copyfileobj( filedata, f )

#-------------------------------------------------------------------------------
def linkDemoFile(username, demo_f):
    # Now only work for fastq files (f_type is set to 1)

    dir = os.path.join( config.DATADIR, username, IO_FOLDER )
    if not os.path.isdir( dir ):
        os.makedirs( dir )

    lpath = os.path.join( dir , demo_f )
    fpath = os.path.join( config.DEMO_DIR, demo_f )

    if not os.path.islink(lpath):
        os.symlink( fpath, lpath )

#-------------------------------------------------------------------------------
def clearFilePart( filename ):
    if os.path.isfile( filename + PARTIAL_SUFFIX ):
        os.remove( filename + PARTIAL_SUFFIX )

#-------------------------------------------------------------------------------
def saveFilePart( filename, filedata ):
    dir = os.path.dirname( filename )
    if not os.path.isdir( dir ):
        os.makedirs( dir )

    with open( filename + PARTIAL_SUFFIX, 'a' ) as f:
        shutil.copyfileobj( filedata, f )

#-------------------------------------------------------------------------------
def endFilePart( filename ):
    if os.path.isfile( filename + PARTIAL_SUFFIX ):
        shutil.move( filename + PARTIAL_SUFFIX, filename )

#-------------------------------------------------------------------------------
def getFileType( filename, option ):
    if option == 'auto':
        lname = filename.lower()
        for k,v in fileExtTable.items():
            if lname.endswith( k ):
                return v
        return FT_UNKNOWN

    return fileOptionTable.get( option.lower(), FT_UNKNOWN )

#-------------------------------------------------------------------------------
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: set fileencoding=utf-8 :
#
# Author:   Makoto Shimazu <makoto.shimaz@gmail.com>
# URL:      https://github.com/Pizaid
# License:  2-Clause BSD License
# Created:  2014-08-09
#

import sys
sys.path.append('gen-py')

from Pizaid import ControllerService
from Pizaid.ttypes import *

from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol

try:
    transport = TSocket.TSocket('localhost', 9090)
    transport = TTransport.TBufferedTransport(transport)
    protocol = TBinaryProtocol.TBinaryProtocol(transport)
    client = ControllerService.Client(protocol)
    transport.open()

#    print client.network_get_ipv4()
#    print client.storage_storage_group_list()
#    print client.storage_join("main", "/dev/sda")
#    print client.storage_capacity_kb("main")
#    print client.storage_usage_kb("main")
#    print client.storage_usage_percent("main")
    print client.storage_disk_list("unused")
    print client.storage_disk_id("/disk/sda")
    print client.storage_disk_port("/disk/sda")
    print client.storage_disk_size("/disk/sda")

except Thrift.TException, tx:
    print '%s' % (tx.message)
<commit_msg>rename: fix sampleclient.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: set fileencoding=utf-8 :
#
# Author:   Makoto Shimazu <makoto.shimaz@gmail.com>
# URL:      https://github.com/Pizaid
# License:  2-Clause BSD License
# Created:  2014-08-09
#

import sys
sys.path.append('gen-py')

from Pizaid import ControllerService
from Pizaid.ttypes import *

from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol

try:
    transport = TSocket.TSocket('localhost', 9090)
    transport = TTransport.TBufferedTransport(transport)
    protocol = TBinaryProtocol.TBinaryProtocol(transport)
    client = ControllerService.Client(protocol)
    transport.open()

#    print client.network_get_ipv4()
#    print client.storage_storage_group_list()
#    print client.storage_join("main", "/dev/sda")
#    print client.storage_capacity_kb("main")
#    print client.storage_usage_kb("main")
#    print client.storage_usage_percent("main")
    print client.storage_disk_list("unused")
    print client.storage_disk_id("/dev/sda")
    print client.storage_disk_port("/dev/sda")
    print client.storage_disk_size("/dev/sda")

except Thrift.TException, tx:
    print '%s' % (tx.message)
<|endoftext|>


<commit_before>import json
import random
from pathlib import Path
from pprint import pprint

import numpy as np
from tqdm import tqdm
from utils import sample, split, read_json, read_img
from moviepy.editor import VideoFileClip

DATASET = Path('~/dataset/').expanduser()
DIRS = sorted([x for x in DATASET.iterdir() if x.is_dir()])

TRAIN_DIRS = DIRS[:-1]
VAL_DIRS = DIRS[-1:]

IMAGE_TRAIN = Path('npy/image_train')
IMAGE_VAL = Path('npy/image_val')
WINDOW_TRAIN = Path('npy/window_train')
WINDOW_VAL = Path('npy/window_val')

N_IMAGE_TRAIN = 25000
N_IMAGE_VAL = 5000
N_WINDOW_TRAIN = 25000
N_WINDOW_VAL = 5000
TIMESTEPS = 30

IMAGE_BATCH_SIZE = 40
WINDOW_BATCH_SIZE = 30

def check():
    for folder in DIRS:
        label = read_json(folder / 'label.json')['label']
        label_len = len(label)
        n_frames = len(list((folder / 'frames/').iterdir()))
        assert n_frames == label_len, '{}: {}, {}'.format(
            folder, label_len, n_frames)

def gen_image_npy(video_dirs, target_dir, n_samples):
    x_all = []
    y_all = []
    for video_dir in video_dirs:
        imgs = sorted((video_dir / 'frames/').iterdir())
        label = json.load((video_dir / 'label.json').open())['label']
        x_all.extend(imgs)
        y_all.extend(label)

    x_use, y_use = sample(x_all, y_all, k=n_samples)

    parts = split(x_use, y_use, k=1000)
    for idx, x_part, y_part in tqdm(parts):
        n = len(x_part)
        xs = np.zeros((n, 224, 224, 3), dtype=np.float32)
        ys = np.zeros((n, 1), dtype=np.uint8)
        for i in range(n):
            xs[i] = read_img(x_part[i])
            ys[i] = y_part[i]
        np.save(str(target_dir / 'x_{:05d}.npy'.format(idx)), xs)
        np.save(str(target_dir / 'y_{:05d}.npy'.format(idx)), ys)
        del xs, ys

def gen_window_npy(video_dirs, target_dir, n_samples):
    x_all = []
    y_all = []
    for video_dir in video_dirs:
        n_frames = len(list((video_dir / 'frames/').iterdir()))
        labels = read_json(video_dir / 'label.json')['label']
        windows = [(video_dir, i, i + TIMESTEPS)
                   for i in range(n_frames - TIMESTEPS)]
        x_all.extend(windows)
        y_all.extend([labels[e - 1] for (_, s, e) in windows])

    x_use, y_use = sample(x_all, y_all, k=n_samples)

    parts = split(x_use, y_use, k=200)
    for idx, x_part, y_part in tqdm(parts):
        n = len(x_part)
        xs = np.zeros((n, TIMESTEPS, 224, 224, 3), dtype=np.float32)
        ys = np.zeros((n, 1), dtype=np.uint8)
        for i in range(n):
            (video_dir, s, e) = x_part[i]
            for f in range(s, e):
                path = video_dir / 'frames' / '{:08d}.jpg'.format(f)
                xs[i][f - s] = read_img(path)
            ys[i] = y_part[i]

        np.save(str(target_dir / 'x_{:05d}.npy'.format(idx)), xs)
        np.save(str(target_dir / 'y_{:05d}.npy'.format(idx)), ys)
        del xs, ys

def image_generator(npy_dir, batch_size):
    x_paths = sorted(npy_dir.glob('x_*.npy'))
    y_paths = sorted(npy_dir.glob('y_*.npy'))

    idx = 0
    x_batch = np.zeros((batch_size, 224, 224, 3), dtype=np.float32)
    y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

    while True:
        for x_path, y_path in zip(x_paths, y_paths):
            x_part = np.load(x_path)
            y_part = np.load(y_path)
            for x, y in zip(x_part, y_part):
                x_batch[idx] = x
                y_batch[idx] = y
                if idx + 1 == batch_size:
                    yield x_batch, y_batch
                idx = (idx + 1) % batch_size
            del x_part, y_part

def window_generator(npy_dir, batch_size):
    x_paths = sorted(npy_dir.glob('x_*.npy'))
    y_paths = sorted(npy_dir.glob('y_*.npy'))

    idx = 0
    x_batch = np.zeros((batch_size, TIMESTEPS, 224, 224, 3), dtype=np.float32)
    y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

    while True:
        for x_path, y_path in zip(x_paths, y_paths):
            x_part = np.load(x_path)
            y_part = np.load(y_path)
            for x, y in zip(x_part, y_part):
                x_batch[idx] = x
                y_batch[idx] = y
                if idx + 1 == batch_size:
                    yield x_batch, y_batch
                idx = (idx + 1) % batch_size
            del x_part, y_part

def window_generator_online(video_dirs, n_samples, batch_size):
    x_all = []
    y_all = []
    for video_dir in video_dirs:
        n_frames = len(list((video_dir / 'frames/').iterdir()))
        labels = read_json(video_dir / 'label.json')['label']
        windows = [(video_dir, i, i + TIMESTEPS)
                   for i in range(n_frames - TIMESTEPS)]
        x_all.extend(windows)
        y_all.extend([labels[e - 1] for (_, s, e) in windows])

    x_use, y_use = sample(x_all, y_all, k=n_samples)

    idx = 0
    x_batch = np.zeros((batch_size, TIMESTEPS, 224, 224, 3), dtype=np.float32)
    y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

    while True:
        for (video_dir, s, e), (label) in zip(x_use, y_use):
            for i in range(e - s):
                img_path = video_dir / 'frames' / '{:08d}.jpg'.format(s + i)
                x_batch[idx][i] = read_img(img_path)
            y_batch[idx] = label

            if idx + 1 == batch_size:
                yield x_batch, y_batch
            idx = (idx + 1) % batch_size

image_train_gen = image_generator(IMAGE_TRAIN, 40)
image_val_gen = image_generator(IMAGE_VAL, 40)
window_train_gen_online = window_generator(TRAIN_DIRS, N_WINDOW_TRAIN, WINDOW_BATCH_SIZE)
window_val_gen_online = window_generator(VAL_DIRS, N_WINDOW_VAL, WINDOW_BATCH_SIZE)

if __name__ == '__main__':
    check()

    for folder in [IMAGE_TRAIN, IMAGE_VAL, WINDOW_TRAIN, WINDOW_VAL]:
        folder.mkdir(parents=True, exist_ok=True)

    print('Train data:')
    pprint(TRAIN_DIRS)
    print('Validation data:')
    pprint(VAL_DIRS)

    gen_image_npy(TRAIN_DIRS, IMAGE_TRAIN, N_IMAGE_TRAIN)
    gen_image_npy(VAL_DIRS, IMAGE_VAL, N_IMAGE_VAL)
    # gen_window_npy(TRAIN_DIRS, WINDOW_TRAIN, N_WINDOW_TRAIN)
    # gen_window_npy(VAL_DIRS, WINDOW_VAL, N_WINDOW_VAL)
<commit_msg>Update at 2017-07-20 12-36-42<commit_after>import json
import random
from pathlib import Path
from pprint import pprint

import numpy as np
from tqdm import tqdm
from utils import sample, split, read_json, read_img
from moviepy.editor import VideoFileClip

DATASET = Path('~/dataset/').expanduser()
DIRS = sorted([x for x in DATASET.iterdir() if x.is_dir()])

TRAIN_DIRS = DIRS[:-1]
VAL_DIRS = DIRS[-1:]

IMAGE_TRAIN = Path('npy/image_train')
IMAGE_VAL = Path('npy/image_val')
WINDOW_TRAIN = Path('npy/window_train')
WINDOW_VAL = Path('npy/window_val')

N_IMAGE_TRAIN = 25000
N_IMAGE_VAL = 5000
N_WINDOW_TRAIN = 25000
N_WINDOW_VAL = 5000
TIMESTEPS = 30

IMAGE_BATCH_SIZE = 40
WINDOW_BATCH_SIZE = 30

def check():
    for folder in DIRS:
        label = read_json(folder / 'label.json')['label']
        label_len = len(label)
        n_frames = len(list((folder / 'frames/').iterdir()))
        assert n_frames == label_len, '{}: {}, {}'.format(
            folder, label_len, n_frames)

def gen_image_npy(video_dirs, target_dir, n_samples):
    x_all = []
    y_all = []
    for video_dir in video_dirs:
        imgs = sorted((video_dir / 'frames/').iterdir())
        label = json.load((video_dir / 'label.json').open())['label']
        x_all.extend(imgs)
        y_all.extend(label)

    x_use, y_use = sample(x_all, y_all, k=n_samples)

    parts = split(x_use, y_use, k=1000)
    for idx, x_part, y_part in tqdm(parts):
        n = len(x_part)
        xs = np.zeros((n, 224, 224, 3), dtype=np.float32)
        ys = np.zeros((n, 1), dtype=np.uint8)
        for i in range(n):
            xs[i] = read_img(x_part[i])
            ys[i] = y_part[i]
        np.save(str(target_dir / 'x_{:05d}.npy'.format(idx)), xs)
        np.save(str(target_dir / 'y_{:05d}.npy'.format(idx)), ys)
        del xs, ys

def gen_window_npy(video_dirs, target_dir, n_samples):
    x_all = []
    y_all = []
    for video_dir in video_dirs:
        n_frames = len(list((video_dir / 'frames/').iterdir()))
        labels = read_json(video_dir / 'label.json')['label']
        windows = [(video_dir, i, i + TIMESTEPS)
                   for i in range(n_frames - TIMESTEPS)]
        x_all.extend(windows)
        y_all.extend([labels[e - 1] for (_, s, e) in windows])

    x_use, y_use = sample(x_all, y_all, k=n_samples)

    parts = split(x_use, y_use, k=200)
    for idx, x_part, y_part in tqdm(parts):
        n = len(x_part)
        xs = np.zeros((n, TIMESTEPS, 224, 224, 3), dtype=np.float32)
        ys = np.zeros((n, 1), dtype=np.uint8)
        for i in range(n):
            (video_dir, s, e) = x_part[i]
            for f in range(s, e):
                path = video_dir / 'frames' / '{:08d}.jpg'.format(f)
                xs[i][f - s] = read_img(path)
            ys[i] = y_part[i]

        np.save(str(target_dir / 'x_{:05d}.npy'.format(idx)), xs)
        np.save(str(target_dir / 'y_{:05d}.npy'.format(idx)), ys)
        del xs, ys

def image_generator(npy_dir, batch_size):
    x_paths = sorted(npy_dir.glob('x_*.npy'))
    y_paths = sorted(npy_dir.glob('y_*.npy'))

    idx = 0
    x_batch = np.zeros((batch_size, 224, 224, 3), dtype=np.float32)
    y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

    while True:
        for x_path, y_path in zip(x_paths, y_paths):
            x_part = np.load(x_path)
            y_part = np.load(y_path)
            for x, y in zip(x_part, y_part):
                x_batch[idx] = x
                y_batch[idx] = y
                if idx + 1 == batch_size:
                    yield x_batch, y_batch
                idx = (idx + 1) % batch_size
            del x_part, y_part

def window_generator(npy_dir, batch_size):
    x_paths = sorted(npy_dir.glob('x_*.npy'))
    y_paths = sorted(npy_dir.glob('y_*.npy'))

    idx = 0
    x_batch = np.zeros((batch_size, TIMESTEPS, 224, 224, 3), dtype=np.float32)
    y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

    while True:
        for x_path, y_path in zip(x_paths, y_paths):
            x_part = np.load(x_path)
            y_part = np.load(y_path)
            for x, y in zip(x_part, y_part):
                x_batch[idx] = x
                y_batch[idx] = y
                if idx + 1 == batch_size:
                    yield x_batch, y_batch
                idx = (idx + 1) % batch_size
            del x_part, y_part

def window_generator_online(video_dirs, n_samples, batch_size):
    x_all = []
    y_all = []
    for video_dir in video_dirs:
        n_frames = len(list((video_dir / 'frames/').iterdir()))
        labels = read_json(video_dir / 'label.json')['label']
        windows = [(video_dir, i, i + TIMESTEPS)
                   for i in range(n_frames - TIMESTEPS)]
        x_all.extend(windows)
        y_all.extend([labels[e - 1] for (_, s, e) in windows])

    x_use, y_use = sample(x_all, y_all, k=n_samples)

    idx = 0
    x_batch = np.zeros((batch_size, TIMESTEPS, 224, 224, 3), dtype=np.float32)
    y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

    while True:
        for (video_dir, s, e), (label) in zip(x_use, y_use):
            for i in range(e - s):
                img_path = video_dir / 'frames' / '{:08d}.jpg'.format(s + i)
                x_batch[idx][i] = read_img(img_path)
            y_batch[idx] = label

            if idx + 1 == batch_size:
                yield x_batch, y_batch
            idx = (idx + 1) % batch_size

image_train_gen = image_generator(IMAGE_TRAIN, 40)
image_val_gen = image_generator(IMAGE_VAL, 40)
window_train_gen = window_generator(WINDOW_TRAIN, WINDOW_BATCH_SIZE)
window_val_gen = window_generator(WINDOW_VAL, WINDOW_BATCH_SIZE)
window_train_gen_online = window_generator_online(TRAIN_DIRS, N_WINDOW_TRAIN, WINDOW_BATCH_SIZE)
window_val_gen_online = window_generator_online(VAL_DIRS, N_WINDOW_VAL, WINDOW_BATCH_SIZE)

if __name__ == '__main__':
    check()

    for folder in [IMAGE_TRAIN, IMAGE_VAL, WINDOW_TRAIN, WINDOW_VAL]:
        folder.mkdir(parents=True, exist_ok=True)

    print('Train data:')
    pprint(TRAIN_DIRS)
    print('Validation data:')
    pprint(VAL_DIRS)

    gen_image_npy(TRAIN_DIRS, IMAGE_TRAIN, N_IMAGE_TRAIN)
    gen_image_npy(VAL_DIRS, IMAGE_VAL, N_IMAGE_VAL)
    # gen_window_npy(TRAIN_DIRS, WINDOW_TRAIN, N_WINDOW_TRAIN)
    # gen_window_npy(VAL_DIRS, WINDOW_VAL, N_WINDOW_VAL)
<|endoftext|>


<commit_before>import unittest

import numpy
import six

import chainer
from chainer import cuda
import chainer.functions as F
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition

class _TestMatMul(unittest.TestCase):

    def check_forward(self, x1_data, x2_data):
        x1 = chainer.Variable(x1_data)
        x2 = chainer.Variable(x2_data)
        y = self.op(x1, x2)
        testing.assert_allclose(self.forward_answer, y.data)

    @condition.retry(3)
    def test_matmul_forward_cpu(self):
        self.check_forward(self.x1, self.x2)

    @attr.gpu
    @condition.retry(3)
    def test_matmul_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x1), cuda.to_gpu(self.x2))

    def check_backward(self, x1_data, x2_data, y_grad, atol, rtol):
        gradient_check.check_backward(
            self.op, (x1_data, x2_data), y_grad, atol=atol, rtol=rtol,
            dtype=numpy.float32)

    @condition.retry(3)
    def test_matmul_backward_cpu(self):
        self.check_backward(self.x1, self.x2, self.gy, atol=1e-2, rtol=5e-2)

    @attr.gpu
    @condition.retry(3)
    def test_matmul_backward_gpu(self):
        self.check_backward(
            cuda.to_gpu(self.x1), cuda.to_gpu(self.x2),
            cuda.to_gpu(self.gy), atol=1e-2)

m = 2
k = 5
n = 10

class TestMatMulMatrixMatrix(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (m, k)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(.5, 1, (k, n)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, (m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.matmul(x, y)
        self.forward_answer = numpy.dot(self.x1, self.x2)

class TestMatMulMatrixMatrixFP16(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (m, k)).astype(numpy.float16)
        self.x2 = numpy.random.uniform(.5, 1, (k, n)).astype(numpy.float16)
        self.gy = numpy.random.uniform(-1, 1, (m, n)).astype(numpy.float16)
        self.op = lambda x, y: F.matmul(x, y)
        self.forward_answer = numpy.dot(self.x1, self.x2)

class TestMatMulMatrixMatrixFP64(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (m, k)).astype(numpy.float64)
        self.x2 = numpy.random.uniform(.5, 1, (k, n)).astype(numpy.float64)
        self.gy = numpy.random.uniform(-1, 1, (m, n)).astype(numpy.float64)
        self.op = lambda x, y: F.matmul(x, y)
        self.forward_answer = numpy.dot(self.x1, self.x2)

class TestMatMulMatrixTMatrix(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (k, m)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(.5, 1, (k, n)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, (m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.matmul(x, y, transa=True)
        self.forward_answer = numpy.dot(self.x1.T, self.x2)

class TestMatMulMatrixMatrixT(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (m, k)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(.5, 1, (n, k)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, (m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.matmul(x, y, transb=True)
        self.forward_answer = numpy.dot(self.x1, self.x2.T)

class TestMatMulMatrixTMatrixT(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (k, m)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(.5, 1, (n, k)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, (m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.matmul(x, y, transa=True, transb=True)
        self.forward_answer = numpy.dot(self.x1.T, self.x2.T)

class TestMatMulVectorTVector(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (m,)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(.5, 1, (m,)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, (1, 1)).astype(numpy.float32)
        self.op = lambda x, y: F.matmul(x, y, transa=True)
        self.forward_answer = numpy.dot(self.x1, self.x2).reshape(1, 1)

class TestMatMulVectorVectorT(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (m,)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(.5, 1, (m,)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, (m, m)).astype(numpy.float32)
        self.op = lambda x, y: F.matmul(x, y, transb=True)
        self.forward_answer = numpy.dot(
            self.x1.reshape(m, 1), self.x2.reshape(1, m))

batch_size = 10

class TestBatchMatMulMatrixMatrix(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, m, k)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (batch_size, k, n)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(x, y)
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i], self.x2[i])
            for i in six.moves.range(batch_size)])

class TestBatchMatMulMatrixTMatrix(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, k, m)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (batch_size, k, n)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(x, y, transa=True)
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i].T, self.x2[i])
            for i in six.moves.range(batch_size)])

class TestBatchMatMulMatrixMatrixT(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, m, k)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (batch_size, n, k)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(x, y, transb=True)
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i], self.x2[i].T)
            for i in six.moves.range(batch_size)])

class TestBatchMatMulMatrixTMatrixT(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, k, m)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (batch_size, n, k)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(x, y, transa=True, transb=True)
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i].T, self.x2[i].T)
            for i in six.moves.range(batch_size)])

class TestBatchMatMulVectorTVector(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, m,)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (batch_size, m,)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, 1, 1)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(x, y, transa=True)
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i], self.x2[i])
            for i in six.moves.range(batch_size)]).reshape(batch_size, 1, 1)

class TestBatchMatMulVectorVectorT(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, m,)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (batch_size, m,)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, m, m)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(x, y, transb=True)
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i].reshape(m, 1), self.x2[i].reshape(1, m))
            for i in six.moves.range(batch_size)])

class TestBatchMatMulMatrixMatrixBatchSize1(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (1, m, k)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (1, k, n)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (1, m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(x, y)
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i], self.x2[i])
            for i in six.moves.range(1)])

class TestBatchMatMulBroadcastedMatrix1(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, m, k)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (1, k, n)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(
            x, F.broadcast_to(y, (batch_size, k, n)))
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i], self.x2[0])
            for i in six.moves.range(batch_size)])

class TestBatchMatMulBroadcastedMatrix2(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, m, k)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (k, n)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(
            x, F.broadcast_to(F.expand_dims(y, 0), (batch_size, k, n)))
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i], self.x2)
            for i in six.moves.range(batch_size)])

testing.run_module(__name__, __file__)
<commit_msg>Fix matmul test<commit_after>import unittest

import numpy
import six

import chainer
from chainer import cuda
import chainer.functions as F
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition

class _TestMatMul(unittest.TestCase):

    def check_forward(self, x1_data, x2_data):
        x1 = chainer.Variable(x1_data)
        x2 = chainer.Variable(x2_data)
        y = self.op(x1, x2)
        testing.assert_allclose(self.forward_answer, y.data)

    @condition.retry(3)
    def test_matmul_forward_cpu(self):
        self.check_forward(self.x1, self.x2)

    @attr.gpu
    @condition.retry(3)
    def test_matmul_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x1), cuda.to_gpu(self.x2))

    def check_backward(self, x1_data, x2_data, y_grad, atol, rtol):
        gradient_check.check_backward(
            self.op, (x1_data, x2_data), y_grad, atol=atol, rtol=rtol,
            dtype=numpy.float32)

    @condition.retry(3)
    def test_matmul_backward_cpu(self):
        self.check_backward(self.x1, self.x2, self.gy, atol=1e-2, rtol=5e-2)

    @attr.gpu
    @condition.retry(3)
    def test_matmul_backward_gpu(self):
        self.check_backward(
            cuda.to_gpu(self.x1), cuda.to_gpu(self.x2),
            cuda.to_gpu(self.gy), atol=1e-2, rtol=1e-2)

m = 2
k = 5
n = 10

class TestMatMulMatrixMatrix(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (m, k)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(.5, 1, (k, n)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, (m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.matmul(x, y)
        self.forward_answer = numpy.dot(self.x1, self.x2)

class TestMatMulMatrixMatrixFP16(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (m, k)).astype(numpy.float16)
        self.x2 = numpy.random.uniform(.5, 1, (k, n)).astype(numpy.float16)
        self.gy = numpy.random.uniform(-1, 1, (m, n)).astype(numpy.float16)
        self.op = lambda x, y: F.matmul(x, y)
        self.forward_answer = numpy.dot(self.x1, self.x2)

class TestMatMulMatrixMatrixFP64(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (m, k)).astype(numpy.float64)
        self.x2 = numpy.random.uniform(.5, 1, (k, n)).astype(numpy.float64)
        self.gy = numpy.random.uniform(-1, 1, (m, n)).astype(numpy.float64)
        self.op = lambda x, y: F.matmul(x, y)
        self.forward_answer = numpy.dot(self.x1, self.x2)

class TestMatMulMatrixTMatrix(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (k, m)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(.5, 1, (k, n)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, (m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.matmul(x, y, transa=True)
        self.forward_answer = numpy.dot(self.x1.T, self.x2)

class TestMatMulMatrixMatrixT(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (m, k)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(.5, 1, (n, k)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, (m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.matmul(x, y, transb=True)
        self.forward_answer = numpy.dot(self.x1, self.x2.T)

class TestMatMulMatrixTMatrixT(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (k, m)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(.5, 1, (n, k)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, (m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.matmul(x, y, transa=True, transb=True)
        self.forward_answer = numpy.dot(self.x1.T, self.x2.T)

class TestMatMulVectorTVector(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (m,)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(.5, 1, (m,)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, (1, 1)).astype(numpy.float32)
        self.op = lambda x, y: F.matmul(x, y, transa=True)
        self.forward_answer = numpy.dot(self.x1, self.x2).reshape(1, 1)

class TestMatMulVectorVectorT(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(.5, 1, (m,)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(.5, 1, (m,)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, (m, m)).astype(numpy.float32)
        self.op = lambda x, y: F.matmul(x, y, transb=True)
        self.forward_answer = numpy.dot(
            self.x1.reshape(m, 1), self.x2.reshape(1, m))

batch_size = 10

class TestBatchMatMulMatrixMatrix(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, m, k)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (batch_size, k, n)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(x, y)
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i], self.x2[i])
            for i in six.moves.range(batch_size)])

class TestBatchMatMulMatrixTMatrix(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, k, m)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (batch_size, k, n)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(x, y, transa=True)
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i].T, self.x2[i])
            for i in six.moves.range(batch_size)])

class TestBatchMatMulMatrixMatrixT(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, m, k)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (batch_size, n, k)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(x, y, transb=True)
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i], self.x2[i].T)
            for i in six.moves.range(batch_size)])

class TestBatchMatMulMatrixTMatrixT(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, k, m)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (batch_size, n, k)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(x, y, transa=True, transb=True)
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i].T, self.x2[i].T)
            for i in six.moves.range(batch_size)])

class TestBatchMatMulVectorTVector(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, m,)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (batch_size, m,)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, 1, 1)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(x, y, transa=True)
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i], self.x2[i])
            for i in six.moves.range(batch_size)]).reshape(batch_size, 1, 1)

class TestBatchMatMulVectorVectorT(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, m,)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (batch_size, m,)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, m, m)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(x, y, transb=True)
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i].reshape(m, 1), self.x2[i].reshape(1, m))
            for i in six.moves.range(batch_size)])

class TestBatchMatMulMatrixMatrixBatchSize1(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (1, m, k)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (1, k, n)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (1, m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(x, y)
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i], self.x2[i])
            for i in six.moves.range(1)])

class TestBatchMatMulBroadcastedMatrix1(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, m, k)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (1, k, n)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(
            x, F.broadcast_to(y, (batch_size, k, n)))
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i], self.x2[0])
            for i in six.moves.range(batch_size)])

class TestBatchMatMulBroadcastedMatrix2(_TestMatMul):

    def setUp(self):
        self.x1 = numpy.random.uniform(
            .5, 1, (batch_size, m, k)).astype(numpy.float32)
        self.x2 = numpy.random.uniform(
            .5, 1, (k, n)).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (batch_size, m, n)).astype(numpy.float32)
        self.op = lambda x, y: F.batch_matmul(
            x, F.broadcast_to(F.expand_dims(y, 0), (batch_size, k, n)))
        self.forward_answer = numpy.array([
            numpy.dot(self.x1[i], self.x2)
            for i in six.moves.range(batch_size)])

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>import requests
from requests_oauthlib import OAuth1

import json

from credentials import *

url = 'https://api.twitter.com/1.1/search/tweets.json?f=tweets&q=to%3Abbt_chart%20from%3Alorenaelise'

consumer_key = CONSUMER_KEY
consumer_secret = CONSUMER_SECRET
access_token = ACCESS_TOKEN
access_token_secret = ACCESS_TOKEN_SECRET

auth = OAuth1(consumer_key, consumer_secret, access_token, access_token_secret)

r = requests.get(url, auth=auth)
data = r.json()

tweet_data = data['statuses']
tweet_count = len(tweet_data)

temps = []
dates = []

def get_temp(tweet):
    temp = filter(lambda x: x.isdigit(), tweet)
    temp = float(temp) / 100
    return temp

get_temp(tweet_data[0]['text'])

# may have to reverse lists to display correctly - 0 is newest tweet and should be oldest

# date is 'created_at'

# change to list comprehension
# for i in range(tweet_count):
#     print tweet_data[i]['text']

# dump data for viewing
# with open('data.json', 'w') as f:
#     json.dump(data, f)
<commit_msg>changed function to return temp<commit_after>import requests
from requests_oauthlib import OAuth1

import json

from credentials import *

url = 'https://api.twitter.com/1.1/search/tweets.json?f=tweets&q=to%3Abbt_chart%20from%3Alorenaelise'

consumer_key = CONSUMER_KEY
consumer_secret = CONSUMER_SECRET
access_token = ACCESS_TOKEN
access_token_secret = ACCESS_TOKEN_SECRET

auth = OAuth1(consumer_key, consumer_secret, access_token, access_token_secret)

r = requests.get(url, auth=auth)
data = r.json()

tweet_data = data['statuses']
tweet_count = len(tweet_data)

dates = []

def get_temp(tweet):
    temp = filter(lambda x: x.isdigit(), tweet)
    temp = float(temp) / 100
    return temp

temps = list(get_temp(tweet_data[0]['text']))
print temps

# may have to reverse lists to display correctly - 0 is newest tweet and should be oldest

# date is 'created_at'

# change to list comprehension
# for i in range(tweet_count):
#     print tweet_data[i]['text']

# dump data for viewing
# with open('data.json', 'w') as f:
#     json.dump(data, f)
<|endoftext|>


<commit_before>import logging

import salt.transport.tcp
import salt.utils.msgpack
from tornado import gen
from tornado.iostream import StreamClosedError
from tornado.tcpserver import TCPServer

log = logging.getLogger(__name__)

async def test_message_client_reconnect(io_loop):
    """
    Verify that the tcp MessageClient class re-sets it's unpacker after a
    stream disconnect.
    """
    config = {
        "master_ip": "127.0.0.1",
        "publish_port": 5679,
    }

    class TestServer(TCPServer):
        send = []
        stop = False

        async def handle_stream(self, stream, address):
            while self.stop is False:
                for msg in self.send[:]:
                    msg = self.send.pop(0)
                    try:
                        await stream.write(msg)
                    except StreamClosedError:
                        break
                else:
                    await gen.sleep(1)
            stream.close()

    received = []
    server = TestServer()
    server.listen(config["publish_port"])

    client = salt.transport.tcp.TCPPubClient(config.copy(), io_loop)
    await client.connect(config["publish_port"])

    def handler(msg):
        received.append(msg)

    client.on_recv(handler)

    # Prepare two packed messages
    msg = salt.utils.msgpack.dumps({"test": "test1"})
    pmsg = salt.utils.msgpack.dumps({"head": {}, "body": msg})
    assert len(pmsg) == 26
    pmsg += salt.utils.msgpack.dumps({"head": {}, "body": msg})

    # Send one full and one partial msg to the client.
    partial = pmsg[:40]
    server.send.append(partial)

    while not received:
        await gen.sleep(1)
    assert received == [msg]

    # The message client has unpacked one msg and there is a partial msg left in
    # the unpacker. Closing the stream now leaves the unpacker in a bad state
    # since the rest of the partil message will never be received.
    server.stop = True
    await gen.sleep(1)
    server.stop = False
    received = []

    # Prior to the fix for #60831, the unpacker would be left in a broken state
    # resulting in either a TypeError or BufferFull error from msgpack. The
    # rest of this test would fail.
    server.send.append(pmsg)
    while not received:
        await gen.sleep(1)
    assert received == [msg, msg]
    server.stop = True
<commit_msg>clean up tests<commit_after>import logging

import pytest
import salt.ext.tornado.gen
import salt.ext.tornado.iostream
import salt.ext.tornado.tcpserver
import salt.transport.tcp
import salt.utils.msgpack

log = logging.getLogger(__name__)

@pytest.fixture
def config():
    yield {
        "master_ip": "127.0.0.1",
        "publish_port": 5679,
    }

@pytest.fixture
def server(config):
    class TestServer(salt.ext.tornado.tcpserver.TCPServer):
        send = []
        disconnect = False

        async def handle_stream(self, stream, address):
            while self.disconnect is False:
                for msg in self.send[:]:
                    msg = self.send.pop(0)
                    try:
                        await stream.write(msg)
                    except salt.ext.tornado.iostream.StreamClosedError:
                        break
                else:
                    await salt.ext.tornado.gen.sleep(1)
            stream.close()

    server = TestServer()
    try:
        yield server
    finally:
        server.disconnect = True
        server.stop()

@pytest.fixture
def client(io_loop, config):
    client = salt.transport.tcp.TCPPubClient(config.copy(), io_loop)
    try:
        yield client
    finally:
        client.close()

async def test_message_client_reconnect(io_loop, config, client, server):
    """
    Verify that the tcp MessageClient class re-sets it's unpacker after a
    stream disconnect.
    """

    server.listen(config["publish_port"])
    await client.connect(config["publish_port"])

    received = []

    def handler(msg):
        received.append(msg)

    client.on_recv(handler)

    # Prepare two packed messages
    msg = salt.utils.msgpack.dumps({"test": "test1"})
    pmsg = salt.utils.msgpack.dumps({"head": {}, "body": msg})
    assert len(pmsg) == 26
    pmsg += salt.utils.msgpack.dumps({"head": {}, "body": msg})

    # Send one full and one partial msg to the client.
    partial = pmsg[:40]
    server.send.append(partial)

    while not received:
        await salt.ext.tornado.gen.sleep(1)
    assert received == [msg]

    # The message client has unpacked one msg and there is a partial msg left in
    # the unpacker. Closing the stream now leaves the unpacker in a bad state
    # since the rest of the partil message will never be received.
    server.disconnect = True
    await salt.ext.tornado.gen.sleep(1)
    server.disconnect = False
    received = []

    # Prior to the fix for #60831, the unpacker would be left in a broken state
    # resulting in either a TypeError or BufferFull error from msgpack. The
    # rest of this test would fail.
    server.send.append(pmsg)
    while not received:
        await salt.ext.tornado.gen.sleep(1)
    assert received == [msg, msg]
<|endoftext|>


<commit_before>import dbus
import os

import unittest
from time import sleep

import sys
sys.path.append("./python/")
import ContextProvider

# FIXME: Use the fake dbus.

# FIXME: Missing testcases
# - Using system bus flag in init
# - Test that callbacks work

class LibraryTestCase(unittest.TestCase):
    def setUp(self):
        pass
    def tearDown(self):
        print "LibraryTestCase tearDown"
        pass

class Startup(LibraryTestCase):
    def setUp(self):
        self.bus = dbus.SessionBus() # Note: using session bus

        # Start a listener which listens to dbus NameOwnerChanged signals
        os.system("python tests/python-test-library/stubs/dbus_listener.py &")
        sleep(0.5)
        # Command the listener to listen to FakeProvider
        self.listener_proxy = self.bus.get_object("org.freedesktop.ContextKit.Testing.Listener","/org/freedesktop/ContextKit/Testing/Listener")
        self.listener_iface = dbus.Interface(self.listener_proxy, "org.freedesktop.ContextKit.Testing.Listener")
        self.listener_iface.SetTarget("org.freedesktop.ContextKit.Testing.Provider")

        # Start a provider stub
        os.system("python tests/python-test-library/stubs/provider_stub.py &")
        sleep(0.5)
        # Command the provider stub to start exposing services over dbus
        self.provider_proxy = self.bus.get_object("org.freedesktop.ContextKit.Testing.Provider","/org/freedesktop/ContextKit/Testing/Provider")
        self.provider_iface = dbus.Interface(self.provider_proxy, "org.freedesktop.ContextKit.Testing.Provider")
        self.provider_iface.DoInit()
        sleep(0.5)

    def tearDown(self):
        print "Startup tearDown"
        # Stop the provider
        self.provider_iface.Exit()
        sleep(0.5)

        # Stop the listener
        self.listener_iface.Exit()
        pass # FIXME: Call parent class func explicitly

    def test_startProvider(self):
        # Verify that the provider is started and exposed over dbus.

        # Get the log from the listener
        log = self.listener_iface.GetLog()
        self.assert_(log == "(TargetAppeared)")

        # Note: we could also test whether disappearing is noticed
        # but that is dropped for simplicity.

    def test_managerInterfaceImplemented(self):
        # Verify that the provider exposes the Manager interface over dbus properly

        # Try to get the manager object
        proxy_object_manager = self.bus.get_object("org.freedesktop.ContextKit.Testing.Provider","/org/freedesktop/ContextKit/Manager")
        iface_manager = dbus.Interface(proxy_object_manager, "org.freedesktop.ContextKit.Manager")
        self.assert_ (proxy_object_manager != None) # FIXME: What is the correct validity check?
        self.assert_ (iface_manager != None) # FIXME: What is the correct validity check?
        try:
            iface_manager.Get("temp")
        except:
            print "Exception caught"
            self.assert_ (False) # Manager interface is not implemented properly
            # After this, only tearDown will be executed

class Subscription(LibraryTestCase):
    def setUp(self):
        # Start a provider stub
        os.system("python tests/python-test-library/stubs/provider_stub.py &")
        sleep(0.5)
        # Command the provider stub to start exposing services over dbus
        bus = dbus.SessionBus()
        self.provider_proxy = bus.get_object("org.freedesktop.ContextKit.Testing.Provider","/org/freedesktop/ContextKit/Testing/Provider")
        self.provider_iface = dbus.Interface(self.provider_proxy, "org.freedesktop.ContextKit.Testing.Provider")
        self.provider_iface.DoInit()
        sleep(0.5)

    def tearDown(self):
        print "Subscription tearDown"
        self.provider_iface.Exit()
        # FIXME: Try to ensure that the stubs are killed properly
        pass # FIXME: Call parent class func explicitly

    def test_subscriberCallbacks(self):
        bus = dbus.SessionBus()
        # Start a subscriber
        #os.system("python tests/python-test-library/stubs/subscriber_stub.py &")
        #sleep(1)

        #subscriber_proxy = bus.get_object("org.freedesktop.ContextKit.Testing.SubscriberStub","/org/freedesktop/ContextKit/Testing/SubscriberStub")
        #subscriber_iface = dbus.Interface(subscriber_proxy, "org.freedesktop.ContextKit.Testing.SubscriberStub")
        #subscriber_iface.DoSubscribe()

        # Verify that the provider got FirstSubscribed event
        # Stop the subscriber

        # Verify that the provider got LastSubscribed event

class ChangeSets(LibraryTestCase):
    def test_createChangeSet(self):
        # FIXME: This functionality is missing from the lib!
        #self.libc.context_provider_init (['Context.Test.keyInt'], None, None, None, None, None, None)
        #changeset = self.libc.context_provider_change_set_create()
        #self.assert_(ret != None)
        #ret = self.libc.context_provider_change_set_add_int(ret, "Context.Test.keyInt", 5)
        #self.assert_(ret == 0) # Success return value
        #ret = self.libc.context_provider_change_set_send_notification(ret)
        #self.assert_(ret == 0) # Success return value
        #ret = self.libc.context_change_set_delete(ret)
        pass

class KeyCounting(LibraryTestCase):
    def test_initialCount(self):
        # Initial count of a key is zero

        #i = c_int()
        #ret = self.libc.context_provider_no_of_subscribers("Context.Test.keyInt", byref(i))

        #self.assert_(ret == 0) # Success return value
        #self.assert_(i.value == 0) # 0 subscribers
        pass

def runTests():
    suiteStartup = unittest.TestLoader().loadTestsFromTestCase(Startup)
    suiteSubscription = unittest.TestLoader().loadTestsFromTestCase(Subscription)
    suiteChangeSets = unittest.TestLoader().loadTestsFromTestCase(ChangeSets)
    suiteKeyCounting = unittest.TestLoader().loadTestsFromTestCase(KeyCounting)

    unittest.TextTestRunner(verbosity=2).run(suiteStartup)
    #unittest.TextTestRunner(verbosity=2).run(suiteSubscription)
    #unittest.TextTestRunner(verbosity=2).run(suiteChangeSets)
    #unittest.TextTestRunner(verbosity=2).run(suiteKeyCounting)

if __name__ == "__main__":
    # Start listening to NameOwnerChanged dbus signal
    #listener = DBusListener()

    runTests()
    #libc.context_kit_key_usage_counter_add(None, None)
<commit_msg>Modifications because the library changed.<commit_after>import dbus
import os

import unittest
from time import sleep

import sys
sys.path.append("./python/")
import ContextProvider

# FIXME: Use the fake dbus.

# FIXME: Missing testcases
# - Using system bus flag in init
# - Test that callbacks work

class LibraryTestCase(unittest.TestCase):
    def setUp(self):
        pass
    def tearDown(self):
        print "LibraryTestCase tearDown"
        pass

class Startup(LibraryTestCase):
    def setUp(self):
        self.bus = dbus.SessionBus() # Note: using session bus

        # Start a listener which listens to dbus NameOwnerChanged signals
        os.system("python tests/python-test-library/stubs/dbus_listener.py &")
        sleep(0.5)
        # Command the listener to listen to FakeProvider
        self.listener_proxy = self.bus.get_object("org.freedesktop.ContextKit.Testing.Listener","/org/freedesktop/ContextKit/Testing/Listener")
        self.listener_iface = dbus.Interface(self.listener_proxy, "org.freedesktop.ContextKit.Testing.Listener")
        self.listener_iface.SetTarget("org.freedesktop.ContextKit.Testing.Provider")

        # Start a provider stub
        os.system("python tests/python-test-library/stubs/provider_stub.py &")
        sleep(0.5)
        # Command the provider stub to start exposing services over dbus
        self.provider_proxy = self.bus.get_object("org.freedesktop.ContextKit.Testing.Provider.Command","/org/freedesktop/ContextKit/Testing/Provider")
        self.provider_iface = dbus.Interface(self.provider_proxy, "org.freedesktop.ContextKit.Testing.Provider")
        self.provider_iface.DoInit()
        sleep(0.5)

    def tearDown(self):
        print "Startup tearDown"
        # Stop the provider
        self.provider_iface.Exit()
        sleep(0.5)

        # Stop the listener
        self.listener_iface.Exit()
        pass # FIXME: Call parent class func explicitly

    def test_startProvider(self):
        # Verify that the provider is started and exposed over dbus.

        # Get the log from the listener
        log = self.listener_iface.GetLog()
        self.assert_(log == "(TargetAppeared)")

        # Note: we could also test whether disappearing is noticed
        # but that is dropped for simplicity.

    def test_managerInterfaceImplemented(self):
        # Verify that the provider exposes the Manager interface over dbus properly

        # Try to get the manager object
        proxy_object_manager = self.bus.get_object("org.freedesktop.ContextKit.Testing.Provider","/org/freedesktop/ContextKit/Manager")
        iface_manager = dbus.Interface(proxy_object_manager, "org.freedesktop.ContextKit.Manager")
        self.assert_ (proxy_object_manager != None) # FIXME: What is the correct validity check?
        self.assert_ (iface_manager != None) # FIXME: What is the correct validity check?
        try:
            iface_manager.Get(["temp"])
        except:
            print "Exception caught"
            self.assert_ (False) # Manager interface is not implemented properly
            # After this, only tearDown will be executed

class Subscription(LibraryTestCase):
    def setUp(self):
        # Start a provider stub
        os.system("python tests/python-test-library/stubs/provider_stub.py &")
        sleep(0.5)
        # Command the provider stub to start exposing services over dbus
        bus = dbus.SessionBus()
        self.provider_proxy = bus.get_object("org.freedesktop.ContextKit.Testing.Provider","/org/freedesktop/ContextKit/Testing/Provider")
        self.provider_iface = dbus.Interface(self.provider_proxy, "org.freedesktop.ContextKit.Testing.Provider")
        self.provider_iface.DoInit()
        sleep(0.5)

    def tearDown(self):
        print "Subscription tearDown"
        self.provider_iface.Exit()
        # FIXME: Try to ensure that the stubs are killed properly
        pass # FIXME: Call parent class func explicitly

    def test_subscriberCallbacks(self):
        bus = dbus.SessionBus()
        # Start a subscriber
        #os.system("python tests/python-test-library/stubs/subscriber_stub.py &")
        #sleep(1)

        #subscriber_proxy = bus.get_object("org.freedesktop.ContextKit.Testing.SubscriberStub","/org/freedesktop/ContextKit/Testing/SubscriberStub")
        #subscriber_iface = dbus.Interface(subscriber_proxy, "org.freedesktop.ContextKit.Testing.SubscriberStub")
        #subscriber_iface.DoSubscribe()

        # Verify that the provider got FirstSubscribed event
        # Stop the subscriber

        # Verify that the provider got LastSubscribed event

class ChangeSets(LibraryTestCase):
    def test_createChangeSet(self):
        # FIXME: This functionality is missing from the lib!
        #self.libc.context_provider_init (['Context.Test.keyInt'], None, None, None, None, None, None)
        #changeset = self.libc.context_provider_change_set_create()
        #self.assert_(ret != None)
        #ret = self.libc.context_provider_change_set_add_int(ret, "Context.Test.keyInt", 5)
        #self.assert_(ret == 0) # Success return value
        #ret = self.libc.context_provider_change_set_send_notification(ret)
        #self.assert_(ret == 0) # Success return value
        #ret = self.libc.context_change_set_delete(ret)
        pass

class KeyCounting(LibraryTestCase):
    def test_initialCount(self):
        # Initial count of a key is zero

        #i = c_int()
        #ret = self.libc.context_provider_no_of_subscribers("Context.Test.keyInt", byref(i))

        #self.assert_(ret == 0) # Success return value
        #self.assert_(i.value == 0) # 0 subscribers
        pass

def runTests():
    suiteStartup = unittest.TestLoader().loadTestsFromTestCase(Startup)
    suiteSubscription = unittest.TestLoader().loadTestsFromTestCase(Subscription)
    suiteChangeSets = unittest.TestLoader().loadTestsFromTestCase(ChangeSets)
    suiteKeyCounting = unittest.TestLoader().loadTestsFromTestCase(KeyCounting)

    unittest.TextTestRunner(verbosity=2).run(suiteStartup)
    #unittest.TextTestRunner(verbosity=2).run(suiteSubscription)
    #unittest.TextTestRunner(verbosity=2).run(suiteChangeSets)
    #unittest.TextTestRunner(verbosity=2).run(suiteKeyCounting)

if __name__ == "__main__":
    # Start listening to NameOwnerChanged dbus signal
    #listener = DBusListener()

    runTests()
    #libc.context_kit_key_usage_counter_add(None, None)
<|endoftext|>