<commit_before>#!python3

# License: Apache 2.0. See LICENSE file in root directory.
# Copyright(c) 2020 Intel Corporation. All Rights Reserved.

import sys, os, subprocess, locale, re, platform, getopt
from abc import ABC, abstractmethod

# Add our py/ module directory to Python's list so we can use them
current_dir = os.path.dirname( os.path.abspath( __file__ ))
sys.path.insert( 1, current_dir + os.sep + "py" )

def usage():
    ourname = os.path.basename(sys.argv[0])
    print( 'Syntax: ' + ourname + ' [options] [dir]' )
    print( '        dir: the directory holding the executable tests to run (default to the build directory')
    print( 'Options:' )
    print( '        --debug        Turn on debugging information' )
    print( '        -v, --verbose  Errors will dump the log to stdout' )
    print( '        -q, --quiet    Suppress output; rely on exit status (0=no failures)' )
    print( '        -r, --regex    run all tests that fit the following regular expression')
    sys.exit(2)
def debug(*args):
    pass

def stream_has_color( stream ):
    if not hasattr(stream, "isatty"):
        return False
    if not stream.isatty():
        return False # auto color only on TTYs
    try:
        import curses
        curses.setupterm()
        return curses.tigetnum( "colors" ) > 2
    except:
        # guess false in case of error
        return False

# Set up the default output system; if not a terminal, disable colors!
if stream_has_color( sys.stdout ):
    red = '\033[91m'
    gray = '\033[90m'
    reset = '\033[0m'
    cr = '\033[G'
    clear_eos = '\033[J'
    clear_eol = '\033[K'
    _progress = ''
    def out(*args):
        print( *args, end = clear_eol + '\n' )
        global _progress
        if len(_progress):
            progress( *_progress )
    def progress(*args):
        print( *args, end = clear_eol + '\r' )
        global _progress
        _progress = args
else:
    red = gray = reset = cr = clear_eos = ''
    def out(*args):
        print( *args )
    def progress(*args):
        print( *args )

n_errors = 0
def error(*args):
    global red
    global reset
    out( red + '-E-' + reset, *args )
    global n_errors
    n_errors = n_errors + 1
def info(*args):
    out( '-I-', *args)

def filesin( root ):
    # Yield all files found in root, using relative names ('root/a' would be yielded as 'a')
    for (path,subdirs,leafs) in os.walk( root ):
        for leaf in leafs:
            # We have to stick to Unix conventions because CMake on Windows is fubar...
            yield os.path.relpath( path + '/' + leaf, root ).replace( '\\', '/' )

def find( dir, mask ):
    pattern = re.compile( mask )
    for leaf in filesin( dir ):
        if pattern.search( leaf ):
            debug(leaf)
            yield leaf

# get os and directories for future use
# NOTE: WSL will read as 'Linux' but the build is Windows-based!
system = platform.system()
if system == 'Linux'  and  "microsoft" not in platform.uname()[3].lower():
    linux = True
else:
    linux = False

# this script is located in librealsense/unit-tests, so one directory up is the main repository
librealsense = os.path.dirname(current_dir)

# function for checking if a file is an executable

def is_executable(path_to_test):
    if linux:
        return os.access(path_to_test, os.X_OK)
    else:
        return path_to_test.endswith('.exe')

# Parse command-line:
try:
    opts,args = getopt.getopt( sys.argv[1:], 'hvqr:',
        longopts = [ 'help', 'verbose', 'debug', 'quiet', 'regex=' ])
except getopt.GetoptError as err:
    error( err )   # something like "option -a not recognized"
    usage()
verbose = False
regex = None
for opt,arg in opts:
    if opt in ('-h','--help'):
        usage()
    elif opt in ('--debug'):
        def debug(*args):
            global gray
            global reset
            out( gray + '-D-', *args, reset )
    elif opt in ('-v','--verbose'):
        verbose = True
    elif opt in ('-q','--quiet'):
        def out(*args):
            pass
    elif opt in ('-r', '--regex'):
        regex = arg

if len(args) > 1:
    usage()
target = None
if len(args) == 1:
    if os.path.isdir( args[0] ):
        target = args[0]
    else:
        usage()
# Trying to assume target directory from inside build directory. Only works if there is only one location with tests
if not target:
    build = librealsense + os.sep + 'build'
    for executable in find(build, '(^|/)test-.*'):
        if not is_executable(executable):
            continue
        dir_with_test = build + os.sep + os.path.dirname(executable)
        if target and target != dir_with_test:
            error("Found executable tests in 2 directories:", target, "and", dir_with_test, ". Can't default to directory")
            usage()
        target = dir_with_test

if target:
    logdir = target + os.sep + 'unit-tests'
else: # no test executables were found. We put the logs directly in build directory
    logdir = librealsense + os.sep + 'build'
os.makedirs( logdir, exist_ok = True )
n_tests = 0

# wrapper function for subprocess.run
def subprocess_run(cmd, stdout = None):
    debug( 'Running:', cmd )
    handle = None
    try:
        if stdout  and  stdout != subprocess.PIPE:
            handle = open( stdout, "w" )
            stdout = handle
        rv = subprocess.run( cmd,
                             stdout = stdout,
                             stderr = subprocess.STDOUT,
                             universal_newlines = True,
                             check = True)
        result = rv.stdout
        if not result:
            result = []
        else:
            result = result.split( '\n' )
        return result
    finally:
        if handle:
            handle.close()

# Python scripts should be able to find the pyrealsense2 .pyd or else they won't work. We don't know
# if the user (Travis included) has pyrealsense2 installed but even if so, we want to use the one we compiled.
# we search the librealsense repository for the .pyd file (.so file in linux)
pyrs = ""
if linux:
    for so in find(librealsense, '(^|/)pyrealsense2.*\.so$'):
        pyrs = so
else:
    for pyd in find(librealsense, '(^|/)pyrealsense2.*\.pyd$'):
        pyrs = pyd

if pyrs:
    # After use of find, pyrs contains the path from librealsense to the pyrealsense that was found
    # We append it to the librealsense path to get an absolute path to the file to add to PYTHONPATH so it can be found by the tests
    pyrs_path = librealsense + os.sep + pyrs
    # We need to add the directory not the file itself
    pyrs_path = os.path.dirname(pyrs_path)
    # Add the necessary path to the PYTHONPATH environment variable so python will look for modules there
    os.environ["PYTHONPATH"] = pyrs_path
    # We also need to add the path to the python packages that the tests use
    os.environ["PYTHONPATH"] += os.pathsep + (current_dir + os.sep + "py")
    # We can simply change `sys.path` but any child python scripts won't see it. We change the environment instead.

def remove_newlines (lines):
    for line in lines:
        if line[-1] == '\n':
            line = line[:-1]    # excluding the endline
        yield line

def grep_( pattern, lines, context ):
    index = 0
    matches = 0
    for line in lines:
        index = index + 1
        match = pattern.search( line )
        if match:
            context['index'] = index
            context['line']  = line
            context['match'] = match
            yield context
            matches = matches + 1
    if matches:
        del context['index']
        del context['line']
        del context['match']

def grep( expr, *args ):
    #debug( f'grep {expr} {args}' )
    pattern = re.compile( expr )
    context = dict()
    for filename in args:
        context['filename'] = filename
        with open( filename, errors = 'ignore' ) as file:
            for line in grep_( pattern, remove_newlines( file ), context ):
                yield context

def cat( filename ):
    with open( filename, errors = 'ignore' ) as file:
        for line in remove_newlines( file ):
            out( line )

def check_log_for_fails(log, testname, exe):
    # Normal logs are expected to have in last line:
    #     "All tests passed (11 assertions in 1 test case)"
    # Tests that have failures, however, will show:
    #     "test cases: 1 | 1 failed
    #      assertions: 9 | 6 passed | 3 failed"
    global verbose
    for ctx in grep( r'^test cases:\s*(\d+) \|\s*(\d+) (passed|failed)', log ):
        m = ctx['match']
        total = int(m.group(1))
        passed = int(m.group(2))
        if m.group(3) == 'failed':
            # "test cases: 1 | 1 failed"
            passed = total - passed
        if passed < total:
            if total == 1  or  passed == 0:
                desc = 'failed'
            else:
                desc = str(total - passed) + ' of ' + str(total) + ' failed'

            if verbose:
                error( red + testname + reset + ': ' + desc )
                info( 'Executable:', exe )
                info( 'Log: >>>' )
                out()
                cat( log )
                out( '<<<' )
            else:
                error( red + testname + reset + ': ' + desc + '; see ' + log )
            return True
        return False

# definition of classes for tests
class Test(ABC): # Abstract Base Class
    """
    Abstract class for a test. Holds the name of the test
    """
    def __init__(self, testname):
        self.testname = testname

    @abstractmethod
    def run_test(self):
        pass

class PyTest(Test):
    """
    Class for python tests. Hold the path to the script of the test
    """
    def __init__(self, testname, path_to_test):
        """
        :param testname: name of the test
        :param path_to_test: the relative path from the current directory to the path
        """
        global current_dir
        Test.__init__(self, testname)
        self.path_to_script = current_dir + os.sep + path_to_test

    @property
    def command(self):
        return [sys.executable, self.path_to_script]

    def run_test(self):
        global n_tests
        n_tests +=1
        log = logdir + os.sep + self.testname + ".log"
        progress(self.testname, '>', log, '...')
        try:
            subprocess_run(self.command, stdout=log)
        except FileNotFoundError:
            error(red + self.testname + reset + ': executable not found! (' + self.path_to_script + ')')
        except subprocess.CalledProcessError as cpe:
            if not check_log_for_fails(log, self.testname, self.path_to_script):
                # An unexpected error occurred
                error(red + self.testname + reset + ': exited with non-zero value! (' + str(cpe.returncode) + ')')

class ExeTest(Test):
    """
    Class for c/cpp tests. Hold the path to the executable for the test
    """
    def __init__(self, testname, exe):
        """
        :param testname: name of the test
        :param exe: full path to executable
        """
        Test.__init__(self, testname)
        self.exe = exe

    @property
    def command(self):
        return [self.exe]

    def run_test(self):
        global n_tests
        n_tests += 1
        log = logdir + os.sep + self.testname + ".log"
        progress(self.testname, '>', log, '...')
        try:
            subprocess_run(self.command, stdout=log)
        except FileNotFoundError:
            error(red + self.testname + reset + ': executable not found! (' + self.exe + ')')
        except subprocess.CalledProcessError as cpe:
            if not check_log_for_fails(log, self.testname, self.exe):
                # An unexpected error occurred
                error(red + self.testname + reset + ': exited with non-zero value! (' + str(cpe.returncode) + ')')

def get_tests():
    global regex, target, pyrs, current_dir, linux
    if regex:
        pattern = re.compile(regex)
    if target:
        # In Linux, the build targets are located elsewhere than on Windows
        # Go over all the tests from a "manifest" we take from the result of the last CMake
        # run (rather than, for example, looking for test-* in the build-directory):
        if linux:
            manifestfile = target + '/CMakeFiles/TargetDirectories.txt'
        else:
            manifestfile = target + '/../CMakeFiles/TargetDirectories.txt'
        for manifest_ctx in grep(r'(?<=unit-tests/build/)\S+(?=/CMakeFiles/test-\S+.dir$)', manifestfile):
            # We need to first create the test name so we can see if it fits the regex
            testdir = manifest_ctx['match'].group(0)  # "log/internal/test-all"
            testparent = os.path.dirname(testdir)  # "log/internal"
            if testparent:
                testname = 'test-' + testparent.replace('/', '-') + '-' + os.path.basename(testdir)[5:]  # "test-log-internal-all"
            else:
                testname = testdir  # no parent folder so we get "test-all"

            if regex and not pattern.search( testname ):
                continue

            if linux:
                exe = target + '/unit-tests/build/' + testdir + '/' + testname
            else:
                exe = target + '/' + testname + '.exe'

            yield ExeTest(testname, exe)

    # If we run python tests with no .pyd/.so file they will crash. Therefore we only run them if such a file was found
    if pyrs:
        # unit-test scripts are in the same directory as this script
        for py_test in find(current_dir, '(^|/)test-.*\.py'):
            testparent = os.path.dirname(py_test)  # "log/internal" <-  "log/internal/test-all.py"
            if testparent:
                testname = 'test-' + testparent.replace('/', '-') + '-' + os.path.basename(py_test)[5:-3]  # remove .py
            else:
                testname = os.path.basename(py_test)[:-3]

            if regex and not pattern.search( testname ):
                continue

            yield PyTest(testname, py_test)

# Before we run any tests, recycle all ports and make sure they're set to USB3
try:
    import acroname
    acroname.connect()
    acroname.enable_ports()     # so ports() will return all
    portlist = acroname.ports()
    debug( 'Recycling found ports:', portlist )
    acroname.set_ports_usb3( portlist )   # will recycle them, too
except ModuleNotFoundError:
    # Error should have already been printed
    # We assume there's no brainstem library, meaning no acroname either
    pass

# Run all tests
for test in get_tests():
    test.run_test()

progress()
if n_errors:
    out( red + str(n_errors) + reset + ' of ' + str(n_tests) + ' test(s) failed!' + clear_eos )
    sys.exit(1)
out( str(n_tests) + ' unit-test(s) completed successfully' + clear_eos )
sys.exit(0)<commit_msg>added run-unit-tests delay after acroname reset; added --stdout, --asis<commit_after>#!python3

# License: Apache 2.0. See LICENSE file in root directory.
# Copyright(c) 2020 Intel Corporation. All Rights Reserved.

import sys, os, subprocess, locale, re, platform, getopt
from abc import ABC, abstractmethod

# Add our py/ module directory to Python's list so we can use them
current_dir = os.path.dirname( os.path.abspath( __file__ ))
sys.path.insert( 1, current_dir + os.sep + "py" )

def usage():
    ourname = os.path.basename(sys.argv[0])
    print( 'Syntax: ' + ourname + ' [options] [dir]' )
    print( '        dir: the directory holding the executable tests to run (default to the build directory')
    print( 'Options:' )
    print( '        --debug        Turn on debugging information' )
    print( '        -v, --verbose  Errors will dump the log to stdout' )
    print( '        -q, --quiet    Suppress output; rely on exit status (0=no failures)' )
    print( '        -r, --regex    run all tests that fit the following regular expression')
    print( '        -s, --stdout   do not redirect stdout to logs')
    print( '        -a, --asis     do not init acroname; just run the tests as-is' )
    sys.exit(2)
def debug(*args):
    pass

def stream_has_color( stream ):
    if not hasattr(stream, "isatty"):
        return False
    if not stream.isatty():
        return False # auto color only on TTYs
    try:
        import curses
        curses.setupterm()
        return curses.tigetnum( "colors" ) > 2
    except:
        # guess false in case of error
        return False

# Set up the default output system; if not a terminal, disable colors!
if stream_has_color( sys.stdout ):
    red = '\033[91m'
    gray = '\033[90m'
    reset = '\033[0m'
    cr = '\033[G'
    clear_eos = '\033[J'
    clear_eol = '\033[K'
    _progress = ''
    def out(*args):
        print( *args, end = clear_eol + '\n' )
        global _progress
        if len(_progress):
            progress( *_progress )
    def progress(*args):
        print( *args, end = clear_eol + '\r' )
        global _progress
        _progress = args
else:
    red = gray = reset = cr = clear_eos = ''
    def out(*args):
        print( *args )
    def progress(*args):
        print( *args )

n_errors = 0
def error(*args):
    global red
    global reset
    out( red + '-E-' + reset, *args )
    global n_errors
    n_errors = n_errors + 1
def info(*args):
    out( '-I-', *args)

def filesin( root ):
    # Yield all files found in root, using relative names ('root/a' would be yielded as 'a')
    for (path,subdirs,leafs) in os.walk( root ):
        for leaf in leafs:
            # We have to stick to Unix conventions because CMake on Windows is fubar...
            yield os.path.relpath( path + '/' + leaf, root ).replace( '\\', '/' )

def find( dir, mask ):
    pattern = re.compile( mask )
    for leaf in filesin( dir ):
        if pattern.search( leaf ):
            debug(leaf)
            yield leaf

# get os and directories for future use
# NOTE: WSL will read as 'Linux' but the build is Windows-based!
system = platform.system()
if system == 'Linux'  and  "microsoft" not in platform.uname()[3].lower():
    linux = True
else:
    linux = False

# this script is located in librealsense/unit-tests, so one directory up is the main repository
librealsense = os.path.dirname(current_dir)

# function for checking if a file is an executable

def is_executable(path_to_test):
    if linux:
        return os.access(path_to_test, os.X_OK)
    else:
        return path_to_test.endswith('.exe')

# Parse command-line:
try:
    opts,args = getopt.getopt( sys.argv[1:], 'hvqr:sa',
        longopts = [ 'help', 'verbose', 'debug', 'quiet', 'regex=', 'stdout', 'asis' ])
except getopt.GetoptError as err:
    error( err )   # something like "option -a not recognized"
    usage()
verbose = False
regex = None
to_stdout = False
asis = False
for opt,arg in opts:
    if opt in ('-h','--help'):
        usage()
    elif opt in ('--debug'):
        def debug(*args):
            global gray
            global reset
            out( gray + '-D-', *args, reset )
    elif opt in ('-v','--verbose'):
        verbose = True
    elif opt in ('-q','--quiet'):
        def out(*args):
            pass
    elif opt in ('-r', '--regex'):
        regex = arg
    elif opt in ('-s', '--stdout'):
        to_stdout = True
    elif opt in ('-a', '--asis'):
        asis = True

if len(args) > 1:
    usage()
target = None
if len(args) == 1:
    if os.path.isdir( args[0] ):
        target = args[0]
    else:
        usage()
# Trying to assume target directory from inside build directory. Only works if there is only one location with tests
if not target:
    build = librealsense + os.sep + 'build'
    for executable in find(build, '(^|/)test-.*'):
        if not is_executable(executable):
            continue
        dir_with_test = build + os.sep + os.path.dirname(executable)
        if target and target != dir_with_test:
            error("Found executable tests in 2 directories:", target, "and", dir_with_test, ". Can't default to directory")
            usage()
        target = dir_with_test

if target:
    logdir = target + os.sep + 'unit-tests'
else: # no test executables were found. We put the logs directly in build directory
    logdir = librealsense + os.sep + 'build'
os.makedirs( logdir, exist_ok = True )
n_tests = 0

# wrapper function for subprocess.run
def subprocess_run(cmd, stdout = None):
    debug( 'Running:', cmd )
    handle = None
    try:
        if stdout  and  stdout != subprocess.PIPE:
            handle = open( stdout, "w" )
            stdout = handle
        rv = subprocess.run( cmd,
                             stdout = stdout,
                             stderr = subprocess.STDOUT,
                             universal_newlines = True,
                             check = True)
        result = rv.stdout
        if not result:
            result = []
        else:
            result = result.split( '\n' )
        return result
    finally:
        if handle:
            handle.close()

# Python scripts should be able to find the pyrealsense2 .pyd or else they won't work. We don't know
# if the user (Travis included) has pyrealsense2 installed but even if so, we want to use the one we compiled.
# we search the librealsense repository for the .pyd file (.so file in linux)
pyrs = ""
if linux:
    for so in find(librealsense, '(^|/)pyrealsense2.*\.so$'):
        pyrs = so
else:
    for pyd in find(librealsense, '(^|/)pyrealsense2.*\.pyd$'):
        pyrs = pyd

if pyrs:
    # After use of find, pyrs contains the path from librealsense to the pyrealsense that was found
    # We append it to the librealsense path to get an absolute path to the file to add to PYTHONPATH so it can be found by the tests
    pyrs_path = librealsense + os.sep + pyrs
    # We need to add the directory not the file itself
    pyrs_path = os.path.dirname(pyrs_path)
    # Add the necessary path to the PYTHONPATH environment variable so python will look for modules there
    os.environ["PYTHONPATH"] = pyrs_path
    # We also need to add the path to the python packages that the tests use
    os.environ["PYTHONPATH"] += os.pathsep + (current_dir + os.sep + "py")
    # We can simply change `sys.path` but any child python scripts won't see it. We change the environment instead.

def remove_newlines (lines):
    for line in lines:
        if line[-1] == '\n':
            line = line[:-1]    # excluding the endline
        yield line

def grep_( pattern, lines, context ):
    index = 0
    matches = 0
    for line in lines:
        index = index + 1
        match = pattern.search( line )
        if match:
            context['index'] = index
            context['line']  = line
            context['match'] = match
            yield context
            matches = matches + 1
    if matches:
        del context['index']
        del context['line']
        del context['match']

def grep( expr, *args ):
    #debug( f'grep {expr} {args}' )
    pattern = re.compile( expr )
    context = dict()
    for filename in args:
        context['filename'] = filename
        with open( filename, errors = 'ignore' ) as file:
            for line in grep_( pattern, remove_newlines( file ), context ):
                yield context

def cat( filename ):
    with open( filename, errors = 'ignore' ) as file:
        for line in remove_newlines( file ):
            out( line )

def check_log_for_fails(log, testname, exe):
    # Normal logs are expected to have in last line:
    #     "All tests passed (11 assertions in 1 test case)"
    # Tests that have failures, however, will show:
    #     "test cases: 1 | 1 failed
    #      assertions: 9 | 6 passed | 3 failed"
    if log is None:
        return False
    global verbose
    for ctx in grep( r'^test cases:\s*(\d+) \|\s*(\d+) (passed|failed)', log ):
        m = ctx['match']
        total = int(m.group(1))
        passed = int(m.group(2))
        if m.group(3) == 'failed':
            # "test cases: 1 | 1 failed"
            passed = total - passed
        if passed < total:
            if total == 1  or  passed == 0:
                desc = 'failed'
            else:
                desc = str(total - passed) + ' of ' + str(total) + ' failed'

            if verbose:
                errOR( Red + testname + reset + ': ' + desc )
                info( 'Executable:', exe )
                info( 'Log: >>>' )
                out()
                cat( log )
                out( '<<<' )
            else:
                error( red + testname + reset + ': ' + desc + '; see ' + log )
            return True
        return False

# definition of classes for tests
class Test(ABC): # Abstract Base Class
    """
    Abstract class for a test. Holds the name of the test
    """
    def __init__(self, testname):
        self.testname = testname

    @abstractmethod
    def run_test(self):
        pass

class PyTest(Test):
    """
    Class for python tests. Hold the path to the script of the test
    """
    def __init__(self, testname, path_to_test):
        """
        :param testname: name of the test
        :param path_to_test: the relative path from the current directory to the path
        """
        global current_dir
        Test.__init__(self, testname)
        self.path_to_script = current_dir + os.sep + path_to_test

    @property
    def command(self):
        return [sys.executable, self.path_to_script]

    def run_test(self):
        global n_tests, to_stdout
        n_tests += 1
        if to_stdout:
            log = None
        else:
            log = logdir + os.sep + self.testname + ".log"
        progress(self.testname, '>', log, '...')
        try:
            subprocess_run(self.command, stdout=log)
        except FileNotFoundError:
            error(red + self.testname + reset + ': executable not found! (' + self.path_to_script + ')')
        except subprocess.CalledProcessError as cpe:
            if not check_log_for_fails(log, self.testname, self.path_to_script):
                # An unexpected error occurred
                error(red + self.testname + reset + ': exited with non-zero value! (' + str(cpe.returncode) + ')')

class ExeTest(Test):
    """
    Class for c/cpp tests. Hold the path to the executable for the test
    """
    def __init__( self, testname, exe ):
        """
        :param testname: name of the test
        :param exe: full path to executable
        """
        Test.__init__(self, testname)
        self.exe = exe

    @property
    def command(self):
        return [self.exe]

    def run_test(self):
        global n_tests, to_stdout
        n_tests += 1
        if to_stdout:
            log = None
        else:
            log = logdir + os.sep + self.testname + ".log"
        progress( self.testname, '>', log, '...' )
        try:
            subprocess_run(self.command, stdout=log)
        except FileNotFoundError:
            error(red + self.testname + reset + ': executable not found! (' + self.exe + ')')
        except subprocess.CalledProcessError as cpe:
            if not check_log_for_fails(log, self.testname, self.exe):
                # An unexpected error occurred
                error(red + self.testname + reset + ': exited with non-zero value! (' + str(cpe.returncode) + ')')

def get_tests():
    global regex, target, pyrs, current_dir, linux
    if regex:
        pattern = re.compile(regex)
    if target:
        # In Linux, the build targets are located elsewhere than on Windows
        # Go over all the tests from a "manifest" we take from the result of the last CMake
        # run (rather than, for example, looking for test-* in the build-directory):
        if linux:
            manifestfile = target + '/CMakeFiles/TargetDirectories.txt'
        else:
            manifestfile = target + '/../CMakeFiles/TargetDirectories.txt'
        for manifest_ctx in grep(r'(?<=unit-tests/build/)\S+(?=/CMakeFiles/test-\S+.dir$)', manifestfile):
            # We need to first create the test name so we can see if it fits the regex
            testdir = manifest_ctx['match'].group(0)  # "log/internal/test-all"
            testparent = os.path.dirname(testdir)  # "log/internal"
            if testparent:
                testname = 'test-' + testparent.replace('/', '-') + '-' + os.path.basename(testdir)[5:]  # "test-log-internal-all"
            else:
                testname = testdir  # no parent folder so we get "test-all"

            if regex and not pattern.search( testname ):
                continue

            if linux:
                exe = target + '/unit-tests/build/' + testdir + '/' + testname
            else:
                exe = target + '/' + testname + '.exe'

            yield ExeTest(testname, exe)

    # If we run python tests with no .pyd/.so file they will crash. Therefore we only run them if such a file was found
    if pyrs:
        # unit-test scripts are in the same directory as this script
        for py_test in find(current_dir, '(^|/)test-.*\.py'):
            testparent = os.path.dirname(py_test)  # "log/internal" <-  "log/internal/test-all.py"
            if testparent:
                testname = 'test-' + testparent.replace('/', '-') + '-' + os.path.basename(py_test)[5:-3]  # remove .py
            else:
                testname = os.path.basename(py_test)[:-3]

            if regex and not pattern.search( testname ):
                continue

            yield PyTest(testname, py_test)

# Before we run any tests, recycle all ports and make sure they're set to USB3
if not asis:
    try:
        import acroname
        acroname.connect()
        acroname.enable_ports()     # so ports() will return all
        portlist = acroname.ports()
        debug( 'Recycling found ports:', portlist )
        acroname.set_ports_usb3( portlist )   # will recycle them, too
        acroname.disconnect()
        import time
        time.sleep(5)
    except ModuleNotFoundError:
        # Error should have already been printed
        # We assume there's no brainstem library, meaning no acroname either
        pass

# Run all tests
for test in get_tests():
    test.run_test()

progress()
if n_errors:
    out( red + str(n_errors) + reset + ' of ' + str(n_tests) + ' test(s) failed!' + clear_eos )
    sys.exit(1)
out( str(n_tests) + ' unit-test(s) completed successfully' + clear_eos )
sys.exit(0)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import unittest
import numpy as np

from pymatgen.symmetry.settings import *

__author__ = "Matthew Horton"
__copyright__ = "Copyright 2017, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Matthew Horton"
__email__ = "mkhorton@lbl.gov"
__status__ = "Development"
__date__ = "Apr 2017"

class JonesFaithfulTransformationTest(unittest.TestCase):

    def setUp(self):
        self.test_strings = ['a,b,c;0,0,0',  # identity
                             'a-b,a+b,2c;0,0,1/2',
                             'a/4+b/4-c/2,a/4-b/4,-a/2-b/2;0,0,0',
                             'a,b,c;1/4,1/2,3/4']  # pure translation
        self.test_Pps = [([[1, 0, 0], [0, 1, 0], [0, 0, 1]], [0, 0, 0]),
                         ([[1, 1, 0], [-1, 1, 0], [0, 0, 2]], [0, 0, 0.5]),
                         ([[0.25, 0.25, -0.5], [0.25, -0.25, -0.5], [-0.5, 0, 0]], [0, 0, 0]),
                         ([[1, 0, 0], [0, 1, 0], [0, 0, 1]], [0.25, 0.5, 0.75])]

    def test_init(self):
        for test_string, test_Pp in zip(self.test_strings, self.test_Pps):
            jft = JonesFaithfulTransformation.from_transformation_string(test_string)
            jft2 = JonesFaithfulTransformation(test_Pp[0], test_Pp[1])
            self.assertTrue(np.allclose(jft.P, jft2.P))
            self.assertTrue(np.allclose(jft.p, jft2.p))
            self.assertEqual(test_string, jft.transformation_string)
            self.assertEqual(test_string, jft2.transformation_string)

    def test_inverse(self):
        for test_string in self.test_strings:
            jft = JonesFaithfulTransformation.from_transformation_string(test_string)
            self.assertEqual(jft, jft.inverse.inverse)
            self.assertEqual(jft.transformation_string, jft.inverse.inverse.transformation_string)

    def test_transform_lattice(self):
        lattice = Lattice.cubic(5)

        all_ref_lattices = [[[5., 0., 0.], [0., 5., 0.], [0., 0., 5.]],
                            [[5., 5., 0.], [-5., 5., 0.], [0., 0., 10.]],
                            [[1.25, 1.25, -2.5], [1.25, -1.25, -2.5], [-2.5, 0., 0.]],
                            [[5., 0., 0.], [0., 5., 0.], [0., 0., 5.]]]

        for ref_lattice, (P, p) in zip(all_ref_lattices, self.test_Pps):
            jft = JonesFaithfulTransformation(P, p)
            self.assertTrue(np.allclose(jft.transform_lattice(lattice).matrix, ref_lattice))

    def test_transform_coords(self):
        coords = [[0, 0, 0], [0.5, 0.5, 0.5]]

        all_ref_coords = [[[0.0, 0.0, 0.0], [0.5, 0.5, 0.5]],
                          [[0.0, 0.0, -0.25], [0.0, 0.5, 0.0]],
                          [[0.0, 0.0, 0.0], [-1.0, 0.0, -1.5]],
                          [[-0.25, -0.5, -0.75], [0.25, 0.0, -0.25]]]

        for ref_coords, (P, p) in zip(all_ref_coords, self.test_Pps):
            jft = JonesFaithfulTransformation(P, p)
            transformed_coords = jft.transform_coords(coords)
            for coord, ref_coord in zip(transformed_coords, ref_coords):
                self.assertTrue(np.allclose(coord, ref_coord))

    def test_transform_symmops(self):

        # reference data for this test taken from GENPOS
        # http://cryst.ehu.es/cryst/get_gen.html

        # Fm-3m
        input_symmops = """x,y,z
-x,-y,z
-x,y,-z
x,-y,-z
z,x,y
z,-x,-y
-z,-x,y
-z,x,-y
y,z,x
-y,z,-x
y,-z,-x
-y,-z,x
y,x,-z
-y,-x,-z
y,-x,z
-y,x,z
x,z,-y
-x,z,y
-x,-z,-y
x,-z,y
z,y,-x
z,-y,x
-z,y,x
-z,-y,-x
-x,-y,-z
x,y,-z
x,-y,z
-x,y,z
-z,-x,-y
-z,x,y
z,x,-y
z,-x,y
-y,-z,-x
y,-z,x
-y,z,x
y,z,-x
-y,-x,z
y,x,z
-y,x,-z
y,-x,-z
-x,-z,y
x,-z,-y
x,z,y
-x,z,-y
-z,-y,x
-z,y,-x
z,-y,-x
z,y,x"""

        # Fm-3m transformed by (a-b,a+b,2c;0,0,1/2)
        ref_transformed_symmops = """x,y,z
-x,-y,z
-y,-x,-z+1/2
y,x,-z+1/2
-1/2x-1/2y+z+1/4,1/2x+1/2y+z+1/4,-1/2x+1/2y-1/4
1/2x+1/2y+z+1/4,-1/2x-1/2y+z+1/4,1/2x-1/2y-1/4
1/2x+1/2y-z-1/4,-1/2x-1/2y-z-1/4,-1/2x+1/2y-1/4
-1/2x-1/2y-z-1/4,1/2x+1/2y-z-1/4,1/2x-1/2y-1/4
-1/2x+1/2y-z-1/4,-1/2x+1/2y+z+1/4,1/2x+1/2y-1/4
1/2x-1/2y-z-1/4,1/2x-1/2y+z+1/4,-1/2x-1/2y-1/4
-1/2x+1/2y+z+1/4,-1/2x+1/2y-z-1/4,-1/2x-1/2y-1/4
1/2x-1/2y+z+1/4,1/2x-1/2y-z-1/4,1/2x+1/2y-1/4
-x,y,-z+1/2
x,-y,-z+1/2
y,-x,z
-y,x,z
1/2x+1/2y-z-1/4,1/2x+1/2y+z+1/4,1/2x-1/2y-1/4
-1/2x-1/2y-z-1/4,-1/2x-1/2y+z+1/4,-1/2x+1/2y-1/4
-1/2x-1/2y+z+1/4,-1/2x-1/2y-z-1/4,1/2x-1/2y-1/4
1/2x+1/2y+z+1/4,1/2x+1/2y-z-1/4,-1/2x+1/2y-1/4
1/2x-1/2y+z+1/4,-1/2x+1/2y+z+1/4,-1/2x-1/2y-1/4
-1/2x+1/2y+z+1/4,1/2x-1/2y+z+1/4,1/2x+1/2y-1/4
1/2x-1/2y-z-1/4,-1/2x+1/2y-z-1/4,1/2x+1/2y-1/4
-1/2x+1/2y-z-1/4,1/2x-1/2y-z-1/4,-1/2x-1/2y-1/4
-x,-y,-z+1/2
x,y,-z+1/2
y,x,z
-y,-x,z
1/2x+1/2y-z-1/4,-1/2x-1/2y-z-1/4,1/2x-1/2y-1/4
-1/2x-1/2y-z-1/4,1/2x+1/2y-z-1/4,-1/2x+1/2y-1/4
-1/2x-1/2y+z+1/4,1/2x+1/2y+z+1/4,1/2x-1/2y-1/4
1/2x+1/2y+z+1/4,-1/2x-1/2y+z+1/4,-1/2x+1/2y-1/4
1/2x-1/2y+z+1/4,1/2x-1/2y-z-1/4,-1/2x-1/2y-1/4
-1/2x+1/2y+z+1/4,-1/2x+1/2y-z-1/4,1/2x+1/2y-1/4
1/2x-1/2y-z-1/4,1/2x-1/2y+z+1/4,1/2x+1/2y-1/4
-1/2x+1/2y-z-1/4,-1/2x+1/2y+z+1/4,-1/2x-1/2y-1/4
x,-y,z
-x,y,z
-y,x,-z+1/2
y,-x,-z+1/2
-1/2x-1/2y+z+1/4,-1/2x-1/2y-z-1/4,-1/2x+1/2y-1/4
1/2x+1/2y+z+1/4,1/2x+1/2y-z-1/4,1/2x-1/2y-1/4
1/2x+1/2y-z-1/4,1/2x+1/2y+z+1/4,-1/2x+1/2y-1/4
-1/2x-1/2y-z-1/4,-1/2x-1/2y+z+1/4,1/2x-1/2y-1/4
-1/2x+1/2y-z-1/4,1/2x-1/2y-z-1/4,1/2x+1/2y-1/4
1/2x-1/2y-z-1/4,-1/2x+1/2y-z-1/4,-1/2x-1/2y-1/4
-1/2x+1/2y+z+1/4,1/2x-1/2y+z+1/4,-1/2x-1/2y-1/4
1/2x-1/2y+z+1/4,-1/2x+1/2y+z+1/4,1/2x+1/2y-1/4"""

        jft = JonesFaithfulTransformation.from_transformation_string(self.test_strings[1])

        input_symmops = [SymmOp.from_xyz_string(s) for s in input_symmops.split()]
        ref_transformed_symmops = [SymmOp.from_xyz_string(s) for s in ref_transformed_symmops.split()]

        transformed_symmops = [jft.transform_symmop(op) for op in input_symmops]

        for transformed_op, ref_transformed_op in zip(transformed_symmops, ref_transformed_symmops):
            self.assertEqual(transformed_op, ref_transformed_op)

if __name__ == '__main__':
    unittest.main()
<commit_msg>More LJT test fixes<commit_after>#!/usr/bin/env python

import unittest
import numpy as np

from pymatgen.symmetry.settings import *

__author__ = "Matthew Horton"
__copyright__ = "Copyright 2017, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Matthew Horton"
__email__ = "mkhorton@lbl.gov"
__status__ = "Development"
__date__ = "Apr 2017"

class JonesFaithfulTransformationTest(unittest.TestCase):

    def setUp(self):
        self.test_strings = ['a,b,c;0,0,0',  # identity
                             'a-b,a+b,2c;0,0,1/2',
                             'a/4+b/4-c/2,a/4-b/4,-a/2-b/2;0,0,0',
                             'a,b,c;1/4,1/2,3/4']  # pure translation
        self.test_Pps = [([[1, 0, 0], [0, 1, 0], [0, 0, 1]], [0, 0, 0]),
                         ([[1, 1, 0], [-1, 1, 0], [0, 0, 2]], [0, 0, 0.5]),
                         ([[0.25, 0.25, -0.5], [0.25, -0.25, -0.5], [-0.5, 0, 0]], [0, 0, 0]),
                         ([[1, 0, 0], [0, 1, 0], [0, 0, 1]], [0.25, 0.5, 0.75])]

    def test_init(self):
        for test_string, test_Pp in zip(self.test_strings, self.test_Pps):
            jft = JonesFaithfulTransformation.from_transformation_string(test_string)
            jft2 = JonesFaithfulTransformation(test_Pp[0], test_Pp[1])
            self.assertTrue(np.allclose(jft.P, jft2.P))
            self.assertTrue(np.allclose(jft.p, jft2.p))
            self.assertEqual(test_string, jft.transformation_string)
            self.assertEqual(test_string, jft2.transformation_string)

    def test_inverse(self):
        for test_string in self.test_strings:
            jft = JonesFaithfulTransformation.from_transformation_string(test_string)
            self.assertEqual(jft, jft.inverse.inverse)
            self.assertEqual(jft.transformation_string, jft.inverse.inverse.transformation_string)

    def test_transform_lattice(self):
        lattice = Lattice.cubic(5)

        all_ref_lattices = [[[5., 0., 0.], [0., 5., 0.], [0., 0., 5.]],
                            [[5., 5., 0.], [-5., 5., 0.], [0., 0., 10.]],
                            [[1.25, 1.25, -2.5], [1.25, -1.25, -2.5], [-2.5, 0., 0.]],
                            [[5., 0., 0.], [0., 5., 0.], [0., 0., 5.]]]

        for ref_lattice, (P, p) in zip(all_ref_lattices, self.test_Pps):
            jft = JonesFaithfulTransformation(P, p)
            self.assertTrue(np.allclose(jft.transform_lattice(lattice).matrix, ref_lattice))

    def test_transform_coords(self):
        coords = [[0, 0, 0], [0.5, 0.5, 0.5]]

        all_ref_coords = [[[0.0, 0.0, 0.0], [0.5, 0.5, 0.5]],
                          [[0.0, 0.0, -0.25], [0.0, 0.5, 0.0]],
                          [[0.0, 0.0, 0.0], [-1.0, 0.0, -1.5]],
                          [[-0.25, -0.5, -0.75], [0.25, 0.0, -0.25]]]

        for ref_coords, (P, p) in zip(all_ref_coords, self.test_Pps):
            jft = JonesFaithfulTransformation(P, p)
            transformed_coords = jft.transform_coords(coords)
            for coord, ref_coord in zip(transformed_coords, ref_coords):
                self.assertTrue(np.allclose(coord, ref_coord))

    def test_transform_symmops(self):

        # reference data for this test taken from GENPOS
        # http://cryst.ehu.es/cryst/get_gen.html

        # Fm-3m
        input_symmops = """x,y,z
-x,-y,z
-x,y,-z
x,-y,-z
z,x,y
z,-x,-y
-z,-x,y
-z,x,-y
y,z,x
-y,z,-x
y,-z,-x
-y,-z,x
y,x,-z
-y,-x,-z
y,-x,z
-y,x,z
x,z,-y
-x,z,y
-x,-z,-y
x,-z,y
z,y,-x
z,-y,x
-z,y,x
-z,-y,-x
-x,-y,-z
x,y,-z
x,-y,z
-x,y,z
-z,-x,-y
-z,x,y
z,x,-y
z,-x,y
-y,-z,-x
y,-z,x
-y,z,x
y,z,-x
-y,-x,z
y,x,z
-y,x,-z
y,-x,-z
-x,-z,y
x,-z,-y
x,z,y
-x,z,-y
-z,-y,x
-z,y,-x
z,-y,-x
z,y,x"""

        # Fm-3m transformed by (a-b,a+b,2c;0,0,1/2)
        ref_transformed_symmops = """x,y,z
-x,-y,z
-y,-x,-z+1/2
y,x,-z+1/2
-1/2x-1/2y+z+1/4,1/2x+1/2y+z+1/4,-1/2x+1/2y+3/4
1/2x+1/2y+z+1/4,-1/2x-1/2y+z+1/4,1/2x-1/2y+3/4
1/2x+1/2y-z+3/4,-1/2x-1/2y-z+3/4,-1/2x+1/2y+3/4
-1/2x-1/2y-z+3/4,1/2x+1/2y-z+3/4,1/2x-1/2y+3/4
-1/2x+1/2y-z+3/4,-1/2x+1/2y+z+1/4,1/2x+1/2y+3/4
1/2x-1/2y-z+3/4,1/2x-1/2y+z+1/4,-1/2x-1/2y+3/4
-1/2x+1/2y+z+1/4,-1/2x+1/2y-z+3/4,-1/2x-1/2y+3/4
1/2x-1/2y+z+1/4,1/2x-1/2y-z+3/4,1/2x+1/2y+3/4
-x,y,-z+1/2
x,-y,-z+1/2
y,-x,z
-y,x,z
1/2x+1/2y-z+3/4,1/2x+1/2y+z+1/4,1/2x-1/2y+3/4
-1/2x-1/2y-z+3/4,-1/2x-1/2y+z+1/4,-1/2x+1/2y+3/4
-1/2x-1/2y+z+1/4,-1/2x-1/2y-z+3/4,1/2x-1/2y+3/4
1/2x+1/2y+z+1/4,1/2x+1/2y-z+3/4,-1/2x+1/2y+3/4
1/2x-1/2y+z+1/4,-1/2x+1/2y+z+1/4,-1/2x-1/2y+3/4
-1/2x+1/2y+z+1/4,1/2x-1/2y+z+1/4,1/2x+1/2y+3/4
1/2x-1/2y-z+3/4,-1/2x+1/2y-z+3/4,1/2x+1/2y+3/4
-1/2x+1/2y-z+3/4,1/2x-1/2y-z+3/4,-1/2x-1/2y+3/4
-x,-y,-z+1/2
x,y,-z+1/2
y,x,z
-y,-x,z
1/2x+1/2y-z+3/4,-1/2x-1/2y-z+3/4,1/2x-1/2y+3/4
-1/2x-1/2y-z+3/4,1/2x+1/2y-z+3/4,-1/2x+1/2y+3/4
-1/2x-1/2y+z+1/4,1/2x+1/2y+z+1/4,1/2x-1/2y+3/4
1/2x+1/2y+z+1/4,-1/2x-1/2y+z+1/4,-1/2x+1/2y+3/4
1/2x-1/2y+z+1/4,1/2x-1/2y-z+3/4,-1/2x-1/2y+3/4
-1/2x+1/2y+z+1/4,-1/2x+1/2y-z+3/4,1/2x+1/2y+3/4
1/2x-1/2y-z+3/4,1/2x-1/2y+z+1/4,1/2x+1/2y+3/4
-1/2x+1/2y-z+3/4,-1/2x+1/2y+z+1/4,-1/2x-1/2y+3/4
x,-y,z
-x,y,z
-y,x,-z+1/2
y,-x,-z+1/2
-1/2x-1/2y+z+1/4,-1/2x-1/2y-z+3/4,-1/2x+1/2y+3/4
1/2x+1/2y+z+1/4,1/2x+1/2y-z+3/4,1/2x-1/2y+3/4
1/2x+1/2y-z+3/4,1/2x+1/2y+z+1/4,-1/2x+1/2y+3/4
-1/2x-1/2y-z+3/4,-1/2x-1/2y+z+1/4,1/2x-1/2y+3/4
-1/2x+1/2y-z+3/4,1/2x-1/2y-z+3/4,1/2x+1/2y+3/4
1/2x-1/2y-z+3/4,-1/2x+1/2y-z+3/4,-1/2x-1/2y+3/4
-1/2x+1/2y+z+1/4,1/2x-1/2y+z+1/4,-1/2x-1/2y+3/4
1/2x-1/2y+z+1/4,-1/2x+1/2y+z+1/4,1/2x+1/2y+3/4"""

        jft = JonesFaithfulTransformation.from_transformation_string(self.test_strings[1])

        input_symmops = [SymmOp.from_xyz_string(s) for s in input_symmops.split()]
        ref_transformed_symmops = [SymmOp.from_xyz_string(s) for s in ref_transformed_symmops.split()]

        transformed_symmops = [jft.transform_symmop(op) for op in input_symmops]

        for transformed_op, ref_transformed_op in zip(transformed_symmops, ref_transformed_symmops):
            self.assertEqual(transformed_op, ref_transformed_op)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import synapse.lib.atomfile as s_atomfile
import synapse.lib.blobfile as s_blobfile

from synapse.tests.common import *

class BlobFileTest(SynTest):

    def blobfile_basic_assumptions(self, fd, blob):
        '''
        Basic assumptions for a Blobfile

        Args:
            fd (file): The FD backing the blob
            blob (s_blobfile.BlobFile): BlobFile under test.
        '''
        self.eq(blob.size(), 0)

        off0 = blob.alloc(8)

        self.eq(blob.size(), 8 + s_blobfile.headsize)
        # The blob and atomfile grow in lockstep
        self.eq(blob.atom.size, blob.size())

        off1 = blob.alloc(8)

        # do interlaced writes
        blob.writeoff(off0, b'asdf')
        blob.writeoff(off1, b'hehe')

        blob.writeoff(off0 + 4, b'qwer')
        blob.writeoff(off1 + 4, b'haha')

        self.eq(blob.readoff(off0, 8), b'asdfqwer')
        self.eq(blob.readoff(off1, 8), b'hehehaha')

        # Do a third allocation and write
        off2 = blob.alloc(8)
        # Ensure our alloc did not smash old data
        self.eq(blob.readoff(off1, 8), b'hehehaha')
        byts = 4 * b':)'
        blob.writeoff(off2, byts)

        self.eq(blob.readoff(off0, 8), b'asdfqwer')
        self.eq(blob.readoff(off1, 8), b'hehehaha')
        self.eq(blob.readoff(off2, 8), byts)

        # The blob and atomfile continue too grow in lockstep
        self.eq(blob.atom.size, blob.size())
        esisze = (3 * (s_blobfile.headsize + 8))
        self.eq(blob.size(), esisze)

        # Ensure bad reads are caught
        self.raises(BadBlobFile, blob.readoff, blob.size() - 4, 8)

    def test_blob_base(self):

        # FIXME test these on windows...
        #self.thisHostMust(platform='linux')

        fd = tempfile.TemporaryFile()
        blob = s_blobfile.BlobFile(fd)
        self.blobfile_basic_assumptions(fd, blob)

        blob.fini()
        # fini the blob closes the underlying fd
        self.true(fd.closed)

    def test_blob_simple_atom(self):
        # FIXME test these on windows...
        #self.thisHostMust(platform='linux')

        fd = tempfile.TemporaryFile()
        atom = s_atomfile.AtomFile(fd)
        blob = s_blobfile.BlobFile(fd, atom=atom)
        self.blobfile_basic_assumptions(fd, blob)

        blob.fini()
        # fini the blob does not close the underlying fd since
        # atom was passed in directly
        self.false(fd.closed)
        atom.fini()
        self.true(fd.closed)

    def test_blob_resize(self):

        fd = tempfile.TemporaryFile()

        with s_blobfile.BlobFile(fd) as blob:  # type: s_blobfile.BlobFile

            blocks = []
            w = blob.waiter(5, 'blob:alloc')
            esize = (s_blobfile.headsize + 8) * 5
            while blob.size() < esize:
                blocks.append(blob.alloc(8))

            self.eq(w.count, 5)
            w.fini()
            self.eq(blob.size(), esize)

            # Firing a single event will work properly.
            # This is for test coverage - normal use case would
            # not use syncing except in the file-duplication case

            mesg0 = ('blob:alloc', {'size': 32})
            mesg = ('blob:sync', {'mesg': mesg0})
            blob.sync(mesg)
            # Ensure the blob was changed
            self.eq(blob.size(), esize + 32 + s_blobfile.headsize)

    def test_blob_save(self):

        #self.thisHostMust(platform='linux')

        msgs = []

        fd0 = tempfile.TemporaryFile()
        blob0 = s_blobfile.BlobFile(fd0)

        blob0.on('blob:sync', msgs.append)

        off0 = blob0.alloc(8)
        off1 = blob0.alloc(8)

        # do interlaced writes
        blob0.writeoff(off0, b'asdf')
        blob0.writeoff(off1, b'hehe')

        blob0.writeoff(off0 + 4, b'qwer')
        blob0.writeoff(off1 + 4, b'haha')

        fd1 = tempfile.TemporaryFile()

        blob1 = s_blobfile.BlobFile(fd1, isclone=True)
        blob1.syncs(msgs)

        self.eq(blob0.readoff(off0, 8), blob1.readoff(off0, 8))
        self.eq(blob0.readoff(off1, 8), blob1.readoff(off1, 8))

        self.eq(blob0.size(), blob1.size())

        # Replaying messages to blob0 doesn't do anything to is since
        # the reactor is not wired up
        blob0.syncs(msgs)
        self.eq(blob0.size(), blob1.size())

        # Calling alloc / writeoff apis to the clone fails
        self.raises(BlobFileIsClone, blob1.alloc, 1)
        self.raises(BlobFileIsClone, blob1.writeoff, 1, b'1')

        blob0.fini()
        blob1.fini()

    def test_blob_readiter(self):
        #self.thisHostMust(platform='linux')

        fd = tempfile.TemporaryFile()

        with s_blobfile.BlobFile(fd) as blob:

            rand = os.urandom(2048)
            off = blob.alloc(2048)
            blob.writeoff(off, rand)

            blocks = [b for b in blob.readiter(off, 2048, itersize=9)]
            byts = b''.join(blocks)

            self.eq(rand, byts)

    def test_blob_walk(self):
        edata = {}
        with self.getTestDir() as fdir:
            fp = os.path.join(fdir, 'test.blob')
            bkup_fp = fp + '.bak'
            fd = genfile(fp)

            with s_blobfile.BlobFile(fd) as blob:  # type: s_blobfile.BlobFile

                off0 = blob.alloc(8)
                off1 = blob.alloc(8)

                edata[off0] = b'asdfqwer'
                edata[off1] = b'hehehaha'

                # do interlaced writes
                blob.writeoff(off0, b'asdf')
                blob.writeoff(off1, b'hehe')

                blob.writeoff(off0 + 4, b'qwer')
                blob.writeoff(off1 + 4, b'haha')
                # self.eq(blob.readoff(off1, 8), edata[off1])
                # Do a large write
                off2 = blob.alloc(1024)
                byts = 512 * b':)'
                blob.writeoff(off2, byts)
                edata[off2] = byts

            # Backup the file
            shutil.copy(fp, bkup_fp)

            fd = genfile(fp)
            with s_blobfile.BlobFile(fd) as blob:  # type: s_blobfile.BlobFile
                dcheck = {}

                def data_check(fd, baseoff, off, size):
                    fd.seek(off)
                    byts = fd.read(size)
                    dcheck[off] = byts == edata.get(off)

                w1 = blob.waiter(1, 'blob:walk:done')
                blob.walk(callback=data_check)
                w1.wait(1)
                self.eq(w1.count, 1)
                self.eq({True}, set(dcheck.values()))

            # Restore backup file
            shutil.copy(bkup_fp, fp)
            fd = genfile(fp)

            # Truncate the file short
            fd.seek(-2, os.SEEK_END)
            fd.truncate()
            with s_blobfile.BlobFile(fd) as blob:  # type: s_blobfile.BlobFile
                w0 = blob.waiter(1, 'blob:walk:truncated')
                w1 = blob.waiter(1, 'blob:walk:done')
                blob.walk()
                w0.wait(1)
                w1.wait(1)
                self.eq(w0.count, 1)
                self.eq(w1.count, 1)
                w0.fini()
                w1.fini()

            # Restore backup file
            shutil.copy(bkup_fp, fp)
            fd = genfile(fp)

            # Add bytes
            fd.seek(0, os.SEEK_END)
            fd.write(b':(')
            with s_blobfile.BlobFile(fd) as blob:  # type: s_blobfile.BlobFile
                w0 = blob.waiter(1, 'blob:walk:unpkerr')
                w1 = blob.waiter(1, 'blob:walk:done')
                blob.walk()
                w0.wait(1)
                w1.wait(1)
                self.eq(w0.count, 1)
                self.eq(w1.count, 1)
                w0.fini()
                w1.fini()
<commit_msg>Fix blobfile test<commit_after>import synapse.lib.atomfile as s_atomfile
import synapse.lib.blobfile as s_blobfile

from synapse.tests.common import *

class BlobFileTest(SynTest):

    def blobfile_basic_assumptions(self, fd, blob):
        '''
        Basic assumptions for a Blobfile

        Args:
            fd (file): The FD backing the blob
            blob (s_blobfile.BlobFile): BlobFile under test.
        '''
        self.false(blob.isclone)
        self.eq(blob.size(), 0)

        off0 = blob.alloc(8)

        self.eq(blob.size(), 8 + s_blobfile.headsize)
        # The blob and atomfile grow in lockstep
        self.eq(blob.atom.size, blob.size())

        off1 = blob.alloc(8)

        # do interlaced writes
        blob.writeoff(off0, b'asdf')
        blob.writeoff(off1, b'hehe')

        blob.writeoff(off0 + 4, b'qwer')
        blob.writeoff(off1 + 4, b'haha')

        self.eq(blob.readoff(off0, 8), b'asdfqwer')
        self.eq(blob.readoff(off1, 8), b'hehehaha')

        # Do a third allocation and write
        off2 = blob.alloc(8)
        # Ensure our alloc did not smash old data
        self.eq(blob.readoff(off1, 8), b'hehehaha')
        byts = 4 * b':)'
        blob.writeoff(off2, byts)

        self.eq(blob.readoff(off0, 8), b'asdfqwer')
        self.eq(blob.readoff(off1, 8), b'hehehaha')
        self.eq(blob.readoff(off2, 8), byts)

        # The blob and atomfile continue too grow in lockstep
        self.eq(blob.atom.size, blob.size())
        esisze = (3 * (s_blobfile.headsize + 8))
        self.eq(blob.size(), esisze)

        # Ensure bad reads are caught
        self.raises(BadBlobFile, blob.readoff, blob.size() - 4, 8)

    def test_blob_base(self):

        # FIXME test these on windows...
        #self.thisHostMust(platform='linux')

        fd = tempfile.TemporaryFile()
        blob = s_blobfile.BlobFile(fd)
        self.blobfile_basic_assumptions(fd, blob)

        blob.fini()
        # fini the blob closes the underlying fd
        self.true(fd.closed)

    def test_blob_simple_atom(self):
        # FIXME test these on windows...
        #self.thisHostMust(platform='linux')

        fd = tempfile.TemporaryFile()
        atom = s_atomfile.AtomFile(fd)
        blob = s_blobfile.BlobFile(fd, atom=atom)
        self.blobfile_basic_assumptions(fd, blob)

        blob.fini()
        # fini the blob does not close the underlying fd since
        # atom was passed in directly
        self.false(fd.closed)
        atom.fini()
        self.true(fd.closed)

    def test_blob_resize(self):

        fd = tempfile.TemporaryFile()

        with s_blobfile.BlobFile(fd) as blob:  # type: s_blobfile.BlobFile

            blocks = []
            w = blob.waiter(5, 'blob:alloc')
            esize = (s_blobfile.headsize + 8) * 5
            while blob.size() < esize:
                blocks.append(blob.alloc(8))

            self.eq(w.count, 5)
            w.fini()
            self.eq(blob.size(), esize)

            # Firing a single event will work properly.
            # This is for test coverage - normal use case would
            # not use syncing except in the file-duplication case

            mesg0 = ('blob:alloc', {'size': 32})
            mesg = ('blob:sync', {'mesg': mesg0})
            blob.sync(mesg)
            # Ensure the blob was not changed since isclone is false
            self.eq(blob.size(), esize)

    def test_blob_save(self):

        #self.thisHostMust(platform='linux')

        msgs = []

        fd0 = tempfile.TemporaryFile()
        blob0 = s_blobfile.BlobFile(fd0)

        blob0.on('blob:sync', msgs.append)

        off0 = blob0.alloc(8)
        off1 = blob0.alloc(8)

        # do interlaced writes
        blob0.writeoff(off0, b'asdf')
        blob0.writeoff(off1, b'hehe')

        blob0.writeoff(off0 + 4, b'qwer')
        blob0.writeoff(off1 + 4, b'haha')

        fd1 = tempfile.TemporaryFile()

        blob1 = s_blobfile.BlobFile(fd1, isclone=True)
        self.true(blob1.isclone)
        blob1.syncs(msgs)

        self.eq(blob0.readoff(off0, 8), blob1.readoff(off0, 8))
        self.eq(blob0.readoff(off1, 8), blob1.readoff(off1, 8))

        self.eq(blob0.size(), blob1.size())

        # Replaying messages to blob0 doesn't do anything to is since
        # the reactor is not wired up
        blob0.syncs(msgs)
        self.eq(blob0.size(), blob1.size())

        # Calling alloc / writeoff apis to the clone fails
        self.raises(BlobFileIsClone, blob1.alloc, 1)
        self.raises(BlobFileIsClone, blob1.writeoff, 1, b'1')

        blob0.fini()
        blob1.fini()

    def test_blob_readiter(self):
        #self.thisHostMust(platform='linux')

        fd = tempfile.TemporaryFile()

        with s_blobfile.BlobFile(fd) as blob:

            rand = os.urandom(2048)
            off = blob.alloc(2048)
            blob.writeoff(off, rand)

            blocks = [b for b in blob.readiter(off, 2048, itersize=9)]
            byts = b''.join(blocks)

            self.eq(rand, byts)

    def test_blob_walk(self):
        edata = {}
        with self.getTestDir() as fdir:
            fp = os.path.join(fdir, 'test.blob')
            bkup_fp = fp + '.bak'
            fd = genfile(fp)

            with s_blobfile.BlobFile(fd) as blob:  # type: s_blobfile.BlobFile

                off0 = blob.alloc(8)
                off1 = blob.alloc(8)

                edata[off0] = b'asdfqwer'
                edata[off1] = b'hehehaha'

                # do interlaced writes
                blob.writeoff(off0, b'asdf')
                blob.writeoff(off1, b'hehe')

                blob.writeoff(off0 + 4, b'qwer')
                blob.writeoff(off1 + 4, b'haha')
                # self.eq(blob.readoff(off1, 8), edata[off1])
                # Do a large write
                off2 = blob.alloc(1024)
                byts = 512 * b':)'
                blob.writeoff(off2, byts)
                edata[off2] = byts

            # Backup the file
            shutil.copy(fp, bkup_fp)

            fd = genfile(fp)
            with s_blobfile.BlobFile(fd) as blob:  # type: s_blobfile.BlobFile
                dcheck = {}

                def data_check(fd, baseoff, off, size):
                    fd.seek(off)
                    byts = fd.read(size)
                    dcheck[off] = byts == edata.get(off)

                w1 = blob.waiter(1, 'blob:walk:done')
                blob.walk(callback=data_check)
                w1.wait(1)
                self.eq(w1.count, 1)
                self.eq({True}, set(dcheck.values()))

            # Restore backup file
            shutil.copy(bkup_fp, fp)
            fd = genfile(fp)

            # Truncate the file short
            fd.seek(-2, os.SEEK_END)
            fd.truncate()
            with s_blobfile.BlobFile(fd) as blob:  # type: s_blobfile.BlobFile
                w0 = blob.waiter(1, 'blob:walk:truncated')
                w1 = blob.waiter(1, 'blob:walk:done')
                blob.walk()
                w0.wait(1)
                w1.wait(1)
                self.eq(w0.count, 1)
                self.eq(w1.count, 1)
                w0.fini()
                w1.fini()

            # Restore backup file
            shutil.copy(bkup_fp, fp)
            fd = genfile(fp)

            # Add bytes
            fd.seek(0, os.SEEK_END)
            fd.write(b':(')
            with s_blobfile.BlobFile(fd) as blob:  # type: s_blobfile.BlobFile
                w0 = blob.waiter(1, 'blob:walk:unpkerr')
                w1 = blob.waiter(1, 'blob:walk:done')
                blob.walk()
                w0.wait(1)
                w1.wait(1)
                self.eq(w0.count, 1)
                self.eq(w1.count, 1)
                w0.fini()
                w1.fini()
<|endoftext|>


<commit_before>"""
Codes for correcting the differential phase and estimating KDP.

@title: phase
@author: Valentin Louf <valentin.louf@monash.edu>
@institutions: Monash University and the Australian Bureau of Meteorology
@date: 20/11/2017

.. autosummary::
    :toctree: generated/

    check_phidp
    fix_phidp_from_kdp
    phidp_bringi
    phidp_giangrande
    unfold_raw_phidp
"""
# Python Standard Library
import time
import datetime

# Other Libraries
import pyart
import scipy
import netCDF4
import numpy as np

from scipy import integrate
from csu_radartools import csu_kdp

def check_phidp(radar, phi_name="PHIDP"):
    """
    Check if PHIDP range is 180 degrees (half-circle) or 360 degrees.
    Parameters:
    ===========
    radar:
        Py-ART radar structure.
    phi_name: str
        Name of the PHIDP field.

    Return:
    =======
    half_phi: bool
        Is PHIDP range is half circle.
    """
    phi = radar.fields[phi_name]['data']
    if phi.max() - phi.min() <= 200:  # 180 degrees plus some margin for noise...
        half_phi = True
    else:
        half_phi = False

    return half_phi

def fix_phidp_from_kdp(radar, gatefilter, kdp_name="KDP_BRINGI", phidp_name="PHIDP_BRINGI"):
    """
    Correct PHIDP and KDP from spider webs.

    Parameters
    ==========
    radar:
        Py-ART radar data structure.
    gatefilter:
        Gate filter.
    kdp_name: str
        Differential phase key name.
    phidp_name: str
        Differential phase key name.

    Returns:
    ========
    phidp: ndarray
        Differential phase array.
    """
    kdp = radar.fields[kdp_name]['data'].copy()
    phidp = radar.fields[phidp_name]['data'].copy()
    kdp[gatefilter.gate_excluded] = 0
    kdp[(kdp < -2)] = 0
    kdp[kdp > 10] = 10
    interg = integrate.cumtrapz(kdp, radar.range['data'], axis=1)

    phidp[:, :-1] = interg / (len(radar.range['data']))
    return phidp

def phidp_bringi(radar, gatefilter, unfold_phidp_name="PHI_UNF", ncp_name="NCP", rhohv_name="RHOHV_CORR", refl_field='DBZ'):
    """
    Compute PHIDP and KDP Bringi.

    Parameters
    ==========
    radar:
        Py-ART radar data structure.
    gatefilter:
        Gate filter.
    unfold_phidp_name: str
        Differential phase key name.
    refl_field: str
        Reflectivity key name.

    Returns:
    ========
    phidpb: ndarray
        Bringi differential phase array.
    kdpb: ndarray
        Bringi specific differential phase array.
    """
    dp = radar.fields[unfold_phidp_name]['data'].copy()
    dz = radar.fields[refl_field]['data'].copy().filled(-9999)

    # Extract dimensions
    rng = radar.range['data']
    azi = radar.azimuth['data']
    dgate = rng[1] - rng[0]
    [R, A] = np.meshgrid(rng, azi)

    # Compute KDP bringi.
    kdpb, phidpb, _ = csu_kdp.calc_kdp_bringi(dp, dz, R / 1e3, gs=dgate, bad=-9999)

    # Mask array
    phidpb = np.ma.masked_where(phidpb == -9999, phidpb)
    kdpb = np.ma.masked_where(kdpb == -9999, kdpb)

    # Get metadata.
    phimeta = pyart.config.get_metadata("differential_phase")
    phimeta['data'] = phidpb
    kdpmeta = pyart.config.get_metadata("specific_differential_phase")
    kdpmeta['data'] = kdpb

    return phimeta, kdpmeta

def phidp_giangrande(radar, gatefilter, refl_field='DBZ', ncp_field='NCP',
                     rhv_field='RHOHV_CORR', phidp_field='PHI_UNF'):
    """
    Phase processing using the LP method in Py-ART. A LP solver is required,

    Parameters:
    ===========
        radar:
            Py-ART radar structure.
        refl_field: str
            Reflectivity field name.
        ncp_field: str
            Normalised coherent power field name.
        rhv_field: str
            Cross correlation ration field name.
        phidp_field: str
            Phase field name.

    Returns:
    ========
        phidp_gg: dict
            Field dictionary containing processed differential phase shifts.
        kdp_gg: dict
            Field dictionary containing recalculated differential phases.
    """
    phidp_gg, kdp_gg = pyart.correct.phase_proc_lp(radar, 0.0,
                                                   min_rhv=0.95,
                                                   LP_solver='cylp',
                                                   refl_field=refl_field,
                                                   ncp_field=ncp_field,
                                                   rhv_field=rhv_field,
                                                   phidp_field=phidp_field)

    return phidp_gg, kdp_gg

def unfold_raw_phidp(radar, gatefilter, phi_name="PHIDP"):
    """
    Unfold raw PHIDP

    Parameters:
    ===========
    radar:
        Py-ART radar structure.
    gatefilter:
        Gate filter.
    phi_name: str
        Name of the PHIDP field.

    Returns:
    ========
    tru_phi: ndarray
        Unfolded raw PHIDP.
    """
    # Extract data
    phi = radar.fields[phi_name]['data'].copy()
    # For CPOL, PHIDP is properly unfolded before season 2003/2004
    CPOL_DATE_PHIDP_FOLD = datetime.datetime(2003, 10, 1)
    dtime = netCDF4.num2date(radar.time['data'][0], radar.time['units'])
    if dtime < CPOL_DATE_PHIDP_FOLD:
        tru_phi = phi
    else:
        tru_phi = phidp_unfold + 225
        pmin = phi[gatefilter.gate_included].min()
        tru_phi -= pmin
        tru_phi[tru_phi > 360] -= 360

    return tru_phi
<commit_msg>no more masked array in unfolding phidp<commit_after>"""
Codes for correcting the differential phase and estimating KDP.

@title: phase
@author: Valentin Louf <valentin.louf@monash.edu>
@institutions: Monash University and the Australian Bureau of Meteorology
@date: 20/11/2017

.. autosummary::
    :toctree: generated/

    check_phidp
    fix_phidp_from_kdp
    phidp_bringi
    phidp_giangrande
    unfold_raw_phidp
"""
# Python Standard Library
import time
import datetime

# Other Libraries
import pyart
import scipy
import netCDF4
import numpy as np

from scipy import integrate
from csu_radartools import csu_kdp

def check_phidp(radar, phi_name="PHIDP"):
    """
    Check if PHIDP range is 180 degrees (half-circle) or 360 degrees.
    Parameters:
    ===========
    radar:
        Py-ART radar structure.
    phi_name: str
        Name of the PHIDP field.

    Return:
    =======
    half_phi: bool
        Is PHIDP range is half circle.
    """
    phi = radar.fields[phi_name]['data']
    if phi.max() - phi.min() <= 200:  # 180 degrees plus some margin for noise...
        half_phi = True
    else:
        half_phi = False

    return half_phi

def fix_phidp_from_kdp(radar, gatefilter, kdp_name="KDP_BRINGI", phidp_name="PHIDP_BRINGI"):
    """
    Correct PHIDP and KDP from spider webs.

    Parameters
    ==========
    radar:
        Py-ART radar data structure.
    gatefilter:
        Gate filter.
    kdp_name: str
        Differential phase key name.
    phidp_name: str
        Differential phase key name.

    Returns:
    ========
    phidp: ndarray
        Differential phase array.
    """
    kdp = radar.fields[kdp_name]['data'].copy()
    phidp = radar.fields[phidp_name]['data'].copy()
    kdp[gatefilter.gate_excluded] = 0
    kdp[(kdp < -2)] = 0
    kdp[kdp > 10] = 10
    interg = integrate.cumtrapz(kdp, radar.range['data'], axis=1)

    phidp[:, :-1] = interg / (len(radar.range['data']))
    return phidp

def phidp_bringi(radar, gatefilter, unfold_phidp_name="PHI_UNF", ncp_name="NCP", rhohv_name="RHOHV_CORR", refl_field='DBZ'):
    """
    Compute PHIDP and KDP Bringi.

    Parameters
    ==========
    radar:
        Py-ART radar data structure.
    gatefilter:
        Gate filter.
    unfold_phidp_name: str
        Differential phase key name.
    refl_field: str
        Reflectivity key name.

    Returns:
    ========
    phidpb: ndarray
        Bringi differential phase array.
    kdpb: ndarray
        Bringi specific differential phase array.
    """
    dp = radar.fields[unfold_phidp_name]['data'].copy()
    dz = radar.fields[refl_field]['data'].copy().filled(-9999)

    # Extract dimensions
    rng = radar.range['data']
    azi = radar.azimuth['data']
    dgate = rng[1] - rng[0]
    [R, A] = np.meshgrid(rng, azi)

    # Compute KDP bringi.
    kdpb, phidpb, _ = csu_kdp.calc_kdp_bringi(dp, dz, R / 1e3, gs=dgate, bad=-9999)

    # Mask array
    phidpb = np.ma.masked_where(phidpb == -9999, phidpb)
    kdpb = np.ma.masked_where(kdpb == -9999, kdpb)

    # Get metadata.
    phimeta = pyart.config.get_metadata("differential_phase")
    phimeta['data'] = phidpb
    kdpmeta = pyart.config.get_metadata("specific_differential_phase")
    kdpmeta['data'] = kdpb

    return phimeta, kdpmeta

def phidp_giangrande(radar, gatefilter, refl_field='DBZ', ncp_field='NCP',
                     rhv_field='RHOHV_CORR', phidp_field='PHI_UNF'):
    """
    Phase processing using the LP method in Py-ART. A LP solver is required,

    Parameters:
    ===========
        radar:
            Py-ART radar structure.
        refl_field: str
            Reflectivity field name.
        ncp_field: str
            Normalised coherent power field name.
        rhv_field: str
            Cross correlation ration field name.
        phidp_field: str
            Phase field name.

    Returns:
    ========
        phidp_gg: dict
            Field dictionary containing processed differential phase shifts.
        kdp_gg: dict
            Field dictionary containing recalculated differential phases.
    """
    phidp_gg, kdp_gg = pyart.correct.phase_proc_lp(radar, 0.0,
                                                   min_rhv=0.95,
                                                   LP_solver='cylp',
                                                   refl_field=refl_field,
                                                   ncp_field=ncp_field,
                                                   rhv_field=rhv_field,
                                                   phidp_field=phidp_field)

    return phidp_gg, kdp_gg

def unfold_raw_phidp(radar, gatefilter, phi_name="PHIDP"):
    """
    Unfold raw PHIDP

    Parameters:
    ===========
    radar:
        Py-ART radar structure.
    gatefilter:
        Gate filter.
    phi_name: str
        Name of the PHIDP field.

    Returns:
    ========
    tru_phi: ndarray
        Unfolded raw PHIDP.
    """
    # Extract data
    phi = radar.fields[phi_name]['data'].copy()
    # For CPOL, PHIDP is properly unfolded before season 2003/2004
    CPOL_DATE_PHIDP_FOLD = datetime.datetime(2003, 10, 1)
    dtime = netCDF4.num2date(radar.time['data'][0], radar.time['units'])
    if dtime < CPOL_DATE_PHIDP_FOLD:
        tru_phi = phi
    else:
        tru_phi = phi + 225
        pmin = phi[gatefilter.gate_included].min()
        tru_phi -= pmin
        tru_phi[tru_phi > 360] -= 360

    return tru_phi
<|endoftext|>


<commit_before>from openvenues.jobs.common_crawl import *
from openvenues.extract.soup import *

contains_microdata_regex = re.compile('vcard|itemtype|typeof|maps\.google|(?:(?:google.[a-z]+(?:\.[a-z]{2,3}){0,1})|(?:goo.gl))/maps|property|og:', re.I | re.UNICODE)

logger = logging.getLogger('microdata_job')

class MicrodataJob(CommonCrawlJob):
    valid_charsets = set(['utf-8', 'iso-8859-1', 'latin-1', 'ascii'])

    def prefix_key(self, prefix, key):
        return u'|'.join([prefix, key])

    def encoding_from_headers(self, content_type):
        if not content_type:
            return None

        content_type, params = cgi.parse_header(content_type)

        if 'charset' in params:
            return params['charset'].strip("'\"")

        if 'text' in content_type:
            return 'ISO-8859-1'

    def detect_encoding(self, content):
        return cchardet.detect(content)['encoding']

    def report_items(self, items):
        self.increment_counter('commoncrawl', 'sites with places', 1)
        for item in items:
            item_type = item.get('item_type')
            for prop in item:
                if prop != 'item_type':
                    self.increment_counter('commoncrawl', u':'.join([item_type, prop]), 1)

    def report_social(self, social):
        for k, vals in social.iteritems():
            self.increment_counter('commoncrawl', 'url type {}'.format(k), len(values))

    def process_record(self, record):
        content = None
        try:
            payload = record.payload.read()
            s = FakeSocket(payload)
            response = HTTPResponse(s)
            response.begin()

            status_code = response.status
            if status_code != 200:
                return

            content_type = response.getheader('Content-Type', '')
            if 'text/html' not in content_type:
                return

            content = response.read(len(payload))
        except Exception:
            return

        if content is not None:
            content = content.strip()

        if not content:
            return

        html = None

        try:
            encoding = self.encoding_from_headers(content_type)
            if encoding and encoding.lower() not in self.valid_charsets:
                return

            doc = UnicodeDammit(content, is_html=True)
            if not doc.unicode_markup or (doc.original_encoding and doc.original_encoding.lower() not in self.valid_charsets) or not contains_microdata_regex.search(doc.unicode_markup):
                return

            doc = doc.unicode_markup

            soup = BeautifulSoup(doc)

            have_item_scope = False

            items = []

            schema_dot_org_items = extract_schema_dot_org(soup)
            rdfa_items = extract_schema_dot_org(soup, use_rdfa=True)
            vcards = extract_vcards(soup)
            address_elements = extract_address_elements(soup)
            geotags = extract_geotags(soup)
            opengraph_tags = extract_opengraph_tags(soup)
            google_maps_embeds = extract_google_map_embeds(soup)

            basic_metadata = extract_basic_metadata(soup)

            items = list(chain(*(c for c in (schema_dot_org_items, rdfa_items, vcards, address_elements, geotags, google_maps_embeds) if c)))
            if opengraph_tags:
                i = opengraph_item(opengraph_tags)
                if i:
                    items.append(i)

            social_handles = extract_social_handles(soup)

            ret = {}
            if items:
                self.report_items(items)
                ret['items'] = items

                if social_handles:
                    ret['social'] = social_handles
                    self.report_social(social_handles)

                if og_tags:
                    ret['og'] = og_tags

                if basic_metadata:
                    ret.update(basic_metadata)

                yield record.url, ret
                items = []

        except Exception as e:
            logger.error(traceback.format_exc())
        finally:
            if html is not None:
                html.clear()
            return

if __name__ == '__main__':
   MicrodataJob.run() <commit_msg>[fix] derp<commit_after>from openvenues.jobs.common_crawl import *
from openvenues.extract.soup import *

contains_microdata_regex = re.compile('vcard|itemtype|typeof|maps\.google|(?:(?:google.[a-z]+(?:\.[a-z]{2,3}){0,1})|(?:goo.gl))/maps|property|og:', re.I | re.UNICODE)

logger = logging.getLogger('microdata_job')

class MicrodataJob(CommonCrawlJob):
    valid_charsets = set(['utf-8', 'iso-8859-1', 'latin-1', 'ascii'])

    def prefix_key(self, prefix, key):
        return u'|'.join([prefix, key])

    def encoding_from_headers(self, content_type):
        if not content_type:
            return None

        content_type, params = cgi.parse_header(content_type)

        if 'charset' in params:
            return params['charset'].strip("'\"")

        if 'text' in content_type:
            return 'ISO-8859-1'

    def detect_encoding(self, content):
        return cchardet.detect(content)['encoding']

    def report_items(self, items):
        self.increment_counter('commoncrawl', 'sites with places', 1)
        for item in items:
            item_type = item.get('item_type')
            for prop in item:
                if prop != 'item_type':
                    self.increment_counter('commoncrawl', u':'.join([item_type, prop]), 1)

    def report_social(self, social):
        for k, vals in social.iteritems():
            self.increment_counter('commoncrawl', 'url type {}'.format(k), len(vals))

    def process_record(self, record):
        content = None
        try:
            payload = record.payload.read()
            s = FakeSocket(payload)
            response = HTTPResponse(s)
            response.begin()

            status_code = response.status
            if status_code != 200:
                return

            content_type = response.getheader('Content-Type', '')
            if 'text/html' not in content_type:
                return

            content = response.read(len(payload))
        except Exception:
            return

        if content is not None:
            content = content.strip()

        if not content:
            return

        html = None

        try:
            encoding = self.encoding_from_headers(content_type)
            if encoding and encoding.lower() not in self.valid_charsets:
                return

            doc = UnicodeDammit(content, is_html=True)
            if not doc.unicode_markup or (doc.original_encoding and doc.original_encoding.lower() not in self.valid_charsets) or not contains_microdata_regex.search(doc.unicode_markup):
                return

            doc = doc.unicode_markup

            soup = BeautifulSoup(doc)

            have_item_scope = False

            items = []

            schema_dot_org_items = extract_schema_dot_org(soup)
            rdfa_items = extract_schema_dot_org(soup, use_rdfa=True)
            vcards = extract_vcards(soup)
            address_elements = extract_address_elements(soup)
            geotags = extract_geotags(soup)
            opengraph_tags = extract_opengraph_tags(soup)
            google_maps_embeds = extract_google_map_embeds(soup)

            basic_metadata = extract_basic_metadata(soup)

            items = list(chain(*(c for c in (schema_dot_org_items, rdfa_items, vcards, address_elements, geotags, google_maps_embeds) if c)))
            if opengraph_tags:
                i = opengraph_item(opengraph_tags)
                if i:
                    items.append(i)

            social_handles = extract_social_handles(soup)

            ret = {}
            if items:
                self.report_items(items)
                ret['items'] = items

                if social_handles:
                    ret['social'] = social_handles
                    self.report_social(social_handles)

                if og_tags:
                    ret['og'] = og_tags

                if basic_metadata:
                    ret.update(basic_metadata)

                yield record.url, ret
                items = []

        except Exception as e:
            logger.error(traceback.format_exc())
        finally:
            if html is not None:
                html.clear()
            return

if __name__ == '__main__':
   MicrodataJob.run() <|endoftext|>


<commit_before>import unittest
from pymsascoring.distancematrix.impl.pam250 import PAM250
from pymsascoring.distancematrix.impl.blosum62 import Blosum62
from pymsascoring.impl.sumofpairs import SumOfPairs

class TestMethods(unittest.TestCase):

    def setUp(self):
        print("setUp: INICIANDO TEST")

    def tearDown(self):
        print("tearDown: FINALIZANDO TEST")

    def test_basic_score_of_12_with_PAM250(self):
        print("Ejecutando test1")
        matrix = PAM250()
        seqs = [('ID1',
                 'AA'),
                ('ID2',
                 'AA'),
                ('ID3',
                 'AA')]
        final_score = SumOfPairs(seqs, matrix)
        self.assertEqual(12, final_score.calc_final_score())

    def test_basic_score_of_12_with_BLOSUM62(self):
        print("Ejecutando test2")
        matrix = Blosum62()
        seqs = [('ID1',
                 'AA'),
                ('ID2',
                 'AA'),
                ('ID3',
                 'AA')]
        final_score = SumOfPairs(seqs, matrix)
        self.assertEqual(24, final_score.calc_final_score())

    def test_basic_score_with_gaps_BLOSUM62(self):
        print("Ejecutando test3")
        matrix = Blosum62()
        seqs = [('ID1',
                 'FA'),
                ('ID2',
                 'A-')]
        final_score = SumOfPairs(seqs, matrix)
        self.assertEqual(-10, final_score.calc_final_score())

    def test_only_gaps_with_BLOSUM62(self):
        print("Ejecutando test4")
        matrix = Blosum62()
        seqs = [('ID1',
                 '---'),
                ('ID2',
                 '---')]
        final_score = SumOfPairs(seqs, matrix)
        self.assertEqual(3, final_score.calc_final_score())<commit_msg>Added main() in testcase<commit_after>import unittest
from pymsascoring.distancematrix.impl.pam250 import PAM250
from pymsascoring.distancematrix.impl.blosum62 import Blosum62
from pymsascoring.impl.sumofpairs import SumOfPairs

class SumOfPairsTestCase(unittest.TestCase):

    def setUp(self):
        print("setUp: INICIANDO TEST")

    def tearDown(self):
        print("tearDown: FINALIZANDO TEST")

    def test_basic_score_of_12_with_PAM250(self):
        print("Ejecutando test1")
        matrix = PAM250()
        seqs = [('ID1',
                 'AA'),
                ('ID2',
                 'AA'),
                ('ID3',
                 'AA')]
        final_score = SumOfPairs(seqs, matrix)
        self.assertEqual(12, final_score.calc_final_score())

    def test_basic_score_of_12_with_BLOSUM62(self):
        print("Ejecutando test2")
        matrix = Blosum62()
        seqs = [('ID1',
                 'AA'),
                ('ID2',
                 'AA'),
                ('ID3',
                 'AA')]
        final_score = SumOfPairs(seqs, matrix)
        self.assertEqual(24, final_score.calc_final_score())

    def test_basic_score_with_gaps_BLOSUM62(self):
        print("Ejecutando test3")
        matrix = Blosum62()
        seqs = [('ID1',
                 'FA'),
                ('ID2',
                 'A-')]
        final_score = SumOfPairs(seqs, matrix)
        self.assertEqual(-10, final_score.calc_final_score())

    def test_only_gaps_with_BLOSUM62(self):
        print("Ejecutando test4")
        matrix = Blosum62()
        seqs = [('ID1',
                 '---'),
                ('ID2',
                 '---')]
        final_score = SumOfPairs(seqs, matrix)
        self.assertEqual(3, final_score.calc_final_score())

if __name__ == "__main__":
    unittest.main()<|endoftext|>


<commit_before>import re
import string
import unicodedata

from .random_ import random

__all__ = [
    'random_string',
    'number_to_string', 'string_to_number', 'number_to_bytes', 'bytes_to_number',
    'dollars_to_cents',
    'to_str', 'to_unicode', 'to_int',
    'slugify',
]

def random_string(length=6, alphabet=string.letters+string.digits):
    """
    Return a random string of given length and alphabet.

    Default alphabet is url-friendly (base62).
    """
    return ''.join([random.choice(alphabet) for i in xrange(length)])

def number_to_string(n, alphabet):
    """
    Given an non-negative integer ``n``, convert it to a string composed of
    the given ``alphabet`` mapping, where the position of each element in
    ``alphabet`` is its radix value.

    Examples::

        >>> number_to_string(12345678, '01')
        '101111000110000101001110'

        >>> number_to_string(12345678, 'ab')
        'babbbbaaabbaaaababaabbba'

        >>> number_to_string(12345678, string.letters + string.digits)
        'ZXP0'

        >>> number_to_string(12345, ['zero ', 'one ', 'two ', 'three ', 'four ', 'five ', 'six ', 'seven ', 'eight ', 'nine '])
        'one two three four five '

    """
    result = ''
    base = len(alphabet)
    current = int(n)
    if current < 0:
        raise ValueError("invalid n (must be non-negative): %s", n)
    while current:
        result = alphabet[current % base] + result
        current = current // base

    return result

def string_to_number(s, alphabet):
    """
    Given a string ``s``, convert it to an integer composed of the given
    ``alphabet`` mapping, where the position of each element in ``alphabet`` is
    its radix value.

    Examples::

        >>> string_to_number('101111000110000101001110', '01')
        12345678

        >>> string_to_number('babbbbaaabbaaaababaabbba', 'ab')
        12345678

        >>> string_to_number('ZXP0', string.letters + string.digits)
        12345678

    """
    base = len(alphabet)
    inverse_alphabet = dict(zip(alphabet, xrange(0, base)))
    n = 0
    exp = 0
    for i in reversed(s):
        n += inverse_alphabet[i] * (base ** exp)
        exp += 1

    return n

def bytes_to_number(b, endian='big'):
    """
    Convert a string to an integer.

    :param b:
        String or bytearray to convert.

    :param endian:
        Byte order to convert into ('big' or 'little' endian-ness, default
        'big')

    Assumes bytes are 8 bits.

    This is a special-case version of string_to_number with a full base-256
    ASCII alphabet. It is the reverse of ``number_to_bytes(n)``.

    Examples::

        >>> bytes_to_number('*')
        42
        >>> bytes_to_number('\\xff')
        255
        >>> bytes_to_number('\\x01\\x00')
        256
        >>> bytes_to_number('\\x00\\x01', endian='little')
        256
    """
    if endian == 'big':
        b = reversed(b)

    n = 0
    for i, ch in enumerate(bytearray(b)):
        n ^= ch << i * 8

    return n

def number_to_bytes(n, endian='big'):
    """
    Convert an integer to a corresponding string of bytes..

    :param n:
        Integer to convert.

    :param endian:
        Byte order to convert into ('big' or 'little' endian-ness, default
        'big')

    Assumes bytes are 8 bits.

    This is a special-case version of number_to_string with a full base-256
    ASCII alphabet. It is the reverse of ``bytes_to_number(b)``.

    Examples::

        >>> number_to_bytes(42)
        '*'
        >>> number_to_bytes(255)
        '\\xff'
        >>> number_to_bytes(256)
        '\\x01\\x00'
        >>> number_to_bytes(256, endian='little')
        '\\x00\\x01'
    """
    b = ''
    while n:
        n, ch = divmod(n, 256)
        b += chr(ch)

    if endian == 'big':
        return b[::-1]

    return b

def to_str(obj, encoding='utf-8', **encode_args):
    r"""
    Returns a ``str`` of ``obj``, encoding using ``encoding`` if necessary. For
    example::

        >>> some_str = "\xff"
        >>> some_unicode = u"\u1234"
        >>> some_exception = Exception(u'Error: ' + some_unicode)
        >>> to_str(some_str)
        '\xff'
        >>> to_str(some_unicode)
        '\xe1\x88\xb4'
        >>> to_str(some_exception)
        'Error: \xe1\x88\xb4'
        >>> to_str([u'\u1234', 42])
        "[u'\\u1234', 42]"

    See source code for detailed semantics.
    """
    # We coerce to unicode if '__unicode__' is available because there is no
    # way to specify encoding when calling ``str(obj)``, so, eg,
    # ``str(Exception(u'\u1234'))`` will explode.
    if isinstance(obj, unicode) or hasattr(obj, "__unicode__"):
        # Note: unicode(u'foo') is O(1) (by experimentation)
        return unicode(obj).encode(encoding, **encode_args)

    # Note: it's just as fast to do `if isinstance(obj, str): return obj` as it
    # is to simply return `str(obj)`.
    return str(obj)

def to_unicode(obj, encoding='utf-8', fallback='latin1', **decode_args):
    r"""
    Returns a ``unicode`` of ``obj``, decoding using ``encoding`` if necessary.
    If decoding fails, the ``fallback`` encoding (default ``latin1``) is used.

    Examples::

        >>> to_unicode('\xe1\x88\xb4')
        u'\u1234'
        >>> to_unicode('\xff')
        u'\xff'
        >>> to_unicode(u'\u1234')
        u'\u1234'
        >>> to_unicode(Exception(u'\u1234'))
        u'\u1234'
        >>> to_unicode([u'\u1234', 42])
        u"[u'\\u1234', 42]"

    See source code for detailed semantics.
    """

    if isinstance(obj, unicode) or hasattr(obj, "__unicode__"):
        return unicode(obj)

    obj_str = str(obj)
    try:
        return unicode(obj_str, encoding, **decode_args)
    except UnicodeDecodeError:
        return unicode(obj_str, fallback, **decode_args)

def to_int(s, default=0):
    """
    Return input converted into an integer. If failed, then return ``default``.

    Examples::

        >>> to_int('1')
        1
        >>> to_int(1)
        1
        >>> to_int('')
        0
        >>> to_int(None)
        0
        >>> to_int(0, default='Empty')
        0
        >>> to_int(None, default='Empty')
        'Empty'
    """
    try:
        return int(s)
    except (TypeError, ValueError):
        return default

def dollars_to_cents(s, allow_negative=False):
    """
    Given a string or integer representing dollars, return an integer of
    equivalent cents, in an input-resilient way.

    Examples::

        >>> dollars_to_cents('$1')
        100
        >>> dollars_to_cents('1')
        100
        >>> dollars_to_cents(1)
        100
    """
    # TODO: Add support for "-$100"
    # TODO: Implement cents_to_dollars
    if not s:
        return

    if isinstance(s, basestring):
        s = s.lstrip('$')

    dollars = int(round(float(s) * 100))
    if not allow_negative and dollars < 0:
        raise ValueError('Negative values not permitted.')

    return dollars

RE_SLUG = re.compile(r'\W+')

def slugify(s, delimiter='-'):
    """
    Normalize `s` into ASCII and replace non-word characters with `delimiter`.
    """
    s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore').decode('ascii')
    return RE_SLUG.sub(delimiter, s).strip(delimiter).lower()

if __name__ == "__main__":
    import doctest
    doctest.testmod(optionflags=doctest.ELLIPSIS)
<commit_msg>Proposing a string_.format_int<commit_after>import re
import string
import unicodedata

from .random_ import random

__all__ = [
    'random_string',
    'number_to_string', 'string_to_number', 'number_to_bytes', 'bytes_to_number',
    'dollars_to_cents',
    'to_str', 'to_unicode', 'to_int',
    'format_int',
    'slugify',
]

_Default = object()

def random_string(length=6, alphabet=string.letters+string.digits):
    """
    Return a random string of given length and alphabet.

    Default alphabet is url-friendly (base62).
    """
    return ''.join([random.choice(alphabet) for i in xrange(length)])

def number_to_string(n, alphabet):
    """
    Given an non-negative integer ``n``, convert it to a string composed of
    the given ``alphabet`` mapping, where the position of each element in
    ``alphabet`` is its radix value.

    Examples::

        >>> number_to_string(12345678, '01')
        '101111000110000101001110'

        >>> number_to_string(12345678, 'ab')
        'babbbbaaabbaaaababaabbba'

        >>> number_to_string(12345678, string.letters + string.digits)
        'ZXP0'

        >>> number_to_string(12345, ['zero ', 'one ', 'two ', 'three ', 'four ', 'five ', 'six ', 'seven ', 'eight ', 'nine '])
        'one two three four five '

    """
    result = ''
    base = len(alphabet)
    current = int(n)
    if current < 0:
        raise ValueError("invalid n (must be non-negative): %s", n)
    while current:
        result = alphabet[current % base] + result
        current = current // base

    return result

def string_to_number(s, alphabet):
    """
    Given a string ``s``, convert it to an integer composed of the given
    ``alphabet`` mapping, where the position of each element in ``alphabet`` is
    its radix value.

    Examples::

        >>> string_to_number('101111000110000101001110', '01')
        12345678

        >>> string_to_number('babbbbaaabbaaaababaabbba', 'ab')
        12345678

        >>> string_to_number('ZXP0', string.letters + string.digits)
        12345678

    """
    base = len(alphabet)
    inverse_alphabet = dict(zip(alphabet, xrange(0, base)))
    n = 0
    exp = 0
    for i in reversed(s):
        n += inverse_alphabet[i] * (base ** exp)
        exp += 1

    return n

def bytes_to_number(b, endian='big'):
    """
    Convert a string to an integer.

    :param b:
        String or bytearray to convert.

    :param endian:
        Byte order to convert into ('big' or 'little' endian-ness, default
        'big')

    Assumes bytes are 8 bits.

    This is a special-case version of string_to_number with a full base-256
    ASCII alphabet. It is the reverse of ``number_to_bytes(n)``.

    Examples::

        >>> bytes_to_number('*')
        42
        >>> bytes_to_number('\\xff')
        255
        >>> bytes_to_number('\\x01\\x00')
        256
        >>> bytes_to_number('\\x00\\x01', endian='little')
        256
    """
    if endian == 'big':
        b = reversed(b)

    n = 0
    for i, ch in enumerate(bytearray(b)):
        n ^= ch << i * 8

    return n

def number_to_bytes(n, endian='big'):
    """
    Convert an integer to a corresponding string of bytes..

    :param n:
        Integer to convert.

    :param endian:
        Byte order to convert into ('big' or 'little' endian-ness, default
        'big')

    Assumes bytes are 8 bits.

    This is a special-case version of number_to_string with a full base-256
    ASCII alphabet. It is the reverse of ``bytes_to_number(b)``.

    Examples::

        >>> number_to_bytes(42)
        '*'
        >>> number_to_bytes(255)
        '\\xff'
        >>> number_to_bytes(256)
        '\\x01\\x00'
        >>> number_to_bytes(256, endian='little')
        '\\x00\\x01'
    """
    b = ''
    while n:
        n, ch = divmod(n, 256)
        b += chr(ch)

    if endian == 'big':
        return b[::-1]

    return b

def to_str(obj, encoding='utf-8', **encode_args):
    r"""
    Returns a ``str`` of ``obj``, encoding using ``encoding`` if necessary. For
    example::

        >>> some_str = "\xff"
        >>> some_unicode = u"\u1234"
        >>> some_exception = Exception(u'Error: ' + some_unicode)
        >>> to_str(some_str)
        '\xff'
        >>> to_str(some_unicode)
        '\xe1\x88\xb4'
        >>> to_str(some_exception)
        'Error: \xe1\x88\xb4'
        >>> to_str([u'\u1234', 42])
        "[u'\\u1234', 42]"

    See source code for detailed semantics.
    """
    # We coerce to unicode if '__unicode__' is available because there is no
    # way to specify encoding when calling ``str(obj)``, so, eg,
    # ``str(Exception(u'\u1234'))`` will explode.
    if isinstance(obj, unicode) or hasattr(obj, "__unicode__"):
        # Note: unicode(u'foo') is O(1) (by experimentation)
        return unicode(obj).encode(encoding, **encode_args)

    # Note: it's just as fast to do `if isinstance(obj, str): return obj` as it
    # is to simply return `str(obj)`.
    return str(obj)

def to_unicode(obj, encoding='utf-8', fallback='latin1', **decode_args):
    r"""
    Returns a ``unicode`` of ``obj``, decoding using ``encoding`` if necessary.
    If decoding fails, the ``fallback`` encoding (default ``latin1``) is used.

    Examples::

        >>> to_unicode('\xe1\x88\xb4')
        u'\u1234'
        >>> to_unicode('\xff')
        u'\xff'
        >>> to_unicode(u'\u1234')
        u'\u1234'
        >>> to_unicode(Exception(u'\u1234'))
        u'\u1234'
        >>> to_unicode([u'\u1234', 42])
        u"[u'\\u1234', 42]"

    See source code for detailed semantics.
    """

    if isinstance(obj, unicode) or hasattr(obj, "__unicode__"):
        return unicode(obj)

    obj_str = str(obj)
    try:
        return unicode(obj_str, encoding, **decode_args)
    except UnicodeDecodeError:
        return unicode(obj_str, fallback, **decode_args)

def to_int(s, default=0):
    """
    Return input converted into an integer. If failed, then return ``default``.

    Examples::

        >>> to_int('1')
        1
        >>> to_int(1)
        1
        >>> to_int('')
        0
        >>> to_int(None)
        0
        >>> to_int(0, default='Empty')
        0
        >>> to_int(None, default='Empty')
        'Empty'
    """
    try:
        return int(s)
    except (TypeError, ValueError):
        return default

def format_int(n, singular=_Default, plural=_Default):
    """
    Return `singular.format(n)` if n is 1, or `plural.format(n)` otherwise. If
    plural is not specified, then it is assumed to be same as singular but
    suffixed with an 's'.

    :param n:
        Integer which determines pluralness.

    :param singular:
        String with a format() placeholder for n. (Default: `u"{:,}"`)

    :param plural:
        String with a format() placeholder for n. (Default: If singular is not
        default, then it's `singular + u"s"`. Otherwise it's same as singular.)

    Example: ::

        >>> format_int(1000)
        u'1,000'
        >>> format_int(1, u"{} day")
        u'1 day'
        >>> format_int(2, u"{} day")
        u'2 days'
        >>> format_int(2, u"{} box", u"{} boxen")
        u'2 boxen'
        >>> format_int(20000, u"{:,} box", u"%d boxen")
        u'20,000 boxen'
    """
    n = int(n)

    if plural is _Default:
        plural = singular is not _Default and singular

    if singular in (None, _Default):
        singular = '{:,}'

    if n == 1 or not plural:
        return singular.format(n)

    return plural.format(n)

def dollars_to_cents(s, allow_negative=False):
    """
    Given a string or integer representing dollars, return an integer of
    equivalent cents, in an input-resilient way.

    Examples::

        >>> dollars_to_cents('$1')
        100
        >>> dollars_to_cents('1')
        100
        >>> dollars_to_cents(1)
        100
    """
    # TODO: Add support for "-$100"
    # TODO: Implement cents_to_dollars
    if not s:
        return

    if isinstance(s, basestring):
        s = s.lstrip('$')

    dollars = int(round(float(s) * 100))
    if not allow_negative and dollars < 0:
        raise ValueError('Negative values not permitted.')

    return dollars

RE_SLUG = re.compile(r'\W+')

def slugify(s, delimiter='-'):
    """
    Normalize `s` into ASCII and replace non-word characters with `delimiter`.
    """
    s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore').decode('ascii')
    return RE_SLUG.sub(delimiter, s).strip(delimiter).lower()

if __name__ == "__main__":
    import doctest
    doctest.testmod(optionflags=doctest.ELLIPSIS)
<|endoftext|>