<commit_before># Copyright 2011, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""WebSocket opening handshake processor. This class try to apply available
opening handshake processors for each protocol version until a connection is
successfully established.
"""

import logging

from mod_pywebsocket import common
from mod_pywebsocket.handshake import draft75
from mod_pywebsocket.handshake import hybi00
from mod_pywebsocket.handshake import hybi
# Export AbortedByUserException, HandshakeException, and VersionException
# symbol from this module.
from mod_pywebsocket.handshake._base import AbortedByUserException
from mod_pywebsocket.handshake._base import HandshakeException
from mod_pywebsocket.handshake._base import VersionException

_LOGGER = logging.getLogger(__name__)

def do_handshake(request, dispatcher, allowDraft75=False, strict=False):
    """Performs WebSocket handshake.

    Args:
        request: mod_python request.
        dispatcher: Dispatcher (dispatch.Dispatcher).
        allowDraft75: allow draft 75 handshake protocol.
        strict: Strictly check handshake request in draft 75.
            Default: False. If True, request.connection must provide
            get_memorized_lines method.

    Handshaker will add attributes such as ws_resource in performing
    handshake.
    """

    _LOGGER.debug('Client\'s opening handshake resource: %r', request.uri)
    # To print mimetools.Message as escaped one-line string, we converts
    # headers_in to dict object. Without conversion, if we use %r, it just
    # prints the type and address, and if we use %s, it prints the original
    # header string as multiple lines.
    #
    # Both mimetools.Message and MpTable_Type of mod_python can be
    # converted to dict.
    #
    # mimetools.Message.__str__ returns the original header string.
    # dict(mimetools.Message object) returns the map from header names to
    # header values. While MpTable_Type doesn't have such __str__ but just
    # __repr__ which formats itself as well as dictionary object.
    _LOGGER.debug(
        'Client\'s opening handshake headers: %r', dict(request.headers_in))

    handshakers = []
    handshakers.append(
        ('IETF HyBi latest', hybi.Handshaker(request, dispatcher)))
    handshakers.append(
        ('IETF HyBi 00', hybi00.Handshaker(request, dispatcher)))
    if allowDraft75:
        handshakers.append(
            ('IETF Hixie 75', draft75.Handshaker(request, dispatcher, strict)))

    for name, handshaker in handshakers:
        _LOGGER.info('Trying %s protocol', name)
        try:
            handshaker.do_handshake()
            return
        except HandshakeException, e:
            _LOGGER.info(
                'Failed to complete opening handshake as %s protocol: %r',
                name, e)
            if e.status:
                raise e
        except AbortedByUserException, e:
            raise
        except VersionException, e:
            raise

    # TODO(toyoshim): Add a test to cover the case all handshakers fail.
    raise HandshakeException(
        'Failed to complete opening handshake for all available protocols',
        status=common.HTTP_STATUS_BAD_REQUEST)

# vi:sts=4 sw=4 et
<commit_msg>Use log debug level for protocol trials. Log info when established. Review URL: http://codereview.appspot.com/5316057<commit_after># Copyright 2011, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""WebSocket opening handshake processor. This class try to apply available
opening handshake processors for each protocol version until a connection is
successfully established.
"""

import logging

from mod_pywebsocket import common
from mod_pywebsocket.handshake import draft75
from mod_pywebsocket.handshake import hybi00
from mod_pywebsocket.handshake import hybi
# Export AbortedByUserException, HandshakeException, and VersionException
# symbol from this module.
from mod_pywebsocket.handshake._base import AbortedByUserException
from mod_pywebsocket.handshake._base import HandshakeException
from mod_pywebsocket.handshake._base import VersionException

_LOGGER = logging.getLogger(__name__)

def do_handshake(request, dispatcher, allowDraft75=False, strict=False):
    """Performs WebSocket handshake.

    Args:
        request: mod_python request.
        dispatcher: Dispatcher (dispatch.Dispatcher).
        allowDraft75: allow draft 75 handshake protocol.
        strict: Strictly check handshake request in draft 75.
            Default: False. If True, request.connection must provide
            get_memorized_lines method.

    Handshaker will add attributes such as ws_resource in performing
    handshake.
    """

    _LOGGER.debug('Client\'s opening handshake resource: %r', request.uri)
    # To print mimetools.Message as escaped one-line string, we converts
    # headers_in to dict object. Without conversion, if we use %r, it just
    # prints the type and address, and if we use %s, it prints the original
    # header string as multiple lines.
    #
    # Both mimetools.Message and MpTable_Type of mod_python can be
    # converted to dict.
    #
    # mimetools.Message.__str__ returns the original header string.
    # dict(mimetools.Message object) returns the map from header names to
    # header values. While MpTable_Type doesn't have such __str__ but just
    # __repr__ which formats itself as well as dictionary object.
    _LOGGER.debug(
        'Client\'s opening handshake headers: %r', dict(request.headers_in))

    handshakers = []
    handshakers.append(
        ('IETF HyBi latest', hybi.Handshaker(request, dispatcher)))
    handshakers.append(
        ('IETF HyBi 00', hybi00.Handshaker(request, dispatcher)))
    if allowDraft75:
        handshakers.append(
            ('IETF Hixie 75', draft75.Handshaker(request, dispatcher, strict)))

    for name, handshaker in handshakers:
        _LOGGER.debug('Trying %s protocol', name)
        try:
            handshaker.do_handshake()
            _LOGGER.info('Established (%s protocol)', name)
            return
        except HandshakeException, e:
            _LOGGER.debug(
                'Failed to complete opening handshake as %s protocol: %r',
                name, e)
            if e.status:
                raise e
        except AbortedByUserException, e:
            raise
        except VersionException, e:
            raise

    # TODO(toyoshim): Add a test to cover the case all handshakers fail.
    raise HandshakeException(
        'Failed to complete opening handshake for all available protocols',
        status=common.HTTP_STATUS_BAD_REQUEST)

# vi:sts=4 sw=4 et
<|endoftext|>


<commit_before>#[PROTEXCAT]
#\License: ALL RIGHTS RESERVED
#\Author: Wang, Jing <jing.j.wang@intel.com>

from oeqa.oetest import oeRuntimeTest
from oeqa.utils.decorators import tag
import re

@tag(TestType="FVT", FeatureID="IOTOS-638")
class BaseOsTest(oeRuntimeTest):
    '''Base os health check
    @class BaseOsTest
    '''
    def test_baseos_dmesg(self):
        '''check dmesg command
        @fn test_baseos_dmesg
        @param self
        @return
        '''
        (status, output) = self.target.run('dmesg')
        ##
        # TESTPOINT: #1, test_baseos_dmesg
        #
        self.assertEqual(status, 0, msg="Error messages: %s" % output)

    def test_baseos_lsmod(self):
        '''check lsmod command
        @fn test_baseos_lsmod
        @param self
        @return
        '''
        (status, output) = self.target.run('lsmod')
        ##
        # TESTPOINT: #1, test_baseos_lsmod
        #
        # lsmod should show at least 1 module
        if output.count('\n') > 0:
            pass
        else:
            status=1
        self.assertEqual(status, 0, msg="Error messages: %s" % output)

    def test_baseos_ps(self):
        '''check ps command
        @fn test_baseos_ps
        @param self
        @return
        '''
        (status, output) = self.target.run('ps')
        ##
        # TESTPOINT: #1, test_baseos_ps
        #
        # ps should show at least 1 process
        if output.count('\n') > 0:
            pass
        else:
            status=1
        self.assertEqual(status, 0, msg="Error messages: %s" % output)

    def test_baseos_df(self):
        '''check df command
        @fn test_baseos_df
        @param self
        @return
        '''
        (status, output) = self.target.run('df')
        ##
        # TESTPOINT: #1, test_baseos_df
        #
        # df should show at least 1 mounting point
        if output.count('\n') > 0:
            pass
        else:
            status=1
        self.assertEqual(status, 0, msg="Error messages: %s" % output)

    def test_baseos_systemd_process(self):
        '''check systemd process
        @fn test_baseos_systemd_process
        @param self
        @return
        '''
        (status, output) = self.target.run("ls -l /proc/1/exe")
        ##
        # TESTPOINT: #1, test_baseos_systemd_process
        #
        if output.endswith("systemd"):
            pass
        else:
            status=1
        self.assertEqual(status, 0, msg="Error messages: %s" % output)

    def test_baseos_systemd_boot_error(self):
        ''' check systemd boot journal error'''
        known_issues_list = [
            "GPT: Use GNU Parted to correct GPT errors",
            # IOTOS-1575 [Edison] wpa_supplicant error during system booting
            "Failed to open config file '/etc/wpa_supplicant/wpa_supplicant-wlan0.conf'",
            # ignore kernel errors
            "^\w{3,} \d{,2} \d{2}:\d{2}:\d{2} \S+ kernel:",
            # IOTOS-1691 Ethernet relevant errors at system startup
            "Error changing net interface name \S+ to \S+: Device or resource busy",
            "open error Permission denied for /var/lib/connman/ethernet_\S+_cable/data",
            # Error from Beaglebone Black USB keyboard or ethernet emulation
            "0003:8086:BEEF.0001",
            # Harmless error
            "[drm] parse error at position 4 in video mode 'efifb'",
            # Errors from bad Intel(r) 500 series support
            "ACPI Error: Could not enable PowerButton event",
            "button: probe of LNXPWRBN:00 failed with error -22",
            "probe of LNXPWRBN:00 failed",
            "Direct firmware load for i915/bxt_dmc_ver1_07.bin failed",
            "Direct firmware load for iwlwifi-8000C",
            "ACPI Error: Could not enable RealTimeClock event",
            "hci_intel: probe of INT33E1:00 failed with error -2"
            ]
        self.longMessage = True
        cmd = "journalctl -ab"
        (status, output) = self.target.run("journalctl -ab")
        self.assertEqual(status, 0, "Fail to run %s,status is %s, output is: %s\n"
                                              % (cmd,status,output))

        self.assertTrue(output.strip(), "No systemd journal log")
        journal = output
        errors = []
        for line in output.split('\n'):
            if 'error' in line.lower():
                flag = 0
                for issue in known_issues_list:
                    if issue in line:
                        flag = 1
                        break
                if flag == 0 :
                    errors.append(line)

        self.assertEqual(len(errors), 0, msg="\nErrors in boot log:\n" +
                                             "\n".join(errors))
<commit_msg>meta-iotqa: Ignore net interface name changing error<commit_after>#[PROTEXCAT]
#\License: ALL RIGHTS RESERVED
#\Author: Wang, Jing <jing.j.wang@intel.com>

from oeqa.oetest import oeRuntimeTest
from oeqa.utils.decorators import tag
import re

@tag(TestType="FVT", FeatureID="IOTOS-638")
class BaseOsTest(oeRuntimeTest):
    '''Base os health check
    @class BaseOsTest
    '''
    def test_baseos_dmesg(self):
        '''check dmesg command
        @fn test_baseos_dmesg
        @param self
        @return
        '''
        (status, output) = self.target.run('dmesg')
        ##
        # TESTPOINT: #1, test_baseos_dmesg
        #
        self.assertEqual(status, 0, msg="Error messages: %s" % output)

    def test_baseos_lsmod(self):
        '''check lsmod command
        @fn test_baseos_lsmod
        @param self
        @return
        '''
        (status, output) = self.target.run('lsmod')
        ##
        # TESTPOINT: #1, test_baseos_lsmod
        #
        # lsmod should show at least 1 module
        if output.count('\n') > 0:
            pass
        else:
            status=1
        self.assertEqual(status, 0, msg="Error messages: %s" % output)

    def test_baseos_ps(self):
        '''check ps command
        @fn test_baseos_ps
        @param self
        @return
        '''
        (status, output) = self.target.run('ps')
        ##
        # TESTPOINT: #1, test_baseos_ps
        #
        # ps should show at least 1 process
        if output.count('\n') > 0:
            pass
        else:
            status=1
        self.assertEqual(status, 0, msg="Error messages: %s" % output)

    def test_baseos_df(self):
        '''check df command
        @fn test_baseos_df
        @param self
        @return
        '''
        (status, output) = self.target.run('df')
        ##
        # TESTPOINT: #1, test_baseos_df
        #
        # df should show at least 1 mounting point
        if output.count('\n') > 0:
            pass
        else:
            status=1
        self.assertEqual(status, 0, msg="Error messages: %s" % output)

    def test_baseos_systemd_process(self):
        '''check systemd process
        @fn test_baseos_systemd_process
        @param self
        @return
        '''
        (status, output) = self.target.run("ls -l /proc/1/exe")
        ##
        # TESTPOINT: #1, test_baseos_systemd_process
        #
        if output.endswith("systemd"):
            pass
        else:
            status=1
        self.assertEqual(status, 0, msg="Error messages: %s" % output)

    def test_baseos_systemd_boot_error(self):
        ''' check systemd boot journal error'''
        known_issues_list = [
            "GPT: Use GNU Parted to correct GPT errors",
            # IOTOS-1575 [Edison] wpa_supplicant error during system booting
            "Failed to open config file '/etc/wpa_supplicant/wpa_supplicant-wlan0.conf'",
            # ignore kernel errors
            "^\w{3,} \d{,2} \d{2}:\d{2}:\d{2} \S+ kernel:",
            # IOTOS-1691 Ethernet relevant errors at system startup
            "Error changing net interface name \S+ to \S+: Device or resource busy",
            "open error Permission denied for /var/lib/connman/ethernet_\S+_cable/data",
            # Error from Beaglebone Black USB keyboard or ethernet emulation
            "0003:8086:BEEF.0001",
            # Harmless error
            "[drm] parse error at position 4 in video mode 'efifb'",
            # Errors from bad Intel(r) 500 series support
            "ACPI Error: Could not enable PowerButton event",
            "button: probe of LNXPWRBN:00 failed with error -22",
            "probe of LNXPWRBN:00 failed",
            "Direct firmware load for i915/bxt_dmc_ver1_07.bin failed",
            "Direct firmware load for iwlwifi-8000C",
            "ACPI Error: Could not enable RealTimeClock event",
            "hci_intel: probe of INT33E1:00 failed with error -2",
            "Error changing net interface name 'usb0' to"
            ]
        self.longMessage = True
        cmd = "journalctl -ab"
        (status, output) = self.target.run("journalctl -ab")
        self.assertEqual(status, 0, "Fail to run %s,status is %s, output is: %s\n"
                                              % (cmd,status,output))

        self.assertTrue(output.strip(), "No systemd journal log")
        journal = output
        errors = []
        for line in output.split('\n'):
            if 'error' in line.lower():
                flag = 0
                for issue in known_issues_list:
                    if issue in line:
                        flag = 1
                        break
                if flag == 0 :
                    errors.append(line)

        self.assertEqual(len(errors), 0, msg="\nErrors in boot log:\n" +
                                             "\n".join(errors))
<|endoftext|>


<commit_before><commit_msg>Fix sporadically failing test test_with_items_action_context<commit_after><|endoftext|>


<commit_before>from __future__ import print_function, absolute_import, division
import numpy as np

def get_dist_speed(movement, start, end):
    r"""
    Return a tuple containing distance covered in a path and
    average speed within a path.

    Parameters
    ----------
    movement : pandas.DataFrame
        CT, CX, CY coordinates and homebase status for the unique
        combination of strain, mouse, and day

    start : int
        positive integer indicating the starting index of a path

    end : int
        positive integer indicating the end index of a path

    Returns
    -------
    dist : distance travelled along a path

    speed : average speed along a path

    Examples
    --------
    >>> movement = data.load_movement(1, 2, 1)
    >>> dist, speed = get_dist_speed(movement, 0, 500)
    (554.5573324808769, 0.080910098911799441)
    """

    # Check whether inputs are valid.
    if (start < 0) or (end < 0):
        raise ValueError("Start and end indices must be positive")
    if (type(start) != int) or (type(end) != int):
        raise TypeError("Start and end indices must be integers")
    if start > end:
        raise ValueError("Start index must be smaller than end index")
    if end - start > len(movement) - 1:
        raise ValueError("Number of observations must be less than \
        or equal to total observations")

    if start == end:
        return(0, 0)

    x = movement['x'][start:(end+1)].ravel()
    y = movement['y'][start:(end+1)].ravel()
    t = movement['t']
    time = t[end] - t[start]

    dist = sum(np.sqrt((x[1:] - x[:-1])**2 + (y[1:] - y[:-1])**2))
    speed = dist / time

    return(dist, speed)
<commit_msg>STY: fix spacing issue<commit_after>from __future__ import print_function, absolute_import, division

import numpy as np

def get_dist_speed(movement, start, end):
    r"""
    Return a tuple containing distance covered in a path and
    average speed within a path.

    Parameters
    ----------
    movement : pandas.DataFrame
        CT, CX, CY coordinates and homebase status for the unique
        combination of strain, mouse, and day

    start : int
        positive integer indicating the starting index of a path

    end : int
        positive integer indicating the end index of a path

    Returns
    -------
    dist : distance travelled along a path

    speed : average speed along a path

    Examples
    --------
    >>> movement = data.load_movement(1, 2, 1)
    >>> dist, speed = get_dist_speed(movement, 0, 500)
    (554.5573324808769, 0.080910098911799441)
    """

    # Check whether inputs are valid.
    if (start < 0) or (end < 0):
        raise ValueError("Start and end indices must be positive")
    if (type(start) != int) or (type(end) != int):
        raise TypeError("Start and end indices must be integers")
    if start > end:
        raise ValueError("Start index must be smaller than end index")
    if end - start > len(movement) - 1:
        raise ValueError("Number of observations must be less than \
        or equal to total observations")

    if start == end:
        return(0, 0)

    x = movement['x'][start:(end+1)].ravel()
    y = movement['y'][start:(end+1)].ravel()
    t = movement['t']
    time = t[end] - t[start]

    dist = sum(np.sqrt((x[1:] - x[:-1])**2 + (y[1:] - y[:-1])**2))
    speed = dist / time
    return(dist, speed)
<|endoftext|>


<commit_before>import numpy as np
import scipy.stats as spst
import scipy.sparse as sprs
from scipy.sparse import csgraph
from collections import namedtuple

__all__ = [
    'trim_disconnected_clusters',
    'site_percolation_orig',
    'bond_percolation_orig',
    'ispercolating',
    'remove_isolated_clusters',
]

def trim_disconnected_clusters(b_labels, s_labels, inlets):
    r"""
    Computes actual node and edge occupancy based on connectivity to the given
    inlets

    Parameters
    ----------
    b_labels : ndarray
        An array of cluster labels assigned to each bond.  -1 indicates
        unoccupied
    s_labels : ndarray
        An array of cluster labels assigned to each site. -1 indicates
        unoccupied. Site cluster numbers must correspond to the bond
        clusters, such that if bond j has a cluster number N, then both
        sites on each end of j are also labeled N.
    inlets : ndarray
        An array containing node indices that are to be treated as inlets.
        Any clusters labels not found in these nodes will be considered
        disconnected and set to -1.

    Returns
    -------
    occupancy : tuple of ndarrays
        The returned tuple contains boolean arrays of ``occupied_sites``
        and ``occupied_bonds``, after accounting for connection to the
        ``inlets``.

    Notes
    -----
    The ``b_labels`` and ``s_labels`` arrays are returned from the
    ``bond_percolation`` function.

    """
    hits = np.unique(s_labels[inlets])
    hits = hits[hits >= 0]
    occupied_bonds = np.isin(b_labels, hits)
    occupied_sites = np.isin(s_labels, hits)
    return occupied_sites, occupied_bonds

def remove_isolated_clusters(labels, inlets):
    r"""
    Finds cluster labels not attached to the inlets, and sets them to
    unoccupied (-1)

    Parameters
    ----------
    labels : tuple of node and edge labels
        This information is provided by the ``site_percolation`` or
        ``bond_percolation`` functions

    inlets : array_like
        A list of which nodes are inlets.  Can be a boolean mask or an
        array of indices.

    Returns
    -------
    A tuple containing a list of node and edge labels, with all clusters
    not connected to the inlets set to not occupied (-1).

    """
    # Identify clusters of invasion sites
    inv_clusters = np.unique(labels.sites[inlets])
    # Remove cluster numbers == -1, if any
    inv_clusters = inv_clusters[inv_clusters >= 0]
    # Find all pores in invading clusters
    p_invading = np.in1d(labels.sites, inv_clusters)
    labels.sites[~p_invading] = -1
    t_invading = np.in1d(labels.bonds, inv_clusters)
    labels.bonds[~t_invading] = -1
    return labels

def ispercolating(am, inlets, outlets, mode='site'):
    r"""
    Determines if a percolating clusters exists in the network spanning
    the given inlet and outlet nodes

    Parameters
    ----------
    am : adjacency_matrix
        The adjacency matrix with the ``data`` attribute indicating
        if an egde is occupied or not.

    inlets : array_like
        An array of indices indicating which nodes are part of the inlets

    outlets : array_like
        An array of indices indicating which nodes are part of the outlets

    mode : str
        Indicates which type of percolation to apply,. Options are:

        ===========  =====================================================
        mode         meaning
        ===========  =====================================================
        'site'       Applies site percolation
        'bond'       Applies bond percolation
        ===========  =====================================================

    """
    if am.format != 'coo':
        am = am.to_coo()
    ij = np.vstack((am.col, am.row)).T
    if mode.startswith('site'):
        occupied_sites = np.zeros(shape=am.shape[0], dtype=bool)
        occupied_sites[ij[am.data].flatten()] = True
        clusters = site_percolation_orig(ij, occupied_sites)
    elif mode.startswith('bond'):
        occupied_bonds = am.data
        clusters = bond_percolation_orig(ij, occupied_bonds)
    ins = np.unique(clusters.sites[inlets])
    if ins[0] == -1:
        ins = ins[1:]
    outs = np.unique(clusters.sites[outlets])
    if outs[0] == -1:
        outs = outs[1:]
    hits = np.in1d(ins, outs)
    return np.any(hits)

def bond_percolation_orig(conns, inds):
    r"""
    Calculates the site and bond occupancy status for a bond percolation
    process given a list of occupied bonds.

    Parameters
    ----------
    conns : array_like
        An N x 2 array of connections whether occupied or not.
        If the adjacency matrix is in the COO sparse format then this
        can be obtained using ``np.vstack(am.row, am.col).T``.  The
        matrix can be symmetric but only the upper triangular portion
        is kept.

    inds : ndarray, boolean
        A list of boolean values indicating whether a bond is occupied
        (``True``) or not (``False``). This must be the same length as
        ``conns``, but the lower triangular protion is ignored and the
        process is treated as undirected.

    Returns
    -------
    A tuple containing a list of site and bond labels, indicating which
    cluster each belongs to.  A value of -1 indicates uninvaded.

    Notes
    -----
    The ``connected_components`` function of ``scipy.sparse.csgraph`` will give
     a cluster number to ALL sites whether they are occupied or not, so this
    function essentially adjusts the cluster numbers to represent a
    percolation process.

    """
    edge_mask = np.array(inds)
    Np = np.amax(conns) + 1
    # Find occupied sites based on occupied bonds
    # (the following 2 lines are not needed but worth keeping for future ref)
    # occupied_sites = np.zeros([Np, ], dtype=bool)
    # np.add.at(occupied_sites, ij[occupied_bonds].flatten(), True)
    adj_mat = sprs.csr_matrix((edge_mask, (conns[:, 0], conns[:, 1])),
                              shape=(Np, Np))
    adj_mat.eliminate_zeros()
    clusters = csgraph.connected_components(csgraph=adj_mat, directed=False)[1]
    # Clusters of size 1 only occur if all a site's bonds are uninvaded
    valid_clusters = np.bincount(clusters) > 1
    mapping = -np.ones(shape=(clusters.max()+1, ), dtype=int)
    mapping[valid_clusters] = np.arange(0, valid_clusters.sum())
    s_labels = mapping[clusters]
    # Bond inherit the cluster number of its connected sites
    b_labels = np.amin(s_labels[conns], axis=1)
    # Set bond cluster to -1 if not actually occupied
    b_labels[~edge_mask] = -1
    tup = namedtuple('cluster_labels', ('sites', 'bonds'))
    return tup(s_labels, b_labels)

def site_percolation_orig(conns, inds):
    r"""
    Calculates the node and edge occupancy status for a site percolation
    process given a list of potentially occupied nodes.

    Parameters
    ----------
    conns : array_like
        An N x 2 array of [site_A, site_B] connections.  If two connected
        nodes are both occupied then they are part of the same cluster, as is
        the edge connecting them.
    inds : ndarray
        A boolean array with ``True`` values indicating occupied nodes.

    Returns
    -------
    A tuple containing a list of node and edge labels indicating which
    cluster each belongs to. A value of -1 indicates unoccupied.

    Notes
    -----
    The ``connected_components`` function of ``scipy.sparse.csgraph`` gives
    ALL nodes a cluster number whether they are occupied or not, so this
    function essentially adjusts the cluster numbers to represent a
    percolation process.

    """
    node_mask = np.array(inds)
    Np = np.size(inds)
    occupied_bonds = np.all(node_mask[conns], axis=1)
    adj_mat = sprs.csr_matrix((occupied_bonds, (conns[:, 0], conns[:, 1])),
                              shape=(Np, Np))
    adj_mat.eliminate_zeros()
    clusters = csgraph.connected_components(csgraph=adj_mat, directed=False)[1]
    clusters[~node_mask] = -1
    s_labels = spst.rankdata(clusters + 1, method="dense") - 1
    if np.any(~node_mask):
        s_labels -= 1
    b_labels = np.amin(s_labels[conns], axis=1)
    tup = namedtuple('cluster_labels', ('sites', 'bonds'))
    return tup(s_labels, b_labels)
<commit_msg>removing whitespace<commit_after>import numpy as np
import scipy.stats as spst
import scipy.sparse as sprs
from scipy.sparse import csgraph
from collections import namedtuple

__all__ = [
    'trim_disconnected_clusters',
    'site_percolation_orig',
    'bond_percolation_orig',
    'ispercolating',
    'remove_isolated_clusters',
]

def trim_disconnected_clusters(b_labels, s_labels, inlets):
    r"""
    Computes actual node and edge occupancy based on connectivity to the given
    inlets

    Parameters
    ----------
    b_labels : ndarray
        An array of cluster labels assigned to each bond.  -1 indicates
        unoccupied
    s_labels : ndarray
        An array of cluster labels assigned to each site. -1 indicates
        unoccupied. Site cluster numbers must correspond to the bond
        clusters, such that if bond j has a cluster number N, then both
        sites on each end of j are also labeled N.
    inlets : ndarray
        An array containing node indices that are to be treated as inlets.
        Any clusters labels not found in these nodes will be considered
        disconnected and set to -1.

    Returns
    -------
    occupancy : tuple of ndarrays
        The returned tuple contains boolean arrays of ``occupied_sites``
        and ``occupied_bonds``, after accounting for connection to the
        ``inlets``.

    Notes
    -----
    The ``b_labels`` and ``s_labels`` arrays are returned from the
    ``bond_percolation`` function.

    """
    hits = np.unique(s_labels[inlets])
    hits = hits[hits >= 0]
    occupied_bonds = np.isin(b_labels, hits)
    occupied_sites = np.isin(s_labels, hits)
    return occupied_sites, occupied_bonds

def remove_isolated_clusters(labels, inlets):
    r"""
    Finds cluster labels not attached to the inlets, and sets them to
    unoccupied (-1)

    Parameters
    ----------
    labels : tuple of node and edge labels
        This information is provided by the ``site_percolation`` or
        ``bond_percolation`` functions

    inlets : array_like
        A list of which nodes are inlets.  Can be a boolean mask or an
        array of indices.

    Returns
    -------
    A tuple containing a list of node and edge labels, with all clusters
    not connected to the inlets set to not occupied (-1).

    """
    # Identify clusters of invasion sites
    inv_clusters = np.unique(labels.sites[inlets])
    # Remove cluster numbers == -1, if any
    inv_clusters = inv_clusters[inv_clusters >= 0]
    # Find all pores in invading clusters
    p_invading = np.in1d(labels.sites, inv_clusters)
    labels.sites[~p_invading] = -1
    t_invading = np.in1d(labels.bonds, inv_clusters)
    labels.bonds[~t_invading] = -1
    return labels

def ispercolating(am, inlets, outlets, mode='site'):
    r"""
    Determines if a percolating clusters exists in the network spanning
    the given inlet and outlet nodes

    Parameters
    ----------
    am : adjacency_matrix
        The adjacency matrix with the ``data`` attribute indicating
        if an egde is occupied or not.
    inlets : array_like
        An array of indices indicating which nodes are part of the inlets
    outlets : array_like
        An array of indices indicating which nodes are part of the outlets
    mode : str
        Indicates which type of percolation to apply,. Options are:

        ===========  =====================================================
        mode         meaning
        ===========  =====================================================
        'site'       Applies site percolation
        'bond'       Applies bond percolation
        ===========  =====================================================

    """
    if am.format != 'coo':
        am = am.to_coo()
    ij = np.vstack((am.col, am.row)).T
    if mode.startswith('site'):
        occupied_sites = np.zeros(shape=am.shape[0], dtype=bool)
        occupied_sites[ij[am.data].flatten()] = True
        clusters = site_percolation_orig(ij, occupied_sites)
    elif mode.startswith('bond'):
        occupied_bonds = am.data
        clusters = bond_percolation_orig(ij, occupied_bonds)
    ins = np.unique(clusters.sites[inlets])
    if ins[0] == -1:
        ins = ins[1:]
    outs = np.unique(clusters.sites[outlets])
    if outs[0] == -1:
        outs = outs[1:]
    hits = np.in1d(ins, outs)
    return np.any(hits)

def bond_percolation_orig(conns, inds):
    r"""
    Calculates the site and bond occupancy status for a bond percolation
    process given a list of occupied bonds.

    Parameters
    ----------
    conns : array_like
        An N x 2 array of connections whether occupied or not.
        If the adjacency matrix is in the COO sparse format then this
        can be obtained using ``np.vstack(am.row, am.col).T``.  The
        matrix can be symmetric but only the upper triangular portion
        is kept.
    inds : ndarray, boolean
        A list of boolean values indicating whether a bond is occupied
        (``True``) or not (``False``). This must be the same length as
        ``conns``, but the lower triangular protion is ignored and the
        process is treated as undirected.

    Returns
    -------
    A tuple containing a list of site and bond labels, indicating which
    cluster each belongs to.  A value of -1 indicates uninvaded.

    Notes
    -----
    The ``connected_components`` function of ``scipy.sparse.csgraph`` will give
     a cluster number to ALL sites whether they are occupied or not, so this
    function essentially adjusts the cluster numbers to represent a
    percolation process.

    """
    edge_mask = np.array(inds)
    Np = np.amax(conns) + 1
    # Find occupied sites based on occupied bonds
    # (the following 2 lines are not needed but worth keeping for future ref)
    # occupied_sites = np.zeros([Np, ], dtype=bool)
    # np.add.at(occupied_sites, ij[occupied_bonds].flatten(), True)
    adj_mat = sprs.csr_matrix((edge_mask, (conns[:, 0], conns[:, 1])),
                              shape=(Np, Np))
    adj_mat.eliminate_zeros()
    clusters = csgraph.connected_components(csgraph=adj_mat, directed=False)[1]
    # Clusters of size 1 only occur if all a site's bonds are uninvaded
    valid_clusters = np.bincount(clusters) > 1
    mapping = -np.ones(shape=(clusters.max()+1, ), dtype=int)
    mapping[valid_clusters] = np.arange(0, valid_clusters.sum())
    s_labels = mapping[clusters]
    # Bond inherit the cluster number of its connected sites
    b_labels = np.amin(s_labels[conns], axis=1)
    # Set bond cluster to -1 if not actually occupied
    b_labels[~edge_mask] = -1
    tup = namedtuple('cluster_labels', ('sites', 'bonds'))
    return tup(s_labels, b_labels)

def site_percolation_orig(conns, inds):
    r"""
    Calculates the node and edge occupancy status for a site percolation
    process given a list of potentially occupied nodes.

    Parameters
    ----------
    conns : array_like
        An N x 2 array of [site_A, site_B] connections.  If two connected
        nodes are both occupied then they are part of the same cluster, as is
        the edge connecting them.
    inds : ndarray
        A boolean array with ``True`` values indicating occupied nodes.

    Returns
    -------
    A tuple containing a list of node and edge labels indicating which
    cluster each belongs to. A value of -1 indicates unoccupied.

    Notes
    -----
    The ``connected_components`` function of ``scipy.sparse.csgraph`` gives
    ALL nodes a cluster number whether they are occupied or not, so this
    function essentially adjusts the cluster numbers to represent a
    percolation process.

    """
    node_mask = np.array(inds)
    Np = np.size(inds)
    occupied_bonds = np.all(node_mask[conns], axis=1)
    adj_mat = sprs.csr_matrix((occupied_bonds, (conns[:, 0], conns[:, 1])),
                              shape=(Np, Np))
    adj_mat.eliminate_zeros()
    clusters = csgraph.connected_components(csgraph=adj_mat, directed=False)[1]
    clusters[~node_mask] = -1
    s_labels = spst.rankdata(clusters + 1, method="dense") - 1
    if np.any(~node_mask):
        s_labels -= 1
    b_labels = np.amin(s_labels[conns], axis=1)
    tup = namedtuple('cluster_labels', ('sites', 'bonds'))
    return tup(s_labels, b_labels)
<|endoftext|>


<commit_before>from nlpia.loaders import get_data
from tqdm import tqdm

df = get_data('deu')
input_texts, target_texts = [], []  # <1>
input_vocabulary = set()  # <3>
output_vocabulary = set()
start_token, stop_token = '\t\n'  # <2>
n = len(df)

for input_text, target_text in tqdm(zip(df.eng, df.deu), total=n):
    target_text = start_token + target_text \
        + stop_token  # <7>
    input_texts.append(input_text)
    target_texts.append(target_text)
    for char in input_text:  # <8>
        if char not in input_vocabulary:
            input_vocabulary.add(char)
    for char in target_text:
        if char not in output_vocabulary:
            output_vocabulary.add(char)

input_vocabulary = sorted(input_vocabulary)  # <1>
output_vocabulary = sorted(output_vocabulary)

input_vocab_size = len(input_vocabulary)  # <2>
output_vocab_size = len(output_vocabulary)
max_encoder_seq_length = max(
    [len(txt) for txt in input_texts])  # <3>
max_decoder_seq_length = max(
    [len(txt) for txt in target_texts])

input_token_index = dict([(char, i) for i, char in
                          enumerate(input_vocabulary)])  # <4>
target_token_index = dict(
    [(char, i) for i, char in enumerate(output_vocabulary)])
reverse_input_char_index = dict((i, char) for char, i in
                                input_token_index.items())  # <5>
reverse_target_char_index = dict((i, char) for char, i in
                                 target_token_index.items())

import numpy as np  # <1>  # noqa

encoder_input_data = np.zeros((n, max_encoder_seq_length, input_vocab_size),
                              dtype='float32')  # <2>
decoder_input_data = np.zeros((n, max_decoder_seq_length, output_vocab_size),
                              dtype='float32')
decoder_target_data = np.zeros((n, max_decoder_seq_length, output_vocab_size),
                               dtype='float32')
for i, (input_text, target_text) in enumerate(tqdm(
        zip(input_texts, target_texts), total=len(target_texts))):  # <3>
    for t, char in enumerate(input_text):  # <4>
        encoder_input_data[
            i, t, input_token_index[char]] = 1.  # <5>
    for t, char in enumerate(target_text):  # <6>
        decoder_input_data[
            i, t, target_token_index[char]] = 1.
        if t > 0:
            decoder_target_data[i, t - 1, target_token_index[char]] = 1

import pandas as pd  # noqa
encoder_input_data = pd.DataFrame(encoder_input_data)

from keras.models import Model  # noqa
from keras.layers import Input, LSTM, Dense  # noqa

batch_size = 64    # <1>
epochs = 100       # <2>
num_neurons = 256  # <3>

encoder_inputs = Input(shape=(None, input_vocab_size))
encoder = LSTM(num_neurons, return_state=True)
encoder_outputs, state_h, state_c = encoder(encoder_inputs)
encoder_states = [state_h, state_c]

decoder_inputs = Input(shape=(None, output_vocab_size))
decoder_lstm = LSTM(num_neurons, return_sequences=True,
                    return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_inputs,
                                     initial_state=encoder_states)
decoder_dense = Dense(output_vocab_size, activation='softmax')
decoder_outputs = decoder_dense(decoder_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)

model.compile(optimizer='rmsprop', loss='categorical_crossentropy',
              metrics=['acc'])

import os  # noqa
from keras.callbacks import ModelCheckpoint  # noqa
from nlpia.constants import BIGDATA_PATH  # noqa
checkpoint_path = os.path.join(BIGDATA_PATH, 'checkpoints')
checkpoint_path = os.path.join(checkpoint_path, 'nlpia-seq2seq-translation-weights.{epoch:02d}-{val_loss:.2f}.hdf5')

checkpoint_callback = ModelCheckpoint(checkpoint_path,
                                      monitor='val_loss', verbose=0, save_best_only=False, mode='auto')
model.fit([encoder_input_data, decoder_input_data],
          decoder_target_data,
          callbacks=[checkpoint_callback],
          batch_size=batch_size, epochs=epochs, validation_split=0.1)  # <4>
<commit_msg>ch10 fixed DataFrame->np.save for 3D np.arrays<commit_after>from nlpia.loaders import get_data
from tqdm import tqdm

df = get_data('deu')
input_texts, target_texts = [], []  # <1>
input_vocabulary = set()  # <3>
output_vocabulary = set()
start_token, stop_token = '\t\n'  # <2>
n = len(df)

for input_text, target_text in tqdm(zip(df.eng, df.deu), total=n):
    target_text = start_token + target_text \
        + stop_token  # <7>
    input_texts.append(input_text)
    target_texts.append(target_text)
    for char in input_text:  # <8>
        if char not in input_vocabulary:
            input_vocabulary.add(char)
    for char in target_text:
        if char not in output_vocabulary:
            output_vocabulary.add(char)

input_vocabulary = sorted(input_vocabulary)  # <1>
output_vocabulary = sorted(output_vocabulary)

input_vocab_size = len(input_vocabulary)  # <2>
output_vocab_size = len(output_vocabulary)
max_encoder_seq_length = max(
    [len(txt) for txt in input_texts])  # <3>
max_decoder_seq_length = max(
    [len(txt) for txt in target_texts])

input_token_index = dict([(char, i) for i, char in
                          enumerate(input_vocabulary)])  # <4>
target_token_index = dict(
    [(char, i) for i, char in enumerate(output_vocabulary)])
reverse_input_char_index = dict((i, char) for char, i in
                                input_token_index.items())  # <5>
reverse_target_char_index = dict((i, char) for char, i in
                                 target_token_index.items())

import numpy as np  # <1>  # noqa

encoder_input_data = np.zeros((n, max_encoder_seq_length, input_vocab_size),
                              dtype='float32')  # <2>
decoder_input_data = np.zeros((n, max_decoder_seq_length, output_vocab_size),
                              dtype='float32')
decoder_target_data = np.zeros((n, max_decoder_seq_length, output_vocab_size),
                               dtype='float32')
for i, (input_text, target_text) in enumerate(tqdm(
        zip(input_texts, target_texts), total=len(target_texts))):  # <3>
    for t, char in enumerate(input_text):  # <4>
        encoder_input_data[
            i, t, input_token_index[char]] = 1.  # <5>
    for t, char in enumerate(target_text):  # <6>
        decoder_input_data[
            i, t, target_token_index[char]] = 1.
        if t > 0:
            decoder_target_data[i, t - 1, target_token_index[char]] = 1

np.save('encoder_input_data.np', encoder_input_data, allow_pickle=False)
np.save('decoder_input_data.np', decoder_input_data, allow_pickle=False)
np.save('decoder_target_data.np', decoder_target_data, allow_pickle=False)

from keras.models import Model  # noqa
from keras.layers import Input, LSTM, Dense  # noqa

batch_size = 64    # <1>
epochs = 100       # <2>
num_neurons = 256  # <3>

encoder_inputs = Input(shape=(None, input_vocab_size))
encoder = LSTM(num_neurons, return_state=True)
encoder_outputs, state_h, state_c = encoder(encoder_inputs)
encoder_states = [state_h, state_c]

decoder_inputs = Input(shape=(None, output_vocab_size))
decoder_lstm = LSTM(num_neurons, return_sequences=True,
                    return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_inputs,
                                     initial_state=encoder_states)
decoder_dense = Dense(output_vocab_size, activation='softmax')
decoder_outputs = decoder_dense(decoder_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)

model.compile(optimizer='rmsprop', loss='categorical_crossentropy',
              metrics=['acc'])

import os  # noqa
from keras.callbacks import ModelCheckpoint  # noqa
from nlpia.constants import BIGDATA_PATH  # noqa
checkpoint_path = os.path.join(BIGDATA_PATH, 'checkpoints')
checkpoint_path = os.path.join(checkpoint_path, 'nlpia-seq2seq-translation-weights.{epoch:02d}-{val_loss:.2f}.hdf5')

checkpoint_callback = ModelCheckpoint(checkpoint_path,
                                      monitor='val_loss', verbose=0, save_best_only=False, mode='auto')
model.fit([encoder_input_data, decoder_input_data],
          decoder_target_data,
          callbacks=[checkpoint_callback],
          batch_size=batch_size, epochs=epochs, validation_split=0.1)  # <4>
<|endoftext|>


<commit_before><commit_msg>Telemetry: stop supporting browserless benchmarks in story runner<commit_after><|endoftext|>


<commit_before><commit_msg>tests: Add o3 classic init x86 boot tests<commit_after><|endoftext|>


<commit_before>#!/usr/local/bin/python3

import json, os, sys
from diff_adt import DiffConfig, DiffResult
from time import localtime, strftime
from subprocess import call
from diff_lev import *

CURRENT_TIMESTAMP = strftime("%Y-%m-%d-%H%M", localtime())
DEBUG_MODE = True

def main():
    print('Getting config and preparing run ...', end=' ')
    config = get_config()
    print('done!')

    print('Initiating grading', *config.labs, '...', end='\n\n')

    # Detect initial run or directory structure corruption and run setup
    if not (os.path.isdir(config.csv_path)
            and os.path.isdir(config.rosters_dir)
            and os.path.isdir(config.results_dir)
            and os.path.isdir(config.submissions_dir)):
        run_init_setup(config)

    rosters = build_rosters(config.roster_paths)

    write_lab_list_for_MATLAB(config)
    if not setup_solution_files(config):
        print('\n\nUnable to set up reference solutions. Exiting.')
        exit(1)

    print('   Running MATLAB script to generate student outputs ...', end='\n\n')
    print('\n\nMATLAB run ' + ('finished!' if generate_MATLAB_output() else '\n\nFAILED!'), end='\n\n')

    print('Comparing results and writing output ...', end=' ')
    result = DiffResult()

    for lab in config.labs:
        diff_lab_outputs(result, lab[:-2], config)

    output_result_to_csv(result, config, rosters)
    for lab in config.labs:
        output_result_to_csv(result, config, rosters, lab_num=lab[3:-2])

    print('ALL DONE!', end='\n\n')

def run_init_setup(config):
    print('Looks like this is the first time you are running this script.\n'
          'Let me set up some directories ...', end='\n\n')
    for p in [config.csv_path, config.rosters_dir, config.results_dir, config.submissions_dir]:
        if p is config.submissions_dir:
            mkdir(config.submissions_dir)
            for lab in config.labs:
                mkdir(config.submissions_dir + lab[:-2])
        else:
            mkdir(p)
    print('\nAll set up! Now, copy student submissions into {}labXX/, and'.format(config.submissions_dir),
          '\nplace the class rosters (CSV exported from PolyLearn) into {}.'.format(config.rosters_dir),
          '\nOnce copying is done, please re-run:', *sys.argv)
    exit(0)

def mkdir(directory):
    print('   mkdir', directory)
    if not os.path.isdir(directory):
        os.mkdir(directory)

def write_lab_list_for_MATLAB(config):
    # Write list of labs to .dat file for MATLAB to read which items to execute
    lab_list_dat = open(config.submissions_dir + 'lab_list.dat', 'w')
    for name in config.labs:
        lab_list_dat.write(name + '\n')
    lab_list_dat.close()

def setup_solution_files(config):
    new_solutions_success = False
    default_solution_success = False

    if check_solution_source(config):
        print('Solution source for all labs detected.\n',
              '  Firing up MATLAB to generate new solutions ...', end='\n\n')
        sys.stdout.flush()
        new_solutions_success = generate_new_solutions(config)
        print('Solution generation', 'successful!' if new_solutions_success else 'failed :(', end='\n\n')

    if not new_solutions_success:
        print('Could not find solution sources for all labs.\n',
              '  Copying default solutions over instead ... ', end='\n   ')
        sys.stdout.flush()
        default_solution_success = copy_default_solutions(config)
        print('\nCopy complete!' if default_solution_success
              else 'Copy failed! Please check permissions.', end='\n\n')

    return new_solutions_success or default_solution_success

def check_solution_source(config):
    result = True
    for lab in config.labs:
        result &= os.path.isfile(config.solutions_dir + 'source/' + lab)
    return result

def generate_new_solutions(config):
    return not call(['matlab', '-nodesktop', '-nosplash', '-nodisplay', '-r',
                     "try, cd '{}', pwd, run('./generate_solution'), catch exc, getReport(exc), end, exit".format(
                         os.getcwd())])

def copy_default_solutions(config):
    result = True
    default_dir = config.solutions_dir + 'default/'
    for file_name in os.listdir(default_dir):
        if '.txt' in file_name:
            result &= not call(['cp', default_dir + file_name, config.solutions_dir])
    return result

def generate_MATLAB_output():
    script = 'generate_output_vm.m' if len(sys.argv) > 1 and sys.argv[1].lower() == '-vm' else 'generate_output.m'
    return not call(['matlab', '-nodesktop', '-nosplash', '-nodisplay', '-r',
                     "try, cd '{}', pwd, run('./{}'), catch exc, getReport(exc), end, exit".format(
                         os.getcwd(), script)])

def diff_lab_outputs(result_obj, lab_dir_name, config):
    submissions_dir = config.submissions_dir
    solutions_dir = config.solutions_dir
    results_dir = config.results_dir

    files = [f for f in os.listdir(results_dir)
             if os.path.isfile(os.path.join(results_dir, f)) and lab_dir_name in f]

    solution_file = solutions_dir + lab_dir_name + '.out.txt'
    alt_solution_file = solutions_dir + lab_dir_name + '.alt.txt'

    for f in files:
        lab_index = f.find('_lab')
        author_name = join_last_name(f[:lab_index])

        if DEBUG_MODE:
            print('comparing', solution_file, 'and', submissions_dir + f, 'for ' + author_name, end='')

        if os.path.isfile(alt_solution_file):
            diff_result = max(cmp(solution_file, results_dir + f),
                              cmp(alt_solution_file, results_dir + f))
        else:
            diff_result = cmp(solution_file, results_dir + f)

        if DEBUG_MODE:
            print(' ... comparison result', diff_result)

        result_obj.add_result(author_name, lab_dir_name, round(diff_result * config.score_out_of, 2))

def output_result_to_csv(result_obj, config, rosters, lab_num=''):
    if DEBUG_MODE:
        print('Final Result Object:\n', result_obj)

    rosters.append(("", []))

    csv_roster = {}
    for id, roster in rosters:
        csv = open('{}{}_{}{}{}.csv'.format(
            config.csv_path,
            CURRENT_TIMESTAMP,
            config.csv_name,
            ('_' if id else '') + id,
            ('_lab' + lab_num) if lab_num else ''
        ), 'w')

        if lab_num:
            write_to_csv(csv, config.csv_header + 'lab' + lab_num)
        else:
            write_to_csv(csv, config.csv_header + str(config.labs)[1:-1].replace(' ', ''))

        csv_roster[id] = (csv, roster)

    result = result_obj.result
    result_tuple_list = sorted([(k, v) for k, v in result.items()])

    for author_name, diff_results in result_tuple_list:
        id = find_roster_id_for_author(author_name, rosters)
        all_results = per_author_result_to_csv_entry(config.labs, diff_results)

        entry_str = '{},{},{}'.format(
            author_name.replace('_', ','),
            csv_roster[id][1][author_name] if id else '',
            all_results if not lab_num else (
                str(diff_results['lab' + lab_num]) if 'lab' + lab_num in diff_results else ''
            )
        )

        csv_to_write_to = csv_roster[id][0]
        write_to_csv(csv_to_write_to, entry_str)

        if csv_to_write_to is not csv_roster[""][0]:
            write_to_csv(csv_roster[""][0], entry_str)

    for csv, _ in csv_roster.values():
        csv.close()

def per_author_result_to_csv_entry(lab_file_names, author_result):
    csv_entry_str = ''
    for lab_file_name in lab_file_names:
        lab = lab_file_name[:-2]
        csv_entry_str += str(author_result[lab]) if lab in author_result else ''
        csv_entry_str += ','
    return csv_entry_str[:-1]

def find_roster_id_for_author(author_name, rosters):
    for id, roster in rosters:

        if DEBUG_MODE:
            print('Author name used to look up in roster: ' + author_name)
            print('Roster\n' + str(roster))

        if author_name in roster:
            return id
    return ""

def write_to_csv(csv_file, line_to_write):
    if DEBUG_MODE:
        print(line_to_write)
    csv_file.write(line_to_write + '\n')

def get_config():
    with open('diff_config.json') as data_file:
        data = json.load(data_file)

    # get the list of lab file names
    lab_file_names = []
    for num in data['labs']:
        lab_file_names.append('lab{:02}.m'.format(num) if num else 'final.m')

    return DiffConfig(lab_file_names,
                      data['submissions_dir'], data['solutions_dir'], data['rosters_dir'], data['results_dir'],
                      data['result_csv_path'], data['result_csv_name'], data['score_out_of'],
                      data['roster_paths'])

def join_last_name(orig_name_str, wrapper_str='"'):
    tokens = orig_name_str.split('_')
    if len(tokens) > 2:
        return '_'.join([tokens[0], '{}{}{}'.format(wrapper_str, ' '.join(tokens[1:]), wrapper_str)])
    else:
        return orig_name_str

def build_rosters(roster_paths):
    rosters = []

    for id, path in roster_paths:
        roster_file = open(path, 'r')
        lines = roster_file.readlines()[1:]

        roster = {}
        for l in lines:
            tokens = l.split(',')
            author_name = '_'.join(tokens[:2])
            author_email = tokens[2]
            roster[author_name] = author_email

        rosters.append((id, roster))

    return rosters

if __name__ == '__main__':
    main()
<commit_msg>Disabling debug prints.<commit_after>#!/usr/local/bin/python3

import json, os, sys
from diff_adt import DiffConfig, DiffResult
from time import localtime, strftime
from subprocess import call
from diff_lev import *

CURRENT_TIMESTAMP = strftime("%Y-%m-%d-%H%M", localtime())
DEBUG_MODE = False

def main():
    print('Getting config and preparing run ...', end=' ')
    config = get_config()
    print('done!')

    print('Initiating grading', *config.labs, '...', end='\n\n')

    # Detect initial run or directory structure corruption and run setup
    if not (os.path.isdir(config.csv_path)
            and os.path.isdir(config.rosters_dir)
            and os.path.isdir(config.results_dir)
            and os.path.isdir(config.submissions_dir)):
        run_init_setup(config)

    rosters = build_rosters(config.roster_paths)

    write_lab_list_for_MATLAB(config)
    if not setup_solution_files(config):
        print('\n\nUnable to set up reference solutions. Exiting.')
        exit(1)

    print('   Running MATLAB script to generate student outputs ...', end='\n\n')
    print('\n\nMATLAB run ' + ('finished!' if generate_MATLAB_output() else '\n\nFAILED!'), end='\n\n')

    print('Comparing results and writing output ...', end=' ')
    result = DiffResult()

    for lab in config.labs:
        diff_lab_outputs(result, lab[:-2], config)

    output_result_to_csv(result, config, rosters)
    for lab in config.labs:
        output_result_to_csv(result, config, rosters, lab_num=lab[3:-2])

    print('ALL DONE!', end='\n\n')

def run_init_setup(config):
    print('Looks like this is the first time you are running this script.\n'
          'Let me set up some directories ...', end='\n\n')
    for p in [config.csv_path, config.rosters_dir, config.results_dir, config.submissions_dir]:
        if p is config.submissions_dir:
            mkdir(config.submissions_dir)
            for lab in config.labs:
                mkdir(config.submissions_dir + lab[:-2])
        else:
            mkdir(p)
    print('\nAll set up! Now, copy student submissions into {}labXX/, and'.format(config.submissions_dir),
          '\nplace the class rosters (CSV exported from PolyLearn) into {}.'.format(config.rosters_dir),
          '\nOnce copying is done, please re-run:', *sys.argv)
    exit(0)

def mkdir(directory):
    print('   mkdir', directory)
    if not os.path.isdir(directory):
        os.mkdir(directory)

def write_lab_list_for_MATLAB(config):
    # Write list of labs to .dat file for MATLAB to read which items to execute
    lab_list_dat = open(config.submissions_dir + 'lab_list.dat', 'w')
    for name in config.labs:
        lab_list_dat.write(name + '\n')
    lab_list_dat.close()

def setup_solution_files(config):
    new_solutions_success = False
    default_solution_success = False

    if check_solution_source(config):
        print('Solution source for all labs detected.\n',
              '  Firing up MATLAB to generate new solutions ...', end='\n\n')
        sys.stdout.flush()
        new_solutions_success = generate_new_solutions(config)
        print('Solution generation', 'successful!' if new_solutions_success else 'failed :(', end='\n\n')

    if not new_solutions_success:
        print('Could not find solution sources for all labs.\n',
              '  Copying default solutions over instead ... ', end='\n   ')
        sys.stdout.flush()
        default_solution_success = copy_default_solutions(config)
        print('\nCopy complete!' if default_solution_success
              else 'Copy failed! Please check permissions.', end='\n\n')

    return new_solutions_success or default_solution_success

def check_solution_source(config):
    result = True
    for lab in config.labs:
        result &= os.path.isfile(config.solutions_dir + 'source/' + lab)
    return result

def generate_new_solutions(config):
    return not call(['matlab', '-nodesktop', '-nosplash', '-nodisplay', '-r',
                     "try, cd '{}', pwd, run('./generate_solution'), catch exc, getReport(exc), end, exit".format(
                         os.getcwd())])

def copy_default_solutions(config):
    result = True
    default_dir = config.solutions_dir + 'default/'
    for file_name in os.listdir(default_dir):
        if '.txt' in file_name:
            result &= not call(['cp', default_dir + file_name, config.solutions_dir])
    return result

def generate_MATLAB_output():
    script = 'generate_output_vm.m' if len(sys.argv) > 1 and sys.argv[1].lower() == '-vm' else 'generate_output.m'
    return not call(['matlab', '-nodesktop', '-nosplash', '-nodisplay', '-r',
                     "try, cd '{}', pwd, run('./{}'), catch exc, getReport(exc), end, exit".format(
                         os.getcwd(), script)])

def diff_lab_outputs(result_obj, lab_dir_name, config):
    submissions_dir = config.submissions_dir
    solutions_dir = config.solutions_dir
    results_dir = config.results_dir

    files = [f for f in os.listdir(results_dir)
             if os.path.isfile(os.path.join(results_dir, f)) and lab_dir_name in f]

    solution_file = solutions_dir + lab_dir_name + '.out.txt'
    alt_solution_file = solutions_dir + lab_dir_name + '.alt.txt'

    for f in files:
        lab_index = f.find('_lab')
        author_name = join_last_name(f[:lab_index])

        if DEBUG_MODE:
            print('comparing', solution_file, 'and', submissions_dir + f, 'for ' + author_name, end='')

        if os.path.isfile(alt_solution_file):
            diff_result = max(cmp(solution_file, results_dir + f),
                              cmp(alt_solution_file, results_dir + f))
        else:
            diff_result = cmp(solution_file, results_dir + f)

        if DEBUG_MODE:
            print(' ... comparison result', diff_result)

        result_obj.add_result(author_name, lab_dir_name, round(diff_result * config.score_out_of, 2))

def output_result_to_csv(result_obj, config, rosters, lab_num=''):
    if DEBUG_MODE:
        print('Final Result Object:\n', result_obj)

    rosters.append(("", []))

    csv_roster = {}
    for id, roster in rosters:
        csv = open('{}{}_{}{}{}.csv'.format(
            config.csv_path,
            CURRENT_TIMESTAMP,
            config.csv_name,
            ('_' if id else '') + id,
            ('_lab' + lab_num) if lab_num else ''
        ), 'w')

        if lab_num:
            write_to_csv(csv, config.csv_header + 'lab' + lab_num)
        else:
            write_to_csv(csv, config.csv_header + str(config.labs)[1:-1].replace(' ', ''))

        csv_roster[id] = (csv, roster)

    result = result_obj.result
    result_tuple_list = sorted([(k, v) for k, v in result.items()])

    for author_name, diff_results in result_tuple_list:
        id = find_roster_id_for_author(author_name, rosters)
        all_results = per_author_result_to_csv_entry(config.labs, diff_results)

        entry_str = '{},{},{}'.format(
            author_name.replace('_', ','),
            csv_roster[id][1][author_name] if id else '',
            all_results if not lab_num else (
                str(diff_results['lab' + lab_num]) if 'lab' + lab_num in diff_results else ''
            )
        )

        csv_to_write_to = csv_roster[id][0]
        write_to_csv(csv_to_write_to, entry_str)

        if csv_to_write_to is not csv_roster[""][0]:
            write_to_csv(csv_roster[""][0], entry_str)

    for csv, _ in csv_roster.values():
        csv.close()

def per_author_result_to_csv_entry(lab_file_names, author_result):
    csv_entry_str = ''
    for lab_file_name in lab_file_names:
        lab = lab_file_name[:-2]
        csv_entry_str += str(author_result[lab]) if lab in author_result else ''
        csv_entry_str += ','
    return csv_entry_str[:-1]

def find_roster_id_for_author(author_name, rosters):
    for id, roster in rosters:

        if DEBUG_MODE:
            print('Author name used to look up in roster: ' + author_name)
            print('Roster\n' + str(roster))

        if author_name in roster:
            return id
    return ""

def write_to_csv(csv_file, line_to_write):
    if DEBUG_MODE:
        print(line_to_write)
    csv_file.write(line_to_write + '\n')

def get_config():
    with open('diff_config.json') as data_file:
        data = json.load(data_file)

    # get the list of lab file names
    lab_file_names = []
    for num in data['labs']:
        lab_file_names.append('lab{:02}.m'.format(num) if num else 'final.m')

    return DiffConfig(lab_file_names,
                      data['submissions_dir'], data['solutions_dir'], data['rosters_dir'], data['results_dir'],
                      data['result_csv_path'], data['result_csv_name'], data['score_out_of'],
                      data['roster_paths'])

def join_last_name(orig_name_str, wrapper_str='"'):
    tokens = orig_name_str.split('_')
    if len(tokens) > 2:
        return '_'.join([tokens[0], '{}{}{}'.format(wrapper_str, ' '.join(tokens[1:]), wrapper_str)])
    else:
        return orig_name_str

def build_rosters(roster_paths):
    rosters = []

    for id, path in roster_paths:
        roster_file = open(path, 'r')
        lines = roster_file.readlines()[1:]

        roster = {}
        for l in lines:
            tokens = l.split(',')
            author_name = '_'.join(tokens[:2])
            author_email = tokens[2]
            roster[author_name] = author_email

        rosters.append((id, roster))

    return rosters

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before><commit_msg>Add a test for kernel started without user_ns kwarg.<commit_after><|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import Dict
from pandas import DataFrame, concat
from lib.cast import safe_int_cast
from lib.data_source import DataSource

_columns = [
    "date",
    "name",
    "code",
    "new_confirmed",
    "current_confirmed",
    "total_confirmed",
    "new_deceased",
    "current_deceased",
    "total_deceased",
    "new_tested",
    "current_tested",
    "total_tested",
    "new_hospitalized",
    "current_hospitalized",
    "total_hospitalized",
    "new_intensive_care",
    "current_intensive_care",
    "total_intensive_care",
    "new_ventilator",
    "current_ventilator",
    "total_ventilator",
    "_source_link",
]

class FinMangoDataSource(DataSource):
    def parse_dataframes(
        self, dataframes: Dict[str, DataFrame], aux: Dict[str, DataFrame], **parse_opts
    ) -> DataFrame:
        # Data is nested into multiple sheets
        tables = []
        for df in dataframes[0].values():
            df.columns = _columns
            tables.append(df.iloc[2:])

        # Put all sheets together into a single DataFrame
        data = concat(tables)
        print(data)
        data["key"] = parse_opts["country"] + "_" + data["code"].str.replace("-", "_")

        # Ensure date is in ISO format
        data["date"] = data["date"].apply(lambda x: str(x)[:10])

        # Make sure that all data is numeric
        for col in data.columns:
            if col not in ("date", "name", "key", "code", "_source_link"):
                data[col] = data[col].apply(safe_int_cast)

        # Remove the "new" columns since cumulative data is more reliable
        data = data.drop(columns=["new_confirmed", "new_deceased"])

        # Output the results
        return data
<commit_msg>Fix ZA data source<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import Dict
from pandas import DataFrame, concat
from lib.cast import safe_int_cast
from lib.data_source import DataSource

_columns = [
    "date",
    "name",
    "code",
    "new_confirmed",
    "current_confirmed",
    "total_confirmed",
    "new_deceased",
    "current_deceased",
    "total_deceased",
    "new_tested",
    "current_tested",
    "total_tested",
    "new_hospitalized",
    "current_hospitalized",
    "total_hospitalized",
    "new_intensive_care",
    "current_intensive_care",
    "total_intensive_care",
    "new_ventilator",
    "current_ventilator",
    "total_ventilator",
    "_source_link",
]

class FinMangoDataSource(DataSource):
    def parse_dataframes(
        self, dataframes: Dict[str, DataFrame], aux: Dict[str, DataFrame], **parse_opts
    ) -> DataFrame:
        # Data is nested into multiple sheets
        tables = []
        for df in dataframes[0].values():
            df.columns = _columns
            tables.append(df.iloc[2:])

        # Put all sheets together into a single DataFrame
        data = concat(tables)
        data["key"] = parse_opts["country"] + "_" + data["code"].str.replace("-", "_")

        # Ensure date is in ISO format
        data["date"] = data["date"].apply(lambda x: str(x)[:10])

        # Make sure that all data is numeric
        for col in data.columns:
            if col not in ("date", "name", "key", "code", "_source_link"):
                data[col] = data[col].apply(safe_int_cast)

        # Remove the "new" columns since cumulative data is more reliable
        data = data.drop(columns=["new_confirmed", "new_deceased"])

        # Output the results
        return data
<|endoftext|>


<commit_before><commit_msg>Removes hard coding of source name<commit_after><|endoftext|>


<commit_before><commit_msg>(remove obsolete test)<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3

# author : renpeng
# github : https://github.com/laodifang
# description : media information
# date : 2015-09-24

import os, sys, re, subprocess, json

class MediaInfo :
    def __init__(self, **kwargs) :
        self.filename = kwargs.get('filename')
        self.cmd      = kwargs.get('cmd')
        self.info     = dict()

        if self.filename == None :
            self.filename = ''

        if self.cmd == None :
            for cmdpath in os.environ['PATH'].split(':') :
                if os.path.isdir(cmdpath) and 'ffprobe' in os.listdir(cmdpath) :
                    self.cmd = cmdpath + '/ffprobe'
                elif os.path.isdir(cmdpath) and 'mediainfo' in os.listdir(cmdpath) :
                    self.cmd = cmdpath + '/mediainfo'

            if self.cmd == None :
                self.cmd = ''

    def getInfo(self) :
        if not os.path.exists(self.filename) or not os.path.exists(self.cmd) :
            return None

        cmdName = os.path.basename(self.cmd)

        if cmdName == 'ffprobe' :
            self._ffmpegGetInfo()
        elif cmdName == 'mediainfo' :
            self._mediainfoGetInfo()

        return self.info

    def _ffmpegGetInfo(self) :
        cmd         = self.cmd + ' -loglevel quiet -print_format json -show_format -show_streams -show_error -count_frames -i ' + self.filename
        outputBytes = ''

        try :
            outputBytes = subprocess.check_output(cmd, shell = True)
        except subprocess.CalledProcessError as e :
            return ''

        outputText = outputBytes.decode('utf-8')
        self.info  = self._ffmpegGetInfoJson(outputText)

    def _ffmpegGetInfoJson(self, sourceString) :
        mediaInfo = dict()
        infoDict  = dict()

        try :
            infoDict = json.loads(sourceString)
        except json.JSONDecodeError as err :
            return mediaInfo

        mediaInfo['container'] = infoDict.get('format').get('format_name')
        mediaInfo['fileSize']  = infoDict.get('format').get('size')
        mediaInfo['duration']  = infoDict.get('format').get('duration')
        mediaInfo['bitrate']   = infoDict.get('format').get('bit_rate')

        videoStreamIndex = None
        audioStreamIndex = None

        for item in infoDict.get('streams') :
            codec_type = item.get('codec_type')

            if codec_type == 'video' :
                videoStreamIndex       = item.get('index')
                mediaInfo['haveVideo'] = 1
            elif codec_type == 'audio' :
                audioStreamIndex       = item.get('index')
                mediaInfo['haveAudio'] = 1

        if mediaInfo.get('haveVideo') :
            mediaInfo['videoCodec']        = infoDict.get('streams')[videoStreamIndex].get('codec_name')
            mediaInfo['videoCodecProfile'] = infoDict.get('streams')[videoStreamIndex].get('profile')
            mediaInfo['videoDuration']     = infoDict.get('streams')[videoStreamIndex].get('duration')
            mediaInfo['videoBitrate']      = infoDict.get('streams')[videoStreamIndex].get('bit_rate')
            mediaInfo['videoWidth']        = infoDict.get('streams')[videoStreamIndex].get('width')
            mediaInfo['videoHeight']       = infoDict.get('streams')[videoStreamIndex].get('height')
            mediaInfo['videoAspectRatio']  = infoDict.get('streams')[videoStreamIndex].get('display_aspect_ratio')
            mediaInfo['videoFrameRate']    = infoDict.get('streams')[videoStreamIndex].get('r_frame_rate')
            mediaInfo['videoFrameCount']   = infoDict.get('streams')[videoStreamIndex].get('nb_read_frames')

        if mediaInfo.get('haveAudio') :
            mediaInfo['audioCodec']        = infoDict.get('streams')[audioStreamIndex].get('codec_name')
            mediaInfo['audioCodecProfile'] = infoDict.get('streams')[audioStreamIndex].get('profile')
            mediaInfo['audioDuration']     = infoDict.get('streams')[audioStreamIndex].get('duration')
            mediaInfo['audioBitrate']      = infoDict.get('streams')[audioStreamIndex].get('bit_rate')
            mediaInfo['audioChannel']      = infoDict.get('streams')[audioStreamIndex].get('channels')
            mediaInfo['audioSamplingRate'] = infoDict.get('streams')[audioStreamIndex].get('sample_rate')
            mediaInfo['audioFrameCount']   = infoDict.get('streams')[audioStreamIndex].get('nb_read_frames')

        return mediaInfo

    def _mediainfoGetInfo(self) :
        cmd         = self.cmd + ' -f ' + self.filename
        outputBytes = ''

        try :
            outputBytes = subprocess.check_output(cmd, shell = True)
        except subprocess.CalledProcessError as e :
            return ''

        outputText = outputBytes.decode('utf-8')
        self.info  = self._mediainfoGetInfoRegex(outputText)

    def _mediainfoGetInfoRegex(self, sourceString) :
        mediaInfo   = dict()

        general     = re.search("(^General\n.*?\n\n)", sourceString, re.S)
        if general :
            generalInfo = general.group(0)

            container   = re.search("Format\s*:\s*([\w\_\-\\\/\. ]+)\n",    generalInfo, re.S)
            fileSize    = re.search("File size\s*:\s*(\d+)\.?\d*\n",        generalInfo, re.S)
            duration    = re.search("Duration\s*:\s*(\d+)\.?\d*\n",         generalInfo, re.S)
            bitrate     = re.search("Overall bit rate\s*:\s*(\d+)\.?\d*\n", generalInfo, re.S)

            mediaInfo['container'] = container.group(1)
            mediaInfo['fileSize']  = fileSize.group(1)
            mediaInfo['duration']  = duration.group(1)
            mediaInfo['bitrate']   = bitrate.group(1)

        video     = re.search("(Video[\s\#\d]*\n.*?\n\n)", sourceString, re.S)
        if video :
            videoInfo = video.group(0)

            videoCodec        = re.search("Codec\s*:\s*([\w\_\-\\\/\. ]+)\n",           videoInfo, re.S)
            videoCodecProfile = re.search("Codec profile\s*:\s*([\w\_\-\\\/\@\. ]+)\n", videoInfo, re.S)
            videoDuration     = re.search("Duration\s*:\s*(\d+)\.?\d*\n",               videoInfo, re.S)
            videoBitrate      = re.search("Bit rate\s*:\s*(\d+)\n",                     videoInfo, re.S)
            videoWidth        = re.search("Width\s*:\s*(\d+)\n",                        videoInfo, re.S)
            videoHeight       = re.search("Height\s*:\s*(\d+)\n",                       videoInfo, re.S)
            videoAspectRatio  = re.search("Display aspect ratio\s*:\s*([\d\.]+)\n",     videoInfo, re.S)
            videoFrameRate    = re.search("Frame rate\s*:\s*([\d\.]+)\n",               videoInfo, re.S)
            videoFrameCount   = re.search("Frame count\s*:\s*(\d+)\.?\d*\n",            videoInfo, re.S)

            if videoCodec :
                mediaInfo['videoCodec'] = videoCodec.group(1)
            if videoCodecProfile :
                mediaInfo['videoCodecProfile'] = videoCodecProfile.group(1)
            if videoDuration :
                mediaInfo['videoDuration'] = videoDuration.group(1)
            if videoBitrate :
                mediaInfo['videoBitrate'] = videoBitrate.group(1)
            if videoWidth :
                mediaInfo['videoWidth'] = videoWidth.group(1)
            if videoHeight :
                mediaInfo['videoHeight'] = videoHeight.group(1)
            if videoAspectRatio :
                mediaInfo['videoAspectRatio'] = videoAspectRatio.group(1)
            if videoFrameRate :
                mediaInfo['videoFrameRate'] = videoFrameRate.group(1)
            if videoFrameCount :
                mediaInfo['videoFrameCount'] = videoFrameCount.group(1)

        audio     = re.search("(Audio[\s\#\d]*\n.*?\n\n)", sourceString, re.S)
        if audio :
            audioInfo = audio.group(0)

            tmpAudioCodec     = re.search("Codec\s*:\s*([\w\_\-\\\/ ]+)\n",             audioInfo, re.S)
            audioCodec        = re.search("\w+", tmpAudioCodec.group(1), re.S)
            audioCodecProfile = re.search("Codec profile\s*:\s*([\w\_\-\\\/\@\. ]+)\n", audioInfo, re.S)
            if audioCodecProfile is None :
                audioCodecProfile = re.search("Format profile\s*:\s*([\w\_\-\\\/\@\. ]+)\n", audioInfo, re.S)

            audioDuration     = re.search("Duration\s*:\s*(\d+)\.?\d*\n", audioInfo, re.S)
            audioBitrate      = re.search("Bit rate\s*:\s*(\d+)\n",       audioInfo, re.S)
            audioChannel      = re.search("Channel\(s\)\s*:\s*(\d+)\n",   audioInfo, re.S)
            samplingRate      = re.search("Sampling rate\s*:\s*([\w\_\-\\\/\@\. ]+)\n", audioInfo, re.S)
            audioSamplingRate = re.search("\d+", samplingRate.group(1), re.S)

            if audioCodec :
                mediaInfo['audioCodec'] = audioCodec.group(0)
            if audioCodecProfile :
                mediaInfo['audioCodecProfile'] = audioCodecProfile.group(1)
            if audioDuration :
                mediaInfo['audioDuration'] = audioDuration.group(1)
            if audioBitrate :
                mediaInfo['audioBitrate'] = audioBitrate.group(1)
            if audioChannel :
                mediaInfo['audioChannel'] = audioChannel.group(1)
            if audioSamplingRate :
                mediaInfo['audioSamplingRate'] = audioSamplingRate.group(0)

        return mediaInfo

<commit_msg>fix bug<commit_after>#!/usr/bin/env python3

# author : renpeng
# github : https://github.com/laodifang
# description : media information
# date : 2015-09-24

import os, sys, re, subprocess, json

class MediaInfo :
    def __init__(self, **kwargs) :
        self.filename = kwargs.get('filename')
        self.cmd      = kwargs.get('cmd')
        self.info     = dict()

        if self.filename == None :
            self.filename = ''

        if self.cmd == None :
            for cmdpath in os.environ['PATH'].split(':') :
            	if os.path.isdir(cmdpath) and 'mediainfo' in os.listdir(cmdpath) :
                    self.cmd = cmdpath + '/mediainfo'
                elif os.path.isdir(cmdpath) and 'ffprobe' in os.listdir(cmdpath) :
                    self.cmd = cmdpath + '/ffprobe'

            if self.cmd == None :
                self.cmd = ''

    def getInfo(self) :
        if not os.path.exists(self.filename) or not os.path.exists(self.cmd) :
            return None

        cmdName = os.path.basename(self.cmd)

        if cmdName == 'ffprobe' :
            self._ffmpegGetInfo()
        elif cmdName == 'mediainfo' :
            self._mediainfoGetInfo()

        return self.info

    def _ffmpegGetInfo(self) :
        cmd         = self.cmd + ' -loglevel quiet -print_format json -show_format -show_streams -show_error -count_frames -i ' + self.filename
        outputBytes = ''

        try :
            outputBytes = subprocess.check_output(cmd, shell = True)
        except subprocess.CalledProcessError as e :
            return ''

        outputText = outputBytes.decode('utf-8')
        self.info  = self._ffmpegGetInfoJson(outputText)

    def _ffmpegGetInfoJson(self, sourceString) :
        mediaInfo = dict()
        infoDict  = dict()

        try :
            infoDict = json.loads(sourceString)
        except json.JSONDecodeError as err :
            return mediaInfo

        mediaInfo['container'] = infoDict.get('format').get('format_name')
        mediaInfo['fileSize']  = infoDict.get('format').get('size')
        mediaInfo['duration']  = infoDict.get('format').get('duration')
        mediaInfo['bitrate']   = infoDict.get('format').get('bit_rate')

        videoStreamIndex = None
        audioStreamIndex = None

        for item in infoDict.get('streams') :
            codec_type = item.get('codec_type')

            if codec_type == 'video' :
                videoStreamIndex       = item.get('index')
                mediaInfo['haveVideo'] = 1
            elif codec_type == 'audio' :
                audioStreamIndex       = item.get('index')
                mediaInfo['haveAudio'] = 1

        if mediaInfo.get('haveVideo') :
            mediaInfo['videoCodec']        = infoDict.get('streams')[videoStreamIndex].get('codec_name')
            mediaInfo['videoCodecProfile'] = infoDict.get('streams')[videoStreamIndex].get('profile')
            mediaInfo['videoDuration']     = infoDict.get('streams')[videoStreamIndex].get('duration')
            mediaInfo['videoBitrate']      = infoDict.get('streams')[videoStreamIndex].get('bit_rate')
            mediaInfo['videoWidth']        = infoDict.get('streams')[videoStreamIndex].get('width')
            mediaInfo['videoHeight']       = infoDict.get('streams')[videoStreamIndex].get('height')
            mediaInfo['videoAspectRatio']  = infoDict.get('streams')[videoStreamIndex].get('display_aspect_ratio')
            mediaInfo['videoFrameRate']    = infoDict.get('streams')[videoStreamIndex].get('r_frame_rate')
            mediaInfo['videoFrameCount']   = infoDict.get('streams')[videoStreamIndex].get('nb_read_frames')

        if mediaInfo.get('haveAudio') :
            mediaInfo['audioCodec']        = infoDict.get('streams')[audioStreamIndex].get('codec_name')
            mediaInfo['audioCodecProfile'] = infoDict.get('streams')[audioStreamIndex].get('profile')
            mediaInfo['audioDuration']     = infoDict.get('streams')[audioStreamIndex].get('duration')
            mediaInfo['audioBitrate']      = infoDict.get('streams')[audioStreamIndex].get('bit_rate')
            mediaInfo['audioChannel']      = infoDict.get('streams')[audioStreamIndex].get('channels')
            mediaInfo['audioSamplingRate'] = infoDict.get('streams')[audioStreamIndex].get('sample_rate')
            mediaInfo['audioFrameCount']   = infoDict.get('streams')[audioStreamIndex].get('nb_read_frames')

        return mediaInfo

    def _mediainfoGetInfo(self) :
        cmd         = self.cmd + ' -f ' + self.filename
        outputBytes = ''

        try :
            outputBytes = subprocess.check_output(cmd, shell = True)
        except subprocess.CalledProcessError as e :
            return ''

        outputText = outputBytes.decode('utf-8')
        self.info  = self._mediainfoGetInfoRegex(outputText)

    def _mediainfoGetInfoRegex(self, sourceString) :
        mediaInfo   = dict()

        general     = re.search("(^General\n.*?\n\n)", sourceString, re.S)
        if general :
            generalInfo = general.group(0)

            container   = re.search("Format\s*:\s*([\w\_\-\\\/\. ]+)\n",    generalInfo, re.S)
            fileSize    = re.search("File size\s*:\s*(\d+)\.?\d*\n",        generalInfo, re.S)
            duration    = re.search("Duration\s*:\s*(\d+)\.?\d*\n",         generalInfo, re.S)
            bitrate     = re.search("Overall bit rate\s*:\s*(\d+)\.?\d*\n", generalInfo, re.S)

            mediaInfo['container'] = container.group(1)
            mediaInfo['fileSize']  = fileSize.group(1)
            mediaInfo['duration']  = duration.group(1)
            mediaInfo['bitrate']   = bitrate.group(1)

        video     = re.search("(Video[\s\#\d]*\n.*?\n\n)", sourceString, re.S)
        if video :
            videoInfo = video.group(0)

            videoCodec        = re.search("Codec\s*:\s*([\w\_\-\\\/\. ]+)\n",           videoInfo, re.S)
            videoCodecProfile = re.search("Codec profile\s*:\s*([\w\_\-\\\/\@\. ]+)\n", videoInfo, re.S)
            videoDuration     = re.search("Duration\s*:\s*(\d+)\.?\d*\n",               videoInfo, re.S)
            videoBitrate      = re.search("Bit rate\s*:\s*(\d+)\n",                     videoInfo, re.S)
            videoWidth        = re.search("Width\s*:\s*(\d+)\n",                        videoInfo, re.S)
            videoHeight       = re.search("Height\s*:\s*(\d+)\n",                       videoInfo, re.S)
            videoAspectRatio  = re.search("Display aspect ratio\s*:\s*([\d\.]+)\n",     videoInfo, re.S)
            videoFrameRate    = re.search("Frame rate\s*:\s*([\d\.]+)\n",               videoInfo, re.S)
            videoFrameCount   = re.search("Frame count\s*:\s*(\d+)\.?\d*\n",            videoInfo, re.S)

            if videoCodec :
                mediaInfo['videoCodec'] = videoCodec.group(1)
            if videoCodecProfile :
                mediaInfo['videoCodecProfile'] = videoCodecProfile.group(1)
            if videoDuration :
                mediaInfo['videoDuration'] = videoDuration.group(1)
            if videoBitrate :
                mediaInfo['videoBitrate'] = videoBitrate.group(1)
            if videoWidth :
                mediaInfo['videoWidth'] = videoWidth.group(1)
            if videoHeight :
                mediaInfo['videoHeight'] = videoHeight.group(1)
            if videoAspectRatio :
                mediaInfo['videoAspectRatio'] = videoAspectRatio.group(1)
            if videoFrameRate :
                mediaInfo['videoFrameRate'] = videoFrameRate.group(1)
            if videoFrameCount :
                mediaInfo['videoFrameCount'] = videoFrameCount.group(1)

        audio     = re.search("(Audio[\s\#\d]*\n.*?\n\n)", sourceString, re.S)
        if audio :
            audioInfo = audio.group(0)

            tmpAudioCodec     = re.search("Codec\s*:\s*([\w\_\-\\\/ ]+)\n",             audioInfo, re.S)
            audioCodec        = re.search("\w+", tmpAudioCodec.group(1), re.S)
            audioCodecProfile = re.search("Codec profile\s*:\s*([\w\_\-\\\/\@\. ]+)\n", audioInfo, re.S)
            if audioCodecProfile is None :
                audioCodecProfile = re.search("Format profile\s*:\s*([\w\_\-\\\/\@\. ]+)\n", audioInfo, re.S)

            audioDuration     = re.search("Duration\s*:\s*(\d+)\.?\d*\n", audioInfo, re.S)
            audioBitrate      = re.search("Bit rate\s*:\s*(\d+)\n",       audioInfo, re.S)
            audioChannel      = re.search("Channel\(s\)\s*:\s*(\d+)\n",   audioInfo, re.S)
            samplingRate      = re.search("Sampling rate\s*:\s*([\w\_\-\\\/\@\. ]+)\n", audioInfo, re.S)
            audioSamplingRate = re.search("\d+", samplingRate.group(1), re.S)

            if audioCodec :
                mediaInfo['audioCodec'] = audioCodec.group(0)
            if audioCodecProfile :
                mediaInfo['audioCodecProfile'] = audioCodecProfile.group(1)
            if audioDuration :
                mediaInfo['audioDuration'] = audioDuration.group(1)
            if audioBitrate :
                mediaInfo['audioBitrate'] = audioBitrate.group(1)
            if audioChannel :
                mediaInfo['audioChannel'] = audioChannel.group(1)
            if audioSamplingRate :
                mediaInfo['audioSamplingRate'] = audioSamplingRate.group(0)

        return mediaInfo

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import Crawler4xiami
import CrawlerCore
import threading

CrawlerCore.put_song(380246)#小手拉大手
CrawlerCore.put_song(130762)#后来
CrawlerCore.put_song(378181)#爱我别走
CrawlerCore.put_song(1769901638)#Rolling In The Deep
CrawlerCore.put_song(75250)
CrawlerCore.put_song(2067242)#好久不见
#CrawlerCore.put_song(1769938907)#An Der Schönen Blauen Donau Op.314
threads=[]
for i in range(5):
     t = threading.Thread(target=Crawler4xiami.worker)
     threads.append(t)
     t.daemon = True
     t.start()

while True:
	inp=raw_input()
	if inp=="e":
		break

print("Canceled")
CrawlerCore.cancel()
for t in threads:
	t.join()
<commit_msg>add donwloader<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import Crawler4xiami
import CrawlerCore
import threading

CrawlerCore.put_song(380246)#小手拉大手
CrawlerCore.put_song(130762)#后来
CrawlerCore.put_song(378181)#爱我别走
CrawlerCore.put_song(1769901638)#Rolling In The Deep
CrawlerCore.put_song(2067242)#好久不见
#CrawlerCore.put_song(1769938907)#An Der Schönen Blauen Donau Op.314
threads=[]
for i in range(5):
     t = threading.Thread(target=Crawler4xiami.worker)
     threads.append(t)
     t.daemon = True
     t.start()

while True:
	inp=raw_input()
	if inp=="e":
		break

print("Canceled")
CrawlerCore.cancel()
for t in threads:
	t.join()
<|endoftext|>