<commit_before>import logging
import time
import requests
from cis_notifications import common
from cis_notifications import secret

logger = logging.getLogger(__name__)

class Event(object):
    """Handle events from lambda and generate hooks out to publishers."""

    def __init__(self, event):
        """[summary]

        Arguments:
            object {[type]} -- [an instance of the event class.]
            event {[type]} -- [the event as ingested from the kinesis stream.]
            subscriptions {[type]} -- [list of urls to post notifications to.]
        """
        self.config = common.get_config()
        self.event = event
        self.secret_manager = secret.Manager()
        self.access_token = None

    def to_notification(self):
        """[summary]
        Transform the instance of the event from the stream into a notification payload.

        [return] JSON data structure to send using requests.
        """

        logger.debug("An event was received", extra={"event": self.event})

        updated_record = self.event.get("dynamodb")

        operation = "foxy"  # Just a place holder in case we have an unhandled event.

        if self.event.get("eventName") == "INSERT":
            operation = "create"

        if self.event.get("eventName") == "MODIFY":
            operation = "update"

        if self.event.get("eventName") == "DELETE":
            operation = "delete"

        if updated_record is not None:
            # Provided the event is the structure that
            notification = {
                "operation": operation,
                "id": updated_record["Keys"]["id"]["S"],
                "time": updated_record["ApproximateCreationDateTime"],
            }

            logger.debug("Notification generated.", extra={"notification": notification})

            return notification
        else:
            logger.debug("No notification generated.")
            return {}

    def send(self, notification):
        """[summary]
        Get the list of notification endpoints from the object constructor and send a POST with the json payload.

        Arguments:
            object {[type]} -- [an instance of the event class.]
            object {[notification]} -- [A json payload that you would like to send to the RP.]

        [return] Dictionary of status codes by publisher.
        """
        # Not in-memory access token?
        if not self.access_token:
            # Load whatever is in our secrets
            self.access_token_dict = self.secret_manager.secretmgr("az_access_token")

            # Check if what we had in secrets is still valid!
            # This includes 10s leeway for clock sync issues
            if float(self.access_token_dict.get("exp", 60.0)) < time.time() - 10:
                logger.info("Access token has expired, refreshing")
                authzero = self._get_authzero_client()
                self.access_token_dict = authzero.exchange_for_access_token()
                # Auth0 gives us the difference (expires_in) not a time stamp, so we need to calculate when the token
                # expires.
                self.access_token_dict["exp"] = time.time() + float(self.access_token_dict.get("expires_in", 60.0))
                self.secret_manager.secretmgr_store("az_access_token", self.access_token_dict)
            else:
                logger.info("Re-using cached access token")
            self.access_token = self.access_token_dict["access_token"]

        if notification != {}:
            rp_urls = self.config(
                "rp_urls", namespace="cis", default="https://dinopark.k8s.dev.sso.allizom.org/events/update"
            )

            results = {}
            for url in rp_urls.split(","):
                result = self._notify_via_post(url, notification, self.access_token)
                results[url] = result
            return results

    def _get_authzero_client(self):
        authzero = secret.AuthZero(
            client_id=self.secret_manager.secret("client_id"),
            client_secret=self.secret_manager.secret("client_secret"),
            api_identifier=self.config("api_identifier", namespace="cis", default="hook.dev.sso.allizom.org"),
            authzero_tenant=self.config("authzero_tenant", namespace="cis", default="auth.mozilla.auth0.com"),
        )

        return authzero

    def _notify_via_post(self, url, json_payload, access_token):
        """[summary]
        Notify a single publisher of the user_id that was updated and return only the status code.

        Arguments:
            url {[type]} -- [the url of the publisher you woud like to notify.]
            json_payload {[type]} -- [the event to send to the publisher.]
        """

        try:
            response = requests.post(
                url, json=json_payload, headers={"authorization": "Bearer {}".format(access_token)}
            )
            return response.status_code
        except requests.exceptions.RequestException:
            return "Unknown"
        except requests.exceptions.HTTPError:
            return "HTTPError"
        except requests.exceptions.ConnectionError:
            return "ConnectionError"
        except requests.exceptions.Timeout:
            return "Timeout"
<commit_msg>add 15min of leeway<commit_after>import logging
import time
import requests
from cis_notifications import common
from cis_notifications import secret

logger = logging.getLogger(__name__)

class Event(object):
    """Handle events from lambda and generate hooks out to publishers."""

    def __init__(self, event):
        """[summary]

        Arguments:
            object {[type]} -- [an instance of the event class.]
            event {[type]} -- [the event as ingested from the kinesis stream.]
            subscriptions {[type]} -- [list of urls to post notifications to.]
        """
        self.config = common.get_config()
        self.event = event
        self.secret_manager = secret.Manager()
        self.access_token = None

    def to_notification(self):
        """[summary]
        Transform the instance of the event from the stream into a notification payload.

        [return] JSON data structure to send using requests.
        """

        logger.debug("An event was received", extra={"event": self.event})

        updated_record = self.event.get("dynamodb")

        operation = "foxy"  # Just a place holder in case we have an unhandled event.

        if self.event.get("eventName") == "INSERT":
            operation = "create"

        if self.event.get("eventName") == "MODIFY":
            operation = "update"

        if self.event.get("eventName") == "DELETE":
            operation = "delete"

        if updated_record is not None:
            # Provided the event is the structure that
            notification = {
                "operation": operation,
                "id": updated_record["Keys"]["id"]["S"],
                "time": updated_record["ApproximateCreationDateTime"],
            }

            logger.debug("Notification generated.", extra={"notification": notification})

            return notification
        else:
            logger.debug("No notification generated.")
            return {}

    def send(self, notification):
        """[summary]
        Get the list of notification endpoints from the object constructor and send a POST with the json payload.

        Arguments:
            object {[type]} -- [an instance of the event class.]
            object {[notification]} -- [A json payload that you would like to send to the RP.]

        [return] Dictionary of status codes by publisher.
        """
        # Not in-memory access token?
        if not self.access_token:
            # Load whatever is in our secrets
            self.access_token_dict = self.secret_manager.secretmgr("az_access_token")

            # Check if what we had in secrets is still valid!
            # This includes 10s leeway for clock sync issues and 15min (900s) for max-lambda function time.
            # Since tokens are normally valid for 86400s (1 day) that should accomodate for all cases. If these were to
            # be less than 15min for any reason, it would simply bypass the cache
            if float(self.access_token_dict.get("exp", 60.0)) < time.time() - 910:
                logger.info("Access token has expired, refreshing")
                authzero = self._get_authzero_client()
                self.access_token_dict = authzero.exchange_for_access_token()
                # Auth0 gives us the difference (expires_in) not a time stamp, so we need to calculate when the token
                # expires.
                self.access_token_dict["exp"] = time.time() + float(self.access_token_dict.get("expires_in", 60.0))
                self.secret_manager.secretmgr_store("az_access_token", self.access_token_dict)
            else:
                logger.info("Re-using cached access token")
            self.access_token = self.access_token_dict["access_token"]

        if notification != {}:
            rp_urls = self.config(
                "rp_urls", namespace="cis", default="https://dinopark.k8s.dev.sso.allizom.org/events/update"
            )

            results = {}
            for url in rp_urls.split(","):
                result = self._notify_via_post(url, notification, self.access_token)
                results[url] = result
            return results

    def _get_authzero_client(self):
        authzero = secret.AuthZero(
            client_id=self.secret_manager.secret("client_id"),
            client_secret=self.secret_manager.secret("client_secret"),
            api_identifier=self.config("api_identifier", namespace="cis", default="hook.dev.sso.allizom.org"),
            authzero_tenant=self.config("authzero_tenant", namespace="cis", default="auth.mozilla.auth0.com"),
        )

        return authzero

    def _notify_via_post(self, url, json_payload, access_token):
        """[summary]
        Notify a single publisher of the user_id that was updated and return only the status code.

        Arguments:
            url {[type]} -- [the url of the publisher you woud like to notify.]
            json_payload {[type]} -- [the event to send to the publisher.]
        """

        try:
            response = requests.post(
                url, json=json_payload, headers={"authorization": "Bearer {}".format(access_token)}
            )
            return response.status_code
        except requests.exceptions.RequestException:
            return "Unknown"
        except requests.exceptions.HTTPError:
            return "HTTPError"
        except requests.exceptions.ConnectionError:
            return "ConnectionError"
        except requests.exceptions.Timeout:
            return "Timeout"
<|endoftext|>


<commit_before>""" Functions related to writing to and retrieving from cache files
"""
import os

import numpy as np

from log_yatsm import logger

_image_ID_str = 'image_IDs'

def get_line_cache_name(dataset_config, n_images, row, nbands):
    """ Returns cache filename for specified config and line number

    Args:
      dataset_config (dict): configuration information about the dataset
      n_images (int): number of images in dataset
      row (int): line of the dataset for output
      nbands (int): number of bands in dataset

    Returns:
      str: filename of cache file

    """
    path = dataset_config.get('cache_line_dir')
    if not path:
        return

    filename = 'yatsm_r%i_n%i_b%i.npy.npz' % (row, n_images, nbands)

    return os.path.join(path, filename)

def get_line_cache_pattern(row, nbands, regex=False):
    """ Returns a pattern for a cache file from a certain row

    This function is useful for finding all cache files from a line, ignoring
    the number of images in the file.

    Args:
      row (int): line of the dataset for output
      nbands (int): number of bands in dataset
      regex (bool, optional): return a regular expression instead of glob
        style (default: False)

    Returns:
      str: filename pattern for cache files from line `row`

    """
    wildcard = '.*' if regex else '*'
    pattern = 'yatsm_r{l}_n{w}_b{b}.npy.npz'.format(
        l=row, w=wildcard, b=nbands)

    return pattern

def test_cache(dataset_config):
    """ Test cache directory for ability to read from or write to

    Args:
      dataset_config (dict): dictionary of dataset configuration options

    Returns:
      (read_cache, write_cache): tuple of bools describing ability to read from
        and write to cache directory

    """
    # Try to find / use cache
    read_cache = False
    write_cache = False

    cache_dir = dataset_config.get('cache_line_dir')
    if cache_dir:
        # Test existence
        if os.path.isdir(cache_dir):
            if os.access(cache_dir, os.R_OK):
                read_cache = True
            if os.access(cache_dir, os.W_OK):
                write_cache = True
            if read_cache and not write_cache:
                logger.warning('Cache directory exists but is not writable')
        else:
            # If it doesn't already exist, can we create it?
            try:
                os.makedirs(cache_dir)
            except:
                logger.warning('Could not create cache directory')
            else:
                read_cache = True
                write_cache = True

    logger.debug('Attempt reading in from cache directory?: {b}'.format(
        b=read_cache))
    logger.debug('Attempt writing to cache directory?: {b}'.format(
        b=write_cache))

    return read_cache, write_cache

def read_cache_file(cache_filename, image_IDs=None):
    """ Returns image data from a cache file

    If `image_IDs` is not None this function will try to ensure data from cache
    file come from the list of image IDs provided. If cache file does not
    contain a list of image IDs, it will skip the check and return cache data.

    Args:
      cache_filename (str): cache filename
      image_IDs (iterable, optional): list of image IDs corresponding to data
        in cache file. If not specified, function will not check for
        correspondence (default: None)

    Returns:
      np.ndarray, or None: Return Y as np.ndarray if possible and if the
        cache file passes the consistency check specified by `image_IDs`; else
        None

    """
    try:
        cache = np.load(cache_filename)
    except IOError:
        return None

    if _image_ID_str in cache.files and image_IDs is not None:
        if not np.array_equal(image_IDs, cache[_image_ID_str]):
            logger.warning('Cache file data in {f} do not match images '
                           'specified'.format(f=cache_filename))
            return None

    return cache['Y']

def write_cache_file(cache_filename, Y, image_IDs):
    """ Writes data to a cache file using np.savez_compressed

    Args:
      cache_filename (str): cache filename
      Y (np.ndarray)
      image_IDs (iterable): list of image IDs corresponding to data in cache
        file. If not specified, function will not check for correspondence

    """
    np.savez_compressed(cache_filename, **{
        'Y': Y, _image_ID_str: image_IDs
    })

# Cache file updating
def update_cache_file(images, image_IDs,
                      old_cache_filename, new_cache_filename,
                      line, reader, reader_kwargs={}):
    """ Modify an existing cache file to contain data within `images`

    This should be useful for updating a set of cache files to reflect
    modifications to the timeseries dataset without completely reading the
    data into another cache file.

    For example, the cache file could be updated to reflect the deletion of
    a misregistered or cloudy image. Another common example would be for
    updating cache files to include newly acquired observations.

    Note that this updater will not handle updating cache files to include
    new bands.

    Args:
      images (iterable): list of new image filenames
      image_IDs (iterable): list of new image identifying strings
      old_cache_filename (str): filename of cache file to update
      new_cache_filename (str): filename of new cache file which includes
        modified data
      line (int): the line of data to be updated
      reader (callable): GDAL or BIP image reader function from `yatsm.readers`
      reader_kwargs (dict): additional keyword arguments for `reader` other
        than the filenames to read and the line to read

    Raises:
      ValueError: Raise error if old cache file does not record `image_IDs`

    """
    images = np.asarray(images)
    image_IDs = np.asarray(image_IDs)

    # Cannot proceed if old cache file doesn't store filenames
    old_cache = np.load(old_cache_filename)
    if _image_ID_str not in old_cache.files:
        raise ValueError('Cannot update cache.'
                         'Old cache file does not store image IDs.')
    old_IDs = old_cache[_image_ID_str]
    old_Y = old_cache['Y']
    nband, _, ncol = old_Y.shape

    # Create new Y and add in values retained from old cache
    new_Y = np.zeros((nband, image_IDs.size, ncol),
                     dtype=old_Y.dtype.type)
    new_IDs = np.zeros(image_IDs.size, dtype=image_IDs.dtype)

    # Check deletions -- find which indices to retain in new cache
    retain_old = np.where(np.in1d(old_IDs, image_IDs))[0]
    if retain_old.size == 0:
        logger.warning('No image IDs in common in old cache file.')
    else:
        logger.debug('    retaining {r} of {n} images'.format(
            r=retain_old.size, n=old_IDs.size))
        # Find indices of old data to insert into new data
        idx_old_IDs = np.argsort(old_IDs)
        sorted_old_IDs = old_IDs[idx_old_IDs]
        idx_IDs = np.searchsorted(sorted_old_IDs,
                                  image_IDs[np.in1d(image_IDs, old_IDs)])

        retain_old = idx_old_IDs[idx_IDs]

        # Indices to insert into new data
        retain_new = np.where(np.in1d(image_IDs, old_IDs))[0]

        new_Y[:, retain_new, :] = old_Y[:, retain_old, :]
        new_IDs[retain_new] = old_IDs[retain_old]

    # Check additions -- find which indices we need to insert
    insert = np.where(np.in1d(image_IDs, old_IDs) == False)[0]

    if retain_old.size == 0 and insert.size == 0:
        raise ValueError('Cannot update cache file -- '
                         'no data retained or added')

    # Read in the remaining data from disk
    if insert.size > 0:
        logger.debug('Inserting {n} new images into cache'.format(
            n=insert.size))
        insert_Y = reader(images[insert], line, **reader_kwargs)
        new_Y[:, insert, :] = insert_Y
        new_IDs[insert] = image_IDs[insert]

    np.testing.assert_equal(new_IDs, image_IDs)

    # Save
    write_cache_file(new_cache_filename, new_Y, image_IDs)
<commit_msg>PEP8;docstring format;compare fix<commit_after>""" Functions related to writing to and retrieving from cache files
"""
import os

import numpy as np

from log_yatsm import logger

_image_ID_str = 'image_IDs'

def get_line_cache_name(dataset_config, n_images, row, nbands):
    """ Returns cache filename for specified config and line number

    Args:
        dataset_config (dict): configuration information about the dataset
        n_images (int): number of images in dataset
        row (int): line of the dataset for output
        nbands (int): number of bands in dataset

    Returns:
        str: filename of cache file

    """
    path = dataset_config.get('cache_line_dir')
    if not path:
        return

    filename = 'yatsm_r%i_n%i_b%i.npy.npz' % (row, n_images, nbands)

    return os.path.join(path, filename)

def get_line_cache_pattern(row, nbands, regex=False):
    """ Returns a pattern for a cache file from a certain row

    This function is useful for finding all cache files from a line, ignoring
    the number of images in the file.

    Args:
        row (int): line of the dataset for output
        nbands (int): number of bands in dataset
        regex (bool, optional): return a regular expression instead of glob
            style (default: False)

    Returns:
        str: filename pattern for cache files from line ``row``

    """
    wildcard = '.*' if regex else '*'
    pattern = 'yatsm_r{l}_n{w}_b{b}.npy.npz'.format(
        l=row, w=wildcard, b=nbands)

    return pattern

def test_cache(dataset_config):
    """ Test cache directory for ability to read from or write to

    Args:
        dataset_config (dict): dictionary of dataset configuration options

    Returns:
        tuple: tuple of bools describing ability to read from and write to
            cache directory

    """
    # Try to find / use cache
    read_cache = False
    write_cache = False

    cache_dir = dataset_config.get('cache_line_dir')
    if cache_dir:
        # Test existence
        if os.path.isdir(cache_dir):
            if os.access(cache_dir, os.R_OK):
                read_cache = True
            if os.access(cache_dir, os.W_OK):
                write_cache = True
            if read_cache and not write_cache:
                logger.warning('Cache directory exists but is not writable')
        else:
            # If it doesn't already exist, can we create it?
            try:
                os.makedirs(cache_dir)
            except:
                logger.warning('Could not create cache directory')
            else:
                read_cache = True
                write_cache = True

    logger.debug('Attempt reading in from cache directory?: {b}'.format(
        b=read_cache))
    logger.debug('Attempt writing to cache directory?: {b}'.format(
        b=write_cache))

    return read_cache, write_cache

def read_cache_file(cache_filename, image_IDs=None):
    """ Returns image data from a cache file

    If `image_IDs` is not None this function will try to ensure data from cache
    file come from the list of image IDs provided. If cache file does not
    contain a list of image IDs, it will skip the check and return cache data.

    Args:
        cache_filename (str): cache filename
        image_IDs (iterable, optional): list of image IDs corresponding to data
            in cache file. If not specified, function will not check for
            correspondence (default: None)

    Returns:
        np.ndarray, or None: Return Y as np.ndarray if possible and if the
            cache file passes the consistency check specified by `image_IDs`,
            else None

    """
    try:
        cache = np.load(cache_filename)
    except IOError:
        return None

    if _image_ID_str in cache.files and image_IDs is not None:
        if not np.array_equal(image_IDs, cache[_image_ID_str]):
            logger.warning('Cache file data in {f} do not match images '
                           'specified'.format(f=cache_filename))
            return None

    return cache['Y']

def write_cache_file(cache_filename, Y, image_IDs):
    """ Writes data to a cache file using np.savez_compressed

    Args:
        cache_filename (str): cache filename
        Y (np.ndarray)
        image_IDs (iterable): list of image IDs corresponding to data in cache
            file. If not specified, function will not check for correspondence

    """
    np.savez_compressed(cache_filename, **{
        'Y': Y, _image_ID_str: image_IDs
    })

# Cache file updating
def update_cache_file(images, image_IDs,
                      old_cache_filename, new_cache_filename,
                      line, reader, reader_kwargs={}):
    """ Modify an existing cache file to contain data within `images`

    This should be useful for updating a set of cache files to reflect
    modifications to the timeseries dataset without completely reading the
    data into another cache file.

    For example, the cache file could be updated to reflect the deletion of
    a misregistered or cloudy image. Another common example would be for
    updating cache files to include newly acquired observations.

    Note that this updater will not handle updating cache files to include
    new bands.

    Args:
        images (iterable): list of new image filenames
        image_IDs (iterable): list of new image identifying strings
        old_cache_filename (str): filename of cache file to update
        new_cache_filename (str): filename of new cache file which includes
            modified data
        line (int): the line of data to be updated
        reader (callable): GDAL or BIP image reader function from
            ``yatsm.readers``
        reader_kwargs (dict): additional keyword arguments for ``reader`` other
            than the filenames to read and the line to read

    Raises:
        ValueError: Raise error if old cache file does not record ``image_IDs``

    """
    images = np.asarray(images)
    image_IDs = np.asarray(image_IDs)

    # Cannot proceed if old cache file doesn't store filenames
    old_cache = np.load(old_cache_filename)
    if _image_ID_str not in old_cache.files:
        raise ValueError('Cannot update cache.'
                         'Old cache file does not store image IDs.')
    old_IDs = old_cache[_image_ID_str]
    old_Y = old_cache['Y']
    nband, _, ncol = old_Y.shape

    # Create new Y and add in values retained from old cache
    new_Y = np.zeros((nband, image_IDs.size, ncol),
                     dtype=old_Y.dtype.type)
    new_IDs = np.zeros(image_IDs.size, dtype=image_IDs.dtype)

    # Check deletions -- find which indices to retain in new cache
    retain_old = np.where(np.in1d(old_IDs, image_IDs))[0]
    if retain_old.size == 0:
        logger.warning('No image IDs in common in old cache file.')
    else:
        logger.debug('    retaining {r} of {n} images'.format(
            r=retain_old.size, n=old_IDs.size))
        # Find indices of old data to insert into new data
        idx_old_IDs = np.argsort(old_IDs)
        sorted_old_IDs = old_IDs[idx_old_IDs]
        idx_IDs = np.searchsorted(sorted_old_IDs,
                                  image_IDs[np.in1d(image_IDs, old_IDs)])

        retain_old = idx_old_IDs[idx_IDs]

        # Indices to insert into new data
        retain_new = np.where(np.in1d(image_IDs, old_IDs))[0]

        new_Y[:, retain_new, :] = old_Y[:, retain_old, :]
        new_IDs[retain_new] = old_IDs[retain_old]

    # Check additions -- find which indices we need to insert
    insert = np.where(np.in1d(image_IDs, old_IDs, invert=True))[0]

    if retain_old.size == 0 and insert.size == 0:
        raise ValueError('Cannot update cache file -- '
                         'no data retained or added')

    # Read in the remaining data from disk
    if insert.size > 0:
        logger.debug('Inserting {n} new images into cache'.format(
            n=insert.size))
        insert_Y = reader(images[insert], line, **reader_kwargs)
        new_Y[:, insert, :] = insert_Y
        new_IDs[insert] = image_IDs[insert]

    np.testing.assert_equal(new_IDs, image_IDs)

    # Save
    write_cache_file(new_cache_filename, new_Y, image_IDs)
<|endoftext|>


<commit_before>import pytest
import numpy
from numpy.testing import assert_allclose
from hypothesis import given, assume
from hypothesis.extra.numpy import arrays
from hypothesis.strategies import integers

from ...._classes.window_encode import _get_positions
from ...._classes.window_encode import _get_full_gradients
from ...._classes.window_encode import _get_full_inputs
from ...._classes.window_encode import _compute_hidden_layer
from ...._classes.window_encode import _zero_features_past_sequence_boundaries
from ....ops import NumpyOps

from ...strategies import arrays_BOP_BO
from ...strategies import arrays_OPFI_BI_lengths
from ...strategies import lengths

@pytest.mark.parametrize(
    'ids_batch,vectors_batch', [
        (
            (
                ('the', 'cat', 'sat'),
                ('on', 'the', 'cat')
            ),
            (
                ((0.,1.),(1.,2.),(2.,3.)),
                ((3.,4.),(0.,1.),(1.,2.))
            ),
        ),
        (
            (
                ('the',),
                ('the',)
            ),
            (
                ((3.,4.),),
                ((3.,4.),),
            ),
        ),
        (
            (
                ('a', 'b', 'a', 'b', 'd', 'e', 'f'),
            ),
            (
                ((0.,1.), (1.,2.), (0.,1.), (1.,2.), (3.,4.), (5.,6.), (6.,7.)),
            )
        )
    ]
)
def test_get_positions(ids_batch, vectors_batch):
    positions, vectors_table = _get_positions(ids_batch, vectors_batch)
    for key, ijs in positions.items():
        for i, j in ijs:
            assert_allclose(vectors_batch[i][j], vectors_table[key])

@given(arrays_BOP_BO())
def test_only_one_piece_gets_gradient_if_unique_max(x_BOP_d_BO):
    x_BOP, d_BO = x_BOP_d_BO
    assert len(x_BOP.shape) == 3
    assert x_BOP.shape[0] == d_BO.shape[0]
    assert x_BOP.shape[1] == d_BO.shape[1]
    whiches_BO = numpy.argmax(x_BOP, axis=-1)
    d_BOP = numpy.zeros(x_BOP.shape)
    _get_full_gradients(d_BOP, d_BO, whiches_BO)
    for b in range(x_BOP.shape[0]):
        for o in range(x_BOP.shape[1]):
            num_at_max = sum(x_BOP[b,o] >= x_BOP[b, o, whiches_BO[b,o]])
            if num_at_max == 1.:
                if d_BO[b, o] != 0:
                    assert sum(cell != 0. for cell in d_BOP[b, o]) == 1.
                    assert abs(d_BOP[b, o]).argmax(axis=-1) == x_BOP[b, o].argmax()
                assert sum(d_BOP[b, o]) == d_BO[b, o]

@given(
    lengths().flatmap(lambda lenlen:
        arrays('int32', shape=(lenlen,),
        elements=integers(min_value=1, max_value=10))))
def test_zero_features_past_sequence_boundaries(seq_lengths):
    ops = NumpyOps()
    B = sum(seq_lengths)
    O = 1
    P = 1
    H__BFOP = ops.allocate((B, 5, O, P))
    H__BFOP += 1
    _zero_features_past_sequence_boundaries(H__BFOP, seq_lengths)
    sequences = ops.unflatten(H__BFOP, seq_lengths)
    for sequence in sequences:
        w0 = sequence[0].reshape((5, O, P))
        assert_allclose(w0[0], 0)
        assert_allclose(w0[1], 0)
        if len(sequence) >= 2:
            w1 = sequence[1].reshape((5, O, P))
            assert_allclose(w1[0], 0)
        w_m1 = sequence[-1].reshape((5, O, P))
        assert_allclose(w_m1[3], 0)
        assert_allclose(w_m1[4], 0)
        if len(sequence) >= 2:
            w_m2 = sequence[-2].reshape((5, O, P))
            assert_allclose(w_m2[4], 0)

@given(arrays_OPFI_BI_lengths(max_B=10, max_P=10, max_I=10))
def test_compute_hidden_layer(arrays_OPFI_BI_lengths):
    W__OPFI, vectors__BI, lengths = arrays_OPFI_BI_lengths
    # Converting to int saves us from a lot of annoying false positives from
    # floating point arithmetic
    W__OPFI = numpy.asarray(W__OPFI, dtype='int32')
    vectors__BI = numpy.asarray(vectors__BI, dtype='int32')
    O, P, F, I = W__OPFI.shape
    B = sum(lengths)
    assume(F == 5)
    assert sum(lengths) == vectors__BI.shape[0]
    assert vectors__BI.shape[1] == W__OPFI.shape[-1]
    ops = NumpyOps()

    H__BFOP = _compute_hidden_layer(ops, W__OPFI, vectors__BI, lengths)
    assert H__BFOP.shape == (B, F, O, P)

    b = 0
    sequences = ops.unflatten(vectors__BI, lengths)
    for sequence in sequences:
        for i, input__i in enumerate(sequence):
            expected__OPF = numpy.tensordot(W__OPFI, input__i, axes=[[3], [0]])
            assert expected__OPF.shape == (O, P, F)
            expected = expected__OPF.transpose((2, 0, 1))
            computed = H__BFOP[b]
            assert expected.shape == computed.shape
            # Check the centre column first
            assert_allclose(expected[2], computed[2],
                            rtol=1e-4, atol=0.001)
            # Now check the 1 column, which represents the L context.
            if i >= 1:
                assert_allclose(expected[1], computed[1], rtol=1e-4, atol=0.001)
            else:
                assert_allclose(computed[1], 0)
            # Now check the 3 column, which represents the R context.
            if len(sequence)-i >= 2:
                assert_allclose(expected[3], computed[3], rtol=1e-4, atol=0.001)
            else:
                assert_allclose(computed[3], 0)
            # Now check the 4 column, which represents the RR context.
            if len(sequence)-i >= 3:
                assert_allclose(expected[4], computed[4], rtol=1e-4, atol=0.001)
            else:
                assert_allclose(computed[4], 0)
            # Now check the 0 column, which represents the LL context.
            if i >= 2:
                assert_allclose(expected[0], computed[0], rtol=1e-4, atol=0.001)
            else:
                assert_allclose(computed[0], 0)
            b += 1

def im2col(sequence, window):
    assert window == 2
    output = numpy.zeros((sequence.shape[0], window*2+1, sequence.shape[1]))
    output[:, 2] = sequence
    output[2:, 0] = sequence[:-2]
    output[1:, 1] = sequence[:-1]
    output[:-1, 3] = sequence[1:]
    output[:-2, 4] = sequence[2:]
    # Words 0 and 1 have no LL feature
    assert sum(abs(output[0, 0])) == 0
    if len(output) >= 2:
        assert sum(abs(output[1, 0])) == 0
        # Word 0 no L feature
        assert sum(abs(output[1, 0])) == 0
    # Words -1 and -2 have no RR feature
    assert sum(abs(output[-1, 4])) == 0
    if len(output) >= 2:
        assert sum(abs(output[-2, 4])) == 0
        # Word -1 has no R feature
    assert sum(abs(output[-1, 3])) == 0
    return output
<commit_msg>Test get_full_inputs<commit_after>import pytest
import numpy
from numpy.testing import assert_allclose
from hypothesis import given, assume
from hypothesis.extra.numpy import arrays
from hypothesis.strategies import integers

from ...._classes.window_encode import _get_positions
from ...._classes.window_encode import _get_full_gradients
from ...._classes.window_encode import _get_full_inputs
from ...._classes.window_encode import _compute_hidden_layer
from ...._classes.window_encode import _zero_features_past_sequence_boundaries
from ....ops import NumpyOps

from ...strategies import arrays_BOP_BO
from ...strategies import arrays_OPFI_BI_lengths
from ...strategies import lengths

@pytest.mark.parametrize(
    'ids_batch,vectors_batch', [
        (
            (
                ('the', 'cat', 'sat'),
                ('on', 'the', 'cat')
            ),
            (
                ((0.,1.),(1.,2.),(2.,3.)),
                ((3.,4.),(0.,1.),(1.,2.))
            ),
        ),
        (
            (
                ('the',),
                ('the',)
            ),
            (
                ((3.,4.),),
                ((3.,4.),),
            ),
        ),
        (
            (
                ('a', 'b', 'a', 'b', 'd', 'e', 'f'),
            ),
            (
                ((0.,1.), (1.,2.), (0.,1.), (1.,2.), (3.,4.), (5.,6.), (6.,7.)),
            )
        )
    ]
)
def test_get_positions(ids_batch, vectors_batch):
    positions, vectors_table = _get_positions(ids_batch, vectors_batch)
    for key, ijs in positions.items():
        for i, j in ijs:
            assert_allclose(vectors_batch[i][j], vectors_table[key])

def test_get_full_inputs_zeros_edges():
    B = 11
    F = 5
    I = 3
    output = numpy.zeros((B, F, I))
    vectors = numpy.ones((B, I))
    lengths = [5, 6]
    assert sum(lengths) == B
    _get_full_inputs(output,
        vectors, lengths)
    assert_allclose(output[0, 0], 0)
    assert_allclose(output[0, 1], 0)
    assert_allclose(output[1, 0], 0)
    assert_allclose(output[1, 1], 1)
    assert_allclose(output[3, 0], 1)
    assert_allclose(output[3, 1], 1)
    assert_allclose(output[3, 3], 1)
    assert_allclose(output[3, 4], 0)
    assert_allclose(output[4, 0], 1)
    assert_allclose(output[4, 1], 1)
    assert_allclose(output[4, 3], 0)
    assert_allclose(output[4, 4], 0)
    assert_allclose(output[5, 0], 0)
    assert_allclose(output[5, 1], 0)
    assert_allclose(output[6, 0], 0)
    assert_allclose(output[6, 1], 1)
    assert_allclose(output[6, 3], 1)
    assert_allclose(output[6, 4], 1)
    assert_allclose(output[10, 0], 1)
    assert_allclose(output[10, 1], 1)
    assert_allclose(output[10, 2], 1)
    assert_allclose(output[10, 3], 0)
    assert_allclose(output[10, 4], 0)

@given(arrays_BOP_BO())
def test_only_one_piece_gets_gradient_if_unique_max(x_BOP_d_BO):
    x_BOP, d_BO = x_BOP_d_BO
    assert len(x_BOP.shape) == 3
    assert x_BOP.shape[0] == d_BO.shape[0]
    assert x_BOP.shape[1] == d_BO.shape[1]
    whiches_BO = numpy.argmax(x_BOP, axis=-1)
    d_BOP = numpy.zeros(x_BOP.shape)

    _get_full_gradients(d_BOP, d_BO, whiches_BO)

    for b in range(x_BOP.shape[0]):
        for o in range(x_BOP.shape[1]):
            num_at_max = sum(x_BOP[b,o] >= x_BOP[b, o, whiches_BO[b,o]])
            if num_at_max == 1.:
                if d_BO[b, o] != 0:
                    assert sum(cell != 0. for cell in d_BOP[b, o]) == 1.
                    assert abs(d_BOP[b, o]).argmax(axis=-1) == x_BOP[b, o].argmax()
                assert sum(d_BOP[b, o]) == d_BO[b, o]

@given(
    lengths().flatmap(lambda lenlen:
        arrays('int32', shape=(lenlen,),
        elements=integers(min_value=1, max_value=10))))
def test_zero_features_past_sequence_boundaries(seq_lengths):
    ops = NumpyOps()
    B = sum(seq_lengths)
    O = 1
    P = 1
    H__BFOP = ops.allocate((B, 5, O, P))
    H__BFOP += 1
    _zero_features_past_sequence_boundaries(H__BFOP, seq_lengths)
    sequences = ops.unflatten(H__BFOP, seq_lengths)
    for sequence in sequences:
        w0 = sequence[0].reshape((5, O, P))
        assert_allclose(w0[0], 0)
        assert_allclose(w0[1], 0)
        if len(sequence) >= 2:
            w1 = sequence[1].reshape((5, O, P))
            assert_allclose(w1[0], 0)
        w_m1 = sequence[-1].reshape((5, O, P))
        assert_allclose(w_m1[3], 0)
        assert_allclose(w_m1[4], 0)
        if len(sequence) >= 2:
            w_m2 = sequence[-2].reshape((5, O, P))
            assert_allclose(w_m2[4], 0)

@given(arrays_OPFI_BI_lengths(max_B=10, max_P=10, max_I=10))
def test_compute_hidden_layer(arrays_OPFI_BI_lengths):
    W__OPFI, vectors__BI, lengths = arrays_OPFI_BI_lengths
    # Converting to int saves us from a lot of annoying false positives from
    # floating point arithmetic
    W__OPFI = numpy.asarray(W__OPFI, dtype='int32')
    vectors__BI = numpy.asarray(vectors__BI, dtype='int32')
    O, P, F, I = W__OPFI.shape
    B = sum(lengths)
    assume(F == 5)
    assert sum(lengths) == vectors__BI.shape[0]
    assert vectors__BI.shape[1] == W__OPFI.shape[-1]
    ops = NumpyOps()

    H__BFOP = _compute_hidden_layer(ops, W__OPFI, vectors__BI, lengths)
    assert H__BFOP.shape == (B, F, O, P)

    b = 0
    sequences = ops.unflatten(vectors__BI, lengths)
    for sequence in sequences:
        for i, input__i in enumerate(sequence):
            expected__OPF = numpy.tensordot(W__OPFI, input__i, axes=[[3], [0]])
            assert expected__OPF.shape == (O, P, F)
            expected = expected__OPF.transpose((2, 0, 1))
            computed = H__BFOP[b]
            assert expected.shape == computed.shape
            # Check the centre column first
            assert_allclose(expected[2], computed[2],
                            rtol=1e-4, atol=0.001)
            # Now check the 1 column, which represents the L context.
            if i >= 1:
                assert_allclose(expected[1], computed[1], rtol=1e-4, atol=0.001)
            else:
                assert_allclose(computed[1], 0)
            # Now check the 3 column, which represents the R context.
            if len(sequence)-i >= 2:
                assert_allclose(expected[3], computed[3], rtol=1e-4, atol=0.001)
            else:
                assert_allclose(computed[3], 0)
            # Now check the 4 column, which represents the RR context.
            if len(sequence)-i >= 3:
                assert_allclose(expected[4], computed[4], rtol=1e-4, atol=0.001)
            else:
                assert_allclose(computed[4], 0)
            # Now check the 0 column, which represents the LL context.
            if i >= 2:
                assert_allclose(expected[0], computed[0], rtol=1e-4, atol=0.001)
            else:
                assert_allclose(computed[0], 0)
            b += 1
<|endoftext|>


<commit_before>"""
Various errors (exceptions) for OpenNSA.

Author: Henrik Thostrup Jensen <htj@nordu.net>
Copyright: NORDUnet (2011-2012)
"""

# NSI Error codes:
#
# PAYLOAD_ERROR             00100
#   MISSING_PARAMETER         00101
#   UNSUPPORTED_PARAMETER     00102
# NOT_IMPLEMENTED           00103
#   VERSION_NOT_SUPPORTED     00104
#   CONNECTION_ERROR          00200
#   INVALID_TRANSITION        00201
#   CONNECTION_EXISTS         00202
#   CONNECTION_NONEXISTENT    00203
#   CONNECTION_GONE           00204
#   CONNECTION_CREATE_ERROR   00205
# SECURITY_ERROR            00300
#   AUTHENTICATION_FAILURE    00301
#   UNAUTHORIZED              00302
# TOPOLOGY_ERROR            00400
#   UNKNOWN_STP               00401
#   STP_RESOLUTION_ERROR      00402
#   NO_PATH_FOUND             00403
#   VLANID_INTERCANGE_NOT_SUPPORTED 00404
# INTERNAL_ERROR            00500
#   INTERNAL_NRM_ERROR        00501
# RESOURCE_UNAVAILABLE      00600
#   STP_UNAVALABLE            00601
#   BANDWIDTH_UNAVAILABLE     00602

## Errors which are/should only be used internally

class CallbackTimeoutError(Exception):
    pass

class InvalidRequestError(Exception):
    pass

class ResourceNotAvailableError(Exception):
    pass

class TopologyError(Exception):
    pass

class StateTransitionError(Exception):
    pass

## NSI Errors

class NSIError(Exception):
    """
    This class is only used to indicate that an exception class is a "proper"
    NSI error, and have an errorId that can be used to produce a message with
    an errorId in it.

    It should not be instantiated directly.
    """
    errorId = None

class PayloadError(NSIError):

    errorId = '00100'

class ConnectionError(NSIError):

    errorId = '00200'

class ConnectionExistsError(ConnectionError):

    errorId = '00202'

class ConnectionNonExistentError(ConnectionError):

    errorId = '00203'

class SecurityError(NSIError):

    errorId = '00300'

class TopologyError(NSIError):

    errorId = '00400'

class InternalServerError(NSIError):

    errorId = '00500'

class ResourceUnavailableError(NSIError):

    errorId = '00600'

NSI_ERROR_CODE_TABLE = {
    '00100' : PayloadError,
    '00200' : ConnectionError,
    '00202' : ConnectionExistsError,
    '00203' : ConnectionNonExistentError,
    '00300' : SecurityError,
    '00400' : TopologyError,
    '00500' : InternalServerError,
    '00600' : ResourceUnavailableError,
}

def lookup(error_code):

    assert type(error_code) is str and len(error_code) == 5, 'Invalid Error Code (type or length is wrong'

    ex = NSI_ERROR_CODE_TABLE.get(error_code)
    if ex is None:
        error_code = error_code[0:2] + '  '
        ex = NSI_ERROR_CODE_TABLE.get(error_code)

    if ex is None:
        raise ValueError('Could not find error type. Invalid error code: %s' % error_code)

    return ex

# These should really be replaced with proper errors

class ReserveError(Exception):
    pass

class ProvisionError(Exception):
    pass

class ReleaseError(Exception):
    pass

class TerminateError(Exception):
    pass

class QueryError(Exception):
    pass

<commit_msg>make comments in error module more readable<commit_after>"""
Various errors (exceptions) for OpenNSA.

Author: Henrik Thostrup Jensen <htj@nordu.net>
Copyright: NORDUnet (2011-2012)
"""

# NSI Error codes:
#
# PAYLOAD_ERROR               00100
#   MISSING_PARAMETER         00101
#   UNSUPPORTED_PARAMETER     00102
#   NOT_IMPLEMENTED           00103
#   VERSION_NOT_SUPPORTED     00104
#
# CONNECTION_ERROR            00200
#   INVALID_TRANSITION        00201
#   CONNECTION_EXISTS         00202
#   CONNECTION_NONEXISTENT    00203
#   CONNECTION_GONE           00204
#   CONNECTION_CREATE_ERROR   00205
#
# SECURITY_ERROR              00300
#   AUTHENTICATION_FAILURE    00301
#   UNAUTHORIZED              00302
#
# TOPOLOGY_ERROR              00400
#   UNKNOWN_STP               00401
#   STP_RESOLUTION_ERROR      00402
#   NO_PATH_FOUND             00403
#   VLANID_INTERCANGE_NOT_SUPPORTED 00404
#
# INTERNAL_ERROR              00500
#   INTERNAL_NRM_ERROR        00501
#
# RESOURCE_UNAVAILABLE        00600
#   STP_UNAVALABLE            00601
#   BANDWIDTH_UNAVAILABLE     00602

## Errors which are/should only be used internally

class CallbackTimeoutError(Exception):
    pass

class InvalidRequestError(Exception):
    pass

class ResourceNotAvailableError(Exception):
    pass

class TopologyError(Exception):
    pass

class StateTransitionError(Exception):
    pass

## NSI Errors

class NSIError(Exception):
    """
    This class is only used to indicate that an exception class is a "proper"
    NSI error, and have an errorId that can be used to produce a message with
    an errorId in it.

    It should not be instantiated directly.
    """
    errorId = None

class PayloadError(NSIError):

    errorId = '00100'

class ConnectionError(NSIError):

    errorId = '00200'

class ConnectionExistsError(ConnectionError):

    errorId = '00202'

class ConnectionNonExistentError(ConnectionError):

    errorId = '00203'

class SecurityError(NSIError):

    errorId = '00300'

class TopologyError(NSIError):

    errorId = '00400'

class InternalServerError(NSIError):

    errorId = '00500'

class ResourceUnavailableError(NSIError):

    errorId = '00600'

NSI_ERROR_CODE_TABLE = {
    '00100' : PayloadError,
    '00200' : ConnectionError,
    '00202' : ConnectionExistsError,
    '00203' : ConnectionNonExistentError,
    '00300' : SecurityError,
    '00400' : TopologyError,
    '00500' : InternalServerError,
    '00600' : ResourceUnavailableError,
}

def lookup(error_code):

    assert type(error_code) is str and len(error_code) == 5, 'Invalid Error Code (type or length is wrong'

    ex = NSI_ERROR_CODE_TABLE.get(error_code)
    if ex is None:
        error_code = error_code[0:2] + '  '
        ex = NSI_ERROR_CODE_TABLE.get(error_code)

    if ex is None:
        raise ValueError('Could not find error type. Invalid error code: %s' % error_code)

    return ex

# These should really be replaced with proper errors

class ReserveError(Exception):
    pass

class ProvisionError(Exception):
    pass

class ReleaseError(Exception):
    pass

class TerminateError(Exception):
    pass

class QueryError(Exception):
    pass

<|endoftext|>


<commit_before># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

{
  # Default value for all libraries.
  'custom_configure_flags': '',
  'custom_c_compiler_flags': '',
  'custom_cxx_compiler_flags': '',
  'custom_linker_flags': '',
  'run_before_build': '',

  'variables': {
    'verbose_libraries_build%': 0,
  },
  'conditions': [
    ['asan==1', {
      'sanitizer_type': 'asan',
    }],
    ['msan==1', {
      'sanitizer_type': 'msan',
    }],
    ['verbose_libraries_build==1', {
      'verbose_libraries_build_flag': '--verbose',
    }, {
      'verbose_libraries_build_flag': '',
    }],
    ['use_goma==1', {
      'cc': '<(gomadir)/gomacc <!(cd <(DEPTH) && pwd -P)/<(make_clang_dir)/bin/clang',
      'cxx': '<(gomadir)/gomacc <!(cd <(DEPTH) && pwd -P)/<(make_clang_dir)/bin/clang++',
    }, {
      'cc': '<!(cd <(DEPTH) && pwd -P)/<(make_clang_dir)/bin/clang',
      'cxx': '<!(cd <(DEPTH) && pwd -P)/<(make_clang_dir)/bin/clang++',
    }],
  ],
  'targets': [
    {
      'target_name': 'instrumented_libraries',
      'type': 'none',
      'variables': {
        'prune_self_dependency': 1,
      },
      'dependencies': [
        '<(_sanitizer_type)-libcairo2',
        '<(_sanitizer_type)-libexpat1',
        '<(_sanitizer_type)-libffi6',
        '<(_sanitizer_type)-libgcrypt11',
        '<(_sanitizer_type)-libgpg-error0',
        '<(_sanitizer_type)-libnspr4',
        '<(_sanitizer_type)-libp11-kit0',
        '<(_sanitizer_type)-libpcre3',
        '<(_sanitizer_type)-libpng12-0',
        '<(_sanitizer_type)-libx11-6',
        '<(_sanitizer_type)-libxau6',
        '<(_sanitizer_type)-libxcb1',
        '<(_sanitizer_type)-libxcomposite1',
        '<(_sanitizer_type)-libxcursor1',
        '<(_sanitizer_type)-libxdamage1',
        '<(_sanitizer_type)-libxdmcp6',
        '<(_sanitizer_type)-libxext6',
        '<(_sanitizer_type)-libxfixes3',
        '<(_sanitizer_type)-libxi6',
        '<(_sanitizer_type)-libxinerama1',
        '<(_sanitizer_type)-libxrandr2',
        '<(_sanitizer_type)-libxrender1',
        '<(_sanitizer_type)-libxss1',
        '<(_sanitizer_type)-libxtst6',
        '<(_sanitizer_type)-zlib1g',
        '<(_sanitizer_type)-libglib2.0-0',
        '<(_sanitizer_type)-libdbus-1-3',
        '<(_sanitizer_type)-libdbus-glib-1-2',
        '<(_sanitizer_type)-nss',
        '<(_sanitizer_type)-libfontconfig1',
        '<(_sanitizer_type)-pulseaudio',
        '<(_sanitizer_type)-libasound2',
        '<(_sanitizer_type)-libcups2',
        '<(_sanitizer_type)-pango1.0',
        '<(_sanitizer_type)-libcap2',
        '<(_sanitizer_type)-libudev0',
      ],
      'conditions': [
        ['asan==1', {
          'dependencies': [
            '<(_sanitizer_type)-libpixman-1-0',
          ],
        }],
      ],
      'actions': [
        {
          'action_name': 'fix_rpaths',
          'inputs': [
            'fix_rpaths.sh',
          ],
          'outputs': [
            '<(PRODUCT_DIR)/instrumented_libraries/<(_sanitizer_type)/rpaths.fixed.txt',
          ],
          'action': [
            '<(DEPTH)/third_party/instrumented_libraries/fix_rpaths.sh',
            '<(PRODUCT_DIR)/instrumented_libraries/<(_sanitizer_type)'
          ],
        },
      ],
    },
    {
      'library_name': 'freetype',
      'dependencies=': [],
      'custom_configure_flags': '',
      'run_before_build': 'freetype.sh',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libcairo2',
      'dependencies=': [],
      'custom_configure_flags': '--disable-gtk-doc',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libdbus-1-3',
      'dependencies=': [
        '<(_sanitizer_type)-libglib2.0-0',
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libdbus-glib-1-2',
      'dependencies=': [
        '<(_sanitizer_type)-libglib2.0-0',
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libexpat1',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libffi6',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libfontconfig1',
      'dependencies=': [
        '<(_sanitizer_type)-freetype',
      ],
      'custom_configure_flags': '--disable-docs',
      'run_before_build': 'libfontconfig.sh',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libgcrypt11',
      'dependencies=': [],
      'custom_linker_flags': '-Wl,-z,muldefs',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libglib2.0-0',
      'dependencies=': [],
      'custom_configure_flags': [
        '--disable-gtk-doc',
        '--disable-gtk-doc-html',
        '--disable-gtk-doc-pdf',
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libgpg-error0',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libnspr4',
      'dependencies=': [],
      'custom_configure_flags': '--enable-64bit',
      'run_before_build': 'libnspr4.sh',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libp11-kit0',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libpcre3',
      'dependencies=': [],
      'custom_configure_flags': [
        '--enable-utf8',
        '--enable-unicode-properties',
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libpixman-1-0',
      'dependencies=': [
        '<(_sanitizer_type)-libglib2.0-0',
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libpng12-0',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libx11-6',
      'dependencies=': [],
      'custom_configure_flags': '--disable-specs',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxau6',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxcb1',
      'dependencies=': [],
      'custom_configure_flags': '--disable-build-docs',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxcomposite1',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxcursor1',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxdamage1',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxdmcp6',
      'dependencies=': [],
      'custom_configure_flags': '--disable-docs',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxext6',
      'dependencies=': [],
      'custom_configure_flags': '--disable-specs',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxfixes3',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxi6',
      'dependencies=': [],
      'custom_configure_flags': [
        '--disable-specs',
        '--disable-docs',
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxinerama1',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxrandr2',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxrender1',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxss1',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxtst6',
      'dependencies=': [],
      'custom_configure_flags': '--disable-specs',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'zlib1g',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'nss',
      'dependencies=': [
        '<(_sanitizer_type)-libnspr4',
      ],
      'run_before_build': 'nss.sh',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'pulseaudio',
      'dependencies=': [],
      'run_before_build': 'pulseaudio.sh',
      'custom_configure_flags': '--with-udev-rules-dir=<(INTERMEDIATE_DIR)/udev/rules.d',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libasound2',
      'dependencies=': [],
      'run_before_build': 'libasound2.sh',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libcups2',
      'dependencies=': [],
      'run_before_build': 'libcups2.sh',
      'custom_configure_flags': [
        # Do not touch system-wide directories.
        '--with-rcdir=no',
        '--with-xinetd=no',
        '--with-dbusdir=no',
        '--with-menudir=no',
        '--with-icondir=no',
        '--with-docdir=no'
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'pango1.0',
      'dependencies=': [
        '<(_sanitizer_type)-libglib2.0-0',
      ],
      'custom_configure_flags': [
        # Avoid https://bugs.gentoo.org/show_bug.cgi?id=425620
        '--enable-introspection=no',
        # More flags are set in download_build_install.py.
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libcap2',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libudev0',
      'dependencies=': [],
      'custom_configure_flags': [
          # Without this flag there's a linking step that doesn't honor LDFLAGS
          # and fails.
          # TODO(earthdok): find a better fix.
          '--disable-gudev'
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
  ],
}
<commit_msg>Add libtasn to instrumented libraries.<commit_after># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

{
  # Default value for all libraries.
  'custom_configure_flags': '',
  'custom_c_compiler_flags': '',
  'custom_cxx_compiler_flags': '',
  'custom_linker_flags': '',
  'run_before_build': '',

  'variables': {
    'verbose_libraries_build%': 0,
  },
  'conditions': [
    ['asan==1', {
      'sanitizer_type': 'asan',
    }],
    ['msan==1', {
      'sanitizer_type': 'msan',
    }],
    ['verbose_libraries_build==1', {
      'verbose_libraries_build_flag': '--verbose',
    }, {
      'verbose_libraries_build_flag': '',
    }],
    ['use_goma==1', {
      'cc': '<(gomadir)/gomacc <!(cd <(DEPTH) && pwd -P)/<(make_clang_dir)/bin/clang',
      'cxx': '<(gomadir)/gomacc <!(cd <(DEPTH) && pwd -P)/<(make_clang_dir)/bin/clang++',
    }, {
      'cc': '<!(cd <(DEPTH) && pwd -P)/<(make_clang_dir)/bin/clang',
      'cxx': '<!(cd <(DEPTH) && pwd -P)/<(make_clang_dir)/bin/clang++',
    }],
  ],
  'targets': [
    {
      'target_name': 'instrumented_libraries',
      'type': 'none',
      'variables': {
        'prune_self_dependency': 1,
      },
      'dependencies': [
        '<(_sanitizer_type)-libcairo2',
        '<(_sanitizer_type)-libexpat1',
        '<(_sanitizer_type)-libffi6',
        '<(_sanitizer_type)-libgcrypt11',
        '<(_sanitizer_type)-libgpg-error0',
        '<(_sanitizer_type)-libnspr4',
        '<(_sanitizer_type)-libp11-kit0',
        '<(_sanitizer_type)-libpcre3',
        '<(_sanitizer_type)-libpng12-0',
        '<(_sanitizer_type)-libx11-6',
        '<(_sanitizer_type)-libxau6',
        '<(_sanitizer_type)-libxcb1',
        '<(_sanitizer_type)-libxcomposite1',
        '<(_sanitizer_type)-libxcursor1',
        '<(_sanitizer_type)-libxdamage1',
        '<(_sanitizer_type)-libxdmcp6',
        '<(_sanitizer_type)-libxext6',
        '<(_sanitizer_type)-libxfixes3',
        '<(_sanitizer_type)-libxi6',
        '<(_sanitizer_type)-libxinerama1',
        '<(_sanitizer_type)-libxrandr2',
        '<(_sanitizer_type)-libxrender1',
        '<(_sanitizer_type)-libxss1',
        '<(_sanitizer_type)-libxtst6',
        '<(_sanitizer_type)-zlib1g',
        '<(_sanitizer_type)-libglib2.0-0',
        '<(_sanitizer_type)-libdbus-1-3',
        '<(_sanitizer_type)-libdbus-glib-1-2',
        '<(_sanitizer_type)-nss',
        '<(_sanitizer_type)-libfontconfig1',
        '<(_sanitizer_type)-pulseaudio',
        '<(_sanitizer_type)-libasound2',
        '<(_sanitizer_type)-libcups2',
        '<(_sanitizer_type)-pango1.0',
        '<(_sanitizer_type)-libcap2',
        '<(_sanitizer_type)-libudev0',
        '<(_sanitizer_type)-libtasn1-3',
      ],
      'conditions': [
        ['asan==1', {
          'dependencies': [
            '<(_sanitizer_type)-libpixman-1-0',
          ],
        }],
      ],
      'actions': [
        {
          'action_name': 'fix_rpaths',
          'inputs': [
            'fix_rpaths.sh',
          ],
          'outputs': [
            '<(PRODUCT_DIR)/instrumented_libraries/<(_sanitizer_type)/rpaths.fixed.txt',
          ],
          'action': [
            '<(DEPTH)/third_party/instrumented_libraries/fix_rpaths.sh',
            '<(PRODUCT_DIR)/instrumented_libraries/<(_sanitizer_type)'
          ],
        },
      ],
    },
    {
      'library_name': 'freetype',
      'dependencies=': [],
      'custom_configure_flags': '',
      'run_before_build': 'freetype.sh',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libcairo2',
      'dependencies=': [],
      'custom_configure_flags': '--disable-gtk-doc',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libdbus-1-3',
      'dependencies=': [
        '<(_sanitizer_type)-libglib2.0-0',
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libdbus-glib-1-2',
      'dependencies=': [
        '<(_sanitizer_type)-libglib2.0-0',
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libexpat1',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libffi6',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libfontconfig1',
      'dependencies=': [
        '<(_sanitizer_type)-freetype',
      ],
      'custom_configure_flags': '--disable-docs',
      'run_before_build': 'libfontconfig.sh',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libgcrypt11',
      'dependencies=': [],
      'custom_linker_flags': '-Wl,-z,muldefs',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libglib2.0-0',
      'dependencies=': [],
      'custom_configure_flags': [
        '--disable-gtk-doc',
        '--disable-gtk-doc-html',
        '--disable-gtk-doc-pdf',
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libgpg-error0',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libnspr4',
      'dependencies=': [],
      'custom_configure_flags': '--enable-64bit',
      'run_before_build': 'libnspr4.sh',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libp11-kit0',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libpcre3',
      'dependencies=': [],
      'custom_configure_flags': [
        '--enable-utf8',
        '--enable-unicode-properties',
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libpixman-1-0',
      'dependencies=': [
        '<(_sanitizer_type)-libglib2.0-0',
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libpng12-0',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libx11-6',
      'dependencies=': [],
      'custom_configure_flags': '--disable-specs',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxau6',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxcb1',
      'dependencies=': [],
      'custom_configure_flags': '--disable-build-docs',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxcomposite1',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxcursor1',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxdamage1',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxdmcp6',
      'dependencies=': [],
      'custom_configure_flags': '--disable-docs',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxext6',
      'dependencies=': [],
      'custom_configure_flags': '--disable-specs',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxfixes3',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxi6',
      'dependencies=': [],
      'custom_configure_flags': [
        '--disable-specs',
        '--disable-docs',
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxinerama1',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxrandr2',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxrender1',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxss1',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libxtst6',
      'dependencies=': [],
      'custom_configure_flags': '--disable-specs',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'zlib1g',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'nss',
      'dependencies=': [
        '<(_sanitizer_type)-libnspr4',
      ],
      'run_before_build': 'nss.sh',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'pulseaudio',
      'dependencies=': [],
      'run_before_build': 'pulseaudio.sh',
      'custom_configure_flags': '--with-udev-rules-dir=<(INTERMEDIATE_DIR)/udev/rules.d',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libasound2',
      'dependencies=': [],
      'run_before_build': 'libasound2.sh',
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libcups2',
      'dependencies=': [],
      'run_before_build': 'libcups2.sh',
      'custom_configure_flags': [
        # Do not touch system-wide directories.
        '--with-rcdir=no',
        '--with-xinetd=no',
        '--with-dbusdir=no',
        '--with-menudir=no',
        '--with-icondir=no',
        '--with-docdir=no'
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'pango1.0',
      'dependencies=': [
        '<(_sanitizer_type)-libglib2.0-0',
      ],
      'custom_configure_flags': [
        # Avoid https://bugs.gentoo.org/show_bug.cgi?id=425620
        '--enable-introspection=no',
        # More flags are set in download_build_install.py.
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libcap2',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libudev0',
      'dependencies=': [],
      'custom_configure_flags': [
          # Without this flag there's a linking step that doesn't honor LDFLAGS
          # and fails.
          # TODO(earthdok): find a better fix.
          '--disable-gudev'
      ],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
    {
      'library_name': 'libtasn1-3',
      'dependencies=': [],
      'includes': ['standard_instrumented_library_target.gypi'],
    },
  ],
}
<|endoftext|>


<commit_before>import decimal

import pytest
from sympy import im

slow = pytest.mark.skipif(
    not pytest.config.getoption("--runslow"),
    reason="need --runslow option to run"
)

TOTAL_DEGREES = 30

from .crv_coeffs import coeffs as correct_coeffs
from .partfrac_coeffs import part_frac_coeffs
from ..cram import get_CRAM_from_cache, CRAM_coeffs
from ..partialfrac import thetas_alphas

# @slow
@pytest.mark.parametrize('degree', range(1, TOTAL_DEGREES+1))
def test_coefficients(degree):
    generated_coeffs = {}
    expr = get_CRAM_from_cache(degree, 200)
    generated_coeffs[degree] = CRAM_coeffs(expr, 20,
        decimal_rounding=True)
    # pytest won't show the full expr from the assert, so we print it too
    print(expr)
    assert generated_coeffs[degree] == correct_coeffs[degree], expr

# @pytest.mark.xfail
@pytest.mark.parametrize('real_imag', ['real', 'imag'])
@pytest.mark.parametrize('idx', range(16//2))
@pytest.mark.parametrize('typ', ['thetas', 'alphas', 'alpha0'])
@pytest.mark.parametrize('degree', [14, 16])
def test_partial_fraction_coefficients(degree, typ, idx, real_imag):
    if idx >= degree//2:
        # How do I do this in the decorators?
        return
    expr = get_CRAM_from_cache(degree, 200)
    thetas, alphas, alpha0 = thetas_alphas(expr, 200)
    format_str = '{:+.19e}'
    paper_coeffs = part_frac_coeffs[degree]
    # Thetas and alphas in the paper are negative what we have, and are only
    # counted once per conjugate.
    if typ == 'thetas':
        vals = [-i for i in thetas if im(-i) >= 0]
    elif typ == 'alphas':
        vals = [-j for i, j in zip(thetas, alphas) if im(-i) >= 0]
    elif typ == 'alpha0':
        if idx != 0:
            return
        vals = [alpha0]
    val = sorted(vals, key=im)[idx]
    real_val_paper, imag_val_paper = sorted(zip(paper_coeffs[typ]['real'],
        paper_coeffs[typ]['imaginary']), key=lambda i: float(i[1]))[idx]

    real_val, imag_val = val.as_real_imag()
    if real_imag == 'real':
        assert format_str.format(decimal.Decimal(repr(real_val))) == real_val_paper
    else:
        assert format_str.format(decimal.Decimal(repr(imag_val))) == imag_val_paper
<commit_msg>Reactivate xfail for the Pusa coefficient tests<commit_after>import decimal

import pytest
from sympy import im

slow = pytest.mark.skipif(
    not pytest.config.getoption("--runslow"),
    reason="need --runslow option to run"
)

TOTAL_DEGREES = 30

from .crv_coeffs import coeffs as correct_coeffs
from .partfrac_coeffs import part_frac_coeffs
from ..cram import get_CRAM_from_cache, CRAM_coeffs
from ..partialfrac import thetas_alphas

# @slow
@pytest.mark.parametrize('degree', range(1, TOTAL_DEGREES+1))
def test_coefficients(degree):
    generated_coeffs = {}
    expr = get_CRAM_from_cache(degree, 200)
    generated_coeffs[degree] = CRAM_coeffs(expr, 20,
        decimal_rounding=True)
    # pytest won't show the full expr from the assert, so we print it too
    print(expr)
    assert generated_coeffs[degree] == correct_coeffs[degree], expr

@pytest.mark.xfail
@pytest.mark.parametrize('real_imag', ['real', 'imag'])
@pytest.mark.parametrize('idx', range(16//2))
@pytest.mark.parametrize('typ', ['thetas', 'alphas', 'alpha0'])
@pytest.mark.parametrize('degree', [14, 16])
def test_partial_fraction_coefficients(degree, typ, idx, real_imag):
    if idx >= degree//2:
        # How do I do this in the decorators?
        return
    expr = get_CRAM_from_cache(degree, 200)
    thetas, alphas, alpha0 = thetas_alphas(expr, 200)
    format_str = '{:+.19e}'
    paper_coeffs = part_frac_coeffs[degree]
    # Thetas and alphas in the paper are negative what we have, and are only
    # counted once per conjugate.
    if typ == 'thetas':
        vals = [-i for i in thetas if im(-i) >= 0]
    elif typ == 'alphas':
        vals = [-j for i, j in zip(thetas, alphas) if im(-i) >= 0]
    elif typ == 'alpha0':
        if idx != 0:
            return
        vals = [alpha0]
    val = sorted(vals, key=im)[idx]
    real_val_paper, imag_val_paper = sorted(zip(paper_coeffs[typ]['real'],
        paper_coeffs[typ]['imaginary']), key=lambda i: float(i[1]))[idx]

    real_val, imag_val = val.as_real_imag()
    if real_imag == 'real':
        assert format_str.format(decimal.Decimal(repr(real_val))) == real_val_paper
    else:
        assert format_str.format(decimal.Decimal(repr(imag_val))) == imag_val_paper
<|endoftext|>


<commit_before>#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import torch

from bigdl.nano.pytorch.lightning import LightningModule
from bigdl.nano.utils.log4Error import invalidInputError
from ..model import AcceleratedModel

class AcceleratedLightningModule(AcceleratedModel, LightningModule):
    def __init__(self, model):
        super().__init__(model)
        self.model = model
        self.train(False)

    def forward(self, *inputs):
        inputs = self.on_forward_start(inputs)
        outputs = self.forward_step(*inputs)
        return self.on_forward_end(outputs)

    def on_train_start(self) -> None:
        invalidInputError(False, errMsg="This model is not trainable!")

    def train(self, mode=False):
        if mode:
            invalidInputError(False, "This model is not trainable!")
        super().train(mode)

    def forward_step(self, *inputs):
        return self.model(*inputs)

    @staticmethod
    def tensors_to_numpy(tensors):
        np_data = tuple(map(lambda x: x.cpu().numpy(), tensors))
        return np_data

    @staticmethod
    def numpy_to_tensors(np_arrays):
        tensors = tuple(map(lambda x: torch.from_numpy(x), np_arrays))
        if len(tensors) == 1:
            tensors = tensors[0]
        return tensors
<commit_msg>fix python datatype error in Pytorch Dataloader (#5237)<commit_after>#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import torch

from bigdl.nano.pytorch.lightning import LightningModule
from bigdl.nano.utils.log4Error import invalidInputError
from ..model import AcceleratedModel

class AcceleratedLightningModule(AcceleratedModel, LightningModule):
    def __init__(self, model):
        super().__init__(model)
        self.model = model
        self.train(False)

    def forward(self, *inputs):
        inputs = self.on_forward_start(inputs)
        outputs = self.forward_step(*inputs)
        return self.on_forward_end(outputs)

    def on_train_start(self) -> None:
        invalidInputError(False, errMsg="This model is not trainable!")

    def train(self, mode=False):
        if mode:
            invalidInputError(False, "This model is not trainable!")
        super().train(mode)

    def forward_step(self, *inputs):
        return self.model(*inputs)

    @staticmethod
    def tensors_to_numpy(tensors):
        np_data = tuple(map(
            lambda x: x.cpu().numpy() if isinstance(x, torch.Tensor) else x,
            tensors))
        return np_data

    @staticmethod
    def numpy_to_tensors(np_arrays):
        tensors = tuple(map(lambda x: torch.from_numpy(x), np_arrays))
        if len(tensors) == 1:
            tensors = tensors[0]
        return tensors
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Common tools:
#	pip/pip3 : install packages
#	pep8/pylint/pychecker : verify syntax
#	unittest package for xUnit.
""" This is a simple script to recompile ycm everytime it gets updated. """

# Imports
import os
import shutil
import subprocess
import urllib.request
import tarfile

# Data
B_DIR = 'build'
CLANG_SITE = 'http://llvm.org/releases/3.4/'
CLANG_FILE = 'clang+llvm-3.4-x86_64-linux-gnu-ubuntu-13.10.tar.xz'
CLANG_URL = "{}{}".format(CLANG_SITE, CLANG_FILE)
CLANG_DIR = 'clang'

# Classes

# Functions

def check_clang():
    """ Check if we have clang, else retrieve it. """
    file_no_ext = CLANG_FILE[0:CLANG_FILE.rindex(".tar")]
    list_dir = os.listdir()

    if CLANG_DIR in list_dir:
        return

    if CLANG_FILE not in list_dir:
        print("Please wait, downloading clang.")
        urllib.request.urlretrieve(CLANG_URL, CLANG_FILE)
        print("Finished download.")

    if file_no_ext in list_dir:
        shutil.rmtree(file_no_ext)

    tar_file = tarfile.open(CLANG_FILE, 'r')
    tar_file.extractall()
    os.rename(file_no_ext, CLANG_DIR)
    os.remove(CLANG_FILE)

# Main
if __name__ == '__main__':
    if B_DIR in os.listdir():
        shutil.rmtree(B_DIR)

    check_clang()

    os.mkdir(B_DIR)
    os.chdir(B_DIR)

    subprocess.call(['cmake', '-GUnix Makefiles',
                     '-DPATH_TO_LLVM_ROOT=../{}'.format(CLANG_DIR), '.',
                     '~/.vim/bundle/YouCompleteMe/cpp'])
    subprocess.call(['make', '-j8', 'ycm_support_libs'])

    os.chdir('..')
    shutil.rmtree(B_DIR)
<commit_msg>Forgot to remove temp clang dir.<commit_after>#!/usr/bin/env python3
# Common tools:
#	pip/pip3 : install packages
#	pep8/pylint/pychecker : verify syntax
#	unittest package for xUnit.
""" This is a simple script to recompile ycm everytime it gets updated. """

# Imports
import os
import shutil
import subprocess
import urllib.request
import tarfile

# Data
B_DIR = 'build'
CLANG_SITE = 'http://llvm.org/releases/3.4/'
CLANG_FILE = 'clang+llvm-3.4-x86_64-linux-gnu-ubuntu-13.10.tar.xz'
CLANG_URL = "{}{}".format(CLANG_SITE, CLANG_FILE)
CLANG_DIR = 'clang'

# Classes

# Functions

def check_clang():
    """ Check if we have clang, else retrieve it. """
    file_no_ext = CLANG_FILE[0:CLANG_FILE.rindex(".tar")]
    list_dir = os.listdir()

    if CLANG_DIR in list_dir:
        return

    if CLANG_FILE not in list_dir:
        print("Please wait, downloading clang.")
        urllib.request.urlretrieve(CLANG_URL, CLANG_FILE)
        print("Finished download.")

    if file_no_ext in list_dir:
        shutil.rmtree(file_no_ext)

    tar_file = tarfile.open(CLANG_FILE, 'r')
    tar_file.extractall()
    os.rename(file_no_ext, CLANG_DIR)
    os.remove(CLANG_FILE)

# Main
if __name__ == '__main__':
    if B_DIR in os.listdir():
        shutil.rmtree(B_DIR)

    check_clang()

    os.mkdir(B_DIR)
    os.chdir(B_DIR)

    subprocess.call(['cmake', '-GUnix Makefiles',
                     '-DPATH_TO_LLVM_ROOT=../{}'.format(CLANG_DIR), '.',
                     '~/.vim/bundle/YouCompleteMe/cpp'])
    subprocess.call(['make', '-j8', 'ycm_support_libs'])

    os.chdir('..')
    shutil.rmtree(B_DIR)
    shutil.rmtree(CLANG_DIR)
<|endoftext|>


<commit_before>from sqlalchemy import Integer
from sqlalchemy.sql.expression import cast
from sqlalchemy.orm import aliased, joinedload, contains_eager

from clld.web.datatables import Values
from clld.web.datatables.base import Col, LinkCol, LinkToMapCol, DataTable, IntegerIdCol
from clld.web.datatables.contribution import Contributions, CitationCol
from clld.web.datatables import contributor
from clld.web.datatables.parameter import Parameters
from clld.web.util.helpers import link
from clld.web.util.htmllib import HTML
from clld.db.meta import DBSession
from clld.db.util import collkey
import clld.db.models.common
from clld.db.models.common import ValueSet, Value
from clld_glottologfamily_plugin.datatables import MacroareaCol, FamilyCol

from ids.models import Chapter, Entry, ROLES, Dictionary, IdsLanguage
from ids.util import concepticon_link

class IDSCodeCol(Col):
    """special handling for IDS code
    """
    __kw__ = {'sTitle': 'IDS code'}

    def format(self, item):
        return self.get_obj(item).id.replace('-', '.')

    def order(self):
        return Entry.chapter_pk, cast(Entry.sub_code, Integer)

    def search(self, qs):
        return Entry.id.contains(qs.replace('.', '-'))

class ChapterCol(Col):
    def __init__(self, *args, **kw):
        kw['choices'] = [
            (sf.pk, sf.name) for sf in DBSession.query(Chapter).order_by(Chapter.pk)]
        super(ChapterCol, self).__init__(*args, **kw)

    def format(self, item):
        obj = self.get_obj(item)
        return obj.chapter.name

    def order(self):
        return Entry.chapter_pk

    def search(self, qs):
        return Entry.chapter_pk == int(qs)

class FormCol(LinkCol):
    def order(self):
        return collkey(Value.name)

class Counterparts(Values):
    """Lists of counterparts
    """
    def base_query(self, query):
        query = Values.base_query(self, query)
        if not self.parameter and not self.language and not self.contribution:
            query = DBSession.query(Value).join(ValueSet)\
                .join(ValueSet.language)\
                .join(ValueSet.parameter)\
                .options(
                    joinedload(Value.valueset).joinedload(ValueSet.language),
                    joinedload(Value.valueset, ValueSet.parameter))
        return query

    def col_defs(self):
        lang = lambda i: i.valueset.language
        param = lambda i: i.valueset.parameter
        if self.parameter:
            res = [
                LinkToMapCol(self, 'm', get_object=lang),
                LinkCol(
                    self, 'language',
                    model_col=clld.db.models.common.Language.name,
                    get_object=lang),
            ]
        elif self.language or self.contribution:
            res = [
                IDSCodeCol(
                    self, 'ids_code',
                    model_col=clld.db.models.common.Parameter.id,
                    get_object=param),
                LinkCol(
                    self, 'meaning',
                    model_col=clld.db.models.common.Parameter.name,
                    get_object=param),
                ChapterCol(self, 'chapter', get_object=param)]
        else:
            res = [
                IDSCodeCol(
                    self, 'ids_code',
                    model_col=clld.db.models.common.Parameter.id,
                    get_object=param),
                LinkCol(
                    self, 'meaning',
                    model_col=clld.db.models.common.Parameter.name,
                    get_object=param),
                ChapterCol(self, 'chapter', get_object=param),
                LinkCol(
                    self, 'language',
                    model_col=clld.db.models.common.Language.name,
                    get_object=lang),
            ]
        res.extend([
            FormCol(
                self, 'counterparts',
                model_col=clld.db.models.common.Value.name,
                sClass="charissil"),
            Col(self, 'description'),
        ])
        return res

class RoleCol(Col):
    __kw__ = {'choices': list(ROLES.items()), 'sClass': 'left'}

    def format(self, item):
        return ROLES[item.ord]

class Compilers(contributor.Contributors):
    def base_query(self, query):
        return DBSession.query(clld.db.models.common.ContributionContributor).join(clld.db.models.common.Contribution).join(clld.db.models.common.Contributor)

    def col_defs(self):
        return [
            LinkCol(self, 'name', get_object=lambda i: i.contributor, model_col=clld.db.models.common.Contributor.name),
            RoleCol(self, 'role', model_col=clld.db.models.common.ContributionContributor.ord),
            LinkCol(self, 'dictionary', get_object=lambda i: i.contribution, model_col=clld.db.models.common.Contribution.name),
        ]

class ContributorCol(Col):
    __kw__ = {'bSortable': False, 'bSearchable': False}

    def __init__(self, dt, name, roleid, **kw):
        kw['sTitle'] = ROLES[roleid]
        Col.__init__(self, dt, name, **kw)
        self.roleid = roleid

    def format(self, item):
        return HTML.ul(
            *[HTML.li(link(self.dt.req, ca.contributor))
              for ca in item.contributor_assocs if ca.ord == self.roleid])

class IdsFamilyCol(FamilyCol):
    def format(self, item):
        item_ = self.get_obj(item)
        if not item_.glottocode:
            return ''
        return FamilyCol.format(self, item)

class Dictionaries(Contributions):
    def __init__(self, *args, **kw):
        Contributions.__init__(self, *args, **kw)
        self.roles = {}
        for roleid in ROLES:
            self.roles[roleid] = aliased(clld.db.models.common.ContributionContributor, name='role%s' % roleid)

    def base_query(self, query):
        q = DBSession.query(clld.db.models.common.Contribution)\
            .join(Dictionary.language)\
            .outerjoin(IdsLanguage.family)\
            .options(
                contains_eager(Dictionary.language),
                joinedload(Dictionary.language, IdsLanguage.family))
        for roleid, alias in self.roles.items():
            q.join(alias, alias.ord == roleid)
        return q.distinct()

    def col_defs(self):
        res = [
            LinkCol(self, 'name'),
            LinkToMapCol(self, 'm', get_object=lambda i: i.language),
            MacroareaCol(self, 'macroarea', get_object=lambda i: i.language, language_cls=IdsLanguage),
            IdsFamilyCol(self, 'family', get_object=lambda i: i.language, language_cls=IdsLanguage),
        ]
        for roleid in ROLES.keys():
            res.append(ContributorCol(self, 'role%s' % roleid, roleid))
        res.append(CitationCol(self, 'cite'))
        return res

class ConcepticonLinkCol(Col):
    __kw__ = {'bSearchable': False, 'bSortable': False}

    def format(self, item):
        return concepticon_link(self.dt.req, item)

class Entries(Parameters):
    __constraints__ = [Chapter]

    def base_query(self, query):
        query = query.join(Chapter)
        if self.chapter:
            query = query.filter(Chapter.pk == self.chapter.pk)
        return query

    def col_defs(self):
        return list(filter(lambda col: not self.chapter or col.name != 'sf', [
            IDSCodeCol(self, 'ids_code'),
            LinkCol(
                self, 'name', sTitle='Meaning',
                sDescription="This column shows the labels of the Loanword Typology "
                "meanings. By clicking on a meaning label, you get more information "
                "about the meaning, as well as a list of all words that are counterparts "
                "of that meaning."),
            ChapterCol(self, 'sf', sTitle='Chapter'),
            Col(self, 'representation', model_col=Entry.representation),
            ConcepticonLinkCol(self, 'concepticon'),
        ]))

class Chapters(DataTable):
    def col_defs(self):
        return [
            IntegerIdCol(self, 'id'),
            LinkCol(self, 'name'),
            Col(self, 'count_entries', sTitle='# Entries')
        ]

def includeme(config):
    config.register_datatable('chapters', Chapters)
    config.register_datatable('values', Counterparts)
    config.register_datatable('contributors', Compilers)
    config.register_datatable('contributions', Dictionaries)
    config.register_datatable('parameters', Entries)
<commit_msg>added Family col to two views; applied new ROLES structure; ignored ` while ordering forms<commit_after>from sqlalchemy import Integer
from sqlalchemy.sql.expression import cast, func
from sqlalchemy.orm import aliased, joinedload, contains_eager

from clld.web.datatables import Values, Sources
from clld.web.datatables.base import Col, LinkCol, LinkToMapCol, DataTable, IntegerIdCol
from clld.web.datatables.contribution import Contributions, CitationCol
from clld.web.datatables import contributor
from clld.web.datatables.parameter import Parameters
from clld.web.util.helpers import link
from clld.web.util.htmllib import HTML
from clld.db.meta import DBSession
from clld.db.util import collkey
import clld.db.models.common
from clld.db.models.common import ValueSet, Value
from clld_glottologfamily_plugin.datatables import MacroareaCol, FamilyCol
from clld_glottologfamily_plugin.models import Family

from ids.models import Chapter, Entry, ROLES, Dictionary, IdsLanguage
from ids.util import concepticon_link

class IDSCodeCol(Col):
    """special handling for IDS code
    """
    __kw__ = {'sTitle': 'IDS code'}

    def format(self, item):
        return self.get_obj(item).id.replace('-', '.')

    def order(self):
        return Entry.chapter_pk, cast(Entry.sub_code, Integer)

    def search(self, qs):
        return Entry.id.contains(qs.replace('.', '-'))

class ChapterCol(Col):
    def __init__(self, *args, **kw):
        kw['choices'] = [
            (sf.pk, sf.name) for sf in DBSession.query(Chapter).order_by(Chapter.pk)]
        super(ChapterCol, self).__init__(*args, **kw)

    def format(self, item):
        obj = self.get_obj(item)
        return obj.chapter.name

    def order(self):
        return Entry.chapter_pk

    def search(self, qs):
        return Entry.chapter_pk == int(qs)

class FormCol(LinkCol):
    def order(self):
        return collkey(func.replace(func.replace(Value.name, '`', ''), 'ː', ''))

class Counterparts(Values):
    """Lists of counterparts
    """
    def get_options(self):
        opts = super(Values, self).get_options()
        if self.parameter:
            opts["aaSorting"] = [[2, "asc"], [1, "asc"]]
        elif self.contribution:
            opts["aaSorting"] = [[0, "asc"]]
        else:
            opts["aaSorting"] = [[0, "asc"], [4, "asc"], [3, "asc"]]
        return opts

    def base_query(self, query):
        query = Values.base_query(self, query)
        if not self.language and not self.contribution and not self.parameter:
            return DBSession.query(Value).join(ValueSet)\
                .join(ValueSet.language)\
                .join(ValueSet.parameter)\
                .join(Family, isouter=True)\
                .options(joinedload(Value.valueset).joinedload(ValueSet.language),
                         joinedload(Value.valueset, ValueSet.parameter))
        if self.contribution:
            return query
        return query.join(Family, isouter=True)

    def col_defs(self):
        lang = lambda i: i.valueset.language
        param = lambda i: i.valueset.parameter
        if self.parameter:
            res = [
                LinkToMapCol(self, 'm', get_object=lang),
                LinkCol(
                    self, 'language',
                    model_col=clld.db.models.common.Language.name,
                    get_object=lang),
                IdsFamilyCol(
                    self, 'family',
                    language_cls=IdsLanguage,
                    get_object=lambda i: i.valueset.language),
            ]
        elif self.language or self.contribution:
            res = [
                IDSCodeCol(
                    self, 'ids_code',
                    model_col=clld.db.models.common.Parameter.id,
                    get_object=param),
                LinkCol(
                    self, 'meaning',
                    model_col=clld.db.models.common.Parameter.name,
                    get_object=param),
                ChapterCol(self, 'chapter', get_object=param)]
        else:
            res = [
                IDSCodeCol(
                    self, 'ids_code',
                    model_col=clld.db.models.common.Parameter.id,
                    get_object=param),
                LinkCol(
                    self, 'meaning',
                    model_col=clld.db.models.common.Parameter.name,
                    get_object=param),
                ChapterCol(self, 'chapter', get_object=param),
                LinkCol(
                    self, 'language',
                    model_col=clld.db.models.common.Language.name,
                    get_object=lang),
                IdsFamilyCol(
                    self, 'family',
                    language_cls=IdsLanguage,
                    get_object=lambda i: i.valueset.language),
            ]
        res.extend([
            FormCol(
                self, 'counterparts',
                model_col=clld.db.models.common.Value.name,
                sClass="charissil"),
            Col(self, 'description'),
        ])
        return res

class RoleCol(Col):
    __kw__ = {'choices': sorted([r[0] for r in ROLES.values()]), 'sClass': 'left'}

    def format(self, item):
        return ROLES[item.ord][0]

    def search(self, qs):
        ROLE_MAP = {v[0]: k for k, v in ROLES.items()}
        return self.model_col == ROLE_MAP[qs]

class Compilers(contributor.Contributors):
    def base_query(self, query):
        return DBSession.query(clld.db.models.common.ContributionContributor).join(clld.db.models.common.Contribution).join(clld.db.models.common.Contributor)

    def col_defs(self):
        return [
            LinkCol(self, 'name', get_object=lambda i: i.contributor, model_col=clld.db.models.common.Contributor.name),
            RoleCol(self, 'role', model_col=clld.db.models.common.ContributionContributor.ord),
            LinkCol(self, 'dictionary', get_object=lambda i: i.contribution, model_col=clld.db.models.common.Contribution.name),
        ]

class ContributorCol(Col):
    __kw__ = {'bSortable': False, 'bSearchable': False}

    def __init__(self, dt, name, roleid, **kw):
        kw['sTitle'] = ROLES[roleid][0]
        Col.__init__(self, dt, name, **kw)
        self.roleid = roleid

    def format(self, item):
        return HTML.ul(
            *[HTML.li(link(self.dt.req, ca.contributor))
              for ca in item.contributor_assocs if ca.ord == self.roleid])

class IdsFamilyCol(FamilyCol):
    def format(self, item):
        item_ = self.get_obj(item)
        if not item_.glottocode:
            return ''
        return FamilyCol.format(self, item)

class Dictionaries(Contributions):
    def __init__(self, *args, **kw):
        Contributions.__init__(self, *args, **kw)
        self.roles = {}
        for roleid in ROLES.keys():
            self.roles[roleid] = aliased(clld.db.models.common.ContributionContributor, name='role%s' % roleid)

    def base_query(self, query):
        q = DBSession.query(clld.db.models.common.Contribution)\
            .join(Dictionary.language)\
            .outerjoin(IdsLanguage.family)\
            .options(
                contains_eager(Dictionary.language),
                joinedload(Dictionary.language, IdsLanguage.family))
        for roleid, alias in self.roles.items():
            q.join(alias, alias.ord == roleid)
        return q.distinct()

    def col_defs(self):
        res = [
            LinkCol(self, 'name'),
            LinkToMapCol(self, 'm', get_object=lambda i: i.language),
            MacroareaCol(self, 'macroarea', get_object=lambda i: i.language, language_cls=IdsLanguage),
            IdsFamilyCol(self, 'family', get_object=lambda i: i.language, language_cls=IdsLanguage),
        ]
        for roleid in ROLES.keys():
            res.append(ContributorCol(self, 'role%s' % roleid, roleid))
        res.append(CitationCol(self, 'cite'))
        return res

class ConcepticonLinkCol(Col):
    __kw__ = {'bSearchable': False, 'bSortable': False}

    def format(self, item):
        return concepticon_link(self.dt.req, item)

class Entries(Parameters):
    __constraints__ = [Chapter]

    def base_query(self, query):
        query = query.join(Chapter)
        if self.chapter:
            query = query.filter(Chapter.pk == self.chapter.pk)
        return query

    def col_defs(self):
        return list(filter(lambda col: not self.chapter or col.name != 'sf', [
            IDSCodeCol(self, 'ids_code'),
            LinkCol(
                self, 'name', sTitle='Meaning',
                sDescription="This column shows the labels of the Loanword Typology "
                "meanings. By clicking on a meaning label, you get more information "
                "about the meaning, as well as a list of all words that are counterparts "
                "of that meaning."),
            ChapterCol(self, 'sf', sTitle='Chapter'),
            Col(self, 'representation', model_col=Entry.representation),
            ConcepticonLinkCol(self, 'concepticon'),
        ]))

class IDSSources(Sources):
    def get_options(self):
        opts = super(Sources, self).get_options()
        opts["aaSorting"] = [[1, "asc"]]
        return opts

class Chapters(DataTable):
    def col_defs(self):
        return [
            IntegerIdCol(self, 'id'),
            LinkCol(self, 'name'),
            Col(self, 'count_entries', sTitle='# Entries')
        ]

def includeme(config):
    config.register_datatable('chapters', Chapters)
    config.register_datatable('values', Counterparts)
    config.register_datatable('contributors', Compilers)
    config.register_datatable('contributions', Dictionaries)
    config.register_datatable('parameters', Entries)
    config.register_datatable('sources', IDSSources)
<|endoftext|>