<commit_before>"""Factor Analysis.
A latent linear variable model, similar to ProbabilisticPCA.

This implementation is based on David Barber's Book,
Bayesian Reasoning and Machine Learning,
http://www.cs.ucl.ac.uk/staff/d.barber/brml,
Algorithm 21.1
"""

# Author: Christian Osendorfer <osendorf@gmail.com>
#         Alexandre Gramfort <alexandre.gramfort@inria.fr>
# Licence: BSD3

from math import sqrt
import numpy as np
from scipy import linalg

from ..base import BaseEstimator, TransformerMixin
from ..utils import array2d, check_arrays

class FactorAnalysis(BaseEstimator, TransformerMixin):
    """Factor Analysis (FA)

    A simple linear generative model with Gaussian latent variables.

    The observations are assumed to be caused by a linear transformation of
    lower dimensional latent factors and added Gaussian noise.
    Without loss of generality the factors are distributed according to a
    Gaussian with zero mean and unit covariance. The noise is also zero mean
    and has an arbitrary diagonal covariance matrix.

    If we would restrict the model further, by assuming that the Gaussian
    noise is even isotropic (all diagonal entries are the same) we would obtain
    :class:`PPCA`.

    FactorAnalysis performs a maximum likelihood estimate of the so-called
    `loading` matrix, the transformation of the latent variables to the
    observed ones, using expectation-maximization (EM).

    Parameters
    ----------
    n_components : int | None
        Dimensionality of latent space, the number of components
        of ``X`` that are obtained after ``transform``.
        If None, n_components is set to the number of features.

    tol : float
        Stopping tolerance for EM algorithm.

    copy : bool
        Whether to make a copy of X. If ``False``, the input X gets overwritten
        during fitting.

    max_iter : int
        Maximum number of iterations.

    verbose : int | bool
        Print verbose output.

    noise_variance_init : None | array, shape=(n_features,)
        The initial guess of the noise variance for each feature.
        If None, it defaults to np.ones(n_features)

    Attributes
    ----------
    `components_` : array, [n_components, n_features]
        Components with maximum variance.

    `loglike_` : list, [n_iterations]
        The log likelihood at each iteration.

    `noise_variance_` : array, shape=(n_features,)
        The estimated noise variance for each feature.

    References
    ----------
    .. David Barber, Bayesian Reasoning and Machine Learning,
        Algorithm 21.1

    .. Christopher M. Bishop: Pattern Recognition and Machine Learning,
        Chapter 12.2.4

    See also
    --------
    PCA: Principal component analysis, a simliar non-probabilistic
        model model that can be computed in closed form.
    ProbabilisticPCA: probabilistic PCA.
    FastICA: Independent component analysis, a latent variable model with
        non-Gaussian latent variables.
    """
    def __init__(self, n_components=None, tol=1e-2, copy=True, max_iter=1000,
                 verbose=0, noise_variance_init=None):
        self.n_components = n_components
        self.copy = copy
        self.tol = tol
        self.max_iter = max_iter
        self.verbose = verbose
        self.noise_variance_init = noise_variance_init

    def fit(self, X, y=None):
        """Fit the FactorAnalysis model to X using EM

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        self
        """
        X = array2d(check_arrays(X, copy=self.copy, sparse_format='dense',
                    dtype=np.float)[0])

        n_samples, n_features = X.shape
        n_components = self.n_components
        if n_components is None:
            n_components = n_features
        self.mean_ = np.mean(X, axis=0)
        X -= self.mean_

        # some constant terms
        nsqrt = sqrt(n_samples)
        llconst = n_features * np.log(2 * np.pi) + n_components
        var = np.var(X, axis=0)

        if self.noise_variance_init is None:
            psi = np.ones(n_features, dtype=X.dtype)
        else:
            if len(self.noise_variance_init) != n_features:
                raise ValueError("noise_variance_init dimension does not "
                        "with number of featueres : %d != %d" %
                        (len(self.noise_variance_init), n_features))
            psi = np.array(self.noise_variance_init)

        loglike = []
        old_ll = -np.inf
        SMALL = 1e-12
        for i in xrange(self.max_iter):
            # SMALL helps numerics
            print psi
            sqrt_psi = np.sqrt(psi) + SMALL
            Xtilde = X / (sqrt_psi * nsqrt)
            _, s, V = linalg.svd(Xtilde, full_matrices=False)
            V = V[:n_components]
            s **= 2
            # Use 'maximum' here to avoid sqrt problems.
            W = np.sqrt(np.maximum(s[:n_components] - 1, 0))[:, np.newaxis] * V
            W *= sqrt_psi

            # loglikelihood
            ll = llconst + np.sum(np.log(s[:n_components]))
            ll += np.sum(s[n_components:]) + np.sum(np.log(psi))
            ll *= -n_samples / 2.
            loglike.append(ll)
            if (ll - old_ll) < self.tol:
                break
            old_ll = ll

            psi = np.maximum(var - np.sum(W ** 2, axis=0), SMALL)
        else:
            if self.verbose:
                print "Did not converge"

        self.components_ = W
        self.noise_variance_ = psi
        self.loglike_ = loglike
        return self

    def transform(self, X):
        """Apply dimensionality reduction to X using the model.

        Compute the expected mean of the latent variables.
        See Barber, 21.2.33 (or Bishop, 12.66).

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)
            The latent variables of X.
        """
        Ih = np.eye(len(self.components_))

        X_transformed = X - self.mean_

        Wpsi = self.components_ / self.noise_variance_
        cov_z = linalg.inv(Ih + np.dot(Wpsi, self.components_.T))
        tmp = np.dot(X_transformed, Wpsi.T)
        X_transformed = np.dot(tmp, cov_z)

        return X_transformed

    def get_covariance(self):
        """Compute data covariance with the FactorAnalysis model

        cov = components_.T * components_ + diag(noise_variance)

        Returns
        -------
        cov : array, shape=(n_features, n_features)
            The covariance
        """
        cov = np.dot(self.components_.T, self.components_) \
              + np.diag(self.noise_variance_)
        return cov
<commit_msg>Added score function.<commit_after>"""Factor Analysis.
A latent linear variable model, similar to ProbabilisticPCA.

This implementation is based on David Barber's Book,
Bayesian Reasoning and Machine Learning,
http://www.cs.ucl.ac.uk/staff/d.barber/brml,
Algorithm 21.1
"""

# Author: Christian Osendorfer <osendorf@gmail.com>
#         Alexandre Gramfort <alexandre.gramfort@inria.fr>
# Licence: BSD3

from math import sqrt
import numpy as np
from scipy import linalg

from ..base import BaseEstimator, TransformerMixin
from ..utils import array2d, check_arrays

class FactorAnalysis(BaseEstimator, TransformerMixin):
    """Factor Analysis (FA)

    A simple linear generative model with Gaussian latent variables.

    The observations are assumed to be caused by a linear transformation of
    lower dimensional latent factors and added Gaussian noise.
    Without loss of generality the factors are distributed according to a
    Gaussian with zero mean and unit covariance. The noise is also zero mean
    and has an arbitrary diagonal covariance matrix.

    If we would restrict the model further, by assuming that the Gaussian
    noise is even isotropic (all diagonal entries are the same) we would obtain
    :class:`PPCA`.

    FactorAnalysis performs a maximum likelihood estimate of the so-called
    `loading` matrix, the transformation of the latent variables to the
    observed ones, using expectation-maximization (EM).

    Parameters
    ----------
    n_components : int | None
        Dimensionality of latent space, the number of components
        of ``X`` that are obtained after ``transform``.
        If None, n_components is set to the number of features.

    tol : float
        Stopping tolerance for EM algorithm.

    copy : bool
        Whether to make a copy of X. If ``False``, the input X gets overwritten
        during fitting.

    max_iter : int
        Maximum number of iterations.

    verbose : int | bool
        Print verbose output.

    noise_variance_init : None | array, shape=(n_features,)
        The initial guess of the noise variance for each feature.
        If None, it defaults to np.ones(n_features)

    Attributes
    ----------
    `components_` : array, [n_components, n_features]
        Components with maximum variance.

    `loglike_` : list, [n_iterations]
        The log likelihood at each iteration.

    `noise_variance_` : array, shape=(n_features,)
        The estimated noise variance for each feature.

    References
    ----------
    .. David Barber, Bayesian Reasoning and Machine Learning,
        Algorithm 21.1

    .. Christopher M. Bishop: Pattern Recognition and Machine Learning,
        Chapter 12.2.4

    See also
    --------
    PCA: Principal component analysis, a simliar non-probabilistic
        model model that can be computed in closed form.
    ProbabilisticPCA: probabilistic PCA.
    FastICA: Independent component analysis, a latent variable model with
        non-Gaussian latent variables.
    """
    def __init__(self, n_components=None, tol=1e-2, copy=True, max_iter=1000,
                 verbose=0, noise_variance_init=None):
        self.n_components = n_components
        self.copy = copy
        self.tol = tol
        self.max_iter = max_iter
        self.verbose = verbose
        self.noise_variance_init = noise_variance_init

    def fit(self, X, y=None):
        """Fit the FactorAnalysis model to X using EM

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        self
        """
        X = array2d(check_arrays(X, copy=self.copy, sparse_format='dense',
                    dtype=np.float)[0])

        n_samples, n_features = X.shape
        n_components = self.n_components
        if n_components is None:
            n_components = n_features
        self.mean_ = np.mean(X, axis=0)
        X -= self.mean_

        # some constant terms
        nsqrt = sqrt(n_samples)
        llconst = n_features * np.log(2 * np.pi) + n_components
        var = np.var(X, axis=0)

        if self.noise_variance_init is None:
            psi = np.ones(n_features, dtype=X.dtype)
        else:
            if len(self.noise_variance_init) != n_features:
                raise ValueError("noise_variance_init dimension does not "
                        "with number of featueres : %d != %d" %
                        (len(self.noise_variance_init), n_features))
            psi = np.array(self.noise_variance_init)

        loglike = []
        old_ll = -np.inf
        SMALL = 1e-12
        for i in xrange(self.max_iter):
            # SMALL helps numerics
            sqrt_psi = np.sqrt(psi) + SMALL
            Xtilde = X / (sqrt_psi * nsqrt)
            _, s, V = linalg.svd(Xtilde, full_matrices=False)
            V = V[:n_components]
            s **= 2
            # Use 'maximum' here to avoid sqrt problems.
            W = np.sqrt(np.maximum(s[:n_components] - 1, 0))[:, np.newaxis] * V
            W *= sqrt_psi

            # loglikelihood
            ll = llconst + np.sum(np.log(s[:n_components]))
            ll += np.sum(s[n_components:]) + np.sum(np.log(psi))
            ll *= -n_samples / 2.
            loglike.append(ll)
            if (ll - old_ll) < self.tol:
                break
            old_ll = ll

            psi = np.maximum(var - np.sum(W ** 2, axis=0), SMALL)
        else:
            if self.verbose:
                print "Did not converge"

        self.components_ = W
        self.noise_variance_ = psi
        self.loglike_ = loglike
        return self

    def transform(self, X):
        """Apply dimensionality reduction to X using the model.

        Compute the expected mean of the latent variables.
        See Barber, 21.2.33 (or Bishop, 12.66).

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)
            The latent variables of X.
        """
        Ih = np.eye(len(self.components_))

        X_transformed = X - self.mean_

        Wpsi = self.components_ / self.noise_variance_
        cov_z = linalg.inv(Ih + np.dot(Wpsi, self.components_.T))
        tmp = np.dot(X_transformed, Wpsi.T)
        X_transformed = np.dot(tmp, cov_z)

        return X_transformed

    def get_covariance(self):
        """Compute data covariance with the FactorAnalysis model.

        cov = components_.T * components_ + diag(noise_variance)

        Returns
        -------
        cov : array, shape=(n_features, n_features)
            Estimated covariance of data.
        """
        cov = np.dot(self.components_.T, self.components_) \
              + np.diag(self.noise_variance_)
        return cov

    def score(self, X):
        """Compute score of X under FactorAnalysis model.
        """
        n, d = X.shape
        X = X - self.mean_

        psi = self.noise_variance_
        W = self.components_

        Ih = np.eye(self.n_components)
        const = -d / 2. * np.log(2 * np.pi)
        fac = W / psi

        cov_z = linalg.inv(Ih + np.dot(fac, W.T))
        inv_cov_x = np.diag(1. / psi) - np.dot(fac.T, np.dot(cov_z, fac))
        _, _det = np.linalg.slogdet(inv_cov_x)

        ll = n * (const + 0.5 * _det) - 0.5 * np.sum(np.dot(X, inv_cov_x) * X)
        return ll
<|endoftext|>


<commit_before># coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    determine_ext,
    int_or_none,
)
import re

class HotStarIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})'
    _GEO_COUNTRIES = ['IN']
    _TESTS = [{
        'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273',
        'info_dict': {
            'id': '1000076273',
            'ext': 'mp4',
            'title': 'On Air With AIB',
            'description': 'md5:c957d8868e9bc793ccb813691cc4c434',
            'timestamp': 1447227000,
            'upload_date': '20151111',
            'duration': 381,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }, {
        'url': 'http://www.hotstar.com/sports/cricket/rajitha-sizzles-on-debut-with-329/2001477583',
        'only_matching': True,
    }, {
        'url': 'http://www.hotstar.com/1000000515',
        'only_matching': True,
    }]

    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', fatal=True, query=None):
        json_data = super(HotStarIE, self)._download_json(
            url_or_request, video_id, note, fatal=fatal, query=query)
        if json_data['resultCode'] != 'OK':
            if fatal:
                raise ExtractorError(json_data['errorDescription'])
            return None
        return json_data['resultObj']

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_data = self._download_json(
            'http://account.hotstar.com/AVS/besc', video_id, query={
                'action': 'GetAggregatedContentDetails',
                'channel': 'PCTV',
                'contentId': video_id,
            })['contentInfo'][0]
        title = video_data['episodeTitle']

        if video_data.get('encrypted') == 'Y':
            raise ExtractorError('This video is DRM protected.', expected=True)

        formats = []
        for f in ('JIO',):
            format_data = self._download_json(
                'http://getcdn.hotstar.com/AVS/besc',
                video_id, 'Downloading %s JSON metadata' % f,
                fatal=False, query={
                    'action': 'GetCDN',
                    'asJson': 'Y',
                    'channel': f,
                    'id': video_id,
                    'type': 'VOD',
                })
            if format_data:
                format_url = format_data.get('src')
                if not format_url:
                    continue
                ext = determine_ext(format_url)
                if ext == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        format_url, video_id, 'mp4',
                        m3u8_id='hls', fatal=False))
                elif ext == 'f4m':
                    # produce broken files
                    continue
                else:
                    formats.append({
                        'url': format_url,
                        'width': int_or_none(format_data.get('width')),
                        'height': int_or_none(format_data.get('height')),
                    })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': video_data.get('description'),
            'duration': int_or_none(video_data.get('duration')),
            'timestamp': int_or_none(video_data.get('broadcastDate')),
            'formats': formats,
            'episode': title,
            'episode_number': int_or_none(video_data.get('episodeNumber')),
            'series': video_data.get('contentTitle'),
        }

class HotStarPlaylistIE(InfoExtractor):
    IE_NAME = 'hotstar:playlist'
    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/tv/(?P<playlist_title>.+)/(?P<series_id>\d+)/episodes/(?P<playlist_id>\d{1,})'

    _TESTS = [{
        'url': 'http://www.hotstar.com/tv/pow-bandi-yuddh-ke/10999/episodes/10856/9993',
        'info_dict': {
            'id': '10856',
            'title': 'pow-bandi-yuddh-ke',
        },
        'playlist_mincount': 0,
    }, {
        'url': 'http://www.hotstar.com/tv/pow-bandi-yuddh-ke/10999/episodes/10856/9993',
        'only_matching': True,
    }]

    def _extract_episode_info(self, series_id, playlist_title, video):

        picture_url = video.get('urlPictures')
        thumbnail = ''
        if picture_url:
            thumbnail = 'http://media0-starag.startv.in/r1/thumbs/PCTV/%s/%s/PCTV-%s-hs.jpg' % (picture_url[-2:], picture_url, picture_url)

        episode_title = video.get('episodeTitle', '')
        episode_title = episode_title.lower().replace(' ', '-')
        url = "http://www.hotstar.com/tv/%s/%s/%s/%s" % (playlist_title, series_id, episode_title, video.get('contentId'))

        info_dict = {
            'id': video.get('contentId'),
            'title': video.get('episodeTitle'),
            'description': video.get('longDescription'),
            'thumbnail': thumbnail,
            'url': url,
            '_type': 'url',
        }
        return info_dict

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        series_id = mobj.group('series_id')
        playlist_id = mobj.group('playlist_id')
        playlist_title = mobj.group('playlist_title')

        collection = self._download_json(
            "http://search.hotstar.com/AVS/besc?action=SearchContents&appVersion=5.0.39&channel=PCTV&moreFilters=series:%s;&query=*&searchOrder=last_broadcast_date+desc,year+asc,title+asc&type=EPISODE" % playlist_id,
            playlist_id
        )

        videos = collection.get('resultObj', {}).get('response', {}).get('docs', [])
        entries = [
            self._extract_episode_info(series_id, playlist_title, video)
            for video in videos if video.get('contentId')]
        return self.playlist_result(entries, playlist_id, playlist_title)
<commit_msg>[hotstar:playlist] Fix issues and improve (closes #12465)<commit_after># coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    determine_ext,
    ExtractorError,
    int_or_none,
)

class HotStarBaseIE(InfoExtractor):
    _GEO_COUNTRIES = ['IN']

    def _download_json(self, *args, **kwargs):
        response = super(HotStarBaseIE, self)._download_json(*args, **kwargs)
        if response['resultCode'] != 'OK':
            if kwargs.get('fatal'):
                raise ExtractorError(
                    response['errorDescription'], expected=True)
            return None
        return response['resultObj']

    def _download_content_info(self, content_id):
        return self._download_json(
            'https://account.hotstar.com/AVS/besc', content_id, query={
                'action': 'GetAggregatedContentDetails',
                'appVersion': '5.0.40',
                'channel': 'PCTV',
                'contentId': content_id,
            })['contentInfo'][0]

class HotStarIE(HotStarBaseIE):
    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})'
    _TESTS = [{
        'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273',
        'info_dict': {
            'id': '1000076273',
            'ext': 'mp4',
            'title': 'On Air With AIB',
            'description': 'md5:c957d8868e9bc793ccb813691cc4c434',
            'timestamp': 1447227000,
            'upload_date': '20151111',
            'duration': 381,
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        }
    }, {
        'url': 'http://www.hotstar.com/sports/cricket/rajitha-sizzles-on-debut-with-329/2001477583',
        'only_matching': True,
    }, {
        'url': 'http://www.hotstar.com/1000000515',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video_data = self._download_content_info(video_id)

        title = video_data['episodeTitle']

        if video_data.get('encrypted') == 'Y':
            raise ExtractorError('This video is DRM protected.', expected=True)

        formats = []
        for f in ('JIO',):
            format_data = self._download_json(
                'http://getcdn.hotstar.com/AVS/besc',
                video_id, 'Downloading %s JSON metadata' % f,
                fatal=False, query={
                    'action': 'GetCDN',
                    'asJson': 'Y',
                    'channel': f,
                    'id': video_id,
                    'type': 'VOD',
                })
            if format_data:
                format_url = format_data.get('src')
                if not format_url:
                    continue
                ext = determine_ext(format_url)
                if ext == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        format_url, video_id, 'mp4',
                        m3u8_id='hls', fatal=False))
                elif ext == 'f4m':
                    # produce broken files
                    continue
                else:
                    formats.append({
                        'url': format_url,
                        'width': int_or_none(format_data.get('width')),
                        'height': int_or_none(format_data.get('height')),
                    })
        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': video_data.get('description'),
            'duration': int_or_none(video_data.get('duration')),
            'timestamp': int_or_none(video_data.get('broadcastDate')),
            'formats': formats,
            'episode': title,
            'episode_number': int_or_none(video_data.get('episodeNumber')),
            'series': video_data.get('contentTitle'),
        }

class HotStarPlaylistIE(HotStarBaseIE):
    IE_NAME = 'hotstar:playlist'
    _VALID_URL = r'(?P<url>https?://(?:www\.)?hotstar\.com/tv/[^/]+/(?P<content_id>\d+))/(?P<type>[^/]+)/(?P<id>\d+)'
    _TESTS = [{
        'url': 'http://www.hotstar.com/tv/pratidaan/14982/episodes/14812/9993',
        'info_dict': {
            'id': '14812',
        },
        'playlist_mincount': 75,
    }, {
        'url': 'http://www.hotstar.com/tv/pratidaan/14982/popular-clips/9998/9998',
        'only_matching': True,
    }]
    _ITEM_TYPES = {
        'episodes': 'EPISODE',
        'popular-clips': 'CLIPS',
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        base_url = mobj.group('url')
        content_id = mobj.group('content_id')
        playlist_type = mobj.group('type')

        content_info = self._download_content_info(content_id)
        playlist_id = compat_str(content_info['categoryId'])

        collection = self._download_json(
            'https://search.hotstar.com/AVS/besc', playlist_id, query={
                'action': 'SearchContents',
                'appVersion': '5.0.40',
                'channel': 'PCTV',
                'moreFilters': 'series:%s;' % playlist_id,
                'query': '*',
                'searchOrder': 'last_broadcast_date desc,year desc,title asc',
                'type': self._ITEM_TYPES.get(playlist_type, 'EPISODE'),
            })

        entries = [
            self.url_result(
                '%s/_/%s' % (base_url, video['contentId']),
                ie=HotStarIE.ie_key(), video_id=video['contentId'])
            for video in collection['response']['docs']
            if video.get('contentId')]

        return self.playlist_result(entries, playlist_id)
<|endoftext|>


<commit_before>from __future__ import print_function
from __future__ import absolute_import
from . import jet as _jet
from . import io as _io
from astropy.convolution import convolve as _convolve
from astropy.convolution import Box2DKernel as _Box2DKernel
from contextlib import contextmanager as _contextmanager
from .utilities import suppress_stdout as _suppress_stdout
import numpy as _np
import sys as _sys

if _sys.version_info[0] == 2:
    from contextlib2 import ExitStack as _ExitStack
else:
    from contextlib import ExitStack as _ExitStack

def load_simulation_data(ids, directory, suppress_output=None):
    data = []
    with _ExitStack() as stack:
        if suppress_output in [None, True]:
            stack.enter_context(_suppress_stdout())
        for i in ids:
            data.append(_io.pload(i, w_dir=directory))
    return data

def get_nlast_info(directory):
    with _suppress_stdout():
        return _io.nlast_info(w_dir=directory)

def get_last_timestep(simulation_directory):
    with _suppress_stdout():
        return _io.nlast_info(w_dir=simulation_directory)['nlast']

def get_output_count(directory):
    return get_nlast_info(directory)['nlast']

def get_tracer_count(directory):
    data = load_timestep_data(0, directory)
    return get_tracer_count_data(data)

def get_tracer_count_data(sim_data):
    return len([trc for trc in sim_data.vars if 'tr' in trc])

def load_timestep_data(timestep, directory, suppress_output=None):
    with _ExitStack() as stack:
        if suppress_output in [None, True]:
            stack.enter_context(_suppress_stdout())
            return _io.pload(timestep, w_dir=directory)

def load_simulation_variables(ids, directory, var_list, suppress_output=None):
    data = {}
    for v in var_list:
        data[v] = []
    for i in ids:
        current_run_data = load_simulation_data([i], directory,
                                              suppress_output)[0]
        for v in var_list:
            data[v].append(getattr(current_run_data, v))
    return data

def load_simulation_times(run_directory, run_timesteps):
    times = []
    for i in run_timesteps:
        with _suppress_stdout():
            energy_data = _io.pload(i, w_dir=run_directory)
        times.append(energy_data.SimTime)
    return times

def sphericaltocartesian(run_data, rotation=None):

    # default rotation is pi / 2
    # (results in jet pointing up for certain simulations)
    if rotation is None:
        rotation = _np.pi / 2
    # generate the spherical polar grid
    R, Theta = _np.meshgrid(run_data.x1, run_data.x2)
    # rotate theta so that jet is pointing upwards - not necessarily needed
    Theta = Theta - rotation

    # convert spherical polar grid to cartesian
    X1 = R * _np.cos(Theta)
    X2 = R * _np.sin(-Theta)
    return X1, X2

def get_cartesian_grid(irregular_grid, x_count, y_count, min_offset=0.1):
    xmin = irregular_grid[0].min()
    ymin = irregular_grid[1].min()
    xi = _np.linspace(xmin + min_offset, irregular_grid[0].max(),
                      x_count)
    yi = _np.linspace(ymin + min_offset, irregular_grid[1].max(),
                      y_count)
    return tuple(_np.meshgrid(xi, yi))

def get_gridded_data(data,
                     irregular_grid,
                     cart_grid=None,
                     method='cubic',
                     xcount=200,
                     ycount=200,
                     fill_value=0):
    from scipy.interpolate import griddata

    # sort out our grid
    if cart_grid is None:
        cart_grid = get_cartesian_grid(
            irregular_grid, x_count=xcount, y_count=ycount)
    print(len(irregular_grid))
    print(len(cart_grid))
    print('fill value is: {0}'.format(fill_value))
    # interpolate data
    gridded_data = griddata(
        (irregular_grid[0].flatten(), irregular_grid[1].flatten()),
        data.flatten('F'),
        cart_grid,
        method=method,
        fill_value=fill_value)
    #gridded_data = _np.ma.masked_invalid(gridded_data)
    return gridded_data, cart_grid

def calculate_cell_volume(sim_data):
    cell_volumes = _np.zeros((sim_data.n1_tot, sim_data.n2_tot))
    if (sim_data.geometry == 'SPHERICAL'):
        for i in range(0, cell_volumes.shape[0]):
            r = (sim_data.x1r[i + 1]**3) - (sim_data.x1r[i]**3)
            for j in range(0, cell_volumes.shape[1]):
                volume = 2 * _np.pi * (
                    _np.cos(sim_data.x2r[j]) - _np.cos(sim_data.x2r[j + 1])) * (r /
                                                                                3)
                cell_volumes[i, j] = volume
    elif (sim_data.geometry == 'CARTESIAN'):
        for i in range(0, cell_volumes.shape[0]):
            for j in range(0, cell_volumes.shape[1]):
                cell_volumes[i, j] = sim_data.x1r[i] * sim_data.x2r[j] * 1
    return cell_volumes

def calculate_cell_area(sim_data):
    areas = _np.zeros(sim_data.rho.shape)
    if (sim_data.geometry == 'SPHERICAL'):
        for i in range(0, areas.shape[0]):
            r = (sim_data.x1r[i + 1]**2) - (sim_data.x1r[i]**2)
            for j in range(0, areas.shape[1]):
                areas[i, j] = sim_data.x1[i]**2 * _np.sin(sim_data.x2[
                    j]) * sim_data.dx2[j] * _np.pi * 2
    elif (sim_data.geometry == 'CARTESIAN'):
        for i in range(0, areas.shape[0]):
            for j in range(0, areas.shape[1]):
                areas[i, j] = sim_data.x1r[i] * sim_data.x2r[j]
    return areas

def find_last_equal_point_radial(data1, data2, epsilon=1e-5):
    """Returns the last equal points in the first dimension of the data, expects 2D arrays"""
    # find difference between 2 data sets
    difference = abs(data1 - data2)
    indicies = []
    for t_index in range(data1.shape[1]):
        indicies.append(find_last_equal_point(difference[:, t_index]))
    return _np.asarray(indicies)

def find_last_equal_point(difference, epsilon=1e-5):
    """Find the last equal point of two 1D(!) arrays, given the absolute difference between them."""
    return (_np.where(difference < epsilon)[0])[-1]

def replace_with_initial_data(initial_data, new_data, epsilon=1e-5):
    """Replace the 1D new_data array with the 1D intial_data array, from the last equal point onwards"""
    # find the last equal point
    last_index = find_last_equal_point(abs(initial_data - new_data), epsilon)

    # replace with intial data from this point onwards
    ret = _np.copy(new_data)
    ret[last_index:] = initial_data[last_index:]

    return ret

def replace_with_initial_data_radial(initial_data, new_data, epsilon=1e-5):
    """Replaces new_data with inital_data, from the last equal point in the 1st dimensions onwards. Expects 2D arrays."""
    # find the last equal point
    last_index = find_last_equal_point_radial(initial_data, new_data, epsilon)

    # replace with intial data from this point onwards
    ret = _np.copy(new_data)
    for t_index in range(new_data.shape[1]):
        ret[last_index[t_index]:, t_index] = initial_data[last_index[t_index]:,
                                                          t_index]

    return ret

def fix_numerical_errors_single_timestep(run_data, initial_data, var_list):
    for v in var_list:
        va = getattr(run_data, v)
        va = replace_with_initial_data_radial(initial_data[v], va)
        setattr(run_data, v, va)

def fix_numerical_errors(run_data, initial_data, var_list):
    for t_step in range(len(run_data)):
        fix_numerical_errors_single_timestep(run_data[t_step], initial_data,
                                             var_list)

def combine_tracers(simulation_data, ntracers):
    """Helper function to combine multiple tracers into one array. Simply adds them up"""
    ret = _np.zeros_like(simulation_data.tr1)
    for i in range(ntracers):
        ret = ret + getattr(simulation_data, 'tr{0}'.format(i + 1))
    return ret

def clamp_tracers(simulation_data,
                  ntracers,
                  tracer_threshold=1e-7,
                  tracer_effective_zero=1e-20):
    # smooth the tracer data with a 2d box kernel of width 3
    box2d = _Box2DKernel(3)
    radio_combined_tracers = _convolve(
        combine_tracers(simulation_data, ntracers), box2d, boundary='extend')
    radio_tracer_mask = _np.where(radio_combined_tracers > tracer_threshold,
                                  1.0, tracer_effective_zero)

    # create new tracer array that is clamped to tracer values
    clamped_tracers = radio_combined_tracers.copy()
    clamped_tracers[clamped_tracers <=
                    tracer_threshold] = tracer_effective_zero

    return (radio_tracer_mask, clamped_tracers, radio_combined_tracers)

def calculate_actual_jet_opening_angle(run_data, theta_deg):
    indicies = _np.where(run_data.x2 < _np.deg2rad(theta_deg))[0]
    if len(indicies) == 0:
        return (list(range(0, len(run_data.x2 - 1))), theta_deg)
    actual_angle = _np.rad2deg(run_data.x2[indicies[-1]])
    return (indicies, actual_angle)

<commit_msg>[simulations] Fix coordinates in sphericaltocartesian<commit_after>from __future__ import print_function
from __future__ import absolute_import
from . import jet as _jet
from . import io as _io
from astropy.convolution import convolve as _convolve
from astropy.convolution import Box2DKernel as _Box2DKernel
from contextlib import contextmanager as _contextmanager
from .utilities import suppress_stdout as _suppress_stdout
import numpy as _np
import sys as _sys

if _sys.version_info[0] == 2:
    from contextlib2 import ExitStack as _ExitStack
else:
    from contextlib import ExitStack as _ExitStack

def load_simulation_data(ids, directory, suppress_output=None):
    data = []
    with _ExitStack() as stack:
        if suppress_output in [None, True]:
            stack.enter_context(_suppress_stdout())
        for i in ids:
            data.append(_io.pload(i, w_dir=directory))
    return data

def get_nlast_info(directory):
    with _suppress_stdout():
        return _io.nlast_info(w_dir=directory)

def get_last_timestep(simulation_directory):
    with _suppress_stdout():
        return _io.nlast_info(w_dir=simulation_directory)['nlast']

def get_output_count(directory):
    return get_nlast_info(directory)['nlast']

def get_tracer_count(directory):
    data = load_timestep_data(0, directory)
    return get_tracer_count_data(data)

def get_tracer_count_data(sim_data):
    return len([trc for trc in sim_data.vars if 'tr' in trc])

def load_timestep_data(timestep, directory, suppress_output=None):
    with _ExitStack() as stack:
        if suppress_output in [None, True]:
            stack.enter_context(_suppress_stdout())
            return _io.pload(timestep, w_dir=directory)

def load_simulation_variables(ids, directory, var_list, suppress_output=None):
    data = {}
    for v in var_list:
        data[v] = []
    for i in ids:
        current_run_data = load_simulation_data([i], directory,
                                              suppress_output)[0]
        for v in var_list:
            data[v].append(getattr(current_run_data, v))
    return data

def load_simulation_times(run_directory, run_timesteps):
    times = []
    for i in run_timesteps:
        with _suppress_stdout():
            energy_data = _io.pload(i, w_dir=run_directory)
        times.append(energy_data.SimTime)
    return times

def sphericaltocartesian(run_data, rotation=None):

    # default rotation is pi / 2
    # (results in jet pointing up for certain simulations)
    if rotation is None:
        rotation = _np.pi / 2
    # generate the spherical polar grid
    R, Theta = _np.meshgrid(run_data.x1r, run_data.x2r)
    # rotate theta so that jet is pointing upwards - not necessarily needed
    Theta = Theta - rotation

    # convert spherical polar grid to cartesian
    X1 = R * _np.cos(Theta)
    X2 = R * _np.sin(-Theta)
    return X1, X2

def get_cartesian_grid(irregular_grid, x_count, y_count, min_offset=0.1):
    xmin = irregular_grid[0].min()
    ymin = irregular_grid[1].min()
    xi = _np.linspace(xmin + min_offset, irregular_grid[0].max(),
                      x_count)
    yi = _np.linspace(ymin + min_offset, irregular_grid[1].max(),
                      y_count)
    return tuple(_np.meshgrid(xi, yi))

def get_gridded_data(data,
                     irregular_grid,
                     cart_grid=None,
                     method='cubic',
                     xcount=200,
                     ycount=200,
                     fill_value=0):
    from scipy.interpolate import griddata

    # sort out our grid
    if cart_grid is None:
        cart_grid = get_cartesian_grid(
            irregular_grid, x_count=xcount, y_count=ycount)
    print(len(irregular_grid))
    print(len(cart_grid))
    print('fill value is: {0}'.format(fill_value))
    # interpolate data
    gridded_data = griddata(
        (irregular_grid[0].flatten(), irregular_grid[1].flatten()),
        data.flatten('F'),
        cart_grid,
        method=method,
        fill_value=fill_value)
    #gridded_data = _np.ma.masked_invalid(gridded_data)
    return gridded_data, cart_grid

def calculate_cell_volume(sim_data):
    cell_volumes = _np.zeros((sim_data.n1_tot, sim_data.n2_tot))
    if (sim_data.geometry == 'SPHERICAL'):
        for i in range(0, cell_volumes.shape[0]):
            r = (sim_data.x1r[i + 1]**3) - (sim_data.x1r[i]**3)
            for j in range(0, cell_volumes.shape[1]):
                volume = 2 * _np.pi * (
                    _np.cos(sim_data.x2r[j]) - _np.cos(sim_data.x2r[j + 1])) * (r /
                                                                                3)
                cell_volumes[i, j] = volume
    elif (sim_data.geometry == 'CARTESIAN'):
        for i in range(0, cell_volumes.shape[0]):
            for j in range(0, cell_volumes.shape[1]):
                cell_volumes[i, j] = sim_data.x1r[i] * sim_data.x2r[j] * 1
    return cell_volumes

def calculate_cell_area(sim_data):
    areas = _np.zeros(sim_data.rho.shape)
    if (sim_data.geometry == 'SPHERICAL'):
        for i in range(0, areas.shape[0]):
            r = (sim_data.x1r[i + 1]**2) - (sim_data.x1r[i]**2)
            for j in range(0, areas.shape[1]):
                areas[i, j] = sim_data.x1[i]**2 * _np.sin(sim_data.x2[
                    j]) * sim_data.dx2[j] * _np.pi * 2
    elif (sim_data.geometry == 'CARTESIAN'):
        for i in range(0, areas.shape[0]):
            for j in range(0, areas.shape[1]):
                areas[i, j] = sim_data.x1r[i] * sim_data.x2r[j]
    return areas

def find_last_equal_point_radial(data1, data2, epsilon=1e-5):
    """Returns the last equal points in the first dimension of the data, expects 2D arrays"""
    # find difference between 2 data sets
    difference = abs(data1 - data2)
    indicies = []
    for t_index in range(data1.shape[1]):
        indicies.append(find_last_equal_point(difference[:, t_index]))
    return _np.asarray(indicies)

def find_last_equal_point(difference, epsilon=1e-5):
    """Find the last equal point of two 1D(!) arrays, given the absolute difference between them."""
    return (_np.where(difference < epsilon)[0])[-1]

def replace_with_initial_data(initial_data, new_data, epsilon=1e-5):
    """Replace the 1D new_data array with the 1D intial_data array, from the last equal point onwards"""
    # find the last equal point
    last_index = find_last_equal_point(abs(initial_data - new_data), epsilon)

    # replace with intial data from this point onwards
    ret = _np.copy(new_data)
    ret[last_index:] = initial_data[last_index:]

    return ret

def replace_with_initial_data_radial(initial_data, new_data, epsilon=1e-5):
    """Replaces new_data with inital_data, from the last equal point in the 1st dimensions onwards. Expects 2D arrays."""
    # find the last equal point
    last_index = find_last_equal_point_radial(initial_data, new_data, epsilon)

    # replace with intial data from this point onwards
    ret = _np.copy(new_data)
    for t_index in range(new_data.shape[1]):
        ret[last_index[t_index]:, t_index] = initial_data[last_index[t_index]:,
                                                          t_index]

    return ret

def fix_numerical_errors_single_timestep(run_data, initial_data, var_list):
    for v in var_list:
        va = getattr(run_data, v)
        va = replace_with_initial_data_radial(initial_data[v], va)
        setattr(run_data, v, va)

def fix_numerical_errors(run_data, initial_data, var_list):
    for t_step in range(len(run_data)):
        fix_numerical_errors_single_timestep(run_data[t_step], initial_data,
                                             var_list)

def combine_tracers(simulation_data, ntracers):
    """Helper function to combine multiple tracers into one array. Simply adds them up"""
    ret = _np.zeros_like(simulation_data.tr1)
    for i in range(ntracers):
        ret = ret + getattr(simulation_data, 'tr{0}'.format(i + 1))
    return ret

def clamp_tracers(simulation_data,
                  ntracers,
                  tracer_threshold=1e-7,
                  tracer_effective_zero=1e-20):
    # smooth the tracer data with a 2d box kernel of width 3
    box2d = _Box2DKernel(3)
    radio_combined_tracers = _convolve(
        combine_tracers(simulation_data, ntracers), box2d, boundary='extend')
    radio_tracer_mask = _np.where(radio_combined_tracers > tracer_threshold,
                                  1.0, tracer_effective_zero)

    # create new tracer array that is clamped to tracer values
    clamped_tracers = radio_combined_tracers.copy()
    clamped_tracers[clamped_tracers <=
                    tracer_threshold] = tracer_effective_zero

    return (radio_tracer_mask, clamped_tracers, radio_combined_tracers)

def calculate_actual_jet_opening_angle(run_data, theta_deg):
    indicies = _np.where(run_data.x2 < _np.deg2rad(theta_deg))[0]
    if len(indicies) == 0:
        return (list(range(0, len(run_data.x2 - 1))), theta_deg)
    actual_angle = _np.rad2deg(run_data.x2[indicies[-1]])
    return (indicies, actual_angle)

<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2020 Google LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import Dict, Sequence, Any

from absl import logging
import numpy as np
import os
from sklearn.metrics import accuracy_score
import torch
import torch.nn as nn
from transformers import AdamW
from torch.utils.data import DataLoader

from cabby.evals import utils as eu
from cabby.model import util

class Trainer:
  def __init__(
          self,
          model: torch.nn.Module,
          device: torch.device,
          optimizer: torch.optim.Adam,
          file_path: str,
          train_loader: Any,
          valid_loader: DataLoader,
          test_loader: DataLoader,
          unique_cells: Sequence[int],
          num_epochs: int,
          cells_tensor: torch.tensor,
          label_to_cellid: Dict[int, int],
          is_distance_distribution: bool,
          is_single_sample_train: bool = False,
          best_valid_loss: float = float("Inf")

          ):

    self.model = model
    self.device = device
    self.optimizer = optimizer
    self.file_path = file_path
    self.train_loader = train_loader
    self.valid_loader = valid_loader
    self.test_loader = test_loader
    self.unique_cells = unique_cells
    self.num_epochs = num_epochs
    self.cells_tensor = cells_tensor.float().to(self.device)
    self.label_to_cellid = label_to_cellid
    self.cos = nn.CosineSimilarity(dim=2)
    self.best_valid_loss = best_valid_loss
    if not os.path.exists(self.file_path):
      os.mkdir(self.file_path)
    self.model_path = os.path.join(self.file_path, 'model.pt')
    self.metrics_path = os.path.join(self.file_path, 'metrics.tsv')
    self.is_distance_distribution = is_distance_distribution
    self.is_single_sample_train = is_single_sample_train
    self.evaluator = eu.Evaluator()

  def evaluate(self, validation_set: bool = True):
    '''Validate the model.'''

    if validation_set:
      data_loader = self.valid_loader
    else:
      data_loader = self.test_loader
      logging.info("Starting evaluation.")

    # Validation loop.
    true_points_list, pred_points_list = [], []
    predictions_list, true_vals  = [], []
    correct = 0
    total = 0
    loss_val_total = 0

    self.model.eval()
    with torch.no_grad():
      for batch in data_loader:

        text = {key: val.to(self.device) for key, val in batch['text'].items()}
        cellids = batch['cellid'].float().to(self.device)

        batch = {k:v.to(self.device) for k,v in batch.items() if torch.is_tensor(v)}

        loss = self.model(
            text,
            cellids,
            batch
            )

        loss_val_total+=loss

        predictions = self.model.predict(text, self.cells_tensor, self.label_to_cellid)

        predictions_list.append(predictions)
        labels = batch['label'].cpu()
        true_vals.append(labels)
        true_points_list.append(batch['end_point'].cpu())

    true_points_list = np.concatenate(true_points_list, axis=0)
    pred_points_list = np.concatenate(predictions_list, axis=0)
    true_vals = np.concatenate(true_vals, axis=0)
    average_valid_loss = loss_val_total / len(data_loader)

    return (average_valid_loss, predictions_list, true_vals,
    true_points_list, pred_points_list)

  def train_model(self):

    '''Main function for training model.'''
    # Initialize running values.
    global_step = 0

    # Training loop.
    self.model.train()

    for epoch in range(self.num_epochs):
      running_loss = 0.0
      logging.info("Epoch number: {}".format(epoch))
      for batch_idx, batch in enumerate(self.train_loader):
        self.optimizer.zero_grad()
        text = {key: val.to(self.device) for key, val in batch['text'].items()}
        cellids = batch['cellid'].float().to(self.device)
        batch = {k:v.to(self.device) for k,v in batch.items() if torch.is_tensor(v)}

        loss = self.model(
            text,
            cellids,
            batch
        )

        loss.backward()

        self.optimizer.step()

        # Update running values.
        running_loss += loss.item()
        global_step += 1

        if self.is_single_sample_train:
          break

      # Evaluation step.
      valid_loss, predictions, true_vals, true_points, pred_points = self.evaluate()

      average_train_loss = running_loss / (batch_idx + 1)

      # Resetting running values.
      running_loss = 0.0

      logging.info('Epoch [{}/{}], Step [{}/{}], \
          Train Loss: {:.4f}, Valid Loss: {:.4f}'
            .format(epoch+1, self.num_epochs, global_step,
                self.num_epochs*len(self.train_loader),
                average_train_loss, valid_loss))

      # Save model and results in checkpoint.
      if self.best_valid_loss > valid_loss:
        self.best_valid_loss = valid_loss
        util.save_checkpoint(self.model_path, self.model, self.best_valid_loss)

      self.model.train()

      if self.is_single_sample_train:
        return

    logging.info('Finished Training.')

    model_state = util.load_checkpoint(self.model_path, self.model, self.device)
    valid_loss = model_state['valid_loss']

    logging.info(
      f'Loaded best model (with validation loss {valid_loss}) for testing.')

    logging.info('Start testing.')

    test_loss, predictions, true_vals, true_points, pred_points = self.evaluate(
      validation_set = False)

    util.save_metrics_last_only(
          self.metrics_path,
          true_points,
          pred_points)

    evaluator = eu.Evaluator()
    error_distances = evaluator.get_error_distances(self.metrics_path)
    _, mean_distance, median_distance, max_error, norm_auc = evaluator.compute_metrics(error_distances)

    logging.info(f"\
          Mean distance: {mean_distance}, \
          Median distance: {median_distance}, \
          Max error: {max_error}, \
          Norm AUC: {norm_auc}")

    if not self.model.is_generation:
      self.save_cell_embed()

  def multi_train_model(self):

    '''Main function for training model.'''
    # Initialize running values.
    global_step = 0

    # Training loop.
    self.model.train()

    for epoch in range(self.num_epochs):
      running_loss = 0.0
      logging.info("Epoch number: {}".format(epoch))
      for batches in zip(self.train_loader[0], self.train_loader[1], self.train_loader[2]):
        loss = torch.tensor([0.0]).to(self.device)
        self.optimizer.zero_grad()
        for batch_idx in range(len(self.train_loader)):
          batch  = batches[batch_idx]
          text = {key: val.to(self.device) for key, val in batch['text'].items()}
          cellids = batch['cellid'].float().to(self.device)
          batch = {k:v.to(self.device) for k,v in batch.items() if torch.is_tensor(v)}

          loss += self.model(
              text,
              cellids,
              batch
          )

        loss.backward()

        self.optimizer.step()

        # Update running values.
        running_loss += loss.item()
        global_step += 1

      # Evaluation step.
      valid_loss, predictions, true_vals, true_points, pred_points = self.evaluate()

      average_train_loss = running_loss / (batch_idx + 1)

      # Resetting running values.
      running_loss = 0.0

      logging.info('Epoch [{}/{}], Step [{}/{}], \
          Train Loss: {:.4f}, Valid Loss: {:.4f}'
            .format(epoch+1, self.num_epochs, global_step,
                self.num_epochs*len(self.train_loader),
                average_train_loss, valid_loss))

      # Save model and results in checkpoint.
      if self.best_valid_loss > valid_loss:
        self.best_valid_loss = valid_loss
        util.save_checkpoint(self.model_path, self.model, self.best_valid_loss)

      self.model.train()

      if self.is_single_sample_train:
        return

    logging.info('Finished Training.')

    model_state = util.load_checkpoint(self.model_path, self.model, self.device)
    valid_loss = model_state['valid_loss']

    logging.info(
      f'Loaded best model (with validation loss {valid_loss}) for testing.')

    logging.info('Start testing.')

    test_loss, predictions, true_vals, true_points, pred_points = self.evaluate(
      validation_set = False)

    util.save_metrics_last_only(
          self.metrics_path,
          true_points,
          pred_points)

    evaluator = eu.Evaluator()
    error_distances = evaluator.get_error_distances(self.metrics_path)
    _, mean_distance, median_distance, max_error, norm_auc = evaluator.compute_metrics(error_distances)

    logging.info(f"\
          Mean distance: {mean_distance}, \
          Median distance: {median_distance}, \
          Max error: {max_error}, \
          Norm AUC: {norm_auc}")

    if not self.model.is_generation:
      self.save_cell_embed()

  def save_cell_embed(self):
    if isinstance(self.model, nn.DataParallel):
      cell_embed = self.model.module.cellid_main(self.cells_tensor)
    else:
      cell_embed = self.model.cellid_main(self.cells_tensor)
    cellid_to_embed = {
      cell: embed for cell, embed in zip(self.unique_cells, cell_embed)}
    path_to_save = os.path.join(self.file_path, 'cellid_to_embedding.pt')
    torch.save(cellid_to_embed, path_to_save)
    logging.info(f'Cell embedding saved to ==> {path_to_save}')

def infer_text(model: torch.nn.Module, text: str):
  if isinstance(model, nn.DataParallel):
    return model.module.text_embed(text)
  else:
    return model.text_embed(text)
    <commit_msg>fixed use of warmup model<commit_after># coding=utf-8
# Copyright 2020 Google LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import Dict, Sequence, Any

from absl import logging
import numpy as np
import os
from sklearn.metrics import accuracy_score
import torch
import torch.nn as nn
from transformers import AdamW
from torch.utils.data import DataLoader

from cabby.evals import utils as eu
from cabby.model import util

class Trainer:
  def __init__(
          self,
          model: torch.nn.Module,
          device: torch.device,
          optimizer: torch.optim.Adam,
          file_path: str,
          train_loader: Any,
          valid_loader: DataLoader,
          test_loader: DataLoader,
          unique_cells: Sequence[int],
          num_epochs: int,
          cells_tensor: torch.tensor,
          label_to_cellid: Dict[int, int],
          is_distance_distribution: bool,
          is_single_sample_train: bool = False,
          best_valid_loss: float = float("Inf")

          ):

    self.model = model
    self.device = device
    self.optimizer = optimizer
    self.file_path = file_path
    self.train_loader = train_loader
    self.valid_loader = valid_loader
    self.test_loader = test_loader
    self.unique_cells = unique_cells
    self.num_epochs = num_epochs
    self.cells_tensor = cells_tensor.float().to(self.device)
    self.label_to_cellid = label_to_cellid
    self.cos = nn.CosineSimilarity(dim=2)
    self.best_valid_loss = best_valid_loss
    if not os.path.exists(self.file_path):
      os.mkdir(self.file_path)
    self.model_path = os.path.join(self.file_path, 'model.pt')
    self.metrics_path = os.path.join(self.file_path, 'metrics.tsv')
    self.is_distance_distribution = is_distance_distribution
    self.is_single_sample_train = is_single_sample_train
    self.evaluator = eu.Evaluator()

  def evaluate(self, validation_set: bool = True):
    '''Validate the model.'''

    if validation_set:
      data_loader = self.valid_loader
    else:
      data_loader = self.test_loader
      logging.info("Starting evaluation.")

    # Validation loop.
    true_points_list, pred_points_list = [], []
    predictions_list, true_vals  = [], []
    correct = 0
    total = 0
    loss_val_total = 0

    self.model.eval()
    with torch.no_grad():
      for batch in data_loader:

        text = {key: val.to(self.device) for key, val in batch['text'].items()}
        cellids = batch['cellid'].float().to(self.device)

        batch = {k:v.to(self.device) for k,v in batch.items() if torch.is_tensor(v)}

        loss = self.model(
            text,
            cellids,
            batch
            )

        loss_val_total+=loss

        predictions = self.model.predict(text, self.cells_tensor, self.label_to_cellid)

        predictions_list.append(predictions)
        labels = batch['label'].cpu()
        true_vals.append(labels)
        true_points_list.append(batch['end_point'].cpu())

    true_points_list = np.concatenate(true_points_list, axis=0)
    pred_points_list = np.concatenate(predictions_list, axis=0)
    true_vals = np.concatenate(true_vals, axis=0)
    average_valid_loss = loss_val_total / len(data_loader)

    return (average_valid_loss, predictions_list, true_vals,
    true_points_list, pred_points_list)

  def train_model(self):

    '''Main function for training model.'''
    # Initialize running values.
    global_step = 0

    # Training loop.
    self.model.train()

    for epoch in range(self.num_epochs):
      running_loss = 0.0
      logging.info("Epoch number: {}".format(epoch))
      for batch_idx, batch in enumerate(self.train_loader):
        self.optimizer.zero_grad()
        text = {key: val.to(self.device) for key, val in batch['text'].items()}
        cellids = batch['cellid'].float().to(self.device)
        batch = {k:v.to(self.device) for k,v in batch.items() if torch.is_tensor(v)}

        loss = self.model(
            text,
            cellids,
            batch
        )

        loss.backward()

        self.optimizer.step()

        # Update running values.
        running_loss += loss.item()
        global_step += 1

        if self.is_single_sample_train:
          break

      # Evaluation step.
      valid_loss, predictions, true_vals, true_points, pred_points = self.evaluate()

      average_train_loss = running_loss / (batch_idx + 1)

      # Resetting running values.
      running_loss = 0.0

      logging.info('Epoch [{}/{}], Step [{}/{}], \
          Train Loss: {:.4f}, Valid Loss: {:.4f}'
            .format(epoch+1, self.num_epochs, global_step,
                self.num_epochs*len(self.train_loader),
                average_train_loss, valid_loss))

      # Save model and results in checkpoint.
      if self.best_valid_loss > valid_loss:
        self.best_valid_loss = valid_loss
        util.save_checkpoint(self.model_path, self.model, self.best_valid_loss)

      self.model.train()

      if self.is_single_sample_train:
        return

    logging.info('Finished Training.')

    model_state = util.load_checkpoint(self.model_path, self.model, self.device)
    valid_loss = model_state['valid_loss']

    logging.info(
      f'Loaded best model (with validation loss {valid_loss}) for testing.')

    logging.info('Start testing.')

    test_loss, predictions, true_vals, true_points, pred_points = self.evaluate(
      validation_set = False)

    util.save_metrics_last_only(
          self.metrics_path,
          true_points,
          pred_points)

    evaluator = eu.Evaluator()
    error_distances = evaluator.get_error_distances(self.metrics_path)
    _, mean_distance, median_distance, max_error, norm_auc = evaluator.compute_metrics(error_distances)

    logging.info(f"\
          Mean distance: {mean_distance}, \
          Median distance: {median_distance}, \
          Max error: {max_error}, \
          Norm AUC: {norm_auc}")

    if not self.model.is_generation:
      self.save_cell_embed()

  def multi_train_model(self):

    '''Main function for training model.'''
    # Initialize running values.
    global_step = 0

    # Training loop.
    self.model.train()

    for epoch in range(self.num_epochs):
      running_loss = 0.0
      logging.info("Epoch number: {}".format(epoch))
      for batches in zip(self.train_loader[0], self.train_loader[1], self.train_loader[2]):
        loss = torch.tensor([0.0]).to(self.device)
        self.optimizer.zero_grad()
        for batch_idx in range(len(self.train_loader)):
          if batch_idx==2:
            self.model.is_landmarks=False
          else:
            self.model.is_landmarks=True
          batch  = batches[batch_idx]
          text = {key: val.to(self.device) for key, val in batch['text'].items()}
          cellids = batch['cellid'].float().to(self.device)
          batch = {k:v.to(self.device) for k,v in batch.items() if torch.is_tensor(v)}

          loss += self.model(
              text,
              cellids,
              batch
          )

        loss.backward()

        self.optimizer.step()

        # Update running values.
        running_loss += loss.item()
        global_step += 1

      # Evaluation step.
      valid_loss, predictions, true_vals, true_points, pred_points = self.evaluate()

      average_train_loss = running_loss / (batch_idx + 1)

      # Resetting running values.
      running_loss = 0.0

      logging.info('Epoch [{}/{}], Step [{}/{}], \
          Train Loss: {:.4f}, Valid Loss: {:.4f}'
            .format(epoch+1, self.num_epochs, global_step,
                self.num_epochs*len(self.train_loader),
                average_train_loss, valid_loss))

      # Save model and results in checkpoint.
      if self.best_valid_loss > valid_loss:
        self.best_valid_loss = valid_loss
        util.save_checkpoint(self.model_path, self.model, self.best_valid_loss)

      self.model.train()

      if self.is_single_sample_train:
        return

    logging.info('Finished Training.')

    model_state = util.load_checkpoint(self.model_path, self.model, self.device)
    valid_loss = model_state['valid_loss']

    logging.info(
      f'Loaded best model (with validation loss {valid_loss}) for testing.')

    logging.info('Start testing.')

    test_loss, predictions, true_vals, true_points, pred_points = self.evaluate(
      validation_set = False)

    util.save_metrics_last_only(
          self.metrics_path,
          true_points,
          pred_points)

    evaluator = eu.Evaluator()
    error_distances = evaluator.get_error_distances(self.metrics_path)
    _, mean_distance, median_distance, max_error, norm_auc = evaluator.compute_metrics(error_distances)

    logging.info(f"\
          Mean distance: {mean_distance}, \
          Median distance: {median_distance}, \
          Max error: {max_error}, \
          Norm AUC: {norm_auc}")

    if not self.model.is_generation:
      self.save_cell_embed()

  def save_cell_embed(self):
    if isinstance(self.model, nn.DataParallel):
      cell_embed = self.model.module.cellid_main(self.cells_tensor)
    else:
      cell_embed = self.model.cellid_main(self.cells_tensor)
    cellid_to_embed = {
      cell: embed for cell, embed in zip(self.unique_cells, cell_embed)}
    path_to_save = os.path.join(self.file_path, 'cellid_to_embedding.pt')
    torch.save(cellid_to_embed, path_to_save)
    logging.info(f'Cell embedding saved to ==> {path_to_save}')

def infer_text(model: torch.nn.Module, text: str):
  if isinstance(model, nn.DataParallel):
    return model.module.text_embed(text)
  else:
    return model.text_embed(text)
    <|endoftext|>


<commit_before>from rawdisk.filesystems.common import Partition
from rawdisk.util.rawstruct import RawStruct
import hurry.filesize
from mft import *
import hexdump

BPB_SIZE = 25
BPB_OFFSET = 0x0B
EXTENDED_BPB_SIZE = 48

class NTFS_Boot_Sector(RawStruct):
    def __init__(self):
        self.bpb = BPB()

    def load(self, data):
        RawStruct.data.fset(self, data)
        self.oem_id = self.get_string(3, 8)
        self.bpb.load(self.get_chunk(BPB_OFFSET, BPB_SIZE + EXTENDED_BPB_SIZE))

class BPB(RawStruct):
    """Bios parameter block
    """
    def __init__(self):
        pass

    def load(self, data):
        RawStruct.data.fset(self, data)
        self.bytes_per_sector = self.get_ushort(0)
        self.sectors_per_cluster = self.get_ubyte(2)
        self.reserved_sectors = self.get_ushort(3)
        self.media_descriptor = self.get_ubyte(10)
        self.sectors_per_track = self.get_ushort(13)
        self.number_of_heads = self.get_ushort(15)
        self.hidden_sectors = self.get_uint(17)
        self.total_sectors = self.get_ulonglong(29)
        self.mft_cluster = self.get_ulonglong(37)
        self.mft_mirror_cluster = self.get_ulonglong(45)
        self.clusters_per_mft = self.get_uint(53)
        self.clusters_per_index = self.get_ubyte(57)
        self.volume_serial = self.get_ulonglong(58)
        self.checksum = self.get_uint(66)

class NTFS_Partition(Partition):
    def __init__(self):
        Partition.__init__(self)
        self.partition_offset = 0
        self.mft_table = None
        self.bootsector = NTFS_Boot_Sector()

    def load(self, filename, offset):
        self.partition_offset = offset

        try:
            with open(filename, 'rb') as f:
                f.seek(self.partition_offset)
                data = f.read(512)
                self.bootsector.load(data)
                self.mft_table = MFT_Table(self.mft_table_offset)
                self.mft_table.load(f)

        except IOError, e:
            print e

    def __str__(self):
        return "Type: NTFS, Offset: 0x%X, Size: %s" % (
            self.partition_offset,
            hurry.filesize.size(self.size)
        )

    @property
    def size(self):
        return self.bootsector.bpb.bytes_per_sector * \
            self.bootsector.bpb.total_sectors

    @property
    def mft_table_offset(self):
        bytes_per_cluster = self.bootsector.bpb.sectors_per_cluster * \
                self.bootsector.bpb.bytes_per_sector

        return self.partition_offset + \
                bytes_per_cluster * \
                self.bootsector.bpb.mft_cluster
<commit_msg>Moved load methods to constructors<commit_after>from rawdisk.filesystems.common import Partition
from rawdisk.util.rawstruct import RawStruct
import hurry.filesize
from mft import *
import hexdump

BPB_SIZE = 25
BPB_OFFSET = 0x0B
EXTENDED_BPB_SIZE = 48

class NTFS_Boot_Sector(RawStruct):
    def __init__(self, data = None):
        RawStruct.data.fset(self, data)
        self.oem_id = self.get_string(3, 8)
        self.bpb = BPB(self.get_chunk(BPB_OFFSET, BPB_SIZE + EXTENDED_BPB_SIZE))

class BPB(RawStruct):
    """Bios parameter block
    """
    def __init__(self, data):
        RawStruct.data.fset(self, data)
        self.bytes_per_sector = self.get_ushort(0)
        self.sectors_per_cluster = self.get_ubyte(2)
        self.reserved_sectors = self.get_ushort(3)
        self.media_descriptor = self.get_ubyte(10)
        self.sectors_per_track = self.get_ushort(13)
        self.number_of_heads = self.get_ushort(15)
        self.hidden_sectors = self.get_uint(17)
        self.total_sectors = self.get_ulonglong(29)
        self.mft_cluster = self.get_ulonglong(37)
        self.mft_mirror_cluster = self.get_ulonglong(45)
        self.clusters_per_mft = self.get_uint(53)
        self.clusters_per_index = self.get_ubyte(57)
        self.volume_serial = self.get_ulonglong(58)
        self.checksum = self.get_uint(66)

class NTFS_Partition(Partition):
    def __init__(self):
        Partition.__init__(self)
        self.partition_offset = 0
        self.mft_table = None

    def load(self, filename, offset):
        self.partition_offset = offset

        try:
            with open(filename, 'rb') as f:
                f.seek(self.partition_offset)
                data = f.read(512)
                self.bootsector = NTFS_Boot_Sector(data)
                self.mft_table = MFT_Table(self.mft_table_offset)
                self.mft_table.load(f)

        except IOError, e:
            print e

    def __str__(self):
        return "Type: NTFS, Offset: 0x%X, Size: %s" % (
            self.partition_offset,
            hurry.filesize.size(self.size)
        )

    @property
    def size(self):
        return self.bootsector.bpb.bytes_per_sector * \
            self.bootsector.bpb.total_sectors

    @property
    def mft_table_offset(self):
        bytes_per_cluster = self.bootsector.bpb.sectors_per_cluster * \
                self.bootsector.bpb.bytes_per_sector

        return self.partition_offset + \
                bytes_per_cluster * \
                self.bootsector.bpb.mft_cluster
<|endoftext|>


<commit_before>"""Factor Analysis.
A latent linear variable model, similar to ProbabilisticPCA.

This implementation is based on David Barber's Book,
Bayesian Reasoning and Machine Learning,
http://www.cs.ucl.ac.uk/staff/d.barber/brml,
Algorithm 21.1
"""

# Author: Christian Osendorfer <osendorf@gmail.com>
#         Alexandre Gramfort <alexandre.gramfort@inria.fr>
# Licence: BSD3

from math import sqrt
import numpy as np
from scipy import linalg

from ..base import BaseEstimator, TransformerMixin
from ..externals.six.moves import xrange
from ..utils import array2d, check_arrays
from ..utils.extmath import fast_logdet

class FactorAnalysis(BaseEstimator, TransformerMixin):
    """Factor Analysis (FA)

    A simple linear generative model with Gaussian latent variables.

    The observations are assumed to be caused by a linear transformation of
    lower dimensional latent factors and added Gaussian noise.
    Without loss of generality the factors are distributed according to a
    Gaussian with zero mean and unit covariance. The noise is also zero mean
    and has an arbitrary diagonal covariance matrix.

    If we would restrict the model further, by assuming that the Gaussian
    noise is even isotropic (all diagonal entries are the same) we would obtain
    :class:`PPCA`.

    FactorAnalysis performs a maximum likelihood estimate of the so-called
    `loading` matrix, the transformation of the latent variables to the
    observed ones, using expectation-maximization (EM).

    Parameters
    ----------
    n_components : int | None
        Dimensionality of latent space, the number of components
        of ``X`` that are obtained after ``transform``.
        If None, n_components is set to the number of features.

    tol : float
        Stopping tolerance for EM algorithm.

    copy : bool
        Whether to make a copy of X. If ``False``, the input X gets overwritten
        during fitting.

    max_iter : int
        Maximum number of iterations.

    verbose : int | bool
        Print verbose output.

    noise_variance_init : None | array, shape=(n_features,)
        The initial guess of the noise variance for each feature.
        If None, it defaults to np.ones(n_features)

    Attributes
    ----------
    `components_` : array, [n_components, n_features]
        Components with maximum variance.

    `loglike_` : list, [n_iterations]
        The log likelihood at each iteration.

    `noise_variance_` : array, shape=(n_features,)
        The estimated noise variance for each feature.

    References
    ----------
    .. David Barber, Bayesian Reasoning and Machine Learning,
        Algorithm 21.1

    .. Christopher M. Bishop: Pattern Recognition and Machine Learning,
        Chapter 12.2.4

    See also
    --------
    PCA: Principal component analysis, a simliar non-probabilistic
        model model that can be computed in closed form.
    ProbabilisticPCA: probabilistic PCA.
    FastICA: Independent component analysis, a latent variable model with
        non-Gaussian latent variables.
    """
    def __init__(self, n_components=None, tol=1e-2, copy=True, max_iter=1000,
                 verbose=0, noise_variance_init=None):
        self.n_components = n_components
        self.copy = copy
        self.tol = tol
        self.max_iter = max_iter
        self.verbose = verbose
        self.noise_variance_init = noise_variance_init

    def fit(self, X, y=None):
        """Fit the FactorAnalysis model to X using EM

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        self
        """
        X = array2d(check_arrays(X, copy=self.copy, sparse_format='dense',
                    dtype=np.float)[0])

        n_samples, n_features = X.shape
        n_components = self.n_components
        if n_components is None:
            n_components = n_features
        self.mean_ = np.mean(X, axis=0)
        X -= self.mean_

        # some constant terms
        nsqrt = sqrt(n_samples)
        llconst = n_features * np.log(2 * np.pi) + n_components
        var = np.var(X, axis=0)

        if self.noise_variance_init is None:
            psi = np.ones(n_features, dtype=X.dtype)
        else:
            if len(self.noise_variance_init) != n_features:
                raise ValueError("noise_variance_init dimension does not "
                                 "with number of features : %d != %d" %
                                 (len(self.noise_variance_init), n_features))
            psi = np.array(self.noise_variance_init)

        loglike = []
        old_ll = -np.inf
        SMALL = 1e-12
        for i in xrange(self.max_iter):
            # SMALL helps numerics
            sqrt_psi = np.sqrt(psi) + SMALL
            Xtilde = X / (sqrt_psi * nsqrt)
            _, s, V = linalg.svd(Xtilde, full_matrices=False)
            V = V[:n_components]
            s **= 2
            # Use 'maximum' here to avoid sqrt problems.
            W = np.sqrt(np.maximum(s[:n_components] - 1, 0))[:, np.newaxis] * V
            W *= sqrt_psi

            # loglikelihood
            ll = llconst + np.sum(np.log(s[:n_components]))
            ll += np.sum(s[n_components:]) + np.sum(np.log(psi))
            ll *= -n_samples / 2.
            loglike.append(ll)
            if (ll - old_ll) < self.tol:
                break
            old_ll = ll

            psi = np.maximum(var - np.sum(W ** 2, axis=0), SMALL)
        else:
            if self.verbose:
                print("Did not converge")

        self.components_ = W
        self.noise_variance_ = psi
        self.loglike_ = loglike
        return self

    def transform(self, X):
        """Apply dimensionality reduction to X using the model.

        Compute the expected mean of the latent variables.
        See Barber, 21.2.33 (or Bishop, 12.66).

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)
            The latent variables of X.
        """
        X = array2d(X)
        Ih = np.eye(len(self.components_))

        X_transformed = X - self.mean_

        Wpsi = self.components_ / self.noise_variance_
        cov_z = linalg.inv(Ih + np.dot(Wpsi, self.components_.T))
        tmp = np.dot(X_transformed, Wpsi.T)
        X_transformed = np.dot(tmp, cov_z)

        return X_transformed

    def get_covariance(self):
        """Compute data covariance with the FactorAnalysis model.

        ``cov = components_.T * components_ + diag(noise_variance)``

        Returns
        -------
        cov : array, shape=(n_features, n_features)
            Estimated covariance of data.
        """
        cov = (np.dot(self.components_.T, self.components_)
               + np.diag(self.noise_variance_))
        return cov

    def score(self, X):
        """Compute score of X under FactorAnalysis model.

        Parameters
        ----------
        X: array of shape(n_samples, n_features)
            The data to test

        Returns
        -------
        ll: array of shape (n_samples),
            log-likelihood of each row of X under the current model
        """
        Xr = X - self.mean_
        cov = self.get_covariance()
        n_features = X.shape[1]
        log_like = np.zeros(X.shape[0])
        self.precision_ = linalg.inv(cov)
        log_like = -.5 * (Xr * (np.dot(Xr, self.precision_))).sum(axis=1)
        log_like -= .5 * (fast_logdet(cov) + n_features * np.log(2 * np.pi))
        return log_like
<commit_msg>FIX : missing y=None in FactorAnalysis<commit_after>"""Factor Analysis.
A latent linear variable model, similar to ProbabilisticPCA.

This implementation is based on David Barber's Book,
Bayesian Reasoning and Machine Learning,
http://www.cs.ucl.ac.uk/staff/d.barber/brml,
Algorithm 21.1
"""

# Author: Christian Osendorfer <osendorf@gmail.com>
#         Alexandre Gramfort <alexandre.gramfort@inria.fr>
# Licence: BSD3

from math import sqrt
import numpy as np
from scipy import linalg

from ..base import BaseEstimator, TransformerMixin
from ..externals.six.moves import xrange
from ..utils import array2d, check_arrays
from ..utils.extmath import fast_logdet

class FactorAnalysis(BaseEstimator, TransformerMixin):
    """Factor Analysis (FA)

    A simple linear generative model with Gaussian latent variables.

    The observations are assumed to be caused by a linear transformation of
    lower dimensional latent factors and added Gaussian noise.
    Without loss of generality the factors are distributed according to a
    Gaussian with zero mean and unit covariance. The noise is also zero mean
    and has an arbitrary diagonal covariance matrix.

    If we would restrict the model further, by assuming that the Gaussian
    noise is even isotropic (all diagonal entries are the same) we would obtain
    :class:`PPCA`.

    FactorAnalysis performs a maximum likelihood estimate of the so-called
    `loading` matrix, the transformation of the latent variables to the
    observed ones, using expectation-maximization (EM).

    Parameters
    ----------
    n_components : int | None
        Dimensionality of latent space, the number of components
        of ``X`` that are obtained after ``transform``.
        If None, n_components is set to the number of features.

    tol : float
        Stopping tolerance for EM algorithm.

    copy : bool
        Whether to make a copy of X. If ``False``, the input X gets overwritten
        during fitting.

    max_iter : int
        Maximum number of iterations.

    verbose : int | bool
        Print verbose output.

    noise_variance_init : None | array, shape=(n_features,)
        The initial guess of the noise variance for each feature.
        If None, it defaults to np.ones(n_features)

    Attributes
    ----------
    `components_` : array, [n_components, n_features]
        Components with maximum variance.

    `loglike_` : list, [n_iterations]
        The log likelihood at each iteration.

    `noise_variance_` : array, shape=(n_features,)
        The estimated noise variance for each feature.

    References
    ----------
    .. David Barber, Bayesian Reasoning and Machine Learning,
        Algorithm 21.1

    .. Christopher M. Bishop: Pattern Recognition and Machine Learning,
        Chapter 12.2.4

    See also
    --------
    PCA: Principal component analysis, a simliar non-probabilistic
        model model that can be computed in closed form.
    ProbabilisticPCA: probabilistic PCA.
    FastICA: Independent component analysis, a latent variable model with
        non-Gaussian latent variables.
    """
    def __init__(self, n_components=None, tol=1e-2, copy=True, max_iter=1000,
                 verbose=0, noise_variance_init=None):
        self.n_components = n_components
        self.copy = copy
        self.tol = tol
        self.max_iter = max_iter
        self.verbose = verbose
        self.noise_variance_init = noise_variance_init

    def fit(self, X, y=None):
        """Fit the FactorAnalysis model to X using EM

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        self
        """
        X = array2d(check_arrays(X, copy=self.copy, sparse_format='dense',
                    dtype=np.float)[0])

        n_samples, n_features = X.shape
        n_components = self.n_components
        if n_components is None:
            n_components = n_features
        self.mean_ = np.mean(X, axis=0)
        X -= self.mean_

        # some constant terms
        nsqrt = sqrt(n_samples)
        llconst = n_features * np.log(2 * np.pi) + n_components
        var = np.var(X, axis=0)

        if self.noise_variance_init is None:
            psi = np.ones(n_features, dtype=X.dtype)
        else:
            if len(self.noise_variance_init) != n_features:
                raise ValueError("noise_variance_init dimension does not "
                                 "with number of features : %d != %d" %
                                 (len(self.noise_variance_init), n_features))
            psi = np.array(self.noise_variance_init)

        loglike = []
        old_ll = -np.inf
        SMALL = 1e-12
        for i in xrange(self.max_iter):
            # SMALL helps numerics
            sqrt_psi = np.sqrt(psi) + SMALL
            Xtilde = X / (sqrt_psi * nsqrt)
            _, s, V = linalg.svd(Xtilde, full_matrices=False)
            V = V[:n_components]
            s **= 2
            # Use 'maximum' here to avoid sqrt problems.
            W = np.sqrt(np.maximum(s[:n_components] - 1, 0))[:, np.newaxis] * V
            W *= sqrt_psi

            # loglikelihood
            ll = llconst + np.sum(np.log(s[:n_components]))
            ll += np.sum(s[n_components:]) + np.sum(np.log(psi))
            ll *= -n_samples / 2.
            loglike.append(ll)
            if (ll - old_ll) < self.tol:
                break
            old_ll = ll

            psi = np.maximum(var - np.sum(W ** 2, axis=0), SMALL)
        else:
            if self.verbose:
                print("Did not converge")

        self.components_ = W
        self.noise_variance_ = psi
        self.loglike_ = loglike
        return self

    def transform(self, X):
        """Apply dimensionality reduction to X using the model.

        Compute the expected mean of the latent variables.
        See Barber, 21.2.33 (or Bishop, 12.66).

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)
            The latent variables of X.
        """
        X = array2d(X)
        Ih = np.eye(len(self.components_))

        X_transformed = X - self.mean_

        Wpsi = self.components_ / self.noise_variance_
        cov_z = linalg.inv(Ih + np.dot(Wpsi, self.components_.T))
        tmp = np.dot(X_transformed, Wpsi.T)
        X_transformed = np.dot(tmp, cov_z)

        return X_transformed

    def get_covariance(self):
        """Compute data covariance with the FactorAnalysis model.

        ``cov = components_.T * components_ + diag(noise_variance)``

        Returns
        -------
        cov : array, shape=(n_features, n_features)
            Estimated covariance of data.
        """
        cov = (np.dot(self.components_.T, self.components_)
               + np.diag(self.noise_variance_))
        return cov

    def score(self, X, y=None):
        """Compute score of X under FactorAnalysis model.

        Parameters
        ----------
        X: array of shape(n_samples, n_features)
            The data to test

        Returns
        -------
        ll: array of shape (n_samples),
            log-likelihood of each row of X under the current model
        """
        Xr = X - self.mean_
        cov = self.get_covariance()
        n_features = X.shape[1]
        log_like = np.zeros(X.shape[0])
        self.precision_ = linalg.inv(cov)
        log_like = -.5 * (Xr * (np.dot(Xr, self.precision_))).sum(axis=1)
        log_like -= .5 * (fast_logdet(cov) + n_features * np.log(2 * np.pi))
        return log_like
<|endoftext|>


<commit_before>def from_iterable(iterables):
    """
    Backport of `itertools.chain.from_iterable` compatible with Python 2.5
    """
    for it in iterables:
        for element in it:
            if isinstance(element, dict):
                for key in element:
                    yield key
            else:
                yield element

class KeepContext(object):
    """
    Context manager that receives a `django.template.Context` instance and a list of keys

    Once the context manager is exited, it removes `keys` from the context, to avoid
    side effects in later layout objects that may use the same context variables.

    Layout objects should use `extra_context` to introduce context variables, never
    touch context object themselves, that could introduce side effects.
    """
    def __init__(self, context, keys):
        self.context = context
        self.keys = keys

    def __enter__(self):
        pass

    def __exit__(self, type, value, traceback):
        for key in list(self.keys):
            del self.context[key]
<commit_msg>Remove crispy_forms.base.from_iterable. #462<commit_after>

class KeepContext(object):
    """
    Context manager that receives a `django.template.Context` instance and a list of keys

    Once the context manager is exited, it removes `keys` from the context, to avoid
    side effects in later layout objects that may use the same context variables.

    Layout objects should use `extra_context` to introduce context variables, never
    touch context object themselves, that could introduce side effects.
    """
    def __init__(self, context, keys):
        self.context = context
        self.keys = keys

    def __enter__(self):
        pass

    def __exit__(self, type, value, traceback):
        for key in list(self.keys):
            del self.context[key]
<|endoftext|>


<commit_before># coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    float_or_none,
    int_or_none,
    str_or_none,
    try_get,
)

class MedalTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?medal\.tv/clips/(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'https://medal.tv/clips/34934644/3Is9zyGMoBMr',
        'md5': '7b07b064331b1cf9e8e5c52a06ae68fa',
        'info_dict': {
            'id': '34934644',
            'ext': 'mp4',
            'title': 'Quad Cold',
            'description': 'Medal,https://medal.tv/desktop/',
            'uploader': 'MowgliSB',
            'timestamp': 1603165266,
            'upload_date': '20201020',
            'uploader_id': 10619174,
        }
    }, {
        'url': 'https://medal.tv/clips/36787208',
        'md5': 'b6dc76b78195fff0b4f8bf4a33ec2148',
        'info_dict': {
            'id': '36787208',
            'ext': 'mp4',
            'title': 'u tk me i tk u bigger',
            'description': 'Medal,https://medal.tv/desktop/',
            'uploader': 'Mimicc',
            'timestamp': 1605580939,
            'upload_date': '20201117',
            'uploader_id': 5156321,
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        hydration_data = self._parse_json(self._search_regex(
            r'<script[^>]*>\s*(?:var\s*)?hydrationData\s*=\s*({.+?})\s*</script>',
            webpage, 'hydration data', default='{}'), video_id)

        clip = try_get(
            hydration_data, lambda x: x['clips'][video_id], dict) or {}
        if not clip:
            raise ExtractorError(
                'Could not find video information.', video_id=video_id)

        title = clip['contentTitle']

        source_width = int_or_none(clip.get('sourceWidth'))
        source_height = int_or_none(clip.get('sourceHeight'))

        aspect_ratio = source_width / source_height if source_width and source_height else 16 / 9

        def add_item(container, item_url, height, id_key='format_id', item_id=None):
            item_id = item_id or '%dp' % height
            if item_id not in item_url:
                return
            width = int(round(aspect_ratio * height))
            container.append({
                'url': item_url,
                id_key: item_id,
                'width': width,
                'height': height
            })

        formats = []
        thumbnails = []
        for k, v in clip.items():
            if not (v and isinstance(v, compat_str)):
                continue
            mobj = re.match(r'(contentUrl|thumbnail)(?:(\d+)p)?$', k)
            if not mobj:
                continue
            prefix = mobj.group(1)
            height = int_or_none(mobj.group(2))
            if prefix == 'contentUrl':
                add_item(
                    formats, v, height or source_height,
                    item_id=None if height else 'source')
            elif prefix == 'thumbnail':
                add_item(thumbnails, v, height, 'id')

        error = clip.get('error')
        if not formats and error:
            if error == 404:
                raise ExtractorError(
                    'That clip does not exist.',
                    expected=True, video_id=video_id)
            else:
                raise ExtractorError(
                    'An unknown error occurred ({0}).'.format(error),
                    video_id=video_id)

        self._sort_formats(formats)

        # Necessary because the id of the author is not known in advance.
        # Won't raise an issue if no profile can be found as this is optional.
        author = try_get(
            hydration_data, lambda x: list(x['profiles'].values())[0], dict) or {}
        author_id = str_or_none(author.get('id'))
        author_url = 'https://medal.tv/users/{0}'.format(author_id) if author_id else None

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'thumbnails': thumbnails,
            'description': clip.get('contentDescription'),
            'uploader': author.get('displayName'),
            'timestamp': float_or_none(clip.get('created'), 1000),
            'uploader_id': author_id,
            'uploader_url': author_url,
            'duration': int_or_none(clip.get('videoLengthSeconds')),
            'view_count': int_or_none(clip.get('views')),
            'like_count': int_or_none(clip.get('likes')),
            'comment_count': int_or_none(clip.get('comments')),
        }
<commit_msg>[medaltv] Fix extraction (#28807)<commit_after># coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ExtractorError,
    float_or_none,
    int_or_none,
    str_or_none,
    try_get,
)

class MedalTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?medal\.tv/clips/(?P<id>[a-zA-Z0-9]+)'
    _TESTS = [{
        'url': 'https://medal.tv/clips/2mA60jWAGQCBH',
        'md5': '7b07b064331b1cf9e8e5c52a06ae68fa',
        'info_dict': {
            'id': '2mA60jWAGQCBH',
            'ext': 'mp4',
            'title': 'Quad Cold',
            'description': 'Medal,https://medal.tv/desktop/',
            'uploader': 'MowgliSB',
            'timestamp': 1603165266,
            'upload_date': '20201020',
            'uploader_id': '10619174',
        }
    }, {
        'url': 'https://medal.tv/clips/2um24TWdty0NA',
        'md5': 'b6dc76b78195fff0b4f8bf4a33ec2148',
        'info_dict': {
            'id': '2um24TWdty0NA',
            'ext': 'mp4',
            'title': 'u tk me i tk u bigger',
            'description': 'Medal,https://medal.tv/desktop/',
            'uploader': 'Mimicc',
            'timestamp': 1605580939,
            'upload_date': '20201117',
            'uploader_id': '5156321',
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        hydration_data = self._parse_json(self._search_regex(
            r'<script[^>]*>\s*(?:var\s*)?hydrationData\s*=\s*({.+?})\s*</script>',
            webpage, 'hydration data', default='{}'), video_id)

        clip = try_get(
            hydration_data, lambda x: x['clips'][video_id], dict) or {}
        if not clip:
            raise ExtractorError(
                'Could not find video information.', video_id=video_id)

        title = clip['contentTitle']

        source_width = int_or_none(clip.get('sourceWidth'))
        source_height = int_or_none(clip.get('sourceHeight'))

        aspect_ratio = source_width / source_height if source_width and source_height else 16 / 9

        def add_item(container, item_url, height, id_key='format_id', item_id=None):
            item_id = item_id or '%dp' % height
            if item_id not in item_url:
                return
            width = int(round(aspect_ratio * height))
            container.append({
                'url': item_url,
                id_key: item_id,
                'width': width,
                'height': height
            })

        formats = []
        thumbnails = []
        for k, v in clip.items():
            if not (v and isinstance(v, compat_str)):
                continue
            mobj = re.match(r'(contentUrl|thumbnail)(?:(\d+)p)?$', k)
            if not mobj:
                continue
            prefix = mobj.group(1)
            height = int_or_none(mobj.group(2))
            if prefix == 'contentUrl':
                add_item(
                    formats, v, height or source_height,
                    item_id=None if height else 'source')
            elif prefix == 'thumbnail':
                add_item(thumbnails, v, height, 'id')

        error = clip.get('error')
        if not formats and error:
            if error == 404:
                raise ExtractorError(
                    'That clip does not exist.',
                    expected=True, video_id=video_id)
            else:
                raise ExtractorError(
                    'An unknown error occurred ({0}).'.format(error),
                    video_id=video_id)

        self._sort_formats(formats)

        # Necessary because the id of the author is not known in advance.
        # Won't raise an issue if no profile can be found as this is optional.
        author = try_get(
            hydration_data, lambda x: list(x['profiles'].values())[0], dict) or {}
        author_id = str_or_none(author.get('id'))
        author_url = 'https://medal.tv/users/{0}'.format(author_id) if author_id else None

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'thumbnails': thumbnails,
            'description': clip.get('contentDescription'),
            'uploader': author.get('displayName'),
            'timestamp': float_or_none(clip.get('created'), 1000),
            'uploader_id': author_id,
            'uploader_url': author_url,
            'duration': int_or_none(clip.get('videoLengthSeconds')),
            'view_count': int_or_none(clip.get('views')),
            'like_count': int_or_none(clip.get('likes')),
            'comment_count': int_or_none(clip.get('comments')),
        }
<|endoftext|>