<commit_before>from __future__ import division

import os
import unittest
import numpy as np
from desisim.templates import ELG, LRG, QSO, BGS, STAR, FSTD, MWS_STAR, WD

desimodel_data_available = 'DESIMODEL' in os.environ
desi_templates_available = 'DESI_ROOT' in os.environ
desi_basis_templates_available = 'DESI_BASIS_TEMPLATES' in os.environ

class TestTemplates(unittest.TestCase):

    def setUp(self):
        self.wavemin = 5000
        self.wavemax = 8000
        self.dwave = 2.0
        self.wave = np.arange(self.wavemin, self.wavemax+self.dwave/2, self.dwave)
        self.nspec = 5

    def _check_output_size(self, flux, wave, meta):
        self.assertEqual(len(meta), self.nspec)
        self.assertEqual(len(wave), len(self.wave))
        self.assertEqual(flux.shape, (self.nspec, len(self.wave)))

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_input_redshift(self):
        '''Test that we can input the redshift for each spectral class.'''
        zrange = np.array([
            (0.6, 1.6), (0.5, 1.0), (0.5, 4.0), (0.01, 0.4),
            (-0.003, 0.003), (-0.003, 0.003), (-0.003, 0.003), (-0.003, 0.003)
            ])
        for zminmax, T in zip(zrange, [ELG, LRG, QSO, BGS, STAR, FSTD, MWS_STAR, WD]):
            redshift = np.random.uniform(zminmax[0], zminmax[1], self.nspec).astype('f4')
            Tx = T(wave=self.wave)
            flux, wave, meta = Tx.make_templates(self.nspec, redshift=redshift)
            self.assertTrue(np.all(redshift == meta['REDSHIFT']))

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_simple(self):
        '''Confirm that creating templates works at all'''
        for T in [ELG, LRG, QSO, BGS, STAR, FSTD, MWS_STAR, WD]:
            template_factory = T(wave=self.wave)
            flux, wave, meta = template_factory.make_templates(self.nspec)
            self._check_output_size(flux, wave, meta)

        #- Can also specify minwave, maxwave, dwave
        elg = ELG(minwave=self.wavemin, maxwave=self.wavemax, cdelt=self.dwave)
        flux, wave, meta = elg.make_templates(self.nspec)
        self._check_output_size(flux, wave, meta)

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_OII(self):
        '''Confirm that ELG [OII] flux matches meta table description'''
        wave = np.arange(5000, 9800.1, 0.2)
        flux, ww, meta = ELG(wave=wave).make_templates(
            nmodel=20, nocolorcuts=True, nocontinuum=True,
            logvdisp_meansig = [np.log10(75), 0.0])

        for i in range(len(meta)):
            z = meta['REDSHIFT'][i]
            ii = (3722*(1+z) < wave) & (wave < 3736*(1+z))
            OIIflux = np.sum(flux[i,ii]*np.gradient(wave[ii]))
            self.assertAlmostEqual(OIIflux, meta['OIIFLUX'][i], 2)

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_HBETA(self):
        '''Confirm that BGS H-beta flux matches meta table description'''
        wave = np.arange(5000, 7000.1, 0.2)
        flux, ww, meta = BGS(wave=wave).make_templates(zrange=[0.1,0.4],
            nmodel=20, nocolorcuts=True, nocontinuum=True,
            logvdisp_meansig=[np.log10(75),0.0])

        for i in range(len(meta)):
            z = meta['REDSHIFT'][i]
            ii = (4854*(1+z) < wave) & (wave < 4868*(1+z))
            hbetaflux = np.sum(flux[i,ii]*np.gradient(wave[ii]))
            self.assertAlmostEqual(hbetaflux, meta['HBETAFLUX'][i], 2)

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_stars(self):
        '''Test options specific to star templates'''
        star = STAR(wave=self.wave)
        flux, wave, meta = star.make_templates(self.nspec)
        self._check_output_size(flux, wave, meta)
        self.assertTrue('LOGG' in meta.dtype.names)
        self.assertTrue('TEFF' in meta.dtype.names)
        self.assertTrue('FEH' in meta.dtype.names)

        fstd = FSTD(wave=self.wave)
        flux, wave, meta = fstd.make_templates(self.nspec)
        self._check_output_size(flux, wave, meta)
        self.assertTrue('LOGG' in meta.dtype.names)
        self.assertTrue('TEFF' in meta.dtype.names)
        self.assertTrue('FEH' in meta.dtype.names)

        mwsstar = MWS_STAR(wave=self.wave)
        flux, wave, meta = mwsstar.make_templates(self.nspec)
        self._check_output_size(flux, wave, meta)
        self.assertTrue('LOGG' in meta.dtype.names)
        self.assertTrue('TEFF' in meta.dtype.names)
        self.assertTrue('FEH' in meta.dtype.names)

        wd = WD(wave=self.wave)
        flux, wave, meta = wd.make_templates(self.nspec)
        self._check_output_size(flux, wave, meta)
        self.assertTrue('LOGG' in meta.dtype.names)
        self.assertTrue('TEFF' in meta.dtype.names)
        self.assertTrue('FEH' in meta.dtype.names)

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_random_seed(self):
        '''Test that random seed works to get the same results back'''
        for T in [ELG, LRG, QSO, BGS, STAR, FSTD, MWS_STAR, WD]:
            Tx = T(wave=self.wave)
            flux1, wave1, meta1 = Tx.make_templates(self.nspec, seed=1)
            flux2, wave2, meta2 = Tx.make_templates(self.nspec, seed=1)
            flux3, wave3, meta3 = Tx.make_templates(self.nspec, seed=2)
            self.assertTrue(np.all(flux1==flux2))
            self.assertTrue(np.any(flux1!=flux3))
            self.assertTrue(np.all(wave1==wave2))

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_sne(self):
        '''Test options for adding in SNeIa spectra'''
        for T in [ELG, LRG, BGS]:
            template_factory = T(wave=self.wave, add_SNeIa=True)
            flux, wave, meta = template_factory.make_templates(self.nspec, nocolorcuts=True,
                                                               sne_rfluxratiorange=(0.5, 0.7))
            self._check_output_size(flux, wave, meta)
            self.assertTrue('SNE_TEMPLATEID' in meta.dtype.names)
            self.assertTrue('SNE_RFLUXRATIO' in meta.dtype.names)
            self.assertTrue('SNE_EPOCH' in meta.dtype.names)

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_input_meta(self):
        '''Test that input meta table option works.'''
        for T in [ELG, LRG, QSO, BGS, STAR, FSTD, MWS_STAR, WD]:
            Tx = T(wave=self.wave)
            flux1, wave1, meta1 = Tx.make_templates(self.nspec)
            flux2, wave2, meta2 = elgmaker.make_templates(input_meta=meta1)
            badkeys = list()
            for key in meta1.colnames:
                if not np.all(meta1[key] == meta2[key]):
                    badkeys.append(key)
            self.assertEqual(len(badkeys), 0, 'mismatch for spectral type {} in keys {}'.format(meta1['OBJTYPE'][0], badkeys))

if __name__ == '__main__':
    unittest.main()
<commit_msg>fixed typo<commit_after>from __future__ import division

import os
import unittest
import numpy as np
from desisim.templates import ELG, LRG, QSO, BGS, STAR, FSTD, MWS_STAR, WD

desimodel_data_available = 'DESIMODEL' in os.environ
desi_templates_available = 'DESI_ROOT' in os.environ
desi_basis_templates_available = 'DESI_BASIS_TEMPLATES' in os.environ

class TestTemplates(unittest.TestCase):

    def setUp(self):
        self.wavemin = 5000
        self.wavemax = 8000
        self.dwave = 2.0
        self.wave = np.arange(self.wavemin, self.wavemax+self.dwave/2, self.dwave)
        self.nspec = 5

    def _check_output_size(self, flux, wave, meta):
        self.assertEqual(len(meta), self.nspec)
        self.assertEqual(len(wave), len(self.wave))
        self.assertEqual(flux.shape, (self.nspec, len(self.wave)))

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_input_redshift(self):
        '''Test that we can input the redshift for each spectral class.'''
        zrange = np.array([
            (0.6, 1.6), (0.5, 1.0), (0.5, 4.0), (0.01, 0.4),
            (-0.003, 0.003), (-0.003, 0.003), (-0.003, 0.003), (-0.003, 0.003)
            ])
        for zminmax, T in zip(zrange, [ELG, LRG, QSO, BGS, STAR, FSTD, MWS_STAR, WD]):
            redshift = np.random.uniform(zminmax[0], zminmax[1], self.nspec).astype('f4')
            Tx = T(wave=self.wave)
            flux, wave, meta = Tx.make_templates(self.nspec, redshift=redshift)
            self.assertTrue(np.all(redshift == meta['REDSHIFT']))

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_simple(self):
        '''Confirm that creating templates works at all'''
        for T in [ELG, LRG, QSO, BGS, STAR, FSTD, MWS_STAR, WD]:
            template_factory = T(wave=self.wave)
            flux, wave, meta = template_factory.make_templates(self.nspec)
            self._check_output_size(flux, wave, meta)

        #- Can also specify minwave, maxwave, dwave
        elg = ELG(minwave=self.wavemin, maxwave=self.wavemax, cdelt=self.dwave)
        flux, wave, meta = elg.make_templates(self.nspec)
        self._check_output_size(flux, wave, meta)

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_OII(self):
        '''Confirm that ELG [OII] flux matches meta table description'''
        wave = np.arange(5000, 9800.1, 0.2)
        flux, ww, meta = ELG(wave=wave).make_templates(
            nmodel=20, nocolorcuts=True, nocontinuum=True,
            logvdisp_meansig = [np.log10(75), 0.0])

        for i in range(len(meta)):
            z = meta['REDSHIFT'][i]
            ii = (3722*(1+z) < wave) & (wave < 3736*(1+z))
            OIIflux = np.sum(flux[i,ii]*np.gradient(wave[ii]))
            self.assertAlmostEqual(OIIflux, meta['OIIFLUX'][i], 2)

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_HBETA(self):
        '''Confirm that BGS H-beta flux matches meta table description'''
        wave = np.arange(5000, 7000.1, 0.2)
        flux, ww, meta = BGS(wave=wave).make_templates(zrange=[0.1,0.4],
            nmodel=20, nocolorcuts=True, nocontinuum=True,
            logvdisp_meansig=[np.log10(75),0.0])

        for i in range(len(meta)):
            z = meta['REDSHIFT'][i]
            ii = (4854*(1+z) < wave) & (wave < 4868*(1+z))
            hbetaflux = np.sum(flux[i,ii]*np.gradient(wave[ii]))
            self.assertAlmostEqual(hbetaflux, meta['HBETAFLUX'][i], 2)

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_stars(self):
        '''Test options specific to star templates'''
        star = STAR(wave=self.wave)
        flux, wave, meta = star.make_templates(self.nspec)
        self._check_output_size(flux, wave, meta)
        self.assertTrue('LOGG' in meta.dtype.names)
        self.assertTrue('TEFF' in meta.dtype.names)
        self.assertTrue('FEH' in meta.dtype.names)

        fstd = FSTD(wave=self.wave)
        flux, wave, meta = fstd.make_templates(self.nspec)
        self._check_output_size(flux, wave, meta)
        self.assertTrue('LOGG' in meta.dtype.names)
        self.assertTrue('TEFF' in meta.dtype.names)
        self.assertTrue('FEH' in meta.dtype.names)

        mwsstar = MWS_STAR(wave=self.wave)
        flux, wave, meta = mwsstar.make_templates(self.nspec)
        self._check_output_size(flux, wave, meta)
        self.assertTrue('LOGG' in meta.dtype.names)
        self.assertTrue('TEFF' in meta.dtype.names)
        self.assertTrue('FEH' in meta.dtype.names)

        wd = WD(wave=self.wave)
        flux, wave, meta = wd.make_templates(self.nspec)
        self._check_output_size(flux, wave, meta)
        self.assertTrue('LOGG' in meta.dtype.names)
        self.assertTrue('TEFF' in meta.dtype.names)
        self.assertTrue('FEH' in meta.dtype.names)

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_random_seed(self):
        '''Test that random seed works to get the same results back'''
        for T in [ELG, LRG, QSO, BGS, STAR, FSTD, MWS_STAR, WD]:
            Tx = T(wave=self.wave)
            flux1, wave1, meta1 = Tx.make_templates(self.nspec, seed=1)
            flux2, wave2, meta2 = Tx.make_templates(self.nspec, seed=1)
            flux3, wave3, meta3 = Tx.make_templates(self.nspec, seed=2)
            self.assertTrue(np.all(flux1==flux2))
            self.assertTrue(np.any(flux1!=flux3))
            self.assertTrue(np.all(wave1==wave2))

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_sne(self):
        '''Test options for adding in SNeIa spectra'''
        for T in [ELG, LRG, BGS]:
            template_factory = T(wave=self.wave, add_SNeIa=True)
            flux, wave, meta = template_factory.make_templates(self.nspec, nocolorcuts=True,
                                                               sne_rfluxratiorange=(0.5, 0.7))
            self._check_output_size(flux, wave, meta)
            self.assertTrue('SNE_TEMPLATEID' in meta.dtype.names)
            self.assertTrue('SNE_RFLUXRATIO' in meta.dtype.names)
            self.assertTrue('SNE_EPOCH' in meta.dtype.names)

    @unittest.skipUnless(desi_basis_templates_available, '$DESI_BASIS_TEMPLATES was not detected.')
    def test_input_meta(self):
        '''Test that input meta table option works.'''
        for T in [ELG, LRG, QSO, BGS, STAR, FSTD, MWS_STAR, WD]:
            Tx = T(wave=self.wave)
            flux1, wave1, meta1 = Tx.make_templates(self.nspec)
            flux2, wave2, meta2 = Tx.make_templates(input_meta=meta1)
            badkeys = list()
            for key in meta1.colnames:
                if not np.all(meta1[key] == meta2[key]):
                    badkeys.append(key)
            self.assertEqual(len(badkeys), 0, 'mismatch for spectral type {} in keys {}'.format(meta1['OBJTYPE'][0], badkeys))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from 語料庫.models import 語料狀況表
from 語料庫.models import 語料表
from 語料庫.管理.校對 import 校對表管理
from django.utils.timezone import now

class 討論表(語料表):

    class Meta:
        proxy = True
        verbose_name = "討論表"
        verbose_name_plural = verbose_name

    def save(self, *args, **kwargs):
        self.檢查時間 = now()
        super().save(*args, **kwargs)

class 討論表管理(校對表管理):
    # change list
    list_display = [
        'id',
        '目標音檔', '狀況',
        '漢字', '本調臺羅', '口語調臺羅',
        '備註開頭',
        '檢查時間',
    ]
    ordering = ['檢查者', 'id']
    list_per_page = 10
    actions = [
        '設定無問題',
    ]
    list_filter = ['音檔__類別', '語料狀況', '校對者', '檢查者', ]

    def 目標音檔(self, obj):
        return ('''<audio controls>
        <source src='/音檔/{}/{}/{}/audio.wav'>
        Your browser does not support the audio element.</audio>'''.format(
            obj.音檔.id, obj.聲音開始時間, obj.聲音結束時間
        ))
    目標音檔.allow_tags = True

    def 設定無問題(self, request, queryset):
        queryset.update(檢查者=request.user, 檢查時間=now())

    # change view
    change_form_template = 'admin/gi2_liau7_khoo3/語料表/custom_change_form.html'
    readonly_fields = ('音檔', '漢字', '本調臺羅', '口語調臺羅', '語料狀況',)
    fieldsets = (
        ('音檔', {
            'fields': ('音檔', ),
            'classes': ['wide']
        }),
        (None, {
            'fields': ('漢字', '本調臺羅', '口語調臺羅', '備註', ),
            'classes': ['wide']
        }),
        (None, {
            'fields': ('語料狀況', ),
            'classes': ['wide']
        }),
    )

    def get_queryset(self, request):
        qs = super(校對表管理, self).get_queryset(request)
        return qs.filter(語料狀況__狀況="愛討論")
<commit_msg>討論表繼承校對表<commit_after>from 語料庫.管理.校對 import 校對表管理
from 語料庫.管理.校對 import 校對表

class 討論表(校對表):

    class Meta:
        proxy = True
        verbose_name = "討論表"
        verbose_name_plural = verbose_name

class 討論表管理(校對表管理):
    # change list
    list_display = [
        'id',
        '目標音檔', '狀況',
        '漢字', '本調臺羅', '口語調臺羅',
        '備註開頭',
        '校對者', '校對時間'
    ]

    def 目標音檔(self, obj):
        return ('''<audio controls>
        <source src='/音檔/{}/{}/{}/audio.wav'>
        Your browser does not support the audio element.</audio>'''.format(
            obj.音檔.id, obj.聲音開始時間, obj.聲音結束時間
        ))
    目標音檔.allow_tags = True

    def get_queryset(self, request):
        qs = super(校對表管理, self).get_queryset(request)
        return qs.filter(語料狀況__狀況="愛討論")
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Tests Prior functions in Pints
#
# This file is part of PINTS.
#  Copyright (c) 2017-2018, University of Oxford.
#  For licensing information, see the LICENSE file distributed with the PINTS
#  software package.
#
from __future__ import division
import unittest
import pints
import numpy as np

class TestPrior(unittest.TestCase):

    def test_normal_prior(self):
        mean = 10
        std = 2
        p = pints.NormalLogPrior(mean, std)

        n = 10000
        r = 6 * np.sqrt(std)

        # Test left half of distribution
        x = np.linspace(mean - r, mean, n)
        px = [p([i]) for i in x]
        self.assertTrue(np.all(px[1:] >= px[:-1]))

        # Test right half of distribution
        y = np.linspace(mean, mean + std, n)
        py = [p([i]) for i in y]
        self.assertTrue(np.all(py[1:] <= py[:-1]))

    def test_normal_prior_sampling(self):
        mean = 10
        std = 2
        p = pints.NormalLogPrior(mean, std)

        d = 1
        n = 1
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        n = 10
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))

        # Very roughly check distribution (main checks are in numpy!)
        np.random.seed(1)
        p = pints.NormalLogPrior(mean, std)
        x = p.sample(10000)
        self.assertTrue(np.abs(mean - x.mean(axis=0)) < 0.1)
        self.assertTrue(np.abs(std - x.std(axis=0)) < 0.01)

    def test_composed_prior(self):
        import pints
        import numpy as np

        m1 = 10
        c1 = 2
        p1 = pints.NormalLogPrior(m1, c1)

        m2 = -50
        c2 = 100
        p2 = pints.NormalLogPrior(m2, c2)

        p = pints.ComposedLogPrior(p1, p2)

        # Test at center
        peak1 = p1([m1])
        peak2 = p2([m2])
        self.assertEqual(p([m1, m2]), peak1 + peak2)

        # Test at random points
        np.random.seed(1)
        for i in range(100):
            x = np.random.normal(m1, c1)
            y = np.random.normal(m2, c2)
            self.assertAlmostEqual(p([x, y]), p1([x]) + p2([y]))

        # Test effect of increasing covariance
        p = [pints.ComposedLogPrior(
            p1, pints.NormalLogPrior(m2, c)) for c in range(1, 10)]
        p = [f([m1, m2]) for f in p]
        self.assertTrue(np.all(p[:-1] > p[1:]))

    def test_composed_prior_sampling(self):

        m1 = 10
        c1 = 2
        p1 = pints.NormalLogPrior(m1, c1)
        m2 = -50
        c2 = 100
        p2 = pints.NormalLogPrior(m2, c2)
        p = pints.ComposedLogPrior(p1, p2)

        p = pints.ComposedLogPrior(p1, p2)
        d = 2
        n = 1
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        n = 10
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        p = pints.ComposedLogPrior(
            p1,
            pints.MultivariateNormalLogPrior([0, 1, 2], np.diag([2, 4, 6])),
            p2,
            p2,
        )
        d = p.dimension()
        self.assertEqual(d, 6)
        n = 1
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        n = 10
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))

    def test_uniform_prior(self):
        lower = np.array([1, 2])
        upper = np.array([10, 20])

        p = pints.UniformLogPrior(lower, upper)
        m = float('-inf')
        self.assertEqual(p([0, 0]), m)
        self.assertEqual(p([0, 5]), m)
        self.assertEqual(p([0, 19]), m)
        self.assertEqual(p([0, 21]), m)
        self.assertEqual(p([5, 0]), m)
        self.assertEqual(p([5, 21]), m)
        self.assertEqual(p([15, 0]), m)
        self.assertEqual(p([15, 5]), m)
        self.assertEqual(p([15, 19]), m)
        self.assertEqual(p([15, 21]), m)
        self.assertEqual(p([10, 10]), m)
        self.assertEqual(p([5, 20]), m)

        w = -np.log(np.product(upper - lower))
        self.assertEqual(p([1, 2]), w)
        self.assertEqual(p([1, 5]), w)
        self.assertEqual(p([1, 20 - 1e-14]), w)
        self.assertEqual(p([5, 5]), w)
        self.assertEqual(p([5, 20 - 1e-14]), w)

    def test_uniform_prior_sampling(self):
        lower = np.array([1, 2])
        upper = np.array([10, 20])
        p = pints.UniformLogPrior(lower, upper)

        # Test output formats
        d = 2
        n = 1
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        n = 10
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))

        p = pints.UniformLogPrior([0], [1])
        d = 1
        n = 1
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        n = 10
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))

        # Roughly check distribution (main checks are in numpy!)
        np.random.seed(1)
        p = pints.UniformLogPrior(lower, upper)
        x = p.sample(10000)
        self.assertTrue(np.all(lower <= x))
        self.assertTrue(np.all(upper > x))
        self.assertTrue(
            np.linalg.norm(x.mean(axis=0) - 0.5 * (upper + lower)) < 0.1)

    def test_multivariate_normal_prior(self):
        # 1d test
        mean = 0
        covariance = 1

        # Input must be a matrix
        self.assertRaises(
            ValueError, pints.MultivariateNormalLogPrior, mean, covariance)
        covariance = [1]
        self.assertRaises(
            ValueError, pints.MultivariateNormalLogPrior, mean, covariance)

        # Basic test
        covariance = [[1]]
        p = pints.MultivariateNormalLogPrior(mean, covariance)
        p([0])
        p([-1])
        p([11])

        # 5d tests
        mean = [1, 2, 3, 4, 5]
        covariance = np.diag(mean)
        p = pints.MultivariateNormalLogPrior(mean, covariance)
        self.assertRaises(ValueError, p, [1, 2, 3])
        p([1, 2, 3, 4, 5])
        p([-1, 2, -3, 4, -5])

    def test_multivariate_normal_sampling(self):
        d = 1
        mean = 2
        covariance = [[1]]
        p = pints.MultivariateNormalLogPrior(mean, covariance)

        n = 1
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        n = 10
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))

        # 5d tests
        d = 5
        mean = np.array([1, 2, 3, 4, 5])
        covariance = np.diag(mean)
        p = pints.MultivariateNormalLogPrior(mean, covariance)
        n = 1
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        n = 10
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))

        # Roughly check distribution (main checks are in numpy!)
        np.random.seed(1)
        p = pints.MultivariateNormalLogPrior(mean, covariance)
        x = p.sample(10000)
        self.assertTrue(np.all(np.abs(mean - x.mean(axis=0)) < 0.1))
        self.assertTrue(np.all(
            np.abs(np.diag(covariance) - x.std(axis=0)**2) < 0.1))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Added test case to test_log_priors<commit_after>#!/usr/bin/env python3
#
# Tests Prior functions in Pints
#
# This file is part of PINTS.
#  Copyright (c) 2017-2018, University of Oxford.
#  For licensing information, see the LICENSE file distributed with the PINTS
#  software package.
#
from __future__ import division
import unittest
import pints
import numpy as np

class TestPrior(unittest.TestCase):

    def test_normal_prior(self):
        mean = 10
        std = 2
        p = pints.NormalLogPrior(mean, std)

        n = 10000
        r = 6 * np.sqrt(std)

        # Test left half of distribution
        x = np.linspace(mean - r, mean, n)
        px = [p([i]) for i in x]
        self.assertTrue(np.all(px[1:] >= px[:-1]))

        # Test right half of distribution
        y = np.linspace(mean, mean + std, n)
        py = [p([i]) for i in y]
        self.assertTrue(np.all(py[1:] <= py[:-1]))

    def test_normal_prior_sampling(self):
        mean = 10
        std = 2
        p = pints.NormalLogPrior(mean, std)

        d = 1
        n = 1
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        n = 10
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))

        # Very roughly check distribution (main checks are in numpy!)
        np.random.seed(1)
        p = pints.NormalLogPrior(mean, std)
        x = p.sample(10000)
        self.assertTrue(np.abs(mean - x.mean(axis=0)) < 0.1)
        self.assertTrue(np.abs(std - x.std(axis=0)) < 0.01)

    def test_composed_prior(self):
        import pints
        import numpy as np

        m1 = 10
        c1 = 2
        p1 = pints.NormalLogPrior(m1, c1)

        m2 = -50
        c2 = 100
        p2 = pints.NormalLogPrior(m2, c2)

        p = pints.ComposedLogPrior(p1, p2)

        # Test at center
        peak1 = p1([m1])
        peak2 = p2([m2])
        self.assertEqual(p([m1, m2]), peak1 + peak2)

        # Test at random points
        np.random.seed(1)
        for i in range(100):
            x = np.random.normal(m1, c1)
            y = np.random.normal(m2, c2)
            self.assertAlmostEqual(p([x, y]), p1([x]) + p2([y]))

        # Test effect of increasing covariance
        p = [pints.ComposedLogPrior(
            p1, pints.NormalLogPrior(m2, c)) for c in range(1, 10)]
        p = [f([m1, m2]) for f in p]
        self.assertTrue(np.all(p[:-1] > p[1:]))

    def test_composed_prior_sampling(self):

        m1 = 10
        c1 = 2
        p1 = pints.NormalLogPrior(m1, c1)
        m2 = -50
        c2 = 100
        p2 = pints.NormalLogPrior(m2, c2)
        p = pints.ComposedLogPrior(p1, p2)

        p = pints.ComposedLogPrior(p1, p2)
        d = 2
        n = 1
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        n = 10
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        p = pints.ComposedLogPrior(
            p1,
            pints.MultivariateNormalLogPrior([0, 1, 2], np.diag([2, 4, 6])),
            p2,
            p2,
        )
        d = p.dimension()
        self.assertEqual(d, 6)
        n = 1
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        n = 10
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))

    def test_uniform_prior(self):
        lower = np.array([1, 2])
        upper = np.array([10, 20])

        # Test normal construction
        p = pints.UniformLogPrior(lower, upper)
        m = float('-inf')
        self.assertEqual(p([0, 0]), m)
        self.assertEqual(p([0, 5]), m)
        self.assertEqual(p([0, 19]), m)
        self.assertEqual(p([0, 21]), m)
        self.assertEqual(p([5, 0]), m)
        self.assertEqual(p([5, 21]), m)
        self.assertEqual(p([15, 0]), m)
        self.assertEqual(p([15, 5]), m)
        self.assertEqual(p([15, 19]), m)
        self.assertEqual(p([15, 21]), m)
        self.assertEqual(p([10, 10]), m)
        self.assertEqual(p([5, 20]), m)

        w = -np.log(np.product(upper - lower))
        self.assertEqual(p([1, 2]), w)
        self.assertEqual(p([1, 5]), w)
        self.assertEqual(p([1, 20 - 1e-14]), w)
        self.assertEqual(p([5, 5]), w)
        self.assertEqual(p([5, 20 - 1e-14]), w)

        # Test from boundaries object
        b = pints.Boundaries(lower, upper)
        p = pints.UniformLogPrior(b)
        m = float('-inf')
        self.assertEqual(p([0, 0]), m)
        self.assertEqual(p([0, 5]), m)
        self.assertEqual(p([0, 19]), m)
        self.assertEqual(p([0, 21]), m)
        self.assertEqual(p([5, 0]), m)
        self.assertEqual(p([5, 21]), m)
        self.assertEqual(p([15, 0]), m)
        self.assertEqual(p([15, 5]), m)
        self.assertEqual(p([15, 19]), m)
        self.assertEqual(p([15, 21]), m)
        self.assertEqual(p([10, 10]), m)
        self.assertEqual(p([5, 20]), m)

        w = -np.log(np.product(upper - lower))
        self.assertEqual(p([1, 2]), w)
        self.assertEqual(p([1, 5]), w)
        self.assertEqual(p([1, 20 - 1e-14]), w)
        self.assertEqual(p([5, 5]), w)
        self.assertEqual(p([5, 20 - 1e-14]), w)

        # Test bad constructor
        self.assertRaises(ValueError, pints.UniformLogPrior, lower)

    def test_uniform_prior_sampling(self):
        lower = np.array([1, 2])
        upper = np.array([10, 20])
        p = pints.UniformLogPrior(lower, upper)

        # Test output formats
        d = 2
        n = 1
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        n = 10
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))

        p = pints.UniformLogPrior([0], [1])
        d = 1
        n = 1
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        n = 10
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))

        # Roughly check distribution (main checks are in numpy!)
        np.random.seed(1)
        p = pints.UniformLogPrior(lower, upper)
        x = p.sample(10000)
        self.assertTrue(np.all(lower <= x))
        self.assertTrue(np.all(upper > x))
        self.assertTrue(
            np.linalg.norm(x.mean(axis=0) - 0.5 * (upper + lower)) < 0.1)

    def test_multivariate_normal_prior(self):
        # 1d test
        mean = 0
        covariance = 1

        # Input must be a matrix
        self.assertRaises(
            ValueError, pints.MultivariateNormalLogPrior, mean, covariance)
        covariance = [1]
        self.assertRaises(
            ValueError, pints.MultivariateNormalLogPrior, mean, covariance)

        # Basic test
        covariance = [[1]]
        p = pints.MultivariateNormalLogPrior(mean, covariance)
        p([0])
        p([-1])
        p([11])

        # 5d tests
        mean = [1, 2, 3, 4, 5]
        covariance = np.diag(mean)
        p = pints.MultivariateNormalLogPrior(mean, covariance)
        self.assertRaises(ValueError, p, [1, 2, 3])
        p([1, 2, 3, 4, 5])
        p([-1, 2, -3, 4, -5])

    def test_multivariate_normal_sampling(self):
        d = 1
        mean = 2
        covariance = [[1]]
        p = pints.MultivariateNormalLogPrior(mean, covariance)

        n = 1
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        n = 10
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))

        # 5d tests
        d = 5
        mean = np.array([1, 2, 3, 4, 5])
        covariance = np.diag(mean)
        p = pints.MultivariateNormalLogPrior(mean, covariance)
        n = 1
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))
        n = 10
        x = p.sample(n)
        self.assertEqual(x.shape, (n, d))

        # Roughly check distribution (main checks are in numpy!)
        np.random.seed(1)
        p = pints.MultivariateNormalLogPrior(mean, covariance)
        x = p.sample(10000)
        self.assertTrue(np.all(np.abs(mean - x.mean(axis=0)) < 0.1))
        self.assertTrue(np.all(
            np.abs(np.diag(covariance) - x.std(axis=0)**2) < 0.1))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Test the speed of rapidly updating multiple plot curves
"""

## Add path to library (just for examples; you do not need this)
import initExample

from pyqtgraph.Qt import QtGui, QtCore
import numpy as np
import pyqtgraph as pg
from pyqtgraph.ptime import time
#QtGui.QApplication.setGraphicsSystem('raster')
app = QtGui.QApplication([])
#mw = QtGui.QMainWindow()
#mw.resize(800,800)

p = pg.plot()
p.setWindowTitle('pyqtgraph example: MultiPlotSpeedTest')
#p.setRange(QtCore.QRectF(0, -10, 5000, 20))
p.setLabel('bottom', 'Index', units='B')

nPlots = 100
nSamples = 500
#curves = [p.plot(pen=(i,nPlots*1.3)) for i in range(nPlots)]
curves = []
for i in range(nPlots):
    c = pg.PlotCurveItem(pen=(i,nPlots*1.3))
    p.addItem(c)
    c.setPos(0,i*6)
    curves.append(c)

p.setYRange(0, nPlots*6)
p.setXRange(0, nSamples)
p.resize(600,900)

rgn = pg.LinearRegionItem([nSamples/5.,nSamples/3.])
p.addItem(rgn)

data = np.random.normal(size=(nPlots*23,nSamples))
ptr = 0
lastTime = time()
fps = None
count = 0
def update():
    global curve, data, ptr, p, lastTime, fps, nPlots, count
    count += 1
    #print "---------", count
    for i in range(nPlots):
        curves[i].setData(data[(ptr+i)%data.shape[0]])

    #print "   setData done."
    ptr += nPlots
    now = time()
    dt = now - lastTime
    lastTime = now
    if fps is None:
        fps = 1.0/dt
    else:
        s = np.clip(dt*3., 0, 1)
        fps = fps * (1-s) + (1.0/dt) * s
    p.setTitle('%0.2f fps' % fps)
    #app.processEvents()  ## force complete redraw for every plot
timer = QtCore.QTimer()
timer.timeout.connect(update)
timer.start(0)

## Start Qt event loop unless running in interactive mode.
if __name__ == '__main__':
    import sys
    if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
        QtGui.QApplication.instance().exec_()
<commit_msg>Refactor MultiPlotSpeedTest.py<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Test the speed of rapidly updating multiple plot curves
"""

## Add path to library (just for examples; you do not need this)
import initExample

from pyqtgraph.Qt import QtGui, QtCore
import numpy as np
import pyqtgraph as pg
from pyqtgraph.ptime import time
app = QtGui.QApplication([])

plot = pg.plot()
plot.setWindowTitle('pyqtgraph example: MultiPlotSpeedTest')
plot.setLabel('bottom', 'Index', units='B')

nPlots = 100
nSamples = 500
curves = []
for idx in range(nPlots):
    curve = pg.PlotCurveItem(pen=(idx,nPlots*1.3))
    plot.addItem(curve)
    curve.setPos(0,idx*6)
    curves.append(curve)

plot.setYRange(0, nPlots*6)
plot.setXRange(0, nSamples)
plot.resize(600,900)

rgn = pg.LinearRegionItem([nSamples/5.,nSamples/3.])
plot.addItem(rgn)

data = np.random.normal(size=(nPlots*23,nSamples))
ptr = 0
lastTime = time()
fps = None
count = 0
def update():
    global curve, data, ptr, plot, lastTime, fps, nPlots, count
    count += 1

    for i in range(nPlots):
        curves[i].setData(data[(ptr+i)%data.shape[0]])

    ptr += nPlots
    now = time()
    dt = now - lastTime
    lastTime = now
    if fps is None:
        fps = 1.0/dt
    else:
        s = np.clip(dt*3., 0, 1)
        fps = fps * (1-s) + (1.0/dt) * s
    plot.setTitle('%0.2f fps' % fps)
    #app.processEvents()  ## force complete redraw for every plot
timer = QtCore.QTimer()
timer.timeout.connect(update)
timer.start(0)

## Start Qt event loop unless running in interactive mode.
if __name__ == '__main__':
    import sys
    if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
        QtGui.QApplication.instance().exec_()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# test_processing.py
#
# purpose:  Test processing step from ctd.py
# author:   Filipe P. A. Fernandes
# e-mail:   ocefpaf@gmail
# web:      http://ocefpaf.tiddlyspot.com/
# created:  01-Mar-2013
# modified: Fri 19 Jul 2013 01:48:30 PM BRT
#
# obs:
#

import unittest

from ctd import DataFrame

class DataFrameTests(unittest.TestCase):
    def setUp(self):
        self.xbt = DataFrame.from_edf(fname='data/XBT.EDF.zip',
                                      compression='zip')
        self.fsi = DataFrame.from_fsi(fname='data/FSI.txt.gz',
                                      compression='gzip', skiprows=9)
        self.cnv = DataFrame.from_cnv(fname='data/CTD_big.cnv.bz2',
                                      compression='bz2')

    def tearDown(self):
        unittest.TestCase.tearDown(self)

    def test_split_cnv(self):
        self.assertIsInstance(self.cnv.split(), tuple)

    def test_split_fsi(self):
        self.assertIsInstance(self.fsi.split(), tuple)

def main():
    unittest.main()

if __name__ == '__main__':
    main()
<commit_msg>Added tests for Pre-Processing stages.<commit_after># -*- coding: utf-8 -*-
#
# test_processing.py
#
# purpose:  Test processing step from ctd.py
# author:   Filipe P. A. Fernandes
# e-mail:   ocefpaf@gmail
# web:      http://ocefpaf.tiddlyspot.com/
# created:  01-Mar-2013
# modified: Wed 24 Jul 2013 08:17:36 PM BRT
#
# obs:
#

import unittest
import numpy as np
from ctd import DataFrame, lp_filter

class BasicProcessingTests(unittest.TestCase):
    def setUp(self):
        self.raw = DataFrame.from_cnv('data/CTD-spiked-unfiltered.cnv.bz2',
                                      compression='bz2')
        self.prc = DataFrame.from_cnv(fname='data/CTD-spiked-filtered.cnv.bz2',
                                      compression='bz2')

    def tearDown(self):
        unittest.TestCase.tearDown(self)

    # Split.
    def test_split_cnv(self):
        self.assertIsInstance(self.raw.split(), tuple)

    # Despike.
    def test_despike(self):
        dirty = self.prc['c0s/m'].split()[0]  # Looking at downcast only.
        clean = dirty.despike(n1=2, n2=20, block=500)
        spikes = clean.isnull()
        equal = (dirty[~spikes] == clean[~spikes]).all()
        self.assertTrue(spikes.any() and equal)

    # Filter.
    def test_lp_filter_2(self):
        kw = dict(sample_rate=24.0, time_constant=0.15)
        unfiltered = self.raw.index.values
        filtered = lp_filter(unfiltered, **kw)
        # FIXME: Not a good test...
        np.testing.assert_almost_equal(filtered, self.prc.index.values,
                                       decimal=1)

    # Pressure check.
    def test_press_check(self):
        unchecked = self.raw['t090c']
        press_checked = unchecked.press_check()
        reversals = press_checked.isnull()
        equal = (unchecked[~reversals] == press_checked[~reversals]).all()
        self.assertTrue(reversals.any() and equal)

    def test_bindata(self):
        delta = 1.
        down = self.prc['t090c'].split()[0]
        down = down.bindata(delta=delta)
        self.assertTrue(np.unique(np.diff(down.index.values)) == delta)

    # PostProcessingTests.
    def test_smooth(self):
        pass

    def test_mixed_layer_depth(self):
        pass

    def test_barrier_layer_thickness(self):
        pass

def main():
    unittest.main()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># This file is part of beets.
# Copyright 2015, Thomas Scholtes
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

from __future__ import (division, absolute_import, print_function,
                        unicode_literals)

from test._common import unittest
from test.helper import TestHelper, has_program

from beets.mediafile import MediaFile

try:
    import gi
    gi.require_version('Gst', '1.0')
    GST_AVAILABLE = True
except ImportError, ValueError:
    GST_AVAILABLE = False

if any(has_program(cmd, ['-v']) for cmd in ['mp3gain', 'aacgain']):
    GAIN_PROG_AVAILABLE = True
else:
    GAIN_PROG_AVAILABLE = False

class ReplayGainCliTestBase(TestHelper):

    def setUp(self):
        self.setup_beets()

        try:
            self.load_plugins('replaygain')
        except:
            self.teardown_beets()
            self.unload_plugins()
            raise

        self.config['replaygain']['backend'] = self.backend
        album = self.add_album_fixture(2)
        for item in album.items():
            self._reset_replaygain(item)

    def tearDown(self):
        self.teardown_beets()
        self.unload_plugins()

    def _reset_replaygain(self, item):
        item['rg_track_peak'] = None
        item['rg_track_gain'] = None
        item['rg_album_gain'] = None
        item['rg_album_gain'] = None
        item.write()
        item.store()

    def test_cli_saves_track_gain(self):
        for item in self.lib.items():
            self.assertIsNone(item.rg_track_peak)
            self.assertIsNone(item.rg_track_gain)
            mediafile = MediaFile(item.path)
            self.assertIsNone(mediafile.rg_track_peak)
            self.assertIsNone(mediafile.rg_track_gain)

        self.run_command('replaygain')
        for item in self.lib.items():
            self.assertIsNotNone(item.rg_track_peak)
            self.assertIsNotNone(item.rg_track_gain)
            mediafile = MediaFile(item.path)
            self.assertAlmostEqual(
                mediafile.rg_track_peak, item.rg_track_peak, places=6)
            self.assertAlmostEqual(
                mediafile.rg_track_gain, item.rg_track_gain, places=2)

    def test_cli_skips_calculated_tracks(self):
        self.run_command('replaygain')
        item = self.lib.items()[0]
        peak = item.rg_track_peak
        item.rg_track_gain = 0.0
        self.run_command('replaygain')
        self.assertEqual(item.rg_track_gain, 0.0)
        self.assertEqual(item.rg_track_peak, peak)

    def test_cli_saves_album_gain_to_file(self):
        for item in self.lib.items():
            mediafile = MediaFile(item.path)
            self.assertIsNone(mediafile.rg_album_peak)
            self.assertIsNone(mediafile.rg_album_gain)

        self.run_command('replaygain', '-a')

        peaks = []
        gains = []
        for item in self.lib.items():
            mediafile = MediaFile(item.path)
            peaks.append(mediafile.rg_album_peak)
            gains.append(mediafile.rg_album_gain)

        # Make sure they are all the same
        self.assertEqual(max(peaks), min(peaks))
        self.assertEqual(max(gains), min(gains))

        self.assertNotEqual(max(gains), 0.0)
        self.assertNotEqual(max(peaks), 0.0)

@unittest.skipIf(not GST_AVAILABLE, 'gstreamer cannot be found')
class ReplayGainGstCliTest(ReplayGainCliTestBase, unittest.TestCase):
    backend = u'gstreamer'

@unittest.skipIf(not GAIN_PROG_AVAILABLE, 'no *gain command found')
class ReplayGainCmdCliTest(ReplayGainCliTestBase, unittest.TestCase):
    backend = u'command'

def suite():
    return unittest.TestLoader().loadTestsFromName(__name__)

if __name__ == b'__main__':
    unittest.main(defaultTest='suite')
<commit_msg>Replaygain tests: more careful plugins unloading<commit_after># This file is part of beets.
# Copyright 2015, Thomas Scholtes
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

from __future__ import (division, absolute_import, print_function,
                        unicode_literals)

from test._common import unittest
from test.helper import TestHelper, has_program

from beets.mediafile import MediaFile

try:
    import gi
    gi.require_version('Gst', '1.0')
    GST_AVAILABLE = True
except ImportError, ValueError:
    GST_AVAILABLE = False

if any(has_program(cmd, ['-v']) for cmd in ['mp3gain', 'aacgain']):
    GAIN_PROG_AVAILABLE = True
else:
    GAIN_PROG_AVAILABLE = False

class ReplayGainCliTestBase(TestHelper):

    def setUp(self):
        self.setup_beets()

        try:
            self.load_plugins('replaygain')
        except:
            import sys
            # store exception info so an error in teardown does not swallow it
            exc_info = sys.exc_info()
            try:
                self.teardown_beets()
                self.unload_plugins()
            except:
                # if load_plugins() failed then setup is incomplete and
                # teardown operations may fail. In particular # {Item,Album}
                # may not have the _original_types attribute in unload_plugins
                pass
            raise exc_info[1], None, exc_info[2]

        self.config['replaygain']['backend'] = self.backend
        album = self.add_album_fixture(2)
        for item in album.items():
            self._reset_replaygain(item)

    def tearDown(self):
        self.teardown_beets()
        self.unload_plugins()

    def _reset_replaygain(self, item):
        item['rg_track_peak'] = None
        item['rg_track_gain'] = None
        item['rg_album_gain'] = None
        item['rg_album_gain'] = None
        item.write()
        item.store()

    def test_cli_saves_track_gain(self):
        for item in self.lib.items():
            self.assertIsNone(item.rg_track_peak)
            self.assertIsNone(item.rg_track_gain)
            mediafile = MediaFile(item.path)
            self.assertIsNone(mediafile.rg_track_peak)
            self.assertIsNone(mediafile.rg_track_gain)

        self.run_command('replaygain')
        for item in self.lib.items():
            self.assertIsNotNone(item.rg_track_peak)
            self.assertIsNotNone(item.rg_track_gain)
            mediafile = MediaFile(item.path)
            self.assertAlmostEqual(
                mediafile.rg_track_peak, item.rg_track_peak, places=6)
            self.assertAlmostEqual(
                mediafile.rg_track_gain, item.rg_track_gain, places=2)

    def test_cli_skips_calculated_tracks(self):
        self.run_command('replaygain')
        item = self.lib.items()[0]
        peak = item.rg_track_peak
        item.rg_track_gain = 0.0
        self.run_command('replaygain')
        self.assertEqual(item.rg_track_gain, 0.0)
        self.assertEqual(item.rg_track_peak, peak)

    def test_cli_saves_album_gain_to_file(self):
        for item in self.lib.items():
            mediafile = MediaFile(item.path)
            self.assertIsNone(mediafile.rg_album_peak)
            self.assertIsNone(mediafile.rg_album_gain)

        self.run_command('replaygain', '-a')

        peaks = []
        gains = []
        for item in self.lib.items():
            mediafile = MediaFile(item.path)
            peaks.append(mediafile.rg_album_peak)
            gains.append(mediafile.rg_album_gain)

        # Make sure they are all the same
        self.assertEqual(max(peaks), min(peaks))
        self.assertEqual(max(gains), min(gains))

        self.assertNotEqual(max(gains), 0.0)
        self.assertNotEqual(max(peaks), 0.0)

@unittest.skipIf(not GST_AVAILABLE, 'gstreamer cannot be found')
class ReplayGainGstCliTest(ReplayGainCliTestBase, unittest.TestCase):
    backend = u'gstreamer'

@unittest.skipIf(not GAIN_PROG_AVAILABLE, 'no *gain command found')
class ReplayGainCmdCliTest(ReplayGainCliTestBase, unittest.TestCase):
    backend = u'command'

def suite():
    return unittest.TestLoader().loadTestsFromName(__name__)

if __name__ == b'__main__':
    unittest.main(defaultTest='suite')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Display number of pending updates for Arch Linux.

Configuration parameters:
    cache_timeout: How often we refresh this module in seconds (default 600)
    format: display format for this module, see Examples below (default None)
    hide_if_zero: Don't show on bar if True (default False)
    include_aur: Check for AUR updates with auracle or yay (default False)

Format placeholders:
    {aur} Number of pending aur updates
    {pacman} Number of pending pacman updates
    {total} Total updates pending

Requires:
    pacman-contrib: contributed scripts and tools for pacman systems
    auracle: a flexible command line client for arch linux's user repository
    yay: yet another yogurt. pacman wrapper and aur helper written in go

Examples:
```
# default formats
arch_updates {
    format = 'UPD: {pacman}'        # if include_aur is False
    format = 'UPD: {pacman}/{aur}'  # if include_aur is True
}
```

@author Iain Tatch <iain.tatch@gmail.com>
@license BSD

SAMPLE OUTPUT
{'full_text': 'UPD: 5'}

arch_updates_aur
{'full_text': 'UPD: 15/4'}
"""

FORMAT_PACMAN_ONLY = "UPD: {pacman}"
FORMAT_PACMAN_AND_AUR = "UPD: {pacman}/{aur}"
STRING_NOT_INSTALLED = "{} not installed"

class Py3status:
    """
    """

    # available configuration parameters
    cache_timeout = 600
    format = None
    hide_if_zero = False
    include_aur = False

    def post_config_hook(self):
        if not self.format:
            if not self.include_aur:
                self.format = FORMAT_PACMAN_ONLY
            else:
                self.format = FORMAT_PACMAN_AND_AUR
        self.include_aur = self.py3.format_contains(self.format, "aur")
        self.include_pacman = self.py3.format_contains(self.format, "pacman")
        if self.py3.format_contains(self.format, "total"):
            self.include_aur = True
            self.include_pacman = True

        if self.include_aur:
            aurs = ["auracle", "yay", "cower"]
            command = self.py3.check_commands(aurs)
            if command:
                self._check_aur_updates = getattr(
                    self, "_check_aur_updates_{}".format(command)
                )
            else:
                self.include_aur = False
                self.py3.notify_user(STRING_NOT_INSTALLED.format(", ".join(aurs)))

    def arch_updates(self):
        pacman_updates = aur_updates = total = None

        if self.include_pacman:
            pacman_updates = self._check_pacman_updates()

        if self.include_aur:
            aur_updates = self._check_aur_updates()

        if pacman_updates is not None or aur_updates is not None:
            total = (pacman_updates or 0) + (aur_updates or 0)

        if self.hide_if_zero and total == 0:
            full_text = ""
        else:
            full_text = self.py3.safe_format(
                self.format,
                {"aur": aur_updates, "pacman": pacman_updates, "total": total},
            )
        return {
            "cached_until": self.py3.time_in(self.cache_timeout),
            "full_text": full_text,
        }

    def _check_pacman_updates(self):
        try:
            updates = self.py3.command_output(["checkupdates"])
            return len(updates.splitlines())
        except self.py3.CommandError:
            return None

    def _check_aur_updates_auracle(self):
        try:
            updates = self.py3.command_output(["auracle", "sync"])
            return len(updates.splitlines())
        except self.py3.CommandError:
            return None

    def _check_aur_updates_cower(self):
        try:
            self.py3.command_output(["cower", "-u"])
        except self.py3.CommandError as ce:
            return len(ce.output.splitlines())
        return None

    def _check_aur_updates_yay(self):
        try:
            updates = self.py3.command_output(["yay", "-Qua"])
            return len(updates.splitlines())
        except self.py3.CommandError:
            return None

if __name__ == "__main__":
    """
    Run module in test mode.
    """
    from py3status.module_test import module_test

    module_test(Py3status)
<commit_msg>arch_updates: remove include_aur in favor of auto, refactoring (#1727)<commit_after># -*- coding: utf-8 -*-
"""
Display number of pending updates for Arch Linux.

Configuration parameters:
    cache_timeout: refresh interval for this module (default 600)
    format: display format for this module, otherwise auto (default None)
    hide_if_zero: don't show on bar if True (default False)

Format placeholders:
    {aur} Number of pending aur updates
    {pacman} Number of pending pacman updates
    {total} Total updates pending

Requires:
    pacman-contrib: contributed scripts and tools for pacman systems
    auracle: a flexible command line client for arch linux's user repository
    yay: yet another yogurt. pacman wrapper and aur helper written in go
```

@author Iain Tatch <iain.tatch@gmail.com>
@license BSD

SAMPLE OUTPUT
{'full_text': 'UPD: 5'}

aur
{'full_text': 'UPD: 15/4'}
"""

STRING_NOT_INSTALLED = "{} not installed"

class Py3status:
    """
    """

    # available configuration parameters
    cache_timeout = 600
    format = None
    hide_if_zero = False

    class Meta:
        deprecated = {"remove": [{"param": "include_aur", "msg": "obsolete"}]}

    def post_config_hook(self):
        helper = {
            "pacman": self.py3.check_commands(["checkupdates"]),
            "aur": self.py3.check_commands(["auracle", "yay", "cower"]),
        }
        if self.format:
            placeholders = self.py3.get_placeholders_list(self.format)
            if "total" in placeholders:
                pass
            elif not any(helper.values()):
                raise Exception(STRING_NOT_INSTALLED.format("pacman, aur"))
            else:
                for name in helper:
                    if name not in placeholders:
                        helper[name] = None
                    elif not helper[name]:
                        raise Exception(STRING_NOT_INSTALLED.format(name))
        elif all(helper.values()):
            self.format = "UPD: {pacman}/{aur}"
        elif helper["pacman"]:
            self.format = "UPD: {pacman}"
        elif helper["aur"]:
            self.format = "UPD: ?/{aur}"
        else:
            raise Exception(STRING_NOT_INSTALLED.format("pacman, aur"))

        for key in helper:
            value = getattr(self, "_get_{}_updates".format(helper[key]), None)
            setattr(self, "_get_{}_updates".format(key), value)

    def _get_checkupdates_updates(self):
        try:
            updates = self.py3.command_output(["checkupdates"])
            return len(updates.splitlines())
        except self.py3.CommandError:
            return None

    def _get_auracle_updates(self):
        try:
            updates = self.py3.command_output(["auracle", "sync"])
            return len(updates.splitlines())
        except self.py3.CommandError:
            return None

    def _get_cower_updates(self):
        try:
            self.py3.command_output(["cower", "-u"])
            return None
        except self.py3.CommandError as ce:
            return len(ce.output.splitlines())

    def _get_yay_updates(self):
        try:
            updates = self.py3.command_output(["yay", "-Qua"])
            return len(updates.splitlines())
        except self.py3.CommandError:
            return None

    def arch_updates(self):
        pacman, aur, total, full_text = None, None, None, None
        if self._get_pacman_updates:
            pacman = self._get_pacman_updates()
        if self._get_aur_updates:
            aur = self._get_aur_updates()

        if pacman is not None or aur is not None:
            total = (pacman or 0) + (aur or 0)

        if not (self.hide_if_zero and not total):
            arch_data = {"aur": aur, "pacman": pacman, "total": total}
            full_text = self.py3.safe_format(self.format, arch_data)

        return {
            "cached_until": self.py3.time_in(self.cache_timeout),
            "full_text": full_text,
        }

if __name__ == "__main__":
    """
    Run module in test mode.
    """
    from py3status.module_test import module_test

    module_test(Py3status)
<|endoftext|>


<commit_before>from rest_framework import serializers

from .models import (
    Chorus,
    Quartet,
    Contest,
    Convention,
    Contestant,
    Performance,
    Group,
)

class PerformanceSerializer(serializers.ModelSerializer):
    round = serializers.CharField(
        source='get_round_display',
    )

    class Meta:
        model = Performance
        fields = (
            'id',
            'slug',
            'round',
            'queue',
            'session',
            'stagetime',
            'place',
            'song1',
            'mus1',
            'prs1',
            'sng1',
            'song2',
            'mus2',
            'prs2',
            'sng2',
            'men',
            'mus1_rata',
            'prs1_rata',
            'sng1_rata',
            'song1_raw',
            'song1_rata',
            'mus2_rata',
            'prs2_rata',
            'sng2_rata',
            'song2_raw',
            'song2_rata',
            'total_raw',
            'total_rata',
        )

class GroupSerializer(serializers.ModelSerializer):

    district = serializers.CharField(
        source='get_district_tag_display',
    )

    class Meta:
        model = Group
        fields = (
            'id',
            'slug',
            'name',
            'district',
            'location',
            'website',
            'facebook',
            'twitter',
            'email',
            'phone',
            'picture',
            'description',
        )

class ContestantContestSerializer(serializers.ModelSerializer):
    performances = PerformanceSerializer(
        many=True,
        read_only=True,
    )

    group = GroupSerializer(
        read_only=True,
    )

    class Meta:
        model = Contestant
        fields = (
            'id',
            'slug',
            'group',
            'seed',
            'prelim',
            'place',
            'score',
            'performances',
        )

class ContestSerializer(serializers.ModelSerializer):
    contestants = ContestantContestSerializer(
        many=True,
    )

    level = serializers.CharField(
        source='get_level_display',
    )

    kind = serializers.CharField(
        source='get_kind_display',
    )

    year = serializers.CharField(
        source='get_year_display',
    )

    district = serializers.CharField(
        source='get_district_display',
    )

    class Meta:
        model = Contest
        fields = (
            'id',
            'slug',
            'level',
            'kind',
            'year',
            'district',
            'panel',
            'scoresheet_pdf',
            'contestants',
        )

class ContestantGroupSerializer(serializers.ModelSerializer):
    performances = PerformanceSerializer(
        many=True,
        read_only=True,
    )

    contest = serializers.StringRelatedField(
        read_only=True,
    )

    class Meta:
        model = Contestant
        fields = (
            'id',
            'slug',
            'contest',
            'seed',
            'prelim',
            'place',
            'score',
            'performances',
        )

class ConventionSerializer(serializers.ModelSerializer):
    contests = ContestSerializer(
        many=True,
    )

    class Meta:
        model = Convention
        lookup_field = 'slug'
        fields = (
            'id',
            'url',
            'slug',
            'name',
            'dates',
            'timezone',
            'contests',
        )

class ChorusSerializer(serializers.ModelSerializer):
    contestants = ContestantGroupSerializer(
        many=True,
        read_only=True,
    )

    chapterName = serializers.CharField(
        source='chapter_name',
    )

    chapterCode = serializers.CharField(
        source='chapter_code',
    )

    class Meta:
        model = Chorus
        lookup_field = 'slug'
        fields = (
            'id',
            'url',
            'slug',
            'name',
            'location',
            'website',
            'facebook',
            'twitter',
            'email',
            'phone',
            'picture',
            'description',
            'notes',
            'director',
            'chapterName',
            'chapterCode',
            'contestants',
        )

class QuartetSerializer(serializers.ModelSerializer):
    contestants = ContestantGroupSerializer(
        many=True,
        read_only=True,
    )

    lead = serializers.StringRelatedField(
        read_only=True,
    )

    tenor = serializers.StringRelatedField(
        read_only=True,
    )

    baritone = serializers.StringRelatedField(
        read_only=True,
    )

    bass = serializers.StringRelatedField(
        read_only=True,
    )

    class Meta:
        model = Quartet
        lookup_field = 'slug'
        fields = (
            'id',
            'url',
            'slug',
            'name',
            'slug',
            'location',
            'website',
            'facebook',
            'twitter',
            'email',
            'phone',
            'picture',
            'description',
            'notes',
            'lead',
            'tenor',
            'baritone',
            'bass',
            # 'district',
            'contestants',
        )
<commit_msg>Fix serializer<commit_after>from rest_framework import serializers

from .models import (
    Chorus,
    Quartet,
    Contest,
    Convention,
    Contestant,
    Performance,
    Group,
)

class PerformanceSerializer(serializers.ModelSerializer):
    round = serializers.CharField(
        source='get_round_display',
    )

    class Meta:
        model = Performance
        fields = (
            'id',
            'slug',
            'round',
            'queue',
            'session',
            'stagetime',
            'place',
            'song1',
            'mus1',
            'prs1',
            'sng1',
            'song2',
            'mus2',
            'prs2',
            'sng2',
            'men',
            'mus1_rata',
            'prs1_rata',
            'sng1_rata',
            'song1_raw',
            'song1_rata',
            'mus2_rata',
            'prs2_rata',
            'sng2_rata',
            'song2_raw',
            'song2_rata',
            'total_raw',
            'total_rata',
        )

class GroupSerializer(serializers.ModelSerializer):

    district = serializers.CharField(
        source='get_district_display',
    )

    class Meta:
        model = Group
        fields = (
            'id',
            'slug',
            'name',
            'district',
            'location',
            'website',
            'facebook',
            'twitter',
            'email',
            'phone',
            'picture',
            'description',
        )

class ContestantContestSerializer(serializers.ModelSerializer):
    performances = PerformanceSerializer(
        many=True,
        read_only=True,
    )

    group = GroupSerializer(
        read_only=True,
    )

    class Meta:
        model = Contestant
        fields = (
            'id',
            'slug',
            'group',
            'seed',
            'prelim',
            'place',
            'score',
            'performances',
        )

class ContestSerializer(serializers.ModelSerializer):
    contestants = ContestantContestSerializer(
        many=True,
    )

    level = serializers.CharField(
        source='get_level_display',
    )

    kind = serializers.CharField(
        source='get_kind_display',
    )

    year = serializers.CharField(
        source='get_year_display',
    )

    district = serializers.CharField(
        source='get_district_display',
    )

    class Meta:
        model = Contest
        fields = (
            'id',
            'slug',
            'level',
            'kind',
            'year',
            'district',
            'panel',
            'scoresheet_pdf',
            'contestants',
        )

class ContestantGroupSerializer(serializers.ModelSerializer):
    performances = PerformanceSerializer(
        many=True,
        read_only=True,
    )

    contest = serializers.StringRelatedField(
        read_only=True,
    )

    class Meta:
        model = Contestant
        fields = (
            'id',
            'slug',
            'contest',
            'seed',
            'prelim',
            'place',
            'score',
            'performances',
        )

class ConventionSerializer(serializers.ModelSerializer):
    contests = ContestSerializer(
        many=True,
    )

    class Meta:
        model = Convention
        lookup_field = 'slug'
        fields = (
            'id',
            'url',
            'slug',
            'name',
            'dates',
            'timezone',
            'contests',
        )

class ChorusSerializer(serializers.ModelSerializer):
    contestants = ContestantGroupSerializer(
        many=True,
        read_only=True,
    )

    chapterName = serializers.CharField(
        source='chapter_name',
    )

    chapterCode = serializers.CharField(
        source='chapter_code',
    )

    class Meta:
        model = Chorus
        lookup_field = 'slug'
        fields = (
            'id',
            'url',
            'slug',
            'name',
            'location',
            'website',
            'facebook',
            'twitter',
            'email',
            'phone',
            'picture',
            'description',
            'notes',
            'director',
            'chapterName',
            'chapterCode',
            'contestants',
        )

class QuartetSerializer(serializers.ModelSerializer):
    contestants = ContestantGroupSerializer(
        many=True,
        read_only=True,
    )

    lead = serializers.StringRelatedField(
        read_only=True,
    )

    tenor = serializers.StringRelatedField(
        read_only=True,
    )

    baritone = serializers.StringRelatedField(
        read_only=True,
    )

    bass = serializers.StringRelatedField(
        read_only=True,
    )

    class Meta:
        model = Quartet
        lookup_field = 'slug'
        fields = (
            'id',
            'url',
            'slug',
            'name',
            'slug',
            'location',
            'website',
            'facebook',
            'twitter',
            'email',
            'phone',
            'picture',
            'description',
            'notes',
            'lead',
            'tenor',
            'baritone',
            'bass',
            # 'district',
            'contestants',
        )
<|endoftext|>


<commit_before>from sequana import snaketools, sequana_data
from sequana.snaketools import DOTParser
import os, shutil
import tempfile
from sequana import Module, SequanaConfig
from easydev import TempFile
import subprocess

def test_dot_parser():
    s = DOTParser(sequana_data("test_dag.dot", "testing"))
    s.add_urls(mapper={'bwa_fix': "test.html"})
    try:os.remove("test_dag.ann.dot")
    except:pass

    s.mode = "v1"
    s.add_urls(mapper={'bwa_fix': "test.html"})
    try:os.remove("test_dag.ann.dot")
    except:pass

def test_md5():
    from sequana import Module
    m = Module("compressor")
    data = m.md5()

def test_modules():
    assert "dag" in snaketools.modules.keys()
    assert snaketools.modules['dag'].endswith("dag.rules")

def test_getcleanup_rules():
    filename =  snaketools.modules['fastq_sampling']
    try:
        snaketools.get_cleanup_rules(filename)
    except:
        pass

def test_snakemake_stats():
    # this is created using snakemake with the option "--stats stats.txt"
    s = snaketools.SnakeMakeStats(sequana_data("test_snakemake_stats.txt"))
    s.plot()
    with TempFile() as fout:
        s.plot_and_save(filename=fout.name, outputdir=None)

    with tempfile.TemporaryDirectory() as tempdir:
        s.plot_and_save(filename="test.png", outputdir=tempdir)

def test_plot_stats():

    with tempfile.TemporaryDirectory() as indir:
        shutil.copy(sequana_data("test_snakemake_stats.txt"),
            indir + "/stats.txt")
        with tempfile.TemporaryDirectory() as outdir:
            snaketools.plot_stats(indir, outdir)
    snaketools.plot_stats("dummy", "dummy")

def test_module():
    # a rule without README
    m = snaketools.Module('mark_duplicates_dynamic')
    m.description
    print(m)
    m # test __repr__
    m.__repr__()
    m.path
    m.snakefile
    m.overview
    # a rule with README
    m = snaketools.Module('dag')
    m.description
    m.overview
    assert m.is_executable()
    m.check()

    # a pipeline
    m = snaketools.Module('compressor')
    m.is_executable()
    m.check()
    m.snakefile
    m.name
    m
    print(m)
    assert m.cluster_config.endswith("cluster_config.json")
    assert m.schema_config.endswith("schema.yaml")

def test_valid_config():
    config = snaketools.SequanaConfig(None)

    s = snaketools.Module("compressor")
    config = snaketools.SequanaConfig(s.config)

    from easydev import TempFile
    with TempFile() as fh:
        config.save(fh.name)

def test_sequana_config():
    s = snaketools.Module("compressor")
    config = snaketools.SequanaConfig(s.config)

    assert config.config.get("compressor")["source"] == "fastq.gz"
    assert config.config.get("kraken:dummy") == None

    # --------------------------------- tests different constructors
    config = snaketools.SequanaConfig()
    config = snaketools.SequanaConfig({"test":1})
    assert config.config.test == 1
    # with a dictionary
    config = snaketools.SequanaConfig(config.config)
    # with a sequanaConfig instance
    config = snaketools.SequanaConfig(config)
    # with a non-yaml file
    try:
        json = sequana_data('test_summary_fastq_stats.json')
        config = snaketools.SequanaConfig(json)
        assert False
    except:
        assert True
    try:
        config = snaketools.SequanaConfig("dummy_dummy")
        assert False
    except:
        assert True

    # Test an exception
    s = snaketools.Module("compressor")
    config = snaketools.SequanaConfig(s.config)
    config._recursive_update(config._yaml_code, {"input_directory_dummy": "test"})

    #config.check_config_with_schema(s.schema_config)
    # loop over all pipelines, read the config, save it and check the content is
    # identical. This requires to remove the templates. We want to make sure the
    # empty strings are kept and that "no value" are kept as well
    #
    #    field1: ""
    #    field2:
    #
    # is unchanged
    from easydev import TempFile
    output = TempFile(suffix=".yaml")
    for pipeline in snaketools.pipeline_names:
        config_filename = Module(pipeline)._get_config()
        cfg1 = SequanaConfig(config_filename)
        cfg1.cleanup() # remove templates and strip strings

        cfg1.save(output.name)
        cfg2 = SequanaConfig(output.name)
        assert cfg2._yaml_code == cfg1._yaml_code
        cfg2._update_config()
        assert cfg1.config == cfg2.config
    output.delete()

def test_check_config_with_schema():
    schema = Module("compressor").schema_config
    SequanaConfig(Module("compressor").config).check_config_with_schema(schema)

def test_module_version():

    Module("snpeff/1.0").version == "1.0"

def test_message():
    snaketools.message("test")

def __test_dummy_manager():
    ss = snaketools.DummyManager()
    ss = snaketools.DummyManager(["test1.fastq.gz", "test2.fastq.gz"])
    assert ss.paired == True
    ss = snaketools.DummyManager(["test1.fastq.gz"])
    assert ss.paired == False
    ss = snaketools.DummyManager("test1.fastq.gz")
    assert ss.paired == False

def test_pipeline_manager():

    # test missing input_directory
    cfg = SequanaConfig({})
    try:
        pm = snaketools.PipelineManager("custom", cfg)
        assert False
    except:
        assert True

    # normal behaviour but no input provided:
    config = Module("compressor")._get_config()
    cfg = SequanaConfig(config)
    cfg.cleanup() # remove templates
    try:
        pm = snaketools.PipelineManager("custome", cfg)
        assert False
    except:
        assert True

    cfg = SequanaConfig(config)
    cfg.cleanup() # remove templates
    file1 = sequana_data("Hm2_GTGAAA_L005_R1_001.fastq.gz")
    cfg.config.input_directory, cfg.config.input_pattern = os.path.split(file1)
    #file2 = sequana_data("Hm2_GTGAAA_L005_R2_001.fastq.gz")
    pm = snaketools.PipelineManager("custom", cfg)
    assert pm.paired == False

    cfg = SequanaConfig(config)
    cfg.cleanup() # remove templates
    cfg.config.input_directory, cfg.config.input_pattern = os.path.split(file1)
    cfg.config.input_pattern = "Hm*gz"
    #file1 = sequana_data("Hm2_GTGAAA_L005_R1_001.fastq.gz")
    pm = snaketools.PipelineManager("custom", cfg)
    pm.plot_stats()
    assert pm.paired == True

    pm.getlogdir("fastqc")
    pm.getwkdir("fastqc")
    pm.getrawdata()
    pm.getreportdir("test")
    pm.getname("fastqc")

    # Test different configuration of input_directory, input_readtag,
    # input_pattern
    # Test the _R[12]_ paired
    with tempfile.TemporaryDirectory() as tmpdir:
        cfg = SequanaConfig()
        cfgname = tmpdir + "/config.yaml"
        cfg.config.input_pattern = "*fastq.gz"
        cfg.config.input_directory = tmpdir
        cfg.config.input_readtag = "_R[12]_"
        cfg._update_yaml()
        cfg.save(cfgname)
        cmd = "touch {}/test_R1_.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        cmd = "touch {}/test_R2_.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        pm = snaketools.PipelineManager("test", cfgname)
        assert pm.paired == True

    # Test the _[12]_ paired
    with tempfile.TemporaryDirectory() as tmpdir:
        cfg = SequanaConfig()
        cfgname = tmpdir + "/config.yaml"
        cfg.config.input_pattern = "*fastq.gz"
        cfg.config.input_directory = tmpdir
        cfg.config.input_readtag = "_[12]."
        cfg._update_yaml()
        cfg.save(cfgname)
        cmd = "touch {}/test_1.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        cmd = "touch {}/test_2.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        pm = snaketools.PipelineManager("test", cfgname)
        assert pm.paired is True

    # Test the _R[12]_ single end
    with tempfile.TemporaryDirectory() as tmpdir:
        cfg = SequanaConfig()
        cfgname = tmpdir + "/config.yaml"
        cfg.config.input_pattern = "*fastq.gz"
        cfg.config.input_directory = tmpdir
        cfg.config.input_readtag = "_R[12]_"
        cfg._update_yaml()
        cfg.save(cfgname)
        cmd = "touch {}/test_R1_.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        pm = snaketools.PipelineManager("test", cfgname)
        assert pm.paired is False

    # Test the _R[12]_ single end
    with tempfile.TemporaryDirectory() as tmpdir:
        cfg = SequanaConfig()
        cfgname = tmpdir + "/config.yaml"
        cfg.config.input_pattern = "*fq.gz" # wrong on purpose
        cfg.config.input_directory = tmpdir
        cfg.config.input_readtag = "_R[12]_"
        cfg._update_yaml()
        cfg.save(cfgname)
        cmd = "touch {}/test_R1_.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        try:
            pm = snaketools.PipelineManager("test", cfgname)
            assert False
        except:
            assert True

    # Test the _R[12]_ single end
    with tempfile.TemporaryDirectory() as tmpdir:
        cfg = SequanaConfig()
        cfgname = tmpdir + "/config.yaml"
        cfg.config.input_pattern = "*fastq.gz"
        cfg.config.input_directory = tmpdir
        cfg.config.input_readtag = "R[12]_"
        cfg._update_yaml()
        cfg.save(cfgname)
        cmd = "touch {}/testR1_.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        cmd = "touch {}/testR2_.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        try:
            pm = snaketools.PipelineManager("test", cfgname)
            assert False
        except:
            assert True

def test_pipeline_manager_generic():
    cfg = SequanaConfig({})
    file1 = sequana_data("Hm2_GTGAAA_L005_R1_001.fastq.gz")
    cfg.config.input_directory, cfg.config.input_pattern = os.path.split(file1)
    cfg.config.input_pattern = "Hm*gz"
    pm = snaketools.PipelineManagerGeneric("custom", cfg)
    pm.getlogdir("fastqc")
    pm.getwkdir("fastqc")
    pm.getrawdata()
    pm.getreportdir("test")
    pm.getname("fastqc")

def test_file_name_factory():
    import glob

    def inner_test(ff):
        len(ff)
        print(ff)
        ff.filenames
        ff.realpaths
        ff.all_extensions
        ff.pathnames
        ff.pathname
        ff.extensions

    #list
    list_files = glob.glob("*.py")
    ff = snaketools.FileFactory(list_files)
    inner_test(ff)

    # glob
    ff = snaketools.FileFactory("*py")
    inner_test(ff)

    directory = os.path.dirname(sequana_data("Hm2_GTGAAA_L005_R1_001.fastq.gz"))

    ff = snaketools.FastQFactory(directory + "/Hm2*fastq.gz", verbose=True)
    assert ff.tags == ['Hm2_GTGAAA_L005']

    ff.get_file1(ff.tags[0])
    ff.get_file2(ff.tags[0])
    assert len(ff) == 1

def test_copy_requirements():
    # We need 4 cases:
    # 1- http
    # 2- a sequana file (phix)
    # 3- an existing file elsewhere (here just a temporary file)
    # 4- an existing file in the same directory as the target dir

    from easydev import TempFile
    fh = tempfile.TemporaryDirectory()
    targetdir = fh.name

    # Case 3: a temporary file
    temprequire = TempFile()

    # Case 4: a local file (copy of the temp file)
    # TODO
    #localfile = temprequire.name.split(os.sep)[-1]
    #shutil.copy(temprequire.name, targetdir)

    cfg = snaketools.SequanaConfig()
    cfg.config.requirements = ["phiX174.fa", temprequire.name,
        #localfile,
        "https://raw.githubusercontent.com/sequana/sequana/master/README.rst"]
    cfg._update_yaml()
    cfg.copy_requirements(target=fh.name)

    # error
    cfg.config.requirements = ['dummy']
    try:
        cfg.copy_requirements(target=fh.name)
        assert False
    except:
        assert True

def test_onsuccess(tmpdir):
    directory = tmpdir.mkdir("onsuccess")
    p1 = directory.join("Makefile")
    p2 = directory.join("cleanup.py")
    onsuc = snaketools.OnSuccess()
    onsuc.makefile_filename = p1
    onsuc.makefile_cleanup = p2

def test_onsuccess_cleaner():
    fh = tempfile.TemporaryDirectory()
    onsucc = snaketools.OnSuccessCleaner()
    onsucc.makefile_filename = fh.name + os.sep + "Makefile"
    onsucc.add_bundle()
    onsucc.add_makefile()

def test_build_dynamic_rule():

    code = "whatever"
    fh = tempfile.TemporaryDirectory()
    directory = fh.name
    snaketools.build_dynamic_rule(code, directory)

def test_init():
    snaketools.init("compressor.rules", globals())
    assert "expected_output" in globals()

def test_get_pipeline_statistics():
    df = snaketools.get_pipeline_statistics()

def test_create_cleanup():
    with tempfile.TemporaryDirectory() as fout:
        snaketools.create_cleanup(fout)

def test_fastqfactory():
    try:
        snaketools.FastQFactory("*", read_tag='error')
        assert False
    except:
        assert True

    try:
        snaketools.FastQFactory("*", read_tag='[12]')
        assert False
    except:
        assert True

    directory = os.path.dirname(sequana_data("Hm2_GTGAAA_L005_R1_001.fastq.gz"))

    ff = snaketools.FastQFactory(directory + os.sep + "Hm2*gz", read_tag='R[12]')
    assert ff.paired is True
    assert ff.tags == ['Hm2_GTGAAA_L005_']

    ff = snaketools.FastQFactory(directory + os.sep + "Hm2*gz", read_tag=None)
    assert ff.paired is False
    assert sorted(ff.tags) == sorted(['Hm2_GTGAAA_L005_R2_001', 'Hm2_GTGAAA_L005_R1_001'])

def test_makefile():
    with tempfile.TemporaryDirectory() as fout:
        mk = snaketools.Makefile()
        mk.makefile_filename = fout + "/Makefile"
        mk.add_remove_done()
        mk.add_bundle()
        mk.save()

def test_bundle():
    with tempfile.TemporaryDirectory() as fout:
        os = snaketools.OnSuccess()
        os.makefile_filename = fout + "/Makefile"
        os.cleanup_filename = fout + "/sequana_cleanup.py"
        os.add_makefile()
        os.create_recursive_cleanup()

<commit_msg>fix tests<commit_after>from sequana import snaketools, sequana_data
from sequana.snaketools import DOTParser
import os, shutil
import tempfile
from sequana import Module, SequanaConfig
from easydev import TempFile
import subprocess

def test_dot_parser():
    s = DOTParser(sequana_data("test_dag.dot", "testing"))
    s.add_urls(mapper={'bwa_fix': "test.html"})
    try:os.remove("test_dag.ann.dot")
    except:pass

    s.mode = "v1"
    s.add_urls(mapper={'bwa_fix': "test.html"})
    try:os.remove("test_dag.ann.dot")
    except:pass

def test_md5():
    from sequana import Module
    m = Module("compressor")
    data = m.md5()

def test_modules():
    assert "dag" in snaketools.modules.keys()
    assert snaketools.modules['dag'].endswith("dag.rules")

def test_getcleanup_rules():
    filename =  snaketools.modules['fastq_sampling']
    try:
        snaketools.get_cleanup_rules(filename)
    except:
        pass

def test_snakemake_stats():
    # this is created using snakemake with the option "--stats stats.txt"
    s = snaketools.SnakeMakeStats(sequana_data("test_snakemake_stats.txt"))
    s.plot()
    with TempFile() as fout:
        s.plot_and_save(filename=fout.name, outputdir=None)

    with tempfile.TemporaryDirectory() as tempdir:
        s.plot_and_save(filename="test.png", outputdir=tempdir)

def test_plot_stats():

    with tempfile.TemporaryDirectory() as indir:
        shutil.copy(sequana_data("test_snakemake_stats.txt"),
            indir + "/stats.txt")
        with tempfile.TemporaryDirectory() as outdir:
            snaketools.plot_stats(indir, outdir)
    snaketools.plot_stats("dummy", "dummy")

def test_module():
    # a rule without README
    m = snaketools.Module('mark_duplicates_dynamic')
    m.description
    print(m)
    m # test __repr__
    m.__repr__()
    m.path
    m.snakefile
    m.overview
    # a rule with README
    m = snaketools.Module('dag')
    m.description
    m.overview
    assert m.is_executable()
    m.check()

    # a pipeline
    m = snaketools.Module('compressor')
    m.is_executable()
    m.check()
    m.snakefile
    m.name
    m
    print(m)
    assert m.cluster_config.endswith("cluster_config.json")
    assert m.schema_config.endswith("schema.yaml")

def test_valid_config():
    config = snaketools.SequanaConfig(None)

    s = snaketools.Module("compressor")
    config = snaketools.SequanaConfig(s.config)

    from easydev import TempFile
    with TempFile() as fh:
        config.save(fh.name)

def test_sequana_config():
    s = snaketools.Module("compressor")
    config = snaketools.SequanaConfig(s.config)

    assert config.config.get("compressor")["source"] == "fastq.gz"
    assert config.config.get("kraken:dummy") == None

    # --------------------------------- tests different constructors
    config = snaketools.SequanaConfig()
    config = snaketools.SequanaConfig({"test":1})
    assert config.config.test == 1
    # with a dictionary
    config = snaketools.SequanaConfig(config.config)
    # with a sequanaConfig instance
    config = snaketools.SequanaConfig(config)
    # with a non-yaml file
    try:
        json = sequana_data('test_summary_fastq_stats.json')
        config = snaketools.SequanaConfig(json)
        assert False
    except:
        assert True
    try:
        config = snaketools.SequanaConfig("dummy_dummy")
        assert False
    except:
        assert True

    # Test an exception
    s = snaketools.Module("compressor")
    config = snaketools.SequanaConfig(s.config)
    config._recursive_update(config._yaml_code, {"input_directory_dummy": "test"})

    #config.check_config_with_schema(s.schema_config)
    # loop over all pipelines, read the config, save it and check the content is
    # identical. This requires to remove the templates. We want to make sure the
    # empty strings are kept and that "no value" are kept as well
    #
    #    field1: ""
    #    field2:
    #
    # is unchanged
    from easydev import TempFile
    output = TempFile(suffix=".yaml")
    for pipeline in snaketools.pipeline_names:
        config_filename = Module(pipeline)._get_config()
        cfg1 = SequanaConfig(config_filename)
        cfg1.cleanup() # remove templates and strip strings

        cfg1.save(output.name)
        cfg2 = SequanaConfig(output.name)
        assert cfg2._yaml_code == cfg1._yaml_code
        cfg2._update_config()
        assert cfg1.config == cfg2.config
    output.delete()

def test_check_config_with_schema():
    schema = Module("compressor").schema_config
    SequanaConfig(Module("compressor").config).check_config_with_schema(schema)

def test_module_version():
    Module("snpeff/1.0").version == "1.0"

def test_message():
    snaketools.message("test")

def test_pipeline_manager():

    # test missing input_directory
    cfg = SequanaConfig({})
    try:
        pm = snaketools.PipelineManager("custom", cfg)
        assert False
    except:
        assert True

    # normal behaviour but no input provided:
    config = Module("compressor")._get_config()
    cfg = SequanaConfig(config)
    cfg.cleanup() # remove templates
    try:
        pm = snaketools.PipelineManager("custom", cfg)
        assert False
    except:
        assert True

    # normal behaviour
    cfg = SequanaConfig(config)
    cfg.cleanup() # remove templates
    file1 = sequana_data("Hm2_GTGAAA_L005_R1_001.fastq.gz")
    cfg.config.input_directory, cfg.config.input_pattern = os.path.split(file1)
    pm = snaketools.PipelineManager("custom", cfg)
    assert pm.paired == False

    cfg = SequanaConfig(config)
    cfg.cleanup() # remove templates
    cfg.config.input_directory, cfg.config.input_pattern = os.path.split(file1)
    cfg.config.input_pattern = "Hm*gz"
    #file1 = sequana_data("Hm2_GTGAAA_L005_R1_001.fastq.gz")
    pm = snaketools.PipelineManager("custom", cfg)
    pm.plot_stats()
    assert pm.paired == True

    pm.getlogdir("fastqc")
    pm.getwkdir("fastqc")
    pm.getrawdata()
    pm.getreportdir("test")
    pm.getname("fastqc")

    # Test different configuration of input_directory, input_readtag,
    # input_pattern
    # Test the _R[12]_ paired
    with tempfile.TemporaryDirectory() as tmpdir:
        cfg = SequanaConfig()
        cfgname = tmpdir + "/config.yaml"
        cfg.config.input_pattern = "*fastq.gz"
        cfg.config.input_directory = tmpdir
        cfg.config.input_readtag = "_R[12]_"
        cfg._update_yaml()
        cfg.save(cfgname)
        cmd = "touch {}/test_R1_.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        cmd = "touch {}/test_R2_.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        pm = snaketools.PipelineManager("test", cfgname)
        assert pm.paired == True

    # Test the _[12]_ paired
    with tempfile.TemporaryDirectory() as tmpdir:
        cfg = SequanaConfig()
        cfgname = tmpdir + "/config.yaml"
        cfg.config.input_pattern = "*fastq.gz"
        cfg.config.input_directory = tmpdir
        cfg.config.input_readtag = "_[12]."
        cfg._update_yaml()
        cfg.save(cfgname)
        cmd = "touch {}/test_1.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        cmd = "touch {}/test_2.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        pm = snaketools.PipelineManager("test", cfgname)
        assert pm.paired is True

    # Test the _R[12]_ single end
    with tempfile.TemporaryDirectory() as tmpdir:
        cfg = SequanaConfig()
        cfgname = tmpdir + "/config.yaml"
        cfg.config.input_pattern = "*fastq.gz"
        cfg.config.input_directory = tmpdir
        cfg.config.input_readtag = "_R[12]_"
        cfg._update_yaml()
        cfg.save(cfgname)
        cmd = "touch {}/test_R1_.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        pm = snaketools.PipelineManager("test", cfgname)
        assert pm.paired is False

    # Test the _R[12]_ single end
    with tempfile.TemporaryDirectory() as tmpdir:
        cfg = SequanaConfig()
        cfgname = tmpdir + "/config.yaml"
        cfg.config.input_pattern = "*fq.gz" # wrong on purpose
        cfg.config.input_directory = tmpdir
        cfg.config.input_readtag = "_R[12]_"
        cfg._update_yaml()
        cfg.save(cfgname)
        cmd = "touch {}/test_R1_.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        try:
            pm = snaketools.PipelineManager("test", cfgname)
            assert False
        except:
            assert True

    # Test the _R[12]_ single end
    with tempfile.TemporaryDirectory() as tmpdir:
        cfg = SequanaConfig()
        cfgname = tmpdir + "/config.yaml"
        cfg.config.input_pattern = "*fastq.gz"
        cfg.config.input_directory = tmpdir
        cfg.config.input_readtag = "R[12]_"
        cfg._update_yaml()
        cfg.save(cfgname)
        cmd = "touch {}/testR1_.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        cmd = "touch {}/testR2_.fastq.gz".format(tmpdir)
        subprocess.call(cmd.split())
        try:
            pm = snaketools.PipelineManager("test", cfgname)
            assert False
        except:
            assert True

def test_pipeline_manager_generic():
    cfg = SequanaConfig({})
    file1 = sequana_data("Hm2_GTGAAA_L005_R1_001.fastq.gz")
    cfg.config.input_directory, cfg.config.input_pattern = os.path.split(file1)
    cfg.config.input_pattern = "Hm*gz"
    pm = snaketools.PipelineManagerGeneric("quality_control", cfg)
    pm.getlogdir("fastqc")
    pm.getwkdir("fastqc")
    pm.getrawdata()
    pm.getreportdir("test")
    pm.getname("fastqc")
    gg = globals()
    gg['__snakefile__'] = "dummy"
    pm.setup(gg)
    del gg['__snakefile__']
    class WF():
        included_stack = ["dummy", 'dummy']
    wf = WF()
    gg['workflow'] = wf
    pm.setup(gg)
    pm.teardown()

    with tempfile.TemporaryDirectory() as dd:
        multiqc = open(dd + "/multiqc.html", "w")
        multiqc.write("test")
        multiqc.close()
        newfile = dd + "/multiqc.html_tmp_"
        pm.clean_multiqc(dd + "/multiqc.html")

def test_file_name_factory():
    import glob

    def inner_test(ff):
        len(ff)
        print(ff)
        ff.filenames
        ff.realpaths
        ff.all_extensions
        ff.pathnames
        ff.pathname
        ff.extensions

    #list
    list_files = glob.glob("*.py")
    ff = snaketools.FileFactory(list_files)
    inner_test(ff)

    # glob
    ff = snaketools.FileFactory("*py")
    inner_test(ff)

    directory = os.path.dirname(sequana_data("Hm2_GTGAAA_L005_R1_001.fastq.gz"))

    ff = snaketools.FastQFactory(directory + "/Hm2*fastq.gz", verbose=True)
    assert ff.tags == ['Hm2_GTGAAA_L005']

    ff.get_file1(ff.tags[0])
    ff.get_file2(ff.tags[0])
    assert len(ff) == 1

def test_copy_requirements():
    # We need 4 cases:
    # 1- http
    # 2- a sequana file (phix)
    # 3- an existing file elsewhere (here just a temporary file)
    # 4- an existing file in the same directory as the target dir

    from easydev import TempFile
    fh = tempfile.TemporaryDirectory()
    targetdir = fh.name

    # Case 3: a temporary file
    temprequire = TempFile()

    # Case 4: a local file (copy of the temp file)
    # TODO
    #localfile = temprequire.name.split(os.sep)[-1]
    #shutil.copy(temprequire.name, targetdir)

    cfg = snaketools.SequanaConfig()
    cfg.config.requirements = ["phiX174.fa", temprequire.name,
        #localfile,
        "https://raw.githubusercontent.com/sequana/sequana/master/README.rst"]
    cfg._update_yaml()
    cfg.copy_requirements(target=fh.name)

    # error
    cfg.config.requirements = ['dummy']
    try:
        cfg.copy_requirements(target=fh.name)
        assert False
    except:
        assert True

def test_onsuccess(tmpdir):
    directory = tmpdir.mkdir("onsuccess")
    p1 = directory.join("Makefile")
    p2 = directory.join("cleanup.py")
    onsuc = snaketools.OnSuccess()
    onsuc.makefile_filename = p1
    onsuc.makefile_cleanup = p2

def test_onsuccess_cleaner():
    fh = tempfile.TemporaryDirectory()
    onsucc = snaketools.OnSuccessCleaner()
    onsucc.makefile_filename = fh.name + os.sep + "Makefile"
    onsucc.add_bundle()
    onsucc.add_makefile()

def test_build_dynamic_rule():

    code = "whatever"
    fh = tempfile.TemporaryDirectory()
    directory = fh.name
    snaketools.build_dynamic_rule(code, directory)

def test_init():
    snaketools.init("compressor.rules", globals())
    assert "expected_output" in globals()

def test_get_pipeline_statistics():
    df = snaketools.get_pipeline_statistics()

def test_create_cleanup():
    with tempfile.TemporaryDirectory() as fout:
        snaketools.create_cleanup(fout)

def test_fastqfactory():
    try:
        snaketools.FastQFactory("*", read_tag='error')
        assert False
    except:
        assert True

    try:
        snaketools.FastQFactory("*", read_tag='[12]')
        assert False
    except:
        assert True

    directory = os.path.dirname(sequana_data("Hm2_GTGAAA_L005_R1_001.fastq.gz"))

    ff = snaketools.FastQFactory(directory + os.sep + "Hm2*gz", read_tag='R[12]')
    assert ff.paired is True
    assert ff.tags == ['Hm2_GTGAAA_L005_']

    ff = snaketools.FastQFactory(directory + os.sep + "Hm2*gz", read_tag=None)
    assert ff.paired is False
    assert sorted(ff.tags) == sorted(['Hm2_GTGAAA_L005_R2_001', 'Hm2_GTGAAA_L005_R1_001'])

def test_makefile():
    with tempfile.TemporaryDirectory() as fout:
        mk = snaketools.Makefile()
        mk.makefile_filename = fout + "/Makefile"
        mk.add_remove_done()
        mk.add_bundle()
        mk.save()

def test_bundle():
    with tempfile.TemporaryDirectory() as fout:
        os = snaketools.OnSuccess()
        os.makefile_filename = fout + "/Makefile"
        os.cleanup_filename = fout + "/sequana_cleanup.py"
        os.add_makefile()
        os.create_recursive_cleanup()

<|endoftext|>