<commit_before>#!/usr/bin/python2.4
#
# Copyright (c) 2004-2005 rPath, Inc.
#

import testsuite
testsuite.setup()

from mint_rephelp import MintRepositoryHelper
from mint.projectlisting import PROJECTNAME_ASC, PROJECTNAME_DES, LASTMODIFIED_ASC, LASTMODIFIED_DES, CREATED_ASC, CREATED_DES, NUMDEVELOPERS_ASC, NUMDEVELOPERS_DES, ACTIVITY_ASC, ACTIVITY_DES, ordersql
from mint import userlevels
from mint.searcher import SearchTermsError

class BrowseTest(MintRepositoryHelper):

    def _changeTimestamps(self, projectId, timeCreated, timeModified):
        cu = self.db.cursor()
        r = cu.execute("UPDATE Projects SET timeCreated=?, timeModified=? WHERE projectId=?", timeCreated, timeModified, projectId)
        self.db.commit()

    def _fakeCommit(self, projectId, timestamp, userId):
        cu = self.db.cursor()
        r = cu.execute("INSERT INTO Commits VALUES(?, ?, 'whoCares', '1.0', ?)", projectId, timestamp, userId)
        self.db.commit()

    def _fakePackage(self, projectId, name):
        cu = self.db.cursor()
        cu.execute("SELECT IFNULL(MAX(pkgId) + 1, 1) FROM PackageIndex")
        pkgId = cu.fetchone()[0]

        r = cu.execute("INSERT INTO PackageIndex VALUES(?, ?, ?, 'whoCares')", (pkgId, projectId, name))
        self.db.commit()

    def _sortOrderDict(self):
        return {
            PROJECTNAME_ASC:   ['bal', 'bar', 'baz', 'biz'],
            PROJECTNAME_DES:   ['biz', 'baz', 'bar', 'bal'],
            LASTMODIFIED_ASC:  ['biz', 'bal', 'bar', 'baz'],
            LASTMODIFIED_DES:  ['baz', 'bar', 'bal', 'biz'],
            CREATED_ASC:       ['bar', 'baz', 'bal', 'biz'],
            CREATED_DES:       ['biz', 'bal', 'baz', 'bar'],
            NUMDEVELOPERS_ASC: ['bar', 'biz', 'baz', 'bal'],
            NUMDEVELOPERS_DES: ['bal', 'baz', 'bar', 'biz'],
            ACTIVITY_ASC:      ['biz', 'baz', 'bal', 'bar'],
            ACTIVITY_DES:      ['bar', 'bal', 'baz', 'biz'],
            }

    def testBrowse(self):
        client, userId = self.quickMintUser("testuser", "testpass")
        client2, userId2 = self.quickMintUser("testuser2", "testpass")
        client3, userId3 = self.quickMintUser("testuser3", "testpass")
        client4, userId4 = self.quickMintUser("testuser4", "testpass")

        fooId = client.newProject("Foo", "foo", "rpath.org")
        self._changeTimestamps(fooId, 1128540046, 1128540046)
        barId = client.newProject("Bar", "bar", "rpath.org")
        self._changeTimestamps(barId, 1124540046, 1124540046)
        bazId = client.newProject("Baz", "baz", "rpath.org")
        self._changeTimestamps(bazId, 1126540046, 1126640046)
        balId = client.newProject("Bal", "bal", "rpath.org")
        self._changeTimestamps(balId, 1128540003, 1129540003)
        bizId = client.newProject("Biz", "biz", "rpath.org")
        self._changeTimestamps(balId, 1128540003, 1129540003)

        fooProject = client.getProject(fooId)
        barProject = client.getProject(barId)
        bazProject = client.getProject(bazId)
        balProject = client.getProject(balId)

        # add some fake commits for sorting
        self._fakeCommit(barId, 1129550003, userId)
        self._fakeCommit(bazId, 1129560003, userId2)
        self._fakeCommit(balId, 1129541003, userId4)
        self._fakeCommit(balId, 1129542003, userId3)
        self._fakeCommit(barId, 1129543003, userId)
        self._fakeCommit(barId, 1129544003, userId)
        # and a couple of really old ones that won't show up...
        self._fakeCommit(bazId, 1120040003, userId2)
        self._fakeCommit(bazId, 1120040013, userId2)
        self._fakeCommit(bizId, 1120040008, userId2)

        # add some members to each project to make the popularity sort useful
        fooProject.addMemberById(userId2, userlevels. DEVELOPER)
        fooProject.addMemberById(userId3, userlevels. DEVELOPER)
        fooProject.addMemberById(userId4, userlevels. DEVELOPER)

        bazProject.addMemberById(userId3, userlevels. DEVELOPER)

        balProject.addMemberById(userId3, userlevels. DEVELOPER)
        balProject.addMemberById(userId4, userlevels. DEVELOPER)

        sortOrderDict = self._sortOrderDict()
        for sortOrder in range(10):
            results, count = client.getProjects(sortOrder, 30, 0)
            if [x[1] for x in results] != sortOrderDict[sortOrder]:
                self.fail("sort problem during sort: %s. expected %s, but got %s"% (ordersql[sortOrder], sortOrderDict[sortOrder],[x[1] for x in results]))
        if client.getProjectsList() != [(4, 0, 0, 'bal - Bal'), (2, 0, 0, 'bar - Bar'), (3, 0, 0, 'baz - Baz'), (5, 0, 0, 'biz - Biz'), (1, 0, 0, 'foo - Foo')]:
            self.fail("getProjectsList did not return alphabetical results")

    def testSearchProjects(self):
        client, userId = self.quickMintUser("testuser", "testpass")
        fooId = client.newProject("Foo Project", "foo", "rpath.org")
        self._changeTimestamps(fooId, 1128540046, 1128540046)
        booId = client.newProject("Boo Project", "boo", "rpath.org")
        self._changeTimestamps(booId, 1124540046, 1124540046)
        snoozeId = client.newProject("Snooze Project", "snooze", "rpath.org")
        self._changeTimestamps(snoozeId, 1126540046, 1126640046)
        snoofId = client.newProject("Snoof Project", "snoof", "rpath.org")
        self._changeTimestamps(snoofId, 1128540003, 1129540003)

        try:
            client.getProjectSearchResults('oo ')
            self.fail("Search for illegal values 'oo ' succeeded")
        except SearchTermsError:
            pass

        if client.getProjectSearchResults('Foo') != ([[1, 'foo', 'Foo Project', '', 1128540046]], 1):
            self.fail("Search for 'Foo' did not return the correct results.")
        res = client.getProjectSearchResults('Project')
        assert(res[1] == 4)
        assert(len(res[0]) == res[1])
        assert(client.getProjectSearchResults('Snoo')[1] == 2)
        assert(client.getProjectSearchResults('Camp Town Racers sing this song... doo dah... doo dah...') == ([], 0) )
        assert(client.getProjectSearchResults('snooze OR Boo')[1] == 2)

    def testSearchUsers(self):
        client, userId = self.quickMintUser("testuser", "testpass")
        client2, userId2 = self.quickMintUser("testuser2", "testpass")
        client3, userId3 = self.quickMintUser("testuser3", "testpass")
        client4, userId4 = self.quickMintUser("testuser4", "testpass")
        assert(len(client.getUserSearchResults('test')[0]) == 4)
        assert(client.getUserSearchResults('er3') == ([[3, 'testuser3', 'Test User', 'test at example.com', '', 0]], 1))
        assert(client.getUserSearchResults('Sir Not Appearing In This Film') == ([], 0) )

    def testSearchPackages(self):
        client, userId = self.quickMintUser("testuser", "testpass")
        projectId = client.newProject("Foo", "foo", "rpath.org")
        self._fakePackage(projectId, "brokenPackage")
        self._fakePackage(projectId, "foomatic")
        self._fakePackage(projectId, "barmatic")
        self._fakePackage(projectId, "foobar")
        assert(client.getPackageSearchResults('broken') == ([['brokenPackage', 'whoCares', 1]], 1))
        assert(client.getPackageSearchResults('foo')[1] == 2)

if __name__ == "__main__":
    testsuite.main()
<commit_msg>tweak test case for browsing<commit_after>#!/usr/bin/python2.4
#
# Copyright (c) 2004-2005 rPath, Inc.
#

import testsuite
testsuite.setup()

from mint_rephelp import MintRepositoryHelper
from mint.projectlisting import PROJECTNAME_ASC, PROJECTNAME_DES, LASTMODIFIED_ASC, LASTMODIFIED_DES, CREATED_ASC, CREATED_DES, NUMDEVELOPERS_ASC, NUMDEVELOPERS_DES, ACTIVITY_ASC, ACTIVITY_DES, ordersql
from mint import userlevels
from mint.searcher import SearchTermsError

class BrowseTest(MintRepositoryHelper):

    def _changeTimestamps(self, projectId, timeCreated, timeModified):
        cu = self.db.cursor()
        r = cu.execute("UPDATE Projects SET timeCreated=?, timeModified=? WHERE projectId=?", timeCreated, timeModified, projectId)
        self.db.commit()

    def _fakeCommit(self, projectId, timestamp, userId):
        cu = self.db.cursor()
        r = cu.execute("INSERT INTO Commits VALUES(?, ?, 'whoCares', '1.0', ?)", projectId, timestamp, userId)
        self.db.commit()

    def _fakePackage(self, projectId, name):
        cu = self.db.cursor()
        cu.execute("SELECT IFNULL(MAX(pkgId) + 1, 1) FROM PackageIndex")
        pkgId = cu.fetchone()[0]

        r = cu.execute("INSERT INTO PackageIndex VALUES(?, ?, ?, 'whoCares')", (pkgId, projectId, name))
        self.db.commit()

    def _sortOrderDict(self):
        return {
            PROJECTNAME_ASC:   ['bal', 'bar', 'baz', 'biz'],
            PROJECTNAME_DES:   ['biz', 'baz', 'bar', 'bal'],
            LASTMODIFIED_ASC:  ['biz', 'bal', 'bar', 'baz'],
            LASTMODIFIED_DES:  ['baz', 'bar', 'bal', 'biz'],
            CREATED_ASC:       ['bar', 'baz', 'bal', 'biz'],
            CREATED_DES:       ['biz', 'bal', 'baz', 'bar'],
            NUMDEVELOPERS_ASC: ['bar', 'biz', 'baz', 'bal'],
            NUMDEVELOPERS_DES: ['bal', 'baz', 'bar', 'biz'],
            ACTIVITY_ASC:      ['biz', 'baz', 'bal', 'bar'],
            ACTIVITY_DES:      ['bar', 'bal', 'baz', 'biz'],
            }

    def testBrowse(self):
        client, userId = self.quickMintUser("testuser", "testpass")
        client2, userId2 = self.quickMintUser("testuser2", "testpass")
        client3, userId3 = self.quickMintUser("testuser3", "testpass")
        client4, userId4 = self.quickMintUser("testuser4", "testpass")

        fooId = client.newProject("Foo", "foo", "rpath.org")
        self._changeTimestamps(fooId, 1128540046, 1128540046)
        barId = client.newProject("Bar", "bar", "rpath.org")
        self._changeTimestamps(barId, 1124540046, 1124540046)
        bazId = client.newProject("Baz", "baz", "rpath.org")
        self._changeTimestamps(bazId, 1126540046, 1126640046)
        balId = client.newProject("Bal", "bal", "rpath.org")
        self._changeTimestamps(balId, 1128540003, 1129540003)
        bizId = client.newProject("Biz", "biz", "rpath.org")
        self._changeTimestamps(balId, 1128540003, 1129540003)

        fooProject = client.getProject(fooId)
        barProject = client.getProject(barId)
        bazProject = client.getProject(bazId)
        balProject = client.getProject(balId)

        # add some fake commits for sorting
        self._fakeCommit(barId, 1129550003, userId)
        self._fakeCommit(bazId, 1129560003, userId2)
        self._fakeCommit(balId, 1129541003, userId4)
        self._fakeCommit(balId, 1129542003, userId3)
        self._fakeCommit(barId, 1129543003, userId)
        self._fakeCommit(barId, 1129544003, userId)
        # and a couple of really old ones that won't show up...
        self._fakeCommit(bazId, 1120040003, userId2)
        self._fakeCommit(bazId, 1120040013, userId2)
        self._fakeCommit(bizId, 1120040008, userId2)

        # add some members to each project to make the popularity sort useful
        fooProject.addMemberById(userId2, userlevels. DEVELOPER)
        fooProject.addMemberById(userId3, userlevels. DEVELOPER)
        fooProject.addMemberById(userId4, userlevels. DEVELOPER)

        bazProject.addMemberById(userId3, userlevels. DEVELOPER)

        balProject.addMemberById(userId3, userlevels. DEVELOPER)
        balProject.addMemberById(userId4, userlevels. DEVELOPER)

        sortOrderDict = self._sortOrderDict()
        for sortOrder in range(10):
            results, count = client.getProjects(sortOrder, 30, 0)
            if count != 4:
                self.fail('getProjects returned the wrong project count.')
            if [x[1] for x in results] != sortOrderDict[sortOrder]:
                self.fail("sort problem during sort: %s. expected %s, but got %s"% (ordersql[sortOrder], sortOrderDict[sortOrder],[x[1] for x in results]))
        if client.getProjectsList() != [(4, 0, 0, 'bal - Bal'), (2, 0, 0, 'bar - Bar'), (3, 0, 0, 'baz - Baz'), (5, 0, 0, 'biz - Biz'), (1, 0, 0, 'foo - Foo')]:
            self.fail("getProjectsList did not return alphabetical results")

    def testSearchProjects(self):
        client, userId = self.quickMintUser("testuser", "testpass")
        fooId = client.newProject("Foo Project", "foo", "rpath.org")
        self._changeTimestamps(fooId, 1128540046, 1128540046)
        booId = client.newProject("Boo Project", "boo", "rpath.org")
        self._changeTimestamps(booId, 1124540046, 1124540046)
        snoozeId = client.newProject("Snooze Project", "snooze", "rpath.org")
        self._changeTimestamps(snoozeId, 1126540046, 1126640046)
        snoofId = client.newProject("Snoof Project", "snoof", "rpath.org")
        self._changeTimestamps(snoofId, 1128540003, 1129540003)

        try:
            client.getProjectSearchResults('oo ')
            self.fail("Search for illegal values 'oo ' succeeded")
        except SearchTermsError:
            pass

        if client.getProjectSearchResults('Foo') != ([[1, 'foo', 'Foo Project', '', 1128540046]], 1):
            self.fail("Search for 'Foo' did not return the correct results.")
        res = client.getProjectSearchResults('Project')
        assert(res[1] == 4)
        assert(len(res[0]) == res[1])
        assert(client.getProjectSearchResults('Snoo')[1] == 2)
        assert(client.getProjectSearchResults('Camp Town Racers sing this song... doo dah... doo dah...') == ([], 0) )
        assert(client.getProjectSearchResults('snooze OR Boo')[1] == 2)

    def testSearchUsers(self):
        client, userId = self.quickMintUser("testuser", "testpass")
        client2, userId2 = self.quickMintUser("testuser2", "testpass")
        client3, userId3 = self.quickMintUser("testuser3", "testpass")
        client4, userId4 = self.quickMintUser("testuser4", "testpass")
        assert(len(client.getUserSearchResults('test')[0]) == 4)
        assert(client.getUserSearchResults('er3') == ([[3, 'testuser3', 'Test User', 'test at example.com', '', 0]], 1))
        assert(client.getUserSearchResults('Sir Not Appearing In This Film') == ([], 0) )

    def testSearchPackages(self):
        client, userId = self.quickMintUser("testuser", "testpass")
        projectId = client.newProject("Foo", "foo", "rpath.org")
        self._fakePackage(projectId, "brokenPackage")
        self._fakePackage(projectId, "foomatic")
        self._fakePackage(projectId, "barmatic")
        self._fakePackage(projectId, "foobar")
        assert(client.getPackageSearchResults('broken') == ([['brokenPackage', 'whoCares', 1]], 1))
        assert(client.getPackageSearchResults('foo')[1] == 2)

if __name__ == "__main__":
    testsuite.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#  This file is part of Sequana software
#
#  Copyright (c) 2016 - Sequana Development Team
#
#  File author(s):
#      Thomas Cokelaer <thomas.cokelaer@pasteur.fr>
#      Dimitri Desvillechabrol <dimitri.desvillechabrol@pasteur.fr>,
#          <d.desvillechabrol@gmail.com>
#
#  Distributed under the terms of the 3-clause BSD license.
#  The full license is in the LICENSE file, distributed with this software.
#
#  website: https://github.com/sequana/sequana
#  documentation: http://sequana.readthedocs.io
#
##############################################################################
import os
import glob
import json

from sequana.modules_report.base_module import SequanaBaseModule

from easydev import DevTools
from sequana.resources.canvas.bar import CanvasBar

from sequana.lazy import pandas as pd

import colorlog
logger = colorlog.getLogger(__name__)

__all__ = ['SequanaMultipleSummary']

class ReadSummary(object):
    def __init__(self, filename):
        self.filename = filename
        self.data = json.load(open(self.filename, "r"))

    def get_phix_percent(self):
        return self.data['phix_section']['contamination']

    def get_cutadapt_stats(self):
        return self.data["cutadapt_json"]

    def get_fastq_stats_samples(self):
        return self.data["fastq_stats_samples_json"]

    def get_mean_quality_samples(self):
        this = self.data["fastq_stats_samples_json"]
        return this["mean quality"]['R1']

    def get_nreads_raw(self):
        this = self.data["fastq_stats_samples_json"]
        return this["n_reads"]['R1']

    def get_average_read_length(self):
        this = self.data["fastq_stats_samples_json"]
        return this["average read length"]['R1']

    def get_trimming_percent(self):
        d = self.get_cutadapt_stats()
        try:
            trimming = d["percent"]["Pairs too short"]
        except:
            trimming = d["percent"]["Reads too short"]
        return float(trimming.replace("%", "").replace("(","").replace(")","").strip())

    def get_adapters_percent(self):
        d = self.get_cutadapt_stats()
        try:
            read1 = d["percent"]["Read1 with adapters"]
            # e.g. (8.3%)
            read1 = float(read1.strip("%)").strip("("))
            try:
                read2 = d["percent"]["Read2 with adapters"]
                read2 = float(read2.strip("%)").strip("("))
                read1 = (read1 + read2 ) /2. # FIXME crude approximation
            except:
                pass # single-end
        except:
            read1 = d["percent"]["Reads with adapters"]
            read1 = float(read1.strip("%)").strip("("))
        return read1

    def get_read1_with_adapters_percent(self):
        return self._get_read_with_adapters_percent("1")

    def get_read2_with_adapters_percent(self):
        return self._get_read_with_adapters_percent("2")

    def _get_read_with_adapters_percent(self, tag):
        d = self.get_cutadapt_stats()
        trimming = d["percent"]["Read%s with adapters" % tag]
        trimming = trimming.strip()
        for this in [",", "(", ")", "%"]:
            trimming = trimming.replace(this, "")
        trimming = float(trimming)
        return trimming

    def get_output_total_reads(self):
        d = self.get_cutadapt_stats()
        try:
            trimming = d["Number of reads"]["Pairs kept"]
        except:
            trimming = d["Number of reads"]["Reads kept"]
        try: # previous version stored strings in the json; TODO add test
            trimming = trimming.strip()
            for this in [",", "(", ")", "%"]:
                trimming = trimming.replace(this, "")
            trimming = int(trimming)
        except:
            pass
        return trimming

class MultiSummary(SequanaBaseModule):
    """
    Used by the pipelines to create a summary based on the content of the
    directory. Also used by the standalone application, in which case
    config and pipeline files are not required.

    For developers:

    1. In class Summary:

    2 In class SequanaMultipleSummary:

        try:self.populate_gc()
        except:pass
        def populate_gc():
            do something
        def get_gc(self):
            return [x.get_gc() for x in self.summaries]

    3: update the jinja file report_multiple_summay

    """
    def __init__(self, pattern="**/summary.json", output_filename=None,
                 verbose=True, **kargs):
        super().__init__()

        from sequana import logger
        logger.setLevel("INFO")
        if verbose is False:
            logger.setLevel("WARNING")

        logger.info("Sequana Summary is still a tool in progress and have been " +
              "  tested with the quality_control pipeline only for now.")
        self.title = "Sequana multiple summary"
        self.devtools = DevTools()

        self.filenames = list(glob.iglob(pattern, recursive=True))
        self.summaries = [ReadSummary(filename) for filename in self.filenames]
        self.projects = [ReadSummary(filename).data['project'] for filename in self.filenames]
        self.create_report_content()
        self.create_html(output_filename)

    def create_report_content(self):
        self.sections = list()
        self.add_section()

    def add_section(self):
        logger.info("Found %s projects/samples/ directories" % len(self.summaries))
        for filename in self.filenames:
            logger.info(filename)

        self.jinja = {}

        self.jinja['canvas'] = ''
        self.jinja['canvas'] += """"""

        caption = """
The table below gives a brief summary of the analysis. The
first column contains clickable sample name that redirects to complete summary
page. The table contains the following columns:

   Table caption

N_raw	Number of reads in the data

GC_raw_(%)	GC content in percentage in the raw data
across all reads

Mean_quality_raw	Mean quality across all reads all bases
in the raw data

Phix_content_(%)	Percentage of reads found with Phix174

Adapters_content_(%)	Percentage of reads with adapters (after phix
removal if applied)

Trimmed_reads_(%)	Percentage of reads trimmed (after
phix and adapter removal)

N_final	Final number of reads (after phix and adapter
removal and trimming)

"""
        infohtml = self.create_hide_section('information',
            '(Show information)', caption, True)
        infohtml = "\n".join(infohtml)

        self.intro = introhtml + """
Summary: """ + infohtml +html

        self.sections.append({
            'name': None,
            'anchor': None,
            'content': self.jinja['canvas'] + "\n".join(self.jinja['sections'])
        })

    def _get_div(self, name, title):
        div = "
%s
" %  title
        div += '

'
        return div

    def _get_df(self, method_name):
        data = getattr(self, method_name)()
        df = pd.DataFrame({
            "name": self.get_projects(),
            "value": data,
            "url": self.get_urls()})
        return df

    def populate_output_total_reads(self):
        df = self._get_df("get_output_total_reads")
        self.df['N_final'] = df.copy()
        self.df['N_final'].rename({'value': 'N_final'}, axis=1, inplace=True)

    def populate_adapters(self):
        title = "Adapters content"
        df = self._get_df("get_adapters_percent")
        self.df['Adapters'] = df.copy()
        self.df['Adapters'].rename({'value': 'Adapters_content_(%)'}, axis=1,
            inplace=True)
        cb = CanvasBar(df, "Adapters content", "adapters", xlabel="Percentage")
        self.jinja['canvas'] += cb.to_html()
        self.jinja['sections'].append(self._get_div("adapters", title))

    def populate_nreads_raw(self):
        title = "Number of reads"
        df = self._get_df("get_nreads_raw")
        self.df['N_raw'] = df.copy()
        self.df['N_raw'].rename({'value': 'N_raw'}, axis=1, inplace=True)
        cb = CanvasBar(df, "Number of reads (raw data)", "nreads_raw",
                    xlabel="Number of reads")
        self.jinja['canvas'] += cb.to_html()
        self.jinja['sections'].append(self._get_div("nreads_raw",title))

    def populate_mean_quality(self):
        title = "Mean quality (raw data)"
        df = self._get_df("get_mean_quality_samples")
        self.df['Mean_quality_raw'] = df.copy()
        self.df['Mean_quality_raw'].rename({'value': 'Mean_quality_raw'},
            axis=1, inplace=True)
        cb = CanvasBar(df, title, "mean_quality", xlabel="mean quality")
        self.jinja['canvas'] += cb.to_html(options={'maxrange':40})
        self.jinja['sections'].append(self._get_div("mean_quality", title))

    def populate_gc_samples(self):
        title = "GC content (raw)"
        df = self._get_df("get_gc_content_samples")
        self.df['GC_raw'] = df.copy()
        self.df['GC_raw'].rename({'value': 'GC_raw_(%)'}, axis=1, inplace=True)
        cb = CanvasBar(df, title, "populate_gc_samples", xlabel="Percentage")
        self.jinja['canvas'] += cb.to_html(options={"maxrange":100})
        self.jinja['sections'].append(self._get_div("populate_gc_samples",title))

    def populate_phix(self):
        title = "Phix content"
        df = self._get_df("get_phix_percent")
        self.df['Phix'] = df.copy()
        self.df['Phix'].rename({'value': 'Phix_content_(%)'}, inplace=True,
            axis=1)
        cb = CanvasBar(df, title, "phix", xlabel="Percentage")
        self.jinja['canvas'] += cb.to_html()
        self.jinja['sections'].append(self._get_div("phix", title))

    def populate_trimming(self):
        title = "Trimming (raw data)"
        df = self._get_df("get_trimming_percent")
        self.df['Trimmed'] = df.copy()
        self.df['Trimmed'].rename({'value': 'Trimmed_reads_(%)'}, axis=1,
            inplace=True)
        cb = CanvasBar(df, title, "trimming", xlabel="Percentage")
        self.jinja['canvas'] += cb.to_html()
        self.jinja['sections'].append(self._get_div("trimming", title))

    def get_projects(self):
        return [x.data['project'] for x in self.summaries]

    def get_urls(self):
        return [x.replace("summary.json", "summary.html") for x in self.get_unique_names()]

    def get_unique_names(self):
        """reduce the filenames length removing the common suffix
        remove also the filename itself.
        """
        projects = self.get_projects()
        if len(projects) == set(projects):
            return projects
        else:
            return self.filenames

    ##########################################################################
    # The retrieval of all specific data for all summaries

    def get_adapters_percent(self):
        return [x.get_adapters_percent() for x in self.summaries]

    def get_nreads_raw(self):
        return [x.get_nreads_raw() for x in self.summaries]

    def get_phix_percent(self):
        return [x.get_phix_percent() for x in self.summaries]

    def get_gc_content_samples(self):
        return [x.get_fastq_stats_samples()['GC content']["R1"] for x in self.summaries]

    def get_trimming_percent(self):
        return [x.get_trimming_percent() for x in self.summaries]

    def get_mean_quality_samples(self):
        return [x.get_mean_quality_samples() for x in self.summaries]

    def get_output_total_reads(self):
        return [x.get_output_total_reads() for x in self.summaries]

    def parse(self):
        pass

<commit_msg>Fix __all__ and some pep8<commit_after>#
#  Copyright (c) 2016-2021 - Sequana Development Team
#
#  The full license is in the LICENSE file, distributed with this software.
#
#  website: https://github.com/sequana/sequana
#  documentation: http://sequana.readthedocs.io
#
##############################################################################
import glob
import json

from sequana.modules_report.base_module import SequanaBaseModule

from easydev import DevTools
from sequana.resources.canvas.bar import CanvasBar

from sequana.lazy import pandas as pd

import colorlog
logger = colorlog.getLogger(__name__)

__all__ = ["ReadSummary", "MultiSummary"]

class ReadSummary(object):
    def __init__(self, filename):
        self.filename = filename
        self.data = json.load(open(self.filename, "r"))

    def get_phix_percent(self):
        return self.data['phix_section']['contamination']

    def get_cutadapt_stats(self):
        return self.data["cutadapt_json"]

    def get_fastq_stats_samples(self):
        return self.data["fastq_stats_samples_json"]

    def get_mean_quality_samples(self):
        this = self.data["fastq_stats_samples_json"]
        return this["mean quality"]['R1']

    def get_nreads_raw(self):
        this = self.data["fastq_stats_samples_json"]
        return this["n_reads"]['R1']

    def get_average_read_length(self):
        this = self.data["fastq_stats_samples_json"]
        return this["average read length"]['R1']

    def get_trimming_percent(self):
        d = self.get_cutadapt_stats()
        try:
            trimming = d["percent"]["Pairs too short"]
        except KeyError:
            trimming = d["percent"]["Reads too short"]
        return float(trimming.replace("%", "").replace("(","").replace(")","").strip())

    def get_adapters_percent(self):
        d = self.get_cutadapt_stats()
        try:
            read1 = d["percent"]["Read1 with adapters"]
            # e.g. (8.3%)
            read1 = float(read1.strip("%)").strip("("))
            try:
                read2 = d["percent"]["Read2 with adapters"]
                read2 = float(read2.strip("%)").strip("("))
                read1 = (read1 + read2 ) /2. # FIXME crude approximation
            except:
                pass # single-end
        except:
            read1 = d["percent"]["Reads with adapters"]
            read1 = float(read1.strip("%)").strip("("))
        return read1

    def get_read1_with_adapters_percent(self):
        return self._get_read_with_adapters_percent("1")

    def get_read2_with_adapters_percent(self):
        return self._get_read_with_adapters_percent("2")

    def _get_read_with_adapters_percent(self, tag):
        d = self.get_cutadapt_stats()
        trimming = d["percent"]["Read%s with adapters" % tag]
        trimming = trimming.strip()
        for this in [",", "(", ")", "%"]:
            trimming = trimming.replace(this, "")
        trimming = float(trimming)
        return trimming

    def get_output_total_reads(self):
        d = self.get_cutadapt_stats()
        try:
            trimming = d["Number of reads"]["Pairs kept"]
        except:
            trimming = d["Number of reads"]["Reads kept"]
        try: # previous version stored strings in the json; TODO add test
            trimming = trimming.strip()
            for this in [",", "(", ")", "%"]:
                trimming = trimming.replace(this, "")
            trimming = int(trimming)
        except:
            pass
        return trimming

class MultiSummary(SequanaBaseModule):
    """
    Used by the pipelines to create a summary based on the content of the
    directory. Also used by the standalone application, in which case
    config and pipeline files are not required.

    For developers:

    1. In class Summary:

    2 In class SequanaMultipleSummary:

        try:self.populate_gc()
        except:pass
        def populate_gc():
            do something
        def get_gc(self):
            return [x.get_gc() for x in self.summaries]

    3: update the jinja file report_multiple_summay

    """
    def __init__(self, pattern="**/summary.json", output_filename=None,
                 verbose=True, **kargs):
        super().__init__()

        from sequana import logger
        logger.setLevel("INFO")
        if verbose is False:
            logger.setLevel("WARNING")

        logger.info("Sequana Quality control Summary will not be maintained. Please use sequana_fastqc for QCs, sequana_multitax for Taxonomy.")
        self.title = "Sequana multiple summary"
        self.devtools = DevTools()

        self.filenames = list(glob.iglob(pattern, recursive=True))

        self.summaries = [ReadSummary(filename) for filename in self.filenames]
        self.projects = [s.data['project'] for s in self.summaries]

        self.create_report_content()
        self.create_html(output_filename)

    def create_report_content(self):
        self.sections = list()
        self.add_section()

    def add_section(self):
        logger.info("Found %s projects/samples/ directories" % len(self.summaries))
        for filename in self.filenames:
            logger.info(filename)

        self.jinja = {}

        self.jinja['canvas'] = ''
        self.jinja['canvas'] += """"""

        caption = """
The table below gives a brief summary of the analysis. The
first column contains clickable sample name that redirects to complete summary
page. The table contains the following columns:

   Table caption

N_raw	Number of reads in the data

GC_raw_(%)	GC content in percentage in the raw data
across all reads

Mean_quality_raw	Mean quality across all reads all bases
in the raw data

Phix_content_(%)	Percentage of reads found with Phix174

Adapters_content_(%)	Percentage of reads with adapters (after phix
removal if applied)

Trimmed_reads_(%)	Percentage of reads trimmed (after
phix and adapter removal)

N_final	Final number of reads (after phix and adapter
removal and trimming)

"""
        infohtml = self.create_hide_section('information',
            '(Show information)', caption, True)
        infohtml = "\n".join(infohtml)

        self.intro = introhtml + """
Summary: """ + infohtml +html

        self.sections.append({
            'name': None,
            'anchor': None,
            'content': self.jinja['canvas'] + "\n".join(self.jinja['sections'])
        })

    def _get_div(self, name, title):
        div = "
%s
" %  title
        div += '

'
        return div

    def _get_df(self, method_name):
        data = getattr(self, method_name)()
        df = pd.DataFrame({
            "name": self.get_projects(),
            "value": data,
            "url": self.get_urls()})
        return df

    def populate_output_total_reads(self):
        df = self._get_df("get_output_total_reads")
        self.df['N_final'] = df.copy()
        self.df['N_final'].rename({'value': 'N_final'}, axis=1, inplace=True)

    def populate_adapters(self):
        title = "Adapters content"
        df = self._get_df("get_adapters_percent")
        self.df['Adapters'] = df.copy()
        self.df['Adapters'].rename({'value': 'Adapters_content_(%)'}, axis=1,
            inplace=True)
        cb = CanvasBar(df, "Adapters content", "adapters", xlabel="Percentage")
        self.jinja['canvas'] += cb.to_html()
        self.jinja['sections'].append(self._get_div("adapters", title))

    def populate_nreads_raw(self):
        title = "Number of reads"
        df = self._get_df("get_nreads_raw")
        self.df['N_raw'] = df.copy()
        self.df['N_raw'].rename({'value': 'N_raw'}, axis=1, inplace=True)
        cb = CanvasBar(df, "Number of reads (raw data)", "nreads_raw",
                    xlabel="Number of reads")
        self.jinja['canvas'] += cb.to_html()
        self.jinja['sections'].append(self._get_div("nreads_raw",title))

    def populate_mean_quality(self):
        title = "Mean quality (raw data)"
        df = self._get_df("get_mean_quality_samples")
        self.df['Mean_quality_raw'] = df.copy()
        self.df['Mean_quality_raw'].rename({'value': 'Mean_quality_raw'},
            axis=1, inplace=True)
        cb = CanvasBar(df, title, "mean_quality", xlabel="mean quality")
        self.jinja['canvas'] += cb.to_html(options={'maxrange':40})
        self.jinja['sections'].append(self._get_div("mean_quality", title))

    def populate_gc_samples(self):
        title = "GC content (raw)"
        df = self._get_df("get_gc_content_samples")
        self.df['GC_raw'] = df.copy()
        self.df['GC_raw'].rename({'value': 'GC_raw_(%)'}, axis=1, inplace=True)
        cb = CanvasBar(df, title, "populate_gc_samples", xlabel="Percentage")
        self.jinja['canvas'] += cb.to_html(options={"maxrange":100})
        self.jinja['sections'].append(self._get_div("populate_gc_samples",title))

    def populate_phix(self):
        title = "Phix content"
        df = self._get_df("get_phix_percent")
        self.df['Phix'] = df.copy()
        self.df['Phix'].rename({'value': 'Phix_content_(%)'}, inplace=True,
            axis=1)
        cb = CanvasBar(df, title, "phix", xlabel="Percentage")
        self.jinja['canvas'] += cb.to_html()
        self.jinja['sections'].append(self._get_div("phix", title))

    def populate_trimming(self):
        title = "Trimming (raw data)"
        df = self._get_df("get_trimming_percent")
        self.df['Trimmed'] = df.copy()
        self.df['Trimmed'].rename({'value': 'Trimmed_reads_(%)'}, axis=1,
            inplace=True)
        cb = CanvasBar(df, title, "trimming", xlabel="Percentage")
        self.jinja['canvas'] += cb.to_html()
        self.jinja['sections'].append(self._get_div("trimming", title))

    def get_projects(self):
        return [x.data['project'] for x in self.summaries]

    def get_urls(self):
        return [x.replace("summary.json", "summary.html") for x in self.get_unique_names()]

    def get_unique_names(self):
        """reduce the filenames length removing the common suffix
        remove also the filename itself.
        """
        projects = self.get_projects()
        if len(projects) == set(projects):
            return projects
        else:
            return self.filenames

    ##########################################################################
    # The retrieval of all specific data for all summaries

    def get_adapters_percent(self):
        return [x.get_adapters_percent() for x in self.summaries]

    def get_nreads_raw(self):
        return [x.get_nreads_raw() for x in self.summaries]

    def get_phix_percent(self):
        return [x.get_phix_percent() for x in self.summaries]

    def get_gc_content_samples(self):
        return [x.get_fastq_stats_samples()['GC content']["R1"] for x in self.summaries]

    def get_trimming_percent(self):
        return [x.get_trimming_percent() for x in self.summaries]

    def get_mean_quality_samples(self):
        return [x.get_mean_quality_samples() for x in self.summaries]

    def get_output_total_reads(self):
        return [x.get_output_total_reads() for x in self.summaries]

    def parse(self):
        pass

<|endoftext|>


<commit_before>#!/usr/bin/python
from cloud_retester import do_test, do_test_cloud, report_cloud, setup_testing_nodes, terminate_testing_nodes

# Clean the repo
do_test("cd ../src/; make clean")

# Make every target first, as some tests depend on multiple targets
for mode in ["debug", "release"]:
    # Build our targets
    do_test("cd ../src/; nice make -j",
            { "DEBUG"            : 1 if mode    == "debug"    else 0 },
            cmd_format="make")

for special in ["NO_EPOLL", "MOCK_IO_LAYER", "MOCK_CACHE_CHECK", "VALGRIND"]:
    do_test("cd ../src/; nice make -j",
            { "DEBUG" : 1,
              special : 1},
            cmd_format="make")

# Make sure auxillary tools compile
do_test("cd ../bench/stress-client/; make clean; make",
        {},
        cmd_format="make")
do_test("cd ../bench/serializer-bench/; make clean; make",
        {},
        cmd_format="make")

# For safety: ensure that nodes are terminated in case of exceptions
try:
    # Setup the EC2 testing nodes
    setup_testing_nodes()

    # Do quick smoke tests in some environments
    for (mode, checker, protocol) in [("debug", "valgrind", "text")]:
        # VERY basic tests
        do_test_cloud("integration/append_prepend.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol })

        do_test_cloud("integration/cas.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol })

        do_test_cloud("integration/flags.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol })

        do_test_cloud("integration/incr_decr.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol })

        do_test_cloud("integration/expiration.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol })

        do_test_cloud("integration/serial_mix.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol,
                  "duration"    : 10,
                  "fsck"        : True},
                repeat=1, timeout=60)

        do_test_cloud("integration/rget.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol }, timeout=45)

        # More advanced tests in various cores/slices configuration
        for (cores, slices) in [(1, 1)]:
            do_test_cloud("integration/many_keys.py",
                    { "auto"        : True,
                      "mode"        : mode,
                      "no-valgrind" : not checker,
                      "protocol"    : protocol,
                      "cores"       : cores,
                      "slices"      : slices,
                      "num-keys"    : 500 })

            do_test_cloud("integration/serial_mix.py",
                    { "auto"        : True,
                      "mode"        : mode,
                      "no-valgrind" : not checker,
                      "protocol"    : protocol,
                      "cores"       : cores,
                      "slices"      : slices,
                      "duration"    : 10 },
                    repeat=2, timeout=35)

            do_test_cloud("integration/serial_mix.py",
                    { "auto"        : True,
                      "mode"        : mode,
                      "no-valgrind" : not checker,
                      "protocol"    : protocol,
                      "cores"       : cores,
                      "slices"      : slices,
                      "duration"    : 10,
                      "restart-server-prob" : "0.0005" },
                    repeat=2, timeout=35)

            do_test_cloud("integration/multi_serial_mix.py",
                    { "auto"        : True,
                      "mode"        : mode,
                      "no-valgrind" : not checker,
                      "protocol"    : protocol,
                      "cores"       : cores,
                      "slices"      : slices,
                      "duration"    : 10 },
                    repeat=2, timeout=35)

            do_test_cloud("integration/multi_serial_mix.py",
                    { "auto"        : True,
                      "mode"        : mode,
                      "no-valgrind" : not checker,
                      "protocol"    : protocol,
                      "cores"       : cores,
                      "slices"      : slices,
                      "memory"      : 7,
                      "diff-log-size" : 2,
                      "duration"    : 10 },
                    repeat=2, timeout=35)

    # Report the results
    report_cloud()

finally:
    terminate_testing_nodes()

<commit_msg>Added big_values.py to smoke_test.<commit_after>#!/usr/bin/python
from cloud_retester import do_test, do_test_cloud, report_cloud, setup_testing_nodes, terminate_testing_nodes

# Clean the repo
do_test("cd ../src/; make clean")

# Make every target first, as some tests depend on multiple targets
for mode in ["debug", "release"]:
    # Build our targets
    do_test("cd ../src/; nice make -j",
            { "DEBUG"            : 1 if mode    == "debug"    else 0 },
            cmd_format="make")

for special in ["NO_EPOLL", "MOCK_IO_LAYER", "MOCK_CACHE_CHECK", "VALGRIND"]:
    do_test("cd ../src/; nice make -j",
            { "DEBUG" : 1,
              special : 1},
            cmd_format="make")

# Make sure auxillary tools compile
do_test("cd ../bench/stress-client/; make clean; make",
        {},
        cmd_format="make")
do_test("cd ../bench/serializer-bench/; make clean; make",
        {},
        cmd_format="make")

# For safety: ensure that nodes are terminated in case of exceptions
try:
    # Setup the EC2 testing nodes
    setup_testing_nodes()

    # Do quick smoke tests in some environments
    for (mode, checker, protocol) in [("debug", "valgrind", "text")]:
        # VERY basic tests
        do_test_cloud("integration/append_prepend.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol })

        do_test_cloud("integration/cas.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol })

        do_test_cloud("integration/flags.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol })

        do_test_cloud("integration/incr_decr.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol })

        do_test_cloud("integration/expiration.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol })

        do_test_cloud("integration/big_values.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol })

        do_test_cloud("integration/serial_mix.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol,
                  "duration"    : 10,
                  "fsck"        : True},
                repeat=1, timeout=60)

        do_test_cloud("integration/rget.py",
                { "auto"        : True,
                  "mode"        : mode,
                  "no-valgrind" : not checker,
                  "protocol"    : protocol }, timeout=45)

        # More advanced tests in various cores/slices configuration
        for (cores, slices) in [(1, 1)]:
            do_test_cloud("integration/many_keys.py",
                    { "auto"        : True,
                      "mode"        : mode,
                      "no-valgrind" : not checker,
                      "protocol"    : protocol,
                      "cores"       : cores,
                      "slices"      : slices,
                      "num-keys"    : 500 })

            do_test_cloud("integration/serial_mix.py",
                    { "auto"        : True,
                      "mode"        : mode,
                      "no-valgrind" : not checker,
                      "protocol"    : protocol,
                      "cores"       : cores,
                      "slices"      : slices,
                      "duration"    : 10 },
                    repeat=2, timeout=35)

            do_test_cloud("integration/serial_mix.py",
                    { "auto"        : True,
                      "mode"        : mode,
                      "no-valgrind" : not checker,
                      "protocol"    : protocol,
                      "cores"       : cores,
                      "slices"      : slices,
                      "duration"    : 10,
                      "restart-server-prob" : "0.0005" },
                    repeat=2, timeout=35)

            do_test_cloud("integration/multi_serial_mix.py",
                    { "auto"        : True,
                      "mode"        : mode,
                      "no-valgrind" : not checker,
                      "protocol"    : protocol,
                      "cores"       : cores,
                      "slices"      : slices,
                      "duration"    : 10 },
                    repeat=2, timeout=35)

            do_test_cloud("integration/multi_serial_mix.py",
                    { "auto"        : True,
                      "mode"        : mode,
                      "no-valgrind" : not checker,
                      "protocol"    : protocol,
                      "cores"       : cores,
                      "slices"      : slices,
                      "memory"      : 7,
                      "diff-log-size" : 2,
                      "duration"    : 10 },
                    repeat=2, timeout=35)

    # Report the results
    report_cloud()

finally:
    terminate_testing_nodes()

<|endoftext|>


<commit_before>import pytest
import datefinder
from dateutil import tz, parser
from datetime import datetime
try:
    from unittest import mock
except ImportError:
    import mock
import sys, logging
logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
logger = logging.getLogger(__name__)

@pytest.mark.parametrize('date_string, expected_parse_arg, expected_captures, expected_date', [
    (
        'due on Tuesday Jul 22, 2014 eastern standard time',
        'tuesday jul 22, 2014',
        { 'timezones': ['eastern'] },
        datetime(2014, 7, 22).replace(tzinfo=tz.gettz('EST'))
    ),
    (
        'Friday pAcific stanDard time',
        'friday',
        { 'timezones': ['pacific'] },
        parser.parse('friday').replace(tzinfo=tz.gettz('PST'))
    ),
    (
        '13/03/2014 Central Daylight Savings Time',
        '13/03/2014',
        { 'timezones': ['central'] },
        datetime(2014, 3, 13).replace(tzinfo=tz.gettz('CST'))
    ),
    # assert dateutil.parse successfully
    # handles tz-naive date strings
    # and returns naive datetime objects
    (
        ' on 12/24/2015 at 2pm ',
        '12/24/2015 at 2pm',
        { 'timezones': [] },
        datetime(2015, 12, 24, 14, 0)
    ),
])
def test_parse_date_string_find_replace(date_string, expected_parse_arg, expected_captures, expected_date):
    dt = datefinder.DateFinder()
    with mock.patch.object(parser, 'parse', wraps=parser.parse) as spy:
        actual_datetime = dt.parse_date_string(date_string, expected_captures)
        spy.assert_called_with(expected_parse_arg)
        logger.debug("acutal={}  expected={}".format(actual_datetime, expected_date))
        assert actual_datetime == expected_date

@pytest.mark.parametrize('date_string, expected_parse_arg, expected_captures, expected_date', [
    # test a tz abbreviation that
    # dateutil.tz.gettz cannot find
    # and will return None for
    (
        ' on 11-20-2015 6pm CST ',
        '11-20-2015 6pm',
        { 'timezones': ['CST'] },
        datetime(2015, 11, 20, 18, 0)
    ),
    # test a tz abbreviation that
    # dateutil.tz.gettz cannot find
    # and will return None for
    (
        ' on 11-20-2015 6am IRST ',
        '11-20-2015 6am',
        { 'timezones': ['IRST'] },
        datetime(2015, 11, 20, 6, 0)
    )
])
def test_parse_date_string_find_replace_nonexistent_tzinfo(date_string, expected_parse_arg, expected_captures, expected_date):
    '''
    mimic what happens when dateutil.tz.gettz tries
    to find a non-existent tzinfo string with mocks
    because some operating systems might resolve 'CST' and 'IRST'

    :param date_string:
    :param expected_parse_arg:
    :param expected_captures:
    :param expected_date:
    :return:
    '''
    dt = datefinder.DateFinder()
    with mock.patch.object(tz, 'gettz', wraps=tz.gettz) as mock_gettz:
        mock_gettz.return_value = None
        actual_datetime = dt.parse_date_string(date_string, expected_captures)
        mock_gettz.assert_called_with(expected_captures['timezones'][0])
        logger.debug("acutal={}  expected={}".format(actual_datetime, expected_date))
        assert actual_datetime == expected_date

# @pytest.mark.parametrize('date_string, expected_exception', [
#     # assert dateutil.parser.parse
#     # throws ValueError on bad date string input
#     (
#         'to barf',
#         ValueError
#     ),
# ])
# def test_dateutil_parse_throws_value_error(date_string, expected_exception):
#     dt = datefinder.DateFinder()
#     pytest.raises(expected_exception, dt.parse_date_string, *[date_string], **{'captures':{}})
<commit_msg>Update nonexistent_tzinfo tests<commit_after>import pytest
import warnings
import datefinder
from dateutil import tz, parser
from datetime import datetime
try:
    from unittest import mock
except ImportError:
    import mock
import sys, logging
logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
logger = logging.getLogger(__name__)

@pytest.mark.parametrize('date_string, expected_parse_args, expected_parse_kwargs, expected_captures, expected_date', [
    (
        'due on Tuesday Jul 22, 2014 eastern standard time',
        ['tuesday jul 22, 2014',],
        {'default': None},
        { 'timezones': ['eastern'] },
        datetime(2014, 7, 22).replace(tzinfo=tz.gettz('EST'))
    ),
    (
        'Friday pAcific stanDard time',
        ['friday',],
        {'default': None},
        { 'timezones': ['pacific'] },
        parser.parse('friday').replace(tzinfo=tz.gettz('PST'))
    ),
    (
        '13/03/2014 Central Daylight Savings Time',
        ['13/03/2014',],
        {'default': None},
        { 'timezones': ['central'] },
        datetime(2014, 3, 13).replace(tzinfo=tz.gettz('CST'))
    ),
    # assert dateutil.parse successfully
    # handles tz-naive date strings
    # and returns naive datetime objects
    (
        '12/24/2015 at 2pm',
        ['12/24/2015 at 2pm',],
        {'default': None},
        { 'timezones': [] },
        datetime(2015, 12, 24, 14, 0)
    ),
])
def test_parse_date_string_find_replace(date_string, expected_parse_args, expected_parse_kwargs, expected_captures, expected_date):
    dt = datefinder.DateFinder()
    with mock.patch.object(parser, 'parse', wraps=parser.parse) as spy:
        actual_datetime = dt.parse_date_string(date_string, expected_captures)
        spy.assert_called_with(*expected_parse_args, **expected_parse_kwargs)
        logger.debug("actual={}  expected={}".format(actual_datetime, expected_date))
        assert actual_datetime == expected_date

@pytest.mark.parametrize('date_string, expected_parse_arg, expected_captures, expected_date', [
    # test a tz abbreviation that
    # dateutil.tz.gettz cannot find
    # and will return None for
    (
        ' on 11-20-2015 6pm CST ',
        '11-20-2015 6pm',
        { 'timezones': ['CST'] },
        datetime(2015, 11, 20, 18, 0)
    ),
    # test a tz abbreviation that
    # dateutil.tz.gettz cannot find
    # and will return None for
    (
        ' on 11-20-2015 6am IRST ',
        '11-20-2015 6am',
        { 'timezones': ['IRST'] },
        datetime(2015, 11, 20, 6, 0)
    )
])
def test_parse_date_string_find_replace_nonexistent_tzinfo(date_string, expected_parse_arg, expected_captures, expected_date):
    '''
    mimic what happens when dateutil tries
    to find a non-existent tzinfo string
    because some operating systems might resolve 'CST' and 'IRST' this should raise a warning.

    :param date_string:
    :param expected_parse_arg:
    :param expected_captures:
    :param expected_date:
    :return:
    '''
    with pytest.warns(parser.UnknownTimezoneWarning):
        warnings.simplefilter('always')
        dt = datefinder.DateFinder()
        actual_datetime = dt.parse_date_string(date_string, expected_captures)
        logger.debug("actual={}  expected={}".format(actual_datetime, expected_date))
        assert actual_datetime == expected_date

# @pytest.mark.parametrize('date_string, expected_exception', [
#     # assert dateutil.parser.parse
#     # throws ValueError on bad date string input
#     (
#         'to barf',
#         ValueError
#     ),
# ])
# def test_dateutil_parse_throws_value_error(date_string, expected_exception):
#     dt = datefinder.DateFinder()
#     pytest.raises(expected_exception, dt.parse_date_string, *[date_string], **{'captures':{}})
<|endoftext|>


<commit_before>import retrieve_new

import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn import cross_validation

class BasicClassifier:
    ''' Simple wrapper class for a sklearn pipeline, with
    Multinomial Naive Bayes as the final estimator.
    '''
    def __init__(self, corpus=None, target=None):
        ''' Constructs pipeline, and trains itself if corpus and
        target params are set.
        '''
        vectorizer  = ('vect',  CountVectorizer())
        transformer = ('tfidf', TfidfTransformer())
        classifier  = ('clf',   MultinomialNB())
        self.pipeline = Pipeline([vectorizer, transformer, classifier])

        if corpus and target:
            self.train(corpus, target)

    def train(self, corpus, target):
        ''' Fit pipeline to feature corpus and target set of labels. '''
        return self.pipeline.fit(corpus, target)

    def test(self, corpus, target):
        ''' Score pipeline using test corpus and labels. '''
        return self.pipeline.score(corpus, target)

    def predict(self, properties):
        ''' Predict the class of a single string of property names. '''
        return self.pipeline.predict([properties])[0]

def keystrings(entities):
    return  [ ' '.join(entity["properties"]) for entity in entities ]

def class_labels(instances):
    return [ instance["class"] for instance in instances ]

def split_dataset(ratio):
    corpus, target = retrieve_new.dataset()
    return cross_validation.train_test_split(corpus, target, test_size=ratio)

def evalclassifier(test_ratio):
    split = split_dataset(test_ratio)
    train_corpus, test_corpus, train_labels, test_labels = split

    print("Training size:", len(train_corpus))
    classifier = BasicClassifier(train_corpus, train_labels)

    print("Testing size: ", len(test_corpus))
    return classifier.test(test_corpus, test_labels)

def main():
    sizes = [0.1, 0.2, 0.3, 0.4, 0.5]

    for i, size in enumerate(sizes):
        print("Trial", i+1)
        print("\nAccuracy:", evalclassifier(size), "\n")
<commit_msg>added support for kfold cross-validation in basicclassifier<commit_after>import retrieve_new

import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn import cross_validation

class BasicClassifier:
    ''' Simple wrapper class for a sklearn pipeline, with
    Multinomial Naive Bayes as the final estimator.
    '''
    def __init__(self, corpus=None, target=None):
        ''' Constructs pipeline, and trains itself if corpus and
        target params are set.
        '''
        vectorizer  = ('vect',  CountVectorizer())
        transformer = ('tfidf', TfidfTransformer())
        classifier  = ('clf',   MultinomialNB())
        self.pipeline = Pipeline([vectorizer, transformer, classifier])

        if corpus is not None and target is not None:
            self.train(corpus, target)

    def train(self, corpus, target):
        ''' Fit pipeline to feature corpus and target set of labels. '''
        return self.pipeline.fit(corpus, target)

    def test(self, corpus, target):
        ''' Score pipeline using test corpus and labels. '''
        return self.pipeline.score(corpus, target)

    def predict(self, properties):
        ''' Predict the class of a single string of property names. '''
        return self.pipeline.predict([properties])[0]

def keystrings(entities):
    return  [ ' '.join(entity["properties"]) for entity in entities ]

def class_labels(instances):
    return [ instance["class"] for instance in instances ]

def split_dataset(ratio):
    corpus, target = retrieve_new.dataset()
    return cross_validation.train_test_split(corpus, target, test_size=ratio)

def evalclassifier(test_ratio):
    split = split_dataset(test_ratio)
    train_corpus, test_corpus, train_labels, test_labels = split

    print("Training size:", len(train_corpus))
    classifier = BasicClassifier(train_corpus, train_labels)

    print("Testing size: ", len(test_corpus))
    return classifier.test(test_corpus, test_labels)

def kfold_eval():
    corpus, target = retrieve_new.dataset()
    corpus, target = np.array(corpus), np.array(target)
    fold_indices = cross_validation.KFold(len(corpus), n_folds=5)

    for train, test in fold_indices:
        train_corpus, train_labels = corpus[train], target[train]
        test_corpus,  test_labels  = corpus[test],  target[test]
        classifier = BasicClassifier(train_corpus, train_labels)
        return classifier.test(test_corpus, test_labels)

def main():
    print(kfold_eval())
<|endoftext|>


<commit_before>#!/usr/bin/env python

###
# Copyright (c) 2002, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

"""
Handles configuration of the bot while it's running.
"""

import plugins

import conf
import utils
import ircdb
import ircutils
import registry
import privmsgs
import callbacks

###
# Now, to setup the registry.
###
import registry

class InvalidRegistryName(callbacks.Error):
    pass

def getWrapper(name):
    parts = name.split('.')
    if not parts or parts[0] != 'supybot':
        raise InvalidRegistryName, name
    group = conf.supybot
    parts.pop(0)
    while parts:
        try:
            group = getattr(group, parts.pop(0))
        except registry.NonExistentRegistryEntry:
            raise InvalidRegistryName, name
    return group

def getCapability(name):
    capability = 'owner' # Default to requiring the owner capability.
    parts = name.split('.')
    while parts:
        part = parts.pop()
        if ircutils.isChannel(part):
            # If a registry value has a channel in it, it requires a channel.op
            # capability, or so we assume.  We'll see if we're proven wrong.
            capability = ircdb.makeChannelCapability(part, 'op')
        ### Do more later, for specific capabilities/sections.
    return capability

class Config(callbacks.Privmsg):
    def callCommand(self, method, irc, msg, *L):
        try:
            callbacks.Privmsg.callCommand(self, method, irc, msg, *L)
        except InvalidRegistryName, e:
            irc.error('%r is not a valid configuration variable.' % e.args[0])
        except registry.InvalidRegistryValue, e:
            irc.error(str(e))

    def list(self, irc, msg, args):
        """<group>

        Returns the configuration variables available under the given
        configuration <group>.
        """
        name = privmsgs.getArgs(args)
        group = getWrapper(name)
        if hasattr(group, 'getValues'):
            try:
                L = zip(*group.getValues())[0]
                irc.reply(utils.commaAndify(L))
            except TypeError:
                irc.error('There don\'t seem to be any values in %r' % name)
        else:
            irc.error('%r is not a valid configuration group.' % name)

    def get(self, irc, msg, args):
        """<name>

        Shows the current value of the configuration variable <name>.
        """
        name = privmsgs.getArgs(args)
        wrapper = getWrapper(name)
        irc.reply(str(wrapper))

    def set(self, irc, msg, args):
        """<name> <value>

        Sets the current value of the configuration variable <name> to <value>.
        """
        (name, value) = privmsgs.getArgs(args, required=2)
        capability = getCapability(name)
        if ircdb.checkCapability(msg.prefix, capability):
            wrapper = getWrapper(name)
            wrapper.set(value)
            irc.replySuccess()
        else:
            irc.errorNoCapability(capability)

    def help(self, irc, msg, args):
        """<name>

        Returns the description of the configuration variable <name>.
        """
        name = privmsgs.getArgs(args)
        wrapper = getWrapper(name)
        irc.reply(wrapper.help)

    def reset(self, irc, msg, args):
        """<name>

        Resets the configuration variable <name> to its original value.
        """
        name = privmsgs.getArgs(args)
        capability = getCapability(name)
        if ircdb.checkCapability(msg.prefix, capability):
            wrapper = getWrapper(name)
            wrapper.reset()
            irc.replySuccess()
        else:
            irc.errorNoCapability(capability)

    def default(self, irc, msg, args):
        """<name>

        Returns the default value of the configuration variable <name>.
        """
        name = privmsgs.getArgs(args)
        wrapper = getWrapper(name)
        irc.reply(wrapper.default)

    def flush(self, irc, msg, args):
        """<filename>

        Flushes the current registry to the file given.
        """
        filename = privmsgs.getArgs(args)
        registry.close(conf.supybot, filename)
        irc.replySuccess()
    flush = privmsgs.checkCapability(flush, 'owner')

Class = Config

# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:
<commit_msg>Removed the reset command.  We can nest default, if we want.<commit_after>#!/usr/bin/env python

###
# Copyright (c) 2002, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

"""
Handles configuration of the bot while it's running.
"""

import plugins

import conf
import utils
import ircdb
import ircutils
import registry
import privmsgs
import callbacks

###
# Now, to setup the registry.
###
import registry

class InvalidRegistryName(callbacks.Error):
    pass

def getWrapper(name):
    parts = name.split('.')
    if not parts or parts[0] != 'supybot':
        raise InvalidRegistryName, name
    group = conf.supybot
    parts.pop(0)
    while parts:
        try:
            group = getattr(group, parts.pop(0))
        except registry.NonExistentRegistryEntry:
            raise InvalidRegistryName, name
    return group

def getCapability(name):
    capability = 'owner' # Default to requiring the owner capability.
    parts = name.split('.')
    while parts:
        part = parts.pop()
        if ircutils.isChannel(part):
            # If a registry value has a channel in it, it requires a channel.op
            # capability, or so we assume.  We'll see if we're proven wrong.
            capability = ircdb.makeChannelCapability(part, 'op')
        ### Do more later, for specific capabilities/sections.
    return capability

class Config(callbacks.Privmsg):
    def callCommand(self, method, irc, msg, *L):
        try:
            callbacks.Privmsg.callCommand(self, method, irc, msg, *L)
        except InvalidRegistryName, e:
            irc.error('%r is not a valid configuration variable.' % e.args[0])
        except registry.InvalidRegistryValue, e:
            irc.error(str(e))

    def list(self, irc, msg, args):
        """<group>

        Returns the configuration variables available under the given
        configuration <group>.
        """
        name = privmsgs.getArgs(args)
        group = getWrapper(name)
        if hasattr(group, 'getValues'):
            try:
                L = zip(*group.getValues())[0]
                irc.reply(utils.commaAndify(L))
            except TypeError:
                irc.error('There don\'t seem to be any values in %r' % name)
        else:
            irc.error('%r is not a valid configuration group.' % name)

    def get(self, irc, msg, args):
        """<name>

        Shows the current value of the configuration variable <name>.
        """
        name = privmsgs.getArgs(args)
        wrapper = getWrapper(name)
        irc.reply(str(wrapper))

    def set(self, irc, msg, args):
        """<name> <value>

        Sets the current value of the configuration variable <name> to <value>.
        """
        (name, value) = privmsgs.getArgs(args, required=2)
        capability = getCapability(name)
        if ircdb.checkCapability(msg.prefix, capability):
            wrapper = getWrapper(name)
            wrapper.set(value)
            irc.replySuccess()
        else:
            irc.errorNoCapability(capability)

    def help(self, irc, msg, args):
        """<name>

        Returns the description of the configuration variable <name>.
        """
        name = privmsgs.getArgs(args)
        wrapper = getWrapper(name)
        irc.reply(wrapper.help)

    def default(self, irc, msg, args):
        """<name>

        Returns the default value of the configuration variable <name>.
        """
        name = privmsgs.getArgs(args)
        wrapper = getWrapper(name)
        irc.reply(wrapper.default)

    def flush(self, irc, msg, args):
        """<filename>

        Flushes the current registry to the file given.
        """
        filename = privmsgs.getArgs(args)
        registry.close(conf.supybot, filename)
        irc.replySuccess()
    flush = privmsgs.checkCapability(flush, 'owner')

Class = Config

# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:
<|endoftext|>


<commit_before>from sanic import Sanic
from sanic.response import text
from sanic.exceptions import PayloadTooLarge

data_received_app = Sanic('data_received')
data_received_app.config.REQUEST_MAX_SIZE = 1
data_received_default_app = Sanic('data_received_default')
data_received_default_app.config.REQUEST_MAX_SIZE = 1
on_header_default_app = Sanic('on_header')
on_header_default_app.config.REQUEST_MAX_SIZE = 500

@data_received_app.route('/1')
async def handler1(request):
    return text('OK')

@data_received_app.exception(PayloadTooLarge)
def handler_exception(request, exception):
    return text('Payload Too Large from error_handler.', 413)

def test_payload_too_large_from_error_handler():
    response = data_received_app.test_client.get('/1', gather_request=False)
    assert response.status == 413
    assert response.text == 'Payload Too Large from error_handler.'

@data_received_default_app.route('/1')
async def handler2(request):
    return text('OK')

def test_payload_too_large_at_data_received_default():
    response = data_received_default_app.test_client.get(
        '/1', gather_request=False)
    assert response.status == 413
    assert response.text == 'Error: Payload Too Large'

@on_header_default_app.route('/1')
async def handler3(request):
    return text('OK')

def test_payload_too_large_at_on_header_default():
    data = 'a' * 1000
    response = on_header_default_app.test_client.post(
        '/1', gather_request=False, data=data)
    assert response.status == 413
    assert response.text == 'Error: Payload Too Large'
<commit_msg>Make these tests not so far apart<commit_after>from sanic import Sanic
from sanic.response import text
from sanic.exceptions import PayloadTooLarge

def test_payload_too_large_from_error_handler():
    data_received_app = Sanic('data_received')
    data_received_app.config.REQUEST_MAX_SIZE = 1

    @data_received_app.route('/1')
    async def handler1(request):
        return text('OK')

    @data_received_app.exception(PayloadTooLarge)
    def handler_exception(request, exception):
        return text('Payload Too Large from error_handler.', 413)

    response = data_received_app.test_client.get('/1', gather_request=False)
    assert response.status == 413
    assert response.text == 'Payload Too Large from error_handler.'

def test_payload_too_large_at_data_received_default():
    data_received_default_app = Sanic('data_received_default')
    data_received_default_app.config.REQUEST_MAX_SIZE = 1

    @data_received_default_app.route('/1')
    async def handler2(request):
        return text('OK')

    response = data_received_default_app.test_client.get(
        '/1', gather_request=False)
    assert response.status == 413
    assert response.text == 'Error: Payload Too Large'

def test_payload_too_large_at_on_header_default():
    on_header_default_app = Sanic('on_header')
    on_header_default_app.config.REQUEST_MAX_SIZE = 500

    @on_header_default_app.post('/1')
    async def handler3(request):
        return text('OK')

    data = 'a' * 1000
    response = on_header_default_app.test_client.post(
        '/1', gather_request=False, data=data)
    assert response.status == 413
    assert response.text == 'Error: Payload Too Large'
<|endoftext|>