<commit_before>"""
byceps.services.brand.settings_service
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:Copyright: 2006-2018 Jochen Kupperschmidt
:License: Modified BSD, see LICENSE for details.
"""

from typing import List, Optional

from ...database import db
from ...typing import BrandID

from .models.setting import Setting as DbSetting
from .transfer.models import BrandSetting

def create_setting(brand_id: BrandID, name: str, value: str) -> BrandSetting:
    """Create a setting for that brand."""
    setting = DbSetting(brand_id, name, value)

    db.session.add(setting)
    db.session.commit()

    return _db_entity_to_brand_setting(setting)

def find_setting(brand_id: BrandID, name: str) -> Optional[BrandSetting]:
    """Return the setting for that brand and with that name, or `None`
    if not found.
    """
    setting = DbSetting.query.get((brand_id, name))

    if setting is None:
        return None

    return _db_entity_to_brand_setting(setting)

def find_setting_value(brand_id: BrandID, name: str) -> Optional[str]:
    """Return the value of the setting for that brand and with that
    name, or `None` if not found.
    """
    setting = find_setting(brand_id, name)

    if setting is None:
        return None

    return setting.value

def get_settings(brand_id: BrandID) -> List[BrandSetting]:
    """Return all settings for that brand."""
    settings = DbSetting.query \
        .filter_by(brand_id=brand_id) \
        .all()

    return {_db_entity_to_party_setting(setting) for setting in settings}

def _db_entity_to_brand_setting(setting: DbSetting) -> BrandSetting:
    return BrandSetting(
        setting.brand_id,
        setting.name,
        setting.value,
    )
<commit_msg>Fix function reference<commit_after>"""
byceps.services.brand.settings_service
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:Copyright: 2006-2018 Jochen Kupperschmidt
:License: Modified BSD, see LICENSE for details.
"""

from typing import List, Optional

from ...database import db
from ...typing import BrandID

from .models.setting import Setting as DbSetting
from .transfer.models import BrandSetting

def create_setting(brand_id: BrandID, name: str, value: str) -> BrandSetting:
    """Create a setting for that brand."""
    setting = DbSetting(brand_id, name, value)

    db.session.add(setting)
    db.session.commit()

    return _db_entity_to_brand_setting(setting)

def find_setting(brand_id: BrandID, name: str) -> Optional[BrandSetting]:
    """Return the setting for that brand and with that name, or `None`
    if not found.
    """
    setting = DbSetting.query.get((brand_id, name))

    if setting is None:
        return None

    return _db_entity_to_brand_setting(setting)

def find_setting_value(brand_id: BrandID, name: str) -> Optional[str]:
    """Return the value of the setting for that brand and with that
    name, or `None` if not found.
    """
    setting = find_setting(brand_id, name)

    if setting is None:
        return None

    return setting.value

def get_settings(brand_id: BrandID) -> List[BrandSetting]:
    """Return all settings for that brand."""
    settings = DbSetting.query \
        .filter_by(brand_id=brand_id) \
        .all()

    return {_db_entity_to_brand_setting(setting) for setting in settings}

def _db_entity_to_brand_setting(setting: DbSetting) -> BrandSetting:
    return BrandSetting(
        setting.brand_id,
        setting.name,
        setting.value,
    )
<|endoftext|>


<commit_before><commit_msg>Create core_Python_9-4.py<commit_after><|endoftext|>


<commit_before>from corehq.apps.domain.models import Domain
from corehq.apps.hqadmin.reports import AdminUserReport, AdminAppReport
from corehq.apps.hqpillow_retry.views import PillowErrorsReport
from corehq.apps.reports.standard import (monitoring, inspect, export,
    deployments, sms, ivr)
from corehq.apps.receiverwrapper import reports as receiverwrapper
import phonelog.reports as phonelog
from corehq.apps.reports.commtrack import standard as commtrack_reports
from corehq.apps.reports.commtrack import maps as commtrack_maps
from corehq.apps.reports.commconnect import system_overview
from corehq.apps.fixtures.interface import FixtureViewInterface, FixtureEditInterface
import hashlib
from dimagi.utils.modules import to_function
import logging

from django.utils.translation import ugettext_noop as _, ugettext_lazy

def REPORTS(project):
    from corehq.apps.reports.standard.cases.basic import CaseListReport
    from corehq.apps.reports.standard.cases.careplan import make_careplan_reports
    from corehq.apps.reports.standard.maps import DemoMapReport, DemoMapReport2, DemoMapCaseList

    reports = [
        (ugettext_lazy("Monitor Workers"), (
            monitoring.WorkerActivityReport,
            monitoring.DailyFormStatsReport,
            monitoring.SubmissionsByFormReport,
            monitoring.FormCompletionTimeReport,
            monitoring.CaseActivityReport,
            monitoring.FormCompletionVsSubmissionTrendsReport,
            monitoring.WorkerActivityTimes,
        )),
        (ugettext_lazy("Inspect Data"), (
            inspect.SubmitHistory, CaseListReport,
        )),
        (ugettext_lazy("Manage Deployments"), (
            deployments.ApplicationStatusReport,
            receiverwrapper.SubmissionErrorReport,
            phonelog.FormErrorReport,
            phonelog.DeviceLogDetailsReport
        )),
        (ugettext_lazy("Demos for Previewers"), (
            DemoMapReport, DemoMapReport2, DemoMapCaseList,
        )),
    ]

    if project.commtrack_enabled:
        reports.insert(0, (ugettext_lazy("Commtrack"), (
            commtrack_reports.CurrentStockStatusReport,
            commtrack_reports.AggregateStockStatusReport,
            commtrack_reports.ReportingRatesReport,
            # commtrack_reports.RequisitionReport,
            commtrack_maps.StockStatusMapReport,
            commtrack_maps.ReportingStatusMapReport,
        )))

    if project.has_careplan:
        from corehq.apps.app_manager.models import CareplanConfig
        config = CareplanConfig.for_domain(project.name)
        if config:
            cp_reports = tuple(make_careplan_reports(config))
            reports.insert(0, (ugettext_lazy("Care Plans"), cp_reports))

    from corehq.apps.accounting.utils import domain_has_privilege
    messaging_reports = []

    project_can_use_sms = domain_has_privilege(project.name, privileges.OUTBOUND_SMS)
    if project_can_use_sms:
        messaging_reports.extend([
            sms.MessagesReport,
        ])
    # always have this historical report visible
    messaging_reports.append(sms.MessageLogReport)

    project_can_use_inbound_sms = domain_has_privilege(project.name, privileges.INBOUND_SMS)
    if project_can_use_inbound_sms:
        messaging_reports.extend([
            ivr.CallLogReport,
            ivr.ExpectedCallbackReport,
            system_overview.SystemOverviewReport,
            system_overview.SystemUsersReport,
        ])

    messaging_reports += getattr(Domain.get_module_by_name(project.name), 'MESSAGING_REPORTS', ())

    messaging = (lambda project, user: (
        ugettext_lazy("Logs") if project.commtrack_enabled else ugettext_lazy("Messaging")), messaging_reports)

    if project.commconnect_enabled:
        reports.insert(0, messaging)
    else:
        reports.append(messaging)

    reports.extend(dynamic_reports(project))

    return reports

def dynamic_reports(project):
    """include any reports that can be configured/customized with static parameters for this domain"""
    for reportset in project.dynamic_reports:
        yield (reportset.section_title, filter(None, (make_dynamic_report(report, [reportset.section_title]) for report in reportset.reports)))

def make_dynamic_report(report_config, keyprefix):
    """create a report class the descends from a generic report class but has specific parameters set"""
    # a unique key to distinguish this particular configuration of the generic report
    report_key = keyprefix + [report_config.report, report_config.name]
    slug = hashlib.sha1(':'.join(report_key)).hexdigest()[:12]
    kwargs = dict(report_config.kwargs)
    kwargs.update({
            'name': report_config.name,
            'slug': slug,
        })
    if report_config.previewers_only:
        # note this is a classmethod that will be injected into the dynamic class below
        @classmethod
        def show_in_navigation(cls, domain=None, project=None, user=None):
            return user and user.is_previewer()
        kwargs['show_in_navigation'] = show_in_navigation

    try:
        metaclass = to_function(report_config.report, failhard=True)
    except StandardError:
        logging.error('dynamic report config for [%s] is invalid' % report_config.report)
        return None

    # dynamically create a report class
    return type('DynamicReport%s' % slug, (metaclass,), kwargs)

from corehq.apps.data_interfaces.interfaces import CaseReassignmentInterface
from corehq.apps.importer.base import ImportCases

DATA_INTERFACES = (
    (ugettext_lazy("Export Data"), (
        export.ExcelExportReport,
        export.CaseExportReport,
        export.DeidExportReport,
    )),
)

EDIT_DATA_INTERFACES = (
    (ugettext_lazy('Edit Data'), (
        CaseReassignmentInterface,
        ImportCases
    )),
)

FIXTURE_INTERFACES = (
    (_('Lookup Tables'), (
        FixtureEditInterface,
        FixtureViewInterface,
    )),
)

from corehq.apps.adm.reports.supervisor import SupervisorReportsADMSection

ADM_SECTIONS = (
    (_('Supervisor Report'), (
        SupervisorReportsADMSection,
    )),
)

from corehq.apps.adm.admin import columns, reports

ADM_ADMIN_INTERFACES = (
    (_("ADM Default Columns"), (
        columns.ReducedADMColumnInterface,
        columns.DaysSinceADMColumnInterface,
        columns.ConfigurableADMColumnInterface
    )),
    (_("ADM Default Reports"), (
        reports.ADMReportAdminInterface,
    ))
)

from corehq.apps.indicators.admin import document_indicators, couch_indicators, dynamic_indicators

INDICATOR_ADMIN_INTERFACES = (
    (_("Form Based Indicators"), (
        document_indicators.FormLabelIndicatorDefinitionAdminInterface,
        document_indicators.FormAliasIndicatorDefinitionAdminInterface,
        document_indicators.CaseDataInFormAdminInterface,
    )),
    (_("Case Based Indicators"), (
        document_indicators.FormDataInCaseAdminInterface,
    )),
    (_("Dynamic Indicators"), (
        dynamic_indicators.CombinedIndicatorAdminInterface,
    )),
    (_("Couch Based Indicators"), (
        couch_indicators.CouchIndicatorAdminInterface,
        couch_indicators.CountUniqueCouchIndicatorAdminInterface,
        couch_indicators.MedianCouchIndicatorAdminInterface,
        couch_indicators.SumLastEmittedCouchIndicatorAdminInterface,
    )),
)

from corehq.apps.announcements.interface import (
    ManageGlobalHQAnnouncementsInterface,
    ManageReportAnnouncementsInterface,
)

ANNOUNCEMENTS_ADMIN_INTERFACES = (
    (_("Manage Announcements"), (
        ManageGlobalHQAnnouncementsInterface,
        ManageReportAnnouncementsInterface,
    )),
)

from corehq.apps.accounting.interface import (
    AccountingInterface,
    SubscriptionInterface,
    SoftwarePlanInterface,
    InvoiceInterface,
    PaymentRecordInterface,
)

from corehq.apps.smsbillables.interface import (
    SMSBillablesInterface,
    SMSGatewayFeeCriteriaInterface,
)

ACCOUNTING_ADMIN_INTERFACES = (
    (_("Accounting Admin"), (
        AccountingInterface,
        SubscriptionInterface,
        SoftwarePlanInterface,
        InvoiceInterface,
        SMSBillablesInterface,
        SMSGatewayFeeCriteriaInterface,
        PaymentRecordInterface,
    )),
)

from corehq.apps.appstore.interfaces import CommCareExchangeAdvanced

APPSTORE_INTERFACES = (
    (_('App Store'), (
        CommCareExchangeAdvanced,
    )),
)

from corehq.apps.reports.standard.domains import OrgDomainStatsReport, AdminDomainStatsReport

BASIC_REPORTS = (
    (_('Project Stats'), (
        OrgDomainStatsReport,
    )),
)

ADMIN_REPORTS = (
    (_('Domain Stats'), (
        AdminDomainStatsReport,
        AdminUserReport,
        AdminAppReport,
        PillowErrorsReport
    )),
)

from corehq.apps.hqwebapp.models import *

TABS = (
    ProjectInfoTab,
    ReportsTab,
    ProjectDataTab,
    CommTrackSetupTab,
    ProjectUsersTab,
    ApplicationsTab,
    CloudcareTab,
    MessagingTab,
    ExchangeTab,
    OrgReportTab,
    OrgSettingsTab, # separate menu?
    AdminTab,
)
<commit_msg>Reordered items in CommTrack Reports sub-menu<commit_after>from corehq.apps.domain.models import Domain
from corehq.apps.hqadmin.reports import AdminUserReport, AdminAppReport
from corehq.apps.hqpillow_retry.views import PillowErrorsReport
from corehq.apps.reports.standard import (monitoring, inspect, export,
    deployments, sms, ivr)
from corehq.apps.receiverwrapper import reports as receiverwrapper
import phonelog.reports as phonelog
from corehq.apps.reports.commtrack import standard as commtrack_reports
from corehq.apps.reports.commtrack import maps as commtrack_maps
from corehq.apps.reports.commconnect import system_overview
from corehq.apps.fixtures.interface import FixtureViewInterface, FixtureEditInterface
import hashlib
from dimagi.utils.modules import to_function
import logging

from django.utils.translation import ugettext_noop as _, ugettext_lazy

def REPORTS(project):
    from corehq.apps.reports.standard.cases.basic import CaseListReport
    from corehq.apps.reports.standard.cases.careplan import make_careplan_reports
    from corehq.apps.reports.standard.maps import DemoMapReport, DemoMapReport2, DemoMapCaseList

    reports = [
        (ugettext_lazy("Monitor Workers"), (
            monitoring.WorkerActivityReport,
            monitoring.DailyFormStatsReport,
            monitoring.SubmissionsByFormReport,
            monitoring.FormCompletionTimeReport,
            monitoring.CaseActivityReport,
            monitoring.FormCompletionVsSubmissionTrendsReport,
            monitoring.WorkerActivityTimes,
        )),
        (ugettext_lazy("Inspect Data"), (
            inspect.SubmitHistory, CaseListReport,
        )),
        (ugettext_lazy("Manage Deployments"), (
            deployments.ApplicationStatusReport,
            receiverwrapper.SubmissionErrorReport,
            phonelog.FormErrorReport,
            phonelog.DeviceLogDetailsReport
        )),
        (ugettext_lazy("Demos for Previewers"), (
            DemoMapReport, DemoMapReport2, DemoMapCaseList,
        )),
    ]

    if project.commtrack_enabled:
        reports.insert(0, (ugettext_lazy("Commtrack"), (
            commtrack_reports.AggregateStockStatusReport,
            commtrack_reports.CurrentStockStatusReport,
            commtrack_maps.StockStatusMapReport,
            commtrack_reports.ReportingRatesReport,
            commtrack_maps.ReportingStatusMapReport,
            # commtrack_reports.RequisitionReport,
        )))

    if project.has_careplan:
        from corehq.apps.app_manager.models import CareplanConfig
        config = CareplanConfig.for_domain(project.name)
        if config:
            cp_reports = tuple(make_careplan_reports(config))
            reports.insert(0, (ugettext_lazy("Care Plans"), cp_reports))

    from corehq.apps.accounting.utils import domain_has_privilege
    messaging_reports = []

    project_can_use_sms = domain_has_privilege(project.name, privileges.OUTBOUND_SMS)
    if project_can_use_sms:
        messaging_reports.extend([
            sms.MessagesReport,
        ])
    # always have this historical report visible
    messaging_reports.append(sms.MessageLogReport)

    project_can_use_inbound_sms = domain_has_privilege(project.name, privileges.INBOUND_SMS)
    if project_can_use_inbound_sms:
        messaging_reports.extend([
            ivr.CallLogReport,
            ivr.ExpectedCallbackReport,
            system_overview.SystemOverviewReport,
            system_overview.SystemUsersReport,
        ])

    messaging_reports += getattr(Domain.get_module_by_name(project.name), 'MESSAGING_REPORTS', ())

    messaging = (lambda project, user: (
        ugettext_lazy("Logs") if project.commtrack_enabled else ugettext_lazy("Messaging")), messaging_reports)

    if project.commconnect_enabled:
        reports.insert(0, messaging)
    else:
        reports.append(messaging)

    reports.extend(dynamic_reports(project))

    return reports

def dynamic_reports(project):
    """include any reports that can be configured/customized with static parameters for this domain"""
    for reportset in project.dynamic_reports:
        yield (reportset.section_title, filter(None, (make_dynamic_report(report, [reportset.section_title]) for report in reportset.reports)))

def make_dynamic_report(report_config, keyprefix):
    """create a report class the descends from a generic report class but has specific parameters set"""
    # a unique key to distinguish this particular configuration of the generic report
    report_key = keyprefix + [report_config.report, report_config.name]
    slug = hashlib.sha1(':'.join(report_key)).hexdigest()[:12]
    kwargs = dict(report_config.kwargs)
    kwargs.update({
            'name': report_config.name,
            'slug': slug,
        })
    if report_config.previewers_only:
        # note this is a classmethod that will be injected into the dynamic class below
        @classmethod
        def show_in_navigation(cls, domain=None, project=None, user=None):
            return user and user.is_previewer()
        kwargs['show_in_navigation'] = show_in_navigation

    try:
        metaclass = to_function(report_config.report, failhard=True)
    except StandardError:
        logging.error('dynamic report config for [%s] is invalid' % report_config.report)
        return None

    # dynamically create a report class
    return type('DynamicReport%s' % slug, (metaclass,), kwargs)

from corehq.apps.data_interfaces.interfaces import CaseReassignmentInterface
from corehq.apps.importer.base import ImportCases

DATA_INTERFACES = (
    (ugettext_lazy("Export Data"), (
        export.ExcelExportReport,
        export.CaseExportReport,
        export.DeidExportReport,
    )),
)

EDIT_DATA_INTERFACES = (
    (ugettext_lazy('Edit Data'), (
        CaseReassignmentInterface,
        ImportCases
    )),
)

FIXTURE_INTERFACES = (
    (_('Lookup Tables'), (
        FixtureEditInterface,
        FixtureViewInterface,
    )),
)

from corehq.apps.adm.reports.supervisor import SupervisorReportsADMSection

ADM_SECTIONS = (
    (_('Supervisor Report'), (
        SupervisorReportsADMSection,
    )),
)

from corehq.apps.adm.admin import columns, reports

ADM_ADMIN_INTERFACES = (
    (_("ADM Default Columns"), (
        columns.ReducedADMColumnInterface,
        columns.DaysSinceADMColumnInterface,
        columns.ConfigurableADMColumnInterface
    )),
    (_("ADM Default Reports"), (
        reports.ADMReportAdminInterface,
    ))
)

from corehq.apps.indicators.admin import document_indicators, couch_indicators, dynamic_indicators

INDICATOR_ADMIN_INTERFACES = (
    (_("Form Based Indicators"), (
        document_indicators.FormLabelIndicatorDefinitionAdminInterface,
        document_indicators.FormAliasIndicatorDefinitionAdminInterface,
        document_indicators.CaseDataInFormAdminInterface,
    )),
    (_("Case Based Indicators"), (
        document_indicators.FormDataInCaseAdminInterface,
    )),
    (_("Dynamic Indicators"), (
        dynamic_indicators.CombinedIndicatorAdminInterface,
    )),
    (_("Couch Based Indicators"), (
        couch_indicators.CouchIndicatorAdminInterface,
        couch_indicators.CountUniqueCouchIndicatorAdminInterface,
        couch_indicators.MedianCouchIndicatorAdminInterface,
        couch_indicators.SumLastEmittedCouchIndicatorAdminInterface,
    )),
)

from corehq.apps.announcements.interface import (
    ManageGlobalHQAnnouncementsInterface,
    ManageReportAnnouncementsInterface,
)

ANNOUNCEMENTS_ADMIN_INTERFACES = (
    (_("Manage Announcements"), (
        ManageGlobalHQAnnouncementsInterface,
        ManageReportAnnouncementsInterface,
    )),
)

from corehq.apps.accounting.interface import (
    AccountingInterface,
    SubscriptionInterface,
    SoftwarePlanInterface,
    InvoiceInterface,
    PaymentRecordInterface,
)

from corehq.apps.smsbillables.interface import (
    SMSBillablesInterface,
    SMSGatewayFeeCriteriaInterface,
)

ACCOUNTING_ADMIN_INTERFACES = (
    (_("Accounting Admin"), (
        AccountingInterface,
        SubscriptionInterface,
        SoftwarePlanInterface,
        InvoiceInterface,
        SMSBillablesInterface,
        SMSGatewayFeeCriteriaInterface,
        PaymentRecordInterface,
    )),
)

from corehq.apps.appstore.interfaces import CommCareExchangeAdvanced

APPSTORE_INTERFACES = (
    (_('App Store'), (
        CommCareExchangeAdvanced,
    )),
)

from corehq.apps.reports.standard.domains import OrgDomainStatsReport, AdminDomainStatsReport

BASIC_REPORTS = (
    (_('Project Stats'), (
        OrgDomainStatsReport,
    )),
)

ADMIN_REPORTS = (
    (_('Domain Stats'), (
        AdminDomainStatsReport,
        AdminUserReport,
        AdminAppReport,
        PillowErrorsReport
    )),
)

from corehq.apps.hqwebapp.models import *

TABS = (
    ProjectInfoTab,
    ReportsTab,
    ProjectDataTab,
    CommTrackSetupTab,
    ProjectUsersTab,
    ApplicationsTab,
    CloudcareTab,
    MessagingTab,
    ExchangeTab,
    OrgReportTab,
    OrgSettingsTab, # separate menu?
    AdminTab,
)
<|endoftext|>


<commit_before>import dedupe
import unittest
import numpy
import random
import itertools

class CoreTest(unittest.TestCase):
  def setUp(self) :
    random.seed(123)

  def test_random_pair(self) :
    self.assertRaises(ValueError, dedupe.core.randomPairs, 1, 10)
    assert dedupe.core.randomPairs(10, 10).any()
    assert dedupe.core.randomPairs(10*1000000000, 10).any()
    assert numpy.array_equal(dedupe.core.randomPairs(10, 5),
                             numpy.array([[ 1,  8],
                                          [ 5,  7],
                                          [ 1,  2],
                                          [ 3,  7],
                                          [ 2,  9]]))

class DedupeClassTest(unittest.TestCase):
  def test_initialize(self) :
    fields =  { 'name' : {'type': 'String'},
                'age'  : {'type': 'String'},
            }
    deduper = dedupe.Dedupe(fields)

    string_predicates = (dedupe.predicates.wholeFieldPredicate,
                         dedupe.predicates.tokenFieldPredicate,
                         dedupe.predicates.commonIntegerPredicate,
                         dedupe.predicates.sameThreeCharStartPredicate,
                         dedupe.predicates.sameFiveCharStartPredicate,
                         dedupe.predicates.sameSevenCharStartPredicate,
                         dedupe.predicates.nearIntegersPredicate,
                         dedupe.predicates.commonFourGram,
                         dedupe.predicates.commonSixGram)

    tfidf_string_predicates = tuple([dedupe.tfidf.TfidfPredicate(threshold)
                                     for threshold
                                     in [0.2, 0.4, 0.6, 0.8]])

    assert deduper.blocker_types == {'String' : string_predicates + tfidf_string_predicates}

class AffineGapTest(unittest.TestCase):
  def setUp(self):
    self.affineGapDistance = dedupe.affinegap.affineGapDistance
    self.normalizedAffineGapDistance = dedupe.affinegap.normalizedAffineGapDistance

  def test_affine_gap_correctness(self):
    assert self.affineGapDistance('a', 'b', -5, 5, 5, 1, 0.5) == 5
    assert self.affineGapDistance('ab', 'cd', -5, 5, 5, 1, 0.5) == 10
    assert self.affineGapDistance('ab', 'cde', -5, 5, 5, 1, 0.5) == 13
    assert self.affineGapDistance('a', 'cde', -5, 5, 5, 1, 0.5) == 8.5
    assert self.affineGapDistance('a', 'cd', -5, 5, 5, 1, 0.5) == 8
    assert self.affineGapDistance('b', 'a', -5, 5, 5, 1, 0.5) == 5
    assert self.affineGapDistance('a', 'a', -5, 5, 5, 1, 0.5) == -5
    assert numpy.isnan(self.affineGapDistance('a', '', -5, 5, 5, 1, 0.5))
    assert numpy.isnan(self.affineGapDistance('', '', -5, 5, 5, 1, 0.5))
    assert self.affineGapDistance('aba', 'aaa', -5, 5, 5, 1, 0.5) == -5
    assert self.affineGapDistance('aaa', 'aba', -5, 5, 5, 1, 0.5) == -5
    assert self.affineGapDistance('aaa', 'aa', -5, 5, 5, 1, 0.5) == -7
    assert self.affineGapDistance('aaa', 'a', -5, 5, 5, 1, 0.5) == -1.5
    assert numpy.isnan(self.affineGapDistance('aaa', '', -5, 5, 5, 1, 0.5))
    assert self.affineGapDistance('aaa', 'abba', -5, 5, 5, 1, 0.5) == 1

  def test_normalized_affine_gap_correctness(self):
    assert numpy.isnan(self.normalizedAffineGapDistance('', '', -5, 5, 5, 1, 0.5))

class ClusteringTest(unittest.TestCase):
  def setUp(self):
    # Fully connected star network
    self.dupes = (((1,2), .86),
                  ((1,3), .72),
                  ((1,4), .2),
                  ((1,5), .6),
                  ((2,3), .86),
                  ((2,4), .2),
                  ((2,5), .72),
                  ((3,4), .3),
                  ((3,5), .5),
                  ((4,5), .72))

  def test_hierarchical(self):
    hierarchical = dedupe.clustering.cluster
    assert hierarchical(self.dupes, 1) == []
    assert hierarchical(self.dupes, 0.5) == [set([1, 2, 3]), set([4,5])]
    assert hierarchical(self.dupes, 0) == [set([1, 2, 3, 4, 5])]

class BlockingTest(unittest.TestCase):
  def setUp(self):
    self.frozendict = dedupe.core.frozendict
    fields =  { 'name' : {'type': 'String'},
                'age'  : {'type': 'String'},
              }
    self.deduper = dedupe.Dedupe(fields)
    self.wholeFieldPredicate = dedupe.predicates.wholeFieldPredicate
    self.sameThreeCharStartPredicate = dedupe.predicates.sameThreeCharStartPredicate
    self.training_pairs = {
        0: [(self.frozendict({"name": "Bob", "age": "50"}),
             self.frozendict({"name": "Charlie", "age": "75"})),
            (self.frozendict({"name": "Meredith", "age": "40"}),
             self.frozendict({"name": "Sue", "age": "10"}))],
        1: [(self.frozendict({"name": "Jimmy", "age": "20"}),
             self.frozendict({"name": "Jimbo", "age": "21"})),
            (self.frozendict({"name": "Willy", "age": "35"}),
             self.frozendict({"name": "William", "age": "35"}))]
      }
    self.predicate_functions = (self.wholeFieldPredicate, self.sameThreeCharStartPredicate)

class PredicatesTest(unittest.TestCase):
  def test_predicates_correctness(self):
    field = '123 16th st'
    assert dedupe.predicates.wholeFieldPredicate(field) == ('123 16th st',)
    assert dedupe.predicates.tokenFieldPredicate(field) == ('123', '16th', 'st')
    assert dedupe.predicates.commonIntegerPredicate(field) == ('123', '16')
    assert dedupe.predicates.sameThreeCharStartPredicate(field) == ('123',)
    assert dedupe.predicates.sameFiveCharStartPredicate(field) == ('123 1',)
    assert dedupe.predicates.sameSevenCharStartPredicate(field) == ('123 16t',)
    assert dedupe.predicates.nearIntegersPredicate(field) == (15, 16, 17, 122, 123, 124)
    assert dedupe.predicates.commonFourGram(field) == ('123 ', '23 1', '3 16', ' 16t', '16th', '6th ', 'th s', 'h st')
    assert dedupe.predicates.commonSixGram(field) == ('123 16', '23 16t', '3 16th', ' 16th ', '16th s', '6th st')

class CoreTest(unittest.TestCase):
  def setUp(self):
    lst = ['name','age','address','dataset']
    data = [dict.fromkeys(xrange(20)) for x in range(10000)]
    data_for_constrained_matching = [dict.fromkeys(lst) for x in range(10000)]
    keys = range(len(data))
    self.data_d = dict(itertools.izip(keys,data))
    self.data_d_constrained_matching = dict(itertools.izip(keys,data_for_constrained_matching))

  def test_data_sample(self):
    assert dedupe.convenience.dataSample(self.data_d,1)
    assert dedupe.convenience.dataSample(self.data_d,150000)

if __name__ == "__main__":
    unittest.main()

<commit_msg>Rename to ConvenienceTest<commit_after>import dedupe
import unittest
import numpy
import random
import itertools

class CoreTest(unittest.TestCase):
  def setUp(self) :
    random.seed(123)

  def test_random_pair(self) :
    self.assertRaises(ValueError, dedupe.core.randomPairs, 1, 10)
    assert dedupe.core.randomPairs(10, 10).any()
    assert dedupe.core.randomPairs(10*1000000000, 10).any()
    assert numpy.array_equal(dedupe.core.randomPairs(10, 5),
                             numpy.array([[ 1,  8],
                                          [ 5,  7],
                                          [ 1,  2],
                                          [ 3,  7],
                                          [ 2,  9]]))

class DedupeClassTest(unittest.TestCase):
  def test_initialize(self) :
    fields =  { 'name' : {'type': 'String'},
                'age'  : {'type': 'String'},
            }
    deduper = dedupe.Dedupe(fields)

    string_predicates = (dedupe.predicates.wholeFieldPredicate,
                         dedupe.predicates.tokenFieldPredicate,
                         dedupe.predicates.commonIntegerPredicate,
                         dedupe.predicates.sameThreeCharStartPredicate,
                         dedupe.predicates.sameFiveCharStartPredicate,
                         dedupe.predicates.sameSevenCharStartPredicate,
                         dedupe.predicates.nearIntegersPredicate,
                         dedupe.predicates.commonFourGram,
                         dedupe.predicates.commonSixGram)

    tfidf_string_predicates = tuple([dedupe.tfidf.TfidfPredicate(threshold)
                                     for threshold
                                     in [0.2, 0.4, 0.6, 0.8]])

    assert deduper.blocker_types == {'String' : string_predicates + tfidf_string_predicates}

class AffineGapTest(unittest.TestCase):
  def setUp(self):
    self.affineGapDistance = dedupe.affinegap.affineGapDistance
    self.normalizedAffineGapDistance = dedupe.affinegap.normalizedAffineGapDistance

  def test_affine_gap_correctness(self):
    assert self.affineGapDistance('a', 'b', -5, 5, 5, 1, 0.5) == 5
    assert self.affineGapDistance('ab', 'cd', -5, 5, 5, 1, 0.5) == 10
    assert self.affineGapDistance('ab', 'cde', -5, 5, 5, 1, 0.5) == 13
    assert self.affineGapDistance('a', 'cde', -5, 5, 5, 1, 0.5) == 8.5
    assert self.affineGapDistance('a', 'cd', -5, 5, 5, 1, 0.5) == 8
    assert self.affineGapDistance('b', 'a', -5, 5, 5, 1, 0.5) == 5
    assert self.affineGapDistance('a', 'a', -5, 5, 5, 1, 0.5) == -5
    assert numpy.isnan(self.affineGapDistance('a', '', -5, 5, 5, 1, 0.5))
    assert numpy.isnan(self.affineGapDistance('', '', -5, 5, 5, 1, 0.5))
    assert self.affineGapDistance('aba', 'aaa', -5, 5, 5, 1, 0.5) == -5
    assert self.affineGapDistance('aaa', 'aba', -5, 5, 5, 1, 0.5) == -5
    assert self.affineGapDistance('aaa', 'aa', -5, 5, 5, 1, 0.5) == -7
    assert self.affineGapDistance('aaa', 'a', -5, 5, 5, 1, 0.5) == -1.5
    assert numpy.isnan(self.affineGapDistance('aaa', '', -5, 5, 5, 1, 0.5))
    assert self.affineGapDistance('aaa', 'abba', -5, 5, 5, 1, 0.5) == 1

  def test_normalized_affine_gap_correctness(self):
    assert numpy.isnan(self.normalizedAffineGapDistance('', '', -5, 5, 5, 1, 0.5))

class ClusteringTest(unittest.TestCase):
  def setUp(self):
    # Fully connected star network
    self.dupes = (((1,2), .86),
                  ((1,3), .72),
                  ((1,4), .2),
                  ((1,5), .6),
                  ((2,3), .86),
                  ((2,4), .2),
                  ((2,5), .72),
                  ((3,4), .3),
                  ((3,5), .5),
                  ((4,5), .72))

  def test_hierarchical(self):
    hierarchical = dedupe.clustering.cluster
    assert hierarchical(self.dupes, 1) == []
    assert hierarchical(self.dupes, 0.5) == [set([1, 2, 3]), set([4,5])]
    assert hierarchical(self.dupes, 0) == [set([1, 2, 3, 4, 5])]

class BlockingTest(unittest.TestCase):
  def setUp(self):
    self.frozendict = dedupe.core.frozendict
    fields =  { 'name' : {'type': 'String'},
                'age'  : {'type': 'String'},
              }
    self.deduper = dedupe.Dedupe(fields)
    self.wholeFieldPredicate = dedupe.predicates.wholeFieldPredicate
    self.sameThreeCharStartPredicate = dedupe.predicates.sameThreeCharStartPredicate
    self.training_pairs = {
        0: [(self.frozendict({"name": "Bob", "age": "50"}),
             self.frozendict({"name": "Charlie", "age": "75"})),
            (self.frozendict({"name": "Meredith", "age": "40"}),
             self.frozendict({"name": "Sue", "age": "10"}))],
        1: [(self.frozendict({"name": "Jimmy", "age": "20"}),
             self.frozendict({"name": "Jimbo", "age": "21"})),
            (self.frozendict({"name": "Willy", "age": "35"}),
             self.frozendict({"name": "William", "age": "35"}))]
      }
    self.predicate_functions = (self.wholeFieldPredicate, self.sameThreeCharStartPredicate)

class PredicatesTest(unittest.TestCase):
  def test_predicates_correctness(self):
    field = '123 16th st'
    assert dedupe.predicates.wholeFieldPredicate(field) == ('123 16th st',)
    assert dedupe.predicates.tokenFieldPredicate(field) == ('123', '16th', 'st')
    assert dedupe.predicates.commonIntegerPredicate(field) == ('123', '16')
    assert dedupe.predicates.sameThreeCharStartPredicate(field) == ('123',)
    assert dedupe.predicates.sameFiveCharStartPredicate(field) == ('123 1',)
    assert dedupe.predicates.sameSevenCharStartPredicate(field) == ('123 16t',)
    assert dedupe.predicates.nearIntegersPredicate(field) == (15, 16, 17, 122, 123, 124)
    assert dedupe.predicates.commonFourGram(field) == ('123 ', '23 1', '3 16', ' 16t', '16th', '6th ', 'th s', 'h st')
    assert dedupe.predicates.commonSixGram(field) == ('123 16', '23 16t', '3 16th', ' 16th ', '16th s', '6th st')

class ConvenienceTest(unittest.TestCase):
  def setUp(self):
    lst = ['name','age','address','dataset']
    data = [dict.fromkeys(xrange(20)) for x in range(10000)]
    data_for_constrained_matching = [dict.fromkeys(lst) for x in range(10000)]
    keys = range(len(data))
    self.data_d = dict(itertools.izip(keys,data))
    self.data_d_constrained_matching = dict(itertools.izip(keys,data_for_constrained_matching))

  def test_data_sample(self):
    assert dedupe.convenience.dataSample(self.data_d,1)
    assert dedupe.convenience.dataSample(self.data_d,150000)

if __name__ == "__main__":
    unittest.main()

<|endoftext|>


<commit_before>from collections import defaultdict
from itertools import islice
import traceback

from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import HtmlFormatter

from celery.utils.mail import ErrorMail
from django.core import mail
from django.utils.log import AdminEmailHandler
from django.views.debug import get_exception_reporter_filter
from django.template.loader import render_to_string

def clean_exception(exception):
    """
    Takes an Exception instance and strips potentially sensitive information
    """
    from django.conf import settings
    if settings.DEBUG:
        return exception

    # couchdbkit doesn't provide a better way for us to catch this exception
    if (
        isinstance(exception, AssertionError) and
        exception.message.startswith('received an invalid response of type')
    ):
        message = ("It looks like couch returned an invalid response to "
                   "couchdbkit.  This could contain sensitive information, "
                   "so it's being redacted.")
        return exception.__class__(message)

    return exception

class HqAdminEmailHandler(AdminEmailHandler):
    """
    Custom AdminEmailHandler to include additional details which can be supplied as follows:

    logger.error(message,
        extra={
            'details': {'domain': 'demo', 'user': 'user1'}
        }
    )
    """
    def get_context(self, record):
        request = None
        try:
            request = record.request
            filter = get_exception_reporter_filter(request)
            request_repr = filter.get_request_repr(request)
        except Exception:
            request_repr = "Request repr() unavailable."

        tb_list = []
        code = None
        if record.exc_info:
            etype, _value, tb = record.exc_info
            value = clean_exception(_value)
            tb_list = ['Traceback (most recent call first):\n']
            formatted_exception = traceback.format_exception_only(etype, value)
            tb_list.extend(formatted_exception)
            extracted_tb = list(reversed(traceback.extract_tb(tb)))
            code = self.get_code(extracted_tb)
            tb_list.extend(traceback.format_list(extracted_tb))
            stack_trace = '\n'.join(tb_list)
            subject = '%s: %s' % (record.levelname,
                                  formatted_exception[0].strip() if formatted_exception else record.getMessage())
        else:
            stack_trace = 'No stack trace available'
            subject = '%s: %s' % (
                record.levelname,
                record.getMessage()
            )
        context = defaultdict(lambda: '')
        context.update({
            'subject': self.format_subject(subject),
            'message': record.getMessage(),
            'details': getattr(record, 'details', None),
            'tb_list': tb_list,
            'request_repr': request_repr,
            'stack_trace': stack_trace,
            'code': code,
        })
        if request:
            context.update({
                'get': request.GET,
                'post': request.POST,
                'method': request.method,
                'url': request.build_absolute_uri(),
            })
        return context

    def emit(self, record):
        context = self.get_context(record)

        message = "\n\n".join(filter(None, [
            self._clean_subject(context['message']),
            self.format_details(context['details']),
            context['stack_trace'],
            context['request_repr'],
        ]))
        html_message = render_to_string('hqadmin/email/error_email.html', context)
        mail.mail_admins(context['subject'], message, fail_silently=True,
                         html_message=html_message)

    def format_details(self, details):
        if details:
            formatted = '\n'.join('{item[0]}: {item[1]}'.format(item=item) for item in details.items())
            return 'Details:\n{}'.format(formatted)

    def get_code(self, extracted_tb):
        trace = next((trace for trace in extracted_tb if 'site-packages' not in trace[0]), None)
        if not trace:
            return None

        filename = trace[0]
        lineno = trace[1]
        offset = 10
        with open(filename) as f:
            code_context = list(islice(f, lineno - offset, lineno + offset))

        return highlight(''.join(code_context),
            PythonLexer(),
            HtmlFormatter(
                noclasses=True,
                linenos='table',
                hl_lines=[offset, offset],
                linenostart=(lineno - offset + 1),
        )
        )

    @classmethod
    def _clean_subject(cls, subject):
        # Django raises BadHeaderError if subject contains following bad_strings
        # to guard against Header Inejction.
        # see https://docs.djangoproject.com/en/1.8/topics/email/#preventing-header-injection
        # bad-strings list from http://nyphp.org/phundamentals/8_Preventing-Email-Header-Injection
        bad_strings = ["\r", "\n", "%0a", "%0d", "Content-Type:", "bcc:", "to:", "cc:"]
        replacement = "-"
        for i in bad_strings:
            subject.replace(i, replacement)
        return subject

class NotifyExceptionEmailer(HqAdminEmailHandler):
    def get_context(self, record):
        context = super(NotifyExceptionEmailer, self).get_context(record)
        context['subject'] = record.getMessage()
        return context

class SensitiveErrorMail(ErrorMail):
    """
    Extends Celery's ErrorMail class to prevents task args and kwargs from being printed in error emails.
    """
    replacement = '(excluded due to sensitive nature)'

    def format_body(self, context):
        context['args'] = self.replacement
        context['kwargs'] = self.replacement
        return self.body.strip() % context
<commit_msg>clean the subject not message<commit_after>from collections import defaultdict
from itertools import islice
import traceback

from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import HtmlFormatter

from celery.utils.mail import ErrorMail
from django.core import mail
from django.utils.log import AdminEmailHandler
from django.views.debug import get_exception_reporter_filter
from django.template.loader import render_to_string

def clean_exception(exception):
    """
    Takes an Exception instance and strips potentially sensitive information
    """
    from django.conf import settings
    if settings.DEBUG:
        return exception

    # couchdbkit doesn't provide a better way for us to catch this exception
    if (
        isinstance(exception, AssertionError) and
        exception.message.startswith('received an invalid response of type')
    ):
        message = ("It looks like couch returned an invalid response to "
                   "couchdbkit.  This could contain sensitive information, "
                   "so it's being redacted.")
        return exception.__class__(message)

    return exception

class HqAdminEmailHandler(AdminEmailHandler):
    """
    Custom AdminEmailHandler to include additional details which can be supplied as follows:

    logger.error(message,
        extra={
            'details': {'domain': 'demo', 'user': 'user1'}
        }
    )
    """
    def get_context(self, record):
        request = None
        try:
            request = record.request
            filter = get_exception_reporter_filter(request)
            request_repr = filter.get_request_repr(request)
        except Exception:
            request_repr = "Request repr() unavailable."

        tb_list = []
        code = None
        if record.exc_info:
            etype, _value, tb = record.exc_info
            value = clean_exception(_value)
            tb_list = ['Traceback (most recent call first):\n']
            formatted_exception = traceback.format_exception_only(etype, value)
            tb_list.extend(formatted_exception)
            extracted_tb = list(reversed(traceback.extract_tb(tb)))
            code = self.get_code(extracted_tb)
            tb_list.extend(traceback.format_list(extracted_tb))
            stack_trace = '\n'.join(tb_list)
            subject = '%s: %s' % (record.levelname,
                                  formatted_exception[0].strip() if formatted_exception else record.getMessage())
        else:
            stack_trace = 'No stack trace available'
            subject = '%s: %s' % (
                record.levelname,
                record.getMessage()
            )
        context = defaultdict(lambda: '')
        context.update({
            'subject': self.format_subject(subject),
            'message': record.getMessage(),
            'details': getattr(record, 'details', None),
            'tb_list': tb_list,
            'request_repr': request_repr,
            'stack_trace': stack_trace,
            'code': code,
        })
        if request:
            context.update({
                'get': request.GET,
                'post': request.POST,
                'method': request.method,
                'url': request.build_absolute_uri(),
            })
        return context

    def emit(self, record):
        context = self.get_context(record)

        message = "\n\n".join(filter(None, [
            context['message'],
            self.format_details(context['details']),
            context['stack_trace'],
            context['request_repr'],
        ]))
        html_message = render_to_string('hqadmin/email/error_email.html', context)
        mail.mail_admins(self._clean_subject(context['subject']), message, fail_silently=True,
                         html_message=html_message)

    def format_details(self, details):
        if details:
            formatted = '\n'.join('{item[0]}: {item[1]}'.format(item=item) for item in details.items())
            return 'Details:\n{}'.format(formatted)

    def get_code(self, extracted_tb):
        trace = next((trace for trace in extracted_tb if 'site-packages' not in trace[0]), None)
        if not trace:
            return None

        filename = trace[0]
        lineno = trace[1]
        offset = 10
        with open(filename) as f:
            code_context = list(islice(f, lineno - offset, lineno + offset))

        return highlight(''.join(code_context),
            PythonLexer(),
            HtmlFormatter(
                noclasses=True,
                linenos='table',
                hl_lines=[offset, offset],
                linenostart=(lineno - offset + 1),
        )
        )

    @classmethod
    def _clean_subject(cls, subject):
        # Django raises BadHeaderError if subject contains following bad_strings
        # to guard against Header Inejction.
        # see https://docs.djangoproject.com/en/1.8/topics/email/#preventing-header-injection
        # bad-strings list from http://nyphp.org/phundamentals/8_Preventing-Email-Header-Injection
        bad_strings = ["\r", "\n", "%0a", "%0d", "Content-Type:", "bcc:", "to:", "cc:"]
        replacement = "-"
        for i in bad_strings:
            subject.replace(i, replacement)
        return subject

class NotifyExceptionEmailer(HqAdminEmailHandler):
    def get_context(self, record):
        context = super(NotifyExceptionEmailer, self).get_context(record)
        context['subject'] = record.getMessage()
        return context

class SensitiveErrorMail(ErrorMail):
    """
    Extends Celery's ErrorMail class to prevents task args and kwargs from being printed in error emails.
    """
    replacement = '(excluded due to sensitive nature)'

    def format_body(self, context):
        context['args'] = self.replacement
        context['kwargs'] = self.replacement
        return self.body.strip() % context
<|endoftext|>


<commit_before>import unittest
import tempfile
import os
from gimmemotifs.denovo import gimme_motifs
from gimmemotifs.motif import read_motifs, motif_from_consensus
from gimmemotifs.comparison import MotifComparer
import shutil
from tempfile import mkdtemp
from time import sleep

class TestDenovo(unittest.TestCase):
    """ A test class to test gimme_motifs denovo """

    def setUp(self):
        self.outdir = mkdtemp()

    def test1_denovo(self):
        """ de novo motif prediction """

        gimme_motifs("test/data/denovo/input.fa", self.outdir,
            params={
                "tools":"Homer",#,MDmodule,BioProspector",
                "fraction":0.5,
                "background":"random"
                },
            filter_significant=True,
            cluster=True)

        fnames = ["motifs.pwm", "motif_report.html", "cluster_report.html",
                    "params.txt", "stats.random.txt"]

        with open(os.path.join(self.outdir, 'gimmemotifs.log')) as f:
            log = f.read()
        self.assertIn("clustering significant", log)

        # Check if all output files are there
        for fname in fnames:
            self.assertTrue(os.path.exists(os.path.join(self.outdir, fname)))

        # Check if correct motif is predicted
        predicted_motifs = read_motifs(open(os.path.join(self.outdir, "motifs.pwm")))
        ap1 = motif_from_consensus("TGASTCA")

        mc = MotifComparer()
        ap1_predicted = False
        for motif in predicted_motifs:
            match = mc.get_closest_match(ap1, motif)
            if match["TGASTCA"][1][3] < 1e-6:
                ap1_predicted = True
                break

        self.assertTrue(ap1_predicted)

    def tearDown(self):
        # remove output
        shutil.rmtree(self.outdir)

if __name__ == '__main__':
    unittest.main()

<commit_msg>updated test<commit_after>import unittest
import tempfile
import os
from gimmemotifs.denovo import gimme_motifs
from gimmemotifs.motif import read_motifs, motif_from_consensus
from gimmemotifs.comparison import MotifComparer
import shutil
from tempfile import mkdtemp
from time import sleep

class TestDenovo(unittest.TestCase):
    """ A test class to test gimme_motifs denovo """

    def setUp(self):
        self.outdir = mkdtemp()

    def test1_denovo(self):
        """ de novo motif prediction """

        gimme_motifs("test/data/denovo/input.fa", self.outdir,
            params={
                "tools":"Homer",#,MDmodule,BioProspector",
                "fraction":0.5,
                "background":"random"
                },
            filter_significant=True,
            cluster=True)

        fnames = ["motifs.pwm", "motif_report.html", "cluster_report.html",
                    "params.txt", "stats.random.txt"]

        with open(os.path.join(self.outdir, 'gimmemotifs.log')) as f:
            log = f.read()
        self.assertIn("clustering significant", log)

        # Check if all output files are there
        for fname in fnames:
            self.assertTrue(os.path.exists(os.path.join(self.outdir, fname)))

        # Check if correct motif is predicted
        with open(os.path.join(self.outdir, "motifs.pwm")) as f:
            predicted_motifs = read_motifs(f)
        ap1 = motif_from_consensus("TGASTCA")

        mc = MotifComparer()
        ap1_predicted = False
        for motif in predicted_motifs:
            match = mc.get_closest_match(ap1, motif)
            if match["TGASTCA"][1][3] < 1e-6:
                ap1_predicted = True
                break

        self.assertTrue(ap1_predicted)

    def tearDown(self):
        # remove output
        shutil.rmtree(self.outdir)

if __name__ == '__main__':
    unittest.main()

<|endoftext|>


<commit_before>import discord
from discord.ext import commands
from cogs.utils.dataIO import dataIO, fileIO
from collections import namedtuple, defaultdict
from datetime import datetime
from random import randint
from copy import deepcopy
from .utils import checks
from __main__ import send_cmd_help
import os
import time
import logging
from .economy import Bank, BankError, AccountAlreadyExists, NoAccount, InsufficientBalance, SameSenderAndReceiver

class Counter:
    """My custom cog that does stuff"""
    def __init__(self, bot):
        self.bot = bot
        self.bank = Bank(bot, "data/economy/bank.json")

    @commands.command(pass_context=True, no_pm=True)
    async def boop(self, ctx, user : discord.Member=None):
        """Boop!"""
        if user != None:
            author = ctx.message.author
            if user != author:
                await self.bot.say("Boop! /) " + user.mention)
            else:
                await self.bot.say("Boop! /) " + user.mention + "    \n\n*ponk*")
        else:
            await self.bot.say("That does not appear to be something I can boop.")

    @commands.command(pass_context=True, no_pm=True)
    async def unflip(self, ctx, user : discord.Member=None):
        """Now with the ability to unflip!"""
        if user != None:
            author = ctx.message.author
            if user != author:
                await self.bot.say(user.mention + " ノ( ゜-゜ノ)")
            else:
                await self.bot.say(user.mention + " ノ( ゜-゜ノ)    \n\n`It's okay - you look better this way`")
        else:
            await self.bot.say("Really? You want me to unflip *that*?")

    @commands.command()
    async def pong(self, user : discord.Member=None):
        """You've seen ping, now get ready for pong!"""
        await self.bot.say("`Ping!`")

    @commands.command(pass_context=True, no_pm=True)
    async def ship(self, ctx, user_1 : discord.Member=None, user_2 : discord.Member=None):
        "Ship yourself, someone else, or two random people"

        msg = " :heart: "
        extra = ""
        member_list = list()
        author = ctx.message.author
        server = ctx.message.server

        if user_1 == None:
            for member in server.members:
                if member.id != author.id:
                    member_list.append(member)
            user_1 = member_list[randint(0, len(member_list)-1)]

        if user_2 == None:
            for member in server.members:
                if member.id != author.id:
                    member_list.append(member)
            user_2 = member_list[randint(0, len(member_list)-1)]

        if user_1 == self.bot.user and user_2 == self.bot.user:
            extra = "\nOh, how funny. I've never seen that response before, imbecile."
        elif user_1 == self.bot.user or user_2 == self.bot.user:
            extra = "\nYou're not even worth the cheapest drink I bought today in this pub."
        elif user_1 == self.bot.user and user_1 == author:
            extra = "\nI'm flattered, really, but all I would do is merge your DNA into my programming, thus permanently erasing you from this world. Besides, you aren't my type."
        else:
            if user_1 == author and user_2 == author:
                extra = "\n\n`I wonder how desperate you can get.`"
            elif user_1 == author or user_2 == author:
                extra = "\n\n`You know, there's pathetic, and then there's you.`"
        await self.bot.say(user_1.mention + msg + user_2.mention + extra)

    @commands.command(pass_context=True, no_pm=True)
    async def playing(self, ctx, game=None):
        "Finds users playing a particular game"

        member_list = list()
        number = 0
        server = ctx.message.server

        for member in server.members:
            if member.game != None:
                if str(member.game).lower() == str(game).lower():
                    member_list.append(member.name)
                    number += 1

        if number > 0:
            await self.bot.say("```{} users playing {}\n{}```".format(number, game, "\n".join(member_list)))
        else:
            await self.bot.say("```I either can't find {}, or you are a master of Engrish.```".format(game))

    @commands.command(pass_context=True, no_pm=True)
    @checks.admin_or_permissions(administrator=True)
    async def updateboard(self, ctx):
        """Updates usernames"""
        bank_list = self.bank.get_server_accounts(ctx.message.server)

        for account in bank_list:
            if account.member != None:
                user = discord.utils.get(ctx.message.server.members, id=account.member.id)
                if user != None:
                    self.bank.accounts[ctx.message.server.id][account.member.id]["name"] = user.name
                    self.bank._save_bank()

        await self.bot.say("Updated all registered bank names!\n```It's recommended you reload modules tha rely on accounts```")

    @commands.group(pass_context=True)
    async def top(self, ctx):
        """Top posters"""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)
            return

    @top.command(pass_context=True, no_pm=True, name="update")
    @checks.admin_or_permissions(administrator=True)
    async def top_update(self, ctx):
        "Get the highest posters"
        ranks = dataIO.load_json("data/counter/ranks.json")
        ranks[ctx.message.server.id] = {}
        await self.bot.say("Running post leaderboard update.")
        for channel in ctx.message.server.channels:
            number = 0
            async for message in self.bot.logs_from(channel, limit=100):
                number = number + 1
                if self.bot.user not in message.mentions and message.author.bot != True:
                    if len(message.content.split(" ")) >= 4 and len(message.content) >= 8 and not message.content.startswith("!"):
                        try:
                            ranks[ctx.message.server.id][message.author.id]["posts"] += 1
                        except KeyError:
                            ranks[ctx.message.server.id][message.author.id] = {}
                            ranks[ctx.message.server.id][message.author.id]["posts"] = 1
                            ranks[ctx.message.server.id][message.author.id]["username"] = message.author.name
                last = message

            while number == 100:
                number = 0
                async for message in self.bot.logs_from(channel, limit=100, before=last):
                    number = number + 1
                    if self.bot.user not in message.mentions and message.author.bot != True:
                        if len(message.content.split(" ")) >= 4 and len(message.content) >= 8 and not message.content.startswith("!"):
                            try:
                                ranks[ctx.message.server.id][message.author.id]["posts"] += 1
                            except KeyError:
                                ranks[ctx.message.server.id][message.author.id] = {}
                                ranks[ctx.message.server.id][message.author.id]["posts"] = 1
                                ranks[ctx.message.server.id][message.author.id]["username"] = message.author.name
                    last = message
        dataIO.save_json("data/counter/ranks.json", ranks)
        await self.bot.say("All messages counted")

    @top.command(pass_context=True, no_pm=True, name="display")
    async def top_display(self, ctx):
        "Display top posters"
        ranks = dataIO.load_json("data/counter/ranks.json")

        sorted_users = sorted(ranks[ctx.message.server.id], key=lambda x: ranks[ctx.message.server.id][x]["posts"], reverse=True)

        top = 10
        if len(sorted_users) < top:
            top = len(sorted_users)
        topfive = sorted_users[:top]
        highscore = "Top Posters:\n"
        place = 1
        for user in topfive:
            highscore += str(place).ljust(len(str(top))+1)
            highscore += (ranks[ctx.message.server.id][user]["username"]+" ").ljust(23-len(str(ranks[ctx.message.server.id][user]["posts"])))
            highscore += str(ranks[ctx.message.server.id][user]["posts"]) + "\n"
            place += 1

        if highscore:
            if len(highscore) < 1985:
                await self.bot.say("```py\n"+highscore+"```")
            else:
                await self.bot.say("The leaderboard is too big to be displayed.")
        else:
            await self.bot.say("The server messages have not been counted yet")

def check_folders():
    if not os.path.exists("data/economy"):
        print("Creating data/economy folder...")
        os.makedirs("data/economy")

    if not os.path.exists("data/counter"):
        print("Creating data/counter folder...")
        os.makedirs("data/counter")

def check_files():

    f = "data/economy/settings.json"
    if not fileIO(f, "check"):
        print("Creating default economy's settings.json...")
        fileIO(f, "save", {})

    f = "data/economy/bank.json"
    if not fileIO(f, "check"):
        print("Creating empty bank.json...")
        fileIO(f, "save", {})

    f = "data/counter/ranks.json"
    if not fileIO(f, "check"):
        print("Creating empty ranks.json...")
        fileIO(f, "save", {})

def setup(bot):
    global logger
    check_folders()
    check_files()
    bot.add_cog(Counter(bot))
<commit_msg>Update counter.py<commit_after>import discord
from discord.ext import commands
from cogs.utils.dataIO import dataIO, fileIO
from collections import namedtuple, defaultdict
from datetime import datetime
from random import randint
from copy import deepcopy
from .utils import checks
from __main__ import send_cmd_help
import os
import time
import logging
from .economy import Bank, BankError, AccountAlreadyExists, NoAccount, InsufficientBalance, SameSenderAndReceiver

class Counter:
    """My custom cog that does stuff"""
    def __init__(self, bot):
        self.bot = bot
        self.bank = Bank(bot, "data/economy/bank.json")

    @commands.command(pass_context=True, no_pm=True)
    async def boop(self, ctx, user : discord.Member=None):
        """Boop!"""
        if user != None:
            author = ctx.message.author
            if user != author:
                await self.bot.say("Boop! /) " + user.mention)
            else:
                await self.bot.say("Boop! /) " + user.mention + "    \n\n*ponk*")
        else:
            await self.bot.say("That does not appear to be something I can boop.")

    @commands.command(pass_context=True, no_pm=True)
    async def unflip(self, ctx, user : discord.Member=None):
        """Now with the ability to unflip!"""
        if user != None:
            author = ctx.message.author
            if user != author:
                await self.bot.say(user.mention + " ノ( ゜-゜ノ)")
            else:
                await self.bot.say(user.mention + " ノ( ゜-゜ノ)    \n\n`It's okay - you look better this way`")
        else:
            await self.bot.say("Really? You want me to unflip *that*?")

    @commands.command()
    async def pong(self, user : discord.Member=None):
        """You've seen ping, now get ready for pong!"""
        await self.bot.say("`Ping!`")

    @commands.command(pass_context=True, no_pm=True)
    async def ship(self, ctx, user_1 : discord.Member=None, user_2 : discord.Member=None):
        "Ship yourself, someone else, or two random people"

        msg = " :heart: "
        extra = ""
        member_list = list()
        author = ctx.message.author
        server = ctx.message.server

        if user_1 == None:
            for member in server.members:
                if member.id != author.id:
                    member_list.append(member)
            user_1 = member_list[randint(0, len(member_list)-1)]

        if user_2 == None:
            for member in server.members:
                if member.id != author.id:
                    member_list.append(member)
            user_2 = member_list[randint(0, len(member_list)-1)]

        if user_1 == self.bot.user and user_2 == self.bot.user:
            extra = "\nOh, how funny. I've never seen that response before, imbecile."
        elif user_1 == self.bot.user or user_2 == self.bot.user:
            extra = "\nYou're not even worth the cheapest drink I bought today in this pub."
        elif user_1 == self.bot.user and user_1 == author:
            extra = "\nI'm flattered, really, but all I would do is merge your DNA into my programming, thus permanently erasing you from this world. Besides, you aren't my type."
        else:
            if user_1 == author and user_2 == author:
                extra = "\n\n`I wonder how desperate you can get.`"
            elif user_1 == author or user_2 == author:
                extra = "\n\n`You know, there's pathetic, and then there's you.`"
        await self.bot.say(user_1.mention + msg + user_2.mention + extra)

    @commands.command(pass_context=True, no_pm=True)
    async def playing(self, ctx, game=None):
        "Finds users playing a particular game"

        member_list = list()
        number = 0
        server = ctx.message.server

        for member in server.members:
            if member.game != None:
                if str(member.game).lower() == str(game).lower():
                    member_list.append(member.name)
                    number += 1

        if number > 0:
            await self.bot.say("```{} users playing {}\n{}```".format(number, game, "\n".join(member_list)))
        else:
            await self.bot.say("```I either can't find {}, or you are a master of Engrish.```".format(game))

    @commands.command(pass_context=True, no_pm=True)
    @checks.admin_or_permissions(administrator=True)
    async def updateboard(self, ctx):
        """Updates usernames"""
        bank_list = self.bank.get_server_accounts(ctx.message.server)

        for account in bank_list:
            if account.member != None:
                user = discord.utils.get(ctx.message.server.members, id=account.member.id)
                if user != None:
                    self.bank.accounts[ctx.message.server.id][account.member.id]["name"] = user.name
                    self.bank._save_bank()

        await self.bot.say("Updated all registered bank names!\n```It's recommended you reload modules tha rely on accounts```")

    @commands.group(pass_context=True)
    async def top(self, ctx):
        """Top posters"""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)
            return

    @top.command(pass_context=True, no_pm=True, name="update")
    @checks.admin_or_permissions(administrator=True)
    async def top_update(self, ctx):
        "Get the highest posters"
        ranks = dataIO.load_json("data/counter/ranks.json")
        ranks[ctx.message.server.id] = {}
        await self.bot.say("Running post leaderboard update.")
        for channel in ctx.message.server.channels:
            number = 0
            async for message in self.bot.logs_from(channel, limit=100):
                number = number + 1
                if self.bot.user not in message.mentions and message.author.bot != True:
                    if len(message.content.split(" ")) >= 4 and len(message.content) >= 8 and not message.content.startswith("!"):
                        try:
                            ranks[ctx.message.server.id][message.author.id]["posts"] += 1
                        except KeyError:
                            ranks[ctx.message.server.id][message.author.id] = {}
                            ranks[ctx.message.server.id][message.author.id]["posts"] = 1
                            ranks[ctx.message.server.id][message.author.id]["username"] = message.author.name
                last = message

            while number == 100:
                number = 0
                async for message in self.bot.logs_from(channel, limit=100, before=last):
                    number = number + 1
                    if self.bot.user not in message.mentions and message.author.bot != True:
                        if len(message.content.split(" ")) >= 4 and len(message.content) >= 8 and not message.content.startswith("!"):
                            try:
                                ranks[ctx.message.server.id][message.author.id]["posts"] += 1
                            except KeyError:
                                ranks[ctx.message.server.id][message.author.id] = {}
                                ranks[ctx.message.server.id][message.author.id]["posts"] = 1
                                ranks[ctx.message.server.id][message.author.id]["username"] = message.author.name
                    last = message
        dataIO.save_json("data/counter/ranks.json", ranks)
        await self.bot.say("All messages counted")

    @top.command(pass_context=True, no_pm=True, name="display")
    async def top_display(self, ctx, user : discord.Member=None):
        "Display top posters"
        ranks = dataIO.load_json("data/counter/ranks.json")
        if user == None:
            sorted_users = sorted(ranks[ctx.message.server.id], key=lambda x: ranks[ctx.message.server.id][x]["posts"], reverse=True)

            top = 10
            if len(sorted_users) < top:
                top = len(sorted_users)
            topfive = sorted_users[:top]
            highscore = "Top Posters:\n"
            place = 1
            for user in topfive:
                highscore += str(place).ljust(len(str(top))+1)
                highscore += (ranks[ctx.message.server.id][user]["username"]+" ").ljust(23-len(str(ranks[ctx.message.server.id][user]["posts"])))
                highscore += str(ranks[ctx.message.server.id][user]["posts"]) + "\n"
                place += 1

            if highscore:
                if len(highscore) < 1985:
                    await self.bot.say("```py\n"+highscore+"```")
                else:
                    await self.bot.say("The leaderboard is too big to be displayed.")
            else:
                await self.bot.say("The server messages have not been counted yet")
        else:
            try:
                posts = ranks[ctx.message.server.id][user.id]["posts"]
                await self.bot.say("{} has {} posts".format(user.name, posts))
            except:
                await self.bot.say("Something went terribly wrong")

def check_folders():
    if not os.path.exists("data/economy"):
        print("Creating data/economy folder...")
        os.makedirs("data/economy")

    if not os.path.exists("data/counter"):
        print("Creating data/counter folder...")
        os.makedirs("data/counter")

def check_files():

    f = "data/economy/settings.json"
    if not fileIO(f, "check"):
        print("Creating default economy's settings.json...")
        fileIO(f, "save", {})

    f = "data/economy/bank.json"
    if not fileIO(f, "check"):
        print("Creating empty bank.json...")
        fileIO(f, "save", {})

    f = "data/counter/ranks.json"
    if not fileIO(f, "check"):
        print("Creating empty ranks.json...")
        fileIO(f, "save", {})

def setup(bot):
    global logger
    check_folders()
    check_files()
    bot.add_cog(Counter(bot))
<|endoftext|>


<commit_before>import numpy as np
from numpy import logaddexp, vstack
from numpy.random import uniform
from functools import reduce

def logsubexp(x,y):
        assert np.all(x >= y), 'cannot take log of negative number %s - %s'%(str(x),str(y))

        return x + np.log1p(-np.exp(y-x))

def log_integrate_log_trap(log_func,log_support):
    """
    Trapezoidal integration of given log(func)
    Returns log of the integral
    """

    log_func_sum = logaddexp(log_func[:-1], log_func[1:]) - np.log(2)
    log_dxs = logsubexp(log_support[:-1], log_support[1:])

    return logaddexp.reduce(log_func_sum + log_dxs)

def compute_weights(data, Nlive):
    """Returns log_ev, log_wts for the log-likelihood samples in data,
    assumed to be a result of nested sampling with Nlive live points."""

    start_data=np.concatenate(([float('-inf')], data[:-Nlive]))
    end_data=data[-Nlive:]

    log_wts=np.zeros(data.shape[0])

    log_vols_start=np.cumsum(np.ones(len(start_data)+1)*np.log1p(-1./Nlive))-np.log1p(-1./Nlive)
    log_vols_end=log_vols_start[-1]+np.concatenate((np.log(np.linspace(1,0,len(end_data)+1)[1:-1]),[np.NINF]))

    log_likes = np.concatenate((start_data,end_data,[end_data[-1]]))

    log_vols=np.concatenate((log_vols_start,log_vols_end))
    log_ev = log_integrate_log_trap(log_likes, log_vols)

    log_dXs = logsubexp(log_vols[:-1], log_vols[1:])
    log_wts = log_likes[1:-1] + log_vols[1:-1]  #(np.log(0.5) + logaddexp(log_dXs[:-1], log_dXs[1:]))

    log_wts -= log_ev

    return log_ev, log_wts

def draw_posterior(data, log_wts, verbose=False):
    """Draw points from the given data (of shape (Nsamples, Ndim))
    with associated log(weight) (of shape (Nsamples,)). Draws uniquely so
    there are no repeated samples"""
    maxWt=max(log_wts)
    normalised_wts=log_wts-maxWt
    selection=[n > np.log(uniform()) for n in normalised_wts]
    idx=list(filter(lambda i: selection[i], range(len(selection))))
    return data[idx]

def draw_posterior_many(datas, Nlives, verbose=False):
    """Draw samples from the posteriors represented by the
    (Nruns, Nsamples, Nparams)-shaped array datas, each sampled with
    the corresponding Nlive number of live points. Will draw without repetition,
    and weight according to the evidence in each input run"""
    # list of log_evidences, log_weights
    log_evs,log_wts=zip(*[compute_weights(data['logL'],Nlive) for data,Nlive in zip(datas, Nlives)])
    if verbose: print('Computed log_evidences: %s'%(str(log_evs)))

    log_total_evidence=reduce(logaddexp, log_evs)
    log_max_evidence=max(log_evs)
    #print 'evidences: %s'%(str(log_evs))
    fracs=[np.exp(log_ev-log_max_evidence) for log_ev in log_evs]
    if verbose: print('Relative weights of input files: %s'%(str(fracs)))
    Ns=[fracs[i]/len(datas[i]) for i in range(len(fracs))]
    Ntot=max(Ns)
    fracs=[n/Ntot for n in Ns]
    if verbose: print('Relative weights of input files taking into account their length: %s'%(str(fracs)))

    posts=[draw_posterior(data,logwt) for (data,logwt,logZ) in zip(datas,log_wts,log_evs)]
    if verbose: print('Number of input samples: %s'%(str([len(x) for x in log_wts])))
    if verbose: print('Expected number of samples from each input file %s'%(str([int(f*len(p)) for f,p in zip(fracs,posts)])))
    bigpos=[]
    for post,frac in zip(posts,fracs):
      mask = uniform(size=len(post))<frac
      bigpos.append(post[mask])
    result = vstack(bigpos)
    if verbose: print('Samples produced: %s'%(str(result.shape)))
    return result

def draw_N_posterior(data,log_wts, N, verbose=False):
    """
    Draw N samples from the input data, weighted by log_wt.
    For large N there may be repeated samples
    """
    if(N==0): return []
    log_cumsums=np.zeros(log_wts.shape[0]+1)
    log_cumsums[0]=-float('inf')
    for i,log_wt in enumerate(log_wts):
        log_cumsums[i+1]=logaddexp(log_cumsums[i], log_wt)

    us=np.log(uniform(size=N))
    idxs=np.digitize(us, log_cumsums)

    return data[idxs-1]

def draw_N_posterior_many(datas, Nlives, Npost, verbose=False):
    """
    Draw Npost samples from the posteriors represented by the
    (Nruns, Nsamples, Nparams)-shaped array datas, each sampled with
    the corresponding number of live points Nlive. The returned number
    of samples may not be exactly Npost due to rounding
    """
    # get log_evidences, log_weights.
    log_evs,log_wts=zip(*[compute_weights(data['logL'],Nlive) for data,Nlive in zip(datas, Nlives)])

    log_total_evidence=reduce(logaddexp, log_evs)
    Ns=[int(round(Npost*np.exp(log_ev-log_total_evidence))) for log_ev in log_evs]
    posts=[draw_N_posterior(data,logwt,N) for (data,logwt,N) in zip(datas,log_wts,Ns)]
    return vstack(posts)

<commit_msg>Migrated `%` string formating<commit_after>import numpy as np
from numpy import logaddexp, vstack
from numpy.random import uniform
from functools import reduce

def logsubexp(x,y):
        assert np.all(x >= y), 'cannot take log of negative number {0!s} - {1!s}'.format(str(x), str(y))

        return x + np.log1p(-np.exp(y-x))

def log_integrate_log_trap(log_func,log_support):
    """
    Trapezoidal integration of given log(func)
    Returns log of the integral
    """

    log_func_sum = logaddexp(log_func[:-1], log_func[1:]) - np.log(2)
    log_dxs = logsubexp(log_support[:-1], log_support[1:])

    return logaddexp.reduce(log_func_sum + log_dxs)

def compute_weights(data, Nlive):
    """Returns log_ev, log_wts for the log-likelihood samples in data,
    assumed to be a result of nested sampling with Nlive live points."""

    start_data=np.concatenate(([float('-inf')], data[:-Nlive]))
    end_data=data[-Nlive:]

    log_wts=np.zeros(data.shape[0])

    log_vols_start=np.cumsum(np.ones(len(start_data)+1)*np.log1p(-1./Nlive))-np.log1p(-1./Nlive)
    log_vols_end=log_vols_start[-1]+np.concatenate((np.log(np.linspace(1,0,len(end_data)+1)[1:-1]),[np.NINF]))

    log_likes = np.concatenate((start_data,end_data,[end_data[-1]]))

    log_vols=np.concatenate((log_vols_start,log_vols_end))
    log_ev = log_integrate_log_trap(log_likes, log_vols)

    log_dXs = logsubexp(log_vols[:-1], log_vols[1:])
    log_wts = log_likes[1:-1] + log_vols[1:-1]  #(np.log(0.5) + logaddexp(log_dXs[:-1], log_dXs[1:]))

    log_wts -= log_ev

    return log_ev, log_wts

def draw_posterior(data, log_wts, verbose=False):
    """Draw points from the given data (of shape (Nsamples, Ndim))
    with associated log(weight) (of shape (Nsamples,)). Draws uniquely so
    there are no repeated samples"""
    maxWt=max(log_wts)
    normalised_wts=log_wts-maxWt
    selection=[n > np.log(uniform()) for n in normalised_wts]
    idx=list(filter(lambda i: selection[i], range(len(selection))))
    return data[idx]

def draw_posterior_many(datas, Nlives, verbose=False):
    """Draw samples from the posteriors represented by the
    (Nruns, Nsamples, Nparams)-shaped array datas, each sampled with
    the corresponding Nlive number of live points. Will draw without repetition,
    and weight according to the evidence in each input run"""
    # list of log_evidences, log_weights
    log_evs,log_wts=zip(*[compute_weights(data['logL'],Nlive) for data,Nlive in zip(datas, Nlives)])
    if verbose: print('Computed log_evidences: {0!s}'.format((str(log_evs))))

    log_total_evidence=reduce(logaddexp, log_evs)
    log_max_evidence=max(log_evs)
    #print 'evidences: %s'%(str(log_evs))
    fracs=[np.exp(log_ev-log_max_evidence) for log_ev in log_evs]
    if verbose: print('Relative weights of input files: {0!s}'.format((str(fracs))))
    Ns=[fracs[i]/len(datas[i]) for i in range(len(fracs))]
    Ntot=max(Ns)
    fracs=[n/Ntot for n in Ns]
    if verbose: print('Relative weights of input files taking into account their length: {0!s}'.format((str(fracs))))

    posts=[draw_posterior(data,logwt) for (data,logwt,logZ) in zip(datas,log_wts,log_evs)]
    if verbose: print('Number of input samples: {0!s}'.format((str([len(x) for x in log_wts]))))
    if verbose: print('Expected number of samples from each input file {0!s}'.format((str([int(f*len(p)) for f,p in zip(fracs,posts)]))))
    bigpos=[]
    for post,frac in zip(posts,fracs):
      mask = uniform(size=len(post))<frac
      bigpos.append(post[mask])
    result = vstack(bigpos)
    if verbose: print('Samples produced: {0!s}'.format((str(result.shape))))
    return result

def draw_N_posterior(data,log_wts, N, verbose=False):
    """
    Draw N samples from the input data, weighted by log_wt.
    For large N there may be repeated samples
    """
    if(N==0): return []
    log_cumsums=np.zeros(log_wts.shape[0]+1)
    log_cumsums[0]=-float('inf')
    for i,log_wt in enumerate(log_wts):
        log_cumsums[i+1]=logaddexp(log_cumsums[i], log_wt)

    us=np.log(uniform(size=N))
    idxs=np.digitize(us, log_cumsums)

    return data[idxs-1]

def draw_N_posterior_many(datas, Nlives, Npost, verbose=False):
    """
    Draw Npost samples from the posteriors represented by the
    (Nruns, Nsamples, Nparams)-shaped array datas, each sampled with
    the corresponding number of live points Nlive. The returned number
    of samples may not be exactly Npost due to rounding
    """
    # get log_evidences, log_weights.
    log_evs,log_wts=zip(*[compute_weights(data['logL'],Nlive) for data,Nlive in zip(datas, Nlives)])

    log_total_evidence=reduce(logaddexp, log_evs)
    Ns=[int(round(Npost*np.exp(log_ev-log_total_evidence))) for log_ev in log_evs]
    posts=[draw_N_posterior(data,logwt,N) for (data,logwt,N) in zip(datas,log_wts,Ns)]
    return vstack(posts)

<|endoftext|>


<commit_before># coding: utf-8
from __future__ import unicode_literals

import tutils
import unittest

import netsecus

class TestHelper(unittest.TestCase):
    def test_escape_filename(self):
        self.assertEqual(netsecus.helper.escape_filename('foo123ABZ.bar'), 'foo123ABZ.bar')
        self.assertEqual(netsecus.helper.escape_filename(''), '_')
        self.assertEqual(netsecus.helper.escape_filename('.'), '_')
        self.assertEqual(netsecus.helper.escape_filename('..'), '_.')
        self.assertEqual(netsecus.helper.escape_filename('../a/b/c'), '_._a_b_c')
        self.assertEqual(netsecus.helper.escape_filename('C:ö'), 'C__')
        self.assertEqual(netsecus.helper.escape_filename('foo bar.json'), 'foo bar.json')

    def test_get_header(self):
        self.assertEqual(
            netsecus.helper.get_header({'Subject': 'Abgabe 1'}, 'Subject'),
            'Abgabe 1')
        self.assertEqual(
            netsecus.helper.get_header({
                'Subject': '=?utf-8?Q?Subject=c3=a4?=X=?utf-8?Q?=c3=bc?=',
            }, 'Subject'),
            'SubjectäXü')

    def test_encode_mail_words(self):
        self.assertEqual(netsecus.helper.encode_mail_words(''), '')
        self.assertEqual(netsecus.helper.encode_mail_words('foobar'), 'foobar')
        self.assertEqual(netsecus.helper.encode_mail_words(
            'Philipp Hagemeister <phi-hag@phihag.de>'),
            'Philipp Hagemeister <phi-hag@phihag.de>')
        self.assertEqual(netsecus.helper.encode_mail_words(
            'Philipp Hägemeister <phi-hag@phihag.de>'),
            '=?utf-8?q?Philipp_H=C3=A4?=gemeister <phi-hag@phihag.de>')

tutils.main(__name__)
<commit_msg>add tests for alias2mail<commit_after># coding: utf-8
from __future__ import unicode_literals

import tutils
import unittest

import netsecus

class TestHelper(unittest.TestCase):
    def test_escape_filename(self):
        self.assertEqual(netsecus.helper.escape_filename('foo123ABZ.bar'), 'foo123ABZ.bar')
        self.assertEqual(netsecus.helper.escape_filename(''), '_')
        self.assertEqual(netsecus.helper.escape_filename('.'), '_')
        self.assertEqual(netsecus.helper.escape_filename('..'), '_.')
        self.assertEqual(netsecus.helper.escape_filename('../a/b/c'), '_._a_b_c')
        self.assertEqual(netsecus.helper.escape_filename('C:ö'), 'C__')
        self.assertEqual(netsecus.helper.escape_filename('foo bar.json'), 'foo bar.json')

    def test_get_header(self):
        self.assertEqual(
            netsecus.helper.get_header({'Subject': 'Abgabe 1'}, 'Subject'),
            'Abgabe 1')
        self.assertEqual(
            netsecus.helper.get_header({
                'Subject': '=?utf-8?Q?Subject=c3=a4?=X=?utf-8?Q?=c3=bc?=',
            }, 'Subject'),
            'SubjectäXü')

    def test_encode_mail_words(self):
        self.assertEqual(netsecus.helper.encode_mail_words(''), '')
        self.assertEqual(netsecus.helper.encode_mail_words('foobar'), 'foobar')
        self.assertEqual(netsecus.helper.encode_mail_words(
            'Philipp Hagemeister <phi-hag@phihag.de>'),
            'Philipp Hagemeister <phi-hag@phihag.de>')
        self.assertEqual(netsecus.helper.encode_mail_words(
            'Philipp Hägemeister <phi-hag@phihag.de>'),
            '=?utf-8?q?Philipp_H=C3=A4?=gemeister <phi-hag@phihag.de>')

    def test_alias2mail(self):
        self.assertEqual(netsecus.helper.alias2mail('a@b.de'), 'a@b.de')
        self.assertEqual(
            netsecus.helper.alias2mail('Philipp Hagemeister <phi-hag_@phihag.de>'),
            'phi-hag_@phihag.de')
        self.assertEqual(
            netsecus.helper.alias2mail('=?UTF-8?Q?D=c3=bcsseldorf_-_Philipp_Hagemeist?=\n'
                '=?UTF-8?Q?er?= <foo@bar.de>'),
            'foo@bar.de')
        self.assertEqual(netsecus.helper.alias2mail('<x@y.de> <foo@bar.museum>'),
            'foo@bar.museum')
        self.assertEqual(netsecus.helper.alias2mail('test <abcdefz0123456789.!#$%&\'*+-/=?^_`{|}~@example.org>'),
            'abcdefz0123456789.!#$%&\'*+-/=?^_`{|}~@example.org')

tutils.main(__name__)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from django.shortcuts import render
import simplejson
import datetime
from django.http import HttpResponse

class GenericItemBase(object):
    ITEM_ATTRS = []

    def __init__(self, identifier):
        self.identifier = identifier

    def jsonify(self, value):
        """
        Method to convert non JSON serializable objects into
        an equivalent JSON serializable form.
        """
        return value

    def json(self):
        raise NotImplementedError

    def render_json(self):
        raise NotImplementedError

    def render_html(self):
        raise NotImplementedError

class GenericItem(GenericItemBase):
    TEMPLATE = 'customer_support/item.html'

    def __init__(self, *args, **kwargs):
        super(GenericItem, self).__init__(*args, **kwargs)
        self._item = {}

    def get_item(self, identifier):
        raise NotImplementedError

    def set_item(self, data):
        self._item = {}
        for key, value in data.items():
            if key in self.ITEM_ATTRS:
                self._item[key] = value

    def json(self):
        item = {}
        for attr_name in self.ITEM_ATTRS:
            attr = self.jsonify(self._item[attr_name])
            if isinstance(attr, datetime):
                attr = attr.strftime('%Y-%m-%d %H:%M')
            item[attr_name] = attr
        return simplejson.dumps(item)

    def render_json(self):
        return HttpResponse(
            self.json(), mimetype='application/json')

    def render_html(self):
        return render(self.TEMPLATE, {'item': self._item})

class GenericItems(GenericItemBase):
    TEMPLATE = 'customer_support/items.html'

    def __init__(self, *args, **kwargs):
        super(GenericItem, self).__init__(*args, **kwargs)
        self._items = []

    def get_items(self, for_entity):
        raise NotImplementedError

    def set_items(self, items):
        self._items = items

    def json(self):
        items = []
        for item in self._items:
            item_dict = {}
            for attr_name in self.ITEM_ATTRS:
                attr = self.jsonify(item[attr_name])
                if isinstance(attr, datetime):
                    attr = attr.strftime('%Y-%m-%d %H:%M')
                item_dict[attr_name] = attr
            items.append(item)
        return simplejson.dumps(items)

    def render_json(self):
        return HttpResponse(
            self.json(), mimetype='application/json')

    def render_html(self):
        return render(self.TEMPLATE, {'items': self._items})
<commit_msg>Added interface for rendering Generic action(s).<commit_after>from __future__ import absolute_import
from django.shortcuts import render
import simplejson
import datetime
from django.http import HttpResponse

class GenericItemBase(object):
    ITEM_ATTRS = []

    def __init__(self, identifier):
        self.identifier = identifier

    def jsonify(self, value):
        """
        Method to convert non JSON serializable objects into
        an equivalent JSON serializable form.
        """
        return value

    def json(self):
        raise NotImplementedError

    def render_json(self):
        raise NotImplementedError

    def render_html(self):
        raise NotImplementedError

class GenericItem(GenericItemBase):
    TEMPLATE = 'customer_support/item.html'

    def __init__(self, *args, **kwargs):
        super(GenericItem, self).__init__(*args, **kwargs)
        self._item = {}

    def get_item(self, identifier):
        raise NotImplementedError

    def set_item(self, data):
        self._item = {}
        for key, value in data.items():
            if key in self.ITEM_ATTRS:
                self._item[key] = value

    def json(self):
        item = {}
        for attr_name in self.ITEM_ATTRS:
            attr = self.jsonify(self._item[attr_name])
            if isinstance(attr, datetime):
                attr = attr.strftime('%Y-%m-%d %H:%M')
            item[attr_name] = attr
        return simplejson.dumps(item)

    def render_json(self):
        return HttpResponse(
            self.json(), mimetype='application/json')

    def render_html(self):
        return render(self.TEMPLATE, {'item': self._item})

class GenericItems(GenericItemBase):
    TEMPLATE = 'customer_support/items.html'

    def __init__(self, *args, **kwargs):
        super(GenericItem, self).__init__(*args, **kwargs)
        self._items = []

    def get_items(self, for_entity):
        raise NotImplementedError

    def set_items(self, items):
        self._items = items

    def json(self):
        items = []
        for item in self._items:
            item_dict = {}
            for attr_name in self.ITEM_ATTRS:
                attr = self.jsonify(item[attr_name])
                if isinstance(attr, datetime):
                    attr = attr.strftime('%Y-%m-%d %H:%M')
                item_dict[attr_name] = attr
            items.append(item)
        return simplejson.dumps(items)

    def render_json(self):
        return HttpResponse(
            self.json(), mimetype='application/json')

    def render_html(self):
        return render(self.TEMPLATE, {'items': self._items})

class GenericActions(object):

    def __init__(self, item_id):
        self.item_id = item_id
        self.actions = []

    def get_actions_for_item(self):
        raise NotImplementedError

    def json(self):
        return simplejson.dumps(self.actions)

    def render_json(self):
        return HttpResponse(self.json(), mimetype='application/json')

    def render_html(self):
        pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import argparse
import hashlib
import json
import os
import re
import time

os.environ['PYWIKIBOT_DIR'] = os.path.dirname(os.path.realpath(__file__))
import pywikibot
from pywikibot.data.api import Request
from config import config_page_name  # pylint: disable=E0611,W0614

parser = argparse.ArgumentParser()
parser.add_argument('--confirm', action='store_true')
parser.set_defaults(confirm=False)
args = parser.parse_args()
print(args)

os.environ['TZ'] = 'UTC'

site = pywikibot.Site()
site.login()

config_page = pywikibot.Page(site, config_page_name)
cfg = config_page.text
cfg = json.loads(cfg)
print(json.dumps(cfg, indent=4, ensure_ascii=False))

if not cfg["enable"]:
    exit("disabled\n")

cvpage = pywikibot.Page(site, cfg["page_name"])
text = cvpage.text

rndstr = hashlib.md5(str(time.time()).encode()).hexdigest()
text = re.sub(r'^(===.*=== *)$', rndstr + r'\1', text, flags=re.M)
sections = text.split(rndstr)

totalcnt = 0
istimeout = False
for secid, section in enumerate(sections[1:], 1):
    title = section[:section.index('\n')]
    print(title, end="\t")
    text = section[section.index('\n') + 1:]

    rndstr = hashlib.md5(str(time.time()).encode()).hexdigest()
    text = re.sub(r"^(.*{{CopyvioEntry\|.+)$", rndstr + r"\1", text, flags=re.M)

    text = text.split(rndstr)

    entrycnt = len(text[1:])
    print(entrycnt)

    newtext = title + '\n' + text[0]
    cnt = 0
    for entry in text[1:]:
        m = re.match(r"{{CopyvioEntry\|1=([^|]+)\|time=(\d+)\|", entry)
        if m:
            pagename = m.group(1)
            entrytime = int(m.group(2))
            print("\t", pagename, entrytime, end="\t")
            if entrytime > time.time():
                istimeout = True
                print("timeout", end="\t")
            page = pywikibot.Page(site, pagename)
            remove = False
            if page.exists():
                for reversion in page.revisions():
                    if reversion.timestamp.astimezone().timestamp() < entrytime - 86400:
                        break
                    if re.search(r"替换为未侵权版本|清理\[\[Category:已完成侵權驗證的頁面]]", reversion.comment):
                        print(reversion.comment)
                        remove = True
                        break
            else:
                data = Request(site=site, parameters={
                    'action': 'query',
                    'letitle': pagename,
                    "list": "logevents",
                    "leprop": "comment",
                    "letype": "delete",
                    "lelimit": "1"
                }).submit()
                print(data['query']['logevents'][0]['comment'])
                remove = True

            if remove:
                cnt += 1
            else:
                print("not remove")
                newtext += entry
        else:
            print("not match")
            newtext += entry

    if cnt == entrycnt:
        print("\t*** remove {} entry and section".format(cnt))
        sections[secid] = ''
    elif cnt > 0:
        print("\t*** remove {} entry".format(cnt))
        sections[secid] = newtext

    totalcnt += cnt

    if istimeout:
        break

text = ''.join(sections)

if cvpage.text == text:
    exit("nothing changed")

pywikibot.showDiff(cvpage.text, text)
cvpage.text = text
summary = cfg["page_summary"].format(totalcnt)
print(summary)
if args.confirm:
    save = input('Save?').lower()
else:
    save = 'yes'
if save in ['yes', 'y', '']:
    cvpage.save(summary=summary, minor=False)
<commit_msg>Fix reversion comment checking time range<commit_after># -*- coding: utf-8 -*-
import argparse
import hashlib
import json
import os
import re
import time

os.environ['PYWIKIBOT_DIR'] = os.path.dirname(os.path.realpath(__file__))
import pywikibot
from pywikibot.data.api import Request
from config import config_page_name  # pylint: disable=E0611,W0614

parser = argparse.ArgumentParser()
parser.add_argument('--confirm', action='store_true')
parser.set_defaults(confirm=False)
args = parser.parse_args()
print(args)

os.environ['TZ'] = 'UTC'

site = pywikibot.Site()
site.login()

config_page = pywikibot.Page(site, config_page_name)
cfg = config_page.text
cfg = json.loads(cfg)
print(json.dumps(cfg, indent=4, ensure_ascii=False))

if not cfg["enable"]:
    exit("disabled\n")

cvpage = pywikibot.Page(site, cfg["page_name"])
text = cvpage.text

rndstr = hashlib.md5(str(time.time()).encode()).hexdigest()
text = re.sub(r'^(===.*=== *)$', rndstr + r'\1', text, flags=re.M)
sections = text.split(rndstr)

totalcnt = 0
istimeout = False
for secid, section in enumerate(sections[1:], 1):
    title = section[:section.index('\n')]
    print(title, end="\t")
    text = section[section.index('\n') + 1:]

    rndstr = hashlib.md5(str(time.time()).encode()).hexdigest()
    text = re.sub(r"^(.*{{CopyvioEntry\|.+)$", rndstr + r"\1", text, flags=re.M)

    text = text.split(rndstr)

    entrycnt = len(text[1:])
    print(entrycnt)

    newtext = title + '\n' + text[0]
    cnt = 0
    for entry in text[1:]:
        m = re.match(r"{{CopyvioEntry\|1=([^|]+)\|time=(\d+)\|", entry)
        if m:
            pagename = m.group(1)
            entrytime = int(m.group(2))
            print("\t", pagename, entrytime, end="\t")
            if entrytime > time.time():
                istimeout = True
                print("timeout", end="\t")
            page = pywikibot.Page(site, pagename)
            remove = False
            if page.exists():
                for reversion in page.revisions():
                    if reversion.timestamp.astimezone().timestamp() < entrytime - 86400 * 7:
                        break
                    if re.search(r"替换为未侵权版本|清理\[\[Category:已完成侵權驗證的頁面]]", reversion.comment):
                        print(reversion.comment)
                        remove = True
                        break
            else:
                data = Request(site=site, parameters={
                    'action': 'query',
                    'letitle': pagename,
                    "list": "logevents",
                    "leprop": "comment",
                    "letype": "delete",
                    "lelimit": "1"
                }).submit()
                print(data['query']['logevents'][0]['comment'])
                remove = True

            if remove:
                cnt += 1
            else:
                print("not remove")
                newtext += entry
        else:
            print("not match")
            newtext += entry

    if cnt == entrycnt:
        print("\t*** remove {} entry and section".format(cnt))
        sections[secid] = ''
    elif cnt > 0:
        print("\t*** remove {} entry".format(cnt))
        sections[secid] = newtext

    totalcnt += cnt

    if istimeout:
        break

text = ''.join(sections)

if cvpage.text == text:
    exit("nothing changed")

pywikibot.showDiff(cvpage.text, text)
cvpage.text = text
summary = cfg["page_summary"].format(totalcnt)
print(summary)
if args.confirm:
    save = input('Save?').lower()
else:
    save = 'yes'
if save in ['yes', 'y', '']:
    cvpage.save(summary=summary, minor=False)
<|endoftext|>