<commit_before># -*- coding:utf-8 -*-
# Author: hankcs
# Date: 2019-12-28 19:26
from os.path import abspath, join, dirname
from setuptools import find_packages, setup

this_dir = abspath(dirname(__file__))
with open(join(this_dir, 'README.md'), encoding='utf-8') as file:
    long_description = file.read()

setup(
    name='hanlp_common',
    version='0.0.6',
    description='HanLP: Han Language Processing',
    long_description=long_description,
    long_description_content_type="text/markdown",
    url='https://github.com/hankcs/HanLP',
    author='hankcs',
    author_email='hankcshe@gmail.com',
    license='Apache License 2.0',
    classifiers=[
        'Intended Audience :: Science/Research',
        'Intended Audience :: Developers',
        "Development Status :: 3 - Alpha",
        'Operating System :: OS Independent',
        "License :: OSI Approved :: Apache Software License",
        'Programming Language :: Python :: 3 :: Only',
        'Topic :: Scientific/Engineering :: Artificial Intelligence',
        "Topic :: Text Processing :: Linguistic"
    ],
    keywords='corpus,machine-learning,NLU,NLP',
    packages=find_packages(exclude=['docs', 'tests*']),
    include_package_data=True,
    install_requires=[
        'phrasetree',
    ],
    extras_require={
        # These AMR dependencies might not be necessary for most people.
        'full': [
            'networkx',
            'penman==0.6.2',
        ],
    },
    python_requires='>=3.6',
)
<commit_msg>Suppress warning<commit_after># -*- coding:utf-8 -*-
# Author: hankcs
# Date: 2019-12-28 19:26
from os.path import abspath, join, dirname
from setuptools import find_packages, setup

this_dir = abspath(dirname(__file__))
with open(join(this_dir, 'README.md'), encoding='utf-8') as file:
    long_description = file.read()

setup(
    name='hanlp_common',
    version='0.0.8',
    description='HanLP: Han Language Processing',
    long_description=long_description,
    long_description_content_type="text/markdown",
    url='https://github.com/hankcs/HanLP',
    author='hankcs',
    author_email='hankcshe@gmail.com',
    license='Apache License 2.0',
    classifiers=[
        'Intended Audience :: Science/Research',
        'Intended Audience :: Developers',
        "Development Status :: 3 - Alpha",
        'Operating System :: OS Independent',
        "License :: OSI Approved :: Apache Software License",
        'Programming Language :: Python :: 3 :: Only',
        'Topic :: Scientific/Engineering :: Artificial Intelligence',
        "Topic :: Text Processing :: Linguistic"
    ],
    keywords='corpus,machine-learning,NLU,NLP',
    packages=find_packages(exclude=['docs', 'tests*']),
    include_package_data=True,
    install_requires=[
        'phrasetree',
    ],
    extras_require={
        # These AMR dependencies might not be necessary for most people.
        'full': [
            'networkx',
            'penman==0.6.2',
        ],
    },
    python_requires='>=3.6',
)
<|endoftext|>


<commit_before>"""holland backup plugin using xtrabackup"""

import os, sys
import shutil
import logging
import tempfile
from subprocess import list2cmdline, check_call, CalledProcessError, STDOUT
from holland.core.exceptions import BackupError
from holland.core.util.path import directory_size
from holland.lib.compression import open_stream
from holland.lib.mysql.option import build_mysql_config, write_options
from holland.lib.mysql.client import connect, MySQLError
from holland.backup.xtrabackup.util import xtrabackup_version, \
                                           get_stream_method, \
                                           run_pre_command

LOG = logging.getLogger(__name__)

CONFIGSPEC = """
[xtrabackup]
global-defaults = string(default='/etc/my.cnf')
innobackupex    = string(default='innobackupex-1.5.1')
ibbackup        = string(default=None)
stream          = option(yes,no,tar,xbstream,default=tar)
slave-info      = boolean(default=no)
no-lock         = boolean(default=no)
tmpdir          = string(default=None)
additional-options = force_list(default=list())
pre-command     = string(default=None)

[compression]
method          = option('none', 'gzip', 'pigz', 'bzip2', 'lzma', 'lzop', default='gzip')
inline          = boolean(default=yes)
level           = integer(min=0, max=9, default=1)

[mysql:client]
defaults-extra-file = force_list(default=list('~/.my.cnf'))
user                = string(default=None)
password            = string(default=None)
socket              = string(default=None)
host                = string(default=None)
port                = integer(min=0, default=None)
""".splitlines()

class XtrabackupPlugin(object):
    """This plugin provides support for backing up a MySQL database using
    xtrabackup from Percona
    """

    def __init__(self, name, config, target_directory, dry_run=False):
        self.name = name
        self.config = config
        self.config.validate_config(CONFIGSPEC)
        self.target_directory = target_directory
        self.dry_run = dry_run

    def estimate_backup_size(self):
        """Estimate the size of the backup this plugin will produce"""
        try:
            mysql_config = build_mysql_config(self.config['mysql:client'])
            client = connect(mysql_config['client'])
            datadir = client.show_variable('datadir')
            return directory_size(datadir)
        except MySQLError, exc:
            raise BackupError("Failed to lookup the MySQL datadir when "
                              "estimating backup size: [%d] %s" % exc.args)

    def backup(self):
        """Run a database backup with xtrabackup"""
        defaults_file = os.path.join(self.target_directory, 'my.xtrabackup.cnf')
        args = [
            self.config['xtrabackup']['innobackupex'],
            '--defaults-file=%s' % defaults_file,
        ]

        if self.config['xtrabackup']['ibbackup']:
            args.append('--ibbackup=' + self.config['xtrabackup']['ibbackup'])

        backup_directory = self.target_directory
        stream_method = get_stream_method(self.config['xtrabackup']['stream'])
        if stream_method:
            args.append('--stream=' + stream_method)
        else:
            backup_directory = os.path.join(backup_directory, 'data')
            try:
                os.makedirs(backup_directory)
            except OSError, exc:
                raise BackupError("Failed to create backup directory %s: %s" %
                                  (backup_directory, exc))
            args.append('--no-timestamp')
        if self.config['xtrabackup']['tmpdir']:
            args.append('--tmpdir')
        if self.config['xtrabackup']['slave-info']:
            args.append('--slave-info')
        if self.config['xtrabackup']['no-lock']:
            args.append('--no-lock')
        if self.config['xtrabackup']['additional-options']:
            args.extend(self.config['xtrabackup']['additional-options'])
        args.append(backup_directory)

        LOG.info("%s", list2cmdline(args))

        if self.dry_run:
            return

        if self.config['xtrabackup']['pre-command']:
            run_pre_command(self.config['xtrabackup']['pre-command'],
                            self.target_directory)

        config = build_mysql_config(self.config['mysql:client'])
        write_options(config, defaults_file)
        shutil.copyfileobj(open(self.config['xtrabackup']['global-defaults'], 'r'),
                           open(defaults_file, 'a'))

        if stream_method:
            stdout_path = os.path.join(self.target_directory, 'backup.tar')
            stdout = open_stream(stdout_path, 'w',
                                 **self.config['compression'])
            stderr_path = os.path.join(self.target_directory, 'xtrabackup.log')
            stderr = open(stderr_path, 'wb')
        else:
            stdout_path = os.path.join(self.target_directory, 'xtrabackup.log')
            stderr_path = stdout_path
            stdout = open(stdout_path, 'wb')
            stderr = STDOUT

        try:
            try:
                check_call(args,
                           stdout=stdout,
                           stderr=stderr,
                           close_fds=True)
            except OSError, exc:
                LOG.info("Command not found: %s", args[0])
                raise BackupError("%s not found. Is xtrabackup installed?" %
                                  args[0])
            except CalledProcessError, exc:
                LOG.info("%s failed", list2cmdline(exc.cmd))
                for line in open(stderr_path, 'rb'):
                    if line.startswith('>>'):
                        continue
                    LOG.info("%s", line.rstrip())
                raise BackupError("%s failed" % exc.cmd[0])
        finally:
            exc_info = sys.exc_info()[1]
            try:
                stdout.close()
                if stderr != STDOUT:
                    stderr.close()
            except IOError, exc:
                if not exc_info:
                    raise BackupError(str(exc))

    def info(self):
        """Provide information about the backup this plugin produced"""
        return ""
<commit_msg>remove incorrect os.makedirs block<commit_after>"""holland backup plugin using xtrabackup"""

import os, sys
import shutil
import logging
import tempfile
from subprocess import list2cmdline, check_call, CalledProcessError, STDOUT
from holland.core.exceptions import BackupError
from holland.core.util.path import directory_size
from holland.lib.compression import open_stream
from holland.lib.mysql.option import build_mysql_config, write_options
from holland.lib.mysql.client import connect, MySQLError
from holland.backup.xtrabackup.util import xtrabackup_version, \
                                           get_stream_method, \
                                           run_pre_command

LOG = logging.getLogger(__name__)

CONFIGSPEC = """
[xtrabackup]
global-defaults = string(default='/etc/my.cnf')
innobackupex    = string(default='innobackupex-1.5.1')
ibbackup        = string(default=None)
stream          = option(yes,no,tar,xbstream,default=tar)
slave-info      = boolean(default=no)
no-lock         = boolean(default=no)
tmpdir          = string(default=None)
additional-options = force_list(default=list())
pre-command     = string(default=None)

[compression]
method          = option('none', 'gzip', 'pigz', 'bzip2', 'lzma', 'lzop', default='gzip')
inline          = boolean(default=yes)
level           = integer(min=0, max=9, default=1)

[mysql:client]
defaults-extra-file = force_list(default=list('~/.my.cnf'))
user                = string(default=None)
password            = string(default=None)
socket              = string(default=None)
host                = string(default=None)
port                = integer(min=0, default=None)
""".splitlines()

class XtrabackupPlugin(object):
    """This plugin provides support for backing up a MySQL database using
    xtrabackup from Percona
    """

    def __init__(self, name, config, target_directory, dry_run=False):
        self.name = name
        self.config = config
        self.config.validate_config(CONFIGSPEC)
        self.target_directory = target_directory
        self.dry_run = dry_run

    def estimate_backup_size(self):
        """Estimate the size of the backup this plugin will produce"""
        try:
            mysql_config = build_mysql_config(self.config['mysql:client'])
            client = connect(mysql_config['client'])
            datadir = client.show_variable('datadir')
            return directory_size(datadir)
        except MySQLError, exc:
            raise BackupError("Failed to lookup the MySQL datadir when "
                              "estimating backup size: [%d] %s" % exc.args)

    def backup(self):
        """Run a database backup with xtrabackup"""
        defaults_file = os.path.join(self.target_directory, 'my.xtrabackup.cnf')
        args = [
            self.config['xtrabackup']['innobackupex'],
            '--defaults-file=%s' % defaults_file,
        ]

        if self.config['xtrabackup']['ibbackup']:
            args.append('--ibbackup=' + self.config['xtrabackup']['ibbackup'])

        backup_directory = self.target_directory
        stream_method = get_stream_method(self.config['xtrabackup']['stream'])
        if stream_method:
            args.append('--stream=' + stream_method)
        else:
            backup_directory = os.path.join(backup_directory, 'data')
            args.append('--no-timestamp')
        if self.config['xtrabackup']['tmpdir']:
            args.append('--tmpdir')
        if self.config['xtrabackup']['slave-info']:
            args.append('--slave-info')
        if self.config['xtrabackup']['no-lock']:
            args.append('--no-lock')
        if self.config['xtrabackup']['additional-options']:
            args.extend(self.config['xtrabackup']['additional-options'])
        args.append(backup_directory)

        LOG.info("%s", list2cmdline(args))

        if self.dry_run:
            return

        if self.config['xtrabackup']['pre-command']:
            run_pre_command(self.config['xtrabackup']['pre-command'],
                            self.target_directory)

        config = build_mysql_config(self.config['mysql:client'])
        write_options(config, defaults_file)
        shutil.copyfileobj(open(self.config['xtrabackup']['global-defaults'], 'r'),
                           open(defaults_file, 'a'))

        if stream_method:
            stdout_path = os.path.join(self.target_directory, 'backup.tar')
            stdout = open_stream(stdout_path, 'w',
                                 **self.config['compression'])
            stderr_path = os.path.join(self.target_directory, 'xtrabackup.log')
            stderr = open(stderr_path, 'wb')
        else:
            stdout_path = os.path.join(self.target_directory, 'xtrabackup.log')
            stderr_path = stdout_path
            stdout = open(stdout_path, 'wb')
            stderr = STDOUT

        try:
            try:
                check_call(args,
                           stdout=stdout,
                           stderr=stderr,
                           close_fds=True)
            except OSError, exc:
                LOG.info("Command not found: %s", args[0])
                raise BackupError("%s not found. Is xtrabackup installed?" %
                                  args[0])
            except CalledProcessError, exc:
                LOG.info("%s failed", list2cmdline(exc.cmd))
                for line in open(stderr_path, 'rb'):
                    if line.startswith('>>'):
                        continue
                    LOG.info("%s", line.rstrip())
                raise BackupError("%s failed" % exc.cmd[0])
        finally:
            exc_info = sys.exc_info()[1]
            try:
                stdout.close()
                if stderr != STDOUT:
                    stderr.close()
            except IOError, exc:
                if not exc_info:
                    raise BackupError(str(exc))

    def info(self):
        """Provide information about the backup this plugin produced"""
        return ""
<|endoftext|>


<commit_before><commit_msg>bug fixes, extra sugar, learnerND_minimizer integration<commit_after><|endoftext|>


<commit_before>
"""

"""

import numpy as np
from ..TS_statistics.utils import build_ngram_arrays, create_ngram,\
    uniform_input_samevals, uniform_input_lags
from ..TS_statistics.probabilitytools import compute_conditional_probs,\
    compute_marginal_probs
from ..TS_statistics.ts_statistics import prob_ngram_x, prob_ngram_xy,\
    prob_ngram_ind, prob_xy, prob_xy_ind, prob_x, compute_joint_probs
from ..TS_statistics.regime_statistics import prob_regimes_x, temporal_counts,\
    temporal_average_counts, count_repeated, temporal_densities, parse_spks,\
    isis_computation, isi_distribution, general_count, counts_normalization,\
    temporal_si, prob_spk_xy, count_into_bursts

def test():
    #####
    X = np.random.randn(1000).cumsum()
    X_disc = np.random.randint(0, 20, 1000)
    X_mdisc = np.random.randint(0, 20, (1000, 5))

    ################
    #Test functions
    ##################

    ## Utils
    #########
    uniform_input_lags([5], X)
    lags = uniform_input_lags(np.array([5]), X)

#    uniform_input_samevals(True, X_disc)
#    uniform_input_samevals(False, np.atleast_2d(X_disc).T)
#    uniform_input_samevals(range(20), X_disc)
#    uniform_input_samevals(np.arange(20), X_disc)
#
#    ngram = create_ngram(X_disc, lags, samevals=True)
#    lags = lags = uniform_input_lags(np.array([5]), X_disc)
#    ngram = create_ngram(X_disc, range(5), samevals=True)
#
#    pres, post = [0], [0]
#    L = 1
#    build_ngram_arrays(np.atleast_2d(X_disc).T, post, pres, L)
#    L = 2
#    build_ngram_arrays(np.atleast_2d(X_disc).T, post, pres, L)

    ## TO IMPLEMENT
#    X_mdisc = np.random.randint(0, 20, (1000, 5))
#    pres, post = [0, 1, 2], [[1, 2, 3], [2, 3, 4], [0, 1, 4]]
#    build_ngram_arrays(X_mdisc, post, pres, L)
#
#    ## probabilitytools
#    ###################
#    probs = np.random.random((10, 7, 5))
#    p_x = compute_marginal_probs(probs, [0])
#    assert(p_x.shape == (7, 5))
#    p_x = compute_marginal_probs(probs, [1])
#    assert(p_x.shape == (10, 5))
#    p_x = compute_marginal_probs(probs, [1, 2])
#    assert(p_x.shape == (10,))
#
#    p_x_y = compute_conditional_probs(probs, [0])
#    assert(p_x_y.shape == probs.shape)
#    p_x_y = compute_conditional_probs(probs, [1])
#    assert(p_x_y.shape == probs.shape)
#    p_x_y = compute_conditional_probs(probs, [1, 2])
#    assert(p_x_y.shape == probs.shape)
#
#    ## probabilitytools
#    ###################
#    L = 3
#    X_m = np.random.randint(0, 5, (1000, 4))
#    x, y = np.random.randint(0, 5, 1000), np.random.randint(0, 5, 1000)
#
#    prob_ngram_ind(x, y, 1, auto=False, samevals=False, normalize=True)
#    prob_ngram_ind(x, y, 1, auto=True, samevals=[range(5)]*2, normalize=True)
#    prob_ngram_ind(x, y, 2, auto=False, samevals=False, normalize=True)
#    prob_ngram_ind(x, y, 2, auto=True, samevals=[np.arange(5)]*2,
#                   normalize=True)
#    prob_ngram_ind(x, y, L, auto=False, samevals=np.arange(5), normalize=True)
#    prob_ngram_ind(x, y, L, auto=True, samevals=True, normalize=True)
#
#    prob_ngram_xy(X_m, 1, bins=None, auto=True, samevals=True, normalize=True)
#    prob_ngram_xy(X_m, 1, bins=None, auto=True, samevals=False, normalize=True)
#    prob_ngram_xy(X_m, 1, bins=None, auto=True, samevals=np.arange(5),
#                  normalize=True)
#    prob_ngram_xy(X_m, 1, bins=None, auto=True, samevals=[range(5)]*4,
#                  normalize=True)
#
#    prob_ngram_x(X_m, 1, bins=None, samevals=True, normalize=True)
#    prob_ngram_x(X_m, 1, bins=None, samevals=False, normalize=True)
#    prob_ngram_x(X_m, 1, bins=None, samevals=np.arange(5), normalize=True)
#    prob_ngram_x(X_m, 1, bins=None, samevals=[np.arange(5)]*4, normalize=True)
#
#    compute_joint_probs(X_m, values=[], normalize=True)
#    compute_joint_probs(X_m, values=np.arange(5), normalize=True)
#    compute_joint_probs(X_m, values=np.arange(5), normalize=True)
#
#    prob_xy_ind(x, y, samevals=True, bins=0, timelag=0, normalize=True)
#    prob_xy_ind(x, y, samevals=False, bins=0, timelag=1, normalize=True)
#    prob_xy_ind(x, y, samevals=True, bins=0, timelag=0, normalize=True)
#    prob_xy_ind(x, y, samevals=False, bins=0, timelag=2, normalize=True)
#    prob_xy_ind(x, y, samevals=False, bins=0, timelag=-2, normalize=True)
#    prob_xy_ind(x, y, samevals=False, bins=0, timelag=-1, normalize=True)
#    prob_xy_ind(x, y, samevals=False, bins=[None, None], timelag=-1,
#                normalize=True)
#    prob_xy_ind(x, y, samevals=False, bins=[0, 0], timelag=-1, normalize=True)
#    prob_xy_ind(x, y, samevals=np.arange(4), bins=2, timelag=-1,
#                normalize=True)
#
#    prob_xy(X_m, bins=0, maxl=1, samevals=True)
#    prob_xy(X_m, bins=0, maxl=1, samevals=False)
#    prob_xy(X_m, bins=0, maxl=1, samevals=np.arange(5))
#    prob_xy(X_m, bins=0, maxl=1, samevals=[np.arange(5)]*4)
#
#    prob_x(X_m, n_bins=0, individually=True, normalize=True)
#    prob_x(X_m, n_bins=2, individually=False, normalize=True)
#
#    #### Regime statistics
#    #######################
#    raster = np.random.randint(0, 10, (1000, 4))
#    times_event = np.random.randn(1000).cumsum()
#    element_event = np.random.randint(0, 5, 1000)
#    regimes_event = np.random.randint(0, 2, 1000)
#    spks = times_event, element_event, regimes_event
#
#    parse_spks(raster)
#    parse_spks(spks)
#
#    prob_regimes_x(spks)
#    prob_regimes_x(spks, 10)
#    prob_regimes_x(spks, normalized=True)
#    prob_regimes_x(spks, 10, normalized=True)
#
#    temporal_counts(spks, normalized=False, collapse_reg=True)
#    temporal_counts(spks, normalized=False, collapse_reg=False)
#    temporal_counts(spks, normalized=True, collapse_reg=True)
#    temporal_counts(spks, normalized=True, collapse_reg=False)
#
#    temporal_average_counts(spks, window=0, collapse_reg=True)
#    temporal_average_counts(spks, window=20, collapse_reg=True)
#    temporal_average_counts(spks, window=0, collapse_reg=False)
#    temporal_average_counts(spks, window=20, collapse_reg=False)
#
#    count_repeated(raster[:, 0])
#
#    temporal_densities(spks, w_limit=0, collapse_reg=True)
#    temporal_densities(spks, w_limit=20, collapse_reg=True)
#    temporal_densities(spks, w_limit=0, collapse_reg=False)
#    temporal_densities(spks, w_limit=20, collapse_reg=False)
#
#    ## Interevent statistics
#    ##
#    isis_computation(spks, logscale=False)
#    isis_computation(spks, logscale=True)
#
#    isi_distribution(spks, 10, globally=False, normalized=True,
#                     logscale=True)
#    isi_distribution(spks, 10, globally=True, normalized=True,
#                     logscale=False)
#    isi_distribution(spks, 10, globally=False, normalized=False,
#                     logscale=False)
#    isi_distribution(spks, 10, globally=True, normalized=False,
#                     logscale=True)
#
#    temporal_si(spks)
#
#    ## Probs utils
#    ##
#    c_xy, max_l = np.random.randint(0, 100, (10, 10, 6)), 5
#    general_count(c_xy)
#    general_count(c_xy, max_l)
#
#    counts_normalization(np.random.randint(0, 100, 1000), 1000)
#    counts_normalization(c_xy, 1000)
#
#    ## Counting probs in the events information information
#    ##
#    prob_spk_xy(spks, max_l=8, normalized=False)
#    prob_spk_xy(spks, max_l=8, normalized=True)
#
#    ## Statistics bursts
#    bursts = [np.arange(1, 10), np.arange(50, 60)]
#    count_into_bursts(spks, bursts, elements=None)
#    count_into_bursts(spks, bursts, elements=range(5))
<commit_msg>Evaluation of ts_statistics test.<commit_after>
"""

"""

import numpy as np
from ..TS_statistics.utils import build_ngram_arrays, create_ngram,\
    uniform_input_samevals, uniform_input_lags
from ..TS_statistics.probabilitytools import compute_conditional_probs,\
    compute_marginal_probs
from ..TS_statistics.ts_statistics import prob_ngram_x, prob_ngram_xy,\
    prob_ngram_ind, prob_xy, prob_xy_ind, prob_x, compute_joint_probs
from ..TS_statistics.regime_statistics import prob_regimes_x, temporal_counts,\
    temporal_average_counts, count_repeated, temporal_densities, parse_spks,\
    isis_computation, isi_distribution, general_count, counts_normalization,\
    temporal_si, prob_spk_xy, count_into_bursts

def test():
    #####
    X = np.random.randn(1000).cumsum()
    X_disc = np.random.randint(0, 20, 1000)
    X_mdisc = np.random.randint(0, 20, (1000, 5))

    ################
    #Test functions
    ##################

    ## Utils
    #########
#    uniform_input_lags([5], X)
#    lags = uniform_input_lags(np.array([5]), X)

#    uniform_input_samevals(True, X_disc)
#    uniform_input_samevals(False, np.atleast_2d(X_disc).T)
#    uniform_input_samevals(range(20), X_disc)
#    uniform_input_samevals(np.arange(20), X_disc)
#
#    ngram = create_ngram(X_disc, lags, samevals=True)
#    lags = lags = uniform_input_lags(np.array([5]), X_disc)
#    ngram = create_ngram(X_disc, range(5), samevals=True)
#
    pres, post = [0], [0]
    L = 1
    build_ngram_arrays(np.atleast_2d(X_disc).T, post, pres, L)
    L = 2
    build_ngram_arrays(np.atleast_2d(X_disc).T, post, pres, L)

    ## TO IMPLEMENT
#    X_mdisc = np.random.randint(0, 20, (1000, 5))
#    pres, post = [0, 1, 2], [[1, 2, 3], [2, 3, 4], [0, 1, 4]]
#    build_ngram_arrays(X_mdisc, post, pres, L)
#
#    ## probabilitytools
#    ###################
#    probs = np.random.random((10, 7, 5))
#    p_x = compute_marginal_probs(probs, [0])
#    assert(p_x.shape == (7, 5))
#    p_x = compute_marginal_probs(probs, [1])
#    assert(p_x.shape == (10, 5))
#    p_x = compute_marginal_probs(probs, [1, 2])
#    assert(p_x.shape == (10,))
#
#    p_x_y = compute_conditional_probs(probs, [0])
#    assert(p_x_y.shape == probs.shape)
#    p_x_y = compute_conditional_probs(probs, [1])
#    assert(p_x_y.shape == probs.shape)
#    p_x_y = compute_conditional_probs(probs, [1, 2])
#    assert(p_x_y.shape == probs.shape)
#
#    ## probabilitytools
#    ###################
#    L = 3
#    X_m = np.random.randint(0, 5, (1000, 4))
#    x, y = np.random.randint(0, 5, 1000), np.random.randint(0, 5, 1000)
#
#    prob_ngram_ind(x, y, 1, auto=False, samevals=False, normalize=True)
#    prob_ngram_ind(x, y, 1, auto=True, samevals=[range(5)]*2, normalize=True)
#    prob_ngram_ind(x, y, 2, auto=False, samevals=False, normalize=True)
#    prob_ngram_ind(x, y, 2, auto=True, samevals=[np.arange(5)]*2,
#                   normalize=True)
#    prob_ngram_ind(x, y, L, auto=False, samevals=np.arange(5), normalize=True)
#    prob_ngram_ind(x, y, L, auto=True, samevals=True, normalize=True)
#
#    prob_ngram_xy(X_m, 1, bins=None, auto=True, samevals=True, normalize=True)
#    prob_ngram_xy(X_m, 1, bins=None, auto=True, samevals=False, normalize=True)
#    prob_ngram_xy(X_m, 1, bins=None, auto=True, samevals=np.arange(5),
#                  normalize=True)
#    prob_ngram_xy(X_m, 1, bins=None, auto=True, samevals=[range(5)]*4,
#                  normalize=True)
#
#    prob_ngram_x(X_m, 1, bins=None, samevals=True, normalize=True)
#    prob_ngram_x(X_m, 1, bins=None, samevals=False, normalize=True)
#    prob_ngram_x(X_m, 1, bins=None, samevals=np.arange(5), normalize=True)
#    prob_ngram_x(X_m, 1, bins=None, samevals=[np.arange(5)]*4, normalize=True)
#
#    compute_joint_probs(X_m, values=[], normalize=True)
#    compute_joint_probs(X_m, values=np.arange(5), normalize=True)
#    compute_joint_probs(X_m, values=np.arange(5), normalize=True)
#
#    prob_xy_ind(x, y, samevals=True, bins=0, timelag=0, normalize=True)
#    prob_xy_ind(x, y, samevals=False, bins=0, timelag=1, normalize=True)
#    prob_xy_ind(x, y, samevals=True, bins=0, timelag=0, normalize=True)
#    prob_xy_ind(x, y, samevals=False, bins=0, timelag=2, normalize=True)
#    prob_xy_ind(x, y, samevals=False, bins=0, timelag=-2, normalize=True)
#    prob_xy_ind(x, y, samevals=False, bins=0, timelag=-1, normalize=True)
#    prob_xy_ind(x, y, samevals=False, bins=[None, None], timelag=-1,
#                normalize=True)
#    prob_xy_ind(x, y, samevals=False, bins=[0, 0], timelag=-1, normalize=True)
#    prob_xy_ind(x, y, samevals=np.arange(4), bins=2, timelag=-1,
#                normalize=True)
#
#    prob_xy(X_m, bins=0, maxl=1, samevals=True)
#    prob_xy(X_m, bins=0, maxl=1, samevals=False)
#    prob_xy(X_m, bins=0, maxl=1, samevals=np.arange(5))
#    prob_xy(X_m, bins=0, maxl=1, samevals=[np.arange(5)]*4)
#
#    prob_x(X_m, n_bins=0, individually=True, normalize=True)
#    prob_x(X_m, n_bins=2, individually=False, normalize=True)
#
#    #### Regime statistics
#    #######################
#    raster = np.random.randint(0, 10, (1000, 4))
#    times_event = np.random.randn(1000).cumsum()
#    element_event = np.random.randint(0, 5, 1000)
#    regimes_event = np.random.randint(0, 2, 1000)
#    spks = times_event, element_event, regimes_event
#
#    parse_spks(raster)
#    parse_spks(spks)
#
#    prob_regimes_x(spks)
#    prob_regimes_x(spks, 10)
#    prob_regimes_x(spks, normalized=True)
#    prob_regimes_x(spks, 10, normalized=True)
#
#    temporal_counts(spks, normalized=False, collapse_reg=True)
#    temporal_counts(spks, normalized=False, collapse_reg=False)
#    temporal_counts(spks, normalized=True, collapse_reg=True)
#    temporal_counts(spks, normalized=True, collapse_reg=False)
#
#    temporal_average_counts(spks, window=0, collapse_reg=True)
#    temporal_average_counts(spks, window=20, collapse_reg=True)
#    temporal_average_counts(spks, window=0, collapse_reg=False)
#    temporal_average_counts(spks, window=20, collapse_reg=False)
#
#    count_repeated(raster[:, 0])
#
#    temporal_densities(spks, w_limit=0, collapse_reg=True)
#    temporal_densities(spks, w_limit=20, collapse_reg=True)
#    temporal_densities(spks, w_limit=0, collapse_reg=False)
#    temporal_densities(spks, w_limit=20, collapse_reg=False)
#
#    ## Interevent statistics
#    ##
#    isis_computation(spks, logscale=False)
#    isis_computation(spks, logscale=True)
#
#    isi_distribution(spks, 10, globally=False, normalized=True,
#                     logscale=True)
#    isi_distribution(spks, 10, globally=True, normalized=True,
#                     logscale=False)
#    isi_distribution(spks, 10, globally=False, normalized=False,
#                     logscale=False)
#    isi_distribution(spks, 10, globally=True, normalized=False,
#                     logscale=True)
#
#    temporal_si(spks)
#
#    ## Probs utils
#    ##
#    c_xy, max_l = np.random.randint(0, 100, (10, 10, 6)), 5
#    general_count(c_xy)
#    general_count(c_xy, max_l)
#
#    counts_normalization(np.random.randint(0, 100, 1000), 1000)
#    counts_normalization(c_xy, 1000)
#
#    ## Counting probs in the events information information
#    ##
#    prob_spk_xy(spks, max_l=8, normalized=False)
#    prob_spk_xy(spks, max_l=8, normalized=True)
#
#    ## Statistics bursts
#    bursts = [np.arange(1, 10), np.arange(50, 60)]
#    count_into_bursts(spks, bursts, elements=None)
#    count_into_bursts(spks, bursts, elements=range(5))
<|endoftext|>


<commit_before>import numpy as np
from numpy.linalg import inv, LinAlgError
from scipy.sparse import csc_matrix
from scipy import matrix

from likelihood import reml_gradient
from likelihood import reml_observed_information_matrix
from likelihood import reml_fisher_information_matrix
from likelihood import reml_average_information_matrix
from likelihood import restricted_loglikelihood
from likelihood import makeP, makeVinv

def is_invertible(m):
    return np.isfinite(np.linalg.cond(m.todense()))

def iterative_scoring_method(mm, starts, method='Fisher', maxiter=250,
                             tol=1e-4, verbose=False):
    """
    Updates variance components for a mixed model in an iterative scheme.
    """
    if method.lower() in {'newton-raphson', 'newton', 'nr'}:
        information_mat = reml_observed_information_matrix
    elif method.lower() in {'fisher scoring', 'fisher', 'fs'}:
        information_mat = reml_fisher_information_matrix
    elif method.lower() in {'average information', 'aireml', 'ai'}:
        information_mat = reml_average_information_matrix
    else:
        raise ValueError('Unknown maximization method')

    if verbose:
        print 'Maximizing model with {} method'.format(method)

    vcs = np.array(starts)

    i = 0
    while True:
        V = mm._makeV(vcs.tolist())

        # Complicated things we only want to calculate once
        Vinv = makeVinv(V)
        P = makeP(mm.X, Vinv)

        # Makes the information matrix and gradient, then performs an iteration
        grad = reml_gradient(mm.y, mm.X, V, mm.random_effects, P=P, Vinv=Vinv)
        mat = information_mat(mm.y, mm.X, V, mm.random_effects, P=P, Vinv=Vinv)
        delta = scoring_iteration(mat, grad)

        if not np.isfinite(delta).all():
            raise LinAlgError('NaNs in scoring update')

        vcs = vcs - delta
        llik = restricted_loglikelihood(mm.y, V, mm.X, P, Vinv)

        if vcs.sum() / np.var(mm.y) > 10:
            raise LinAlgError('Optimizer left parameter space')

        if verbose:
            print i, llik, vcs

        if (np.abs(delta) < tol).all():
            break

        i += 1
        if i > maxiter:
            raise LinAlgError('Ran out of scoring iterations')

    return vcs.tolist()

def scoring_iteration(info_mat, gradient):
    try:
        info_mat = np.matrix(info_mat)
        gradient = np.matrix(gradient)
        return -1.0 * np.array(info_mat.I * gradient.T).T[0]
    except LinAlgError:
        raise LinAlgError('Information matrix not invertible!')

def expectation_maximization_reml(mm, starts, maxiter=100, tol=1e-4,
                                  verbose=False):
    i = 0

    if verbose:
        print 'Maximizing model by expectation-maximization'

    n = mm.nobs()
    y = mm.y
    vcs = np.array(starts)
    while True:
        V = mm._makeV(vcs.tolist())

        # Complicated things we only want to calculate once
        Vinv = makeVinv(V)
        P = makeP(mm.X, Vinv)

        coefficients = np.array([
            matrix.item(y.T * P * cov * P * y - np.trace(P * cov))
            for cov in mm.covariance_matrices])

        delta = (vcs ** 2 / n) * coefficients
        vcs += delta

        llik = restricted_loglikelihood(mm.y, V, mm.X, P, Vinv)

        if (np.abs(delta) < tol).all():
            break

        if verbose:
            print i, llik, vcs

        if i > maxiter:
            raise LinAlgError('Ran out of scoring iterations')

        i += 1
    return vcs.tolist()
<commit_msg>optionally return VC estimates after a few rounds of EM reml<commit_after>import numpy as np
from numpy.linalg import inv, LinAlgError
from scipy.sparse import csc_matrix
from scipy import matrix

from likelihood import reml_gradient
from likelihood import reml_observed_information_matrix
from likelihood import reml_fisher_information_matrix
from likelihood import reml_average_information_matrix
from likelihood import restricted_loglikelihood
from likelihood import makeP, makeVinv

def is_invertible(m):
    return np.isfinite(np.linalg.cond(m.todense()))

def iterative_scoring_method(mm, starts, method='Fisher', maxiter=250,
                             tol=1e-4, verbose=False):
    """
    Updates variance components for a mixed model in an iterative scheme.
    """
    if method.lower() in {'newton-raphson', 'newton', 'nr'}:
        information_mat = reml_observed_information_matrix
    elif method.lower() in {'fisher scoring', 'fisher', 'fs'}:
        information_mat = reml_fisher_information_matrix
    elif method.lower() in {'average information', 'aireml', 'ai'}:
        information_mat = reml_average_information_matrix
    else:
        raise ValueError('Unknown maximization method')

    if verbose:
        print 'Maximizing model with {} method'.format(method)

    vcs = np.array(starts)

    i = 0
    while True:
        V = mm._makeV(vcs.tolist())

        # Complicated things we only want to calculate once
        Vinv = makeVinv(V)
        P = makeP(mm.X, Vinv)

        # Makes the information matrix and gradient, then performs an iteration
        grad = reml_gradient(mm.y, mm.X, V, mm.random_effects, P=P, Vinv=Vinv)
        mat = information_mat(mm.y, mm.X, V, mm.random_effects, P=P, Vinv=Vinv)
        delta = scoring_iteration(mat, grad)

        if not np.isfinite(delta).all():
            raise LinAlgError('NaNs in scoring update')

        vcs = vcs - delta
        llik = restricted_loglikelihood(mm.y, V, mm.X, P, Vinv)

        if vcs.sum() / np.var(mm.y) > 10:
            raise LinAlgError('Optimizer left parameter space')

        if verbose:
            print i, llik, vcs

        if (np.abs(delta) < tol).all():
            break

        i += 1
        if i > maxiter:
            raise LinAlgError('Ran out of scoring iterations')

    return vcs.tolist()

def scoring_iteration(info_mat, gradient):
    try:
        info_mat = np.matrix(info_mat)
        gradient = np.matrix(gradient)
        return -1.0 * np.array(info_mat.I * gradient.T).T[0]
    except LinAlgError:
        raise LinAlgError('Information matrix not invertible!')

def expectation_maximization_reml(mm, starts=None, maxiter=10000, tol=1e-8,
                                  verbose=False, vcs_after_maxiter=False):
    i = 0

    if verbose:
        print 'Maximizing model by expectation-maximization'

    n = mm.nobs()
    y = mm.y
    vcs = np.array(starts)
    while True:
        V = mm._makeV(vcs.tolist())

        # Complicated things we only want to calculate once
        Vinv = makeVinv(V)
        P = makeP(mm.X, Vinv)

        coefficients = np.array([
            matrix.item(y.T * P * cov * P * y - np.trace(P * cov))
            for cov in mm.covariance_matrices])

        delta = (vcs ** 2 / n) * coefficients
        vcs += delta

        llik = restricted_loglikelihood(mm.y, V, mm.X, P, Vinv)

        if (np.abs(delta) < tol).all():
            break

        if verbose:
            print i, llik, vcs

        if i > maxiter and vcs_after_maxiter:
            break

        if i > maxiter:
            raise LinAlgError('Ran out of scoring iterations')

        i += 1
    return vcs.tolist()
<|endoftext|>


<commit_before>"""
A simple relational database interface that stays out of your way.
Designed to be small, fast, and transparent. Integrates with user-defined record classes.

* Record:     base class for user-defined Record classes (inherits from bl.dict.Dict)
* RecordSet:  set of Records (inherits from list)
* Database :  a database connection
                (init with a standard DB-API 2.0 connection string)

---------------------------------------------------------------------------
                                            Memory:  Footprint:
# > python                                  4656 K   4656 K (Python 3.4.3 osx)
# >>> from bl.database import Database   5460 K    804 K (YMMV)
---------------------------------------------------------------------------
Sample session:
>>> d = Database()      # in-memory sqlite3 database
>>> d.execute("create table table1 (name varchar primary key, email varchar not null unique)")
>>> d.execute("insert into table1 (name, email) values ('sah', 'sah@blackearthgroup.com')")
>>> records = d.select("select * from table1")
>>> records[0].name
'sah'
>>> d.connection.close()
>>>
"""

import datetime, imp, re, time
from bl.dict import Dict

class Database(Dict):
    """a database connection object."""

    def __init__(self, connection_string=None, dba=None, tries=3, DEBUG=False, log=print, **args):
        Dict.__init__(self,
            connection_string=connection_string or '',
            dba=dba or imp.load_module('sqlite3', *imp.find_module('sqlite3')),
            DEBUG=DEBUG, tries=tries, log=log, **args)
        if self.dba is None:
            import sqlite3
            self.dba = sqlite3
        elif type(self.dba) in (str, bytes):
            fm = imp.find_module(dba)
            self.dba = imp.load_module(self.dba, fm[0], fm[1], fm[2])

        if self.dba.__name__ == 'psycopg2':
            # make psycopg2 always return unicode strings
            try:
                dba.extensions.register_type(dba.extensions.UNICODE)
                dba.extensions.register_type(dba.extensions.UNICODEARRAY)
            except:
                # if that didn't work for some reason, then just go with the default setup.
                pass

        # try reaching the db "tries" times, with increasing wait times, before raising an exception.
        for i in range(tries):
            try:
                if self.connection_string != None:
                    self.connection = self.dba.connect(self.connection_string)
                else:
                    self.connection = self.dba.connect(**args)
                break
            except:
                if i==list(range(tries))[-1]:       # last try failed
                    raise
                else:                               # wait a bit
                    time.sleep(2*i)
        if self.dba.__name__ == 'sqlite3':
            self.execute("pragma foreign_keys = ON")

    def __repr__(self):
        return "Database(dba=%s, connection_string='%s')" % (self.dba.__name__, self.connection_string)

    def cursor(self):
        """get a cursor for fine-grained transaction control."""
        cursor = self.connection.cursor()
        return cursor

    def execute(self, sql, vals=None, cursor=None):
        """execute SQL transaction, commit it, and return nothing. If a cursor is specified, work within that transaction."""
        if self.DEBUG==True:
            self.log(sql)
        try:
            c = cursor or self.connection.cursor()
            if vals in [None, (), [], {}]:
                c.execute(sql)
            else:
                c.execute(sql, vals)
            if cursor is None:
                self.commit()
        except:
            if cursor is not None:
                cursor.connection.rollback()
            raise

    def commit(self):
        """commit the changes on the current connection."""
        self.connection.commit()

    def rollback(self):
        """rollback the changes on the current connection, aborting the transaction."""
        self.connection.rollback()

    def select(self, sql, vals=None, Record=None, RecordSet=None, cursor=None):
        """select from db and return the full result set.
        Required Arguments:
            sql: the SQL query as a string
        Optional/Named Arguments
            vals: any bound variables
            Record: the class (itself) that the resulting records should be
        """
        if Record is None:
            from .record import Record
        if RecordSet is None:
            from .recordset import RecordSet

        records = RecordSet()                   # Populate a RecordSet (list) with the all resulting
        for record in self.selectgen(sql, vals=vals, Record=Record, cursor=cursor):
            records.append(record)
        return records

    def selectgen(self, sql, vals=None, Record=None, cursor=None):
        """select from db and yield a generator"""
        if self.DEBUG == True: print("==SELECT:==\n", sql)
        c = cursor or self.cursor()
        self.execute(sql, vals=vals, cursor=c)

        if Record is None:
            from .record import Record

        # get a list of attribute names from the cursor.description
        attr_list = list()
        for r in c.description:
            attr_list.append(r[0])

        result = c.fetchone()
        while result is not None:
            record = Record(self)               # whatever the record class is, include another instance
            for i in range(len(attr_list)):     # make each attribute dict-able by name
                record[attr_list[i]] = result[i]
            yield record
            result = c.fetchone()

        if cursor is None:
            c.close()   # closing the cursor without committing rolls back the transaction.

    def select_one(self, sql, vals=None, Record=None, cursor=None):
        """select one record from db
        Required Arguments:
            sql: the SQL query as a string
        Optional/Named Arguments:
            vals: any bound variables
            Record: the class (itself) that the resulting records should be
        """
        c = cursor or self.cursor()
        self.execute(sql, vals, cursor=c)

        if Record is None:
            from .record import Record

        # get a list of attribute names from the cursor.description
        attr_list = list()
        for r in c.description:
            attr_list.append(r[0])
        result = c.fetchone()
        if result is None:
            record = None
        else:
            record = Record(self)
            for i in range(len(attr_list)):
                record[attr_list[i]] = result[i]
        if cursor is None:
            c.close()
        return record

    def quote(self, attr):
        """returns the given attribute in a form that is insertable in the insert() and update() methods."""
        t = type(attr)
        if t == type(None): return 'NULL'
        elif t == datetime.datetime:    # datetime -- put it in quotes
            return "'%s'" % str(attr)
        elif t == str:
            return self._quote_str(attr)
        elif t in [dict, Dict]:
            return "$$%s$$" % attr
        else:                           # boolean or number -- no quoting needed
            return str(attr).lower()

    def _quote_str(self, attr):
        """quote the attr string in a manner fitting the Database server, if known."""
        sn = self.servername().lower()
        if 'sqlserver' in sn or 'sqlite' in sn:
            # quote for sqlserver and sqlite: double '' to escape
            attr = "'" + re.sub("'", "''", attr) + "'"
        elif 'postgres' in sn:
            if type(attr) == str:
                attr = "$$%s$$" % attr
        else:
            if type(attr) == str:
                attr = "'%s'" % attr
            else:
                attr = "'%s'" % str(attr, 'UTF-8')
        return attr

    def servername(self):
        """return a string that describes the database server being used"""
        if 'psycopg' in self.dba.__name__:
            return 'postgresql'
        elif 'sqlite' in self.dba.__name__:
            return 'sqlite'
        elif self.dbconfig is not None and self.dbconfig.server is not None:
            return self.dbconfig.server
        elif 'adodbapi' in str(self.connection):
            return 'sqlserver'
        elif 'port=5432' in str(self.connection):
            return 'postgresql'
        elif 'port=3306' in str(self.connection):
            return 'mysql'
        else:
            return ''

    def table_exists(self, table_name):
        sn = self.servername().lower()
        if 'sqlite' in sn:
            return self.select_one(
                "select * from sqlite_master where name=? and type='table' limit 1", (table_name,))
        elif 'mysql' in sn:
            return self.select_one(
                "show tables like %s", (table_name,))
        else:
            # postgresql and sqlserver both use the sql standard here.
            return self.select_one(
                "select * from information_schema.tables where table_name=%s limit 1", (table_name,))

def doctests():
    """doctests for the bl.db module
    >>> d = Database()
    >>> d.execute("create table table1 (name varchar primary key, email varchar not null unique);")
    >>> d.execute("insert into table1 (name, email) values ('sah', 'sah@blackearthgroup.com')")
    >>> d.execute("insert into table1 (name, email) values ('sah', 'sah.harrison@gmail.com')")
    Traceback (most recent call last):
      ...
    sqlite3.IntegrityError: UNIQUE constraint failed: table1.name
    """

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<commit_msg>normalize database connection string for better display<commit_after>"""
A simple relational database interface that stays out of your way.
Designed to be small, fast, and transparent. Integrates with user-defined record classes.

* Record:     base class for user-defined Record classes (inherits from bl.dict.Dict)
* RecordSet:  set of Records (inherits from list)
* Database :  a database connection
                (init with a standard DB-API 2.0 connection string)

---------------------------------------------------------------------------
                                            Memory:  Footprint:
# > python                                  4656 K   4656 K (Python 3.4.3 osx)
# >>> from bl.database import Database   5460 K    804 K (YMMV)
---------------------------------------------------------------------------
Sample session:
>>> d = Database()      # in-memory sqlite3 database
>>> d.execute("create table table1 (name varchar primary key, email varchar not null unique)")
>>> d.execute("insert into table1 (name, email) values ('sah', 'sah@blackearthgroup.com')")
>>> records = d.select("select * from table1")
>>> records[0].name
'sah'
>>> d.connection.close()
>>>
"""

import datetime, imp, re, time
from bl.dict import Dict

class Database(Dict):
    """a database connection object."""

    def __init__(self, connection_string=None, dba=None, tries=3, DEBUG=False, log=print, **args):
        Dict.__init__(self,
            connection_string=re.sub('\s+', ' ', connection_string or ''),
            dba=dba or imp.load_module('sqlite3', *imp.find_module('sqlite3')),
            DEBUG=DEBUG, tries=tries, log=log, **args)
        if self.dba is None:
            import sqlite3
            self.dba = sqlite3
        elif type(self.dba) in (str, bytes):
            fm = imp.find_module(dba)
            self.dba = imp.load_module(self.dba, fm[0], fm[1], fm[2])

        if self.dba.__name__ == 'psycopg2':
            # make psycopg2 always return unicode strings
            try:
                dba.extensions.register_type(dba.extensions.UNICODE)
                dba.extensions.register_type(dba.extensions.UNICODEARRAY)
            except:
                # if that didn't work for some reason, then just go with the default setup.
                pass

        # try reaching the db "tries" times, with increasing wait times, before raising an exception.
        for i in range(tries):
            try:
                if self.connection_string != None:
                    self.connection = self.dba.connect(self.connection_string)
                else:
                    self.connection = self.dba.connect(**args)
                break
            except:
                if i==list(range(tries))[-1]:       # last try failed
                    raise
                else:                               # wait a bit
                    time.sleep(2*i)
        if self.dba.__name__ == 'sqlite3':
            self.execute("pragma foreign_keys = ON")

    def __repr__(self):
        return "Database(dba=%s, connection_string='%s')" % (self.dba.__name__, self.connection_string)

    def cursor(self):
        """get a cursor for fine-grained transaction control."""
        cursor = self.connection.cursor()
        return cursor

    def execute(self, sql, vals=None, cursor=None):
        """execute SQL transaction, commit it, and return nothing. If a cursor is specified, work within that transaction."""
        if self.DEBUG==True:
            self.log(sql)
        try:
            c = cursor or self.connection.cursor()
            if vals in [None, (), [], {}]:
                c.execute(sql)
            else:
                c.execute(sql, vals)
            if cursor is None:
                self.commit()
        except:
            if cursor is not None:
                cursor.connection.rollback()
            raise

    def commit(self):
        """commit the changes on the current connection."""
        self.connection.commit()

    def rollback(self):
        """rollback the changes on the current connection, aborting the transaction."""
        self.connection.rollback()

    def select(self, sql, vals=None, Record=None, RecordSet=None, cursor=None):
        """select from db and return the full result set.
        Required Arguments:
            sql: the SQL query as a string
        Optional/Named Arguments
            vals: any bound variables
            Record: the class (itself) that the resulting records should be
        """
        if Record is None:
            from .record import Record
        if RecordSet is None:
            from .recordset import RecordSet

        records = RecordSet()                   # Populate a RecordSet (list) with the all resulting
        for record in self.selectgen(sql, vals=vals, Record=Record, cursor=cursor):
            records.append(record)
        return records

    def selectgen(self, sql, vals=None, Record=None, cursor=None):
        """select from db and yield a generator"""
        if self.DEBUG == True: print("==SELECT:==\n", sql)
        c = cursor or self.cursor()
        self.execute(sql, vals=vals, cursor=c)

        if Record is None:
            from .record import Record

        # get a list of attribute names from the cursor.description
        attr_list = list()
        for r in c.description:
            attr_list.append(r[0])

        result = c.fetchone()
        while result is not None:
            record = Record(self)               # whatever the record class is, include another instance
            for i in range(len(attr_list)):     # make each attribute dict-able by name
                record[attr_list[i]] = result[i]
            yield record
            result = c.fetchone()

        if cursor is None:
            c.close()   # closing the cursor without committing rolls back the transaction.

    def select_one(self, sql, vals=None, Record=None, cursor=None):
        """select one record from db
        Required Arguments:
            sql: the SQL query as a string
        Optional/Named Arguments:
            vals: any bound variables
            Record: the class (itself) that the resulting records should be
        """
        c = cursor or self.cursor()
        self.execute(sql, vals, cursor=c)

        if Record is None:
            from .record import Record

        # get a list of attribute names from the cursor.description
        attr_list = list()
        for r in c.description:
            attr_list.append(r[0])
        result = c.fetchone()
        if result is None:
            record = None
        else:
            record = Record(self)
            for i in range(len(attr_list)):
                record[attr_list[i]] = result[i]
        if cursor is None:
            c.close()
        return record

    def quote(self, attr):
        """returns the given attribute in a form that is insertable in the insert() and update() methods."""
        t = type(attr)
        if t == type(None): return 'NULL'
        elif t == datetime.datetime:    # datetime -- put it in quotes
            return "'%s'" % str(attr)
        elif t == str:
            return self._quote_str(attr)
        elif t in [dict, Dict]:
            return "$$%s$$" % attr
        else:                           # boolean or number -- no quoting needed
            return str(attr).lower()

    def _quote_str(self, attr):
        """quote the attr string in a manner fitting the Database server, if known."""
        sn = self.servername().lower()
        if 'sqlserver' in sn or 'sqlite' in sn:
            # quote for sqlserver and sqlite: double '' to escape
            attr = "'" + re.sub("'", "''", attr) + "'"
        elif 'postgres' in sn:
            if type(attr) == str:
                attr = "$$%s$$" % attr
        else:
            if type(attr) == str:
                attr = "'%s'" % attr
            else:
                attr = "'%s'" % str(attr, 'UTF-8')
        return attr

    def servername(self):
        """return a string that describes the database server being used"""
        if 'psycopg' in self.dba.__name__:
            return 'postgresql'
        elif 'sqlite' in self.dba.__name__:
            return 'sqlite'
        elif self.dbconfig is not None and self.dbconfig.server is not None:
            return self.dbconfig.server
        elif 'adodbapi' in str(self.connection):
            return 'sqlserver'
        elif 'port=5432' in str(self.connection):
            return 'postgresql'
        elif 'port=3306' in str(self.connection):
            return 'mysql'
        else:
            return ''

    def table_exists(self, table_name):
        sn = self.servername().lower()
        if 'sqlite' in sn:
            return self.select_one(
                "select * from sqlite_master where name=? and type='table' limit 1", (table_name,))
        elif 'mysql' in sn:
            return self.select_one(
                "show tables like %s", (table_name,))
        else:
            # postgresql and sqlserver both use the sql standard here.
            return self.select_one(
                "select * from information_schema.tables where table_name=%s limit 1", (table_name,))

def doctests():
    """doctests for the bl.db module
    >>> d = Database()
    >>> d.execute("create table table1 (name varchar primary key, email varchar not null unique);")
    >>> d.execute("insert into table1 (name, email) values ('sah', 'sah@blackearthgroup.com')")
    >>> d.execute("insert into table1 (name, email) values ('sah', 'sah.harrison@gmail.com')")
    Traceback (most recent call last):
      ...
    sqlite3.IntegrityError: UNIQUE constraint failed: table1.name
    """

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
@author: Ricardo Ribeiro
@credits: Ricardo Ribeiro
@license: MIT
@version: 0.0
@maintainer: Ricardo Ribeiro
@email: ricardojvr@gmail.com
@status: Development
@lastEditedBy: Carlos Mão de Ferro (carlos.maodeferro@neuro.fchampalimaud.org)
'''

import logging
from pyforms.gui.Controls.ControlBase import ControlBase
import pyforms.utils.tools as tools
from PyQt4 import uic, QtGui
from PyQt4.Qsci import QsciScintilla, QsciLexerPython  # pylint: disable=no-name-in-module
from PyQt4.QtGui import QFontMetrics
from PyQt4.QtGui import QFont
from PyQt4.QtGui import QColor
from PyQt4.QtCore import Qt

from pysettings import conf

class ControlCodeEditor(ControlBase):
    """
    Control that offers a code editor with pretty-print and line numbers and a save button
    """

    ARROW_MARKER_NUM = 8

    def __init__(self, label='', default='', helptext=''):
        super(ControlCodeEditor, self).__init__(label, default, helptext)

        self._changed_func = None

    def init_form(self):
        self.logger = logging.getLogger(__name__)

        control_path = tools.getFileInSameDirectory(__file__, "code_editor.ui")
        self._form = uic.loadUi(control_path)

        self._code_editor = self._form.code_editor

        self._save_button = self._form.save_button
        self._save_button.clicked[bool].connect(self.on_save_changes)

        self.form.font_size.addItem('9')
        self.form.font_size.addItem('10')
        self.form.font_size.addItem('11')
        self.form.font_size.addItem('12')
        self.form.font_size.addItem('14')
        self.form.font_size.addItem('18')
        self.form.font_size.addItem('24')

        # Set the default font size
        index = self.form.font_size.findText(conf.PYFORMS_CONTROL_CODE_EDITOR_DEFAULT_FONT_SIZE)
        self.form.font_size.setCurrentIndex(index)

        self.form.font_size.currentIndexChanged.connect(self.__font_size_index_changed)

        self.form.save_button.setIcon(conf.PYFORMS_ICON_CODEEDITOR_SAVE)

        self.lexer = QsciLexerPython

        self._code_editor.keyPressEvent = self._key_pressed
        self._changed_func = None

        self.value = self._value
        super(ControlCodeEditor, self).init_form()

    def __font_size_index_changed(self, index):
        item = self.form.font_size.currentText()
        if len(item) >= 1:
            self._load_code_editor_settings()

    def _load_code_editor_settings(self):
        """
        Load settings on the code editor like, font style, margins, scroll, etc.
        Based on the example from http://eli.thegreenplace.net/2011/04/01/sample-using-qscintilla-with-pyqt/
        """

        item = self.form.font_size.currentText()
        size = int(item)

        # Set the default font
        font = QFont()
        font.setFamily('Courier')
        font.setFixedPitch(True)
        font.setPointSize(size)
        self._code_editor.setFont(font)
        self._code_editor.setMarginsFont(font)

        # Margin 0 is used for line numbers
        fontmetrics = QFontMetrics(font)
        self._code_editor.setMarginsFont(font)
        self._code_editor.setMarginWidth(0, fontmetrics.width("00000") + 6)
        self._code_editor.setMarginLineNumbers(0, True)
        self._code_editor.setMarginsBackgroundColor(QColor("#cccccc"))

        # Clickable margin 1 for showing markers
        self._code_editor.setMarginSensitivity(1, True)
        self._code_editor.marginClicked.connect(self.on_margin_clicked)
        self._code_editor.markerDefine(QsciScintilla.RightArrow, self.ARROW_MARKER_NUM)
        self._code_editor.setMarkerBackgroundColor(QColor("#ee1111"), self.ARROW_MARKER_NUM)

        # Detect changes to text
        self._code_editor.modificationChanged.connect(self.on_modification_changed)

        # Brace matching: enable for a brace immediately before or after the current position
        self._code_editor.setBraceMatching(QsciScintilla.SloppyBraceMatch)

        # Current line visible with special background color
        self._code_editor.setCaretLineVisible(True)
        self._code_editor.setCaretLineBackgroundColor(QColor("#ffe4e4"))

        # Set Python lexer
        # Set style for Python comments (style number 1) to a fixed-width Courier.
        lexer = self.lexer()
        lexer.setDefaultFont(font)
        self._code_editor.setLexer(lexer)
        self._code_editor.setIndentationWidth(4)
        # self._code_editor.SendScintilla(QsciScintilla.SCI_STYLESETFONT, 1, 'Courier')
        self._code_editor.SendScintilla(QsciScintilla.SCI_STYLESETFONT, 1)

        # Don't want to see the horizontal scrollbar at all
        # Use raw message to Scintilla here (all messages are documented here: http://www.scintilla.org/ScintillaDoc.html)
        self._code_editor.SendScintilla(QsciScintilla.SCI_SETHSCROLLBAR, 0)

        # not too small
        #self._code_editor.setMinimumSize(600, 450)

    ###################################################################
    ############ Events ###############################################
    ###################################################################

    def on_margin_clicked(self, nmargin, nline, modifiers):  # pylint: disable=unused-argument
        """
        On margin clicked, toggle marker for the line the margin was clicked on
        :param nmargin:
        :type nmargin:
        :param nline:
        :type nline:
        :param modifiers:
        :type modifiers:
        """
        if self._code_editor.markersAtLine(nline) != 0:
            self._code_editor.markerDelete(nline, self.ARROW_MARKER_NUM)
        else:
            self._code_editor.markerAdd(nline, self.ARROW_MARKER_NUM)

    def on_modification_changed(self):
        """
        On modification change, re-enable save button
        """
        if self._changed_func: self._save_button.setEnabled(True)

    def on_save_changes(self):
        """
        On button save clicked, save changes made on the code editor to file
        """
        if self.changed_event() and self._changed_func:
            self._code_editor.setModified(False)
            self._save_button.setEnabled(False)

    def _key_pressed(self, event):
        """
        Handle KeyPressed event
        We only care about CTRL-S in order to save changes
        :param event: key event
        """
        QsciScintilla.keyPressEvent(self._code_editor, event)
        if event.key() in [Qt.Key_S, Qt.Key_Save]:
            modifiers = QtGui.QApplication.keyboardModifiers()
            if modifiers == Qt.ControlModifier and self.is_modified():
                self.logger.debug("Saving...")
                self.on_save_changes()

        self.key_pressed_event(event)

    def key_pressed_event(self, event):
        """
        Override KeyPressed event as you like
        :param event: key event
        """
        pass

    @property
    def is_modified(self):
        return self._code_editor.isModified()

    ###################################################################
    ############ Properties ###########################################
    ###################################################################

    @property
    def lexer(self): return self._lexer

    @lexer.setter
    def lexer(self, value):
        self._lexer = value
        self._load_code_editor_settings()

    @property
    def value(self):
        return self._code_editor.text()

    @value.setter
    def value(self, value):
        ControlBase.value.fset(self, value)
        if len(value) > 0:
            self._code_editor.setText(str(value))
            self._code_editor.setModified(False)
            self._save_button.setEnabled(False)

    @property
    def changed_event(self): return self._changed_func if self._changed_func else (lambda : 0)

    @changed_event.setter
    def changed_event(self, value): self._changed_func = value<commit_msg>Fixes bug on code editor: save code key event<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
@author: Ricardo Ribeiro
@credits: Ricardo Ribeiro
@license: MIT
@version: 0.0
@maintainer: Ricardo Ribeiro
@email: ricardojvr@gmail.com
@status: Development
@lastEditedBy: Carlos Mão de Ferro (carlos.maodeferro@neuro.fchampalimaud.org)
'''

import logging
from pyforms.gui.Controls.ControlBase import ControlBase
import pyforms.utils.tools as tools
from PyQt4 import uic, QtGui
from PyQt4.Qsci import QsciScintilla, QsciLexerPython  # pylint: disable=no-name-in-module
from PyQt4.QtGui import QFontMetrics
from PyQt4.QtGui import QFont
from PyQt4.QtGui import QColor
from PyQt4.QtCore import Qt

from pysettings import conf

class ControlCodeEditor(ControlBase):
    """
    Control that offers a code editor with pretty-print and line numbers and a save button
    """

    ARROW_MARKER_NUM = 8

    def __init__(self, label='', default='', helptext=''):
        super(ControlCodeEditor, self).__init__(label, default, helptext)

        self._changed_func = None

    def init_form(self):
        self.logger = logging.getLogger(__name__)

        control_path = tools.getFileInSameDirectory(__file__, "code_editor.ui")
        self._form = uic.loadUi(control_path)

        self._code_editor = self._form.code_editor

        self._save_button = self._form.save_button
        self._save_button.clicked[bool].connect(self.on_save_changes)

        self.form.font_size.addItem('9')
        self.form.font_size.addItem('10')
        self.form.font_size.addItem('11')
        self.form.font_size.addItem('12')
        self.form.font_size.addItem('14')
        self.form.font_size.addItem('18')
        self.form.font_size.addItem('24')

        # Set the default font size
        index = self.form.font_size.findText(conf.PYFORMS_CONTROL_CODE_EDITOR_DEFAULT_FONT_SIZE)
        self.form.font_size.setCurrentIndex(index)

        self.form.font_size.currentIndexChanged.connect(self.__font_size_index_changed)

        self.form.save_button.setIcon(conf.PYFORMS_ICON_CODEEDITOR_SAVE)

        self.lexer = QsciLexerPython

        self._code_editor.keyPressEvent = self._key_pressed
        self._changed_func = None

        self.value = self._value
        super(ControlCodeEditor, self).init_form()

    def __font_size_index_changed(self, index):
        item = self.form.font_size.currentText()
        if len(item) >= 1:
            self._load_code_editor_settings()

    def _load_code_editor_settings(self):
        """
        Load settings on the code editor like, font style, margins, scroll, etc.
        Based on the example from http://eli.thegreenplace.net/2011/04/01/sample-using-qscintilla-with-pyqt/
        """

        item = self.form.font_size.currentText()
        size = int(item)

        # Set the default font
        font = QFont()
        font.setFamily('Courier')
        font.setFixedPitch(True)
        font.setPointSize(size)
        self._code_editor.setFont(font)
        self._code_editor.setMarginsFont(font)

        # Margin 0 is used for line numbers
        fontmetrics = QFontMetrics(font)
        self._code_editor.setMarginsFont(font)
        self._code_editor.setMarginWidth(0, fontmetrics.width("00000") + 6)
        self._code_editor.setMarginLineNumbers(0, True)
        self._code_editor.setMarginsBackgroundColor(QColor("#cccccc"))

        # Clickable margin 1 for showing markers
        self._code_editor.setMarginSensitivity(1, True)
        self._code_editor.marginClicked.connect(self.on_margin_clicked)
        self._code_editor.markerDefine(QsciScintilla.RightArrow, self.ARROW_MARKER_NUM)
        self._code_editor.setMarkerBackgroundColor(QColor("#ee1111"), self.ARROW_MARKER_NUM)

        # Detect changes to text
        self._code_editor.modificationChanged.connect(self.on_modification_changed)

        # Brace matching: enable for a brace immediately before or after the current position
        self._code_editor.setBraceMatching(QsciScintilla.SloppyBraceMatch)

        # Current line visible with special background color
        self._code_editor.setCaretLineVisible(True)
        self._code_editor.setCaretLineBackgroundColor(QColor("#ffe4e4"))

        # Set Python lexer
        # Set style for Python comments (style number 1) to a fixed-width Courier.
        lexer = self.lexer()
        lexer.setDefaultFont(font)
        self._code_editor.setLexer(lexer)
        self._code_editor.setIndentationWidth(4)
        # self._code_editor.SendScintilla(QsciScintilla.SCI_STYLESETFONT, 1, 'Courier')
        self._code_editor.SendScintilla(QsciScintilla.SCI_STYLESETFONT, 1)

        # Don't want to see the horizontal scrollbar at all
        # Use raw message to Scintilla here (all messages are documented here: http://www.scintilla.org/ScintillaDoc.html)
        self._code_editor.SendScintilla(QsciScintilla.SCI_SETHSCROLLBAR, 0)

        # not too small
        #self._code_editor.setMinimumSize(600, 450)

    ###################################################################
    ############ Events ###############################################
    ###################################################################

    def on_margin_clicked(self, nmargin, nline, modifiers):  # pylint: disable=unused-argument
        """
        On margin clicked, toggle marker for the line the margin was clicked on
        :param nmargin:
        :type nmargin:
        :param nline:
        :type nline:
        :param modifiers:
        :type modifiers:
        """
        if self._code_editor.markersAtLine(nline) != 0:
            self._code_editor.markerDelete(nline, self.ARROW_MARKER_NUM)
        else:
            self._code_editor.markerAdd(nline, self.ARROW_MARKER_NUM)

    def on_modification_changed(self):
        """
        On modification change, re-enable save button
        """
        if self._changed_func: self._save_button.setEnabled(True)

    def on_save_changes(self):
        """
        On button save clicked, save changes made on the code editor to file
        """
        if self.changed_event() and self._changed_func:
            self._code_editor.setModified(False)
            self._save_button.setEnabled(False)

    def _key_pressed(self, event):
        """
        Handle KeyPressed event
        We only care about CTRL-S in order to save changes
        :param event: key event
        """
        QsciScintilla.keyPressEvent(self._code_editor, event)
        if event.key() in [Qt.Key_S, Qt.Key_Save]:
            modifiers = QtGui.QApplication.keyboardModifiers()
            if modifiers == Qt.ControlModifier and self.is_modified:
                self.logger.debug("Saving...")
                self.on_save_changes()

        self.key_pressed_event(event)

    def key_pressed_event(self, event):
        """
        Override KeyPressed event as you like
        :param event: key event
        """
        pass

    @property
    def is_modified(self):
        return self._code_editor.isModified()

    ###################################################################
    ############ Properties ###########################################
    ###################################################################

    @property
    def lexer(self): return self._lexer

    @lexer.setter
    def lexer(self, value):
        self._lexer = value
        self._load_code_editor_settings()

    @property
    def value(self):
        return self._code_editor.text()

    @value.setter
    def value(self, value):
        ControlBase.value.fset(self, value)
        if len(value) > 0:
            self._code_editor.setText(str(value))
            self._code_editor.setModified(False)
            self._save_button.setEnabled(False)

    @property
    def changed_event(self): return self._changed_func if self._changed_func else (lambda : 0)

    @changed_event.setter
    def changed_event(self, value): self._changed_func = value<|endoftext|>


<commit_before>"""a module that implements Monte-Carlo logic and supporting utilties when defining a population
"""
__author__ = "Reed Essick (reed.essick@gmail.com)"

#-------------------------------------------------

import numpy as np

from gw_event_gen import eventgen

try:
    from ConfigParser import ConfigParser
except ImportError: ### Python2 vs Python3
    from configparser import ConfigParser

#-------------------------------------------------

DEFAULT_MIN_NUM_SAMPLES = 100
DEFAULT_ERROR = np.infty

#-------------------------------------------------

### basic config parsing

def subclasses(klass):
    ans = dict()
    for obj in klass.__subclasses__():
        ans[obj.__name__] = obj
        ans.update(subclasses(obj))
    return ans

def parse(section, config):
    ans = dict()
    for k in config.options(section):
        try: ### maybe it's an integer
            ans[k] = config.getint(section, k)
            continue
        except ValueError:
            pass

        try: ### maybe a float
            ans[k] = config.getfloat(section, k)
            continue
        except ValueError:
            pass

        try: ### maybe a bool
            ans[k] = config.getbool(section, k)
            continue
        except:
            pass

        ### just grab it as a string
        ans[k] = config.get(section, k)

    return ans

#-------------------------------------------------

### utilities for setting up Monte-Carlo sampling objects

def path2generator(path):
    """a function that reads in an INI file and instantiates the relevant EventGenerator object
    """
    config = ConfigParser()
    config.read(path)

    ### iterate through sections, instantiating a generator for each one
    generator = eventgen.EventGenerator()

    ### keep track of which generators we instantiate
    ### used to make sure we have everything we need and that we don't repeat anything
    gentypes = set()

    # figure out what is declared
    types = eventgen.SamplingDistribution.__subclasses__() ### get the different types of distributions
    samplingdistributions = subclasses(eventgen.SamplingDistribution)       ### get instantiators for all distributions
    attributetransformations = subclasses(eventgen.AttributeTransformation)

    # add generators
    timedistribution = None
    for name in config.get('general', 'generators').split(): ### iterate through generators
        for parent in samplingdistributions[name].__bases__:
            if parent in gentypes: ### check to make sure we haven't repeated anything
                raise RuntimeError('cannot specify more than one instance of %s!'%samplingdistributions[name].__name__)
            gentypes.add(parent)

        gen = samplingdistributions[name](**parse(name, config))
        generator.append_generator(gen)

        if isinstance(gen, eventgen.TimeDistribution):
            timedistribution = gen

    if timedistribution is None:
        timedistribution = eventgen.UniformEventTime(t0=0, dur=10)
        generator.append_generator(timedistribution)
        for parent in eventgen.UniformEventTime.__bases__:
            gentypes.add(parent)

    ### figure out whether we have enough different types of generators
    ### includes declaring required transformations
    # check for things that *must* be specified
    for gentype in [eventgen.TimeDistribution, eventgen.MassDistribution, eventgen.SpinDistribution, eventgen.OrientationDistribution, eventgen.EccentricityDistribution]:
        if gentype not in gentypes:
            raise RuntimeError('must specify an instance of %s!'%gentype.__name__)
    # check for things that may be specified in several ways
    required_transforms = []
    if eventgen.DistanceDistribution in gentypes:
        if eventgen.RedshiftDistribution in gentypes:
            raise RuntimeError('cannot specify both DistanceDistribution and RedshiftDistribution!')
    elif eventgen.RedshiftDistribution in gentypes:
        required_transforms.append(eventgen.Redshift2LuminosityDistance.__name__) ### must include transformation from redshift to distance
    else:
        raise RuntimeError('must specify either DistanceDistribution or RedshiftDistribution!')

    # add attribute transformations
    if config.has_option('general', 'transforms'):
        for name in config.get('general', 'transforms').split(): ### iterate through transforms
            if name in required_transforms:
                required_transforms.remove(name)
            trans = attributetransformations[name]() ### FIXME: some of these take in arguments...
            generator.append_transform(trans)

    if required_transforms:
        raise RuntimeError('required AttributeTransformation (%s) not specified!'%(', '.join(required_transforms)))

    return generator, timedistribution

#-------------------------------------------------

### Monte-Carlo logic

def update_max_redshift(generator, network):
    """update the maximum redshift or distance defined in generator to avoid sampling too distant sources

    **NOTE**: right now, we just skip this and leave the generator untouched...this will not scale well if the detector sensitivities change significantly!
"""
    pass
    '''
find maximum distance for detectable things
    -> difficult when we use a non-central chi2 distrib to measure p(det)...
    -> maybe just require cdf(det) to be less than some small fraction --> control the systematic error from this truncation to an acceptable level?

Maybe we just Monte-Carlo at a fixed distance and determine the maximum SNR observed? With that, we can scale the maximum distance based on that fixed distance and the assumption that everything with SNR below some value has negligible probability of being detected
'''

def update_montecarlo_counters(generator, event, e1, e2, de1, de2, Nparams):
    """update the montecarlo counters for our smart termination condition
    """
    # extract basic params
    pdet = event.pdet

    # compute the effective pdf and jacobian
    pdf = 1.
    jac = np.empty(Nparams, dtype=float)
    i = 0
    for gen in generator._generators:
        variates = [getattr(event, lbl) for lbl in gen._variates]

        pdf *= gen.pdf(*variates)
        jac[i:i+len(gen._params)] = gen.jacobian(*variates)

    # update the counters
    e1 += pdet
    e2 += pdet**2
    de1 += pdet*jac/pdf
    de2 += 2*pdet**2*jac/pdf

    return e1, e2, de1, de2

def montecarlo(generator, network, min_num_samples=DEFAULT_MIN_NUM_SAMPLES, error=DEFAULT_ERROR):
    """generate samples from generator until we reach the termination conditions specified by min_num_samples and error
    """
    ### generate the minimum number of samples required
    for event in generator.generate_events(n_itr=np.infty, n_event=min_num_samples):
        pass

    # figure out whether we want to even attempt the "smart termination condition"
    if error!=np.infty:

        ### set up the further iteration
        Nparams = sum(len(gen._params) for gen in generator._generators)
        e1 = 0
        e2 = 0
        de1 = np.zeros(Nparams, dtype=float)
        de2 = np.zeros(Nparams, dtype=float)
        for event in generator._events:
            e1, e2, de1, de2 = update_montecarlo_counters(generator, event, e1, e2, de1, de2, Nparams)

        for event in generator.generate_events(n_itr=np.infty, n_event=np.infty, generator=True): ### go forever until we are told to stop
            if np.all(2*e1**2 * np.abs(de1) * error >= 3*np.abs(2*e2*de1 - e1*de2)):
                break ### all smart termination conditions are satisfied, so we break the broader iteration

            ### update monte-carlo sums
            e1, e2, de1, de2 = update_montecarlo_counters(generator, event, e1, e2, de1, de2, Nparams)

    ### format the samples into a numpy structured array compatible with writing to disk
    attrs = sorted(generator.attributes)
    return np.array([[getattr(event, attr) for attr in attrs] for event in generator._events], dtype=[(attr, float) for attr in attrs])
<commit_msg>fixed a small bug<commit_after>"""a module that implements Monte-Carlo logic and supporting utilties when defining a population
"""
__author__ = "Reed Essick (reed.essick@gmail.com)"

#-------------------------------------------------

import numpy as np

from gw_event_gen import eventgen

try:
    from ConfigParser import ConfigParser
except ImportError: ### Python2 vs Python3
    from configparser import ConfigParser

#-------------------------------------------------

DEFAULT_MIN_NUM_SAMPLES = 100
DEFAULT_ERROR = np.infty

#-------------------------------------------------

### basic config parsing

def subclasses(klass):
    ans = dict()
    for obj in klass.__subclasses__():
        ans[obj.__name__] = obj
        ans.update(subclasses(obj))
    return ans

def parse(section, config):
    ans = dict()
    for k in config.options(section):
        try: ### maybe it's an integer
            ans[k] = config.getint(section, k)
            continue
        except ValueError:
            pass

        try: ### maybe a float
            ans[k] = config.getfloat(section, k)
            continue
        except ValueError:
            pass

        try: ### maybe a bool
            ans[k] = config.getbool(section, k)
            continue
        except:
            pass

        ### just grab it as a string
        ans[k] = config.get(section, k)

    return ans

#-------------------------------------------------

### utilities for setting up Monte-Carlo sampling objects

def path2generator(path):
    """a function that reads in an INI file and instantiates the relevant EventGenerator object
    """
    config = ConfigParser()
    config.read(path)

    ### iterate through sections, instantiating a generator for each one
    generator = eventgen.EventGenerator()

    ### keep track of which generators we instantiate
    ### used to make sure we have everything we need and that we don't repeat anything
    gentypes = set()

    # figure out what is declared
    types = eventgen.SamplingDistribution.__subclasses__() ### get the different types of distributions
    samplingdistributions = subclasses(eventgen.SamplingDistribution)       ### get instantiators for all distributions
    attributetransformations = subclasses(eventgen.AttributeTransformation)

    # add generators
    timedistribution = None
    for name in config.get('general', 'generators').split(): ### iterate through generators
        for parent in samplingdistributions[name].__bases__:
            if parent in gentypes: ### check to make sure we haven't repeated anything
                raise RuntimeError('cannot specify more than one instance of %s!'%samplingdistributions[name].__name__)
            gentypes.add(parent)

        gen = samplingdistributions[name](**parse(name, config))
        generator.append_generator(gen)

        if isinstance(gen, eventgen.TimeDistribution):
            timedistribution = gen

    if timedistribution is None:
        timedistribution = eventgen.UniformEventTime(t0=0, dur=10)
        generator.append_generator(timedistribution)
        for parent in eventgen.UniformEventTime.__bases__:
            gentypes.add(parent)

    ### figure out whether we have enough different types of generators
    ### includes declaring required transformations
    # check for things that *must* be specified
    for gentype in [eventgen.TimeDistribution, eventgen.MassDistribution, eventgen.SpinDistribution, eventgen.OrientationDistribution, eventgen.EccentricityDistribution]:
        if gentype not in gentypes:
            raise RuntimeError('must specify an instance of %s!'%gentype.__name__)
    # check for things that may be specified in several ways
    required_transforms = []
    if eventgen.DistanceDistribution in gentypes:
        if eventgen.RedshiftDistribution in gentypes:
            raise RuntimeError('cannot specify both DistanceDistribution and RedshiftDistribution!')
    elif eventgen.RedshiftDistribution in gentypes:
        required_transforms.append(eventgen.Redshift2LuminosityDistance.__name__) ### must include transformation from redshift to distance
    else:
        raise RuntimeError('must specify either DistanceDistribution or RedshiftDistribution!')

    # add attribute transformations
    if config.has_option('general', 'transforms'):
        for name in config.get('general', 'transforms').split(): ### iterate through transforms
            if name in required_transforms:
                required_transforms.remove(name)
            trans = attributetransformations[name]() ### FIXME: some of these take in arguments...
            generator.append_transform(trans)

    if required_transforms:
        raise RuntimeError('required AttributeTransformation (%s) not specified!'%(', '.join(required_transforms)))

    return generator, timedistribution

#-------------------------------------------------

### Monte-Carlo logic

def update_max_redshift(generator, network):
    """update the maximum redshift or distance defined in generator to avoid sampling too distant sources

    **NOTE**: right now, we just skip this and leave the generator untouched...this will not scale well if the detector sensitivities change significantly!
"""
    pass
    '''
find maximum distance for detectable things
    -> difficult when we use a non-central chi2 distrib to measure p(det)...
    -> maybe just require cdf(det) to be less than some small fraction --> control the systematic error from this truncation to an acceptable level?

Maybe we just Monte-Carlo at a fixed distance and determine the maximum SNR observed? With that, we can scale the maximum distance based on that fixed distance and the assumption that everything with SNR below some value has negligible probability of being detected
'''

def update_montecarlo_counters(generator, event, e1, e2, de1, de2, Nparams):
    """update the montecarlo counters for our smart termination condition
    """
    # extract basic params
    pdet = event.pdet

    # compute the effective pdf and jacobian
    pdf = 1.
    jac = np.empty(Nparams, dtype=float)
    i = 0
    for gen in generator._generators:
        variates = [getattr(event, lbl) for lbl in gen._variates]

        pdf *= gen.pdf(*variates)
        jac[i:i+len(gen._params)] = gen.jacobian(*variates)

    # update the counters
    e1 += pdet
    e2 += pdet**2
    de1 += pdet*jac/pdf
    de2 += 2*pdet**2*jac/pdf

    return e1, e2, de1, de2

def montecarlo(generator, network, min_num_samples=DEFAULT_MIN_NUM_SAMPLES, error=DEFAULT_ERROR):
    """generate samples from generator until we reach the termination conditions specified by min_num_samples and error
    """
    ### generate the minimum number of samples required
    for event in generator.generate_events(n_itr=np.infty, n_event=min_num_samples):
        pass

    # figure out whether we want to even attempt the "smart termination condition"
    if error!=np.infty:

        ### set up the further iteration
        Nparams = sum(len(gen._params) for gen in generator._generators)
        e1 = 0
        e2 = 0
        de1 = np.zeros(Nparams, dtype=float)
        de2 = np.zeros(Nparams, dtype=float)
        for event in generator._events:
            e1, e2, de1, de2 = update_montecarlo_counters(generator, event, e1, e2, de1, de2, Nparams)

        for event in generator.generate_events(n_itr=np.infty, n_event=np.infty, generator=True): ### go forever until we are told to stop
            if np.all(2*e1**2 * np.abs(de1) * error >= 3*np.abs(2*e2*de1 - e1*de2)):
                break ### all smart termination conditions are satisfied, so we break the broader iteration

            ### update monte-carlo sums
            e1, e2, de1, de2 = update_montecarlo_counters(generator, event, e1, e2, de1, de2, Nparams)

    ### format the samples into a numpy structured array compatible with writing to disk
    attrs = [(attr, 'float') for attr in sorted(generator.attributes)]
    return np.array([tuple(float(getattr(event, attr)) for attr, _ in attrs) for event in generator._events], dtype=attrs)
<|endoftext|>