<commit_before>import numpy as np
import time
import sys

#==================================================
#              timing decorator
#==================================================
import time
def timeit(method):

    def timed(*args, **kw):
        ts = time.time()
        result = method(*args, **kw)
        te = time.time()

        print '%r %2.2f sec' % \
              (method.__name__,  te-ts)
        return result

    return timed

def update_progress(progress):
    barLength = 10 # Modify this to change the length of the progress bar
    status = ""
    if isinstance(progress, int):
        progress = float(progress)
    if not isinstance(progress, float):
        progress = 0
        status = "error: progress var must be float\r\n"
    if progress < 0:
        progress = 0
        status = "Halt...\r\n"
    if progress >= 1:
        progress = 1
        status = "Done...\r\n"
    block = int(round(barLength*progress))
    text = "\rMb/S: [{0}] {1}% {2}".format( "#"*block + "-"*(barLength-block), progress*100, status)
    sys.stdout.write(text)
    sys.stdout.flush()

class Gather(object):
    '''
    data object which contains

     * the gather to be processed
     * its source and destination (memmapped files)
     * the mask used to extract and save
     '''

    def __init__(self, source, dest, mask):
        self.data = np.array(source[mask])
        self.dest = dest
        self.mask = mask

    def __getitem__(self, i):
        return self.data[i]

    def save(self):
        self.dest[self.mask] = self.data

    def close(self):
        self.dest.flush()

class Flow(object):
    '''
    streams in the seismic data in gathers
    needs the sort order to be defined.
    requires input to be .npy file
    needs a list of method objects designed to
    take the gathers as structured arrays
    '''

    def __init__(self, infile, outfile, order=['fldr', 'tracf'], methods=[]): #default to shot gathers
        self.primaryOrder = order[0]
        self.secondaryOrder = order[1]
        self.indata = np.lib.format.open_memmap(infile, mode='r')
        self.outdata = np.lib.format.open_memmap(outfile, dtype=self.indata.dtype, shape= self.indata.shape, mode='w+')
        self.outdata[:] = self.indata[:]
        self.outdata['trace'].fill(0.0)
        self.outdata.flush()

    def __iter__(self):
        keys = np.unique(self.indata[self.primaryOrder])
        steps = (len(keys)*1.0)
        t = time.time()
        for count, key in enumerate(keys):
            mask = self.indata[self.primaryOrder] == key
            gather = Gather(self.indata, self.outdata, mask)
            gather.data.sort(order=self.secondaryOrder)
            update_progress(gather.data.nbytes*1e-6/time.time()-t)
            t = time.time()
            yield gather

    def save(self):
        self.outdata[self.mask] = self.gather

    def close(self):
        del self.outdata

<commit_msg>added method to flow<commit_after>import numpy as np
import time
import sys

#==================================================
#              timing decorator
#==================================================
import time
def timeit(method):

    def timed(*args, **kw):
        ts = time.time()
        result = method(*args, **kw)
        te = time.time()

        print '%r %2.2f sec' % \
              (method.__name__,  te-ts)
        return result

    return timed

def update_progress(progress):
    barLength = 10 # Modify this to change the length of the progress bar
    status = ""
    if isinstance(progress, int):
        progress = float(progress)
    if not isinstance(progress, float):
        progress = 0
        status = "error: progress var must be float\r\n"
    if progress < 0:
        progress = 0
        status = "Halt...\r\n"
    if progress >= 1:
        progress = 1
        status = "Done...\r\n"
    block = int(round(barLength*progress))
    text = "\rMb/S: [{0}] {1}% {2}".format( "#"*block + "-"*(barLength-block), progress*100, status)
    sys.stdout.write(text)
    sys.stdout.flush()

class Gather(object):
    '''
    data object which contains

     * the gather to be processed
     * its source and destination (memmapped files)
     * the mask used to extract and save
     '''

    def __init__(self, source, dest, mask):
        self.data = np.array(source[mask])
        self.dest = dest
        self.mask = mask

    def __getitem__(self, i):
        return self.data[i]

    def save(self):
        self.dest[self.mask] = self.data

    def close(self):
        self.dest.flush()

class Flow(object):
    '''
    streams in the seismic data in gathers
    needs the sort order to be defined.
    requires input to be .npy file
    needs a list of method objects designed to
    take the gathers as structured arrays
    '''

    def __init__(self, infile, outfile=None, order=['fldr', 'tracf'], methods=[]): #default to shot gathers
        self.methods = methods
        self.primaryOrder = order[0]
        self.secondaryOrder = order[1]
        self.indata = np.memmap(infile, mode='r')
        if outfile:
            self.outdata = np.memmap(outfile, dtype=self.indata.dtype, shape= self.indata.shape, mode='w+')
            #self.outdata[:] = self.indata[:]
            self.outdata['trace'].fill(0.0)
            self.outdata.flush()

    def __iter__(self):
        keys = np.unique(self.indata[self.primaryOrder])
        steps = (len(keys)*1.0)
        t = time.time()
        for count, key in enumerate(keys):
            mask = self.indata[self.primaryOrder] == key
            gather = Gather(self.indata, self.outdata, mask)
            gather.data.sort(order=self.secondaryOrder)
            update_progress(gather.data.nbytes*1e-6/time.time()-t)
            t = time.time()
            yield gather

    def save(self):
        self.outdata[self.mask] = self.gather

    def run(self):
        '''iterates a series of gathers through the methods defined in the input.
        if no methods are definied make a copy of the data'''
        for gather in self:
            pass

    def close(self):
        del self.outdata

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import os
import re
import sys
import subprocess
from ConfigParser import SafeConfigParser
from traceback import format_exc

from subprocesses import captureStdout, isVM, normExpUserPath, vdump

def findCommonAncestor(repoDir, a, b):
    return captureStdout(['hg', '-R', repoDir, 'log', '-r', 'ancestor(' + a + ',' + b + ')',
                          '--template={node|short}'])[0]

def getCsetHashFromBisectMsg(str):
    # Example bisect msg: "Testing changeset 41831:4f4c01fb42c3 (2 changesets remaining, ~1 tests)"
    r = re.compile(r"(^|.* )(\d+):(\w{12}).*")
    m = r.match(str)
    if m:
        return m.group(3)

assert getCsetHashFromBisectMsg("x 12345:abababababab") == "abababababab"
assert getCsetHashFromBisectMsg("x 12345:123412341234") == "123412341234"
assert getCsetHashFromBisectMsg("12345:abababababab y") == "abababababab"

def getMcRepoDir():
    '''Returns default m-c repository location and its base directory depending on machine.'''
    if isVM() == ('Windows', True):  # Self-selected presets in custom VMs
        baseDir = os.path.join('z:', os.sep)
    elif isVM() == ('Linux', True):  # Self-selected presets in custom VMs
        baseDir = os.path.join('/', 'mnt', 'hgfs')
    else:
        baseDir = '~'
    mcRepoDir = normExpUserPath(os.path.join(baseDir, 'trees', 'mozilla-central'))
    assert getRepoNameFromHgrc(mcRepoDir) == 'mozilla-central'
    return baseDir, mcRepoDir

def getRepoHashAndId(repoDir, repoRev='parents() and default'):
    '''
    This function returns the repository hash and id, and whether it is on default.
    It also asks what the user would like to do, should the repository not be on default.
    '''
    # This returns null if the repository is not on default.
    hgLogTmplList = ['hg', '-R', repoDir, 'log', '-r', repoRev,
                     '--template', '{node|short} {rev}']
    hgIdFull = captureStdout(hgLogTmplList)[0]
    onDefault = bool(hgIdFull)
    if not onDefault:
        updateDefault = raw_input('Not on default tip! ' + \
            'Would you like to (a)bort, update to (d)efault, or (u)se this rev: ')
        if updateDefault == 'a':
            print 'Aborting...'
            sys.exit(0)
        elif updateDefault == 'd':
            subprocess.check_call(['hg', '-R', repoDir, 'up', 'default'])
            onDefault = True
        elif updateDefault == 'u':
            hgLogTmplList = ['hg', '-R', repoDir, 'log', '-r', 'parents()', '--template',
                             '{node|short} {rev}']
        else:
            raise Exception('Invalid choice.')
        hgIdFull = captureStdout(hgLogTmplList)[0]
    assert hgIdFull != ''
    (hgIdChangesetHash, hgIdLocalNum) = hgIdFull.split(' ')
    vdump('Finished getting the hash and local id number of the repository.')
    return hgIdChangesetHash, hgIdLocalNum, onDefault

def getRepoNameFromHgrc(repoDir):
    '''Looks in the hgrc file in the .hg directory of the repository and returns the name.'''
    hgrcpath = os.path.join(repoDir, '.hg', 'hgrc')
    assert os.path.isfile(hgrcpath)
    hgCfg = SafeConfigParser()
    hgCfg.read(hgrcpath)
    # Not all default entries in [paths] end with "/".
    return [i for i in hgCfg.get('paths', 'default').split('/') if i][-1]

def isAncestor(repoDir, a, b):
    return findCommonAncestor(repoDir, a, b) == a

def patchHgRepoUsingMq(patchLoc, workingDir=os.getcwdu()):
    # We may have passed in the patch with or without the full directory.
    p = os.path.abspath(normExpUserPath(patchLoc))
    pname = os.path.basename(p)
    assert (p, pname) != ('','')
    subprocess.check_call(['hg', '-R', workingDir, 'qimport', p])
    vdump("Patch qimport'ed.")
    try:
        subprocess.check_call(['hg', '-R', workingDir, 'qpush', pname])
        vdump("Patch qpush'ed.")
    except subprocess.CalledProcessError:
        subprocess.check_call(['hg', '-R', workingDir, 'qpop'])
        subprocess.check_call(['hg', '-R', workingDir, 'qdelete', pname])
        print 'You may have untracked .rej files in the repository.'
        print '`hg st` output of the repository in ' + workingDir + ' :'
        subprocess.check_call(['hg', '-R', workingDir, 'st'])
        hgPurgeAns = str(raw_input('Do you want to run `hg purge`? (y/n): '))
        assert hgPurgeAns.lower() in ('y', 'n')
        if hgPurgeAns == 'y':
            subprocess.check_call(['hg', '-R', workingDir, 'purge'])
        raise Exception(format_exc())
    return pname
<commit_msg>Make another WinXP-specific fix in hgCmds.py.<commit_after>#!/usr/bin/env python
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import os
import platform
import re
import sys
import subprocess
from ConfigParser import SafeConfigParser
from traceback import format_exc

from subprocesses import captureStdout, isVM, normExpUserPath, vdump

def findCommonAncestor(repoDir, a, b):
    return captureStdout(['hg', '-R', repoDir, 'log', '-r', 'ancestor(' + a + ',' + b + ')',
                          '--template={node|short}'])[0]

def getCsetHashFromBisectMsg(str):
    # Example bisect msg: "Testing changeset 41831:4f4c01fb42c3 (2 changesets remaining, ~1 tests)"
    r = re.compile(r"(^|.* )(\d+):(\w{12}).*")
    m = r.match(str)
    if m:
        return m.group(3)

assert getCsetHashFromBisectMsg("x 12345:abababababab") == "abababababab"
assert getCsetHashFromBisectMsg("x 12345:123412341234") == "123412341234"
assert getCsetHashFromBisectMsg("12345:abababababab y") == "abababababab"

def getMcRepoDir():
    '''Returns default m-c repository location and its base directory depending on machine.'''
    if isVM() == ('Windows', True):  # Self-selected presets in custom VMs
        baseDir = os.path.join('z:', os.sep)
    elif isVM() == ('Linux', True):  # Self-selected presets in custom VMs
        baseDir = os.path.join('/', 'mnt', 'hgfs')
    elif platform.uname()[2] == 'XP':  # WinXP contains spaces in the user directory
        baseDir = os.path.join('c:\\')
    else:
        baseDir = '~'
    mcRepoDir = normExpUserPath(os.path.join(baseDir, 'trees', 'mozilla-central'))
    assert getRepoNameFromHgrc(mcRepoDir) == 'mozilla-central'
    return baseDir, mcRepoDir

def getRepoHashAndId(repoDir, repoRev='parents() and default'):
    '''
    This function returns the repository hash and id, and whether it is on default.
    It also asks what the user would like to do, should the repository not be on default.
    '''
    # This returns null if the repository is not on default.
    hgLogTmplList = ['hg', '-R', repoDir, 'log', '-r', repoRev,
                     '--template', '{node|short} {rev}']
    hgIdFull = captureStdout(hgLogTmplList)[0]
    onDefault = bool(hgIdFull)
    if not onDefault:
        updateDefault = raw_input('Not on default tip! ' + \
            'Would you like to (a)bort, update to (d)efault, or (u)se this rev: ')
        if updateDefault == 'a':
            print 'Aborting...'
            sys.exit(0)
        elif updateDefault == 'd':
            subprocess.check_call(['hg', '-R', repoDir, 'up', 'default'])
            onDefault = True
        elif updateDefault == 'u':
            hgLogTmplList = ['hg', '-R', repoDir, 'log', '-r', 'parents()', '--template',
                             '{node|short} {rev}']
        else:
            raise Exception('Invalid choice.')
        hgIdFull = captureStdout(hgLogTmplList)[0]
    assert hgIdFull != ''
    (hgIdChangesetHash, hgIdLocalNum) = hgIdFull.split(' ')
    vdump('Finished getting the hash and local id number of the repository.')
    return hgIdChangesetHash, hgIdLocalNum, onDefault

def getRepoNameFromHgrc(repoDir):
    '''Looks in the hgrc file in the .hg directory of the repository and returns the name.'''
    hgrcpath = os.path.join(repoDir, '.hg', 'hgrc')
    assert os.path.isfile(hgrcpath)
    hgCfg = SafeConfigParser()
    hgCfg.read(hgrcpath)
    # Not all default entries in [paths] end with "/".
    return [i for i in hgCfg.get('paths', 'default').split('/') if i][-1]

def isAncestor(repoDir, a, b):
    return findCommonAncestor(repoDir, a, b) == a

def patchHgRepoUsingMq(patchLoc, workingDir=os.getcwdu()):
    # We may have passed in the patch with or without the full directory.
    p = os.path.abspath(normExpUserPath(patchLoc))
    pname = os.path.basename(p)
    assert (p, pname) != ('','')
    subprocess.check_call(['hg', '-R', workingDir, 'qimport', p])
    vdump("Patch qimport'ed.")
    try:
        subprocess.check_call(['hg', '-R', workingDir, 'qpush', pname])
        vdump("Patch qpush'ed.")
    except subprocess.CalledProcessError:
        subprocess.check_call(['hg', '-R', workingDir, 'qpop'])
        subprocess.check_call(['hg', '-R', workingDir, 'qdelete', pname])
        print 'You may have untracked .rej files in the repository.'
        print '`hg st` output of the repository in ' + workingDir + ' :'
        subprocess.check_call(['hg', '-R', workingDir, 'st'])
        hgPurgeAns = str(raw_input('Do you want to run `hg purge`? (y/n): '))
        assert hgPurgeAns.lower() in ('y', 'n')
        if hgPurgeAns == 'y':
            subprocess.check_call(['hg', '-R', workingDir, 'purge'])
        raise Exception(format_exc())
    return pname
<|endoftext|>


<commit_before>"""Time series management and processing."""

from dataclasses import dataclass

import numpy as np
from numpy import nan
from patlib.std import find_1st_ind
from struct_tools import NicePrint

from dapper.dpr_config import rc
from dapper.tools.rounding import log10int, round2, round2sigfig

def auto_cov(xx, nlags=4, zero_mean=False, corr=False):
    """Auto covariance function, computed along axis 0.

    - `nlags`: max lag (offset) for which to compute acf.
    - `corr` : normalize acf by `acf[0]` so as to return auto-CORRELATION.

    With `corr=True`, this is identical to
    `statsmodels.tsa.stattools.acf(xx,True,nlags)`
    """
    assert nlags < len(xx)

    N = len(xx)
    A = xx if zero_mean else (xx - xx.mean(0))
    acovf = np.zeros((nlags+1,)+xx.shape[1:])

    for i in range(nlags+1):
        Left  = A[np.arange(N-i)]
        Right = A[np.arange(i, N)]
        acovf[i] = (Left*Right).sum(0)/(N-i)

    if corr:
        acovf /= acovf[0]

    return acovf

def fit_acf_by_AR1(acf_empir, nlags=None):
    """Fit an empirical auto cov function (ACF) by that of an AR1 process.

    - `acf_empir`: auto-corr/cov-function.
    - `nlags`: length of ACF to use in AR(1) fitting
    """
    if nlags is None:
        nlags = len(acf_empir)

    # geometric_mean = ss.mstats.gmean
    def geometric_mean(xx): return np.exp(np.mean(np.log(xx)))

    def mean_ratio(xx):
        return geometric_mean([xx[i]/xx[i-1] for i in range(1, len(xx))])

    # Negative correlation => Truncate ACF
    neg_ind   = find_1st_ind(np.array(acf_empir) <= 0)
    acf_empir = acf_empir[:neg_ind]

    if len(acf_empir) == 0:
        return 0
    elif len(acf_empir) == 1:
        return 0.01
    else:
        return mean_ratio(acf_empir)

def estimate_corr_length(xx):
    r"""Estimate the correlation length of a time series.

    For explanation, see `dapper.mods.LA.homogeneous_1D_cov`.
    Also note that, for exponential corr function, as assumed here,

    $$\text{corr}(L) = \exp(-1) \approx 0.368$$
    """
    acovf = auto_cov(xx, min(100, len(xx)-2))
    a     = fit_acf_by_AR1(acovf)
    if a == 0:
        L = 0
    else:
        L = 1/np.log(1/a)
    return L

@dataclass
class UncertainQtty():
    """Data container associating uncertainty (confidence) to a quantity.

    Includes intelligent rounding and printing functionality.

    Examples:
    >>> for c in [.01, .1, .2, .9, 1]:
    ...    print(UncertainQtty(1.2345, c))
    1.23 ±0.01
    1.2 ±0.1
    1.2 ±0.2
    1.2 ±0.9
    1 ±1

    >>> for c in [.01, 1e-10, 1e-17, 0]:
    ...    print(UncertainQtty(1.2, c))
    1.20 ±0.01
    1.2000000000 ±1e-10
    1.19999999999999996 ±1e-17
    1.2000000000 ±0

    Note that in the case of a confidence of exactly 0,
    it defaults to 10 decimal places.
    Meanwhile, a NaN confidence yields printing using `rc.sigfig`:

    >>> print(UncertainQtty(1.234567, np.nan))
    1.235 ±nan

    Also note the effect of large uncertainty:

    >>> for c in [1, 9, 10, 11, 20, 100, np.inf]:
    ...    print(UncertainQtty(12, c))
    12 ±1
    12 ±9
    10 ±10
    10 ±10
    10 ±20
    0 ±100
    0 ±inf
    """

    val: float
    conf: float

    def round(self, mult=1.0):  # noqa
        """Round intelligently.

        - `conf` to 1 sig. fig.
        - `val`:
            - to precision: `mult*conf`
            - fallback: `rc.sigfig`
        """
        if np.isnan(self.conf):
            # Fallback to rc.sigfig
            c = self.conf
            v = round2sigfig(self.val, rc.sigfig)
        else:
            # Normal/general case
            c = round2sigfig(self.conf, 1)
            v = round2(self.val, mult*self.conf)
        return v, c

    def __str__(self):
        """Returns 'val ±conf'.

        The value string contains the number of decimals required by the confidence.

        Note: the special cases require processing on top of `round`.
        """
        v, c = self.round()
        if np.isnan(c):
            # Rounding to fallback (rc.sigfig) already took place
            return f"{v} ±{c}"
        if c == 0:
            # 0 (i.e. not 1e-300) never arises "naturally" => Treat it "magically"
            # by truncating to a default. Also see https://stackoverflow.com/a/25899600
            n = -10
        else:
            # Normal/general case.
            n = log10int(c)
        frmt = "%.f"
        if n < 0:
            # Ensure we get 1.30 ±0.01, NOT 1.3 ±0.01:
            frmt = "%%0.%df" % -n
        v = frmt % v
        return f"{v} ±{c}"

    def __repr__(self):
        """Essentially the same as `__str__`."""
        v, c = str(self).split(" ±")
        return self.__class__.__name__ + f"(val={v}, conf={c})"

def mean_with_conf(xx):
    """Compute the mean of a 1d iterable `xx`.

    Also provide confidence of mean,
    as estimated from its correlation-corrected variance.
    """
    mu = np.mean(xx)
    N  = len(xx)
    # TODO 3: review
    if (not np.isfinite(mu)) or N <= 5:
        uq = UncertainQtty(mu, np.nan)
    elif np.allclose(xx, mu):
        uq = UncertainQtty(mu, 0)
    else:
        acovf = auto_cov(xx)
        var   = acovf[0]
        var  /= N
        # Estimate (fit) ACF
        a = fit_acf_by_AR1(acovf)
        # If xx[k] where independent of xx[k-1],
        # then std_of_mu is the end of the story.
        # The following corrects for the correlation in the time series.
        #
        # See https://stats.stackexchange.com/q/90062
        # c = sum([(N-k)*a**k for k in range(1,N)])
        # But this series is analytically tractable:
        c = ((N-1)*a - N*a**2 + a**(N+1)) / (1-a)**2
        confidence_correction = 1 + 2/N * c
        var *= confidence_correction
        uq = UncertainQtty(mu, np.sqrt(var))
    return uq

class StatPrint(NicePrint):
    """Set `NicePrint` options suitable for stats."""

    printopts = dict(
        excluded=NicePrint.printopts["excluded"]+["HMM", "LP_instance"],
        ordering="linenumber",
        reverse=True,
        indent=2,
        aliases={
            'f': 'Forecast  (.f)',
            'a': 'Analysis  (.a)',
            's': 'Smoothed  (.s)',
            'u': 'Universal (.u)',
            'm': 'Field mean (.m)',
            'ma': 'Field mean-abs (.ma)',
            'rms': 'Field root-mean-square (.rms)',
            'gm': 'Field geometric-mean (.gm)',
        },
    )

    # Adjust np.printoptions before NicePrint
    def __repr__(self):
        with np.printoptions(threshold=10, precision=3):
            return super().__repr__()

    def __str__(self):
        with np.printoptions(threshold=10, precision=3):
            return super().__str__()

def monitor_setitem(cls):
    """Modify cls to track of whether its `__setitem__` has been called.

    See sub.py for a sublcass solution (drawback: creates a new class).
    """
    orig_setitem = cls.__setitem__

    def setitem(self, key, val):
        orig_setitem(self, key, val)
        self.were_changed = True
    cls.__setitem__ = setitem

    # Using class var for were_changed => don't need explicit init
    cls.were_changed = False

    if issubclass(cls, NicePrint):
        cls.printopts['excluded'] = \
                cls.printopts.get('excluded', []) + ['were_changed']

    return cls

@monitor_setitem
class DataSeries(StatPrint):
    """Basically just an `np.ndarray`. But adds:

    - Possibility of adding attributes.
    - The class (type) provides way to acertain if an attribute is a series.

    Note: subclassing `ndarray` is too dirty => We'll just use the
    `array` attribute, and provide `{s,g}etitem`.
    """

    def __init__(self, shape, **kwargs):
        self.array = np.full(shape, nan, **kwargs)

    def __len__(self): return len(self.array)
    def __getitem__(self, key): return self.array[key]
    def __setitem__(self, key, val): self.array[key] = val

@monitor_setitem
class FAUSt(DataSeries, StatPrint):
    """Container for time series of a statistic from filtering.

    Four attributes, each of which is an ndarray:

    - `.f` for forecast      , `(KObs+1,)+item_shape`
    - `.a` for analysis      , `(KObs+1,)+item_shape`
    - `.s` for smoothed      , `(KObs+1,)+item_shape`
    - `.u` for universial/all, `(K   +1,)+item_shape`

    If `store_u=False`, then `.u` series has shape `(1,)+item_shape`,
    wherein only the most-recently-written item is stored.

    Series can also be indexed as in

        self[kObs,'a']
        self[whatever,kObs,'a']
        # ... and likewise for 'f' and 's'. For 'u', can use:
        self[k,'u']
        self[k,whatever,'u']

    .. note:: If a data series only pertains to analysis times,
              then you should use a plain np.array instead.
    """

    def __init__(self, K, KObs, item_shape, store_u, store_s, **kwargs):
        """Construct object.

        - `item_shape` : shape of an item in the series.
        - `store_u`    : if False: only the current value is stored.
        - `kwargs`     : passed on to ndarrays.
        """
        self.f     = np.full((KObs+1,)+item_shape, nan, **kwargs)
        self.a     = np.full((KObs+1,)+item_shape, nan, **kwargs)
        if store_s:
            self.s = np.full((KObs+1,)+item_shape, nan, **kwargs)
        if store_u:
            self.u = np.full((K   + 1,)+item_shape, nan, **kwargs)
        else:
            self.u = np.full((1,)+item_shape, nan, **kwargs)

    # We could just store the input values for these attrs, but using
    # property => Won't be listed in vars(self), and un-writeable.
    item_shape = property(lambda self: self.a.shape[1:])
    store_u    = property(lambda self: len(self.u) > 1)

    def _ind(self, key):
        """Aux function to unpack `key` (`k,kObs,faus`)"""
        if key[-1] == 'u':
            return key[0] if self.store_u else 0
        else:
            return key[-2]

    def __setitem__(self, key, item):
        getattr(self, key[-1])[self._ind(key)] = item

    def __getitem__(self, key):
        return getattr(self, key[-1])[self._ind(key)]

class RollingArray:
    """ND-Array that implements "leftward rolling" along axis 0.

    Used for data that gets plotted in sliding graphs.
    """

    def __init__(self, shape, fillval=nan):
        self.array = np.full(shape, fillval)
        self.k1 = 0      # previous k
        self.nFilled = 0

    def insert(self, k, val):
        dk = k-self.k1

        # Old (more readable?) version:
        # if dk in [0,1]: # case: forecast or analysis update
        # self.array = np.roll(self.array, -1, axis=0)
        # elif dk>1:      # case: user has skipped ahead (w/o liveplotting)
        # self.array = np.roll(self.array, -dk, axis=0)
        # self.array[-dk:] = nan
        # self.array[-1] = val

        dk = max(1, dk)
        # TODO 7: Should have used deque?
        self.array = np.roll(self.array, -dk, axis=0)
        self.array[-dk:] = nan
        self.array[-1:] = val

        self.k1 = k
        self.nFilled = min(len(self), self.nFilled+dk)

    def leftmost(self):
        return self[len(self)-self.nFilled]

    def span(self):
        return (self.leftmost(), self[-1])

    @property
    def T(self):
        return self.array.T

    def __array__(self, dtype=None): return self.array(dtype=dtype)
    def __len__(self): return len(self.array)
    def __repr__(self): return 'RollingArray:\n%s' % str(self.array)
    def __getitem__(self, key): return self.array[key]

    def __setitem__(self, key, val):
        # Don't implement __setitem__ coz leftmost() is then
        # not generally meaningful (i.e. if an element is set in the middle).
        # Of course self.array can still be messed with.
        raise AttributeError("Values should be set with update()")
<commit_msg>Fix bug<commit_after>"""Time series management and processing."""

from dataclasses import dataclass

import numpy as np
from numpy import nan
from patlib.std import find_1st_ind
from struct_tools import NicePrint

from dapper.dpr_config import rc
from dapper.tools.rounding import log10int, round2, round2sigfig

def auto_cov(xx, nlags=4, zero_mean=False, corr=False):
    """Auto covariance function, computed along axis 0.

    - `nlags`: max lag (offset) for which to compute acf.
    - `corr` : normalize acf by `acf[0]` so as to return auto-CORRELATION.

    With `corr=True`, this is identical to
    `statsmodels.tsa.stattools.acf(xx,True,nlags)`
    """
    assert nlags < len(xx)

    N = len(xx)
    A = xx if zero_mean else (xx - xx.mean(0))
    acovf = np.zeros((nlags+1,)+xx.shape[1:])

    for i in range(nlags+1):
        Left  = A[np.arange(N-i)]
        Right = A[np.arange(i, N)]
        acovf[i] = (Left*Right).sum(0)/(N-i)

    if corr:
        acovf /= acovf[0]

    return acovf

def fit_acf_by_AR1(acf_empir, nlags=None):
    """Fit an empirical auto cov function (ACF) by that of an AR1 process.

    - `acf_empir`: auto-corr/cov-function.
    - `nlags`: length of ACF to use in AR(1) fitting
    """
    if nlags is None:
        nlags = len(acf_empir)

    # geometric_mean = ss.mstats.gmean
    def geometric_mean(xx): return np.exp(np.mean(np.log(xx)))

    def mean_ratio(xx):
        return geometric_mean([xx[i]/xx[i-1] for i in range(1, len(xx))])

    # Negative correlation => Truncate ACF
    neg_ind   = find_1st_ind(np.array(acf_empir) <= 0)
    acf_empir = acf_empir[:neg_ind]

    if len(acf_empir) == 0:
        return 0
    elif len(acf_empir) == 1:
        return 0.01
    else:
        return mean_ratio(acf_empir)

def estimate_corr_length(xx):
    r"""Estimate the correlation length of a time series.

    For explanation, see `dapper.mods.LA.homogeneous_1D_cov`.
    Also note that, for exponential corr function, as assumed here,

    $$\text{corr}(L) = \exp(-1) \approx 0.368$$
    """
    acovf = auto_cov(xx, min(100, len(xx)-2))
    a     = fit_acf_by_AR1(acovf)
    if a == 0:
        L = 0
    else:
        L = 1/np.log(1/a)
    return L

@dataclass
class UncertainQtty():
    """Data container associating uncertainty (confidence) to a quantity.

    Includes intelligent rounding and printing functionality.

    Examples:
    >>> for c in [.01, .1, .2, .9, 1]:
    ...    print(UncertainQtty(1.2345, c))
    1.23 ±0.01
    1.2 ±0.1
    1.2 ±0.2
    1.2 ±0.9
    1 ±1

    >>> for c in [.01, 1e-10, 1e-17, 0]:
    ...    print(UncertainQtty(1.2, c))
    1.20 ±0.01
    1.2000000000 ±1e-10
    1.19999999999999996 ±1e-17
    1.2000000000 ±0

    Note that in the case of a confidence of exactly 0,
    it defaults to 10 decimal places.
    Meanwhile, a NaN confidence yields printing using `rc.sigfig`:

    >>> print(UncertainQtty(1.234567, np.nan))
    1.235 ±nan

    Also note the effect of large uncertainty:

    >>> for c in [1, 9, 10, 11, 20, 100, np.inf]:
    ...    print(UncertainQtty(12, c))
    12 ±1
    12 ±9
    10 ±10
    10 ±10
    10 ±20
    0 ±100
    0 ±inf
    """

    val: float
    conf: float

    def round(self, mult=1.0):  # noqa
        """Round intelligently.

        - `conf` to 1 sig. fig.
        - `val`:
            - to precision: `mult*conf`
            - fallback: `rc.sigfig`
        """
        if np.isnan(self.conf):
            # Fallback to rc.sigfig
            c = self.conf
            v = round2sigfig(self.val, rc.sigfig)
        else:
            # Normal/general case
            c = round2sigfig(self.conf, 1)
            v = round2(self.val, mult*self.conf)
        return v, c

    def __str__(self):
        """Returns 'val ±conf'.

        The value string contains the number of decimals required by the confidence.

        Note: the special cases require processing on top of `round`.
        """
        v, c = self.round()
        if np.isnan(c):
            # Rounding to fallback (rc.sigfig) already took place
            return f"{v} ±{c}"
        if c == 0:
            # 0 (i.e. not 1e-300) never arises "naturally" => Treat it "magically"
            # by truncating to a default. Also see https://stackoverflow.com/a/25899600
            n = -10
        else:
            # Normal/general case.
            n = log10int(c)
        frmt = "%.f"
        if n < 0:
            # Ensure we get 1.30 ±0.01, NOT 1.3 ±0.01:
            frmt = "%%0.%df" % -n
        v = frmt % v
        return f"{v} ±{c}"

    def __repr__(self):
        """Essentially the same as `__str__`."""
        v, c = str(self).split(" ±")
        return self.__class__.__name__ + f"(val={v}, conf={c})"

def mean_with_conf(xx):
    """Compute the mean of a 1d iterable `xx`.

    Also provide confidence of mean,
    as estimated from its correlation-corrected variance.
    """
    mu = np.mean(xx)
    N  = len(xx)
    # TODO 3: review
    if (not np.isfinite(mu)) or N <= 5:
        uq = UncertainQtty(mu, np.nan)
    elif np.allclose(xx, mu):
        uq = UncertainQtty(mu, 0)
    else:
        acovf = auto_cov(xx)
        var   = acovf[0]
        var  /= N
        # Estimate (fit) ACF
        a = fit_acf_by_AR1(acovf)
        # If xx[k] where independent of xx[k-1],
        # then std_of_mu is the end of the story.
        # The following corrects for the correlation in the time series.
        #
        # See https://stats.stackexchange.com/q/90062
        # c = sum([(N-k)*a**k for k in range(1,N)])
        # But this series is analytically tractable:
        c = ((N-1)*a - N*a**2 + a**(N+1)) / (1-a)**2
        confidence_correction = 1 + 2/N * c
        var *= confidence_correction
        uq = UncertainQtty(mu, np.sqrt(var))
    return uq

class StatPrint(NicePrint):
    """Set `NicePrint` options suitable for stats."""

    printopts = dict(
        excluded=NicePrint.printopts["excluded"]+["HMM", "LP_instance"],
        ordering="linenumber",
        reverse=True,
        indent=2,
        aliases={
            'f': 'Forecast  (.f)',
            'a': 'Analysis  (.a)',
            's': 'Smoothed  (.s)',
            'u': 'Universal (.u)',
            'm': 'Field mean (.m)',
            'ma': 'Field mean-abs (.ma)',
            'rms': 'Field root-mean-square (.rms)',
            'gm': 'Field geometric-mean (.gm)',
        },
    )

    # Adjust np.printoptions before NicePrint
    def __repr__(self):
        with np.printoptions(threshold=10, precision=3):
            return super().__repr__()

    def __str__(self):
        with np.printoptions(threshold=10, precision=3):
            return super().__str__()

def monitor_setitem(cls):
    """Modify cls to track of whether its `__setitem__` has been called.

    See sub.py for a sublcass solution (drawback: creates a new class).
    """
    orig_setitem = cls.__setitem__

    def setitem(self, key, val):
        orig_setitem(self, key, val)
        self.were_changed = True
    cls.__setitem__ = setitem

    # Using class var for were_changed => don't need explicit init
    cls.were_changed = False

    if issubclass(cls, NicePrint):
        cls.printopts['excluded'] = \
                cls.printopts.get('excluded', []) + ['were_changed']

    return cls

@monitor_setitem
class DataSeries(StatPrint):
    """Basically just an `np.ndarray`. But adds:

    - Possibility of adding attributes.
    - The class (type) provides way to acertain if an attribute is a series.

    Note: subclassing `ndarray` is too dirty => We'll just use the
    `array` attribute, and provide `{s,g}etitem`.
    """

    def __init__(self, shape, **kwargs):
        self.array = np.full(shape, nan, **kwargs)

    def __len__(self): return len(self.array)
    def __getitem__(self, key): return self.array[key]
    def __setitem__(self, key, val): self.array[key] = val

@monitor_setitem
class FAUSt(DataSeries, StatPrint):
    """Container for time series of a statistic from filtering.

    Four attributes, each of which is an ndarray:

    - `.f` for forecast      , `(KObs+1,)+item_shape`
    - `.a` for analysis      , `(KObs+1,)+item_shape`
    - `.s` for smoothed      , `(KObs+1,)+item_shape`
    - `.u` for universial/all, `(K   +1,)+item_shape`

    If `store_u=False`, then `.u` series has shape `(1,)+item_shape`,
    wherein only the most-recently-written item is stored.

    Series can also be indexed as in

        self[kObs,'a']
        self[whatever,kObs,'a']
        # ... and likewise for 'f' and 's'. For 'u', can use:
        self[k,'u']
        self[k,whatever,'u']

    .. note:: If a data series only pertains to analysis times,
              then you should use a plain np.array instead.
    """

    def __init__(self, K, KObs, item_shape, store_u, store_s, **kwargs):
        """Construct object.

        - `item_shape` : shape of an item in the series.
        - `store_u`    : if False: only the current value is stored.
        - `kwargs`     : passed on to ndarrays.
        """
        self.f     = np.full((KObs+1,)+item_shape, nan, **kwargs)
        self.a     = np.full((KObs+1,)+item_shape, nan, **kwargs)
        if store_s:
            self.s = np.full((KObs+1,)+item_shape, nan, **kwargs)
        if store_u:
            self.u = np.full((K   + 1,)+item_shape, nan, **kwargs)
        else:
            self.u = np.full((1,)+item_shape, nan, **kwargs)

    # We could just store the input values for these attrs, but using
    # property => Won't be listed in vars(self), and un-writeable.
    item_shape = property(lambda self: self.a.shape[1:])
    store_u    = property(lambda self: len(self.u) > 1)

    def _ind(self, key):
        """Aux function to unpack `key` (`k,kObs,faus`)"""
        if key[-1] == 'u':
            return key[0] if self.store_u else 0
        else:
            return key[-2]

    def __setitem__(self, key, item):
        getattr(self, key[-1])[self._ind(key)] = item

    def __getitem__(self, key):
        return getattr(self, key[-1])[self._ind(key)]

class RollingArray:
    """ND-Array that implements "leftward rolling" along axis 0.

    Used for data that gets plotted in sliding graphs.
    """

    def __init__(self, shape, fillval=nan):
        self.array = np.full(shape, fillval)
        self.k1 = 0      # previous k
        self.nFilled = 0

    def insert(self, k, val):
        dk = k-self.k1

        # Old (more readable?) version:
        # if dk in [0,1]: # case: forecast or analysis update
        # self.array = np.roll(self.array, -1, axis=0)
        # elif dk>1:      # case: user has skipped ahead (w/o liveplotting)
        # self.array = np.roll(self.array, -dk, axis=0)
        # self.array[-dk:] = nan
        # self.array[-1] = val

        dk = max(1, dk)
        # TODO 7: Should have used deque?
        self.array = np.roll(self.array, -dk, axis=0)
        self.array[-dk:] = nan
        self.array[-1:] = val

        self.k1 = k
        self.nFilled = min(len(self), self.nFilled+dk)

    def leftmost(self):
        return self[len(self)-self.nFilled]

    def span(self):
        return (self.leftmost(), self[-1])

    @property
    def T(self):
        return self.array.T

    def __array__(self, _dtype=None): return self.array
    def __len__(self): return len(self.array)
    def __repr__(self): return 'RollingArray:\n%s' % str(self.array)
    def __getitem__(self, key): return self.array[key]

    def __setitem__(self, key, val):
        # Don't implement __setitem__ coz leftmost() is then
        # not generally meaningful (i.e. if an element is set in the middle).
        # Of course self.array can still be messed with.
        raise AttributeError("Values should be set with update()")
<|endoftext|>


<commit_before>import csv
import sys
import collections
import pandas as pd
import numpy as np

####################
# Plate Reader IDs #
####################
plate_reader_ids = {"268449":'b1',
                    "271275":'b2',
                    "1402031D":'b3'}

####################
# Calibration Data #
####################
calibration_data = dict()
calibration_data["GFP"] = {'b3': {61:2261, 100:80850},
                           'b2': {61:1762, 100:62732},
                           'b1': {61:1517, 100:53258}}
calibration_data["Citrine"] = {'b3': {61:2379, 100:82680},
                               'b2': {61:1849, 100:64985},
                               'b1': {61:1487, 100:51725}}
calibration_data["RFP"] = {'b3': {61:70.38, 100:2541},
                           'b2': {61:67.64, 100:2392},
                           'b1': {61:47.96, 100:1689}}
calibration_data["CFP"] = {'b3': {61:578, 100:20722},
                           'b2': {61:513, 100:18136},
                           'b1': {61:387, 100:13809}}
calibration_data["Venus"] = {'b3': {61:2246, 100:79955},
                             'b2': {61:1742, 100:61130},
                             'b1': {61:1455, 100:50405}}
calibration_data["Cherry"] = {'b3': {61:79.39, 100:2850},
                              'b2': {61:80.84, 100:2822},
                              'b1': {61:51.07, 100:1782}}

def standard_channel_name(fp_name):
    upper_name = fp_name.upper()
    for std_name in ["GFP", "Citrine", "RFP", "CFP", "Venus", "Cherry"]:
        if std_name in upper_name:
            return std_name
    raise UserWarning(("Unable to convert channel %s into standard channel " + \
                      "name. Are you sure this is the right name?") % fp_name)
    return fp_name

def raw_to_uM(raw, protein, biotek, gain):
    if not protein in calibration_data or \
       not biotek in calibration_data[protein] or \
       not gain in calibration_data[protein][biotek]:
       return None
    return float(raw) / calibration_data[protein][biotek][gain]

ReadSet = collections.namedtuple('ReadSet', ['name', 'excitation', 'emission',
                                             'gain'])

def read_supplementary_info(input_filename):
    info = dict()
    with open(input_filename, 'rU') as infile:
        reader = csv.reader(infile)
        title_line = reader.next()
        for i in range(1, len(title_line)):
            info[title_line[i]] = dict()
        for line in reader:
            if line[0].strip() == "":
                continue
            for i in range(1, len(title_line)):
                info[title_line[i]][line[0]] = line[i]
    return info

def tidy_biotek_data(input_filename, supplementary_filename = None):
    '''
    Convert the raw output from a Biotek plate reader into tidy data.
    Optionally, also adds columns of metadata specified by a "supplementary
    file", which is a CSV spreadsheet mapping well numbers to metadata.

    Arguments:
        --input_filename: Name of a Biotek output file. Data file should be
                            standard excel output files, saved as a CSV.
        --supplementary_filename: Name of a supplementary file. Supplementary
                                    file must be a CSV wit a header, where the
                                    first column is the name of the well,
                                    additional columns define additional
                                    metadata, and each row after the header is a
                                    single well's metadata. Defaults to None
                                    (no metadata other than what can be mined
                                    from the data file).
    Returns: None
    Side Effects: Creates a new CSV with the same name as the data file with
                    "_tidy" appended to the end. This new file is in tidy
                    format, with each row representing a single channel read
                    from a single well at a single time.

    '''
    supplementary_data = dict()
    if supplementary_filename:
        supplementary_data = read_supplementary_info(supplementary_filename)
    filename_base   = input_filename.rsplit('.')[0]
    output_filename = filename_base + "_tidy.csv"

    with open(input_filename, 'rU') as infile:
        with open(output_filename, 'w') as outfile:
            reader = csv.reader(infile)
            writer = csv.writer(outfile, delimiter = ',')
            title_row = ['Channel', 'Gain', 'Time (sec)', 'Well', 'AFU', 'uM',
                             'Excitation', 'Emission']
            for name in supplementary_data.keys():
                title_row.append(name)
            writer.writerow(title_row)

            # Read plate information
            read_sets = dict()
            for line in reader:
                if line[0].strip() == "Reader Serial Number:":
                    if line[1] in plate_reader_ids:
                        plate_reader_id = plate_reader_ids[line[1]]
                    else:
                        raise ValueError("Unknown plate reader id '%s'" % \
                                         line[1])
                    continue
                if line[0] == "Read":
                    read_name = line[1]
                    for line in reader:
                        if line[1].startswith("Read Height"):
                            break
                        if line[1].startswith("Filter Set"):
                            line = reader.next()
                            lineparts = line[1].split(",")
                            excitation = int(lineparts[0].split(":")[-1].strip())
                            emission = int(lineparts[1].split(":")[-1].strip())
                            line = reader.next()
                            gain = int(line[1].split(",")[-1].split(":")[-1].strip())
                            if not read_name in read_sets:
                                read_sets[read_name] = []
                            read_sets[read_name].append(ReadSet(read_name,
                                                                excitation,
                                                                emission, gain))
                if line[0] == "End Kinetic":
                    break

            # Read data blocks
            # Find a data block
            for line in reader:
                if line[0].strip() == "":
                    continue
                info = line[0].strip()
                if info in ["Layout", "Results"]:
                    continue
                read_name = info.split(":")[0]
                if not info.endswith(']'):
                    read_idx = 0
                else:
                    read_idx = int(info.split('[')[-1][:-1]) - 1
                read_properties = read_sets[read_name][read_idx]
                gain            = read_properties.gain
                excitation      = read_properties.excitation
                emission        = read_properties.emission

                line = reader.next() # Skip a line
                line = reader.next() # Chart title line
                well_names = line
                # Data lines
                for line in reader:
                    if line[1] == "":
                        break
                    raw_time = line[1]
                    time_parts = raw_time.split(':')
                    time = int(time_parts[2]) + 60*int(time_parts[1]) \
                           + 3600*int(time_parts[0])
                    temp = line[2]
                    for i in range(3,len(line)):
                        if line[i].strip() == "":
                            break
                        well_name = well_names[i]
                        afu       = line[i]
                        uM_conc   = raw_to_uM(line[i],
                                              standard_channel_name(read_name),
                                              plate_reader_id, gain)
                        if uM_conc == None:
                            uM_conc = -1
                        row = [read_name, gain, time, well_name, afu, uM_conc,
                               excitation, emission]
                        for name in supplementary_data.keys():
                            row.append(supplementary_data[name][well_name])
                        writer.writerow(row)

def background_subtract(df, negative_control_wells):
    '''
    Create a new version of a dataframe with background removed. Background is
    inferred from one or more negative control wells. If more than one negative
    control is specified, the average of the wells is used as a background
    value.

    Note that this function assumes that every measurement has a corresponding
    negative control measurement in each of the negative control wells (same
    channel and gain).

    Arguments:
        df -- DataFrame of Biotek data, pulled from a tidy dataset of the form
                produced by tidy_biotek_data.
        negative_control_wells -- String or iterable of Strings specifying one
                                    or more negative control wells.
    Returns: A new DataFrame with background subtracted out.
    '''
    if type(negative_control_wells) == str:
        negative_control_wells = [negative_control_wells]
    return_df = pd.DataFrame()
    # Split the dataframe by channel and gain
    for channel in df.Channel.unique():
        channel_df = df[df.Channel == channel]
        for gain in channel_df.Gain.unique():
            condition_df = channel_df[channel_df.Gain == gain]
            neg_ctrl_df  = pd.DataFrame()
            for well in negative_control_wells:
                well_df = condition_df[condition_df.Well == well]
                neg_ctrl_df = neg_ctrl_df.append(well_df)
            grouped_neg_ctrl = neg_ctrl_df.groupby(["Time (sec)"])
            avg_neg_ctrl = grouped_neg_ctrl.aggregate(np.average)
            avg_neg_ctrl.sort_index(inplace = True)
            avg_neg_ctrl.reset_index(inplace = True)
            # Easiest thing to do is to apply the background subtraction to each
            # well separately
            for well in condition_df.Well.unique():
                well_df = condition_df[condition_df.Well == well].copy()
                well_df.sort_values("Time (sec)", inplace = True)
                well_df.reset_index(inplace = True)
                well_df.AFU = well_df.AFU - avg_neg_ctrl.AFU
                if channel in calibration_data and \
                   gain in calibration_data[channel]['b1']:
                    well_df.uM = well_df.uM - avg_neg_ctrl.uM
                return_df = return_df.append(well_df)
    return return_df

<commit_msg>Automated reinstallation push<commit_after>import csv
import sys
import collections
import pandas as pd
import numpy as np
import warnings

####################
# Plate Reader IDs #
####################
plate_reader_ids = {"268449":'b1',
                    "271275":'b2',
                    "1402031D":'b3'}

####################
# Calibration Data #
####################
calibration_data = dict()
calibration_data["GFP"] = {'b3': {61:2261, 100:80850},
                           'b2': {61:1762, 100:62732},
                           'b1': {61:1517, 100:53258}}
calibration_data["Citrine"] = {'b3': {61:2379, 100:82680},
                               'b2': {61:1849, 100:64985},
                               'b1': {61:1487, 100:51725}}
calibration_data["RFP"] = {'b3': {61:70.38, 100:2541},
                           'b2': {61:67.64, 100:2392},
                           'b1': {61:47.96, 100:1689}}
calibration_data["CFP"] = {'b3': {61:578, 100:20722},
                           'b2': {61:513, 100:18136},
                           'b1': {61:387, 100:13809}}
calibration_data["Venus"] = {'b3': {61:2246, 100:79955},
                             'b2': {61:1742, 100:61130},
                             'b1': {61:1455, 100:50405}}
calibration_data["Cherry"] = {'b3': {61:79.39, 100:2850},
                              'b2': {61:80.84, 100:2822},
                              'b1': {61:51.07, 100:1782}}

def standard_channel_name(fp_name, suppress_name_warning = False):
    upper_name = fp_name.upper()
    for std_name in ["GFP", "Citrine", "RFP", "CFP", "Venus", "Cherry"]:
        if std_name in upper_name:
            return std_name
    if not suppress_name_warning:
    warnings.warn(("Unable to convert channel %s into standard channel " + \
                   "name. Are you sure this is the right name?") % fp_name)
    return fp_name

def raw_to_uM(raw, protein, biotek, gain):
    if not protein in calibration_data or \
       not biotek in calibration_data[protein] or \
       not gain in calibration_data[protein][biotek]:
       return None
    return float(raw) / calibration_data[protein][biotek][gain]

ReadSet = collections.namedtuple('ReadSet', ['name', 'excitation', 'emission',
                                             'gain'])

def read_supplementary_info(input_filename):
    info = dict()
    with open(input_filename, 'rU') as infile:
        reader = csv.reader(infile)
        title_line = reader.next()
        for i in range(1, len(title_line)):
            info[title_line[i]] = dict()
        for line in reader:
            if line[0].strip() == "":
                continue
            for i in range(1, len(title_line)):
                info[title_line[i]][line[0]] = line[i]
    return info

def tidy_biotek_data(input_filename, supplementary_filename = None):
    '''
    Convert the raw output from a Biotek plate reader into tidy data.
    Optionally, also adds columns of metadata specified by a "supplementary
    file", which is a CSV spreadsheet mapping well numbers to metadata.

    Arguments:
        --input_filename: Name of a Biotek output file. Data file should be
                            standard excel output files, saved as a CSV.
        --supplementary_filename: Name of a supplementary file. Supplementary
                                    file must be a CSV wit a header, where the
                                    first column is the name of the well,
                                    additional columns define additional
                                    metadata, and each row after the header is a
                                    single well's metadata. Defaults to None
                                    (no metadata other than what can be mined
                                    from the data file).
    Returns: None
    Side Effects: Creates a new CSV with the same name as the data file with
                    "_tidy" appended to the end. This new file is in tidy
                    format, with each row representing a single channel read
                    from a single well at a single time.

    '''
    supplementary_data = dict()
    if supplementary_filename:
        supplementary_data = read_supplementary_info(supplementary_filename)
    filename_base   = input_filename.rsplit('.')[0]
    output_filename = filename_base + "_tidy.csv"

    with open(input_filename, 'rU') as infile:
        with open(output_filename, 'w') as outfile:
            reader = csv.reader(infile)
            writer = csv.writer(outfile, delimiter = ',')
            title_row = ['Channel', 'Gain', 'Time (sec)', 'Well', 'AFU', 'uM',
                             'Excitation', 'Emission']
            for name in supplementary_data.keys():
                title_row.append(name)
            writer.writerow(title_row)

            # Read plate information
            read_sets = dict()
            for line in reader:
                if line[0].strip() == "Reader Serial Number:":
                    if line[1] in plate_reader_ids:
                        plate_reader_id = plate_reader_ids[line[1]]
                    else:
                        raise ValueError("Unknown plate reader id '%s'" % \
                                         line[1])
                    continue
                if line[0] == "Read":
                    read_name = line[1]
                    for line in reader:
                        if line[1].startswith("Read Height"):
                            break
                        if line[1].startswith("Filter Set"):
                            line = reader.next()
                            lineparts = line[1].split(",")
                            excitation = int(lineparts[0].split(":")[-1].strip())
                            emission = int(lineparts[1].split(":")[-1].strip())
                            line = reader.next()
                            gain = int(line[1].split(",")[-1].split(":")[-1].strip())
                            if not read_name in read_sets:
                                read_sets[read_name] = []
                            read_sets[read_name].append(ReadSet(read_name,
                                                                excitation,
                                                                emission, gain))
                if line[0] == "End Kinetic":
                    break

            # Read data blocks
            # Find a data block
            for line in reader:
                if line[0].strip() == "":
                    continue
                info = line[0].strip()
                if info in ["Layout", "Results"]:
                    continue
                read_name = info.split(":")[0]
                if not info.endswith(']'):
                    read_idx = 0
                else:
                    read_idx = int(info.split('[')[-1][:-1]) - 1
                read_properties = read_sets[read_name][read_idx]
                gain            = read_properties.gain
                excitation      = read_properties.excitation
                emission        = read_properties.emission

                line = reader.next() # Skip a line
                line = reader.next() # Chart title line
                well_names = line
                # Data lines
                for line in reader:
                    if line[1] == "":
                        break
                    raw_time = line[1]
                    time_parts = raw_time.split(':')
                    time = int(time_parts[2]) + 60*int(time_parts[1]) \
                           + 3600*int(time_parts[0])
                    temp = line[2]
                    for i in range(3,len(line)):
                        if line[i].strip() == "":
                            break
                        well_name = well_names[i]
                        afu       = line[i]
                        uM_conc   = raw_to_uM(line[i],
                                              standard_channel_name(read_name),
                                              plate_reader_id, gain)
                        if uM_conc == None:
                            uM_conc = -1
                        row = [read_name, gain, time, well_name, afu, uM_conc,
                               excitation, emission]
                        for name in supplementary_data.keys():
                            row.append(supplementary_data[name][well_name])
                        writer.writerow(row)

def background_subtract(df, negative_control_wells):
    '''
    Create a new version of a dataframe with background removed. Background is
    inferred from one or more negative control wells. If more than one negative
    control is specified, the average of the wells is used as a background
    value.

    Note that this function assumes that every measurement has a corresponding
    negative control measurement in each of the negative control wells (same
    channel and gain).

    Arguments:
        df -- DataFrame of Biotek data, pulled from a tidy dataset of the form
                produced by tidy_biotek_data.
        negative_control_wells -- String or iterable of Strings specifying one
                                    or more negative control wells.
    Returns: A new DataFrame with background subtracted out.
    '''
    if type(negative_control_wells) == str:
        negative_control_wells = [negative_control_wells]
    return_df = pd.DataFrame()
    # Split the dataframe by channel and gain
    for channel in df.Channel.unique():
        channel_df = df[df.Channel == channel]
        for gain in channel_df.Gain.unique():
            condition_df = channel_df[channel_df.Gain == gain]
            neg_ctrl_df  = pd.DataFrame()
            for well in negative_control_wells:
                well_df = condition_df[condition_df.Well == well]
                neg_ctrl_df = neg_ctrl_df.append(well_df)
            grouped_neg_ctrl = neg_ctrl_df.groupby(["Time (sec)"])
            avg_neg_ctrl = grouped_neg_ctrl.aggregate(np.average)
            avg_neg_ctrl.sort_index(inplace = True)
            avg_neg_ctrl.reset_index(inplace = True)
            # Easiest thing to do is to apply the background subtraction to each
            # well separately
            for well in condition_df.Well.unique():
                well_df = condition_df[condition_df.Well == well].copy()
                well_df.sort_values("Time (sec)", inplace = True)
                well_df.reset_index(inplace = True)
                well_df.AFU = well_df.AFU - avg_neg_ctrl.AFU
                if channel in calibration_data and \
                   gain in calibration_data[channel]['b1']:
                    well_df.uM = well_df.uM - avg_neg_ctrl.uM
                return_df = return_df.append(well_df)
    return return_df

<|endoftext|>


<commit_before>import mysql.connector
import os

from ... import config

# FIXME: use proper config files
if os.environ.get('PAAS_MANAGER_ENV') == 'test':
    config['mysql']['database'] += '_test'

def db_action(fn):
    def wrapped(*args, **kwargs):
        res = fn(*args, **kwargs)
        DatabaseConnector.connect.commit()
        return res
    return wrapped

class DatabaseConnector():
    connect = mysql.connector.connect(**config['mysql'])
    cursor = connect.cursor()

    def __init__(self, args=None):
        if args is None:
            args = {}
        self.__dict__ = args
        if not hasattr(self, 'id'):
            self.id = None
        for k, v in args.items():
            setattr(self, k, v)

    @classmethod
    def hydrate_obj(cls, values):
        if not values:
            return None
        fields = [i[0] for i in cls.cursor.description]
        obj = cls()
        for i, f in enumerate(fields):
            setattr(obj, f, values[i])
        return obj

    @classmethod
    def _val_to_cond(cls, v):
        return str(v[0]) + "=%s"

    @classmethod
    def _conditions_to_str(cls, conditions):
        return ' AND '.join(map(cls._val_to_cond, conditions.items()))

    @classmethod
    def _make_query(cls, conditions, options):
        if 'fields' in options:
            fields = ', '.join(options['fields'])
        else:
            fields = '*'

        if 'one' in options:
            options['limit'] = 1
        query_template = "select {fields} from {table}"
        args = ()
        if conditions:
            query_template += " where {conditions}"
            args += tuple(conditions.values())

        if 'limit' in options:
            query_template += " limit %s"
            args += (options['limit'],)

        query = query_template.format(
            fields=fields,
            table=cls.table,
            conditions=cls._conditions_to_str(conditions)
        )
        cls.cursor.execute(query, args)
        if 'one' in options:
            return cls.hydrate_obj(cls.cursor.fetchone())
        else:
            return [cls.hydrate_obj(obj) for obj in cls.cursor.fetchall()]

    @classmethod
    def query(cls, conditions=None, options=None):
        if conditions is None:
            conditions = {}
        if options is None:
            options = {}
        return cls._make_query(conditions, {})

    @classmethod
    def count(cls, conditions=None):
        options = {'fields': 'id'}
        result = cls.query(conditions, options)
        return len(result)

    @classmethod
    def exists(cls, conditions=None):
        return cls.count(conditions) > 0

    @classmethod
    def create(cls, args=None):
        obj = cls(args)
        return obj.save()

    @classmethod
    def find(cls, id):
        return cls.find_by({'id': id})

    @classmethod
    def find_by(cls, conditions):
        return cls._make_query(conditions, {'one': True})

    @classmethod
    @db_action
    def remove_all(cls):
        query_template = "delete from {table}"
        query = query_template.format(table=cls.table)
        cls.cursor.execute(query)

    def is_new(self):
        return self.id is None

    def before_save(self):
        pass

    def _insert(self):
        query_template = "insert into {table} ({fields}) values ({values})"
        fields = ', '.join([str(key) for key in self.__dict__.keys()])
        values = ', '.join(['%s' for _ in self.__dict__.values()])
        query = query_template.format(
            table=self.table, fields=fields, values=values)
        self.cursor.execute(query, tuple(self.__dict__.values()))
        self.id = self.cursor.lastrowid

    def _update(self):
        query_template = "update {table} set {fields} where id=%s"
        fields = ', '.join(map(self._val_to_cond, self.__dict__.items()))
        query = query_template.format(table=self.table, fields=fields)
        self.cursor.execute(query, tuple(self.__dict__) + (self.id,))

    @db_action
    def save(self):
        self.before_save()
        if self.is_new():
            self._insert()
        else:
            self._update()
        return self

    @db_action
    def remove(self):
        query_template = "delete from {table} where id=%s"
        query = query_template.format(table=self.table)
        self.cursor.execute(query, (self.id,))
<commit_msg>Fix bug in object hydration.<commit_after>import mysql.connector
import os

from ... import config

# FIXME: use proper config files
if os.environ.get('PAAS_MANAGER_ENV') == 'test':
    config['mysql']['database'] += '_test'

def db_action(fn):
    def wrapped(*args, **kwargs):
        res = fn(*args, **kwargs)
        DatabaseConnector.connect.commit()
        return res
    return wrapped

class DatabaseConnector():
    connect = mysql.connector.connect(**config['mysql'])
    cursor = connect.cursor()

    def __init__(self, args=None):
        if args is None:
            args = {}
        self.__dict__ = args
        if not hasattr(self, 'id'):
            self.id = None
        for k, v in args.items():
            setattr(self, k, v)

    @classmethod
    def hydrate_obj(cls, values):
        if not values:
            return None
        fields = [i[0] for i in cls.cursor.description]
        obj = cls()
        for i, f in enumerate(fields):
            obj._add_attr(f, values[i])
        return obj

    @classmethod
    def _val_to_cond(cls, v):
        return str(v[0]) + "=%s"

    @classmethod
    def _conditions_to_str(cls, conditions):
        return ' AND '.join(map(cls._val_to_cond, conditions.items()))

    @classmethod
    def _make_query(cls, conditions, options):
        if 'fields' in options:
            fields = ', '.join(options['fields'])
        else:
            fields = '*'

        if 'one' in options:
            options['limit'] = 1
        query_template = "select {fields} from {table}"
        args = ()
        if conditions:
            query_template += " where {conditions}"
            args += tuple(conditions.values())

        if 'limit' in options:
            query_template += " limit %s"
            args += (options['limit'],)

        query = query_template.format(
            fields=fields,
            table=cls.table,
            conditions=cls._conditions_to_str(conditions)
        )
        cls.cursor.execute(query, args)
        if 'one' in options:
            return cls.hydrate_obj(cls.cursor.fetchone())
        else:
            return [cls.hydrate_obj(obj) for obj in cls.cursor.fetchall()]

    @classmethod
    def query(cls, conditions=None, options=None):
        if conditions is None:
            conditions = {}
        if options is None:
            options = {}
        return cls._make_query(conditions, {})

    @classmethod
    def count(cls, conditions=None):
        options = {'fields': 'id'}
        result = cls.query(conditions, options)
        return len(result)

    @classmethod
    def exists(cls, conditions=None):
        return cls.count(conditions) > 0

    @classmethod
    def create(cls, args=None):
        obj = cls(args)
        return obj.save()

    @classmethod
    def find(cls, id):
        return cls.find_by({'id': id})

    @classmethod
    def find_by(cls, conditions):
        return cls._make_query(conditions, {'one': True})

    @classmethod
    @db_action
    def remove_all(cls):
        query_template = "delete from {table}"
        query = query_template.format(table=cls.table)
        cls.cursor.execute(query)

    def is_new(self):
        return self.id is None

    def before_save(self):
        pass

    def _insert(self):
        query_template = "insert into {table} ({fields}) values ({values})"
        fields = ', '.join([str(key) for key in self.__dict__.keys()])
        values = ', '.join(['%s' for _ in self.__dict__.values()])
        query = query_template.format(
            table=self.table, fields=fields, values=values)
        self.cursor.execute(query, tuple(self.__dict__.values()))
        self.id = self.cursor.lastrowid

    def _update(self):
        query_template = "update {table} set {fields} where id=%s"
        fields = ', '.join(map(self._val_to_cond, self.__dict__.items()))
        query = query_template.format(table=self.table, fields=fields)
        self.cursor.execute(query, tuple(self.__dict__) + (self.id,))

    @db_action
    def save(self):
        self.before_save()
        if self.is_new():
            self._insert()
        else:
            self._update()
        return self

    @db_action
    def remove(self):
        query_template = "delete from {table} where id=%s"
        query = query_template.format(table=self.table)
        self.cursor.execute(query, (self.id,))
<|endoftext|>


<commit_before>import mimetypes

import boto3
import flask

from .base import AnnexBase
from .compat import string_types

# -----------------------------------------------------------------------------

DEFAULT_URL_EXPIRES_IN = 300

# -----------------------------------------------------------------------------

class S3Annex(AnnexBase):
    def __init__(
        self,
        bucket_name,
        region=None,
        access_key_id=None,
        secret_access_key=None,
        url_expires_in=DEFAULT_URL_EXPIRES_IN,
    ):
        self._client = boto3.client(
            's3',
            region,
            aws_access_key_id=access_key_id,
            aws_secret_access_key=secret_access_key,
        )

        self._bucket_name = bucket_name
        self._url_expires_in = url_expires_in

    def delete(self, key):
        self._client.delete_object(Bucket=self._bucket_name, Key=key)

    def delete_many(self, keys):
        self._client.delete_objects(
            Bucket=self._bucket_name,
            Delete={
                'Objects': tuple({'Key': key} for key in keys),
            },
        )

    def get_file(self, key, out_file):
        if isinstance(out_file, string_types):
            self._client.download_file(self._bucket_name, key, out_file)
        else:
            self._client.download_fileobj(self._bucket_name, key, out_file)

    def list_keys(self, prefix):
        response = self._client.list_objects_v2(
            Bucket=self._bucket_name, Prefix=prefix,
        )
        if 'Contents' not in response:
            return ()
        return tuple(item['Key'] for item in response['Contents'])

    def save_file(self, key, in_file):
        # Get the content type from the key, rather than letting Boto try to
        # figure it out from the file's name, which may be uninformative.
        content_type = mimetypes.guess_type(key)[0]
        if content_type:
            extra_args = {'ContentType': content_type}
        else:
            extra_args = None

        if isinstance(in_file, string_types):
            self._client.upload_file(
                in_file, self._bucket_name, key, extra_args,
            )
        else:
            self._client.upload_fileobj(
                in_file, self._bucket_name, key, extra_args,
            )

    def send_file(self, key):
        url = self._client.generate_presigned_url(
            ClientMethod='get_object',
            Params={
                'Bucket': self._bucket_name,
                'Key': key,
            },
            ExpiresIn=self._url_expires_in,
        )
        return flask.redirect(url)

    def get_upload_info(self, key):
        fields = {}
        conditions = []

        content_type = mimetypes.guess_type(key)[0]
        if content_type:
            fields['Content-Type'] = content_type

        max_content_length = flask.current_app.config['MAX_CONTENT_LENGTH']
        if max_content_length is not None:
            conditions.append(
                ('content-length-range', 0, max_content_length),
            )

        # Boto doesn't automatically add fields to conditions.
        for field_key, field_value in fields.items():
            conditions.append({field_key: field_value})

        post_info = self._client.generate_presigned_post(
            Bucket=self._bucket_name,
            Key=key,
            Fields=fields,
            Conditions=conditions,
            ExpiresIn=self._url_expires_in,
        )

        return {
            'method': 'POST',
            'url': post_info['url'],
            'data': post_info['fields'],
        }
<commit_msg>Rename url_expires_in to expires_in<commit_after>import mimetypes

import boto3
import flask

from .base import AnnexBase
from .compat import string_types

# -----------------------------------------------------------------------------

DEFAULT_EXPIRES_IN = 300

# -----------------------------------------------------------------------------

class S3Annex(AnnexBase):
    def __init__(
        self,
        bucket_name,
        region=None,
        access_key_id=None,
        secret_access_key=None,
        expires_in=DEFAULT_EXPIRES_IN,
    ):
        self._client = boto3.client(
            's3',
            region,
            aws_access_key_id=access_key_id,
            aws_secret_access_key=secret_access_key,
        )

        self._bucket_name = bucket_name
        self._expires_in = expires_in

    def delete(self, key):
        self._client.delete_object(Bucket=self._bucket_name, Key=key)

    def delete_many(self, keys):
        self._client.delete_objects(
            Bucket=self._bucket_name,
            Delete={
                'Objects': tuple({'Key': key} for key in keys),
            },
        )

    def get_file(self, key, out_file):
        if isinstance(out_file, string_types):
            self._client.download_file(self._bucket_name, key, out_file)
        else:
            self._client.download_fileobj(self._bucket_name, key, out_file)

    def list_keys(self, prefix):
        response = self._client.list_objects_v2(
            Bucket=self._bucket_name, Prefix=prefix,
        )
        if 'Contents' not in response:
            return ()
        return tuple(item['Key'] for item in response['Contents'])

    def save_file(self, key, in_file):
        # Get the content type from the key, rather than letting Boto try to
        # figure it out from the file's name, which may be uninformative.
        content_type = mimetypes.guess_type(key)[0]
        if content_type:
            extra_args = {'ContentType': content_type}
        else:
            extra_args = None

        if isinstance(in_file, string_types):
            self._client.upload_file(
                in_file, self._bucket_name, key, extra_args,
            )
        else:
            self._client.upload_fileobj(
                in_file, self._bucket_name, key, extra_args,
            )

    def send_file(self, key):
        url = self._client.generate_presigned_url(
            ClientMethod='get_object',
            Params={
                'Bucket': self._bucket_name,
                'Key': key,
            },
            ExpiresIn=self._expires_in,
        )
        return flask.redirect(url)

    def get_upload_info(self, key):
        fields = {}
        conditions = []

        content_type = mimetypes.guess_type(key)[0]
        if content_type:
            fields['Content-Type'] = content_type

        max_content_length = flask.current_app.config['MAX_CONTENT_LENGTH']
        if max_content_length is not None:
            conditions.append(
                ('content-length-range', 0, max_content_length),
            )

        # Boto doesn't automatically add fields to conditions.
        for field_key, field_value in fields.items():
            conditions.append({field_key: field_value})

        post_info = self._client.generate_presigned_post(
            Bucket=self._bucket_name,
            Key=key,
            Fields=fields,
            Conditions=conditions,
            ExpiresIn=self._expires_in,
        )

        return {
            'method': 'POST',
            'url': post_info['url'],
            'data': post_info['fields'],
        }
<|endoftext|>


<commit_before>import importlib
import weakref

try:
    unicode_str = unicode
except NameError:
    unicode_str = str

def metaclass(mcs):
    def _decorator(cls):
        attrs = dict(cls.__dict__)
        try:
            if isinstance(cls.__slots__, str):
                slots = cls.__slots__,
            else:
                slots = cls.__slots__
            for slot in slots:
                del attrs[slot]
        except AttributeError:
            pass
        for prop in '__weakref__', '__dict__':
            attrs.pop(prop, None)
        return mcs(cls.__name__, cls.__bases__, attrs)
    return _decorator

class CachedDescriptor(object):

    __slots__ = 'cache'

    def __init__(self):
        self.cache = self.init_cache()

    def __get__(self, instance, instance_type=None):
        if instance is None:
            return self  # static access
        try:
            return self.cache[instance]
        except KeyError:
            value = self.cache[instance] = self.get_value(instance)
            return value

    @staticmethod
    def init_cache():
        return weakref.WeakKeyDictionary()

    def get_value(self, instance):
        raise NotImplementedError

class cached_property(CachedDescriptor):

    __slots__ = ('get', 'set', 'delete', '__doc__', 'cache')

    def __init__(self, fget=None, fset=None, fdel=None, doc=None):
        super(cached_property, self).__init__()
        self.get = fget
        self.set = fset
        self.delete = fdel
        self.__doc__ = doc

    def get_value(self, instance):
        if self.get is None:
            raise AttributeError("unreadable attribute")
        return self.get(instance)

    def __set__(self, instance, value):
        if self.set is None:
            self.cache[instance] = value
        else:
            self.set(instance, value)
            self.cache.pop(instance, None)

    def __delete__(self, instance):
        if self.delete is not None:
            self.delete(instance)
        self.cache.pop(instance, None)

    def getter(self, getter):
        self.get = getter
        return self

    def setter(self, setter):
        self.set = setter
        return self

    def deleter(self, deleter):
        self.delete = deleter
        return self

class LazyType(type):

    def __call__(cls, path):
        if isinstance(path, cls):
            return path
        elif isinstance(path, str):
            return super(LazyType, cls).__call__(path)
        return path

@metaclass(LazyType)
class Lazy(object):

    __slots__ = ('_path', '__weakref__')

    def __init__(self, path):
        self._path = path

    def __call__(self, *args, **kwargs):
        return self._obj(*args, **kwargs)

    def __iter__(self):
        return iter(self._obj)

    def __len__(self):
        return len(self._obj)

    def __str__(self):
        return str(self._obj)

    def __unicode__(self):
        return unicode(self._obj)

    def __bytes__(self):
        return bytes(self._obj)

    def __getitem__(self, item):
        return self._obj[item]

    def __getattr__(self, attr):
        return getattr(self._obj, attr)

    @property
    def __class__(self):
        return self._obj.__class__

    @cached_property
    def _obj(self):
        try:
            module_name, obj_name = self._path.rsplit('.', 1)
        except ValueError:
            module_name, obj_name = self._path, None
        module = importlib.import_module(module_name)
        if obj_name is not None:
            return getattr(module, obj_name)
        return module

def to_bytes(obj, encoding='utf-8', error_callback=None):
    try:
        if isinstance(obj, (bytes, bytearray, memoryview)):
            return bytes(obj)
        if obj is None:
            return b''
        try:
            return obj.__bytes__()
        except AttributeError:
            return unicode_str(obj).encode(encoding=encoding)
    except Exception as error:
        if error_callback is not None:
            error_callback(error)
        raise
<commit_msg>fixed __slots__ properties<commit_after>import importlib
import weakref

try:
    unicode_str = unicode
except NameError:
    unicode_str = str

def metaclass(mcs):
    def _decorator(cls):
        attrs = dict(cls.__dict__)
        try:
            if isinstance(cls.__slots__, str):
                slots = cls.__slots__,
            else:
                slots = cls.__slots__
            for slot in slots:
                del attrs[slot]
        except AttributeError:
            pass
        for prop in '__weakref__', '__dict__':
            attrs.pop(prop, None)
        return mcs(cls.__name__, cls.__bases__, attrs)
    return _decorator

class CachedDescriptor(object):

    __slots__ = 'cache',

    def __init__(self):
        self.cache = self.init_cache()

    def __get__(self, instance, instance_type=None):
        if instance is None:
            return self  # static access
        try:
            return self.cache[instance]
        except KeyError:
            value = self.cache[instance] = self.get_value(instance)
            return value

    @staticmethod
    def init_cache():
        return weakref.WeakKeyDictionary()

    def get_value(self, instance):
        raise NotImplementedError

class cached_property(CachedDescriptor):

    __slots__ = 'get', 'set', 'delete', '__doc__'

    def __init__(self, fget=None, fset=None, fdel=None, doc=None):
        super(cached_property, self).__init__()
        self.get = fget
        self.set = fset
        self.delete = fdel
        self.__doc__ = doc

    def get_value(self, instance):
        if self.get is None:
            raise AttributeError("unreadable attribute")
        return self.get(instance)

    def __set__(self, instance, value):
        if self.set is None:
            self.cache[instance] = value
        else:
            self.set(instance, value)
            self.cache.pop(instance, None)

    def __delete__(self, instance):
        if self.delete is not None:
            self.delete(instance)
        self.cache.pop(instance, None)

    def getter(self, getter):
        self.get = getter
        return self

    def setter(self, setter):
        self.set = setter
        return self

    def deleter(self, deleter):
        self.delete = deleter
        return self

class LazyType(type):

    def __call__(cls, path):
        if isinstance(path, cls):
            return path
        elif isinstance(path, str):
            return super(LazyType, cls).__call__(path)
        return path

@metaclass(LazyType)
class Lazy(object):

    __slots__ = '_path', '__weakref__'

    def __init__(self, path):
        self._path = path

    def __call__(self, *args, **kwargs):
        return self._obj(*args, **kwargs)

    def __iter__(self):
        return iter(self._obj)

    def __len__(self):
        return len(self._obj)

    def __str__(self):
        return str(self._obj)

    def __unicode__(self):
        return unicode(self._obj)

    def __bytes__(self):
        return bytes(self._obj)

    def __getitem__(self, item):
        return self._obj[item]

    def __getattr__(self, attr):
        return getattr(self._obj, attr)

    @property
    def __class__(self):
        return self._obj.__class__

    @cached_property
    def _obj(self):
        try:
            module_name, obj_name = self._path.rsplit('.', 1)
        except ValueError:
            module_name, obj_name = self._path, None
        module = importlib.import_module(module_name)
        if obj_name is not None:
            return getattr(module, obj_name)
        return module

def to_bytes(obj, encoding='utf-8', error_callback=None):
    try:
        if isinstance(obj, (bytes, bytearray, memoryview)):
            return bytes(obj)
        if obj is None:
            return b''
        try:
            return obj.__bytes__()
        except AttributeError:
            return unicode_str(obj).encode(encoding=encoding)
    except Exception as error:
        if error_callback is not None:
            error_callback(error)
        raise
<|endoftext|>